RQ #3: How do in-the-moment experiences of youth (i.e., challenge, relevance, learning, and affect) cultivate engagement in STEM activities?

This is what we know:

  • Support for agency is associated with overall engagement (fairly robustly - except when classroom / field is added)
  • Challenge and relevance both are associated with engagement and are very robust, altough relevance is much more strongly related
  • Overall pre-interest is associated with higher engagement

This is what we still need to figure out:

  • We have included prop attend and classroom versus field, but when we considered a) what were important covariates / control variables to be included in past analyses (and analyses for RQ #1 and RQ #2), we removed prop attend, and b) including support for agency, we found that this (support for agency) may be more meaningful than whether the activity was in the classroom or field context - is this okay?

How this document is organized

  • Only one outcome, overall engagement (the engagement measure is constructed on the basis of individual items for hard working, concentrating, enjoy, and interest), with three models, all with challenge and relevance and overall pre-interest and gender as predictors:
    • With youth activity as predictors
    • With support for agency and STEM S-B as predictors
    • With both youth activity and support for agency and STEM S-B as predictors
library(tidyverse)
library(lme4)
library(corrr)
library(jmRtools)
library(sjPlot)
esm <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-esm.csv")
pre_survey_data_processed <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-pre-survey.csv")
post_survey_data_partially_processed <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-post-survey.csv")
video <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-video.csv")
pqa <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-pqa.csv")
attendance <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-attendance.csv")
class_data <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-class-video.csv")
demographics <- read_csv("/Volumes/SCHMIDTLAB/PSE/data/STEM-IE/STEM-IE-demographics.csv")
pm <- read_csv("/Volumes/SCHMIDTLAB/PSE/Data/STEM-IE/STEM-IE-program-match.csv")
attendance <- rename(attendance, participant_ID = ParticipantID)
attendance <- mutate(attendance, prop_attend = DaysAttended / DaysScheduled, 
                     participant_ID = as.integer(participant_ID))
attendance <- select(attendance, participant_ID, prop_attend)

demographics <- filter(demographics, participant_ID!= 7187)
demographics <- left_join(demographics, attendance)

esm$overall_engagement <- jmRtools::composite_mean_maker(esm, hard_working, concentrating, enjoy, interest)
df <- left_join(esm, pre_survey_data_processed, by = "participant_ID") # df & post-survey
df <- left_join(df, video, by = c("program_ID", "response_date", "sociedad_class", "signal_number")) # df & video
df <- left_join(df, demographics, by = c("participant_ID", "program_ID")) # df and demographics
pqa <- mutate(pqa, 
              active = active_part_1 + active_part_2,
              ho_thinking = ho_thinking_1 + ho_thinking_2 + ho_thinking_3,
              belonging = belonging_1 + belonging_2,
              agency = agency_1 + agency_2 + agency_3 + agency_4,
              youth_development_overall = active_part_1 + active_part_2 + ho_thinking_1 + ho_thinking_2 + ho_thinking_3 + belonging_1 + belonging_2 + agency_1 + agency_2 + agency_3 + agency_4,
              making_observations = stem_sb_8,
              data_modeling = stem_sb_2 + stem_sb_3 + stem_sb_9,
              interpreting_communicating = stem_sb_6,
              generating_data = stem_sb_4,
              asking_questions = stem_sb_1)

# pqa <- rename(pqa, sixth_math_sociedad = sixth_math)
# pqa <- rename(pqa, seventh_math_sociedad = seventh_math)
# pqa <- rename(pqa, eighth_math_sociedad = eighth_math)
# pqa <- rename(pqa, dance_sociedad = dance)
# pqa <- rename(pqa, robotics_sociedad = robotics)

pqa$sociedad_class <- ifelse(pqa$eighth_math == 1, "8th Math",
                             ifelse(pqa$seventh_math == 1, "7th Math",
                                    ifelse(pqa$sixth_math == 1, "6th Math",
                                           ifelse(pqa$robotics == 1, "Robotics",
                                                  ifelse(pqa$dance == 1, "Dance", NA)))))

pqa <- rename(pqa, 
              program_ID = SiteIDNumeric,
              response_date = resp_date,
              signal_number = signal)

pqa$program_ID <- as.character(pqa$program_ID)

df <- left_join(df, pqa, by = c("response_date", "program_ID", "signal_number", "sociedad_class"))
df <- df %>% 
    mutate(youth_activity_three = case_when(
        youth_activity_rc == "Creating Product" ~ "Creating Product",
        youth_activity_rc == "Basic Skills Activity" ~ "Basic Skills Activity",
        TRUE ~ "Other"
    ))

df$youth_activity_three <- fct_relevel(df$youth_activity_three, 
                                       "Other")
df <- df %>% 
    mutate(ho_thinking_dummy = ifelse(sum_ho_thinking > 0, 1, 0),
           agency_dummy = ifelse(sum_agency > 0, 1, 0),
           active_dummy = ifelse(sum_ap > 0, 1, 0),
           belonging_dummy = ifelse(sum_belonging > 0, 1, 0),
           stem_sb_dummy = ifelse(sum_stem_sb > 0, 1, 0))

Correlations

df %>% 
    select(overall_engagement, challenge, relevance, overall_pre_interest, prop_attend, classroom_versus_field_enrichment, agency, sum_stem_sb) %>% 
    correlate() %>% 
    shave() %>% 
    fashion()
##                             rowname overall_engagement challenge relevance
## 1                overall_engagement                                       
## 2                         challenge                .32                    
## 3                         relevance                .68       .39          
## 4              overall_pre_interest                .14      -.00       .09
## 5                       prop_attend                .04      -.03       .04
## 6 classroom_versus_field_enrichment               -.03       .00      -.01
## 7                            agency                .06       .06       .02
## 8                       sum_stem_sb                .01       .00       .02
##   overall_pre_interest prop_attend classroom_versus_field_enrichment
## 1                                                                   
## 2                                                                   
## 3                                                                   
## 4                                                                   
## 5                  .06                                              
## 6                 -.10         .06                                  
## 7                  .08         .02                               .21
## 8                  .02         .04                               .04
##   agency sum_stem_sb
## 1                   
## 2                   
## 3                   
## 4                   
## 5                   
## 6                   
## 7                   
## 8    .39

Overall engagement

Overall engagement: With youth activity as predictors

m <- lmer(overall_engagement ~ 1 +
              challenge +
              relevance +

              overall_pre_interest +
              gender +

              youth_activity_three +
              
              (1|program_ID) + (1|participant_ID) + (agency_dummy|beep_ID_new),
          data = df)

sjPlot::sjt.lmer(m, p.kr = T, show.re.var = F, show.ci = F, show.se = T)
    overall_engagement
    B std. Error p
Fixed Parts
(Intercept)   1.14 0.11 <.001
challenge   0.06 0.01 <.001
relevance   0.53 0.02 <.001
overall_pre_interest   0.07 0.03 .021
gender (M)   -0.07 0.05 .198
youth_activity_three (Basic Skills Activity)   0.00 0.04 .895
youth_activity_three (Creating Product)   0.01 0.04 .727
Random Parts
Nbeep_ID_new   236
Nparticipant_ID   180
Nprogram_ID   9
ICCbeep_ID_new   0.064
ICCparticipant_ID   0.249
ICCprogram_ID   0.012
Observations   2549
R2 / Ω02   .682 / .680

Overall engagement: With instructional support for agency and STEM S-B

m <- lmer(overall_engagement ~ 1 +
              challenge +
              relevance +

              overall_pre_interest +
              gender +

              agency_dummy +
              stem_sb_dummy + 
              
              (1|program_ID) + (1|participant_ID) + (agency_dummy|beep_ID_new),
          data = df)

sjPlot::sjt.lmer(m, p.kr = T, show.re.var = F, show.ci = F, show.se = T)
    overall_engagement
    B std. Error p
Fixed Parts
(Intercept)   1.11 0.12 <.001
challenge   0.06 0.01 <.001
relevance   0.54 0.02 <.001
overall_pre_interest   0.07 0.03 .022
gender (M)   -0.07 0.05 .198
agency_dummy   0.09 0.04 .021
stem_sb_dummy   -0.06 0.04 .143
Random Parts
Nbeep_ID_new   236
Nparticipant_ID   180
Nprogram_ID   9
ICCbeep_ID_new   0.056
ICCparticipant_ID   0.251
ICCprogram_ID   0.012
Observations   2549
R2 / Ω02   .681 / .679

Overall engagement: With youth activity and instructional support for agency and STEM S-B

m <- lmer(overall_engagement ~ 1 +
              challenge +
              relevance +

              overall_pre_interest +
              gender +

              agency_dummy +
              stem_sb_dummy + 
              youth_activity_three +
              
              (1|program_ID) + (1|participant_ID) + (agency_dummy|beep_ID_new),
          data = df)

sjPlot::sjt.lmer(m, p.kr = T, show.re.var = F, show.ci = F, show.se = T)
    overall_engagement
    B std. Error p
Fixed Parts
(Intercept)   1.11 0.12 <.001
challenge   0.06 0.01 <.001
relevance   0.54 0.02 <.001
overall_pre_interest   0.07 0.03 .023
gender (M)   -0.07 0.05 .199
agency_dummy   0.09 0.04 .024
stem_sb_dummy   -0.06 0.04 .137
youth_activity_three (Basic Skills Activity)   0.01 0.04 .729
youth_activity_three (Creating Product)   0.00 0.04 .937
Random Parts
Nbeep_ID_new   236
Nparticipant_ID   180
Nprogram_ID   9
ICCbeep_ID_new   0.056
ICCparticipant_ID   0.251
ICCprogram_ID   0.012
Observations   2549
R2 / Ω02   .681 / .680