Prepare scripts for the analysis of interventionla study data

03_INTERVENTIONAL_M0/INTERVENTIONAL_M0_00_R_environment.R

+## INTERVENTIONAL_M0_00_R_environment.R
+## Date     : 2024/10/17
+## Author   : Thomas Obadia
+##
+## This is a general script for the 03_INVENTORY/ R environment, that
+## will load all required packages into R and set global variables
+## used by other scripts.
+## 
+## You should usually not source this file alone: it is sourced in 
+## 02_OBSERVATIONAL/OBSERVATIONAL_01_dump_REDCap_database.R
+######################################################################
+
+
+
+
+
+######################################################################
+### REQUIRED PACKAGES
+######################################################################
+require(dotenv)              # Easy handling personal global variables
+require(redcapAPI)           # REDCap API methods
+require(tidyverse)           # Data manipulation made easy
+# require(OpenStreetMap)       # Set of methods for OpenStreetMap 
+# require(ggmap)               # More general set of methods
+
+
+
+
+
+######################################################################
+### SOURCE .env FILE
+######################################################################
+dotenv::load_dot_env("./03_INTERVENTIONAL_M0/.env")
+
+
+
+
+
+######################################################################
+### DATA SOURCE TO BE USED
+######################################################################
+## Global flags that define which database should be dumped and which
+## will be ignored. There is one flag per data source: IPP, IPM, AHRI.
+#    - TRUE : data will be dumped
+#    - FALSE: data will not be dumped
+DATA_SOURCE_REDCAP_IPP_INT_M0     <- FALSE
+DATA_SOURCE_REDCAP_IPM_INT_M0     <- TRUE
+DATA_SOURCE_REDCAP_AHRI_INT_M0    <- FALSE
+
+DATA_SOURCE_REDCAP_IPP_INT_PCD    <- FALSE
+DATA_SOURCE_REDCAP_IPM_INT_PCD    <- TRUE
+DATA_SOURCE_REDCAP_AHRI_INT_PCD   <- FALSE
+
+
+
+
+
+######################################################################
+### DATA UP-TO-DATE FLAG
+######################################################################
+## Global flag that defines if the database dump is recent enough or
+## if a REDCap extract should be conducted again
+#    - TRUE : data is recent enough, don't dump
+#    - FALSE: new extract is necessary (default)
+DATA_EXTRACT_IS_RECENT_OBS     <- FALSE
+
+## Date at which the last extract was done. Defaults to Epoch time
+## in the R Environment file, so that a new dump is always done after
+## sourcing this file.
+DATA_EXTRACT_TS_DEFAULT    <- as.Date("1970-01-01")
+
+## Number of days after which database should be extracted again
+DATA_EXTRACT_EXPIRY_TIME_D <- 1
+
+
+
+
+
+######################################################################
+### SEED VALUE
+######################################################################
+## A global seed value to be used by set.seed() calls
+SEED <- 12345
+
+
+
+
+
+######################################################################
+### CURATION
+######################################################################
+## Global flags that defines if a curation rule should be applied to 
+## the raw data dumped from REDCap or not.
+#    - TRUE : curation rule will be enforced
+#    - FALSE: curation rule will not be enforced (default)
+CURATE_DATA_RECALCULATE_AGEY_ETHIOPIA   <- FALSE
+CURATE_DATA_RECALCULATE_AGEY_MADAGASCAR <- FALSE
+
+
+
+

03_INTERVENTIONAL_M0/INTERVENTIONAL_M0_01_dump_REDCap_database.R

+## INTERVENTIONAL_M0_01_dump_REDCap_database.R
+## Date     : 2025/05/19
+## Author   : Thomas Obadia
+##
+## This script dumps the REDCap database for the PvSTATEM observational
+## project.
+## Note that it relies on the dotenv package in an effort to not 
+## incorporate API keys into the script.
+## Your API key is strictly personal and should never be shared !
+######################################################################
+
+
+
+
+
+######################################################################
+### SOURCE THE R ENVIRONMENT
+######################################################################
+source("./03_INTERVENTIONAL_M0/INTERVENTIONAL_M0_00_R_environment.R")
+
+
+
+
+
+######################################################################
+### CONNECTION TO REDCAP SERVER
+######################################################################
+## .env file should contain tokens for IPP, IPM and AHRI
+RCON_IPP_INTERVENTIONAL_M0   <- NULL
+RCON_IPM_INTERVENTIONAL_M0   <- NULL
+RCON_AHRI_INTERVENTIONAL_M0  <- NULL
+
+RCON_IPP_INTERVENTIONAL_PCD  <- NULL
+RCON_IPM_INTERVENTIONAL_PCD  <- NULL
+RCON_AHRI_INTERVENTIONAL_PCD <- NULL
+
+# Month 0 Active Follow-Up datasets
+if (DATA_SOURCE_REDCAP_IPP_INT_M0) {
+  RCON_IPP_INTERVENTIONAL_M0  <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_IPP_API_URL"), 
+                                                             token = Sys.getenv("REDCAP_IPP_API_TOKEN_INTERVENTIONAL_M0"))
+}
+if (DATA_SOURCE_REDCAP_IPM_INT_M0) {
+  RCON_IPM_INTERVENTIONAL_M0  <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_IPM_API_URL"), 
+                                                             token = Sys.getenv("REDCAP_IPM_API_TOKEN_INTERVENTIONAL_M0"))
+}
+if (DATA_SOURCE_REDCAP_AHRI_INT_M0) {
+  RCON_AHRI_INTERVENTIONAL_M0 <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_AHRI_API_URL"), 
+                                                             token = Sys.getenv("REDCAP_AHRI_API_TOKEN_INTERVENTIONAL_M0"))
+}
+
+# Passive Case Detection datasets
+if (DATA_SOURCE_REDCAP_IPP_INT_PCD) {
+  RCON_IPP_INTERVENTIONAL_PCD  <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_IPP_API_URL"), 
+                                                             token = Sys.getenv("REDCAP_IPP_API_TOKEN_INTERVENTIONAL_PCD"))
+}
+if (DATA_SOURCE_REDCAP_IPM_INT_PCD) {
+  RCON_IPM_INTERVENTIONAL_PCD  <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_IPM_API_URL"), 
+                                                             token = Sys.getenv("REDCAP_IPM_API_TOKEN_INTERVENTIONAL_PCD"))
+}
+if (DATA_SOURCE_REDCAP_AHRI_INT_PCD) {
+  RCON_AHRI_INTERVENTIONAL_PCD <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_AHRI_API_URL"), 
+                                                             token = Sys.getenv("REDCAP_AHRI_API_TOKEN_INTERVENTIONAL_PCD"))
+}
+
+
+## Allocate one RCON object into a generic one for subsequent calls
+# RCON_INTERVENTIONAL_M0      <- RCON_IPM_INTERVENTIONAL_M0 # Use for single-download tests
+RCON_INTERVENTIONAL_M0  <- list("IPP"  = RCON_IPP_INTERVENTIONAL_M0, 
+                                "IPM"  = RCON_IPM_INTERVENTIONAL_M0, 
+                                "AHRI" = RCON_AHRI_INTERVENTIONAL_M0) %>% 
+  discard(is.null)
+
+RCON_INTERVENTIONAL_PCD <- list("IPP"  = RCON_IPP_INTERVENTIONAL_PCD, 
+                                "IPM"  = RCON_IPM_INTERVENTIONAL_PCD, 
+                                "AHRI" = RCON_AHRI_INTERVENTIONAL_PCD) %>% 
+  discard(is.null)
+
+
+
+
+
+######################################################################
+### DUMP DATABASE
+######################################################################
+## Check if data needs to be extracted again
+DATA_EXTRACT_IS_RECENT_INT_M0    <- as.logical(difftime(time1 = Sys.Date(), 
+                                                        time2 = as.Date(ifelse(exists("DATA_EXTRACT_TS_INT_M0"), DATA_EXTRACT_TS_INT_M0, DATA_EXTRACT_TS_DEFAULT)), 
+                                                        units = "days") <= DATA_EXTRACT_EXPIRY_TIME_D)
+
+## Use the exportRecordsTyped method from package redcapAPI
+if (!DATA_EXTRACT_IS_RECENT_INT_M0) {
+  # Initialize empty element for raw data
+  dat_interventional_m0_raw  <- list()
+  dat_interventional_pcd_raw <- list()
+  
+  ## Active Follow-Up datasets
+  for (RCON in names(RCON_INTERVENTIONAL_M0)) {
+    cat("Extracting data from RCON_INTERVENTIONAL_M0 element: ", 
+        RCON, 
+        "...\n", 
+        sep = "")
+    
+    dat_interventional_m0_raw[[RCON]] <- redcapAPI::exportRecordsTyped(RCON_INTERVENTIONAL_M0[[RCON]], 
+                                                                       cast = list(
+                                                                         # Radio buttons reported as label instead of code
+                                                                         "radio"    = castLabelCharacter, 
+                                                                         "yesno"    = castLabelCharacter, 
+                                                                         "dropdown" = castLabelCharacter
+                                                                       )) %>% 
+      
+      # Convert the '999' code to NA, except in Date/POSIX fields because it'll mess them up
+      mutate(across(-c(where(is.POSIXt), 
+                       where(is.Date)), function(x) (ifelse(x == 999, NA, x)))) %>% 
+      
+      # Add data source for convenience
+      mutate(data_source   = RCON)
+    
+    ## Check for invalid records before moving on
+    print(reviewInvalidRecords(dat_interventional_m0_raw[[RCON]]))
+  }
+  
+  # Cleanup that loop
+  rm(RCON)
+  
+  # Concatenate all data into a single table
+  dat_interventional_m0_raw <- dat_interventional_m0_raw %>% 
+    bind_rows()
+  
+  ## Passive Case Detection datasets
+  for (RCON in names(RCON_INTERVENTIONAL_PCD)) {
+    cat("Extracting data from RCON_INTERVENTIONAL_PCD element: ", 
+        RCON, 
+        "...\n", 
+        sep = "")
+    
+    dat_interventional_pcd_raw[[RCON]] <- redcapAPI::exportRecordsTyped(RCON_INTERVENTIONAL_PCD[[RCON]], 
+                                                                        cast = list(
+                                                                          # Radio buttons reported as label instead of code
+                                                                          "radio"    = castLabelCharacter, 
+                                                                          "yesno"    = castLabelCharacter, 
+                                                                          "dropdown" = castLabelCharacter
+                                                                        )) %>% 
+      
+      # Convert the '999' code to NA, except in Date/POSIX fields because it'll mess them up
+      mutate(across(-c(where(is.POSIXt), 
+                       where(is.Date)), function(x) (ifelse(x == 999, NA, x)))) %>% 
+      
+      # Add data source for convenience
+      mutate(data_source   = RCON)
+    
+    ## Check for invalid records before moving on
+    print(reviewInvalidRecords(dat_interventional_pcd_raw[[RCON]]))
+  }
+  
+  # Cleanup that loop
+  rm(RCON)
+  
+  # Concatenate all data into a single table
+  dat_interventional_pcd_raw <- dat_interventional_pcd_raw %>% 
+    bind_rows()
+  
+  ## Update timestamp of extract
+  DATA_EXTRACT_TS_INT_M0 <- Sys.Date()
+}
+
+
+
+
+
+######################################################################
+### WRITE RAW DATA TO OUTPUT DIRECTORY
+######################################################################
+## Name of output files
+# Month 0 Active Follow-Up datasets
+INTERVENTIONAL_M0_OUT_01_FILENAME  <- paste0("INTERVENTIONAL_M0_OUT_01_raw_data", 
+                                             "_country-", 
+                                             paste(unique(dat_interventional_m0_raw$country), collapse = "-"), 
+                                             "_timestamp-", 
+                                             strftime(Sys.time(), format = "%Y%m%d_%H%M%S"), 
+                                             ".csv")
+
+# Passive Case Detection datasets
+INTERVENTIONAL_PCD_OUT_01_FILENAME <- paste0("INTERVENTIONAL_PCD_OUT_01_raw_data", 
+                                             "_country-", 
+                                             paste(unique(dat_interventional_pcd_raw$country), collapse = "-"), 
+                                             "_timestamp-", 
+                                             strftime(Sys.time(), format = "%Y%m%d_%H%M%S"), 
+                                             ".csv")
+
+## Write to output file
+# Month 0 Active Follow-Up datasets
+write.table(dat_interventional_m0_raw, 
+            file      = paste0("./03_INTERVENTIONAL_M0/outputs/", 
+                               INTERVENTIONAL_M0_OUT_01_FILENAME), 
+            sep       = ",", 
+            dec       = ".", 
+            quote     = TRUE, 
+            col.names = TRUE, 
+            row.names = FALSE)
+
+# Passive Case Detection datasets
+write.table(dat_interventional_pcd_raw, 
+            file      = paste0("./03_INTERVENTIONAL_M0/outputs/", 
+                               INTERVENTIONAL_PCD_OUT_01_FILENAME), 
+            sep       = ",", 
+            dec       = ".", 
+            quote     = TRUE, 
+            col.names = TRUE, 
+            row.names = FALSE)
+
+
+
+
+
+######################################################################
+### CLEANUP
+######################################################################
+rm(INTERVENTIONAL_M0_OUT_01_FILENAME, 
+   INTERVENTIONAL_PCD_OUT_01_FILENAME)
+
+
+
+
+
+######################################################################
+### UPDATE DATA_EXTRACT_IS_RECENT_INT_M0
+######################################################################
+DATA_EXTRACT_IS_RECENT_INT_M0    <- as.logical(difftime(time1 = Sys.Date(), 
+                                                        time2 = as.Date(ifelse(exists("DATA_EXTRACT_TS_INT_M0"), DATA_EXTRACT_TS_INT_M0, DATA_EXTRACT_TS_DEFAULT)), 
+                                                        units = "days") <= DATA_EXTRACT_EXPIRY_TIME_D)
+

03_INTERVENTIONAL_M0/outputs/INTERVENTIONAL_PCD_OUT_01_raw_data_country-_timestamp-20250519_115208.csv

+"record_id","date_saisie_pcd","user_pcd","consult_date","place_consulation","pid_number","subjectid","current_cluster","pfpanresult","pfpvresult","g6pd_result","bloodstage_ttt","primaquine_ttt","drugadmin_01_bs_galenic","drugadmin_01_bs_unit","drugadmin_01_pq_galenic","drugadmin_01_pq_unit","dbs_sticker","patient_initial","pcd_form_complete","data_source"
+"1",NA,NA,2025-05-15,"CSB VASIANA","P-50038",NA,"23","100","c",NA,NA,NA,NA,NA,NA,NA,NA,"FVI",3,"IPM"
+"2",NA,NA,2025-05-15,"AC RINDRA AMBATOLAHIHAZO","P-40083",NA,"23","100","c",NA,NA,NA,NA,NA,NA,NA,NA,"RJE",3,"IPM"
+"3",NA,NA,2025-05-15,"AC RINDRA AMBATOLAHIHAZO","P-12545",NA,"23","100","c",NA,NA,NA,NA,NA,NA,NA,NA,"RCL",3,"IPM"
+"4",NA,NA,2025-05-15,"AC RINDRA AMBATOLAHIHAZO","P-12519",NA,"23","100","c",NA,NA,NA,NA,NA,NA,NA,NA,"TCE",3,"IPM"
+"5",NA,NA,2025-05-15,"AC RINDRA AMBATOLAHIHAZO","P-17600",NA,"23","100","c",NA,NA,NA,NA,NA,NA,NA,NA,"RFA",3,"IPM"
+"6",NA,NA,2025-05-15,"AC RINDRA AMBATOLAHIHAZO","P-17551",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"RSU",3,"IPM"
+"7",NA,NA,2025-05-15,"CSB VASINA","P-12508",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"NOM",3,"IPM"
+"8",NA,NA,2025-05-15,"CSB VASINA","P-12509",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"SIL",3,"IPM"
+"9",NA,NA,2025-05-15,"CSB VASINA","P-20007",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"VKA",3,"IPM"
+"10",NA,NA,2025-05-16,"CSB II VASIANA ","P-35003","M-26-066","26","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"RMI",2,"IPM"
+"17",NA,NA,2025-05-16,"AC RINDRA AMBATOLAHIHAZO","P-45042",NA,"23","110","c/pv","Normal","3","7.5",NA,NA,NA,NA,"M-0028","AST",2,"IPM"
+"18",NA,NA,2025-05-16,"AC RINDRA AMBATOLAHIHAZO","P-20011",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"TNJ",2,"IPM"
+"19",NA,NA,2025-05-16,"AC RINDRA AMBATOLAHIHAZO","P-12578",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"NZO",2,"IPM"
+"20",NA,NA,2025-05-16,"AC RINDRA AMBATOLAHIHAZO","P-45038",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"TRA",2,"IPM"
+"21",NA,NA,2025-05-16,"AC RINDRA AMBATOLAHIHAZO","P-22551",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"RHO",2,"IPM"
+"22",NA,NA,2025-05-16,"AC RINDRA AMBATOLAHIHAZO","P-12581",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"BIE",2,"IPM"
+"23",NA,NA,2025-05-17,"CSB VASIANA","P-50044",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"VCR",2,"IPM"
+"24",NA,NA,2025-05-17,"AC RINDRA AMBATOLAHIHAZO","P-45027",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"RSE",2,"IPM"
+"25",NA,NA,2025-05-17,"AC RINDRA AMBATOLAHIHAZ0","P-20132",NA,"23","100",NA,NA,NA,NA,NA,NA,NA,NA,NA,"RTA",2,"IPM"