Initial commit for the 02_OBSERVATIONAL sub-project including setting up the R...

Initial commit for the 02_OBSERVATIONAL sub-project including setting up the R environment as well as dumping the observational database.

02_OBSERVATIONAL/.env.example

+# Institut Pasteur Paris
+REDCAP_IPP_API_URL='YOUR_REDCAP_API_URL_HERE'
+REDCAP_IPP_API_TOKEN_OBSERVATIONAL='YOUR_REDCAP_API_KEY_HERE'
+
+# Institut Pasteur de Madagascar
+REDCAP_IPM_API_URL='YOUR_REDCAP_API_URL_HERE'
+REDCAP_IPM_API_TOKEN_OBSERVATIONAL='YOUR_REDCAP_API_KEY_HERE'
+
+# Armauer Hansen Research Institute
+REDCAP_AHRI_API_URL='YOUR_REDCAP_API_URL_HERE'
+REDCAP_AHRI_API_TOKEN_OBSERVATIONAL='YOUR_REDCAP_API_KEY_HERE'

02_OBSERVATIONAL/OBSERVATIONAL_00_R_environment.R

+## OBSERVATIONAL_00_R_environment.R
+## Date     : 2024/10/17
+## Author   : Thomas Obadia
+##
+## This is a general script for the 02_INVENTORY/ R environment, that
+## will load all required packages into R and set global variables
+## used by other scripts.
+## 
+## You should usually not source this file alone: it is sourced in 
+## XXXXXX
+######################################################################
+
+
+
+
+
+######################################################################
+### REQUIRED PACKAGES
+######################################################################
+require(dotenv)              # Easy handling personal global variables
+require(redcapAPI)           # REDCap API methods
+require(tidyverse)           # Data manipulation made easy
+# require(OpenStreetMap)       # Set of methods for OpenStreetMap 
+# require(ggmap)               # More general set of methods
+
+
+
+
+
+######################################################################
+### SOURCE .env FILE
+######################################################################
+dotenv::load_dot_env("./02_OBSERVATIONAL/.env")
+
+
+
+
+
+######################################################################
+### DATA SOURCE TO BE USED
+######################################################################
+## Global flags that define which database should be dumped and which
+## will be ignored. There is one flag per data source: IPP, IPM, AHRI.
+#    - TRUE : data will be dumped
+#    - FALSE: data will not be dumped
+DATA_SOURCE_REDCAP_IPP_OBS     <- FALSE
+DATA_SOURCE_REDCAP_IPM_OBS     <- FALSE
+DATA_SOURCE_REDCAP_AHRI_OBS    <- TRUE
+
+
+
+
+
+######################################################################
+### DATA UP-TO-DATE FLAG
+######################################################################
+## Global flag that defines if the database dump is recent enough or
+## if a REDCap extract should be conducted again
+#    - TRUE : data is recent enough, don't dump
+#    - FALSE: new extract is necessary (default)
+DATA_EXTRACT_IS_RECENT     <- FALSE
+
+## Date at which the last extract was done. Defaults to Epoch time
+## in the R Environment file, so that a new dump is always done after
+## sourcing this file.
+DATA_EXTRACT_TS_DEFAULT    <- as.Date("1970-01-01")
+
+## Number of days after which database should be extracted again
+DATA_EXTRACT_EXPIRY_TIME_D <- 1
+
+
+
+
+
+######################################################################
+### SEED VALUE
+######################################################################
+## A global seed value to be used by set.seed() calls
+SEED <- 12345

02_OBSERVATIONAL/OBSERVATIONAL_01_dump_REDCap_database.R

+## OBSERVATIONAL_01_dump_REDCap_database.R
+## Date     : 2024/10/17
+## Author   : Thomas Obadia
+##
+## This script dumps the REDCap database for the PvSTATEM observational
+## project.
+## Note that it relies on the dotenv package in an effort to not 
+## incorporate API keys into the script.
+## Your API key is strictly personal and should never be shared !
+######################################################################
+
+
+
+
+
+######################################################################
+### SOURCE THE R ENVIRONMENT
+######################################################################
+source("./02_OBSERVATIONAL/OBSERVATIONAL_00_R_environment.R")
+
+
+
+
+
+######################################################################
+### CONNECTION TO REDCAP SERVER
+######################################################################
+## .env file should contain tokens for IPP, IPM and AHRI
+RCON_IPP_OBSERVATIONAL   <- NULL
+RCON_IPM_OBSERVATIONAL   <- NULL
+RCON_AHRI_OBSERVATIONAL  <- NULL
+
+if (DATA_SOURCE_REDCAP_IPP_OBS) {
+  RCON_IPP_OBSERVATIONAL  <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_IPP_API_URL"), 
+                                                         token = Sys.getenv("REDCAP_IPP_API_TOKEN_OBSERVATIONAL"))
+}
+if (DATA_SOURCE_REDCAP_IPM_OBS) {
+  RCON_IPM_OBSERVATIONAL  <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_IPM_API_URL"), 
+                                                         token = Sys.getenv("REDCAP_IPM_API_TOKEN_OBSERVATIONAL"))
+}
+if (DATA_SOURCE_REDCAP_AHRI_OBS) {
+  RCON_AHRI_OBSERVATIONAL <- redcapAPI::redcapConnection(url   = Sys.getenv("REDCAP_AHRI_API_URL"), 
+                                                         token = Sys.getenv("REDCAP_AHRI_API_TOKEN_OBSERVATIONAL"))
+}
+
+## Allocate one RCON object into a generic one for subsequent calls
+# RCON_OBSERVATIONAL      <- RCON_IPM_OBSERVATIONAL # Use for single-download tests
+RCON_OBSERVATIONAL <- list("IPP"  = RCON_IPP_OBSERVATIONAL, 
+                           "IPM"  = RCON_IPM_OBSERVATIONAL, 
+                           "AHRI" = RCON_AHRI_OBSERVATIONAL) %>% 
+  discard(is.null)
+
+
+
+
+
+######################################################################
+### DUMP DATABASE
+######################################################################
+## Check if data needs to be extracted again
+DATA_EXTRACT_IS_RECENT    <- as.logical(difftime(time1 = Sys.Date(), 
+                                                 time2 = as.Date(ifelse(exists("DATA_EXTRACT_TS"), DATA_EXTRACT_TS, DATA_EXTRACT_TS_DEFAULT)), 
+                                                 units = "days") <= DATA_EXTRACT_EXPIRY_TIME_D)
+
+## Use the exportRecordsTyped method from package redcapAPI
+if (!DATA_EXTRACT_IS_RECENT) {
+  # Initialize empty element for raw data
+  dat_observational_raw <- list()
+  
+  for (RCON in names(RCON_OBSERVATIONAL)) {
+    cat("Extracting data from RCON element: ", 
+        RCON, 
+        "...\n", 
+        sep = "")
+    
+    dat_observational_raw[[RCON]] <- redcapAPI::exportRecordsTyped(RCON_OBSERVATIONAL[[RCON]], 
+                                                                   cast = list(
+                                                                     # Radio buttons reported as label instead of code
+                                                                     "radio"    = castLabelCharacter, 
+                                                                     "yesno"    = castLabelCharacter, 
+                                                                     "dropdown" = castLabelCharacter
+                                                                   )) %>% 
+      
+      # Convert the '999' code to NA, except in Date/POSIX fields because it'll mess them up
+      mutate(across(-c(where(is.POSIXt), 
+                       where(is.Date)), function(x) (ifelse(x == 999, NA, x)))) %>% 
+      
+      # Add data source for convenience
+      mutate(data_source   = RCON)
+    
+    ## Check for invalid records before moving on
+    print(reviewInvalidRecords(dat_observational_raw[[RCON]]))
+  }
+  
+  ## Cleanup that loop
+  rm(RCON)
+  
+  ## Concatenate all data into a single table
+  dat_observational_raw <- dat_observational_raw %>% 
+    bind_rows()
+  
+  ## Update timestamp of extract
+  DATA_EXTRACT_TS <- Sys.Date()
+}
+
+
+
+
+
+######################################################################
+### UPDATE DATA_EXTRACT_IS_RECENT
+######################################################################
+DATA_EXTRACT_IS_RECENT    <- as.logical(difftime(time1 = Sys.Date(), 
+                                                 time2 = as.Date(ifelse(exists("DATA_EXTRACT_TS"), DATA_EXTRACT_TS, DATA_EXTRACT_TS_DEFAULT)), 
+                                                 units = "days") <= DATA_EXTRACT_EXPIRY_TIME_D)

02_OBSERVATIONAL/outputs/.gitignore

+.DS_Store
+*.csv