Merge branch 'dev' into 'master'

Dev

See merge request christian.ruiz/statbot-swiss!118

.gitlab-ci.yml

@@ -2,6 +2,10 @@ variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_LFS_SKIP_SMUDGE: 1
+  arr: ($(cat config.yml | grep "PATH_NEW_DATASETS:"))
+  PATH_NEW_DATASETS: ${arr[1]}
+  arr: ($(cat config.yml | grep "PATH_GEN_DATASETS:"))
+  PATH_GEN_DATASETS: ${arr[1]}
 
 stages:
   - main
@@ -27,7 +31,7 @@ trigger_feature:
     - if: '$CI_COMMIT_BRANCH =~ /^feature_/ && $CI_PIPELINE_SOURCE != "merge_request_event"'
     - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'dev' && $CI_MERGE_REQUEST_ID
       changes:
-        - inst/extdata/new_datasets/*
+        - $PATH_NEW_DATASETS*
 
 
 trigger_on_merge_request:
@@ -38,7 +42,7 @@ trigger_on_merge_request:
   rules:
     - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'master' && $CI_MERGE_REQUEST_ID
       changes:
-        - inst/extdata/new_datasets/*
+        - $PATH_NEW_DATASETS*
  
 
 trigger_downstream:

NEWS.md

 # Version 4.3.7 - 12.10.2022
 - Separation into two renku datasets (additional to the statbot-core): data and metadata
 - Adaptation of all the codes towards the above separating into metadata 
+- PATH_NEW_DATASETS and PATH_GEN_DATASETS set
 
 # Version 4.3.6 - 06.09.2022
 - Heavy Cleanup

R/dimensions_template.R

@@ -31,7 +31,7 @@ dimension_file_init <- function(dataset_id, author) {
   load_gs(dataset_id = dataset_id)
 
 
-  ds <- dataset_from_yaml(paste0("inst/extdata/new_datasets/", dataset_id, ".yml"))
+  ds <- dataset_from_yaml(paste0(PATH_NEW_DATASETS, dataset_id, ".yml"))
 
   if (ds$institution == "bfs") {
     template_path <- "inst/scripts/templates/template_dimensions.R"

R/global.R

@@ -9,6 +9,9 @@ VALID_ONTOLOGIES <- unlist(constants$SPATIAL_UNIT_ONTOLOGY, use.names = F)
 
 UNICODES <- constants$UNICODES
 
+PATH_NEW_DATASETS<-yaml::yaml.load_file("config.yml")$PATH_NEW_DATASETS
+PATH_GEN_DATASETS<-yaml::yaml.load_file("config.yml")$PATH_GEN_DATASETS
+
 
 month_tibble <- tibble::tribble(
   ~Monat, ~month_id,

R/yaml_from_googlesheet.R

@@ -32,7 +32,7 @@ load_gs <- function(dataset_id = NULL, prio = -1) {
   df <- df[!is.na(df$id), ]
 
   # clean up first - we do not lose anything -they are all auto-generated
-  #do.call(file.remove, list(list.files("inst/extdata/new_datasets/", full.names = TRUE)))
+  #do.call(file.remove, list(list.files(PATH_NEW_DATASETS, full.names = TRUE)))
 
   # convert these two columns to characters in case they are logical
   # the reason is a temporary fix because the cols are still empty and thus he sees them as logical
@@ -105,8 +105,8 @@ load_gs <- function(dataset_id = NULL, prio = -1) {
       flag_constructor = ds$flag_constructor,
       updated = NA_character_
     )
-    dir.create("inst/extdata/new_datasets", showWarnings = F)
-    write_yaml(out, paste0("inst/extdata/new_datasets/", ds$id, ".yml"), precision = 1)
+    dir.create(PATH_NEW_DATASETS, showWarnings = F)
+    write_yaml(out, paste0(PATH_NEW_DATASETS, ds$id, ".yml"), precision = 1)
   }
 
   classes <- apply(classes, 2, as.character)

config.yml

+PATH_NEW_DATASETS: "inst/extdata/new_datasets/"
+PATH_GEN_DATASETS: "inst/extdata/gen_datasets/"

inst/extdata/new_datasets/1_01_009_CH.yml

+id: 1_01_009_CH
+source_id:
+- px-x-0102020202_106
+name:
+  en: Marriages
+  de: Eheschliessungen
+unit_short:
+  de: Ehen
+read_class: px
+unique_dimension_names:
+- age_class_husband_marriages
+- citizenship_selection_husband_alt_names
+- marital_status_husband
+- age_class_wife_marriages
+- citizenship_selection_wife
+- marital_status_wife
+modules:
+- etl_spatialunit
+- join_dimension_values
+- final
+institution: bfs
+smallest_spatial_entity: ch
+time_col: Jahr
+time_interval: year
+institution_dim_names: .na.character
+spatial_col: .na.character
+obs_value_col: .na.character
+description:
+  de: Heiraten nach gegenseitiger Altersklasse, Staatsangehörigkeit (Auswahl) und
+    Zivilstand der Ehepartner, 2007-2020
+unit_long:
+  de: Eheschliessungen
+class_name: marriages_ageclass_citizenship_marital_status
+source: BFS-BEVNAT
+separator_var: .na.character
+melt_vars: .na.character
+max_year: .na
+set_totals: .na.character
+question_type: '2'
+spatial_filter: .na.character
+filter: .na
+prio: 2.0
+version:
+- 4.2
+flag_constructor: .na.character
+updated: .na.character

inst/scripts/dim/dim_1_01_161_TG.R

@@ -23,7 +23,7 @@ if(!file.exists(destfile)){
 
 
 df <- read.csv2(destfile)
-ds<-dataset_from_yaml(paste0("inst/extdata/new_datasets/1_01_161_TG.yml"))
+ds<-dataset_from_yaml(paste0(PATH_NEW_DATASETS,"1_01_161_TG.yml"))
 
 
 

inst/scripts/dim/dim_1_07_023_BL.R

@@ -24,7 +24,7 @@ if(!file.exists(destfile)){
 
 
 df <- read.csv2(destfile)
-ds<-dataset_from_yaml(paste0("inst/extdata/new_datasets/1_07_023_BL.yml"))
+ds<-dataset_from_yaml(paste0(PATH_NEW_DATASETS,"1_07_023_BL.yml"))
 dim_table<-combine_dimension_tables()
 
 

inst/scripts/dim_helper/app.R

@@ -75,7 +75,7 @@ ui <- fluidPage(theme = shinytheme("cerulean"),
     sidebarLayout(
       sidebarPanel(
         selectInput("dataset_selected", "Dataset:",
-                    c("",tools::file_path_sans_ext(list.files("~/work/statbot-swiss/inst/extdata/new_datasets"))),
+                    c("",tools::file_path_sans_ext(list.files(paste0("~/work/statbot-swiss/",PATH_NEW_DATASETS))),
                     selected = F),
         selectInput("lang", "Language of your institution:",
                     c("","English" = "en",
@@ -243,7 +243,7 @@ server <- function(input, output,session) {
       
       }else{
         
-       ds <- dataset_from_yaml(paste0("~/work/statbot-swiss/inst/extdata/new_datasets/", input$dataset_selected, ".yml"))
+       ds <- dataset_from_yaml(paste0("~/work/statbot-swiss/",PATH_NEW_DATASETS, input$dataset_selected, ".yml"))
       
       destfile<-paste0("~/work/statbot-swiss/temp/",ds$source_id,".",ds$read_class)
       

inst/scripts/new_main.R

@@ -2,7 +2,7 @@ devtools::load_all(".")
 
 dir.create("temp", showWarnings = F)
 
-res <-list.files("/home/rstudio/work/statbot-swiss/inst/extdata/new_datasets", pattern = ".yml", full.names = T) 
+res <-list.files(PATH_NEW_DATASETS, pattern = ".yml", full.names = T) 
 #res <- res[!grepl("OLD", res)]
 
 process_multiple_ds(res)
@@ -11,7 +11,7 @@ process_multiple_ds(res[1:25])
 process_multiple_ds(res[26])
 process_multiple_ds(res[27:174])
 
-process_ds("inst/extdata/new_datasets/1_01_001_BS.yml")
+process_ds(paste0(PATH_NEW_DATASETS,"1_01_009_CH.yml"))
 
 
 

inst/scripts/templates/template_dim_regional.R

@@ -24,7 +24,7 @@ if(!file.exists(destfile)){
 
 
 df <- read.csv2(destfile)
-ds<-dataset_from_yaml(paste0("inst/extdata/new_datasets/{{dataset_id}}.yml"))
+ds<-dataset_from_yaml(paste0(PATH_NEW_DATASETS,"{{dataset_id}}.yml"))
 
 
 

inst/scripts/update_prio.sh

 
 echo "Executing Update Script for $1"
 renku dataset ls-files statbot-prio1 -c path | tail -n +3 | xargs -I {} renku dataset unlink $1 -yI {}
-cp -r inst/extdata/new_datasets/* data/$1/
+cp -r $PATH_NEW_DATASETS* data/$1/
 cp -r data/workflow_output/* data/$1/
 cp -r data/dimensions/* data/$1/ 
 echo "Copying done"

process_dataset.sh

@@ -2,9 +2,14 @@
 
 set -euo pipefail
 
-mkdir -p inst/extdata/new_datasets
-mkdir -p inst/extdata/gen_datasets
-cp inst/extdata/new_datasets/* inst/extdata/gen_datasets/
+arr=($(cat config.yml | grep "PATH_NEW_DATASETS:"))
+PATH_NEW_DATASETS= ${arr[1]}
+arr=($(cat config.yml | grep "PATH_GEN_DATASETS:"))
+PATH_GEN_DATASETS= ${arr[1]}
+
+mkdir -p $PATH_NEW_DATASETS
+mkdir -p $PATH_GEN_DATASETS
+cp $PATH_NEW_DATASETS* $PATH_GEN_DATASETS
 
 echo "Run script update_full_fimension_table"
 Rscript update_full_dimension_table.R
@@ -19,13 +24,13 @@ git commit -m "add new datasets to gen_datasets"
 echo "Git commiting finished. Entering first loop now."
 
 
-for yaml_path in inst/extdata/new_datasets/*; do
+for yaml_path in $PATH_NEW_DATASETS*; do
 
   echo "Processing $yaml_path"
 
   yaml_file=${yaml_path##*/}
 
-  new_yaml_path="inst/extdata/gen_datasets/${yaml_file}"
+  new_yaml_path="${PATH_GEN_DATASETS}${yaml_file}"
 
   dataset_name=${yaml_file%.yml}
   
@@ -42,7 +47,7 @@ echo "First loop finished. Entering second loop now."
 
 
 
-for yaml_path in inst/extdata/new_datasets/*; do
+for yaml_path in $PATH_NEW_DATASETS*; do
 
   echo "Processing $yaml_path"
   
@@ -51,7 +56,7 @@ for yaml_path in inst/extdata/new_datasets/*; do
  
   # loop over the new datasets for 1:n datasets
   dataset_to_add="data/statbot-datawarehouse/${dataset_name}/${dataset_name}-0.parquet"
-  yaml_to_add="inst/extdata/gen_datasets/${dataset_name}.yml"
+  yaml_to_add="${PATH_GEN_DATASETS}${dataset_name}.yml"
 
   renku dataset add -o --copy statbot-datawarehouse $dataset_to_add
   renku dataset add -o --copy statbot-metadata $yaml_to_add
@@ -59,5 +64,5 @@ for yaml_path in inst/extdata/new_datasets/*; do
   
 done
 
-rm inst/extdata/new_datasets/*
+rm $PATH_NEW_DATASETS*
 

test_create_dataset.R

@@ -5,7 +5,7 @@ print("test")
 write_full_dimension_file()
 
 
-new_files <- list.files("inst/extdata/new_datasets", full.names = T)
+new_files <- list.files(PATH_NEW_DATASETS, full.names = T)
 
 print(length(new_files))
 

tests/testthat/test-datasets.R

@@ -19,7 +19,7 @@ test_that("datasets are correctly initialized", {
 })
 
 test_that("datasets metadata files respect the schema", {
-  res <-list.files("/home/rstudio/work/statbot-swiss/inst/extdata/gen_datasets", pattern = ".yml", full.names = T)
+  res <-list.files(paste0("/home/rstudio/work/statbot-swiss/",PATH_GEN_DATASETS), pattern = ".yml", full.names = T)
   
   res <- res %>% 
     purrr::map(dataset_from_yaml) %>%

update_dataset.R

@@ -8,7 +8,7 @@ print(yaml_path)
 process_ds(yaml_path)
 
 
-# new_data <- list.files("inst/extdata/new_datasets")
+# new_data <- list.files(PATH_NEW_DATASETS)
 # 
 # dataset_paths <- paste0("data/statbot_datawarehouse/", gsub("\\.yaml", "", new_data),"/")
 #