Commit 047ba084 authored by Christian Ruiz's avatar Christian Ruiz
Browse files

Merge branch 'dev' into 'master'

Dev

See merge request christian.ruiz/statbot-swiss!118
parents 81fcc501 d368c472
Pipeline #435305 failed with stages
in 0 seconds
......@@ -2,6 +2,10 @@ variables:
GIT_STRATEGY: fetch
GIT_SSL_NO_VERIFY: "true"
GIT_LFS_SKIP_SMUDGE: 1
arr: ($(cat config.yml | grep "PATH_NEW_DATASETS:"))
PATH_NEW_DATASETS: ${arr[1]}
arr: ($(cat config.yml | grep "PATH_GEN_DATASETS:"))
PATH_GEN_DATASETS: ${arr[1]}
stages:
- main
......@@ -27,7 +31,7 @@ trigger_feature:
- if: '$CI_COMMIT_BRANCH =~ /^feature_/ && $CI_PIPELINE_SOURCE != "merge_request_event"'
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'dev' && $CI_MERGE_REQUEST_ID
changes:
- inst/extdata/new_datasets/*
- $PATH_NEW_DATASETS*
trigger_on_merge_request:
......@@ -38,7 +42,7 @@ trigger_on_merge_request:
rules:
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'master' && $CI_MERGE_REQUEST_ID
changes:
- inst/extdata/new_datasets/*
- $PATH_NEW_DATASETS*
trigger_downstream:
......
# Version 4.3.7 - 12.10.2022
- Separation into two renku datasets (additional to the statbot-core): data and metadata
- Adaptation of all the codes towards the above separating into metadata
- PATH_NEW_DATASETS and PATH_GEN_DATASETS set
# Version 4.3.6 - 06.09.2022
- Heavy Cleanup
......
......@@ -31,7 +31,7 @@ dimension_file_init <- function(dataset_id, author) {
load_gs(dataset_id = dataset_id)
ds <- dataset_from_yaml(paste0("inst/extdata/new_datasets/", dataset_id, ".yml"))
ds <- dataset_from_yaml(paste0(PATH_NEW_DATASETS, dataset_id, ".yml"))
if (ds$institution == "bfs") {
template_path <- "inst/scripts/templates/template_dimensions.R"
......
......@@ -9,6 +9,9 @@ VALID_ONTOLOGIES <- unlist(constants$SPATIAL_UNIT_ONTOLOGY, use.names = F)
UNICODES <- constants$UNICODES
PATH_NEW_DATASETS<-yaml::yaml.load_file("config.yml")$PATH_NEW_DATASETS
PATH_GEN_DATASETS<-yaml::yaml.load_file("config.yml")$PATH_GEN_DATASETS
month_tibble <- tibble::tribble(
~Monat, ~month_id,
......
......@@ -32,7 +32,7 @@ load_gs <- function(dataset_id = NULL, prio = -1) {
df <- df[!is.na(df$id), ]
# clean up first - we do not lose anything -they are all auto-generated
#do.call(file.remove, list(list.files("inst/extdata/new_datasets/", full.names = TRUE)))
#do.call(file.remove, list(list.files(PATH_NEW_DATASETS, full.names = TRUE)))
# convert these two columns to characters in case they are logical
# the reason is a temporary fix because the cols are still empty and thus he sees them as logical
......@@ -105,8 +105,8 @@ load_gs <- function(dataset_id = NULL, prio = -1) {
flag_constructor = ds$flag_constructor,
updated = NA_character_
)
dir.create("inst/extdata/new_datasets", showWarnings = F)
write_yaml(out, paste0("inst/extdata/new_datasets/", ds$id, ".yml"), precision = 1)
dir.create(PATH_NEW_DATASETS, showWarnings = F)
write_yaml(out, paste0(PATH_NEW_DATASETS, ds$id, ".yml"), precision = 1)
}
classes <- apply(classes, 2, as.character)
......
PATH_NEW_DATASETS: "inst/extdata/new_datasets/"
PATH_GEN_DATASETS: "inst/extdata/gen_datasets/"
No preview for this file type
id: 1_01_009_CH
source_id:
- px-x-0102020202_106
name:
en: Marriages
de: Eheschliessungen
unit_short:
de: Ehen
read_class: px
unique_dimension_names:
- age_class_husband_marriages
- citizenship_selection_husband_alt_names
- marital_status_husband
- age_class_wife_marriages
- citizenship_selection_wife
- marital_status_wife
modules:
- etl_spatialunit
- join_dimension_values
- final
institution: bfs
smallest_spatial_entity: ch
time_col: Jahr
time_interval: year
institution_dim_names: .na.character
spatial_col: .na.character
obs_value_col: .na.character
description:
de: Heiraten nach gegenseitiger Altersklasse, Staatsangehörigkeit (Auswahl) und
Zivilstand der Ehepartner, 2007-2020
unit_long:
de: Eheschliessungen
class_name: marriages_ageclass_citizenship_marital_status
source: BFS-BEVNAT
separator_var: .na.character
melt_vars: .na.character
max_year: .na
set_totals: .na.character
question_type: '2'
spatial_filter: .na.character
filter: .na
prio: 2.0
version:
- 4.2
flag_constructor: .na.character
updated: .na.character
......@@ -23,7 +23,7 @@ if(!file.exists(destfile)){
df <- read.csv2(destfile)
ds<-dataset_from_yaml(paste0("inst/extdata/new_datasets/1_01_161_TG.yml"))
ds<-dataset_from_yaml(paste0(PATH_NEW_DATASETS,"1_01_161_TG.yml"))
......
......@@ -24,7 +24,7 @@ if(!file.exists(destfile)){
df <- read.csv2(destfile)
ds<-dataset_from_yaml(paste0("inst/extdata/new_datasets/1_07_023_BL.yml"))
ds<-dataset_from_yaml(paste0(PATH_NEW_DATASETS,"1_07_023_BL.yml"))
dim_table<-combine_dimension_tables()
......
......@@ -75,7 +75,7 @@ ui <- fluidPage(theme = shinytheme("cerulean"),
sidebarLayout(
sidebarPanel(
selectInput("dataset_selected", "Dataset:",
c("",tools::file_path_sans_ext(list.files("~/work/statbot-swiss/inst/extdata/new_datasets"))),
c("",tools::file_path_sans_ext(list.files(paste0("~/work/statbot-swiss/",PATH_NEW_DATASETS))),
selected = F),
selectInput("lang", "Language of your institution:",
c("","English" = "en",
......@@ -243,7 +243,7 @@ server <- function(input, output,session) {
}else{
ds <- dataset_from_yaml(paste0("~/work/statbot-swiss/inst/extdata/new_datasets/", input$dataset_selected, ".yml"))
ds <- dataset_from_yaml(paste0("~/work/statbot-swiss/",PATH_NEW_DATASETS, input$dataset_selected, ".yml"))
destfile<-paste0("~/work/statbot-swiss/temp/",ds$source_id,".",ds$read_class)
......
......@@ -2,7 +2,7 @@ devtools::load_all(".")
dir.create("temp", showWarnings = F)
res <-list.files("/home/rstudio/work/statbot-swiss/inst/extdata/new_datasets", pattern = ".yml", full.names = T)
res <-list.files(PATH_NEW_DATASETS, pattern = ".yml", full.names = T)
#res <- res[!grepl("OLD", res)]
process_multiple_ds(res)
......@@ -11,7 +11,7 @@ process_multiple_ds(res[1:25])
process_multiple_ds(res[26])
process_multiple_ds(res[27:174])
process_ds("inst/extdata/new_datasets/1_01_001_BS.yml")
process_ds(paste0(PATH_NEW_DATASETS,"1_01_009_CH.yml"))
......
......@@ -24,7 +24,7 @@ if(!file.exists(destfile)){
df <- read.csv2(destfile)
ds<-dataset_from_yaml(paste0("inst/extdata/new_datasets/{{dataset_id}}.yml"))
ds<-dataset_from_yaml(paste0(PATH_NEW_DATASETS,"{{dataset_id}}.yml"))
......
echo "Executing Update Script for $1"
renku dataset ls-files statbot-prio1 -c path | tail -n +3 | xargs -I {} renku dataset unlink $1 -yI {}
cp -r inst/extdata/new_datasets/* data/$1/
cp -r $PATH_NEW_DATASETS* data/$1/
cp -r data/workflow_output/* data/$1/
cp -r data/dimensions/* data/$1/
echo "Copying done"
......
......@@ -2,9 +2,14 @@
set -euo pipefail
mkdir -p inst/extdata/new_datasets
mkdir -p inst/extdata/gen_datasets
cp inst/extdata/new_datasets/* inst/extdata/gen_datasets/
arr=($(cat config.yml | grep "PATH_NEW_DATASETS:"))
PATH_NEW_DATASETS= ${arr[1]}
arr=($(cat config.yml | grep "PATH_GEN_DATASETS:"))
PATH_GEN_DATASETS= ${arr[1]}
mkdir -p $PATH_NEW_DATASETS
mkdir -p $PATH_GEN_DATASETS
cp $PATH_NEW_DATASETS* $PATH_GEN_DATASETS
echo "Run script update_full_fimension_table"
Rscript update_full_dimension_table.R
......@@ -19,13 +24,13 @@ git commit -m "add new datasets to gen_datasets"
echo "Git commiting finished. Entering first loop now."
for yaml_path in inst/extdata/new_datasets/*; do
for yaml_path in $PATH_NEW_DATASETS*; do
echo "Processing $yaml_path"
yaml_file=${yaml_path##*/}
new_yaml_path="inst/extdata/gen_datasets/${yaml_file}"
new_yaml_path="${PATH_GEN_DATASETS}${yaml_file}"
dataset_name=${yaml_file%.yml}
......@@ -42,7 +47,7 @@ echo "First loop finished. Entering second loop now."
for yaml_path in inst/extdata/new_datasets/*; do
for yaml_path in $PATH_NEW_DATASETS*; do
echo "Processing $yaml_path"
......@@ -51,7 +56,7 @@ for yaml_path in inst/extdata/new_datasets/*; do
# loop over the new datasets for 1:n datasets
dataset_to_add="data/statbot-datawarehouse/${dataset_name}/${dataset_name}-0.parquet"
yaml_to_add="inst/extdata/gen_datasets/${dataset_name}.yml"
yaml_to_add="${PATH_GEN_DATASETS}${dataset_name}.yml"
renku dataset add -o --copy statbot-datawarehouse $dataset_to_add
renku dataset add -o --copy statbot-metadata $yaml_to_add
......@@ -59,5 +64,5 @@ for yaml_path in inst/extdata/new_datasets/*; do
done
rm inst/extdata/new_datasets/*
rm $PATH_NEW_DATASETS*
......@@ -5,7 +5,7 @@ print("test")
write_full_dimension_file()
new_files <- list.files("inst/extdata/new_datasets", full.names = T)
new_files <- list.files(PATH_NEW_DATASETS, full.names = T)
print(length(new_files))
......
......@@ -19,7 +19,7 @@ test_that("datasets are correctly initialized", {
})
test_that("datasets metadata files respect the schema", {
res <-list.files("/home/rstudio/work/statbot-swiss/inst/extdata/gen_datasets", pattern = ".yml", full.names = T)
res <-list.files(paste0("/home/rstudio/work/statbot-swiss/",PATH_GEN_DATASETS), pattern = ".yml", full.names = T)
res <- res %>%
purrr::map(dataset_from_yaml) %>%
......
......@@ -8,7 +8,7 @@ print(yaml_path)
process_ds(yaml_path)
# new_data <- list.files("inst/extdata/new_datasets")
# new_data <- list.files(PATH_NEW_DATASETS)
#
# dataset_paths <- paste0("data/statbot_datawarehouse/", gsub("\\.yaml", "", new_data),"/")
#
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment