Merge branch 'AS_redo_renku_datasets' into 'master'

redo renku datasets Closes #5, #3, and s2s-ai-challenge#22 See merge request !14

Merge branch 'AS_redo_renku_datasets' into 'master'
redo renku datasets Closes #5, #3, and s2s-ai-challenge#22 See merge request !14
48fd739a · Aaron Spring · 23e55e79 · 76645b2d · 48fd739a · 48fd739a
Commit 48fd739a authored Jun 19, 2021 by Aaron Spring 🚼
--- a/.gitattributes
+++ b/.gitattributes
@@ -437,3 +437,91 @@ data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.8.0.0 f
 data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.7.0.0 filter=lfs diff=lfs merge=lfs -text
 data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
 data/ecmwf_recalibrated_benchmark_2020_biweekly_terciled.nc filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
--- a/.renku/datasets/a31a4922-5cbd-4d8e-bee3-435b1fd85282/metadata.yml
+++ b/.renku/datasets/a31a4922-5cbd-4d8e-bee3-435b1fd85282/metadata.yml
--- a/.renku/metadata.yml
+++ b/.renku/metadata.yml
@@ -2,7 +2,7 @@
 '@type':
 - http://schema.org/Project
 - http://www.w3.org/ns/prov#Location
-http://schema.org/agent: 0.15.1
+http://schema.org/agent: 0.14.2
 http://schema.org/creator:
  '@id': mailto:aaron.spring@mpimet.mpg.de
  '@type':

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,10 +2,11 @@

 ### unreleased

+- Recreate biweekly renku datasets with `float32` single precision and newly uploaded data to [`climetlab_s2s_ai_challenge`](https://github.com/ecmwf-lab/climetlab-s2s-ai-challenge/releases/tag/0.7.0). Before `lead_time` in `tp` had a one day shift. These new renku datasets will be used by the [`s2saichallengescorer`](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/3). (#7, !9, https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/22, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
 - Order of processing gridded `RPSS` to final score: (#7, !9, [s2s-ai-competition-scoring-image!2](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/2), [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
    1. `RPSS`
-    2. penalize #7 
-    3. `clip(-10,1)`
+    2. penalize where `NaN` submitted but value expected #7 
+    3. `clip(-10,1)`: prevent too negative values
    4. mean over `forecast_time`
    5. spatially weighted mean [90N-60S]
    6. mean over `lead_time` and `data_vars`

--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zmetadata
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zmetadata
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/forecast_time/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/forecast_time/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/lead_time/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/lead_time/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zarray
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zarray
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.0.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.0.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.1.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.1.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.2.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.2.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.3.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.3.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.4.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.4.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/.zarray
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/.zarray
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/0.0.0.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/0.0.0.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/0.0.1.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/0.0.1.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/0.0.2.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/0.0.2.0.0