From ca2d09c99f541c3d4ac67844360733d35178d825 Mon Sep 17 00:00:00 2001 From: "CR (covid cron)" Date: Thu, 24 Sep 2020 07:36:24 +0000 Subject: [PATCH] renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json --- .../14104787d8d6409eb91c0bf4a5a2022e.cwl | 120 ++++++++++++++++ data/covidtracking/states-daily.json | 4 +- data/covidtracking/states-metadata.json | 4 +- runs/download-covidtracking-data.runs.ipynb | 128 +++++++++--------- 4 files changed, 188 insertions(+), 68 deletions(-) create mode 100644 .renku/workflow/14104787d8d6409eb91c0bf4a5a2022e.cwl diff --git a/.renku/workflow/14104787d8d6409eb91c0bf4a5a2022e.cwl b/.renku/workflow/14104787d8d6409eb91c0bf4a5a2022e.cwl new file mode 100644 index 000000000..29ae3f38d --- /dev/null +++ b/.renku/workflow/14104787d8d6409eb91c0bf4a5a2022e.cwl @@ -0,0 +1,120 @@ +class: Workflow +cwlVersion: v1.0 +hints: [] +inputs: + input_1: + default: out_folder + streamable: false + type: string + input_2: + default: data/covidtracking + streamable: false + type: string + input_3: + default: + class: File + path: ../../notebooks/process/download-covidtracking-data.ipynb + streamable: false + type: File + input_4: + default: runs/download-covidtracking-data.runs.ipynb + streamable: false + type: string + input_5: + default: states-daily.json + streamable: false + type: string + input_6: + default: states-metadata.json + streamable: false + type: string +outputs: + output_1: + outputSource: step_1/output_0 + streamable: false + type: File + output_3: + outputSource: step_1/output_1 + streamable: false + type: Directory +requirements: [] +steps: + step_1: + in: + input_1: input_1 + input_2: input_2 + input_3: input_3 + input_4: input_4 + out: + - output_0 + - output_1 + run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl + step_2: + in: + filename: input_5 + input_directory: step_1/output_1 + out: + - output_file + run: + arguments: [] + baseCommand: + - 'true' + class: CommandLineTool + cwlVersion: v1.0 + hints: [] + inputs: + filename: + default: states-daily.json + streamable: false + type: string + input_directory: + streamable: false + type: Directory + outputs: + output_file: + outputBinding: + glob: $(inputs.filename) + streamable: false + type: File + permanentFailCodes: [] + requirements: + - &id001 + class: InlineJavascriptRequirement + - &id002 + class: InitialWorkDirRequirement + listing: $(inputs.input_directory.listing) + successCodes: [] + temporaryFailCodes: [] + step_3: + in: + filename: input_6 + input_directory: step_1/output_1 + out: + - output_file + run: + arguments: [] + baseCommand: + - 'true' + class: CommandLineTool + cwlVersion: v1.0 + hints: [] + inputs: + filename: + default: states-metadata.json + streamable: false + type: string + input_directory: + streamable: false + type: Directory + outputs: + output_file: + outputBinding: + glob: $(inputs.filename) + streamable: false + type: File + permanentFailCodes: [] + requirements: + - *id001 + - *id002 + successCodes: [] + temporaryFailCodes: [] diff --git a/data/covidtracking/states-daily.json b/data/covidtracking/states-daily.json index 8cd963bef..fc773df25 100644 --- a/data/covidtracking/states-daily.json +++ b/data/covidtracking/states-daily.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3a2d54ba98b1707b5ab739f7a08d04feaa9ef0261ba8637fd953675d4b52103c -size 15133509 +oid sha256:ffad157a714e91e50e3420067f6bc43529fdcb19735766903153628ec7bab103 +size 15208829 diff --git a/data/covidtracking/states-metadata.json b/data/covidtracking/states-metadata.json index 39f1b7dab..b3cfc9ac4 100644 --- a/data/covidtracking/states-metadata.json +++ b/data/covidtracking/states-metadata.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:132b77335850da7e0708e078078c476d7595248af97bba2edf4e495a335e2c6e -size 81151 +oid sha256:74b334c56573d929a59f7cb8efa7d2e218a05d81060ac6d287f0704d0db9f40b +size 81413 diff --git a/runs/download-covidtracking-data.runs.ipynb b/runs/download-covidtracking-data.runs.ipynb index 5ebb6f100..18fe29ee3 100644 --- a/runs/download-covidtracking-data.runs.ipynb +++ b/runs/download-covidtracking-data.runs.ipynb @@ -5,10 +5,10 @@ "execution_count": 1, "metadata": { "papermill": { - "duration": 4.181312, - "end_time": "2020-09-23T07:35:21.010050", + "duration": 3.711103, + "end_time": "2020-09-24T07:35:55.897522", "exception": false, - "start_time": "2020-09-23T07:35:16.828738", + "start_time": "2020-09-24T07:35:52.186419", "status": "completed" }, "tags": [] @@ -25,10 +25,10 @@ "execution_count": 2, "metadata": { "papermill": { - "duration": 0.01892, - "end_time": "2020-09-23T07:35:21.042851", + "duration": 0.027146, + "end_time": "2020-09-24T07:35:55.939885", "exception": false, - "start_time": "2020-09-23T07:35:21.023931", + "start_time": "2020-09-24T07:35:55.912739", "status": "completed" }, "tags": [ @@ -46,10 +46,10 @@ "execution_count": 3, "metadata": { "papermill": { - "duration": 0.019395, - "end_time": "2020-09-23T07:35:21.074425", + "duration": 0.031292, + "end_time": "2020-09-24T07:35:55.985668", "exception": false, - "start_time": "2020-09-23T07:35:21.055030", + "start_time": "2020-09-24T07:35:55.954376", "status": "completed" }, "tags": [ @@ -59,7 +59,7 @@ "outputs": [], "source": [ "# Parameters\n", - "PAPERMILL_INPUT_PATH = \"/tmp/o8q1c8tx/notebooks/process/download-covidtracking-data.ipynb\"\n", + "PAPERMILL_INPUT_PATH = \"/tmp/hhqn2uzg/notebooks/process/download-covidtracking-data.ipynb\"\n", "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n", "out_folder = \"data/covidtracking\"\n" ] @@ -68,10 +68,10 @@ "cell_type": "markdown", "metadata": { "papermill": { - "duration": 0.007765, - "end_time": "2020-09-23T07:35:21.089543", + "duration": 0.008503, + "end_time": "2020-09-24T07:35:56.008313", "exception": false, - "start_time": "2020-09-23T07:35:21.081778", + "start_time": "2020-09-24T07:35:55.999810", "status": "completed" }, "tags": [] @@ -87,10 +87,10 @@ "execution_count": 4, "metadata": { "papermill": { - "duration": 0.628909, - "end_time": "2020-09-23T07:35:21.725778", + "duration": 0.325836, + "end_time": "2020-09-24T07:35:56.341140", "exception": false, - "start_time": "2020-09-23T07:35:21.096869", + "start_time": "2020-09-24T07:35:56.015304", "status": "completed" }, "tags": [] @@ -107,10 +107,10 @@ "execution_count": 5, "metadata": { "papermill": { - "duration": 0.018332, - "end_time": "2020-09-23T07:35:21.755795", + "duration": 0.028059, + "end_time": "2020-09-24T07:35:56.382919", "exception": false, - "start_time": "2020-09-23T07:35:21.737463", + "start_time": "2020-09-24T07:35:56.354860", "status": "completed" }, "tags": [] @@ -129,10 +129,10 @@ "execution_count": 6, "metadata": { "papermill": { - "duration": 0.073565, - "end_time": "2020-09-23T07:35:21.836599", + "duration": 0.079429, + "end_time": "2020-09-24T07:35:56.475405", "exception": false, - "start_time": "2020-09-23T07:35:21.763034", + "start_time": "2020-09-24T07:35:56.395976", "status": "completed" }, "tags": [] @@ -270,10 +270,10 @@ "cell_type": "markdown", "metadata": { "papermill": { - "duration": 0.007067, - "end_time": "2020-09-23T07:35:21.854979", + "duration": 0.007668, + "end_time": "2020-09-24T07:35:56.497150", "exception": false, - "start_time": "2020-09-23T07:35:21.847912", + "start_time": "2020-09-24T07:35:56.489482", "status": "completed" }, "tags": [] @@ -287,10 +287,10 @@ "execution_count": 7, "metadata": { "papermill": { - "duration": 25.11081, - "end_time": "2020-09-23T07:35:46.972922", + "duration": 25.363958, + "end_time": "2020-09-24T07:36:21.868028", "exception": false, - "start_time": "2020-09-23T07:35:21.862112", + "start_time": "2020-09-24T07:35:56.504070", "status": "completed" }, "tags": [] @@ -307,10 +307,10 @@ "execution_count": 8, "metadata": { "papermill": { - "duration": 0.034722, - "end_time": "2020-09-23T07:35:47.021928", + "duration": 0.061035, + "end_time": "2020-09-24T07:36:21.948005", "exception": false, - "start_time": "2020-09-23T07:35:46.987206", + "start_time": "2020-09-24T07:36:21.886970", "status": "completed" }, "tags": [] @@ -329,10 +329,10 @@ "execution_count": 9, "metadata": { "papermill": { - "duration": 0.562298, - "end_time": "2020-09-23T07:35:47.592568", + "duration": 0.553683, + "end_time": "2020-09-24T07:36:22.518379", "exception": false, - "start_time": "2020-09-23T07:35:47.030270", + "start_time": "2020-09-24T07:36:21.964696", "status": "completed" }, "tags": [] @@ -342,7 +342,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "11354 data points\n" + "11410 data points\n" ] }, { @@ -392,21 +392,21 @@ " \n", " \n", " 0\n", - " 20200922\n", + " 20200923\n", " AK\n", - " 7886.0\n", - " 423305.0\n", + " 7941.0\n", + " 425257.0\n", " NaN\n", - " 431191.0\n", + " 433198.0\n", " 43.0\n", " NaN\n", " NaN\n", " NaN\n", " ...\n", - " 431191\n", + " 433198\n", " 0\n", " 0\n", - " 24135b606b72e9c822c662be4c07b364d622accd\n", + " 77a29837bdfd97f18040622b1984ee7b87e863b8\n", " 0\n", " 0\n", " 0\n", @@ -416,21 +416,21 @@ " \n", " \n", " 1\n", - " 20200922\n", + " 20200923\n", " AL\n", - " 146584.0\n", - " 933565.0\n", + " 147153.0\n", + " 938451.0\n", " NaN\n", - " 1065553.0\n", - " 796.0\n", - " 16604.0\n", + " 1070903.0\n", + " 768.0\n", + " 16698.0\n", " NaN\n", - " 1659.0\n", + " 1669.0\n", " ...\n", - " 1080149\n", - " 18\n", - " 117\n", - " 536eaa13e29a33e56ab8f49589c94d7bd8cd3164\n", + " 1085604\n", + " 31\n", + " 94\n", + " 64268ac422e3a504e9b8401b29b1a739bf4ee1ba\n", " 0\n", " 0\n", " 0\n", @@ -445,20 +445,20 @@ ], "text/plain": [ " date state positive negative pending totalTestResults \\\n", - "0 20200922 AK 7886.0 423305.0 NaN 431191.0 \n", - "1 20200922 AL 146584.0 933565.0 NaN 1065553.0 \n", + "0 20200923 AK 7941.0 425257.0 NaN 433198.0 \n", + "1 20200923 AL 147153.0 938451.0 NaN 1070903.0 \n", "\n", " hospitalizedCurrently hospitalizedCumulative inIcuCurrently \\\n", "0 43.0 NaN NaN \n", - "1 796.0 16604.0 NaN \n", + "1 768.0 16698.0 NaN \n", "\n", " inIcuCumulative ... posNeg deathIncrease hospitalizedIncrease \\\n", - "0 NaN ... 431191 0 0 \n", - "1 1659.0 ... 1080149 18 117 \n", + "0 NaN ... 433198 0 0 \n", + "1 1669.0 ... 1085604 31 94 \n", "\n", " hash commercialScore \\\n", - "0 24135b606b72e9c822c662be4c07b364d622accd 0 \n", - "1 536eaa13e29a33e56ab8f49589c94d7bd8cd3164 0 \n", + "0 77a29837bdfd97f18040622b1984ee7b87e863b8 0 \n", + "1 64268ac422e3a504e9b8401b29b1a739bf4ee1ba 0 \n", "\n", " negativeRegularScore negativeScore positiveScore score grade \n", "0 0 0 0 0 \n", @@ -498,18 +498,18 @@ "version": "3.7.3" }, "papermill": { - "duration": 32.809978, - "end_time": "2020-09-23T07:35:47.913746", + "duration": 32.434435, + "end_time": "2020-09-24T07:36:22.946894", "environment_variables": {}, "exception": null, - "input_path": "/tmp/o8q1c8tx/notebooks/process/download-covidtracking-data.ipynb", + "input_path": "/tmp/hhqn2uzg/notebooks/process/download-covidtracking-data.ipynb", "output_path": "runs/download-covidtracking-data.runs.ipynb", "parameters": { - "PAPERMILL_INPUT_PATH": "/tmp/o8q1c8tx/notebooks/process/download-covidtracking-data.ipynb", + "PAPERMILL_INPUT_PATH": "/tmp/hhqn2uzg/notebooks/process/download-covidtracking-data.ipynb", "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb", "out_folder": "data/covidtracking" }, - "start_time": "2020-09-23T07:35:15.103768", + "start_time": "2020-09-24T07:35:50.512459", "version": "1.1.0" } }, -- GitLab