Skip to content
Hide whitespace changes
Inline Side-by-side
.renku/metadata/datasets-provenance-tails 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_data_value": {
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
"/datasets/6686acf5d6c743909649d7841217821f",
{
"@renku_data_type": "renku.core.models.dataset.Dataset",
"@renku_oid": "d3ffff15b59a6aef7d4ae74a7d6ab721bb022a03dabac1ef7ea5d9a4c8d81cc3",
"@renku_reference": true
}
]
}
]
}
]
}
]
},
"@renku_oid": "datasets-provenance-tails"
}
\ No newline at end of file
.renku/metadata/datasets-tags 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_data_value": null,
"@renku_oid": "datasets-tags"
}
\ No newline at end of file
.renku/metadata/f8/11/f811a7ad47f1435b83000b0b88c74e915c236e623f7743f682c9b4011214884e 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "BTrees.OOBTree.OOBucket",
"@renku_data_value": {
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
"BTree",
{
"@renku_data_type": "type",
"@renku_data_value": "BTrees.OOBTree.OOBTree"
},
"Bucket",
{
"@renku_data_type": "type",
"@renku_data_value": "BTrees.OOBTree.OOBucket"
},
"Set",
{
"@renku_data_type": "type",
"@renku_data_value": "BTrees.OOBTree.OOSet"
},
"TreeSet",
{
"@renku_data_type": "type",
"@renku_data_value": "BTrees.OOBTree.OOTreeSet"
},
"attrname",
"upstream",
"call",
false,
"difference",
{
"@renku_data_type": "function",
"@renku_data_value": "BTrees._OOBTree.difference"
},
"dump",
{
"@renku_data_type": "function",
"@renku_data_value": "renku.core.metadata.gateway.database_gateway.dump_activity"
},
"element",
{
"@renku_data_type": "zope.interface.interface.Attribute",
"@renku_data_value": {
"_Element__tagged_values": null,
"__doc__": "the upstream activities",
"__name__": "upstream",
"interface": {
"@renku_data_type": "type",
"@renku_data_value": "renku.core.metadata.gateway.database_gateway.IActivityDownstreamRelation"
}
}
},
"interface",
{
"@renku_data_type": "type",
"@renku_data_value": "renku.core.metadata.gateway.database_gateway.IActivityDownstreamRelation"
},
"intersection",
{
"@renku_data_type": "function",
"@renku_data_value": "BTrees._OOBTree.intersection"
},
"load",
{
"@renku_data_type": "function",
"@renku_data_value": "renku.core.metadata.gateway.database_gateway.load_activity"
},
"multiple",
false,
"multiunion",
null,
"name",
"upstream",
"union",
{
"@renku_data_type": "function",
"@renku_data_value": "BTrees._OOBTree.union"
}
]
}
]
},
"@renku_oid": "f811a7ad47f1435b83000b0b88c74e915c236e623f7743f682c9b4011214884e"
}
\ No newline at end of file
.renku/metadata/plans 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_data_value": null,
"@renku_oid": "plans"
}
\ No newline at end of file
.renku/metadata/plans-by-name 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_data_value": null,
"@renku_oid": "plans-by-name"
}
\ No newline at end of file
.renku/metadata/project 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "renku.core.models.project.Project",
"@renku_oid": "project",
"agent_version": "1.0.6",
"annotations": [],
"automated_update": false,
"creator": {
"@renku_data_type": "renku.core.models.provenance.agent.Person",
"@renku_data_value": {
"affiliation": null,
"alternate_name": null,
"email": "anthony.sonrel@uzh.ch",
"id": "mailto:anthony.sonrel@uzh.ch",
"name": "Anthony Sonrel"
}
},
"date_created": {
"@renku_data_type": "datetime.datetime",
"@renku_data_value": "2021-04-28T07:37:09+00:00"
},
"description": null,
"id": "/projects/omnibenchmark/utils",
"immutable_template_files": [],
"keywords": [],
"name": "utils",
"template_id": "minimal",
"template_metadata": "{\"__template_source__\": \"https://github.com/SwissDataScienceCenter/renku-project-template\", \"__template_ref__\": \"0.1.19\", \"__template_id__\": \"minimal\", \"__namespace__\": \"omnibenchmark\", \"__automated_update__\": true, \"__repository__\": \"https://renkulab.io/gitlab\", \"__sanitized_project_name__\": \"utils\", \"__project_slug__\": \"omnibenchmark/utils\", \"name\": \"utils\"}",
"template_ref": "0.1.19",
"template_source": "https://github.com/SwissDataScienceCenter/renku-project-template",
"template_version": "817b03c0f434a79eb54473688c1f897a66228b3d",
"version": "9"
}
\ No newline at end of file
.renku/metadata/root 0 → 100644
View file @ ef3de034
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_data_value": {
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
{
"@renku_data_type": "builtins.tuple",
"@renku_data_value": [
"_downstream_relations",
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "_downstream_relations",
"@renku_reference": true
},
"activities",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "activities-index",
"attribute": "id",
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "activities",
"@renku_reference": true
},
"key_type": null,
"name": "activities",
"object_type": "renku.core.models.provenance.activity.Activity"
},
"activities-by-generation",
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "activities-by-generation",
"@renku_reference": true
},
"activities-by-usage",
{
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "activities-by-usage",
"@renku_reference": true
},
"activity-catalog",
{
"@renku_data_type": "zc.relation.catalog.Catalog",
"@renku_oid": "activity-catalog",
"@renku_reference": true
},
"activity-collections",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "activity-collections-index",
"attribute": "id",
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "activity-collections",
"@renku_reference": true
},
"key_type": null,
"name": "activity-collections",
"object_type": "renku.core.models.provenance.activity.ActivityCollection"
},
"datasets",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "datasets-index",
"attribute": "name",
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "datasets",
"@renku_reference": true
},
"key_type": null,
"name": "datasets",
"object_type": "renku.core.models.dataset.Dataset"
},
"datasets-provenance-tails",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "datasets-provenance-tails-index",
"attribute": "id",
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "datasets-provenance-tails",
"@renku_reference": true
},
"key_type": null,
"name": "datasets-provenance-tails",
"object_type": "renku.core.models.dataset.Dataset"
},
"datasets-tags",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "datasets-tags-index",
"attribute": null,
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "datasets-tags",
"@renku_reference": true
},
"key_type": null,
"name": "datasets-tags",
"object_type": "persistent.list.PersistentList"
},
"plans",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "plans-index",
"attribute": "id",
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "plans",
"@renku_reference": true
},
"key_type": null,
"name": "plans",
"object_type": "renku.core.models.workflow.plan.AbstractPlan"
},
"plans-by-name",
{
"@renku_data_type": "renku.core.metadata.database.Index",
"@renku_oid": "plans-by-name-index",
"attribute": "name",
"entries": {
"@renku_data_type": "renku.core.metadata.database.RenkuOOBTree",
"@renku_oid": "plans-by-name",
"@renku_reference": true
},
"key_type": null,
"name": "plans-by-name",
"object_type": "renku.core.models.workflow.plan.AbstractPlan"
},
"project",
{
"@renku_data_type": "renku.core.models.project.Project",
"@renku_oid": "project",
"@renku_reference": true
}
]
}
]
}
]
}
]
},
"@renku_oid": "root"
}
\ No newline at end of file
Dockerfile
View file @ ef3de034
# For finding latest versions of the base image see
# https://github.com/SwissDataScienceCenter/renkulab-docker
ARG RENKU_BASE_IMAGE=renku/renkulab-py:3.8-0.7.7
ARG RENKU_BASE_IMAGE=renku/renkulab-py:3.9-0.10.3
FROM ${RENKU_BASE_IMAGE}
# Uncomment and adapt if code is to be included in the image
......@@ -11,7 +11,8 @@ FROM ${RENKU_BASE_IMAGE}
# except for the last end with backslash '\' to continue the RUN line
#
USER root
RUN apt-get update && \
RUN apt-get update ; apt-get install ca-certificates && \
apt-get update && \
apt-get install -y --no-install-recommends \
apt-utils \
jq && \
......@@ -36,16 +37,23 @@ USER ${NB_USER}
# RENKU_VERSION determines the version of the renku CLI
# that will be used in this image. To find the latest version,
# visit https://pypi.org/project/renku/#history.
ARG RENKU_VERSION=0.16.1.post1
ARG RENKU_VERSION=1.0.6
########################################################
# Do not edit this section and do not add anything below
# Install renku from pypi or from github if it's a dev version
RUN if [ -n "$RENKU_VERSION" ] ; then \
pipx uninstall renku && \
/home/jovyan/.local/pipx/shared/bin/pip install setuptools==57.5.0; \
pipx install --force renku==${RENKU_VERSION} \
; fi
source .renku/venv/bin/activate ; \
currentversion=$(renku --version) ; \
if [ "$RENKU_VERSION" != "$currentversion" ] ; then \
pip uninstall renku ; \
gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\3/p") ; \
if [ -n "$gitversion" ] ; then \
pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
else \
pip install --force renku==${RENKU_VERSION} ;\
fi \
fi \
fi
########################################################
bash/dataset_utils.sh
View file @ ef3de034
......@@ -158,6 +158,7 @@ help_import_datasets_by_keyword()
echo "Options:"
echo "h Print this Help."
echo "o Orchestrator namespace without gitlab prefix, e.g. defaults to omnibenchmark/orchestrator"
echo "f Forces the dataset import and ignores checks on the orchestrator CICD."
echo
}
......@@ -418,13 +419,15 @@ function import_datasets_by_keyword () {
local OPTIND o_path="omnibenchmark/orchestrator"
# Display help message
while getopts ":o:h" option; do
while getopts ":o:hf" option; do
case $option in
h)
help_import_datasets_by_keyword
return 1;;
o)
o_path=${OPTARG};;
f)
force_import='true';;
\?)
error_exit "Error: Invalid option";;
esac
......@@ -446,8 +449,14 @@ function import_datasets_by_keyword () {
# Test ids to be from the origin project, have a successful cicd and be part of omnibenchmark
origin_ids=$(check_import_from_target "${in_ids[@]}")
omni_ids=$(check_orchestrator -o "$o_path" "${origin_ids[@]}")
success_ids=$(check_project_status -o "$o_path" ${omni_ids[@]})
if [[ force_import ]]
then
success_ids=${origin_ids[@]}
else
omni_ids=$(check_orchestrator -o "$o_path" "${origin_ids[@]}")
success_ids=$(check_project_status -o "$o_path" ${omni_ids[@]})
fi
for dataset_id in ${success_ids}
......
bash/provenance_utils.sh
View file @ ef3de034
......@@ -187,7 +187,7 @@ function find_unique () {
function get_project_graph () {
renku graph export --full --format json-ld > .renku/provenance.json
renku graph export --full --format jsonld > .renku/provenance.json
renku save
}
......

This GitLab is a part of renkulab.io