Commit c896e437 authored by Cyril Matthey-Doret's avatar Cyril Matthey-Doret
Browse files

Merge branch 'refactor_cloning' into 'master'

Refactor cloning

See merge request learn-renku/teaching-on-renku/advanced-teaching-automation!1
parents 6939a25c 86628d2c
Pipeline #388535 passed with stage
in 11 seconds
......@@ -4,7 +4,7 @@ install:
pip install -e .
uninstall:
pip uninstall teach-utils
pip uninstall teach-auto
test:
pytest --doctest-modules teach_utils
pytest --doctest-modules teach_auto
......@@ -4,7 +4,7 @@ with open("requirements.txt", "r") as f:
REQUIREMENTS = f.read()
setup(
name="teach_utils",
name="teach_auto",
version="0.1",
packages=find_packages(),
install_requires=REQUIREMENTS,
......
#!/usr/bin/env python
# Reads JSON output from collect-forks (either as a file or in stdin)
# and clone all target forks into provided directory at the target commit.
# usage ./clone_forks_from_json.sh OUT_DIR forks.json
# ./collect_forks.py --token token.asc URL | ./clone_forks_from_json.sh OUT_DIR
import sys
from pathlib import Path
import json
import click
import git
def clone_fork(url: str, out_path: Path, commit: str):
"""
Clone input repo in target directory
and checkout desired commit."""
cloned = git.Repo.clone_from(url, out_path)
cloned.git.checkout(commit)
@click.command()
@click.argument("out_dir", type=click.Path(file_okay=False, exists=False))
@click.argument("forks", type=click.File(mode="r"), default=sys.stdin)
def main(out_dir, forks, token):
forks = json.load(forks)
for fork in forks:
url, group, commit = [fork.get(key) for key in ["url", "group", "commit"]]
path = Path(out_dir) / group
clone_fork(url, path, commit)
......@@ -15,7 +15,8 @@ import requests
import click
from datetime import datetime
import pytz
from teach_utils.common_requests import parse_repo_url, get_project_id
from class_auto.common_requests import parse_repo_url, get_project_id
from class_auto.utils import ingest_token
def validate_iso_date(date: str) -> str:
......@@ -49,7 +50,7 @@ def collect_forks(project_url: str, header=Dict[str, str]) -> List[Dict]:
def filter_group_forks(forks: List[Dict]) -> List[Dict]:
"""Given a list of forks' metadata, only keep those that belong to a group
Examples
--------
>>> d1 = {'id': 1, 'namespace': {'kind': 'user'}}
......@@ -93,9 +94,9 @@ def get_last_commit_hash(
def format_fork_metadata(
fork: Dict, header: Dict[str, str], deadline: Optional[str] = None
) -> Dict:
"""Format and add fields to a fork's metadata. The
"""Format and add fields to a fork's metadata. The
resulting metadata will have the following fields:
id, http_url_to_repo, autostart_url, commit, members, visibility, group"""
id, url, autostart_url, commit, members, visibility, group"""
meta = {
"id": fork["id"],
"url": fork["http_url_to_repo"],
......@@ -134,17 +135,12 @@ def format_fork_metadata(
)
@click.argument("repo_url", type=str)
@click.option(
"--token",
"--token-path",
type=click.Path(exists=True),
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
def main(repo_url, token, deadline=None):
if token is None:
token = click.prompt(
"Please enter your Gitlab API token", hide_input=True, err=True
)
else:
token = open(token).read().strip()
def main(repo_url, token_path, deadline=None):
token = ingest_token(token_path)
# Check for valid deadline format
if deadline is not None:
validate_iso_date(deadline)
......
......@@ -7,7 +7,8 @@ from typing import Dict
import requests
import pandas as pd
import click
from teach_utils.common_requests import get_group_id, parse_group_url
from teach_auto.common_requests import get_group_id, parse_group_url
from teach_auto.utils import ingest_token
def create_student_group(
......@@ -52,23 +53,17 @@ def generate_student_code(first_name: str, last_name: str) -> str:
@click.argument("student_table")
@click.argument("parent_url")
@click.option(
"--token",
"--token-path",
type=click.Path(exists=True),
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
def main(student_table, parent_url, token):
def main(student_table, parent_url, token_path):
"""Given a parent-group URL and a CSV file of students exported from Moodle,
create one private subgroup for each student in the parent group and invite students
in their personal group. The student table must contain columns: "Email address", "First name"
and "Surname"
"""
if token is None:
token = click.prompt(
"Please enter your Gitlab API token", hide_input=True, err=True
)
else:
token = open(token).read().strip()
token = ingest_token(token_path)
header = {"PRIVATE-TOKEN": token}
students = pd.read_csv(student_table)
......
......@@ -5,7 +5,8 @@ import re
from typing import Dict
import requests
import click
from teach_utils.common_requests import parse_repo_url, get_group_id
from class_auto.common_requests import parse_repo_url, get_group_id
from class_auto.utils import ingest_token
def invite_student_email(
......@@ -24,10 +25,10 @@ def find_email(line: str) -> str:
"""Find and return email address in input string.
Raises an error if none found and only return the first
email if there are more than one.
Examples
--------
>>> find_email('robert,smith,robert.smith@no.where,123')
'robert.smith@no.where'
>>> find_email('mike.rock@yes.no,mrock@yes.no')
......@@ -47,20 +48,15 @@ def find_email(line: str) -> str:
)
@click.argument("group-url")
@click.option(
"--token",
"--token-path",
type=click.Path(exists=True),
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
def main(emails_file, group_url, token):
def main(emails_file, group_url, token_path):
"""
Send invitations to join input gitlab group to all emails in the emails_file.
The file should have one email per line."""
if token is None:
token = click.prompt(
"Please enter your Gitlab API token", hide_input=True, err=True
)
else:
token = open(token).read().strip()
token = ingest_token(token_path)
header = {"PRIVATE-TOKEN": token}
# Recover the email on each line, in case there are multiple columns
......
......@@ -10,11 +10,13 @@
import sys
import json
from string import Formatter
from typing import Dict
from typing import Dict, Optional
import requests
from pathlib import Path
import click
import pandas as pd
from teach_utils.common_requests import parse_repo_url
from class_auto.common_requests import parse_repo_url
from class_auto.utils import ingest_token
FEEDBACK_TEMPLATE = """
......@@ -58,10 +60,39 @@ def fill_template(content: Dict[str, str], template: str) -> str:
raise ValueError(f"Fields missing from content: {missing_flds}") from None
def gather_fill_data(fork: Dict, feedback_csv: Optional[Path] = None) -> Dict:
"""Gather metadata and feedback for a single fork into a dictionary
to use for filling the template. The resulting dictionary contains
fields: project, group, members, url, commit and visibility from the fork
metadata, as well as all columns present in the feedback csv file.
"""
# Get basic fields in fork metadata
_, group, repo = parse_repo_url(fork["url"])
members = "\n".join([f"* {member['name']}" for member in fork["members"]])
fill_data = {
"project": repo.removesuffix(".git"),
"group": group,
"members": members,
}
# add commit, url and visibility fields from fork data
fill_data |= {key: fork.get(key) for key in ["url", "commit", "visibility"]}
# If a CSV file with feedback is provided, read the fields
# of the current fork (row) to the fill data.
if feedback_csv:
fill_data |= (
pd.read_csv(feedback_csv)
.query(f'id == {fork["id"]}')
.iloc[0]
.drop("id")
.to_dict()
)
return fill_data
@click.command()
@click.argument("forks", type=click.File(mode="r"), default=sys.stdin)
@click.option(
"--token",
"--token-path",
type=click.Path(exists=True),
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
......@@ -74,29 +105,22 @@ def fill_template(content: Dict[str, str], template: str) -> str:
default=None,
)
@click.option(
"--feedback",
"--feedback-csv",
type=click.Path(exists=True),
help="CSV file with column 'id', and additional columns matching fields in the template"
"The id column should match a gitlab project ID from the JSON input.",
default=None,
)
@click.option(
"--dry-run",
is_flag=True,
help="Print feedback to stdout instead of opening issues"
"--dry-run", is_flag=True, help="Print feedback to stdout instead of opening issues"
)
def main(forks, feedback, token, template, dry_run):
def main(forks, feedback_csv, token_path, template, dry_run):
"""
Send feedback to repositories in input JSON by opening issues.
"""
# Build header with authentication token
if token is None:
token = click.prompt(
"Please enter your Gitlab API token", hide_input=True, err=True
)
else:
token = open(token).read().strip()
token = ingest_token(token_path)
header = {"PRIVATE-TOKEN": token}
forks = json.load(forks)
......@@ -108,30 +132,11 @@ def main(forks, feedback, token, template, dry_run):
feedback_template = FEEDBACK_TEMPLATE
for fork in forks:
# Get basic fields from repo metadata in json
base, group, repo = parse_repo_url(fork["url"])
members = "\n".join([f"* {member['name']}" for member in fork["members"]])
fill_data = {
"project": repo.removesuffix(".git"),
"group": group,
"members": members,
}
# add commit, url and visibility fields from fork data
fill_data |= {key: fork.get(key) for key in ["url", "commit", "visibility"]}
# If provided, get additional fields from feedback table
if feedback:
feedback_fields = (
pd.read_csv(feedback)
.query(f'id == {fork["id"]}')
.iloc[0]
.drop("id")
.to_dict()
)
fill_data |= feedback_fields
base = parse_repo_url(fork["url"])[0]
fill_data = gather_fill_data(fork, feedback_csv)
# Replace fields in the template by their value for the current fork
description = fill_template(fill_data, feedback_template)
# Create issue
# Create issue, or print to stdout if dry run
if dry_run:
print(fork)
print(description)
......
from typing import Optional
from pathlib import Path
import click
def ingest_token(token_path: Optional[Path]=None) -> str:
"""Read token from target file if provided.
If input is None, prompt for token."""
if token_path is None:
token = click.prompt(
"Please enter your Gitlab API token", hide_input=True, err=True
)
else:
token = open(token_path).read().strip()
return token
#!/usr/bin/env bash
# Reads JSON output from collect_forks.py (either as a file or in stdin)
# and clone all target forks into provided directory at the target commit.
# usage ./clone_forks_from_json.sh OUT_DIR forks.json
# ./collect_forks.py --token token.asc URL | ./clone_forks_from_json.sh OUT_DIR
# Help message
function usage () {
cat <<EOF
Usage:
./$(basename $0) [outdir] [in_file.json]
./collect_forks.py --token token.asc URL | ./$(basename $0) [outdir]
Reads json output from collect_forks.py and clone all target forks into provided directory
at the target commit. Repositories are cloned into outdir/namespace.
Arguments:
outdir: Directory where all forks will be cloned [default: .]
in_file.json: JSON output of collect_forks.py containing fork metadata [default: stdin]
EOF
exit 0
}
# Parsing CL arguments
OUT_DIR=${1:-.}
JSON=${2:-/dev/stdin}
if [[ $# -gt 2 ]] || [[ $1 == '-h' ]] || [[ $1 == '--help' ]]; then
usage
fi
mkdir -p "${OUT_DIR}"
jq ".[] | \"git clone \(.url) $OUT_DIR/\(.group) && cd ${OUT_DIR}/\(.group) && git checkout \(.commit)\"" \
< "${JSON}" \
| xargs -L 1 -I {} sh -c "{}"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment