Merge branch 'refactor_cloning' into 'master'

Refactor cloning

See merge request learn-renku/teaching-on-renku/advanced-teaching-automation!1

Makefile

@@ -4,7 +4,7 @@ install:
 	pip install -e .
 
 uninstall:
-	pip uninstall teach-utils 
+	pip uninstall teach-auto
 
 test:
-	pytest --doctest-modules teach_utils
+	pytest --doctest-modules teach_auto

requirements.txt

 click
+GitPython
 requests
 pytz
 pandas

setup.py

@@ -4,7 +4,7 @@ with open("requirements.txt", "r") as f:
     REQUIREMENTS = f.read()
 
 setup(
-    name="teach_utils",
+    name="teach_auto",
     version="0.1",
     packages=find_packages(),
     install_requires=REQUIREMENTS,

teach_auto/clone_forks.py

+#!/usr/bin/env python
+# Reads JSON output from collect-forks (either as a file or in stdin)
+# and clone all target forks into provided directory at the target commit.
+
+# usage ./clone_forks_from_json.sh OUT_DIR forks.json
+#       ./collect_forks.py --token token.asc URL | ./clone_forks_from_json.sh OUT_DIR
+
+import sys
+from pathlib import Path
+import json
+import click
+import git
+
+
+def clone_fork(url: str, out_path: Path, commit: str):
+    """
+    Clone input repo in target directory
+    and checkout desired commit."""
+    cloned = git.Repo.clone_from(url, out_path)
+    cloned.git.checkout(commit)
+
+
+@click.command()
+@click.argument("out_dir", type=click.Path(file_okay=False, exists=False))
+@click.argument("forks", type=click.File(mode="r"), default=sys.stdin)
+def main(out_dir, forks, token):
+
+    forks = json.load(forks)
+    for fork in forks:
+        url, group, commit = [fork.get(key) for key in ["url", "group", "commit"]]
+        path = Path(out_dir) / group
+        clone_fork(url, path, commit)

teach_utils/collect_forks.py → teach_auto/collect_forks.py

@@ -15,7 +15,8 @@ import requests
 import click
 from datetime import datetime
 import pytz
-from teach_utils.common_requests import parse_repo_url, get_project_id
+from class_auto.common_requests import parse_repo_url, get_project_id
+from class_auto.utils import ingest_token
 
 
 def validate_iso_date(date: str) -> str:
@@ -49,7 +50,7 @@ def collect_forks(project_url: str, header=Dict[str, str]) -> List[Dict]:
 
 def filter_group_forks(forks: List[Dict]) -> List[Dict]:
     """Given a list of forks' metadata, only keep those that belong to a group
-    
+
     Examples
     --------
     >>> d1 = {'id': 1, 'namespace': {'kind': 'user'}}
@@ -93,9 +94,9 @@ def get_last_commit_hash(
 def format_fork_metadata(
     fork: Dict, header: Dict[str, str], deadline: Optional[str] = None
 ) -> Dict:
-    """Format and add fields to a fork's metadata. The 
+    """Format and add fields to a fork's metadata. The
     resulting metadata will have the following fields:
-    id, http_url_to_repo, autostart_url, commit, members, visibility, group"""
+    id, url, autostart_url, commit, members, visibility, group"""
     meta = {
         "id": fork["id"],
         "url": fork["http_url_to_repo"],
@@ -134,17 +135,12 @@ def format_fork_metadata(
 )
 @click.argument("repo_url", type=str)
 @click.option(
-    "--token",
+    "--token-path",
     type=click.Path(exists=True),
     help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
 )
-def main(repo_url, token, deadline=None):
-    if token is None:
-        token = click.prompt(
-            "Please enter your Gitlab API token", hide_input=True, err=True
-        )
-    else:
-        token = open(token).read().strip()
+def main(repo_url, token_path, deadline=None):
+    token = ingest_token(token_path)
     # Check for valid deadline format
     if deadline is not None:
         validate_iso_date(deadline)

teach_utils/moodle_to_student_groups.py → teach_auto/create-groups.py

@@ -7,7 +7,8 @@ from typing import Dict
 import requests
 import pandas as pd
 import click
-from teach_utils.common_requests import get_group_id, parse_group_url
+from teach_auto.common_requests import get_group_id, parse_group_url
+from teach_auto.utils import ingest_token
 
 
 def create_student_group(
@@ -52,23 +53,17 @@ def generate_student_code(first_name: str, last_name: str) -> str:
 @click.argument("student_table")
 @click.argument("parent_url")
 @click.option(
-    "--token",
+    "--token-path",
     type=click.Path(exists=True),
     help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
 )
-def main(student_table, parent_url, token):
+def main(student_table, parent_url, token_path):
     """Given a parent-group URL and a CSV file of students exported from Moodle,
     create one private subgroup for each student in the parent group and invite students
     in their personal group. The student table must contain columns: "Email address", "First name"
     and "Surname"
     """
-    if token is None:
-        token = click.prompt(
-            "Please enter your Gitlab API token", hide_input=True, err=True
-        )
-    else:
-        token = open(token).read().strip()
-
+    token = ingest_token(token_path)
     header = {"PRIVATE-TOKEN": token}
 
     students = pd.read_csv(student_table)

teach_utils/invite_students.py → teach_auto/invite_students.py

@@ -5,7 +5,8 @@ import re
 from typing import Dict
 import requests
 import click
-from teach_utils.common_requests import parse_repo_url, get_group_id
+from class_auto.common_requests import parse_repo_url, get_group_id
+from class_auto.utils import ingest_token
 
 
 def invite_student_email(
@@ -24,10 +25,10 @@ def find_email(line: str) -> str:
     """Find and return email address in input string.
     Raises an error if none found and only return the first
     email if there are more than one.
-    
+
     Examples
     --------
-    
+
     >>> find_email('robert,smith,robert.smith@no.where,123')
     'robert.smith@no.where'
     >>> find_email('mike.rock@yes.no,mrock@yes.no')
@@ -47,20 +48,15 @@ def find_email(line: str) -> str:
 )
 @click.argument("group-url")
 @click.option(
-    "--token",
+    "--token-path",
     type=click.Path(exists=True),
     help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
 )
-def main(emails_file, group_url, token):
+def main(emails_file, group_url, token_path):
     """
     Send invitations to join input gitlab group to all emails in the emails_file.
     The file should have one email per line."""
-    if token is None:
-        token = click.prompt(
-            "Please enter your Gitlab API token", hide_input=True, err=True
-        )
-    else:
-        token = open(token).read().strip()
+    token = ingest_token(token_path)
     header = {"PRIVATE-TOKEN": token}
 
     # Recover the email on each line, in case there are multiple columns

teach_utils/send_feedback.py → teach_auto/send_feedback.py

@@ -10,11 +10,13 @@
 import sys
 import json
 from string import Formatter
-from typing import Dict
+from typing import Dict, Optional
 import requests
+from pathlib import Path
 import click
 import pandas as pd
-from teach_utils.common_requests import parse_repo_url
+from class_auto.common_requests import parse_repo_url
+from class_auto.utils import ingest_token
 
 
 FEEDBACK_TEMPLATE = """
@@ -58,10 +60,39 @@ def fill_template(content: Dict[str, str], template: str) -> str:
         raise ValueError(f"Fields missing from content: {missing_flds}") from None
 
 
+def gather_fill_data(fork: Dict, feedback_csv: Optional[Path] = None) -> Dict:
+    """Gather metadata and feedback for a single fork into a dictionary
+    to use for filling the template. The resulting dictionary contains
+    fields: project, group, members, url, commit and visibility from the fork
+    metadata, as well as all columns present in the feedback csv file.
+    """
+    # Get basic fields in fork metadata
+    _, group, repo = parse_repo_url(fork["url"])
+    members = "\n".join([f"* {member['name']}" for member in fork["members"]])
+    fill_data = {
+        "project": repo.removesuffix(".git"),
+        "group": group,
+        "members": members,
+    }
+    # add commit, url and visibility fields from fork data
+    fill_data |= {key: fork.get(key) for key in ["url", "commit", "visibility"]}
+    # If a CSV file with feedback is provided, read the fields
+    # of the current fork (row) to the fill data.
+    if feedback_csv:
+        fill_data |= (
+            pd.read_csv(feedback_csv)
+            .query(f'id == {fork["id"]}')
+            .iloc[0]
+            .drop("id")
+            .to_dict()
+        )
+    return fill_data
+
+
 @click.command()
 @click.argument("forks", type=click.File(mode="r"), default=sys.stdin)
 @click.option(
-    "--token",
+    "--token-path",
     type=click.Path(exists=True),
     help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
 )
@@ -74,29 +105,22 @@ def fill_template(content: Dict[str, str], template: str) -> str:
     default=None,
 )
 @click.option(
-    "--feedback",
+    "--feedback-csv",
     type=click.Path(exists=True),
     help="CSV file with column 'id', and additional columns matching fields in the template"
     "The id column should match a gitlab project ID from the JSON input.",
     default=None,
 )
 @click.option(
-    "--dry-run",
-    is_flag=True,
-    help="Print feedback to stdout instead of opening issues"
+    "--dry-run", is_flag=True, help="Print feedback to stdout instead of opening issues"
 )
-def main(forks, feedback, token, template, dry_run):
+def main(forks, feedback_csv, token_path, template, dry_run):
     """
     Send feedback to repositories in input JSON by opening issues.
     """
 
     # Build header with authentication token
-    if token is None:
-        token = click.prompt(
-            "Please enter your Gitlab API token", hide_input=True, err=True
-        )
-    else:
-        token = open(token).read().strip()
+    token = ingest_token(token_path)
     header = {"PRIVATE-TOKEN": token}
 
     forks = json.load(forks)
@@ -108,30 +132,11 @@ def main(forks, feedback, token, template, dry_run):
         feedback_template = FEEDBACK_TEMPLATE
 
     for fork in forks:
-        # Get basic fields from repo metadata in json
-        base, group, repo = parse_repo_url(fork["url"])
-        members = "\n".join([f"* {member['name']}" for member in fork["members"]])
-        fill_data = {
-            "project": repo.removesuffix(".git"),
-            "group": group,
-            "members": members,
-        }
-        # add commit, url and visibility fields from fork data
-        fill_data |= {key: fork.get(key) for key in ["url", "commit", "visibility"]}
-        # If provided, get additional fields from feedback table
-        if feedback:
-            feedback_fields = (
-                pd.read_csv(feedback)
-                .query(f'id == {fork["id"]}')
-                .iloc[0]
-                .drop("id")
-                .to_dict()
-            )
-            fill_data |= feedback_fields
-
+        base = parse_repo_url(fork["url"])[0]
+        fill_data = gather_fill_data(fork, feedback_csv)
         # Replace fields in the template by their value for the current fork
         description = fill_template(fill_data, feedback_template)
-        # Create issue
+        # Create issue, or print to stdout if dry run
         if dry_run:
             print(fork)
             print(description)

teach_auto/utils.py

+from typing import Optional
+from pathlib import Path
+import click
+
+
+def ingest_token(token_path: Optional[Path]=None) -> str:
+    """Read token from target file if provided.
+    If input is None, prompt for token."""
+    if token_path is None:
+        token = click.prompt(
+            "Please enter your Gitlab API token", hide_input=True, err=True
+        )
+    else:
+        token = open(token_path).read().strip()
+    return token

teach_utils/clone_forks_from_json.sh

-#!/usr/bin/env bash
-# Reads JSON output from collect_forks.py (either as a file or in stdin)
-# and clone all target forks into provided directory at the target commit.
-
-# usage ./clone_forks_from_json.sh OUT_DIR forks.json
-#       ./collect_forks.py --token token.asc URL | ./clone_forks_from_json.sh OUT_DIR
-
-# Help message
-function usage () {
-   cat <<EOF
-Usage:
-    ./$(basename $0) [outdir] [in_file.json]
-    ./collect_forks.py --token token.asc URL | ./$(basename $0) [outdir]
-
-Reads json output from collect_forks.py and clone all target forks into provided directory
-at the target commit. Repositories are cloned into outdir/namespace.
-
-Arguments:
-    outdir: Directory where all forks will be cloned [default: .]
-    in_file.json: JSON output of collect_forks.py containing fork metadata [default: stdin]
-EOF
-   exit 0
-}
-
-# Parsing CL arguments
-OUT_DIR=${1:-.}
-JSON=${2:-/dev/stdin}
-if [[ $# -gt 2 ]] || [[ $1 == '-h' ]] || [[ $1 == '--help' ]]; then
-    usage
-fi
-
-mkdir -p "${OUT_DIR}"
-
-jq ".[] | \"git clone \(.url) $OUT_DIR/\(.group) && cd ${OUT_DIR}/\(.group) && git checkout \(.commit)\"" \
-    < "${JSON}" \
-    | xargs -L 1 -I {} sh -c "{}"