refactor group creation scripts

README.md

@@ -27,20 +27,20 @@ Creating a single group for the class allows to have a centralizd place to store
 
 The manual process of inviting the whole class to the group can be automated using Gitlab's invitation API. Given a list of student email addresses (e.g. exported from moodle) and the URL to a gitlab group, we provide a script to automatically send an invitation email to each student.
 
-script: [scripts/invite_students.py](scripts/invite_students.py)
+script: [teach_utils/invite_students.py](teach_utils/invite_students.py)
 
-usage: `python invite_moodle.py moodle_students_list.csv https://gitlab-instance.com/group-name`
+usage: `python invite_students.py emails.txt https://gitlab-instance.com/group-name`
 
 
 ### Creating student groups
 
 When students work on graded assignments, they will usually have to work in private groups. The groups must be accessible to the teachers in order to grade the assignments.
 
-One way to achieve this is to have the teacher create all private student groups (they will therefore be the group owner) and then invite students to their respective groups. We provide a script to automate the group creation and invitation process.
+One way to achieve this is to have the teacher create all private student groups (they will therefore be the group owner) and then invite students to their respective groups. We provide a script to automate the group creation and invitation process. It creates student groups based on their information in a CSV file exported from Moodle.
 
-script: [scripts/create_student_groups.py](scripts/create_student_groups.py)
+script: [teach_utils/moodle_to_student_groups.py](teach_utils/moodle_to_student_groups.py)
 
-usage: `python create_student_groups.py] students_moodle.csv`
+usage: `python mootle_to_student_groups.py] students_moodle.csv`
 
 ### Gathering all forks of a project
 
@@ -58,7 +58,7 @@ The teacher can then use the Gitlab API to keep track of all student's forks. We
 * `commit`: The hash of the last commit before the deadline, used for grading the assignment.
 * `autostart_url`: A URL to directly start a Renku session at the last commit before the deadline.
 
-script: [scripts/collect_forks.py](scripts/collect_forks.py)
+script: [teach_utils/collect_forks.py](teach_utils/collect_forks.py)
 
 usage:
 
@@ -73,7 +73,7 @@ python collect_forks.py \
 
 In some cases, the student projects may have to be cloned locally by the teacher. We provide a shell script to automate this process. This script will read the JSON output from `collect_forks.py` and clone all the forks at the deadline commit into a target directory.
 
-script: [scripts/clone_forks_from_json.sh](scripts/clone_forks_from_json.sh)
+script: [teach_utils/clone_forks_from_json.sh](teach_utils/clone_forks_from_json.sh)
 
 usage: `python clone_forks_from_json.sh forks.json clone_dir`
 

requirements.txt

 click
 requests
 pytz
+pandas

scripts/collect_forks.py → teach_utils/collect_forks.py

@@ -15,8 +15,7 @@ import requests
 import click
 from datetime import datetime
 import pytz
-
-# Replace URL
+from teach_utils.common_requests import parse_repo_url, get_project_id
 
 
 def validate_iso_date(date: str) -> str:
@@ -28,55 +27,6 @@ def validate_iso_date(date: str) -> str:
         raise ValueError("Deadline must be in ISO-8601 format.")
 
 
-def parse_repo_url(url: str) -> Tuple[str, str, str]:
-    """Decompose a full repo URL into 3 parts:
-    - the organization base URL
-    - the namespace (i.e. groups and subgroups
-    - the name of the repository
-
-    Examples
-    --------
-    >>> parse_repo_url('https://org.com/group/subgroup/repo')
-    ('https://org.com', 'group/subgroup', 'repo')
-    >>> parse_repo_url('https://org.com/gitlab/group/repo')
-    ('https://org.com/gitlab', 'group', 'repo')
-    """
-    regex = re.compile(
-        (
-            "(?P<base>https://[^/]*(/gitlab|/projects)?)/"
-            "(?P<namespace>([^/]*/)*)"
-            "(?P<repo>[^/]*)$"
-        ),
-        re.IGNORECASE,
-    )
-    captured = re.match(regex, url).groupdict()
-    base, namespace, repo = [captured[group] for group in ["base", "namespace", "repo"]]
-    namespace = namespace.strip("/")
-    return base, namespace, repo
-
-
-def get_project_id(project_url: str, header=Dict[str, str]) -> int:
-    """Given a project's URL, return it's gitlab ID"""
-    base, namespace, repo = parse_repo_url(project_url)
-    project = []
-    page = 1
-    while not len(project):
-        resp = requests.get(
-            f"{base}/api/v4/projects?search={repo}&per_page=100&page={page}",
-            headers=header,
-        )
-        if resp.ok:
-            resp = resp.json()
-        else:
-            resp.raise_for_status()
-        page += 1
-        project = [p for p in resp if p["path_with_namespace"] == f"{namespace}/{repo}"]
-
-    if len(list(project)) > 1:
-        raise ValueError("More than one project matched input url")
-    return project[0]["id"]
-
-
 def collect_forks(project_url: str, header=Dict[str, str]) -> List[Dict]:
     """Retrieve the metadata from all forks of input project"""
     base, namespace, repo = parse_repo_url(project_url)
@@ -180,7 +130,7 @@ def format_fork_metadata(
 @click.option(
     "--token",
     type=click.Path(exists=True),
-    help="Armored ASCII file containing the Gitlab API token. If not provided, you will be prompted for the token.",
+    help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
 )
 def main(repo_url, token, deadline=None):
     if token is None:

teach_utils/common_requests.py

+import re
+from typing import Tuple, List, Dict, Optional
+import requests
+
+
+def parse_repo_url(url: str) -> Tuple[str, str, str]:
+    """Decompose a full repo URL into 3 parts:
+    - the organization base URL
+    - the namespace (i.e. groups and subgroups
+    - the name of the repository
+
+    Examples
+    --------
+    >>> parse_repo_url('https://org.com/group/subgroup/repo')
+    ('https://org.com', 'group/subgroup', 'repo')
+    >>> parse_repo_url('https://org.com/gitlab/group/repo')
+    ('https://org.com/gitlab', 'group', 'repo')
+    """
+    regex = re.compile(
+        (
+            "(?P<base>https://[^/]*(/gitlab|/projects)?)/"
+            "(?P<namespace>([^/]*/)*)"
+            "(?P<repo>[^/]*)$"
+        ),
+        re.IGNORECASE,
+    )
+    captured = re.match(regex, url).groupdict()
+    base, namespace, repo = [captured[group] for group in ["base", "namespace", "repo"]]
+    namespace = namespace.strip("/")
+    return base, namespace, repo
+
+
+def parse_group_url(url: str) -> Tuple[str, str]:
+    """Decompose a full group URL into 2 parts:
+    - the organization base URL
+    - the namespace (i.e. groups and subgroups
+
+    Examples
+    --------
+    >>> parse_group_url('https://org.com/group/subgroup')
+    ('https://org.com', 'group/subgroup')
+    >>> parse_group_url('https://org.com/gitlab/group')
+    ('https://org.com/gitlab', 'group')
+    """
+    regex = re.compile(
+        (
+            "(?P<base>https://[^/]*(/gitlab|/projects)?)/"
+            "(?P<namespace>([^/]*/)*[^/]*)$"
+        ),
+        re.IGNORECASE,
+    )
+    captured = re.match(regex, url).groupdict()
+    base, namespace = [captured[group] for group in ["base", "namespace"]]
+    return base, namespace
+
+
+def get_project_id(project_url: str, header: Dict[str, str]) -> int:
+    """Given a project's URL, return it's gitlab ID"""
+    base, namespace, repo = parse_repo_url(project_url)
+    project = []
+    page = 1
+    # Gitlab only provides max 100 results/page, we need to scan all pages
+    # in case the project is not on the first one
+    while not len(project):
+        resp = requests.get(
+            f"{base}/api/v4/projects?search={repo}&per_page=100&page={page}",
+            headers=header,
+        )
+        if resp.ok:
+            resp = resp.json()
+        else:
+            resp.raise_for_status()
+        page += 1
+        project = [p for p in resp if p["path_with_namespace"] == f"{namespace}/{repo}"]
+
+    if len(list(project)) > 1:
+        raise ValueError("More than one project matched input url")
+    return project[0]["id"]
+
+
+def get_group_id(group_url: str, header: Dict[str, str]) -> int:
+    """Given a group's URL, return it's gitlab ID"""
+    base, namespace = parse_group_url(group_url)
+    if namespace.find("/") > 0:
+        parent_name = namespace.split("/")[0]
+        group_name = namespace.split("/")[-1]
+        query = f"{base}/api/v4/groups/{parent_name}/subgroups?search={group_name}"
+    else:
+        query = f"{base}/api/v4/groups/{namespace}"
+
+    response = requests.get(query, headers=header)
+    if response.ok:
+        content = response.json()
+        # Subgroup response comes as a list and must be unwrapped
+        if isinstance(content, list):
+            content = content[0]
+        return content["id"]
+    else:
+        response.raise_for_status()

teach_utils/invite_students.py

+# Given an input csv file of course participants created from moodle,
+# create corresponding groups in moodle.
+
+import re
+from typing import Dict
+import requests
+import click
+from teach_utils.common_requests import parse_repo_url, get_group_id
+
+
+def invite_student_email(
+    student_email: str, base_url: str, group_id: int, header: Dict[str, str]
+) -> None:
+    """Invite student to group by email"""
+    response = requests.post(
+        f"{base_url}/api/v4/groups/{group_id}/invitations",
+        data={"email": student_email, "access_level": 40},
+        headers=header,
+    )
+    print(response.json())
+
+
+def find_email(line: str) -> str:
+    """Find and return email address in input string.
+    Raises an error if none found and only return the first
+    email if there are more than one.
+    
+    Examples
+    --------
+    
+    >>> find_email('robert,smith,robert.smith@no.where,123')
+    'robert.smith@no.where'
+    >>> find_email('mike.rock@yes.no,mrock@yes.no')
+    'mike.rock@yes.no'
+    """
+    regex = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
+    try:
+        email = re.search(regex, line).groups()[0]
+    except AttributeError:
+        raise ValueError(f"Missing email: {line}")
+    return email
+
+
+@click.command()
+@click.argument(
+    "emails_file", type=click.Path(exists=True),
+)
+@click.argument("group-url")
+@click.option(
+    "--token",
+    type=click.Path(exists=True),
+    help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
+)
+def main(emails_file, group_url, token):
+    """
+    Send invitations to join input gitlab group to all emails in the emails_file.
+    The file should have one email per line."""
+    if token is None:
+        token = click.prompt("Please enter your Gitlab API token", hide_input=True)
+    else:
+        token = open(token).read().strip()
+    header = {"PRIVATE-TOKEN": token}
+
+    # Recover the email on each line, in case there are multiple columns
+    emails = map(find_email, open(emails_file).read().split("\n"))
+
+    group_id = get_group_id(group_url, header)
+    base_url, _, _ = parse_repo_url(group_url)
+
+    for email in emails:
+        # Invite student to their group by email
+        invite_student_email(email, base_url, group_id, header)
+
+
+if __name__ == "__main__":
+    main()

teach_utils/moodle_to_student_groups.py

+# Given an input csv file of course participants created from moodle,
+# create corresponding groups in moodle.
+
+import json
+import datetime
+from typing import Dict
+import requests
+import pandas as pd
+import click
+from teach_utils.common_requests import get_group_id, parse_group_url
+
+
+def create_student_group(
+    group_name: str, parent_url: str, header: Dict[str, str]
+) -> requests.Response:
+    """Create a private group for student"""
+    data = {
+        "path": group_name,
+        "name": group_name,
+        "parent_id": get_group_id(parent_url),
+        "visibility": "private",
+    }
+    base, _ = parse_group_url(parent_url)
+    response = requests.post(
+        f"{base}/groups",
+        data=json.dumps(data),
+        headers={**header, "Content-Type": "application/json"},
+    )
+    return response
+
+
+def invite_student_email(
+    student_email: str, base_url: str, group_id: int, header: Dict[str, str]
+) -> None:
+    """Invite student to group by email"""
+    response = requests.post(
+        f"{base_url}/groups/{group_id}/invitations",
+        data={"email": student_email, "access_level": 50},
+        headers=header,
+    )
+    print(response.json())
+
+
+def generate_student_code(first_name: str, last_name: str) -> str:
+    """Generate a uniform student identifier from their name"""
+    initials = first_name[0] + last_name[0]
+    year = datetime.datetime.now().year
+    return f"ST_{initials}_{year}".upper()
+
+
+@click.command()
+@click.argument("student_table")
+@click.argument("parent_url")
+@click.option(
+    "--token",
+    type=click.Path(exists=True),
+    help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
+)
+def main(student_table, parent_url, token):
+    """Given a parent-group URL and a CSV file of students exported from Moodle,
+    create one private subgroup for each student in the parent group and invite students
+    in their personal group. The student table must contain columns: "Email address", "First name"
+    and "Surname"
+    """
+    if token is None:
+        token = click.prompt("Please enter your Gitlab API token", hide_input=True)
+    else:
+        token = open(token).read().strip()
+
+    header = {"PRIVATE-TOKEN": token}
+
+    students = pd.read_csv(student_table)
+    base_url, _ = parse_group_url(parent_url)
+
+    for _, st in students.iterrows():
+        # Generate a student code, e.g.: Ben Smith -> ST_BS_2022
+        student_code = generate_student_code(st["First name"], st["Surname"])
+
+        # Create student group "parent-group/ST_BS_2022"
+        resp = create_student_group(student_code, parent_url, header)
+        # Skip if already created
+        if not resp.ok:
+            print(f"Could not create {student_code}: {resp.json()['message']}")
+
+        group_id = get_group_id(f"{parent_url}/{student_code}", header)
+        # Invite student to their group by email
+        invite_student_email(st["Email address"], base_url, group_id, header)
+
+
+if __name__ == "__main__":
+    main()