Commit 61df3849 authored by Cyril Matthey-Doret's avatar Cyril Matthey-Doret
Browse files

refactor group creation scripts

parent ca36e3ef
Pipeline #327220 passed with stage
in 3 minutes and 37 seconds
......@@ -27,20 +27,20 @@ Creating a single group for the class allows to have a centralizd place to store
The manual process of inviting the whole class to the group can be automated using Gitlab's invitation API. Given a list of student email addresses (e.g. exported from moodle) and the URL to a gitlab group, we provide a script to automatically send an invitation email to each student.
script: [scripts/invite_students.py](scripts/invite_students.py)
script: [teach_utils/invite_students.py](teach_utils/invite_students.py)
usage: `python invite_moodle.py moodle_students_list.csv https://gitlab-instance.com/group-name`
usage: `python invite_students.py emails.txt https://gitlab-instance.com/group-name`
### Creating student groups
When students work on graded assignments, they will usually have to work in private groups. The groups must be accessible to the teachers in order to grade the assignments.
One way to achieve this is to have the teacher create all private student groups (they will therefore be the group owner) and then invite students to their respective groups. We provide a script to automate the group creation and invitation process.
One way to achieve this is to have the teacher create all private student groups (they will therefore be the group owner) and then invite students to their respective groups. We provide a script to automate the group creation and invitation process. It creates student groups based on their information in a CSV file exported from Moodle.
script: [scripts/create_student_groups.py](scripts/create_student_groups.py)
script: [teach_utils/moodle_to_student_groups.py](teach_utils/moodle_to_student_groups.py)
usage: `python create_student_groups.py] students_moodle.csv`
usage: `python mootle_to_student_groups.py] students_moodle.csv`
### Gathering all forks of a project
......@@ -58,7 +58,7 @@ The teacher can then use the Gitlab API to keep track of all student's forks. We
* `commit`: The hash of the last commit before the deadline, used for grading the assignment.
* `autostart_url`: A URL to directly start a Renku session at the last commit before the deadline.
script: [scripts/collect_forks.py](scripts/collect_forks.py)
script: [teach_utils/collect_forks.py](teach_utils/collect_forks.py)
usage:
......@@ -73,7 +73,7 @@ python collect_forks.py \
In some cases, the student projects may have to be cloned locally by the teacher. We provide a shell script to automate this process. This script will read the JSON output from `collect_forks.py` and clone all the forks at the deadline commit into a target directory.
script: [scripts/clone_forks_from_json.sh](scripts/clone_forks_from_json.sh)
script: [teach_utils/clone_forks_from_json.sh](teach_utils/clone_forks_from_json.sh)
usage: `python clone_forks_from_json.sh forks.json clone_dir`
......
......@@ -15,8 +15,7 @@ import requests
import click
from datetime import datetime
import pytz
# Replace URL
from teach_utils.common_requests import parse_repo_url, get_project_id
def validate_iso_date(date: str) -> str:
......@@ -28,55 +27,6 @@ def validate_iso_date(date: str) -> str:
raise ValueError("Deadline must be in ISO-8601 format.")
def parse_repo_url(url: str) -> Tuple[str, str, str]:
"""Decompose a full repo URL into 3 parts:
- the organization base URL
- the namespace (i.e. groups and subgroups
- the name of the repository
Examples
--------
>>> parse_repo_url('https://org.com/group/subgroup/repo')
('https://org.com', 'group/subgroup', 'repo')
>>> parse_repo_url('https://org.com/gitlab/group/repo')
('https://org.com/gitlab', 'group', 'repo')
"""
regex = re.compile(
(
"(?P<base>https://[^/]*(/gitlab|/projects)?)/"
"(?P<namespace>([^/]*/)*)"
"(?P<repo>[^/]*)$"
),
re.IGNORECASE,
)
captured = re.match(regex, url).groupdict()
base, namespace, repo = [captured[group] for group in ["base", "namespace", "repo"]]
namespace = namespace.strip("/")
return base, namespace, repo
def get_project_id(project_url: str, header=Dict[str, str]) -> int:
"""Given a project's URL, return it's gitlab ID"""
base, namespace, repo = parse_repo_url(project_url)
project = []
page = 1
while not len(project):
resp = requests.get(
f"{base}/api/v4/projects?search={repo}&per_page=100&page={page}",
headers=header,
)
if resp.ok:
resp = resp.json()
else:
resp.raise_for_status()
page += 1
project = [p for p in resp if p["path_with_namespace"] == f"{namespace}/{repo}"]
if len(list(project)) > 1:
raise ValueError("More than one project matched input url")
return project[0]["id"]
def collect_forks(project_url: str, header=Dict[str, str]) -> List[Dict]:
"""Retrieve the metadata from all forks of input project"""
base, namespace, repo = parse_repo_url(project_url)
......@@ -180,7 +130,7 @@ def format_fork_metadata(
@click.option(
"--token",
type=click.Path(exists=True),
help="Armored ASCII file containing the Gitlab API token. If not provided, you will be prompted for the token.",
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
def main(repo_url, token, deadline=None):
if token is None:
......
import re
from typing import Tuple, List, Dict, Optional
import requests
def parse_repo_url(url: str) -> Tuple[str, str, str]:
"""Decompose a full repo URL into 3 parts:
- the organization base URL
- the namespace (i.e. groups and subgroups
- the name of the repository
Examples
--------
>>> parse_repo_url('https://org.com/group/subgroup/repo')
('https://org.com', 'group/subgroup', 'repo')
>>> parse_repo_url('https://org.com/gitlab/group/repo')
('https://org.com/gitlab', 'group', 'repo')
"""
regex = re.compile(
(
"(?P<base>https://[^/]*(/gitlab|/projects)?)/"
"(?P<namespace>([^/]*/)*)"
"(?P<repo>[^/]*)$"
),
re.IGNORECASE,
)
captured = re.match(regex, url).groupdict()
base, namespace, repo = [captured[group] for group in ["base", "namespace", "repo"]]
namespace = namespace.strip("/")
return base, namespace, repo
def parse_group_url(url: str) -> Tuple[str, str]:
"""Decompose a full group URL into 2 parts:
- the organization base URL
- the namespace (i.e. groups and subgroups
Examples
--------
>>> parse_group_url('https://org.com/group/subgroup')
('https://org.com', 'group/subgroup')
>>> parse_group_url('https://org.com/gitlab/group')
('https://org.com/gitlab', 'group')
"""
regex = re.compile(
(
"(?P<base>https://[^/]*(/gitlab|/projects)?)/"
"(?P<namespace>([^/]*/)*[^/]*)$"
),
re.IGNORECASE,
)
captured = re.match(regex, url).groupdict()
base, namespace = [captured[group] for group in ["base", "namespace"]]
return base, namespace
def get_project_id(project_url: str, header: Dict[str, str]) -> int:
"""Given a project's URL, return it's gitlab ID"""
base, namespace, repo = parse_repo_url(project_url)
project = []
page = 1
# Gitlab only provides max 100 results/page, we need to scan all pages
# in case the project is not on the first one
while not len(project):
resp = requests.get(
f"{base}/api/v4/projects?search={repo}&per_page=100&page={page}",
headers=header,
)
if resp.ok:
resp = resp.json()
else:
resp.raise_for_status()
page += 1
project = [p for p in resp if p["path_with_namespace"] == f"{namespace}/{repo}"]
if len(list(project)) > 1:
raise ValueError("More than one project matched input url")
return project[0]["id"]
def get_group_id(group_url: str, header: Dict[str, str]) -> int:
"""Given a group's URL, return it's gitlab ID"""
base, namespace = parse_group_url(group_url)
if namespace.find("/") > 0:
parent_name = namespace.split("/")[0]
group_name = namespace.split("/")[-1]
query = f"{base}/api/v4/groups/{parent_name}/subgroups?search={group_name}"
else:
query = f"{base}/api/v4/groups/{namespace}"
response = requests.get(query, headers=header)
if response.ok:
content = response.json()
# Subgroup response comes as a list and must be unwrapped
if isinstance(content, list):
content = content[0]
return content["id"]
else:
response.raise_for_status()
# Given an input csv file of course participants created from moodle,
# create corresponding groups in moodle.
import re
from typing import Dict
import requests
import click
from teach_utils.common_requests import parse_repo_url, get_group_id
def invite_student_email(
student_email: str, base_url: str, group_id: int, header: Dict[str, str]
) -> None:
"""Invite student to group by email"""
response = requests.post(
f"{base_url}/api/v4/groups/{group_id}/invitations",
data={"email": student_email, "access_level": 40},
headers=header,
)
print(response.json())
def find_email(line: str) -> str:
"""Find and return email address in input string.
Raises an error if none found and only return the first
email if there are more than one.
Examples
--------
>>> find_email('robert,smith,robert.smith@no.where,123')
'robert.smith@no.where'
>>> find_email('mike.rock@yes.no,mrock@yes.no')
'mike.rock@yes.no'
"""
regex = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
try:
email = re.search(regex, line).groups()[0]
except AttributeError:
raise ValueError(f"Missing email: {line}")
return email
@click.command()
@click.argument(
"emails_file", type=click.Path(exists=True),
)
@click.argument("group-url")
@click.option(
"--token",
type=click.Path(exists=True),
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
def main(emails_file, group_url, token):
"""
Send invitations to join input gitlab group to all emails in the emails_file.
The file should have one email per line."""
if token is None:
token = click.prompt("Please enter your Gitlab API token", hide_input=True)
else:
token = open(token).read().strip()
header = {"PRIVATE-TOKEN": token}
# Recover the email on each line, in case there are multiple columns
emails = map(find_email, open(emails_file).read().split("\n"))
group_id = get_group_id(group_url, header)
base_url, _, _ = parse_repo_url(group_url)
for email in emails:
# Invite student to their group by email
invite_student_email(email, base_url, group_id, header)
if __name__ == "__main__":
main()
# Given an input csv file of course participants created from moodle,
# create corresponding groups in moodle.
import json
import datetime
from typing import Dict
import requests
import pandas as pd
import click
from teach_utils.common_requests import get_group_id, parse_group_url
def create_student_group(
group_name: str, parent_url: str, header: Dict[str, str]
) -> requests.Response:
"""Create a private group for student"""
data = {
"path": group_name,
"name": group_name,
"parent_id": get_group_id(parent_url),
"visibility": "private",
}
base, _ = parse_group_url(parent_url)
response = requests.post(
f"{base}/groups",
data=json.dumps(data),
headers={**header, "Content-Type": "application/json"},
)
return response
def invite_student_email(
student_email: str, base_url: str, group_id: int, header: Dict[str, str]
) -> None:
"""Invite student to group by email"""
response = requests.post(
f"{base_url}/groups/{group_id}/invitations",
data={"email": student_email, "access_level": 50},
headers=header,
)
print(response.json())
def generate_student_code(first_name: str, last_name: str) -> str:
"""Generate a uniform student identifier from their name"""
initials = first_name[0] + last_name[0]
year = datetime.datetime.now().year
return f"ST_{initials}_{year}".upper()
@click.command()
@click.argument("student_table")
@click.argument("parent_url")
@click.option(
"--token",
type=click.Path(exists=True),
help="Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token.",
)
def main(student_table, parent_url, token):
"""Given a parent-group URL and a CSV file of students exported from Moodle,
create one private subgroup for each student in the parent group and invite students
in their personal group. The student table must contain columns: "Email address", "First name"
and "Surname"
"""
if token is None:
token = click.prompt("Please enter your Gitlab API token", hide_input=True)
else:
token = open(token).read().strip()
header = {"PRIVATE-TOKEN": token}
students = pd.read_csv(student_table)
base_url, _ = parse_group_url(parent_url)
for _, st in students.iterrows():
# Generate a student code, e.g.: Ben Smith -> ST_BS_2022
student_code = generate_student_code(st["First name"], st["Surname"])
# Create student group "parent-group/ST_BS_2022"
resp = create_student_group(student_code, parent_url, header)
# Skip if already created
if not resp.ok:
print(f"Could not create {student_code}: {resp.json()['message']}")
group_id = get_group_id(f"{parent_url}/{student_code}", header)
# Invite student to their group by email
invite_student_email(st["Email address"], base_url, group_id, header)
if __name__ == "__main__":
main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment