Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Learn Renku
Teaching on Renku
Advanced teaching automation
Commits
86628d2c
Commit
86628d2c
authored
Jul 22, 2022
by
Cyril Matthey-Doret
Browse files
Refactor cloning
parent
6939a25c
Changes
12
Hide whitespace changes
Inline
Side-by-side
Makefile
View file @
86628d2c
...
...
@@ -4,7 +4,7 @@ install:
pip
install
-e
.
uninstall
:
pip uninstall teach-ut
ils
pip uninstall teach-
a
ut
o
test
:
pytest
--doctest-modules
teach_ut
ils
pytest
--doctest-modules
teach_
a
ut
o
requirements.txt
View file @
86628d2c
click
GitPython
requests
pytz
pandas
setup.py
View file @
86628d2c
...
...
@@ -4,7 +4,7 @@ with open("requirements.txt", "r") as f:
REQUIREMENTS
=
f
.
read
()
setup
(
name
=
"teach_ut
ils
"
,
name
=
"teach_
a
ut
o
"
,
version
=
"0.1"
,
packages
=
find_packages
(),
install_requires
=
REQUIREMENTS
,
...
...
teach_ut
ils
/__init__.py
→
teach_
a
ut
o
/__init__.py
View file @
86628d2c
File moved
teach_auto/clone_forks.py
0 → 100644
View file @
86628d2c
#!/usr/bin/env python
# Reads JSON output from collect-forks (either as a file or in stdin)
# and clone all target forks into provided directory at the target commit.
# usage ./clone_forks_from_json.sh OUT_DIR forks.json
# ./collect_forks.py --token token.asc URL | ./clone_forks_from_json.sh OUT_DIR
import
sys
from
pathlib
import
Path
import
json
import
click
import
git
def
clone_fork
(
url
:
str
,
out_path
:
Path
,
commit
:
str
):
"""
Clone input repo in target directory
and checkout desired commit."""
cloned
=
git
.
Repo
.
clone_from
(
url
,
out_path
)
cloned
.
git
.
checkout
(
commit
)
@
click
.
command
()
@
click
.
argument
(
"out_dir"
,
type
=
click
.
Path
(
file_okay
=
False
,
exists
=
False
))
@
click
.
argument
(
"forks"
,
type
=
click
.
File
(
mode
=
"r"
),
default
=
sys
.
stdin
)
def
main
(
out_dir
,
forks
,
token
):
forks
=
json
.
load
(
forks
)
for
fork
in
forks
:
url
,
group
,
commit
=
[
fork
.
get
(
key
)
for
key
in
[
"url"
,
"group"
,
"commit"
]]
path
=
Path
(
out_dir
)
/
group
clone_fork
(
url
,
path
,
commit
)
teach_ut
ils
/collect_forks.py
→
teach_
a
ut
o
/collect_forks.py
View file @
86628d2c
...
...
@@ -15,7 +15,8 @@ import requests
import
click
from
datetime
import
datetime
import
pytz
from
teach_utils.common_requests
import
parse_repo_url
,
get_project_id
from
class_auto.common_requests
import
parse_repo_url
,
get_project_id
from
class_auto.utils
import
ingest_token
def
validate_iso_date
(
date
:
str
)
->
str
:
...
...
@@ -49,7 +50,7 @@ def collect_forks(project_url: str, header=Dict[str, str]) -> List[Dict]:
def
filter_group_forks
(
forks
:
List
[
Dict
])
->
List
[
Dict
]:
"""Given a list of forks' metadata, only keep those that belong to a group
Examples
--------
>>> d1 = {'id': 1, 'namespace': {'kind': 'user'}}
...
...
@@ -93,9 +94,9 @@ def get_last_commit_hash(
def
format_fork_metadata
(
fork
:
Dict
,
header
:
Dict
[
str
,
str
],
deadline
:
Optional
[
str
]
=
None
)
->
Dict
:
"""Format and add fields to a fork's metadata. The
"""Format and add fields to a fork's metadata. The
resulting metadata will have the following fields:
id,
http_url_to_repo
, autostart_url, commit, members, visibility, group"""
id,
url
, autostart_url, commit, members, visibility, group"""
meta
=
{
"id"
:
fork
[
"id"
],
"url"
:
fork
[
"http_url_to_repo"
],
...
...
@@ -134,17 +135,12 @@ def format_fork_metadata(
)
@
click
.
argument
(
"repo_url"
,
type
=
str
)
@
click
.
option
(
"--token"
,
"--token
-path
"
,
type
=
click
.
Path
(
exists
=
True
),
help
=
"Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token."
,
)
def
main
(
repo_url
,
token
,
deadline
=
None
):
if
token
is
None
:
token
=
click
.
prompt
(
"Please enter your Gitlab API token"
,
hide_input
=
True
,
err
=
True
)
else
:
token
=
open
(
token
).
read
().
strip
()
def
main
(
repo_url
,
token_path
,
deadline
=
None
):
token
=
ingest_token
(
token_path
)
# Check for valid deadline format
if
deadline
is
not
None
:
validate_iso_date
(
deadline
)
...
...
teach_ut
ils
/common_requests.py
→
teach_
a
ut
o
/common_requests.py
View file @
86628d2c
File moved
teach_ut
ils/moodle_to_student_
groups.py
→
teach_
a
ut
o/create-
groups.py
View file @
86628d2c
...
...
@@ -7,7 +7,8 @@ from typing import Dict
import
requests
import
pandas
as
pd
import
click
from
teach_utils.common_requests
import
get_group_id
,
parse_group_url
from
teach_auto.common_requests
import
get_group_id
,
parse_group_url
from
teach_auto.utils
import
ingest_token
def
create_student_group
(
...
...
@@ -52,23 +53,17 @@ def generate_student_code(first_name: str, last_name: str) -> str:
@
click
.
argument
(
"student_table"
)
@
click
.
argument
(
"parent_url"
)
@
click
.
option
(
"--token"
,
"--token
-path
"
,
type
=
click
.
Path
(
exists
=
True
),
help
=
"Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token."
,
)
def
main
(
student_table
,
parent_url
,
token
):
def
main
(
student_table
,
parent_url
,
token
_path
):
"""Given a parent-group URL and a CSV file of students exported from Moodle,
create one private subgroup for each student in the parent group and invite students
in their personal group. The student table must contain columns: "Email address", "First name"
and "Surname"
"""
if
token
is
None
:
token
=
click
.
prompt
(
"Please enter your Gitlab API token"
,
hide_input
=
True
,
err
=
True
)
else
:
token
=
open
(
token
).
read
().
strip
()
token
=
ingest_token
(
token_path
)
header
=
{
"PRIVATE-TOKEN"
:
token
}
students
=
pd
.
read_csv
(
student_table
)
...
...
teach_ut
ils
/invite_students.py
→
teach_
a
ut
o
/invite_students.py
View file @
86628d2c
...
...
@@ -5,7 +5,8 @@ import re
from
typing
import
Dict
import
requests
import
click
from
teach_utils.common_requests
import
parse_repo_url
,
get_group_id
from
class_auto.common_requests
import
parse_repo_url
,
get_group_id
from
class_auto.utils
import
ingest_token
def
invite_student_email
(
...
...
@@ -24,10 +25,10 @@ def find_email(line: str) -> str:
"""Find and return email address in input string.
Raises an error if none found and only return the first
email if there are more than one.
Examples
--------
>>> find_email('robert,smith,robert.smith@no.where,123')
'robert.smith@no.where'
>>> find_email('mike.rock@yes.no,mrock@yes.no')
...
...
@@ -47,20 +48,15 @@ def find_email(line: str) -> str:
)
@
click
.
argument
(
"group-url"
)
@
click
.
option
(
"--token"
,
"--token
-path
"
,
type
=
click
.
Path
(
exists
=
True
),
help
=
"Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token."
,
)
def
main
(
emails_file
,
group_url
,
token
):
def
main
(
emails_file
,
group_url
,
token
_path
):
"""
Send invitations to join input gitlab group to all emails in the emails_file.
The file should have one email per line."""
if
token
is
None
:
token
=
click
.
prompt
(
"Please enter your Gitlab API token"
,
hide_input
=
True
,
err
=
True
)
else
:
token
=
open
(
token
).
read
().
strip
()
token
=
ingest_token
(
token_path
)
header
=
{
"PRIVATE-TOKEN"
:
token
}
# Recover the email on each line, in case there are multiple columns
...
...
teach_ut
ils
/send_feedback.py
→
teach_
a
ut
o
/send_feedback.py
View file @
86628d2c
...
...
@@ -10,11 +10,13 @@
import
sys
import
json
from
string
import
Formatter
from
typing
import
Dict
from
typing
import
Dict
,
Optional
import
requests
from
pathlib
import
Path
import
click
import
pandas
as
pd
from
teach_utils.common_requests
import
parse_repo_url
from
class_auto.common_requests
import
parse_repo_url
from
class_auto.utils
import
ingest_token
FEEDBACK_TEMPLATE
=
"""
...
...
@@ -58,10 +60,39 @@ def fill_template(content: Dict[str, str], template: str) -> str:
raise
ValueError
(
f
"Fields missing from content:
{
missing_flds
}
"
)
from
None
def
gather_fill_data
(
fork
:
Dict
,
feedback_csv
:
Optional
[
Path
]
=
None
)
->
Dict
:
"""Gather metadata and feedback for a single fork into a dictionary
to use for filling the template. The resulting dictionary contains
fields: project, group, members, url, commit and visibility from the fork
metadata, as well as all columns present in the feedback csv file.
"""
# Get basic fields in fork metadata
_
,
group
,
repo
=
parse_repo_url
(
fork
[
"url"
])
members
=
"
\n
"
.
join
([
f
"*
{
member
[
'name'
]
}
"
for
member
in
fork
[
"members"
]])
fill_data
=
{
"project"
:
repo
.
removesuffix
(
".git"
),
"group"
:
group
,
"members"
:
members
,
}
# add commit, url and visibility fields from fork data
fill_data
|=
{
key
:
fork
.
get
(
key
)
for
key
in
[
"url"
,
"commit"
,
"visibility"
]}
# If a CSV file with feedback is provided, read the fields
# of the current fork (row) to the fill data.
if
feedback_csv
:
fill_data
|=
(
pd
.
read_csv
(
feedback_csv
)
.
query
(
f
'id ==
{
fork
[
"id"
]
}
'
)
.
iloc
[
0
]
.
drop
(
"id"
)
.
to_dict
()
)
return
fill_data
@
click
.
command
()
@
click
.
argument
(
"forks"
,
type
=
click
.
File
(
mode
=
"r"
),
default
=
sys
.
stdin
)
@
click
.
option
(
"--token"
,
"--token
-path
"
,
type
=
click
.
Path
(
exists
=
True
),
help
=
"Path to a file containing the Gitlab API token. If not provided, you will be prompted for the token."
,
)
...
...
@@ -74,29 +105,22 @@ def fill_template(content: Dict[str, str], template: str) -> str:
default
=
None
,
)
@
click
.
option
(
"--feedback"
,
"--feedback
-csv
"
,
type
=
click
.
Path
(
exists
=
True
),
help
=
"CSV file with column 'id', and additional columns matching fields in the template"
"The id column should match a gitlab project ID from the JSON input."
,
default
=
None
,
)
@
click
.
option
(
"--dry-run"
,
is_flag
=
True
,
help
=
"Print feedback to stdout instead of opening issues"
"--dry-run"
,
is_flag
=
True
,
help
=
"Print feedback to stdout instead of opening issues"
)
def
main
(
forks
,
feedback
,
token
,
template
,
dry_run
):
def
main
(
forks
,
feedback
_csv
,
token
_path
,
template
,
dry_run
):
"""
Send feedback to repositories in input JSON by opening issues.
"""
# Build header with authentication token
if
token
is
None
:
token
=
click
.
prompt
(
"Please enter your Gitlab API token"
,
hide_input
=
True
,
err
=
True
)
else
:
token
=
open
(
token
).
read
().
strip
()
token
=
ingest_token
(
token_path
)
header
=
{
"PRIVATE-TOKEN"
:
token
}
forks
=
json
.
load
(
forks
)
...
...
@@ -108,30 +132,11 @@ def main(forks, feedback, token, template, dry_run):
feedback_template
=
FEEDBACK_TEMPLATE
for
fork
in
forks
:
# Get basic fields from repo metadata in json
base
,
group
,
repo
=
parse_repo_url
(
fork
[
"url"
])
members
=
"
\n
"
.
join
([
f
"*
{
member
[
'name'
]
}
"
for
member
in
fork
[
"members"
]])
fill_data
=
{
"project"
:
repo
.
removesuffix
(
".git"
),
"group"
:
group
,
"members"
:
members
,
}
# add commit, url and visibility fields from fork data
fill_data
|=
{
key
:
fork
.
get
(
key
)
for
key
in
[
"url"
,
"commit"
,
"visibility"
]}
# If provided, get additional fields from feedback table
if
feedback
:
feedback_fields
=
(
pd
.
read_csv
(
feedback
)
.
query
(
f
'id ==
{
fork
[
"id"
]
}
'
)
.
iloc
[
0
]
.
drop
(
"id"
)
.
to_dict
()
)
fill_data
|=
feedback_fields
base
=
parse_repo_url
(
fork
[
"url"
])[
0
]
fill_data
=
gather_fill_data
(
fork
,
feedback_csv
)
# Replace fields in the template by their value for the current fork
description
=
fill_template
(
fill_data
,
feedback_template
)
# Create issue
# Create issue
, or print to stdout if dry run
if
dry_run
:
print
(
fork
)
print
(
description
)
...
...
teach_auto/utils.py
0 → 100644
View file @
86628d2c
from
typing
import
Optional
from
pathlib
import
Path
import
click
def
ingest_token
(
token_path
:
Optional
[
Path
]
=
None
)
->
str
:
"""Read token from target file if provided.
If input is None, prompt for token."""
if
token_path
is
None
:
token
=
click
.
prompt
(
"Please enter your Gitlab API token"
,
hide_input
=
True
,
err
=
True
)
else
:
token
=
open
(
token_path
).
read
().
strip
()
return
token
teach_utils/clone_forks_from_json.sh
deleted
100644 → 0
View file @
6939a25c
#!/usr/bin/env bash
# Reads JSON output from collect_forks.py (either as a file or in stdin)
# and clone all target forks into provided directory at the target commit.
# usage ./clone_forks_from_json.sh OUT_DIR forks.json
# ./collect_forks.py --token token.asc URL | ./clone_forks_from_json.sh OUT_DIR
# Help message
function
usage
()
{
cat
<<
EOF
Usage:
./
$(
basename
$0
)
[outdir] [in_file.json]
./collect_forks.py --token token.asc URL | ./
$(
basename
$0
)
[outdir]
Reads json output from collect_forks.py and clone all target forks into provided directory
at the target commit. Repositories are cloned into outdir/namespace.
Arguments:
outdir: Directory where all forks will be cloned [default: .]
in_file.json: JSON output of collect_forks.py containing fork metadata [default: stdin]
EOF
exit
0
}
# Parsing CL arguments
OUT_DIR
=
${
1
:-
.
}
JSON
=
${
2
:-
/dev/stdin
}
if
[[
$#
-gt
2
]]
||
[[
$1
==
'-h'
]]
||
[[
$1
==
'--help'
]]
;
then
usage
fi
mkdir
-p
"
${
OUT_DIR
}
"
jq
".[] |
\"
git clone
\(
.url)
$OUT_DIR
/
\(
.group) && cd
${
OUT_DIR
}
/
\(
.group) && git checkout
\(
.commit)
\"
"
\
<
"
${
JSON
}
"
\
| xargs
-L
1
-I
{}
sh
-c
"{}"
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment