JOJ/Joint-Teapot
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: canvas score utils

Browse Source
This commit is contained in:
张泊明518370910136 2021-10-07 01:12:22 +08:00
parent 7fa19c3cb9
commit 2dbe321d43
No known key found for this signature in database
GPG Key ID: FBEF5DE8B9F4C629
5 changed files with 406 additions and 332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
joint_teapot/__main__.py
View File

@ -1,6 +1,7 @@
__version__ = "0.0.0" __version__ = "0.0.0"
from datetime import datetime from datetime import datetime
from pathlib import Path
from typing import List from typing import List
from typer import Typer, echo from typer import Typer, echo
@ -66,22 +67,39 @@ def checkout_to_repos_by_release_name(
"close-all-issues", help="close all issues and pull requests in gitea organization" "close-all-issues", help="close all issues and pull requests in gitea organization"
) )
def close_all_issues() -> None: def close_all_issues() -> None:
teapot.close_all_issues() teapot.gitea.close_all_issues()
@app.command("archieve-all-repos", help="archieve all repos in gitea organization") @app.command("archieve-all-repos", help="archieve all repos in gitea organization")
def archieve_all_repos() -> None: def archieve_all_repos() -> None:
teapot.archieve_all_repos() teapot.gitea.archieve_all_repos()
@app.command("get-no-collaborator-repos", help="list all repos with no collaborators") @app.command("get-no-collaborator-repos", help="list all repos with no collaborators")
def get_no_collaborator_repos() -> None: def get_no_collaborator_repos() -> None:
teapot.get_no_collaborator_repos() teapot.gitea.get_no_collaborator_repos()
@app.command("get-no-commit-repos", help="list all repos with no commit") @app.command("get-no-commit-repos", help="list all repos with no commit")
def get_no_commit_repos() -> None: def get_no_commit_repos() -> None:
teapot.get_no_commit_repos() teapot.gitea.get_no_commit_repos()
@app.command(
"prepare-assignment-dir",
help='prepare assignment dir from extracted canvas "Download Submissions" zip',
)
def prepare_assignment_dir(dir: Path) -> None:
teapot.canvas.prepare_assignment_dir(str(dir))
@app.command(
"upload-assignment-scores",
help="upload assignment scores to canvas from score file (SCORE.txt by default), "
+ "read the first line as score, the rest as comments",
)
def upload_assignment_scores(dir: Path, assignment_name: str) -> None:
teapot.canvas.upload_assignment_scores(str(dir), assignment_name)
if __name__ == "__main__": if __name__ == "__main__":

19
joint_teapot/teapot.py
View File

@ -68,8 +68,13 @@ class Teapot:
) )
def create_teams_and_repos_by_canvas_groups(self) -> List[str]: def create_teams_and_repos_by_canvas_groups(self) -> List[str]:
def convertor(name: str) -> str:
team_name, number_str = name.split(" ")
number = int(number_str)
return f"{team_name}-{number:02}"
return self.gitea.create_teams_and_repos_by_canvas_groups( return self.gitea.create_teams_and_repos_by_canvas_groups(
self.canvas.students, self.canvas.groups self.canvas.students, self.canvas.groups, convertor, convertor
) )
def get_public_key_of_all_canvas_students(self) -> List[str]: def get_public_key_of_all_canvas_students(self) -> List[str]:
@ -116,18 +121,6 @@ class Teapot:
self.git.repo_clean_and_checkout(repo_name, f"tags/{release['tag_name']}") self.git.repo_clean_and_checkout(repo_name, f"tags/{release['tag_name']}")
return failed_repos return failed_repos
def close_all_issues(self) -> None:
self.gitea.close_all_issues()
def archieve_all_repos(self) -> None:
self.gitea.archieve_all_repos()
def get_no_collaborator_repos(self) -> None:
self.gitea.get_no_collaborator_repos()
def get_no_commit_repos(self) -> None:
self.gitea.get_no_commit_repos()
if __name__ == "__main__": if __name__ == "__main__":
teapot = Teapot() teapot = Teapot()

60
joint_teapot/workers/canvas.py
View File

@ -1,9 +1,13 @@
import os
from glob import glob
from canvasapi import Canvas as PyCanvas from canvasapi import Canvas as PyCanvas
from patoolib import extract_archive
from patoolib.util import PatoolError
from joint_teapot.config import settings from joint_teapot.config import settings
from joint_teapot.utils.logger import logger from joint_teapot.utils.logger import logger
from joint_teapot.utils.main import first
# from canvasapi.group import Group, GroupMembership
class Canvas: class Canvas:
@ -11,12 +15,13 @@ class Canvas:
self, self,
access_token: str = settings.canvas_access_token, access_token: str = settings.canvas_access_token,
course_id: int = settings.canvas_course_id, course_id: int = settings.canvas_course_id,
score_filename: str = "SCORE.txt",
): ):
self.canvas = PyCanvas("https://umjicanvas.com/", access_token) self.canvas = PyCanvas("https://umjicanvas.com/", access_token)
self.course = self.canvas.get_course(course_id) self.course = self.canvas.get_course(course_id)
logger.info(f"Canvas course loaded. {self.course}") logger.info(f"Canvas course loaded. {self.course}")
self.students = self.course.get_users( self.students = self.course.get_users(
enrollment_type=["student", "observer"], include=["email"] enrollment_type=["student"], include=["email"]
) )
for attr in ["sis_login_id", "sortable_name", "name"]: for attr in ["sis_login_id", "sortable_name", "name"]:
if not hasattr(self.students[0], attr): if not hasattr(self.students[0], attr):
@ -28,8 +33,57 @@ class Canvas:
logger.debug(f"Canvas assignments loaded") logger.debug(f"Canvas assignments loaded")
self.groups = self.course.get_groups() self.groups = self.course.get_groups()
logger.debug(f"Canvas groups loaded") logger.debug(f"Canvas groups loaded")
self.score_filename = score_filename
logger.debug("Canvas initialized") logger.debug("Canvas initialized")
def prepare_assignment_dir(self, dir: str, create_score_file: bool = True) -> None:
login_ids = {stu.id: stu.login_id for stu in self.students}
for v in login_ids.values():
new_path = os.path.join(dir, v)
if not os.path.exists(new_path):
os.mkdir(new_path)
for path in glob(os.path.join(dir, "*")):
file_name = os.path.basename(path)
if "_" not in file_name:
continue
segments = file_name.split("_")
if segments[1] == "late":
file_id = int(segments[2])
student = first(
self.students, lambda x: x.login_id == login_ids[file_id]
)
logger.info(f"{student} submits late")
else:
file_id = int(segments[1])
target_dir = os.path.join(dir, login_ids[file_id])
try:
extract_archive(path, outdir=target_dir, verbosity=-1)
os.remove(path)
except PatoolError:
os.rename(path, os.path.join(target_dir, file_name))
if create_score_file:
open(os.path.join(target_dir, self.score_filename), mode="w")
def upload_assignment_scores(self, dir: str, assignment_name: str) -> None:
assignment = first(self.assignments, lambda x: x.name == assignment_name)
if assignment is None:
logger.info(f"Canvas assignment {assignment_name} not found")
return
for submission in assignment.get_submissions():
student = first(self.students, lambda x: x.id == submission.user_id)
if student is None:
continue
score_file_path = os.path.join(
dir, student.sis_login_id, self.score_filename
)
score, *comments = list(open(score_file_path))
data = {
"submission": {"posted_grade": float(score)},
"comment": {"text_comment": "".join(comments)},
}
logger.info(f"{assignment} {student} {data.__repr__()}")
submission.edit(**data)
if __name__ == "__main__": if __name__ == "__main__":
canvas = Canvas() canvas = Canvas()

632
joint_teapot/workers/gitea.py
View File

@ -1,312 +1,320 @@
import re import re
from enum import Enum from enum import Enum
from functools import lru_cache from functools import lru_cache
from typing import Any, Callable, Dict, List, Optional from typing import Any, Callable, Dict, List, Optional
import focs_gitea import focs_gitea
from canvasapi.group import Group, GroupMembership from canvasapi.group import Group, GroupMembership
from canvasapi.paginated_list import PaginatedList from canvasapi.paginated_list import PaginatedList
from canvasapi.user import User from canvasapi.user import User
from focs_gitea.rest import ApiException from focs_gitea.rest import ApiException
from joint_teapot.config import settings from joint_teapot.config import settings
from joint_teapot.utils.logger import logger from joint_teapot.utils.logger import logger
from joint_teapot.utils.main import first from joint_teapot.utils.main import first
class PermissionEnum(Enum): class PermissionEnum(Enum):
read = "read" read = "read"
write = "write" write = "write"
admin = "admin" admin = "admin"
def default_repo_name_convertor(user: User) -> Optional[str]: def default_repo_name_convertor(user: User) -> Optional[str]:
id, name = user.sis_login_id, user.name id, name = user.sis_login_id, user.name
eng = re.sub("[\u4e00-\u9fa5]", "", name) eng = re.sub("[\u4e00-\u9fa5]", "", name)
eng = eng.replace(",", "") eng = eng.replace(",", "")
eng = "".join([word[0].capitalize() + word[1:] for word in eng.split()]) eng = "".join([word[0].capitalize() + word[1:] for word in eng.split()])
return f"{eng}{id}" return f"{eng}{id}"
def list_all(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: def list_all(method: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
all_res = [] all_res = []
page = 1 page = 1
while True: while True:
res = method(*args, **kwargs, page=page) res = method(*args, **kwargs, page=page)
if not res: if not res:
break break
for item in res: for item in res:
all_res.append(item) all_res.append(item)
page += 1 page += 1
return all_res return all_res
class Gitea: class Gitea:
def __init__( def __init__(
self, self,
access_token: str = settings.gitea_access_token, access_token: str = settings.gitea_access_token,
org_name: str = settings.gitea_org_name, org_name: str = settings.gitea_org_name,
): ):
self.org_name = org_name self.org_name = org_name
configuration = focs_gitea.Configuration() configuration = focs_gitea.Configuration()
configuration.api_key["access_token"] = access_token configuration.api_key["access_token"] = access_token
self.api_client = focs_gitea.ApiClient(configuration) self.api_client = focs_gitea.ApiClient(configuration)
self.admin_api = focs_gitea.AdminApi(self.api_client) self.admin_api = focs_gitea.AdminApi(self.api_client)
self.miscellaneous_api = focs_gitea.MiscellaneousApi(self.api_client) self.miscellaneous_api = focs_gitea.MiscellaneousApi(self.api_client)
self.organization_api = focs_gitea.OrganizationApi(self.api_client) self.organization_api = focs_gitea.OrganizationApi(self.api_client)
self.issue_api = focs_gitea.IssueApi(self.api_client) self.issue_api = focs_gitea.IssueApi(self.api_client)
self.repository_api = focs_gitea.RepositoryApi(self.api_client) self.repository_api = focs_gitea.RepositoryApi(self.api_client)
self.settings_api = focs_gitea.SettingsApi(self.api_client) self.settings_api = focs_gitea.SettingsApi(self.api_client)
self.user_api = focs_gitea.UserApi(self.api_client) self.user_api = focs_gitea.UserApi(self.api_client)
logger.debug("Gitea initialized") logger.debug("Gitea initialized")
@lru_cache() @lru_cache()
def _get_team_id_by_name(self, name: str) -> int: def _get_team_id_by_name(self, name: str) -> int:
res = self.organization_api.team_search(self.org_name, q=str(name), limit=1) res = self.organization_api.team_search(self.org_name, q=str(name), limit=1)
if len(res["data"]) == 0: if len(res["data"]) == 0:
raise Exception(f"{name} not found by name in Gitea") raise Exception(f"{name} not found by name in Gitea")
return res["data"][0]["id"] return res["data"][0]["id"]
@lru_cache() @lru_cache()
def _get_username_by_canvas_student(self, student: User) -> str: def _get_username_by_canvas_student(self, student: User) -> str:
res = self.user_api.user_search(q=student.sis_login_id, limit=1) res = self.user_api.user_search(q=student.sis_login_id, limit=1)
if len(res["data"]) == 0: if len(res["data"]) == 0:
raise Exception(f"{student} not found in Gitea") raise Exception(f"{student} not found in Gitea")
return res["data"][0]["username"] return res["data"][0]["username"]
def add_canvas_students_to_teams( def add_canvas_students_to_teams(
self, students: PaginatedList, team_names: List[str] self, students: PaginatedList, team_names: List[str]
) -> None: ) -> None:
for team_name in team_names: for team_name in team_names:
team_id = self._get_team_id_by_name(team_name) team_id = self._get_team_id_by_name(team_name)
team_members = self.organization_api.org_list_team_members(team_id) team_members = self.organization_api.org_list_team_members(team_id)
for student in students: for student in students:
try: try:
username = self._get_username_by_canvas_student(student) username = self._get_username_by_canvas_student(student)
team_member = first(team_members, lambda x: x.login == username) team_member = first(team_members, lambda x: x.login == username)
if team_member is None: if team_member is None:
self.organization_api.org_add_team_member(team_id, username) self.organization_api.org_add_team_member(team_id, username)
logger.info(f"{student} added to team {team_name}") logger.info(f"{student} added to team {team_name}")
else: else:
team_members.remove(team_member) team_members.remove(team_member)
logger.warning(f"{student} already in team {team_name}") logger.warning(f"{student} already in team {team_name}")
except Exception as e: except Exception as e:
logger.error(e) logger.error(e)
for team_member in team_members: for team_member in team_members:
logger.error( logger.error(
f"{team_member.full_name} found in team {team_name} " f"{team_member.full_name} found in team {team_name} "
+ "but not found in Canvas students" + "but not found in Canvas students"
) )
def create_personal_repos_for_canvas_students( def create_personal_repos_for_canvas_students(
self, self,
students: PaginatedList, students: PaginatedList,
repo_name_convertor: Callable[ repo_name_convertor: Callable[
[User], Optional[str] [User], Optional[str]
] = default_repo_name_convertor, ] = default_repo_name_convertor,
) -> List[str]: ) -> List[str]:
repo_names = [] repo_names = []
for student in students: for student in students:
repo_name = repo_name_convertor(student) repo_name = repo_name_convertor(student)
if repo_name is None: if repo_name is None:
continue continue
repo_names.append(repo_name) repo_names.append(repo_name)
body = { body = {
"auto_init": False, "auto_init": False,
"default_branch": "master", "default_branch": "master",
"name": repo_name, "name": repo_name,
"private": True, "private": True,
"template": False, "template": False,
"trust_model": "default", "trust_model": "default",
} }
try: try:
try: try:
repo = self.organization_api.create_org_repo( repo = self.organization_api.create_org_repo(
self.org_name, body=body self.org_name, body=body
) )
logger.info( logger.info(
f"Personal repo {self.org_name}/{repo_name} for {student} created" f"Personal repo {self.org_name}/{repo_name} for {student} created"
) )
except ApiException as e: except ApiException as e:
if e.status == 409: if e.status == 409:
logger.warning( logger.warning(
f"Personal repo {self.org_name}/{repo_name} for {student} already exists" f"Personal repo {self.org_name}/{repo_name} for {student} already exists"
) )
else: else:
raise (e) raise (e)
username = self._get_username_by_canvas_student(student) username = self._get_username_by_canvas_student(student)
self.repository_api.repo_add_collaborator( self.repository_api.repo_add_collaborator(
self.org_name, repo_name, username self.org_name, repo_name, username
) )
except Exception as e: except Exception as e:
logger.error(e) logger.error(e)
return repo_names return repo_names
def create_teams_and_repos_by_canvas_groups( def create_teams_and_repos_by_canvas_groups(
self, self,
students: PaginatedList, students: PaginatedList,
groups: PaginatedList, groups: PaginatedList,
team_name_convertor: Callable[[str], Optional[str]] = lambda name: name, team_name_convertor: Callable[[str], Optional[str]] = lambda name: name,
repo_name_convertor: Callable[[str], Optional[str]] = lambda name: name, repo_name_convertor: Callable[[str], Optional[str]] = lambda name: name,
permission: PermissionEnum = PermissionEnum.write, permission: PermissionEnum = PermissionEnum.write,
) -> List[str]: ) -> List[str]:
repo_names = [] repo_names = []
group: Group teams = list_all(self.organization_api.org_list_teams, self.org_name)
for group in groups: repos = list_all(self.organization_api.org_list_repos, self.org_name)
team_name = team_name_convertor(group.name) group: Group
repo_name = repo_name_convertor(group.name) for group in groups:
if team_name is None or repo_name is None: team_name = team_name_convertor(group.name)
continue repo_name = repo_name_convertor(group.name)
repo_names.append(repo_name) if team_name is None or repo_name is None:
team = self.organization_api.org_create_team( continue
self.org_name, team = first(teams, lambda team: team.name == team_name)
body={ if team is None:
"can_create_org_repo": False, team = self.organization_api.org_create_team(
"includes_all_repositories": False, self.org_name,
"name": team_name, body={
"permission": permission.value, "can_create_org_repo": False,
"units": [ "includes_all_repositories": False,
"repo.code", "name": team_name,
"repo.issues", "permission": permission.value,
"repo.ext_issues", "units": [
"repo.wiki", "repo.code",
"repo.pulls", "repo.issues",
"repo.releases", "repo.ext_issues",
"repo.projects", "repo.wiki",
"repo.ext_wiki", "repo.pulls",
], "repo.releases",
}, "repo.projects",
) "repo.ext_wiki",
repo = self.organization_api.create_org_repo( ],
self.org_name, },
body={ )
"auto_init": False, logger.info(f"{self.org_name}/{team_name} created")
"default_branch": "master", repo = first(repos, lambda repo: repo.name == repo_name)
"name": repo_name, if repo is None:
"private": True, repo_names.append(repo_name)
"template": False, repo = self.organization_api.create_org_repo(
"trust_model": "default", self.org_name,
}, body={
) "auto_init": False,
self.organization_api.org_add_team_repository( "default_branch": "master",
team.id, self.org_name, repo_name "name": repo_name,
) "private": True,
membership: GroupMembership "template": False,
for membership in group.get_memberships(): "trust_model": "default",
student = first(students, lambda s: s.id == membership.user_id) },
if student is None: )
raise Exception( logger.info(f"Team {team_name} created")
f"student with user_id {membership.user_id} not found" self.organization_api.org_add_team_repository(
) team.id, self.org_name, repo_name
username = self._get_username_by_canvas_student(student) )
self.organization_api.org_add_team_member(team.id, username) membership: GroupMembership
self.repository_api.repo_add_collaborator( for membership in group.get_memberships():
self.org_name, repo_name, username student = first(students, lambda s: s.id == membership.user_id)
) if student is None:
return repo_names raise Exception(
f"student with user_id {membership.user_id} not found"
def get_public_key_of_canvas_students(self, students: PaginatedList) -> List[str]: )
res = [] username = self._get_username_by_canvas_student(student)
for student in students: self.organization_api.org_add_team_member(team.id, username)
try: self.repository_api.repo_add_collaborator(
username = self._get_username_by_canvas_student(student) self.org_name, repo_name, username
res.extend( )
[ return repo_names
item.key
for item in list_all(self.user_api.user_list_keys, username) def get_public_key_of_canvas_students(self, students: PaginatedList) -> List[str]:
] res = []
) for student in students:
except Exception as e: try:
logger.error(e) username = self._get_username_by_canvas_student(student)
return res res.extend(
[
def get_repos_releases(self, repo_names: List[str]) -> List[List[Dict[str, Any]]]: item.key
return [ for item in list_all(self.user_api.user_list_keys, username)
list_all(self.repository_api.repo_list_releases, self.org_name, repo_name) ]
for repo_name in repo_names )
] except Exception as e:
logger.error(e)
def get_all_repo_names(self) -> List[str]: return res
return [
data.name def get_repos_releases(self, repo_names: List[str]) -> List[List[Dict[str, Any]]]:
for data in list_all(self.organization_api.org_list_repos, self.org_name) return [
] list_all(self.repository_api.repo_list_releases, self.org_name, repo_name)
for repo_name in repo_names
def get_no_collaborator_repos(self) -> List[str]: ]
res = []
for data in list_all(self.organization_api.org_list_repos, self.org_name): def get_all_repo_names(self) -> List[str]:
collaborators = self.repository_api.repo_list_collaborators( return [
self.org_name, data.name data.name
) for data in list_all(self.organization_api.org_list_repos, self.org_name)
if collaborators: ]
continue
logger.info(f"{self.org_name}/{data.name} has no collaborators") def get_no_collaborator_repos(self) -> List[str]:
res.append(data.name) res = []
return res for data in list_all(self.organization_api.org_list_repos, self.org_name):
collaborators = self.repository_api.repo_list_collaborators(
def get_no_commit_repos(self) -> List[str]: self.org_name, data.name
res = [] )
for data in list_all(self.organization_api.org_list_repos, self.org_name): if collaborators:
try: continue
commits = self.repository_api.repo_get_all_commits( logger.info(f"{self.org_name}/{data.name} has no collaborators")
self.org_name, data.name res.append(data.name)
) return res
except ApiException as e:
if e.status == 409: def get_no_commit_repos(self) -> List[str]:
logger.info(f"{self.org_name}/{data.name} has no commits") res = []
res.append(data.name) for data in list_all(self.organization_api.org_list_repos, self.org_name):
else: try:
raise (e) commits = self.repository_api.repo_get_all_commits(
return res self.org_name, data.name
)
def create_issue( except ApiException as e:
self, if e.status == 409:
repo_name: str, logger.info(f"{self.org_name}/{data.name} has no commits")
title: str, res.append(data.name)
body: str, else:
assign_every_collaborators: bool = True, raise (e)
) -> None: return res
assignees = []
if assign_every_collaborators: def create_issue(
assignees = [ self,
item.username repo_name: str,
for item in list_all( title: str,
self.repository_api.repo_list_collaborators, body: str,
self.org_name, assign_every_collaborators: bool = True,
repo_name, ) -> None:
) assignees = []
] if assign_every_collaborators:
self.issue_api.issue_create_issue( assignees = [
self.org_name, item.username
repo_name, for item in list_all(
body={"title": title, "body": body, "assignees": assignees}, self.repository_api.repo_list_collaborators,
) self.org_name,
repo_name,
def check_exist_issue_by_title(self, repo_name: str, title: str) -> bool: )
for issue in list_all( ]
self.issue_api.issue_list_issues, self.org_name, repo_name self.issue_api.issue_create_issue(
): self.org_name,
if issue.title == title: repo_name,
return True body={"title": title, "body": body, "assignees": assignees},
return False )
def close_all_issues(self) -> None: def check_exist_issue_by_title(self, repo_name: str, title: str) -> bool:
for repo in list_all(self.organization_api.org_list_repos, self.org_name): for issue in list_all(
for issue in list_all( self.issue_api.issue_list_issues, self.org_name, repo_name
self.issue_api.issue_list_issues, self.org_name, repo.name ):
): if issue.title == title:
if issue.state != "closed": return True
self.issue_api.issue_edit_issue( return False
self.org_name, repo.name, issue.number, body={"state": "closed"}
) def close_all_issues(self) -> None:
for repo in list_all(self.organization_api.org_list_repos, self.org_name):
def archieve_all_repos(self) -> None: for issue in list_all(
for repo in list_all(self.organization_api.org_list_repos, self.org_name): self.issue_api.issue_list_issues, self.org_name, repo.name
self.repository_api.repo_edit( ):
self.org_name, repo.name, body={"archived": True} if issue.state != "closed":
) self.issue_api.issue_edit_issue(
self.org_name, repo.name, issue.number, body={"state": "closed"}
)
if __name__ == "__main__":
gitea = Gitea() def archieve_all_repos(self) -> None:
res = gitea.get_no_commit_repos() for repo in list_all(self.organization_api.org_list_repos, self.org_name):
self.repository_api.repo_edit(
self.org_name, repo.name, body={"archived": True}
)
if __name__ == "__main__":
gitea = Gitea()
res = gitea.get_no_commit_repos()

1
requirements.txt
View File

@ -2,5 +2,6 @@ canvasapi>=2.2.0
focs_gitea>=1.0.0 focs_gitea>=1.0.0
GitPython>=3.1.18 GitPython>=3.1.18
loguru>=0.5.3 loguru>=0.5.3
patool>=1.12
pydantic[dotenv]>=1.8.1 pydantic[dotenv]>=1.8.1
typer[all]>=0.3.2 typer[all]>=0.3.2