diff --git a/.gitignore b/.gitignore index 36a781e..cbc4898 100644 --- a/.gitignore +++ b/.gitignore @@ -293,6 +293,9 @@ dmypy.json !.vscode/extensions.json *.code-workspace +# vim +.vimspector.json + # End of https://www.toptal.com/developers/gitignore/api/vscode,python repos/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e292c07..8f989a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: hooks: - id: pyupgrade - repo: https://github.com/hadialqattan/pycln - rev: v0.0.1-beta.3 # Possible releases: https://github.com/hadialqattan/pycln/tags + rev: v1.3.2 # Possible releases: https://github.com/hadialqattan/pycln/tags hooks: - id: pycln - repo: https://github.com/PyCQA/bandit diff --git a/README.md b/README.md index 0c4a3d6..f8e50ce 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,10 @@ This tool is still under heavy development. The docs may not be updated on time, ```bash python3 -m venv env # you only need to do that once -source env/Scripts/activate # each time when you need this venv +# each time when you need this venv, if on Linux / macOS use +source env/bin/activate +# or this if on Windows +source env/Scripts/activate ``` ### Install @@ -33,20 +36,57 @@ pytest -svv ## Commands & Features -- `archive-all-repos`: archive all repos in gitea organization -- `check-issues`: check the existence of issue by title on gitea -- `checkout-releases`: checkout git repo to git tag fetched from gitea by release name, with due date -- `clone-all-repos`: clone all gitea repos to local -- `close-all-issues`: close all issues and pull requests in gitea organization -- `create-issues`: create issues on gitea -- `create-personal-repos`: create personal repos on gitea for all canvas students -- `create-teams`: create teams on gitea by canvas groups -- `get-no-collaborator-repos`: list all repos with no collaborators -- `get-public-keys`: list all public keys on gitea -- `get-repos-status`: list status of all repos with conditions -- `invite-to-teams`: invite all canvas students to gitea teams by team name -- `prepare-assignment-dir`: prepare assignment dir from extracted canvas "Download Submissions" zip -- `upload-assignment-grades`: upload assignment grades to canvas from grade file (GRADE.txt by default), read the first line as grade, the rest as comments +### `archive-all-repos` +archive all repos in gitea organization + +### `check-issues` +check the existence of issue by title on gitea + +### `checkout-releases` +checkout git repo to git tag fetched from gitea by release name, with due date + +### `clone-all-repos` +clone all gitea repos to local + +### `close-all-issues` +close all issues and pull requests in gitea organization + +### `create-channels-on-mm` +create channels for student groups according to group information on gitea. Optionally specify a prefix to ignore all repos whose names do not start with it. + +Example: `python3 -m joint_teapot create_channels_for_groups p1` will fetch all repos whose names start with `"p1"` and create same-name channels on mm for these repos. Members of a repo will be added to the corresponding channel. + +### `create-issues` +create issues on gitea + +### `create-personal-repos` +create personal repos on gitea for all canvas students + +### `create-teams` +create teams on gitea by canvas groups + +### `create-webhooks-for-mm` +Create a pair of webhooks on gitea and mm for all student groups on gitea, and configure them so that updates on gitea will be pushed to the mm channel. Optionally specify a prefix to ignore all repos whose names do not start with it. + +Example: `python3 -m joint_teapot create-webhooks-for-mm p1` will fetch all repos whose names start with `"p1"` and create two-way webhooks for these repos. All repos should already have same-name mm channels. If not, use `create-channels-on-mm` to create them. + +### `get-no-collaborator-repos` +list all repos with no collaborators + +### `get-public-keys` +list all public keys on gitea + +### `get-repos-status` +list status of all repos with conditions + +### `invite-to-teams` +invite all canvas students to gitea teams by team name + +### `prepare-assignment-dir` +prepare assignment dir from extracted canvas "Download Submissions" zip + +### `upload-assignment-grades` +upload assignment grades to canvas from grade file (GRADE.txt by default), read the first line as grade, the rest as comments ## License diff --git a/joint_teapot/app.py b/joint_teapot/app.py index 5b735a8..38e5d3b 100644 --- a/joint_teapot/app.py +++ b/joint_teapot/app.py @@ -135,6 +135,38 @@ def upload_assignment_grades(assignments_dir: Path, assignment_name: str) -> Non tea.pot.canvas.upload_assignment_grades(str(assignments_dir), assignment_name) +@app.command( + "create-channels-on-mm", + help="create channels for student groups according to group information on" + " gitea", +) +def create_channels_on_mm(prefix: str = Argument("")) -> None: + groups = [ + group + for group in tea.pot.gitea.get_all_teams() + if isinstance(group["name"], str) and group["name"].startswith(prefix) + ] + logger.info( + f"{len(groups)} channels to be created: {groups[0]['name']} ... {groups[-1]['name']}" + ) + tea.pot.mattermost.create_channels_for_groups(groups) + + +@app.command( + "create-webhooks-for-mm", + help="create a pair of webhooks on gitea and mm for all student groups on gitea, " + "and configure them so that updates on gitea will be pushed to the mm channel", +) +def create_webhooks_for_mm(prefix: str = Argument("")) -> None: + groups = [ + group["name"] + for group in tea.pot.gitea.get_all_teams() + if isinstance(group["name"], str) and group["name"].startswith(prefix) + ] + logger.info(f"{len(groups)} pairs to be created: {groups[0]} ... {groups[-1]}") + tea.pot.mattermost.create_webhooks_for_repos(groups, tea.pot.gitea) + + if __name__ == "__main__": try: app() diff --git a/joint_teapot/config.py b/joint_teapot/config.py index bcb590c..912d7c6 100644 --- a/joint_teapot/config.py +++ b/joint_teapot/config.py @@ -9,16 +9,26 @@ class Settings(BaseSettings): """ # canvas + canvas_domain_name: str = "umjicanvas.com" + canvas_suffix: str = "/" canvas_access_token: str = "" canvas_course_id: int = 0 # gitea + gitea_domain_name: str = "focs.ji.sjtu.edu.cn" + gitea_suffix: str = "/git" gitea_access_token: str = "" gitea_org_name: str = "" # git repos_dir: str = "./repos" + # mattermost + mattermost_domain_name: str = "focs.ji.sjtu.edu.cn" + mattermost_suffix: str = "/mm" + mattermost_access_token: str = "" + mattermost_team: str = "" + # sid joj_sid: str = "" diff --git a/joint_teapot/teapot.py b/joint_teapot/teapot.py index 0369ed5..5ef7d06 100644 --- a/joint_teapot/teapot.py +++ b/joint_teapot/teapot.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Optional, TypeVar from joint_teapot.config import settings from joint_teapot.utils.logger import logger from joint_teapot.utils.main import first -from joint_teapot.workers import Canvas, Git, Gitea +from joint_teapot.workers import Canvas, Git, Gitea, Mattermost from joint_teapot.workers.joj import JOJ _T = TypeVar("_T") @@ -41,6 +41,7 @@ class Teapot: _gitea = None _git = None _joj = None + _mattermost = None @property def canvas(self) -> Canvas: @@ -66,11 +67,18 @@ class Teapot: self._joj = JOJ() return self._joj + @property + def mattermost(self) -> Mattermost: + if not self._mattermost: + self._mattermost = Mattermost() + return self._mattermost + def __init__(self) -> None: logger.info( "Settings loaded. " f"Canvas Course ID: {settings.canvas_course_id}, " - f"Gitea Organization name: {settings.gitea_org_name}" + f"Gitea Organization name: {settings.gitea_org_name}, " + f"Mattermost Team name: {settings.mattermost_team}@{settings.mattermost_domain_name}{settings.mattermost_suffix}" ) logger.debug("Teapot initialized.") diff --git a/joint_teapot/workers/__init__.py b/joint_teapot/workers/__init__.py index aa8d92c..2ac53fe 100644 --- a/joint_teapot/workers/__init__.py +++ b/joint_teapot/workers/__init__.py @@ -1,3 +1,4 @@ from joint_teapot.workers.canvas import Canvas as Canvas from joint_teapot.workers.git import Git as Git from joint_teapot.workers.gitea import Gitea as Gitea +from joint_teapot.workers.mattermost import Mattermost as Mattermost diff --git a/joint_teapot/workers/canvas.py b/joint_teapot/workers/canvas.py index 518566c..c622a9a 100644 --- a/joint_teapot/workers/canvas.py +++ b/joint_teapot/workers/canvas.py @@ -15,11 +15,13 @@ from joint_teapot.utils.main import first, percentile class Canvas: def __init__( self, + domain_name: str = settings.canvas_domain_name, + suffix: str = settings.canvas_suffix, access_token: str = settings.canvas_access_token, course_id: int = settings.canvas_course_id, grade_filename: str = "GRADE.txt", ): - self.canvas = PyCanvas("https://umjicanvas.com/", access_token) + self.canvas = PyCanvas(f"https://{domain_name}{suffix}", access_token) self.course = self.canvas.get_course(course_id) logger.info(f"Canvas course loaded. {self.course}") # types = ["student", "observer"] diff --git a/joint_teapot/workers/gitea.py b/joint_teapot/workers/gitea.py index df8ba29..efb7bdf 100644 --- a/joint_teapot/workers/gitea.py +++ b/joint_teapot/workers/gitea.py @@ -1,7 +1,7 @@ from datetime import datetime from enum import Enum from functools import lru_cache -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union import focs_gitea from canvasapi.group import Group, GroupMembership @@ -41,10 +41,13 @@ class Gitea: self, access_token: str = settings.gitea_access_token, org_name: str = settings.gitea_org_name, + domain_name: str = settings.gitea_domain_name, + suffix: str = settings.gitea_suffix, ): self.org_name = org_name configuration = focs_gitea.Configuration() configuration.api_key["access_token"] = access_token + configuration.host = f"https://{domain_name}{suffix}/api/v1" self.api_client = focs_gitea.ApiClient(configuration) self.admin_api = focs_gitea.AdminApi(self.api_client) self.miscellaneous_api = focs_gitea.MiscellaneousApi(self.api_client) @@ -148,8 +151,8 @@ class Gitea: repos = list_all(self.organization_api.org_list_repos, self.org_name) group: Group for group in groups: - team_name = team_name_convertor(group.name) - repo_name = repo_name_convertor(group.name) + team_name = team_name_convertor(group["name"]) + repo_name = repo_name_convertor(group["name"]) if team_name is None or repo_name is None: continue team = first(teams, lambda team: team.name == team_name) @@ -375,6 +378,32 @@ class Gitea: self.org_name, repo.name, body={"archived": True} ) + def get_all_teams( + self, + ) -> List[Dict[str, Union[str, List[str]]]]: + ret: List[Dict[str, Union[str, List[str]]]] = [] + try: + teams_raw = self.organization_api.org_list_teams(self.org_name) + except ApiException as e: + logger.error(f"Failed to get teams from organization {self.org_name}: {e}") + exit(1) + for team_raw in teams_raw: + if team_raw.name == "Owners": + continue + team_id = team_raw.id + try: + members = [ + m.login.lower() + for m in self.organization_api.org_list_team_members(team_id) + ] + except ApiException as e: + logger.warning( + f"Failed to get members of team {team_id} in {self.org_name}: {e}" + ) + continue + ret.append({"name": team_raw.name, "members": members}) + return ret + if __name__ == "__main__": gitea = Gitea() diff --git a/joint_teapot/workers/mattermost.py b/joint_teapot/workers/mattermost.py new file mode 100644 index 0000000..5415a4f --- /dev/null +++ b/joint_teapot/workers/mattermost.py @@ -0,0 +1,150 @@ +from typing import Dict, List, Union + +import focs_gitea +from mattermostdriver import Driver + +from joint_teapot.config import settings +from joint_teapot.utils.logger import logger +from joint_teapot.workers.gitea import Gitea + + +class Mattermost: + def __init__( + self, + access_token: str = settings.mattermost_access_token, + team_name: str = settings.mattermost_team, + domain_name: str = settings.mattermost_domain_name, + suffix: str = settings.mattermost_suffix, + ): + self.url = domain_name + self.url_suffix = suffix + self.endpoint = Driver( + { + "url": domain_name, + "port": 443, + "basepath": suffix + "/api/v4", + "token": access_token, + } + ) + try: + operator = self.endpoint.login() + except Exception: + logger.error("Cannot login to Mattermost") + return + if "admin" not in operator["roles"]: + logger.error("Please make sure you have enough permission") + try: + self.team = self.endpoint.teams.get_team_by_name(team_name) + except Exception as e: + logger.error(f"Cannot get team {team_name}: {e}") + return + + def create_channels_for_groups( + self, groups: List[Dict[str, Union[str, List[str]]]] + ) -> None: + for group in groups: + try: + channel = self.endpoint.channels.create_channel( + { + "team_id": self.team["id"], + "name": group["name"], + "display_name": group["name"], + "type": "P", # create private channels + } + ) + logger.info(f"Added group {group['name']} to Mattermost") + except Exception as e: + logger.warning( + f"Error when creating channel {group['name']}: {e} Perhaps channel already exists?" + ) + continue + for member in group["members"]: + try: + mmuser = self.endpoint.users.get_user_by_username(member) + except Exception: + logger.warning( + f"User {member} is not found on the Mattermost server" + ) + continue + # code for adding student to mm, disabled since there is no need to do that + # try: + # mmuser = self.endpoint.users.create_user({'email':f"{member}@sjtu.edu.cn", 'username':member, auth_service:"gitlab"}) + # except e: + # logger.error(f"Error creating user {member}") + # continue + try: + self.endpoint.channels.add_user( + channel["id"], {"user_id": mmuser["id"]} + ) + except Exception: + logger.warning(f"User {member} is not in the team") + logger.info(f"Added member {member} to channel {group['name']}") + + def create_webhooks_for_repos(self, repos: List[str], gitea: Gitea) -> None: + # one group corresponds to one repo so these concepts can be used interchangably + for repo in repos: + logger.info(f"Creating webhooks for repo {gitea.org_name}/{repo}") + try: + mm_channel = self.endpoint.channels.get_channel_by_name( + self.team["id"], repo + ) + except Exception as e: + logger.warning( + f"Error when getting channel {repo} from Mattermost team {self.team['name']}: {e}" + ) + continue + try: + mm_webhook = self.endpoint.webhooks.create_incoming_hook( + { + "channel_id": mm_channel["id"], + "display_name": f"Gitea integration for {self.team['name']}/{repo}", + "channel_locked": True, + } + ) + except Exception as e: + logger.error(f"Error when creating incoming webhook at Mattermost: {e}") + continue + try: + gitea.repository_api.repo_create_hook( + gitea.org_name, + repo, + body=focs_gitea.CreateHookOption( + active=True, + type="slack", + events=[ + "create", + "delete", + "push", + "release", + "issues_only", + "issue_assign", + "issue_comment", + "pull_request_only", + "pull_request_assign", + "pull_request_comment", + "pull_request_review", + ], + config={ + "url": f"https://{self.url}{self.url_suffix}/hooks/{mm_webhook['id']}", + "username": "FOCS Gitea", + "icon_url": f"https://{self.url}{self.url_suffix}/api/v4/brand/image", + "content_type": "json", + "channel": repo, + }, + ), + ) + except Exception as e: + logger.warning(f"Error when creating outgoing webhook at Gitea: {e}") + + # unused since we can give students invitation links instead + def invite_students_to_team(self, students: List[str]) -> None: + for student in students: + try: + mmuser = self.endpoint.users.get_user_by_username(student) + except Exception: + logger.warning(f"User {student} is not found on the Mattermost server") + continue + self.endpoint.teams.add_user_to_team( + self.team["id"], {"user_id": mmuser["id"], "team_id": self.team["id"]} + ) + logger.info(f"Added user {student} to team {self.team['name']}") diff --git a/requirements.txt b/requirements.txt index 6388f15..71cd70d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ focs_gitea>=1.0.0 GitPython>=3.1.18 joj-submitter>=0.0.8 loguru>=0.5.3 +mattermostdriver>=7.3.2 patool>=1.12 pydantic[dotenv]>=1.8.1 typer[all]>=0.3.2 diff --git a/setup.py b/setup.py index 1aa0d0e..e571b72 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ setup( version=get_version("joint_teapot"), url="https://github.com/BoYanZh/joint-teapot", license="MIT", - description="A handy tool for TAs in JI to handle stuffs through Gitea, Canvas, and JOJ.", + description="A handy tool for TAs in JI to handle stuffs through Gitea, Canvas, JOJ and Mattermost.", long_description=get_long_description(), long_description_content_type="text/markdown", author="BoYanZh",