From 13e7278cddb433e1b13cd7ec8bb7f4624199e239 Mon Sep 17 00:00:00 2001 From: BoYanZh Date: Mon, 11 Oct 2021 03:44:51 +0800 Subject: [PATCH] feat: git retry --- joint_teapot/__main__.py | 56 ++++++++++++++++++--------- joint_teapot/teapot.py | 45 +++++++++++++-------- joint_teapot/workers/git.py | 73 +++++++++++++++++++++++++++++------ joint_teapot/workers/gitea.py | 38 +++++++++++------- 4 files changed, 150 insertions(+), 62 deletions(-) diff --git a/joint_teapot/__main__.py b/joint_teapot/__main__.py index 92d9fd7..deaae97 100644 --- a/joint_teapot/__main__.py +++ b/joint_teapot/__main__.py @@ -4,20 +4,32 @@ from datetime import datetime from pathlib import Path from typing import List -from typer import Typer, echo +from typer import Argument, Typer, echo from joint_teapot.teapot import Teapot from joint_teapot.utils.logger import logger app = Typer(add_completion=False) -teapot = Teapot() + + +class Tea: + _teapot = None + + @property + def pot(self) -> Teapot: + if not self._teapot: + self._teapot = Teapot() + return self._teapot + + +tea = Tea() # lazy loader @app.command( "invite-to-teams", help="invite all canvas students to gitea teams by team name" ) def add_all_canvas_students_to_teams(team_names: List[str]) -> None: - teapot.add_all_canvas_students_to_teams(team_names) + tea.pot.add_all_canvas_students_to_teams(team_names) @app.command( @@ -25,32 +37,32 @@ def add_all_canvas_students_to_teams(team_names: List[str]) -> None: help="create personal repos on gitea for all canvas students", ) def create_personal_repos_for_all_canvas_students() -> None: - teapot.create_personal_repos_for_all_canvas_students() + tea.pot.create_personal_repos_for_all_canvas_students() @app.command("create-teams", help="create teams on gitea by canvas groups") def create_teams_and_repos_by_canvas_groups(group_prefix: str) -> None: - teapot.create_teams_and_repos_by_canvas_groups(group_prefix) + tea.pot.create_teams_and_repos_by_canvas_groups(group_prefix) @app.command("get-public-keys", help="list all public keys on gitea") def get_public_key_of_all_canvas_students() -> None: - echo("\n".join(teapot.get_public_key_of_all_canvas_students())) + echo("\n".join(tea.pot.get_public_key_of_all_canvas_students())) @app.command("clone-all-repos", help="clone all gitea repos to local") def clone_all_repos() -> None: - teapot.clone_all_repos() + tea.pot.clone_all_repos() @app.command("create-issues", help="create issues on gitea") def create_issue_for_repos(repo_names: List[str], title: str, body: str) -> None: - teapot.create_issue_for_repos(repo_names, title, body) + tea.pot.create_issue_for_repos(repo_names, title, body) @app.command("check-issues", help="check the existence of issue by title on gitea") def check_exist_issue_by_title(repo_names: List[str], title: str) -> None: - echo("\n".join(teapot.check_exist_issue_by_title(repo_names, title))) + echo("\n".join(tea.pot.check_exist_issue_by_title(repo_names, title))) @app.command( @@ -58,31 +70,37 @@ def check_exist_issue_by_title(repo_names: List[str], title: str) -> None: help="checkout git repo to git tag fetched from gitea by release name, with due date", ) def checkout_to_repos_by_release_name( - repo_names: List[str], release_name: str, due: datetime = datetime(3000, 1, 1) + repo_names: List[str], release_name: str, due: datetime = Argument("3000-01-01") ) -> None: - teapot.checkout_to_repos_by_release_name(repo_names, release_name, due) + failed_repos = tea.pot.checkout_to_repos_by_release_name( + repo_names, release_name, due + ) + echo(f"failed repos: {failed_repos}") @app.command( "close-all-issues", help="close all issues and pull requests in gitea organization" ) def close_all_issues() -> None: - teapot.gitea.close_all_issues() + tea.pot.gitea.close_all_issues() @app.command("archieve-all-repos", help="archieve all repos in gitea organization") def archieve_all_repos() -> None: - teapot.gitea.archieve_all_repos() + tea.pot.gitea.archieve_all_repos() @app.command("get-no-collaborator-repos", help="list all repos with no collaborators") def get_no_collaborator_repos() -> None: - teapot.gitea.get_no_collaborator_repos() + tea.pot.gitea.get_no_collaborator_repos() -@app.command("get-no-commit-repos", help="list all repos with no commit") -def get_no_commit_repos() -> None: - teapot.gitea.get_no_commit_repos() +@app.command("get-repos-status", help="list status of all repos with conditions") +def get_repos_status( + commit_lt: int = Argument(100000, help="commit count less than"), + issue_lt: int = Argument(100000, help="issue count less than"), +) -> None: + tea.pot.get_repos_status(commit_lt, issue_lt) @app.command( @@ -90,7 +108,7 @@ def get_no_commit_repos() -> None: help='prepare assignment dir from extracted canvas "Download Submissions" zip', ) def prepare_assignment_dir(dir: Path) -> None: - teapot.canvas.prepare_assignment_dir(str(dir)) + tea.pot.canvas.prepare_assignment_dir(str(dir)) @app.command( @@ -99,7 +117,7 @@ def prepare_assignment_dir(dir: Path) -> None: + "read the first line as score, the rest as comments", ) def upload_assignment_scores(dir: Path, assignment_name: str) -> None: - teapot.canvas.upload_assignment_scores(str(dir), assignment_name) + tea.pot.canvas.upload_assignment_scores(str(dir), assignment_name) if __name__ == "__main__": diff --git a/joint_teapot/teapot.py b/joint_teapot/teapot.py index 781ab22..e95cf70 100644 --- a/joint_teapot/teapot.py +++ b/joint_teapot/teapot.py @@ -1,14 +1,19 @@ import functools from datetime import datetime -from typing import Any, Callable, List, Optional +from typing import Any, Callable, List, Optional, TypeVar from joint_teapot.config import settings from joint_teapot.utils.logger import logger from joint_teapot.utils.main import first from joint_teapot.workers import Canvas, Git, Gitea +_T = TypeVar("_T") -def for_all_methods(decorator: Callable[..., Any]) -> Callable[..., Any]: + +def for_all_methods( + decorator: Callable[[Callable[[_T], _T]], Any] +) -> Callable[[_T], _T]: + @functools.wraps(decorator) def decorate(cls: Any) -> Any: for attr in cls.__dict__: # there's propably a better way to do this if callable(getattr(cls, attr)): @@ -55,7 +60,9 @@ class Teapot: def __init__(self) -> None: logger.info( - f"Settings loaded. Canvas Course ID: {settings.canvas_course_id}, Gitea Organization name: {settings.gitea_org_name}" + "Settings loaded. " + f"Canvas Course ID: {settings.canvas_course_id}, " + f"Gitea Organization name: {settings.gitea_org_name}" ) logger.debug("Teapot initialized.") @@ -84,11 +91,10 @@ class Teapot: def get_public_key_of_all_canvas_students(self) -> List[str]: return self.gitea.get_public_key_of_canvas_students(self.canvas.students) - def clone_all_repos(self) -> List[str]: - return [ + def clone_all_repos(self) -> None: + for i, repo_name in enumerate(self.gitea.get_all_repo_names()): + logger.info(f"{i}, {self.gitea.org_name}/{repo_name} cloning...") self.git.repo_clean_and_checkout(repo_name, "master") - for repo_name in self.gitea.get_all_repo_names() - ] def create_issue_for_repos( self, repo_names: List[str], title: str, body: str @@ -112,19 +118,26 @@ class Teapot: due: datetime = datetime(3000, 1, 1), ) -> List[str]: failed_repos = [] - repos_releases = self.gitea.get_repos_releases(repo_names) - for repo_name, repo_releases in zip(repo_names, repos_releases): - release = first(repo_releases, lambda item: item["name"] == release_name) - if ( - release is None - or datetime.strptime(release["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ") - >= due - ): + for repo_name in repo_names: + repo_releases = self.gitea.get_repo_releases(repo_name) + release = first(repo_releases, lambda item: item.name == release_name) + if release is None or release.created_at.replace(tzinfo=None) >= due: failed_repos.append(repo_name) continue - self.git.repo_clean_and_checkout(repo_name, f"tags/{release['tag_name']}") + self.git.repo_clean_and_checkout(repo_name, f"tags/{release.tag_name}") + logger.info( + f"{self.gitea.org_name}/{repo_name} checkout to tags/{release.tag_name} succeed" + ) return failed_repos + def get_repos_status(self, commit_lt: int, issue_lt: int) -> None: + for repo_name, commit_count, issue_count in self.gitea.get_repos_status(): + if commit_count < commit_lt or issue_count < issue_lt: + logger.info( + f"{self.gitea.org_name}/{repo_name} has " + f"{commit_count} commit(s), {issue_count} issue(s)" + ) + if __name__ == "__main__": teapot = Teapot() diff --git a/joint_teapot/workers/git.py b/joint_teapot/workers/git.py index 1b80c77..015945d 100644 --- a/joint_teapot/workers/git.py +++ b/joint_teapot/workers/git.py @@ -1,5 +1,9 @@ import os import sys +from time import sleep +from typing import Optional + +from git.exc import GitCommandError from joint_teapot.utils.logger import logger @@ -24,25 +28,70 @@ class Git: self.repos_dir = repos_dir logger.debug("Git initialized") - def clone_repo(self, repo_name: str, branch: str = "master") -> Repo: + def clone_repo( + self, repo_name: str, branch: str = "master", auto_retry: bool = True + ) -> Optional[Repo]: + repo = None repo_dir = os.path.join(self.repos_dir, repo_name) - return Repo.clone_from( - f"ssh://git@focs.ji.sjtu.edu.cn:2222/{self.org_name}/{repo_name}.git", - repo_dir, - branch=branch, - ) + retry_interval = 2 + while retry_interval and auto_retry: + try: + repo = Repo.clone_from( + f"ssh://git@focs.ji.sjtu.edu.cn:2222/{self.org_name}/{repo_name}.git", + repo_dir, + branch=branch, + ) + except GitCommandError as e: + if "Connection refused" in e.stderr or "Connection reset" in e.stderr: + logger.warning( + f"{repo_name} connection refused/reset in clone. " + "Probably by JI firewall." + ) + logger.info(f"wait for {retry_interval} seconds to retry...") + sleep(retry_interval) + if retry_interval < 64: + retry_interval *= 2 + elif f"Remote branch {branch} not found in upstream origin" in e.stderr: + retry_interval = 0 + logger.error(f"{repo_name} origin/{branch} not found") + else: + raise + return repo - def get_repo(self, repo_name: str) -> Repo: + def get_repo(self, repo_name: str) -> Optional[Repo]: repo_dir = os.path.join(self.repos_dir, repo_name) if os.path.exists(repo_dir): return Repo(repo_dir) return self.clone_repo(repo_name) - def repo_clean_and_checkout(self, repo_name: str, checkout_dest: str) -> str: + def repo_clean_and_checkout( + self, repo_name: str, checkout_dest: str, auto_retry: bool = True + ) -> str: repo_dir = os.path.join(self.repos_dir, repo_name) repo = self.get_repo(repo_name) - repo.git.fetch("--tags", "--all", "-f") - repo.git.reset("--hard", f"origin/master") - repo.git.clean("-d", "-f", "-x") - repo.git.checkout(checkout_dest) + if not repo: + return repo_dir + retry_interval = 2 + while retry_interval and auto_retry: + try: + repo.git.fetch("--tags", "--all", "-f") + repo.git.reset("--hard", f"origin/master") + repo.git.clean("-d", "-f", "-x") + repo.git.checkout(checkout_dest) + retry_interval = 0 + except GitCommandError as e: + if "Connection refused" in e.stderr or "Connection reset" in e.stderr: + logger.warning( + f"{repo_name} connection refused/reset in clone. " + "Probably by JI firewall." + ) + logger.info(f"wait for {retry_interval} seconds to retry...") + sleep(retry_interval) + if retry_interval < 64: + retry_interval *= 2 + elif "Remote branch master not found in upstream origin" in e.stderr: + retry_interval = 0 + logger.error(f"{repo_name} origin/master not found") + else: + raise return repo_dir diff --git a/joint_teapot/workers/gitea.py b/joint_teapot/workers/gitea.py index 08835b3..428bb3b 100644 --- a/joint_teapot/workers/gitea.py +++ b/joint_teapot/workers/gitea.py @@ -1,7 +1,7 @@ import re from enum import Enum from functools import lru_cache -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, List, Optional, Tuple import focs_gitea from canvasapi.group import Group, GroupMembership @@ -267,11 +267,15 @@ class Gitea: logger.error(e) return res - def get_repos_releases(self, repo_names: List[str]) -> List[List[Dict[str, Any]]]: - return [ - list_all(self.repository_api.repo_list_releases, self.org_name, repo_name) - for repo_name in repo_names - ] + def get_repo_releases(self, repo_name: str) -> List[Any]: + res = [] + try: + args = self.repository_api.repo_list_releases, self.org_name, repo_name + res = list_all(*args) + except ApiException as e: + if e.status != 404: + raise + return res def get_all_repo_names(self) -> List[str]: return [ @@ -291,19 +295,24 @@ class Gitea: res.append(data.name) return res - def get_no_commit_repos(self) -> List[str]: + def get_repos_status(self) -> List[Tuple[str, int, int]]: res = [] - for data in list_all(self.organization_api.org_list_repos, self.org_name): + for repo in list_all(self.organization_api.org_list_repos, self.org_name): try: commits = self.repository_api.repo_get_all_commits( - self.org_name, data.name + self.org_name, repo.name ) except ApiException as e: - if e.status == 409: - logger.info(f"{self.org_name}/{data.name} has no commits") - res.append(data.name) - else: - raise (e) + if e.status != 409: + raise + commits = [] + issues = self.issue_api.issue_list_issues( + self.org_name, repo.name, state="all" + ) + # if not commits: + # logger.info(f"{self.org_name}/{repo.name} has no commits") + # res.append(repo.name) + res.append((repo.name, len(commits), len(issues))) return res def create_issue( @@ -356,4 +365,3 @@ class Gitea: if __name__ == "__main__": gitea = Gitea() - res = gitea.get_no_commit_repos()