JOJ/Joint-Teapot
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: git retry

Browse Source
This commit is contained in:
张泊明518370910136 2021-10-11 03:44:51 +08:00
parent c2333d88b1
commit 13e7278cdd
No known key found for this signature in database
GPG Key ID: FBEF5DE8B9F4C629
4 changed files with 150 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
joint_teapot/__main__.py
View File

@ -4,20 +4,32 @@ from datetime import datetime
from pathlib import Path
from typing import List
from typer import Typer, echo
from typer import Argument, Typer, echo
from joint_teapot.teapot import Teapot
from joint_teapot.utils.logger import logger
app = Typer(add_completion=False)
teapot = Teapot()
class Tea:
_teapot = None
@property
def pot(self) -> Teapot:
if not self._teapot:
self._teapot = Teapot()
return self._teapot
tea = Tea() # lazy loader
@app.command(
"invite-to-teams", help="invite all canvas students to gitea teams by team name"
)
def add_all_canvas_students_to_teams(team_names: List[str]) -> None:
teapot.add_all_canvas_students_to_teams(team_names)
tea.pot.add_all_canvas_students_to_teams(team_names)
@app.command(
@ -25,32 +37,32 @@ def add_all_canvas_students_to_teams(team_names: List[str]) -> None:
help="create personal repos on gitea for all canvas students",
)
def create_personal_repos_for_all_canvas_students() -> None:
teapot.create_personal_repos_for_all_canvas_students()
tea.pot.create_personal_repos_for_all_canvas_students()
@app.command("create-teams", help="create teams on gitea by canvas groups")
def create_teams_and_repos_by_canvas_groups(group_prefix: str) -> None:
teapot.create_teams_and_repos_by_canvas_groups(group_prefix)
tea.pot.create_teams_and_repos_by_canvas_groups(group_prefix)
@app.command("get-public-keys", help="list all public keys on gitea")
def get_public_key_of_all_canvas_students() -> None:
echo("\n".join(teapot.get_public_key_of_all_canvas_students()))
echo("\n".join(tea.pot.get_public_key_of_all_canvas_students()))
@app.command("clone-all-repos", help="clone all gitea repos to local")
def clone_all_repos() -> None:
teapot.clone_all_repos()
tea.pot.clone_all_repos()
@app.command("create-issues", help="create issues on gitea")
def create_issue_for_repos(repo_names: List[str], title: str, body: str) -> None:
teapot.create_issue_for_repos(repo_names, title, body)
tea.pot.create_issue_for_repos(repo_names, title, body)
@app.command("check-issues", help="check the existence of issue by title on gitea")
def check_exist_issue_by_title(repo_names: List[str], title: str) -> None:
echo("\n".join(teapot.check_exist_issue_by_title(repo_names, title)))
echo("\n".join(tea.pot.check_exist_issue_by_title(repo_names, title)))
@app.command(
@ -58,31 +70,37 @@ def check_exist_issue_by_title(repo_names: List[str], title: str) -> None:
help="checkout git repo to git tag fetched from gitea by release name, with due date",
)
def checkout_to_repos_by_release_name(
repo_names: List[str], release_name: str, due: datetime = datetime(3000, 1, 1)
repo_names: List[str], release_name: str, due: datetime = Argument("3000-01-01")
) -> None:
teapot.checkout_to_repos_by_release_name(repo_names, release_name, due)
failed_repos = tea.pot.checkout_to_repos_by_release_name(
repo_names, release_name, due
)
echo(f"failed repos: {failed_repos}")
@app.command(
"close-all-issues", help="close all issues and pull requests in gitea organization"
)
def close_all_issues() -> None:
teapot.gitea.close_all_issues()
tea.pot.gitea.close_all_issues()
@app.command("archieve-all-repos", help="archieve all repos in gitea organization")
def archieve_all_repos() -> None:
teapot.gitea.archieve_all_repos()
tea.pot.gitea.archieve_all_repos()
@app.command("get-no-collaborator-repos", help="list all repos with no collaborators")
def get_no_collaborator_repos() -> None:
teapot.gitea.get_no_collaborator_repos()
tea.pot.gitea.get_no_collaborator_repos()
@app.command("get-no-commit-repos", help="list all repos with no commit")
def get_no_commit_repos() -> None:
teapot.gitea.get_no_commit_repos()
@app.command("get-repos-status", help="list status of all repos with conditions")
def get_repos_status(
commit_lt: int = Argument(100000, help="commit count less than"),
issue_lt: int = Argument(100000, help="issue count less than"),
) -> None:
tea.pot.get_repos_status(commit_lt, issue_lt)
@app.command(
@ -90,7 +108,7 @@ def get_no_commit_repos() -> None:
help='prepare assignment dir from extracted canvas "Download Submissions" zip',
)
def prepare_assignment_dir(dir: Path) -> None:
teapot.canvas.prepare_assignment_dir(str(dir))
tea.pot.canvas.prepare_assignment_dir(str(dir))
@app.command(
@ -99,7 +117,7 @@ def prepare_assignment_dir(dir: Path) -> None:
+ "read the first line as score, the rest as comments",
)
def upload_assignment_scores(dir: Path, assignment_name: str) -> None:
teapot.canvas.upload_assignment_scores(str(dir), assignment_name)
tea.pot.canvas.upload_assignment_scores(str(dir), assignment_name)
if __name__ == "__main__":

45
joint_teapot/teapot.py
View File

@ -1,14 +1,19 @@
import functools
from datetime import datetime
from typing import Any, Callable, List, Optional
from typing import Any, Callable, List, Optional, TypeVar
from joint_teapot.config import settings
from joint_teapot.utils.logger import logger
from joint_teapot.utils.main import first
from joint_teapot.workers import Canvas, Git, Gitea
_T = TypeVar("_T")
def for_all_methods(decorator: Callable[..., Any]) -> Callable[..., Any]:
def for_all_methods(
decorator: Callable[[Callable[[_T], _T]], Any]
) -> Callable[[_T], _T]:
@functools.wraps(decorator)
def decorate(cls: Any) -> Any:
for attr in cls.__dict__: # there's propably a better way to do this
if callable(getattr(cls, attr)):
@ -55,7 +60,9 @@ class Teapot:
def __init__(self) -> None:
logger.info(
f"Settings loaded. Canvas Course ID: {settings.canvas_course_id}, Gitea Organization name: {settings.gitea_org_name}"
"Settings loaded. "
f"Canvas Course ID: {settings.canvas_course_id}, "
f"Gitea Organization name: {settings.gitea_org_name}"
)
logger.debug("Teapot initialized.")
@ -84,11 +91,10 @@ class Teapot:
def get_public_key_of_all_canvas_students(self) -> List[str]:
return self.gitea.get_public_key_of_canvas_students(self.canvas.students)
def clone_all_repos(self) -> List[str]:
return [
def clone_all_repos(self) -> None:
for i, repo_name in enumerate(self.gitea.get_all_repo_names()):
logger.info(f"{i}, {self.gitea.org_name}/{repo_name} cloning...")
self.git.repo_clean_and_checkout(repo_name, "master")
for repo_name in self.gitea.get_all_repo_names()
]
def create_issue_for_repos(
self, repo_names: List[str], title: str, body: str
@ -112,19 +118,26 @@ class Teapot:
due: datetime = datetime(3000, 1, 1),
) -> List[str]:
failed_repos = []
repos_releases = self.gitea.get_repos_releases(repo_names)
for repo_name, repo_releases in zip(repo_names, repos_releases):
release = first(repo_releases, lambda item: item["name"] == release_name)
if (
release is None
or datetime.strptime(release["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ")
>= due
):
for repo_name in repo_names:
repo_releases = self.gitea.get_repo_releases(repo_name)
release = first(repo_releases, lambda item: item.name == release_name)
if release is None or release.created_at.replace(tzinfo=None) >= due:
failed_repos.append(repo_name)
continue
self.git.repo_clean_and_checkout(repo_name, f"tags/{release['tag_name']}")
self.git.repo_clean_and_checkout(repo_name, f"tags/{release.tag_name}")
logger.info(
f"{self.gitea.org_name}/{repo_name} checkout to tags/{release.tag_name} succeed"
)
return failed_repos
def get_repos_status(self, commit_lt: int, issue_lt: int) -> None:
for repo_name, commit_count, issue_count in self.gitea.get_repos_status():
if commit_count < commit_lt or issue_count < issue_lt:
logger.info(
f"{self.gitea.org_name}/{repo_name} has "
f"{commit_count} commit(s), {issue_count} issue(s)"
)
if __name__ == "__main__":
teapot = Teapot()

73
joint_teapot/workers/git.py
View File

@ -1,5 +1,9 @@
import os
import sys
from time import sleep
from typing import Optional
from git.exc import GitCommandError
from joint_teapot.utils.logger import logger
@ -24,25 +28,70 @@ class Git:
self.repos_dir = repos_dir
logger.debug("Git initialized")
def clone_repo(self, repo_name: str, branch: str = "master") -> Repo:
def clone_repo(
self, repo_name: str, branch: str = "master", auto_retry: bool = True
) -> Optional[Repo]:
repo = None
repo_dir = os.path.join(self.repos_dir, repo_name)
return Repo.clone_from(
f"ssh://git@focs.ji.sjtu.edu.cn:2222/{self.org_name}/{repo_name}.git",
repo_dir,
branch=branch,
)
retry_interval = 2
while retry_interval and auto_retry:
try:
repo = Repo.clone_from(
f"ssh://git@focs.ji.sjtu.edu.cn:2222/{self.org_name}/{repo_name}.git",
repo_dir,
branch=branch,
)
except GitCommandError as e:
if "Connection refused" in e.stderr or "Connection reset" in e.stderr:
logger.warning(
f"{repo_name} connection refused/reset in clone. "
"Probably by JI firewall."
)
logger.info(f"wait for {retry_interval} seconds to retry...")
sleep(retry_interval)
if retry_interval < 64:
retry_interval *= 2
elif f"Remote branch {branch} not found in upstream origin" in e.stderr:
retry_interval = 0
logger.error(f"{repo_name} origin/{branch} not found")
else:
raise
return repo
def get_repo(self, repo_name: str) -> Repo:
def get_repo(self, repo_name: str) -> Optional[Repo]:
repo_dir = os.path.join(self.repos_dir, repo_name)
if os.path.exists(repo_dir):
return Repo(repo_dir)
return self.clone_repo(repo_name)
def repo_clean_and_checkout(self, repo_name: str, checkout_dest: str) -> str:
def repo_clean_and_checkout(
self, repo_name: str, checkout_dest: str, auto_retry: bool = True
) -> str:
repo_dir = os.path.join(self.repos_dir, repo_name)
repo = self.get_repo(repo_name)
repo.git.fetch("--tags", "--all", "-f")
repo.git.reset("--hard", f"origin/master")
repo.git.clean("-d", "-f", "-x")
repo.git.checkout(checkout_dest)
if not repo:
return repo_dir
retry_interval = 2
while retry_interval and auto_retry:
try:
repo.git.fetch("--tags", "--all", "-f")
repo.git.reset("--hard", f"origin/master")
repo.git.clean("-d", "-f", "-x")
repo.git.checkout(checkout_dest)
retry_interval = 0
except GitCommandError as e:
if "Connection refused" in e.stderr or "Connection reset" in e.stderr:
logger.warning(
f"{repo_name} connection refused/reset in clone. "
"Probably by JI firewall."
)
logger.info(f"wait for {retry_interval} seconds to retry...")
sleep(retry_interval)
if retry_interval < 64:
retry_interval *= 2
elif "Remote branch master not found in upstream origin" in e.stderr:
retry_interval = 0
logger.error(f"{repo_name} origin/master not found")
else:
raise
return repo_dir

38
joint_teapot/workers/gitea.py
View File

@ -1,7 +1,7 @@
import re
from enum import Enum
from functools import lru_cache
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, List, Optional, Tuple
import focs_gitea
from canvasapi.group import Group, GroupMembership
@ -267,11 +267,15 @@ class Gitea:
logger.error(e)
return res
def get_repos_releases(self, repo_names: List[str]) -> List[List[Dict[str, Any]]]:
return [
list_all(self.repository_api.repo_list_releases, self.org_name, repo_name)
for repo_name in repo_names
]
def get_repo_releases(self, repo_name: str) -> List[Any]:
res = []
try:
args = self.repository_api.repo_list_releases, self.org_name, repo_name
res = list_all(*args)
except ApiException as e:
if e.status != 404:
raise
return res
def get_all_repo_names(self) -> List[str]:
return [
@ -291,19 +295,24 @@ class Gitea:
res.append(data.name)
return res
def get_no_commit_repos(self) -> List[str]:
def get_repos_status(self) -> List[Tuple[str, int, int]]:
res = []
for data in list_all(self.organization_api.org_list_repos, self.org_name):
for repo in list_all(self.organization_api.org_list_repos, self.org_name):
try:
commits = self.repository_api.repo_get_all_commits(
self.org_name, data.name
self.org_name, repo.name
)
except ApiException as e:
if e.status == 409:
logger.info(f"{self.org_name}/{data.name} has no commits")
res.append(data.name)
else:
raise (e)
if e.status != 409:
raise
commits = []
issues = self.issue_api.issue_list_issues(
self.org_name, repo.name, state="all"
)
# if not commits:
# logger.info(f"{self.org_name}/{repo.name} has no commits")
# res.append(repo.name)
res.append((repo.name, len(commits), len(issues)))
return res
def create_issue(
@ -356,4 +365,3 @@ class Gitea:
if __name__ == "__main__":
gitea = Gitea()
res = gitea.get_no_commit_repos()