feat: joj3 auto retry on git push failed

joint_teapot/app.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime
 from pathlib import Path
+from time import sleep
 from typing import TYPE_CHECKING, List
 
 from filelock import FileLock
@@ -530,7 +531,6 @@ def joj3_all(
         )
     if skip_scoreboard and skip_failed_table:
         return
-    tea.pot.git  # trigger lazy load
     lock_file_path = os.path.join(
         settings.repos_dir, repo_name, settings.joj3_lock_file_path
     )
@@ -539,6 +539,9 @@ def joj3_all(
         + f"timeout: {settings.joj3_lock_file_timeout}"
     )
     with FileLock(lock_file_path, timeout=settings.joj3_lock_file_timeout).acquire():
+        retry_interval = 1
+        git_push_ok = False
+        while not git_push_ok:
             logger.info("file lock acquired")
             repo_path = tea.pot.git.repo_clean_and_checkout(repo_name, "grading")
             repo: Repo = tea.pot.git.get_repo(repo_name)
@@ -586,7 +589,17 @@ def joj3_all(
                         f"gitea issue link: {gitea_issue_url}"
                     ),
                 )
-        tea.pot.git.push(repo_name)
+            push_info_list = tea.pot.git.push(repo_name)
+            git_push_ok = push_info_list.error is None
+            if not git_push_ok:
+                retry_interval *= 2
+                logger.info(
+                    f"git push failed, retry in {retry_interval} seconds: {push_info_list}"
+                )
+                if retry_interval > 64:
+                    logger.error(f"git push failed too many times")
+                    return
+                sleep(retry_interval)
 
 
 if __name__ == "__main__":

joint_teapot/workers/git.py

@@ -9,6 +9,7 @@ current_path = sys.path[0]
 sys.path.remove(current_path)
 from git import Repo
 from git.exc import GitCommandError
+from git.remote import PushInfoList
 
 sys.path.insert(0, current_path)
 
@@ -117,6 +118,6 @@ class Git:
         if repo.is_dirty(untracked_files=True) or repo.index.diff(None):
             repo.index.commit(commit_message)
 
-    def push(self, repo_name: str) -> None:
+    def push(self, repo_name: str) -> PushInfoList:
         repo: Repo = self.get_repo(repo_name)
-        repo.remote(name="origin").push()
+        return repo.remote(name="origin").push()