chore: better logs

This reverts commit 3dc6667716.

.pre-commit-config.yaml

@@ -1,41 +1,41 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v6.0.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
       - id: requirements-txt-fixer
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.4.1"
+    rev: "v1.18.2"
     hooks:
       - id: mypy
         additional_dependencies:
           - pydantic
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.9.0
+    rev: v3.20.0
     hooks:
       - id: pyupgrade
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.4.0
+    rev: v2.5.0
     hooks:
       - id: pycln
         args: [-a]
   - repo: https://github.com/PyCQA/bandit
-    rev: '1.7.5'
+    rev: '1.8.6'
     hooks:
     - id: bandit
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 6.0.1
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 25.9.0
     hooks:
       - id: black
   - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.1
+    rev: v1.5.5
     hooks:
       - id: remove-crlf
       - id: remove-tabs

joint_teapot/app.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, List, Optional
 
-# from filelock import FileLock
+from filelock import FileLock
 from git import Repo
 from typer import Argument, Exit, Option, Typer, echo
 
@@ -34,9 +34,17 @@ class Tea:
 tea = Tea()  # lazy loader
 
 
-@app.command("export-students", help="export students from canvas to csv file")
+@app.command("export-users", help="export users from canvas to csv file")
-def export_students_to_csv(output_file: Path) -> None:
+def export_users_to_csv(output_file: Path = Argument("students.csv")) -> None:
-    tea.pot.canvas.export_students_to_csv(output_file)
+    tea.pot.canvas.export_users_to_csv(output_file)
+
+
+@app.command(
+    "export-wrong-email-users",
+    help="export users with wrong email from canvas in stdout",
+)
+def export_wrong_email_users() -> None:
+    tea.pot.canvas.export_wrong_email_users()
 
 
 @app.command(
@@ -295,6 +303,10 @@ def joj3_all_env(
         False,
         help="skip creating failed table on gitea",
     ),
+    scoreboard_column_by_ref: bool = Option(
+        False,
+        help="use git ref as scoreboard column name",
+    ),
     submitter_in_issue_title: bool = Option(
         True,
         help="whether to include submitter in issue title",
@@ -307,6 +319,10 @@ def joj3_all_env(
         "#795548",
         help="label color for the issue created by this command",
     ),
+    issue_label_exclusive: bool = Option(
+        False,
+        help="label set as exclusive for the issue created by this command",
+    ),
     end_time: Optional[datetime] = Option(None),
     penalty_config: str = Option(
         "",
@@ -333,6 +349,7 @@ def joj3_all_env(
     submitter_repo_name = env.github_repository.split("/")[-1]
     penalty_factor = joj3.get_penalty_factor(end_time, penalty_config)
     total_score = joj3.get_total_score(env.joj3_output_path)
+    total_score = round(total_score - abs(total_score) * (1 - penalty_factor))
     res = {
         "totalScore": total_score,
         "cappedTotalScore": (
@@ -360,6 +377,7 @@ def joj3_all_env(
             submitter_repo_name,
             issue_label_name,
             issue_label_color,
+            issue_label_exclusive,
             penalty_factor,
         )
         res["issue"] = issue_number
@@ -371,13 +389,12 @@ def joj3_all_env(
     lock_file_path = os.path.join(
         settings.repos_dir, grading_repo_name, settings.joj3_lock_file_path
     )
-    logger.info(
+    logger.debug(
         f"try to acquire lock, file path: {lock_file_path}, "
         + f"timeout: {settings.joj3_lock_file_timeout}"
     )
-    if True:  # disable the file lock temporarily
+    with FileLock(lock_file_path, timeout=settings.joj3_lock_file_timeout).acquire():
-        # with FileLock(lock_file_path, timeout=settings.joj3_lock_file_timeout).acquire():
+        logger.debug("file lock acquired")
-        logger.info("file lock acquired")
         retry_interval = 1
         git_push_ok = False
         while not git_push_ok:
@@ -398,12 +415,18 @@ def joj3_all_env(
                 raise Exit(code=1)
             repo.git.reset("--hard", "origin/grading")
             if not skip_scoreboard:
+                exercise_name = env.joj3_conf_name
+                if scoreboard_column_by_ref:
+                    exercise_name = env.github_ref
                 joj3.generate_scoreboard(
                     env.joj3_output_path,
                     env.github_actor,
                     os.path.join(repo_path, scoreboard_filename),
-                    env.joj3_conf_name,
+                    exercise_name,
+                    submitter_repo_name,
+                    total_score,
                 )
+                failed_stage = joj3.get_failed_stage_from_file(env.joj3_output_path)
                 tea.pot.git.add_commit(
                     grading_repo_name,
                     [scoreboard_filename],
@@ -413,6 +436,7 @@ def joj3_all_env(
                         f"gitea actions link: {gitea_actions_url}\n"
                         f"gitea issue link: {gitea_issue_url}\n"
                         f"groups: {env.joj3_groups}\n"
+                        f"failed stage: {failed_stage}\n"
                     ),
                 )
             if not skip_failed_table:
@@ -481,6 +505,9 @@ def joj3_check_env(
             "Example: --penalty-config 24=0.75,48=0.5"
         ),
     ),
+    ignore_submitter: bool = Option(
+        False, help="ignore submitter when checking submission count"
+    ),
 ) -> None:
     app.pretty_exceptions_enable = False
     set_settings(Settings(_env_file=env_path))
@@ -499,7 +526,7 @@ def joj3_check_env(
         penalty_config,
     )
     count_msg, count_failed = tea.pot.joj3_check_submission_count(
-        env, grading_repo_name, group_config, scoreboard_filename
+        env, grading_repo_name, group_config, scoreboard_filename, ignore_submitter
     )
     echo(
         json.dumps(
@@ -512,6 +539,16 @@ def joj3_check_env(
     logger.info("joj3-check-env done")
 
 
+@app.command("joj3-check-gitea-token")
+def joj3_check_gitea_token(
+    env_path: str = Argument("", help="path to .env file")
+) -> None:
+    app.pretty_exceptions_enable = False
+    set_settings(Settings(_env_file=env_path))
+    set_logger(settings.stderr_log_level)
+    tea.pot.gitea.organization_api.org_list_repos(settings.gitea_org_name)
+
+
 if __name__ == "__main__":
     try:
         app()

joint_teapot/config.py

@@ -40,7 +40,7 @@ class Settings(BaseSettings):
     joj_sid: str = ""
 
     # joj3
-    joj3_lock_file_path: str = ".git/teapot.lock"
+    joj3_lock_file_path: str = ".git/teapot-joj3-all-env.lock"
     joj3_lock_file_timeout: int = 30
 
     # moss

joint_teapot/teapot.py

@@ -240,6 +240,7 @@ class Teapot:
         submitter_repo_name: str,
         issue_label_name: str,
         issue_label_color: str,
+        issue_label_exclusive: bool,
         penalty_factor: float,
     ) -> int:
         title, comment = joj3.generate_title_and_comment(
@@ -265,7 +266,6 @@ class Teapot:
         ):
             if issue.title.startswith(title_prefix):
                 joj3_issue = issue
-                logger.info(f"found joj3 issue: #{joj3_issue.number}")
                 break
         else:
             new_issue = True
@@ -280,7 +280,11 @@ class Teapot:
                 label = self.gitea.issue_api.issue_create_label(
                     self.gitea.org_name,
                     submitter_repo_name,
-                    body={"name": issue_label_name, "color": issue_label_color},
+                    body={
+                        "name": issue_label_name,
+                        "color": issue_label_color,
+                        "exclusive": issue_label_exclusive,
+                    },
                 )
                 label_id = label.id
             joj3_issue = self.gitea.issue_api.issue_create_issue(
@@ -288,9 +292,7 @@ class Teapot:
                 submitter_repo_name,
                 body={"title": title, "body": comment, "labels": [label_id]},
             )
-            logger.info(f"created joj3 issue: #{joj3_issue.number}")
         gitea_issue_url = joj3_issue.html_url
-        logger.info(f"gitea issue url: {gitea_issue_url}")
         if not new_issue:
             self.gitea.issue_api.issue_edit_issue(
                 self.gitea.org_name,
@@ -325,7 +327,7 @@ class Teapot:
                     f"[{end_time + timedelta(seconds=1)}, {penalty_end_time}].\n",
                     True,
                 )
-            else:
+            elif now > end_time:
                 return (
                     "### Submission Time Check Passed\n"
                     f"Current time {now} is not in the valid range "
@@ -354,7 +356,9 @@ class Teapot:
         grading_repo_name: str,
         group_config: str,
         scoreboard_filename: str,
+        ignore_submitter: bool,
     ) -> Tuple[str, bool]:
+        submitter = env.github_actor
         submitter_repo_name = env.github_repository.split("/")[-1]
         repo: Repo = self.git.get_repo(grading_repo_name)
         now = datetime.now(timezone.utc)
@@ -381,11 +385,13 @@ class Teapot:
             time_windows.append(since)
             valid_items.append((name, max_count, time_period, since))
         logger.info(f"valid items: {valid_items}, time windows: {time_windows}")
-        all_commits = []
+        matched_commits = []
+        all_commits_length = 0
         if time_windows:
             earliest_since = min(time_windows).strftime("%Y-%m-%dT%H:%M:%S")
             commits = repo.iter_commits(paths=scoreboard_filename, since=earliest_since)
             for commit in commits:
+                all_commits_length += 1
                 lines = commit.message.strip().splitlines()
                 if not lines:
                     continue
@@ -395,25 +401,28 @@ class Teapot:
                 d = match.groupdict()
                 if (
                     env.joj3_conf_name != d["exercise_name"]
-                    or env.github_actor != d["submitter"]
                     or submitter_repo_name != d["submitter_repo_name"]
                 ):
                     continue
+                if not ignore_submitter and submitter != d["submitter"]:
+                    continue
                 groups_line = next((l for l in lines if l.startswith("groups: ")), None)
                 commit_groups = (
                     groups_line[len("groups: ") :].split(",") if groups_line else []
                 )
-                all_commits.append(
+                matched_commits.append(
                     {
                         "time": commit.committed_datetime,
                         "groups": [g.strip() for g in commit_groups],
                     }
                 )
-        logger.info(f"all commits length: {len(all_commits)}")
+        logger.info(
+            f"matched commits length: {len(matched_commits)}, all commits length: {all_commits_length}"
+        )
         for name, max_count, time_period, since in valid_items:
             submit_count = 0
             time_limit = now - timedelta(hours=time_period)
-            for commit in all_commits:
+            for commit in matched_commits:
                 if commit["time"] < time_limit:
                     continue
                 if name:
@@ -423,7 +432,7 @@ class Teapot:
                         continue
                 submit_count += 1
             logger.info(
-                f"submitter {env.github_actor} is submitting for the {submit_count + 1} time, "
+                f"submitter {submitter} is submitting for the {submit_count + 1} time, "
                 f"{min(0, max_count - submit_count - 1)} time(s) remaining, "
                 f"group={name}, "
                 f"time period={time_period} hour(s), "

joint_teapot/utils/joj3.py

@@ -7,6 +7,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from pydantic_settings import BaseSettings
 
+from joint_teapot.config import settings
 from joint_teapot.utils.logger import logger
 
 
@@ -40,6 +41,8 @@ def generate_scoreboard(
     submitter: str,
     scoreboard_file_path: str,
     exercise_name: str,
+    submitter_repo_name: str,
+    exercise_total_score: int,
 ) -> None:
     if not scoreboard_file_path.endswith(".csv"):
         logger.error(
@@ -48,18 +51,25 @@ def generate_scoreboard(
         return
     os.makedirs(os.path.dirname(scoreboard_file_path), exist_ok=True)
     # Load the csv file if it already exists
+    fixed_headers = ["", "repo_name", "last_edit", "total"]
+    fixed_defaults = [submitter, submitter_repo_name, "", "0"]
     if os.path.exists(scoreboard_file_path):
         with open(scoreboard_file_path, newline="") as file:
             reader = csv.reader(file)
             rows = list(reader)
         columns = rows[0]
         data = rows[1:]
+
+        def migrate_scoreboard() -> None:
+            if "repo_name" in columns:
+                return
+            columns.insert(1, "repo_name")
+            for row in data:
+                row.insert(1, "")
+
+        migrate_scoreboard()
     else:
-        columns = [
+        columns = fixed_headers
-            "",
-            "last_edit",
-            "total",
-        ]
         data = []
 
     submitter_found = False
@@ -69,8 +79,7 @@ def generate_scoreboard(
             submitter_found = True
             break
     if not submitter_found:
-        fixed_columns = [submitter, "", "0"]
+        submitter_row = fixed_defaults + [""] * (len(columns) - len(fixed_headers))
-        submitter_row = fixed_columns + [""] * (len(columns) - len(fixed_columns))
         data.append(submitter_row)
 
     # Update data
@@ -85,33 +94,33 @@ def generate_scoreboard(
             exercise_name = comment.split("-")[0]
     # Find if exercise in table:
     if exercise_name not in columns:
-        column_tail = columns[3:]
+        column_tail = columns[len(fixed_headers) :]
         bisect.insort(column_tail, exercise_name)
-        columns[3:] = column_tail
+        columns[len(fixed_headers) :] = column_tail
         index = columns.index(exercise_name)
         for row in data:
             row.insert(index, "")
 
-    exercise_total_score = 0
-    for stage in stages:
-        for result in stage["results"]:
-            exercise_total_score += result["score"]
-    exercise_total_score = exercise_total_score
     submitter_row[columns.index(exercise_name)] = str(exercise_total_score)
 
     total = 0
     for col in columns:
-        if col in ["", "total", "last_edit"]:
+        if col in fixed_headers:
             continue
         idx = columns.index(col)
         if (submitter_row[idx] is not None) and (submitter_row[idx] != ""):
+            try:
                 total += int(submitter_row[idx])
+            except ValueError:
+                pass
 
     submitter_row[columns.index("total")] = str(total)
 
     now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     submitter_row[columns.index("last_edit")] = now
 
+    submitter_row[columns.index("repo_name")] = submitter_repo_name
+
     # Sort data by total, from low to high
     data.sort(key=lambda x: int(x[columns.index("total")]))
 
@@ -134,6 +143,18 @@ def get_failed_table_from_file(table_file_path: str) -> List[List[str]]:
     return data
 
 
+def get_failed_stage_from_file(score_file_path: str) -> str:
+    with open(score_file_path) as json_file:
+        stages: List[Dict[str, Any]] = json.load(json_file)
+
+    failed_stage = ""
+    for stage in stages:
+        if stage["force_quit"] == True:
+            failed_stage = stage["name"]
+            break
+    return failed_stage
+
+
 def update_failed_table_from_score_file(
     data: List[List[str]],
     score_file_path: str,
@@ -141,31 +162,23 @@ def update_failed_table_from_score_file(
     repo_link: str,
     action_link: str,
 ) -> None:
-    # get info from score file
+    failed_stage = get_failed_stage_from_file(score_file_path)
-    with open(score_file_path) as json_file:
-        stages: List[Dict[str, Any]] = json.load(json_file)
-
-    failed_name = ""
-    for stage in stages:
-        if stage["force_quit"] == True:
-            failed_name = stage["name"]
-            break
 
     # append to failed table
     now = datetime.now().strftime("%Y-%m-%d %H:%M")
     repo = f"[{repo_name}]({repo_link})"
-    failure = f"[{failed_name}]({action_link})"
+    failure = f"[{failed_stage}]({action_link})"
     row_found = False
     for i, row in enumerate(data[:]):
         if row[1] == repo:
             row_found = True
-            if failed_name == "":
+            if failed_stage == "":
                 data.remove(row)
             else:
                 data[i][0] = now
                 data[i][2] = failure
             break
-    if not row_found and failed_name != "":
+    if not row_found and failed_stage != "":
         data.append([now, repo, failure])
 
 
@@ -231,14 +244,12 @@ def generate_title_and_comment(
         f"Generated at {now} from [Gitea Actions #{run_number}]({action_link}), "
         f"commit {commit_hash}, "
         f"triggered by @{submitter}, "
-        f"run ID `{run_id}`.\n"
+        f"run ID [`{run_id}`](https://focs.ji.sjtu.edu.cn/joj-mon/d/{settings.gitea_org_name}?var-Filters=RunID%7C%3D%7C{run_id}).\n"
         "Powered by [JOJ3](https://github.com/joint-online-judge/JOJ3) and "
         "[Joint-Teapot](https://github.com/BoYanZh/Joint-Teapot) with ❤️.\n"
     )
     if penalty_factor != 1.0:
-        comment += (
+        comment += f"## ⚠️Total Score Penalty Warning⚠️\n**The total score is multiplied by {penalty_factor}.**\n"
-            f"## Total Score Penalty Warning\nThe total score is multiplied by 0.75.\n"
-        )
     for stage in stages:
         if all(
             result["score"] == 0 and result["comment"].strip() == ""
@@ -260,7 +271,7 @@ def generate_title_and_comment(
             total_score += result["score"]
         comment += "\n"
     if penalty_factor != 1.0:
-        total_score = round(total_score * penalty_factor)
+        total_score = round(total_score - abs(total_score) * (1 - penalty_factor))
     title = get_title_prefix(exercise_name, submitter, submitter_in_title)
     if max_total_score >= 0:
         title += f"{total_score} / {max_total_score}"
@@ -307,8 +318,10 @@ def get_penalty_factor(
 ) -> float:
     if not end_time or not penalty_config:
         return 1.0
-    penalties = parse_penalty_config(penalty_config)
     now = datetime.now()
+    if now < end_time:
+        return 1.0
+    penalties = parse_penalty_config(penalty_config)
     res = 0.0
     for hour, factor in penalties[::-1]:
         if now < end_time + timedelta(hours=hour):

joint_teapot/utils/logger.py

@@ -38,10 +38,7 @@ def set_logger(
 ) -> None:
     logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)
     logger.remove()
-    logger.add(
+    logger.add(stderr, level=stderr_log_level, colorize=stderr.isatty())
-        stderr,
-        level=stderr_log_level,
-    )
     logger.add(settings.log_file_path, level="DEBUG")
 
 

joint_teapot/workers/canvas.py

@@ -35,7 +35,7 @@ class Canvas:
         # types = ["student", "observer"]
         types = ["student"]
 
-        def patch_student(student: User) -> User:
+        def patch_user(student: User) -> User:
             student.name = (
                 re.sub(r"[^\x00-\x7F]+", "", student.name).strip().title()
             )  # We only care english name
@@ -44,7 +44,7 @@ class Canvas:
             return student
 
         self.students = [
-            patch_student(student)
+            patch_user(student)
             for student in self.course.get_users(enrollment_type=types)
         ]
         for attr in ["login_id", "name"]:
@@ -52,6 +52,7 @@ class Canvas:
                 raise Exception(
                     f"Unable to gather students' {attr}, please contact the Canvas site admin"
                 )
+        self.users = [patch_user(student) for student in self.course.get_users()]
         logger.debug("Canvas students loaded")
         self.assignments = self.course.get_assignments()
         logger.debug("Canvas assignments loaded")
@@ -60,12 +61,26 @@ class Canvas:
         self.grade_filename = grade_filename
         logger.debug("Canvas initialized")
 
-    def export_students_to_csv(self, filename: Path) -> None:
+    def export_wrong_email_users(self) -> None:
+        SAMPLE_EMAIL_BODY = """Dear Student,
+
+We have noticed that you have changed your email address on Canvas. While this can clearly cause privacy issues, this also prevents you from joining Gitea which will be intensively used in this course. Please revert back to your SJTU email address (`jaccount@sjtu.edu.cn`) as soon as possible. Note that if your email address is still incorrect in 24 hours, we will have to apply penalties as this is slowing down the whole course progress.
+
+Best regards,
+Teaching Team"""
+        emails = [
+            user.email for user in self.users if not user.email.endswith("@sjtu.edu.cn")
+        ]
+        print(f"To: {','.join(emails)}")
+        print(f"Subject: [{settings.gitea_org_name}] Important: wrong Canvas email")
+        print(f"Body:\n{SAMPLE_EMAIL_BODY}")
+
+    def export_users_to_csv(self, filename: Path) -> None:
         with open(filename, mode="w", newline="") as file:
             writer = csv.writer(file)
-            for student in self.students:
+            for user in self.users:
-                writer.writerow([student.name, student.sis_id, student.login_id])
+                writer.writerow([user.name, user.sis_id, user.login_id])
-        logger.info(f"Students exported to {filename}")
+        logger.info(f"Users exported to {filename}")
 
     def prepare_assignment_dir(
         self, dir_or_zip_file: str, create_grade_file: bool = True

joint_teapot/workers/git.py

@@ -90,25 +90,23 @@ class Git:
         retry_interval = 2
         while retry_interval and auto_retry:
             try:
-                current_branch = ""
-                if repo.head.is_detached:
-                    current_branch = repo.head.commit.hexsha
-                else:
-                    current_branch = repo.active_branch.name
                 if clean_git_lock:
-                    lock_files = [
+                    locks_removed_count = 0
-                        "index.lock",
+                    for root, _, files in os.walk(os.path.join(repo_dir, ".git")):
-                        "HEAD.lock",
+                        for filename in files:
-                        "fetch-pack.lock",
+                            if filename.endswith(".lock"):
-                        "logs/HEAD.lock",
+                                lock_file_path = os.path.join(root, filename)
-                        "packed-refs.lock",
+                                if (
-                        "config.lock",
+                                    os.path.relpath(lock_file_path, repo_dir)
-                        f"refs/remotes/origin/{current_branch}.lock",
+                                    == settings.joj3_lock_file_path
-                    ]
+                                ):
-                    for lock_file in lock_files:
+                                    continue
-                        lock_path = os.path.join(repo_dir, ".git", lock_file)
+                                try:
-                        if os.path.exists(lock_path):
+                                    os.remove(lock_file_path)
-                            os.remove(lock_path)
+                                    locks_removed_count += 1
+                                except OSError as e:
+                                    logger.warning(f"error removing lock file: {e}")
+                    logger.info(f"removed {locks_removed_count} lock files")
                 repo.git.fetch("--tags", "--all", "-f")
                 repo.git.reset("--hard", reset_target)
                 repo.git.clean("-d", "-f", "-x")
@@ -145,9 +143,7 @@ class Git:
             try:
                 repo.index.add(file)
             except OSError:
-                logger.warning(
+                logger.warning(f'file path "{file}" does not exist, skipped')
-                    f'File path "{file}" does not exist. Skipping this file.'
-                )
                 continue
         if repo.is_dirty(untracked_files=True) or repo.index.diff(None):
             repo.index.commit(commit_message)

joint_teapot/workers/gitea.py

@@ -494,6 +494,7 @@ class Gitea:
             self.repository_api.user_current_delete_subscription(
                 self.org_name, repo.name
             )
+            logger.info(f"Unwatched {repo.name}")
 
     def get_all_teams(self) -> Dict[str, List[str]]:
         res: Dict[str, List[str]] = {}