fix: joj3-check-env group match

.gitignore

@@ -298,4 +298,5 @@ dmypy.json
 
 # End of https://www.toptal.com/developers/gitignore/api/vscode,python
 
-repos/
+repos/*
+!repos/.gitkeep

README.md

@@ -36,9 +36,9 @@ pytest -svv
 
 ## Commands & Features
 
-### `archive-all-repos`
+### `archive-repos`
 
-archive all repos in gitea organization
+archive repos in gitea organization according to regex (dry-run enabled by default)
 
 ### `unwatch-all-repos`
 
@@ -87,7 +87,7 @@ Example: `python3 -m joint_teapot create-personal-repos --suffix "-p1"` will cre
 
 ### `create-teams`
 
-create teams on gitea by canvas groups
+create teams on gitea by canvas groups. To integrate with webhooks, it's recommended to set suffix to `-gitea`.
 
 ### `create-webhooks-for-mm`
 

joint_teapot/app.py

@@ -1,7 +1,7 @@
 import json
 import os
 import re
-from datetime import datetime, timedelta, timezone
+from datetime import datetime
 from pathlib import Path
 from time import sleep
 from typing import TYPE_CHECKING, List
@@ -16,7 +16,7 @@ from joint_teapot.utils import joj3
 from joint_teapot.utils.logger import logger, set_logger
 
 if TYPE_CHECKING:
-    import focs_gitea
+    pass
 
 app = Typer(add_completion=False)
 
@@ -90,8 +90,16 @@ def create_issue_for_repos(
     use_regex: bool = Option(
         False, "--regex", help="repo_names takes list of regexes if set"
     ),
+    milesetone: str = Option("", "--milestone", help="milestone title"),
+    labels: List[str] = Option(
+        [],
+        "--label",
+        help="labels to add to the issue (use --label A --label B to add multiple)",
+    ),
 ) -> None:
-    tea.pot.create_issue_for_repos(repo_names, title, body, from_file, use_regex)
+    tea.pot.create_issue_for_repos(
+        repo_names, title, body, from_file, use_regex, milesetone, labels
+    )
 
 
 @app.command("create-comment", help="create a comment for an issue on gitea")
@@ -103,11 +111,17 @@ def create_comment(
     tea.pot.create_comment(repo_name, index, body)
 
 
-@app.command("create-milestones", help="create milestones on gitea")
-def create_milestone_for_repos(
-    repo_names: List[str], title: str, description: str, due_on: datetime
+@app.command(
+    "create-milestones",
+    help="create milestones on gitea",
+)
+def create_milestones(
+    title: str,
+    regex: str = Argument(".+"),
+    due_on: str = Argument("", help="milestone due-on date [%YYYY-%MM-%DD]"),
+    description: str = Argument(""),
 ) -> None:
-    tea.pot.create_milestone_for_repos(repo_names, title, description, due_on)
+    tea.pot.gitea.create_milestones(title, regex, due_on, description)
 
 
 @app.command("check-issues", help="check the existence of issue by title on gitea")
@@ -141,9 +155,11 @@ def close_all_issues() -> None:
     tea.pot.gitea.close_all_issues()
 
 
-@app.command("archive-all-repos", help="archive all repos in gitea organization")
-def archive_all_repos() -> None:
-    tea.pot.gitea.archive_all_repos()
+@app.command(
+    "archive-repos", help="archive repos in gitea organization according to regex"
+)
+def archive_repos(regex: str = Argument(".+"), dry_run: bool = Option(True)) -> None:
+    tea.pot.gitea.archive_repos(regex, dry_run)
 
 
 @app.command("unwatch-all-repos", help="unwatch all repos in gitea organization")
@@ -184,7 +200,7 @@ def upload_assignment_grades(assignments_dir: Path, assignment_name: str) -> Non
 @app.command(
     "create-group-channels-on-mm",
     help="create channels for student groups according to group information on"
-    " gitea",
+    " gitea; to integrate with webhooks, it's recommended to set suffix to '-gitea'",
 )
 def create_group_channels_on_mm(
     prefix: str = Option(""),
@@ -221,7 +237,8 @@ def create_personal_channels_on_mm(
     "and configure them so that updates on gitea will be pushed to the mm channel",
 )
 def create_webhooks_for_mm(
-    regex: str = Argument(""), git_suffix: bool = Option(False)
+    regex: str = Argument(""),
+    gitea_suffix: bool = Option(True, help="append gitea suffix to mm channel names"),
 ) -> None:
     repo_names = [
         group_name
@@ -229,7 +246,9 @@ def create_webhooks_for_mm(
         if re.match(regex, group_name)
     ]
     logger.info(f"{len(repo_names)} pair(s) of webhooks to be created: {repo_names}")
-    tea.pot.mattermost.create_webhooks_for_repos(repo_names, tea.pot.gitea, git_suffix)
+    tea.pot.mattermost.create_webhooks_for_repos(
+        repo_names, tea.pot.gitea, gitea_suffix
+    )
 
 
 @app.command(
@@ -304,57 +323,24 @@ def joj3_all_env(
         "sha": env.github_sha,
         "commitMsg": env.joj3_commit_msg,
     }
-    gitea_actions_url = (
-        f"https://{settings.gitea_domain_name}{settings.gitea_suffix}/"
-        + f"{settings.gitea_org_name}/{submitter_repo_name}/"
-        + f"actions/runs/{env.github_run_number}"
-    )
     submitter_repo_url = (
         f"https://{settings.gitea_domain_name}{settings.gitea_suffix}/"
         + f"{settings.gitea_org_name}/{submitter_repo_name}"
     )
+    gitea_actions_url = f"{submitter_repo_url}/actions/runs/{env.github_run_number}"
     gitea_issue_url = ""
     if not skip_result_issue:
-        title, comment = joj3.generate_title_and_comment(
-            env.joj3_output_path,
-            gitea_actions_url,
-            env.github_run_number,
-            env.joj3_conf_name,
-            env.github_actor,
-            env.github_sha,
-            submitter_in_issue_title,
-            env.joj3_run_id,
+        issue_number = tea.pot.joj3_post_issue(
+            env,
             max_total_score,
-        )
-        title_prefix = joj3.get_title_prefix(
-            env.joj3_conf_name, env.github_actor, submitter_in_issue_title
-        )
-        joj3_issue: focs_gitea.Issue
-        issue: focs_gitea.Issue
-        for issue in tea.pot.gitea.issue_api.issue_list_issues(
-            tea.pot.gitea.org_name, submitter_repo_name, state="open"
-        ):
-            if issue.title.startswith(title_prefix):
-                joj3_issue = issue
-                logger.info(f"found joj3 issue: #{joj3_issue.number}")
-                break
-        else:
-            joj3_issue = tea.pot.gitea.issue_api.issue_create_issue(
-                tea.pot.gitea.org_name,
+            gitea_actions_url,
+            submitter_in_issue_title,
             submitter_repo_name,
-                body={"title": title_prefix + "0", "body": ""},
         )
-            logger.info(f"created joj3 issue: #{joj3_issue.number}")
-        gitea_issue_url = joj3_issue.html_url
+        res["issue"] = issue_number
+        gitea_issue_url = f"{submitter_repo_url}/issues/{issue_number}"
         logger.info(f"gitea issue url: {gitea_issue_url}")
-        tea.pot.gitea.issue_api.issue_edit_issue(
-            tea.pot.gitea.org_name,
-            submitter_repo_name,
-            joj3_issue.number,
-            body={"title": title, "body": comment},
-        )
-        res["issue"] = joj3_issue.number
-    print(json.dumps(res))  # print result to stdout for joj3 log parser
+    echo(json.dumps(res))  # print result to stdout for joj3 log parser
     if skip_scoreboard and skip_failed_table:
         return
     lock_file_path = os.path.join(
@@ -463,6 +449,7 @@ def joj3_check_env(
     app.pretty_exceptions_enable = False
     set_settings(Settings(_env_file=env_path))
     set_logger(settings.stderr_log_level)
+    logger.info(f"debug log to file: {settings.log_file_path}")
     env = joj3.Env()
     if "" in (
         env.github_actor,
@@ -470,98 +457,10 @@ def joj3_check_env(
     ):
         logger.error("missing required env var")
         raise Exit(code=1)
-    submitter_repo_name = env.github_repository.split("/")[-1]
-    repo: Repo = tea.pot.git.get_repo(grading_repo_name)
-    now = datetime.now(timezone.utc)
-    items = group_config.split(",")
-    comment = ""
-    failed = False
-    pattern = re.compile(
-        r"joj3: update scoreboard for (?P<exercise_name>.+?) "
-        r"by @(?P<submitter>.+) in "
-        r"(?P<gitea_org_name>.+)/(?P<submitter_repo_name>.+)@(?P<commit_hash>.+)"
+    msg, failed = tea.pot.joj3_check_submission_count(
+        env, grading_repo_name, group_config, scoreboard_filename
     )
-    time_windows = []
-    valid_items = []
-    for item in items:
-        name, values = item.split("=")
-        max_count, time_period = map(int, values.split(":"))
-        if max_count < 0 or time_period < 0:
-            continue
-        since = now - timedelta(hours=time_period)
-        time_windows.append(since)
-        valid_items.append((name, max_count, time_period, since))
-    all_commits = []
-    if time_windows:
-        earliest_since = min(time_windows).strftime("%Y-%m-%dT%H:%M:%S")
-        commits = repo.iter_commits(paths=scoreboard_filename, since=earliest_since)
-        for commit in commits:
-            lines = commit.message.strip().splitlines()
-            if not lines:
-                continue
-            match = pattern.match(lines[0])
-            if not match:
-                continue
-            d = match.groupdict()
-            if (
-                env.joj3_conf_name != d["exercise_name"]
-                or env.github_actor != d["submitter"]
-                or submitter_repo_name != d["submitter_repo_name"]
-            ):
-                continue
-            groups_line = next((l for l in lines if l.startswith("groups: ")), None)
-            commit_groups = (
-                groups_line[len("groups: ") :].split(",") if groups_line else []
-            )
-            all_commits.append(
-                {
-                    "time": commit.committed_datetime,
-                    "groups": [g.strip() for g in commit_groups],
-                }
-            )
-    for name, max_count, time_period, since in valid_items:
-        submit_count = 0
-        time_limit = now - timedelta(hours=time_period)
-        for commit in all_commits:
-            if commit["time"] < time_limit:
-                continue
-            if name:
-                target_group = name.lower()
-                commit_groups_lower = [g.lower() for g in commit["groups"]]
-                if target_group not in commit_groups_lower:
-                    continue
-            submit_count += 1
-        logger.info(
-            f"submitter {env.github_actor} is submitting for the {submit_count + 1} time, "
-            f"{min(0, max_count - submit_count - 1)} time(s) remaining, "
-            f"group={name}, "
-            f"time period={time_period} hour(s), "
-            f"max count={max_count}, submit count={submit_count}"
-        )
-        use_group = False
-        if name:
-            comment += f"keyword `{name}` "
-            for group in env.joj3_groups or "":
-                if group.lower() == name.lower():
-                    use_group = True
-                    break
-        else:
-            use_group = True
-        comment += (
-            f"in last {time_period} hour(s): "
-            f"submit count {submit_count}, "
-            f"max count {max_count}"
-        )
-        if use_group and submit_count + 1 > max_count:
-            failed = True
-            comment += ", exceeded"
-        comment += "\n"
-    if failed:
-        title = "### Submission Count Check Failed:"
-    else:
-        title = "### Submission Count Check Passed:"
-    msg = f"{title}\n{comment}\n"
-    print(json.dumps({"msg": msg, "failed": failed}))  # print result to stdout for joj3
+    echo(json.dumps({"msg": msg, "failed": failed}))  # print result to stdout for joj3
     logger.info("joj3-check-env done")
 
 

joint_teapot/teapot.py

@@ -2,17 +2,22 @@ import functools
 import glob
 import os
 import re
-from datetime import datetime
-from typing import Any, Callable, Dict, List, Optional, TypeVar
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, TypeVar
 
 import mosspy
+from git import Repo
 
 from joint_teapot.config import settings
+from joint_teapot.utils import joj3
 from joint_teapot.utils.logger import logger
 from joint_teapot.utils.main import default_repo_name_convertor, first
 from joint_teapot.workers import Canvas, Git, Gitea, Mattermost
 from joint_teapot.workers.joj import JOJ
 
+if TYPE_CHECKING:
+    import focs_gitea
+
 _T = TypeVar("_T")
 
 
@@ -106,7 +111,7 @@ class Teapot:
                 return None
             team_name, number_str = name.split(" ")
             number = int(number_str)
-            return f"{team_name}-{number:02}"
+            return f"{team_name}{number:02}"
 
         return self.gitea.create_teams_and_repos_by_canvas_groups(
             self.canvas.students, self.canvas.groups, convertor, convertor
@@ -141,6 +146,8 @@ class Teapot:
         body: str,
         from_file: bool = False,
         use_regex: bool = False,
+        milestone: str = "",
+        labels: List[str] = [],
     ) -> None:
         if from_file:
             try:
@@ -168,7 +175,7 @@ class Teapot:
             affected_repos = repo_names
 
         for repo_name in affected_repos:
-            self.gitea.create_issue(repo_name, title, content)
+            self.gitea.create_issue(repo_name, title, content, True, milestone, labels)
 
     def create_comment(
         self,
@@ -178,12 +185,6 @@ class Teapot:
     ) -> None:
         self.gitea.create_comment(repo_name, index, body)
 
-    def create_milestone_for_repos(
-        self, repo_names: List[str], title: str, description: str, due_on: datetime
-    ) -> None:
-        for repo_name in repo_names:
-            self.gitea.create_milestone(repo_name, title, description, due_on)
-
     def check_exist_issue_by_title(
         self, repo_names: List[str], title: str
     ) -> List[str]:
@@ -229,6 +230,152 @@ class Teapot:
             self.canvas.students, invite_teaching_teams
         )
 
+    def joj3_post_issue(
+        self,
+        env: joj3.Env,
+        max_total_score: int,
+        gitea_actions_url: str,
+        submitter_in_issue_title: bool,
+        submitter_repo_name: str,
+    ) -> int:
+        title, comment = joj3.generate_title_and_comment(
+            env.joj3_output_path,
+            gitea_actions_url,
+            env.github_run_number,
+            env.joj3_conf_name,
+            env.github_actor,
+            env.github_sha,
+            submitter_in_issue_title,
+            env.joj3_run_id,
+            max_total_score,
+        )
+        title_prefix = joj3.get_title_prefix(
+            env.joj3_conf_name, env.github_actor, submitter_in_issue_title
+        )
+        joj3_issue: focs_gitea.Issue
+        issue: focs_gitea.Issue
+        new_issue = False
+        for issue in self.gitea.issue_api.issue_list_issues(
+            self.gitea.org_name, submitter_repo_name, state="open"
+        ):
+            if issue.title.startswith(title_prefix):
+                joj3_issue = issue
+                logger.info(f"found joj3 issue: #{joj3_issue.number}")
+                break
+        else:
+            new_issue = True
+            joj3_issue = self.gitea.issue_api.issue_create_issue(
+                self.gitea.org_name,
+                submitter_repo_name,
+                body={"title": title, "body": comment},
+            )
+            logger.info(f"created joj3 issue: #{joj3_issue.number}")
+        gitea_issue_url = joj3_issue.html_url
+        logger.info(f"gitea issue url: {gitea_issue_url}")
+        if not new_issue:
+            self.gitea.issue_api.issue_edit_issue(
+                self.gitea.org_name,
+                submitter_repo_name,
+                joj3_issue.number,
+                body={"title": title, "body": comment},
+            )
+        return joj3_issue.number
+
+    def joj3_check_submission_count(
+        self,
+        env: joj3.Env,
+        grading_repo_name: str,
+        group_config: str,
+        scoreboard_filename: str,
+    ) -> Tuple[str, bool]:
+        submitter_repo_name = env.github_repository.split("/")[-1]
+        repo: Repo = self.git.get_repo(grading_repo_name)
+        now = datetime.now(timezone.utc)
+        items = group_config.split(",")
+        comment = ""
+        failed = False
+        pattern = re.compile(
+            r"joj3: update scoreboard for (?P<exercise_name>.+?) "
+            r"by @(?P<submitter>.+) in "
+            r"(?P<gitea_org_name>.+)/(?P<submitter_repo_name>.+)@(?P<commit_hash>.+)"
+        )
+        time_windows = []
+        valid_items = []
+        for item in items:
+            name, values = item.split("=")
+            max_count, time_period = map(int, values.split(":"))
+            if max_count < 0 or time_period < 0:
+                continue
+            since = now - timedelta(hours=time_period)
+            time_windows.append(since)
+            valid_items.append((name, max_count, time_period, since))
+        all_commits = []
+        if time_windows:
+            earliest_since = min(time_windows).strftime("%Y-%m-%dT%H:%M:%S")
+            commits = repo.iter_commits(paths=scoreboard_filename, since=earliest_since)
+            for commit in commits:
+                lines = commit.message.strip().splitlines()
+                if not lines:
+                    continue
+                match = pattern.match(lines[0])
+                if not match:
+                    continue
+                d = match.groupdict()
+                if (
+                    env.joj3_conf_name != d["exercise_name"]
+                    or env.github_actor != d["submitter"]
+                    or submitter_repo_name != d["submitter_repo_name"]
+                ):
+                    continue
+                groups_line = next((l for l in lines if l.startswith("groups: ")), None)
+                commit_groups = (
+                    groups_line[len("groups: ") :].split(",") if groups_line else []
+                )
+                all_commits.append(
+                    {
+                        "time": commit.committed_datetime,
+                        "groups": [g.strip() for g in commit_groups],
+                    }
+                )
+        for name, max_count, time_period, since in valid_items:
+            submit_count = 0
+            time_limit = now - timedelta(hours=time_period)
+            for commit in all_commits:
+                if commit["time"] < time_limit:
+                    continue
+                if name:
+                    target_group = name.lower()
+                    commit_groups_lower = [g.lower() for g in commit["groups"]]
+                    if target_group not in commit_groups_lower:
+                        continue
+                submit_count += 1
+            logger.info(
+                f"submitter {env.github_actor} is submitting for the {submit_count + 1} time, "
+                f"{min(0, max_count - submit_count - 1)} time(s) remaining, "
+                f"group={name}, "
+                f"time period={time_period} hour(s), "
+                f"max count={max_count}, submit count={submit_count}"
+            )
+            use_group = True
+            if name:
+                comment += f"keyword `{name}` "
+                use_group = name.lower() in env.joj3_groups.lower()
+            comment += (
+                f"in last {time_period} hour(s): "
+                f"submit count {submit_count}, "
+                f"max count {max_count}"
+            )
+            if use_group and submit_count + 1 > max_count:
+                failed = True
+                comment += ", exceeded"
+            comment += "\n"
+        if failed:
+            title = "### Submission Count Check Failed:"
+        else:
+            title = "### Submission Count Check Passed:"
+        msg = f"{title}\n{comment}\n"
+        return msg, failed
+
 
 if __name__ == "__main__":
     teapot = Teapot()

joint_teapot/workers/gitea.py

@@ -1,5 +1,4 @@
 import re
-from datetime import datetime
 from enum import Enum
 from functools import lru_cache
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar
@@ -345,6 +344,8 @@ class Gitea:
         title: str,
         body: str,
         assign_every_collaborators: bool = True,
+        milestone: str = "",
+        labels: list[str] = [],
     ) -> None:
         assignees = []
         if assign_every_collaborators:
@@ -356,10 +357,33 @@ class Gitea:
                     repo_name,
                 )
             ]
+        milestone_id = None
+        if milestone:
+            milestone_list = self.issue_api.issue_get_milestones_list(
+                self.org_name, repo_name
+            )
+            if milestone not in [m.title for m in milestone_list]:
+                logger.warning(f"Milestone {milestone} does not exist in {repo_name}")
+            else:
+                milestone_id = first(
+                    [m.id for m in milestone_list if m.title == milestone]
+                )
+        labels_id = []
+        if labels:
+            labels_list = self.issue_api.issue_list_labels(self.org_name, repo_name)
+            labels_id = [l.id for l in labels_list if l.name in labels]
+            if not labels_id:
+                logger.warning(f"no label matches {labels}")
         self.issue_api.issue_create_issue(
             self.org_name,
             repo_name,
-            body={"title": title, "body": body, "assignees": assignees},
+            body={
+                "title": title,
+                "body": body,
+                "assignees": assignees,
+                "milestone": milestone_id,
+                "labels": labels_id,
+            },
         )
         logger.info(f'Created issue "{title}" in {repo_name}')
 
@@ -382,15 +406,22 @@ class Gitea:
         repo_name: str,
         title: str,
         description: str,
-        due_on: datetime,
+        due_on: str,
     ) -> None:
+        if due_on == "":
+            self.issue_api.issue_create_milestone(
+                self.org_name,
+                repo_name,
+                body={"title": title, "description": description},
+            )
+            return
         self.issue_api.issue_create_milestone(
             self.org_name,
             repo_name,
             body={
                 "title": title,
                 "description": description,
-                "due_on": due_on.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+                "due_on": due_on + "T23:59:59.999+08:00",
             },
         )
 
@@ -413,10 +444,16 @@ class Gitea:
                         self.org_name, repo_name, issue.number, body={"state": "closed"}
                     )
 
-    def archive_all_repos(self) -> None:
-        for repo in list_all(self.organization_api.org_list_repos, self.org_name):
+    def archive_repos(self, regex: str = ".+", dry_run: bool = True) -> None:
+        if dry_run:
+            logger.info("Dry run enabled. No changes will be made to the repositories.")
+        logger.info(f"Archiving repos with name matching {regex}")
+        for repo_name in self.get_all_repo_names():
+            if re.match(regex, repo_name):
+                logger.info(f"Archived {repo_name}")
+                if not dry_run:
                     self.repository_api.repo_edit(
-                self.org_name, repo.name, body={"archived": True}
+                        self.org_name, repo_name, body={"archived": True}
                     )
 
     def unwatch_all_repos(self) -> None:
@@ -463,6 +500,21 @@ class Gitea:
             )
             logger.info(f"Unsubscribed from {sub.name}")
 
+    def create_milestones(
+        self, milestone: str, regex: str, due_date: str, description: str
+    ) -> None:
+        for repo_name in self.get_all_repo_names():
+            if not re.match(regex, repo_name):
+                continue
+            milestone_list = self.issue_api.issue_get_milestones_list(
+                self.org_name, repo_name
+            )
+            if milestone in [m.title for m in milestone_list]:
+                logger.warning(f"Milestone {milestone} already exists in {repo_name}")
+                continue
+            self.create_milestone(repo_name, milestone, description, due_date)
+            logger.info(f"Created milestone {milestone} in {repo_name}")
+
 
 if __name__ == "__main__":
     gitea = Gitea()

joint_teapot/workers/mattermost.py

@@ -36,7 +36,7 @@ class Mattermost:
         except Exception:
             logger.error("Cannot login to Mattermost")
             return
-        if "admin" not in operator["roles"]:
+        if "admin" not in operator["roles"] and "system_user" not in operator["roles"]:
             logger.error("Please make sure you have enough permission")
         try:
             self.team = self.endpoint.teams.get_team_by_name(team_name)
@@ -167,12 +167,14 @@ class Mattermost:
                 logger.info(f"Added member {member} to channel {channel_name}")
 
     def create_webhooks_for_repos(
-        self, repos: List[str], gitea: Gitea, git_suffix: bool
+        self, repos: List[str], gitea: Gitea, gitea_suffix: bool
     ) -> None:
         # one group corresponds to one repo so these concepts can be used interchangeably
         for repo in repos:
-            logger.info(f"Creating webhooks for repo {gitea.org_name}/{repo}")
-            channel_name = f"{repo}-git" if git_suffix else repo
+            channel_name = f"{repo}-gitea" if gitea_suffix else repo
+            logger.info(
+                f"Creating webhooks for repo {gitea.org_name}/{repo} and channel {channel_name}"
+            )
             try:
                 mm_channel = self.endpoint.channels.get_channel_by_name(
                     self.team["id"], channel_name
@@ -203,9 +205,16 @@ class Mattermost:
                         events=[
                             "issues_only",
                             "issue_comment",
+                            "issue_assign",
                             "pull_request_only",
                             "pull_request_comment",
                             "pull_request_review",
+                            "pull_request_review_request",
+                            "push",
+                            "create",
+                            "delete",
+                            "release",
+                            "wiki",
                         ],
                         config={
                             "url": f"https://{self.url}{self.url_suffix}/hooks/{mm_webhook['id']}",