From acbcda95650b7dd506d21b8a70a58e04ec1a7752 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sun, 9 Feb 2025 13:26:19 -0500 Subject: [PATCH 01/13] ci: only use runs-on --- .gitea/workflows/build.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index d4d0dcf..f7d281d 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -4,8 +4,7 @@ on: - pull_request jobs: build: - container: - image: focs.ji.sjtu.edu.cn:5000/gitea/runner-images:ubuntu-latest + runs-on: ubuntu-latest steps: - name: Check out repository code uses: https://gitea.com/BoYanZh/checkout@focs From 2bf1a225b68f1a57083c95d75f24efe3c6e69240 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Mon, 24 Feb 2025 20:53:09 -0500 Subject: [PATCH 02/13] docs(README): list pdm run --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7590674..e3b98db 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,4 @@ 3. Change dir to the repo, `cd JOJ3-config-generator` 4. Install deps by `pdm install && pdm run pre-commit install` 5. Run the cli app by `pdm run app --help` +6. Check other commands or scripts with `pdm run --list` From bddb67decfeaf4b1c3a952883c748d984af880bc Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Mon, 24 Feb 2025 22:36:08 -0500 Subject: [PATCH 03/13] test: allow non-exist toml file input --- tests/convert/utils.py | 26 ++++++++++++-------------- tests/convert_joj1/utils.py | 18 +++++++----------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/tests/convert/utils.py b/tests/convert/utils.py index 5875138..046f715 100644 --- a/tests/convert/utils.py +++ b/tests/convert/utils.py @@ -1,5 +1,5 @@ import json -import os +from pathlib import Path from typing import Any, Dict, Tuple import rtoml @@ -11,19 +11,17 @@ from joj3_config_generator.models import repo, task def read_convert_files( case_name: str, ) -> Tuple[repo.Config, task.Config, Dict[str, Any]]: - root = os.path.dirname(os.path.realpath(__file__)) - repo_toml_path = os.path.join(root, case_name, "repo.toml") - with open(repo_toml_path) as f: - repo_toml = f.read() - task_toml_path = os.path.join(root, case_name, "task.toml") - with open(task_toml_path) as f: - task_toml = f.read() - result_json_path = os.path.join(root, case_name, "task.json") - with open(result_json_path) as f: - result: Dict[str, Any] = json.load(f) - repo_obj = rtoml.loads(repo_toml) - task_obj = rtoml.loads(task_toml) - return repo.Config(**repo_obj), task.Config(**task_obj), result + root = Path(__file__).resolve().parent + repo_toml_path = root / case_name / "repo.toml" + repo_toml = repo_toml_path.read_text() if repo_toml_path.exists() else "" + task_toml_path = root / case_name / "task.toml" + task_toml = task_toml_path.read_text() if task_toml_path.exists() else "" + result = json.loads((root / case_name / "task.json").read_text()) + return ( + repo.Config(**rtoml.loads(repo_toml)), + task.Config(**rtoml.loads(task_toml)), + result, + ) def load_case(case_name: str) -> None: diff --git a/tests/convert_joj1/utils.py b/tests/convert_joj1/utils.py index 36f6108..7732511 100644 --- a/tests/convert_joj1/utils.py +++ b/tests/convert_joj1/utils.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from typing import Any, Dict, Tuple import rtoml @@ -9,16 +9,12 @@ from joj3_config_generator.models import joj1 def read_convert_joj1_files(case_name: str) -> Tuple[joj1.Config, Dict[str, Any]]: - root = os.path.dirname(os.path.realpath(__file__)) - task_yaml_path = os.path.join(root, case_name, "task.yaml") - with open(task_yaml_path) as f: - task_yaml = f.read() - task_toml_path = os.path.join(root, case_name, "task.toml") - with open(task_toml_path) as f: - task_toml = f.read() - joj1_obj = yaml.safe_load(task_yaml) - task_obj = rtoml.loads(task_toml) - return joj1.Config(**joj1_obj), task_obj + root = Path(__file__).resolve().parent + task_yaml_path = root / case_name / "task.yaml" + task_yaml = task_yaml_path.read_text() + task_toml_path = root / case_name / "task.toml" + task_toml = task_toml_path.read_text() + return joj1.Config(**yaml.safe_load(task_yaml)), rtoml.loads(task_toml) def load_case(case_name: str) -> None: From 73f2581a316ad2603cc3514a5b97129d600780c6 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Tue, 25 Feb 2025 03:12:25 -0500 Subject: [PATCH 04/13] ci: use local checkout --- .gitea/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index f7d281d..2ff3fc6 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository code - uses: https://gitea.com/BoYanZh/checkout@focs + uses: actions/checkout@focs - name: Display Python3 version run: python3 --version - name: Install PDM From 68628c0eae83f2cc52aff57fda0fb149e500e0dd Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Tue, 25 Feb 2025 03:47:50 -0500 Subject: [PATCH 05/13] feat: convert toml files recursively --- joj3_config_generator/main.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index d556071..6847a00 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -1,5 +1,4 @@ import json -import os from pathlib import Path import inquirer @@ -51,18 +50,19 @@ def convert(root: Path = Path(".")) -> None: Convert given dir of JOJ3 toml config files to JOJ3 json config files """ logger.info(f"Converting files in {root.absolute()}") - repo_toml_path = os.path.join(root, "repo.toml") - # TODO: loop through all dirs to find all task.toml - task_toml_path = os.path.join(root, "task.toml") - result_json_path = os.path.join(root, "task.json") - with open(repo_toml_path) as repo_file: - repo_toml = repo_file.read() - with open(task_toml_path) as task_file: - task_toml = task_file.read() - repo_obj = rtoml.loads(repo_toml) - task_obj = rtoml.loads(task_toml) - result_model = convert_conf(repo.Config(**repo_obj), task.Config(**task_obj)) - result_dict = result_model.model_dump(by_alias=True) - with open(result_json_path, "w") as result_file: - json.dump(result_dict, result_file, ensure_ascii=False, indent=4) - result_file.write("\n") + repo_toml_path = root / "repo.toml" + repo_obj = rtoml.loads( + repo_toml_path.read_text() if repo_toml_path.exists() else "" + ) + for task_toml_path in root.glob("**/*.toml"): + toml_name = task_toml_path.name.removesuffix(".toml") + if toml_name == "repo": + continue + result_json_path = task_toml_path.parent / f"{toml_name}.json" + logger.info(f"Converting {task_toml_path} to {result_json_path}") + task_obj = rtoml.loads(task_toml_path.read_text()) + result_model = convert_conf(repo.Config(**repo_obj), task.Config(**task_obj)) + result_dict = result_model.model_dump(by_alias=True) + with result_json_path.open("w") as result_file: + json.dump(result_dict, result_file, ensure_ascii=False, indent=4) + result_file.write("\n") From 67d2fcc4e45e5da331f5c0b65b1bebd2e89d7908 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Tue, 25 Feb 2025 04:15:05 -0500 Subject: [PATCH 06/13] feat: dump with exclude_none --- joj3_config_generator/main.py | 13 +++++++++++-- tests/convert/basic/task.json | 26 -------------------------- tests/convert/utils.py | 4 +++- tests/convert_joj1/utils.py | 2 +- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index 6847a00..9d57dfe 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -5,6 +5,7 @@ import inquirer import rtoml import typer import yaml +from typing_extensions import Annotated from joj3_config_generator.convert import convert as convert_conf from joj3_config_generator.convert import convert_joj1 as convert_joj1_conf @@ -45,7 +46,15 @@ def convert_joj1(yaml_file: typer.FileText, toml_file: typer.FileTextWrite) -> N @app.command() -def convert(root: Path = Path(".")) -> None: +def convert( + root: Annotated[ + Path, + typer.Argument( + help="root directory of config files, " + "located at /home/tt/.config/joj in JTC" + ), + ] = Path(".") +) -> None: """ Convert given dir of JOJ3 toml config files to JOJ3 json config files """ @@ -62,7 +71,7 @@ def convert(root: Path = Path(".")) -> None: logger.info(f"Converting {task_toml_path} to {result_json_path}") task_obj = rtoml.loads(task_toml_path.read_text()) result_model = convert_conf(repo.Config(**repo_obj), task.Config(**task_obj)) - result_dict = result_model.model_dump(by_alias=True) + result_dict = result_model.model_dump(by_alias=True, exclude_none=True) with result_json_path.open("w") as result_file: json.dump(result_dict, result_file, ensure_ascii=False, indent=4) result_file.write("\n") diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 294d15c..ca0a8fc 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -19,9 +19,6 @@ "./h3/ex5.m" ], "env": [], - "stdin": null, - "stdout": null, - "stderr": null, "cpuLimit": 0, "realCpuLimit": 0, "clockLimit": 0, @@ -33,22 +30,12 @@ "copyIn": { "tools/matlab-joj": { "src": "tools/matlab-joj", - "content": null, - "fileId": null, - "name": null, - "max": null, - "symlink": null, "streamIn": false, "streamOut": false, "pipe": false }, "tools/matlab_formatter.py": { "src": "tools/matlab_formatter.py", - "content": null, - "fileId": null, - "name": null, - "max": null, - "symlink": null, "streamIn": false, "streamOut": false, "pipe": false @@ -99,9 +86,6 @@ "./h3/ex5.m" ], "env": [], - "stdin": null, - "stdout": null, - "stderr": null, "cpuLimit": 0, "realCpuLimit": 0, "clockLimit": 0, @@ -113,22 +97,12 @@ "copyIn": { "tools/matlab-joj": { "src": "tools/matlab-joj", - "content": null, - "fileId": null, - "name": null, - "max": null, - "symlink": null, "streamIn": false, "streamOut": false, "pipe": false }, "tools/matlab_formatter.py": { "src": "tools/matlab_formatter.py", - "content": null, - "fileId": null, - "name": null, - "max": null, - "symlink": null, "streamIn": false, "streamOut": false, "pipe": false diff --git a/tests/convert/utils.py b/tests/convert/utils.py index 046f715..f51f073 100644 --- a/tests/convert/utils.py +++ b/tests/convert/utils.py @@ -26,5 +26,7 @@ def read_convert_files( def load_case(case_name: str) -> None: repo, task, expected_result = read_convert_files(case_name) - result = convert(repo, task).model_dump(mode="json", by_alias=True) + result = convert(repo, task).model_dump( + mode="json", by_alias=True, exclude_none=True + ) assert result == expected_result diff --git a/tests/convert_joj1/utils.py b/tests/convert_joj1/utils.py index 7732511..74aa1b5 100644 --- a/tests/convert_joj1/utils.py +++ b/tests/convert_joj1/utils.py @@ -19,5 +19,5 @@ def read_convert_joj1_files(case_name: str) -> Tuple[joj1.Config, Dict[str, Any] def load_case(case_name: str) -> None: joj1, expected_result = read_convert_joj1_files(case_name) - result = convert_joj1(joj1).model_dump(by_alias=True) + result = convert_joj1(joj1).model_dump(by_alias=True, exclude_none=True) assert result == expected_result From ab5bfc6cfeed697467de3a2000ff4e3f62bac2a6 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Tue, 25 Feb 2025 14:05:00 -0500 Subject: [PATCH 07/13] fix: find repo.toml recursively --- joj3_config_generator/main.py | 37 ++++++++++++++++------------ joj3_config_generator/models/repo.py | 2 ++ joj3_config_generator/models/task.py | 2 ++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index 9d57dfe..91d879a 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -59,19 +59,24 @@ def convert( Convert given dir of JOJ3 toml config files to JOJ3 json config files """ logger.info(f"Converting files in {root.absolute()}") - repo_toml_path = root / "repo.toml" - repo_obj = rtoml.loads( - repo_toml_path.read_text() if repo_toml_path.exists() else "" - ) - for task_toml_path in root.glob("**/*.toml"): - toml_name = task_toml_path.name.removesuffix(".toml") - if toml_name == "repo": - continue - result_json_path = task_toml_path.parent / f"{toml_name}.json" - logger.info(f"Converting {task_toml_path} to {result_json_path}") - task_obj = rtoml.loads(task_toml_path.read_text()) - result_model = convert_conf(repo.Config(**repo_obj), task.Config(**task_obj)) - result_dict = result_model.model_dump(by_alias=True, exclude_none=True) - with result_json_path.open("w") as result_file: - json.dump(result_dict, result_file, ensure_ascii=False, indent=4) - result_file.write("\n") + for repo_toml_path in root.glob("**/repo.toml"): + repo_path = repo_toml_path.parent + repo_obj = rtoml.loads(repo_toml_path.read_text()) + for task_toml_path in repo_path.glob("**/*.toml"): + if repo_toml_path == task_toml_path: + continue + toml_name = task_toml_path.name.removesuffix(".toml") + result_json_path = task_toml_path.parent / f"{toml_name}.json" + logger.info( + f"Converting {repo_toml_path} & {task_toml_path} to {result_json_path}" + ) + task_obj = rtoml.loads(task_toml_path.read_text()) + repo_conf = repo.Config(**repo_obj) + repo_conf.path = repo_toml_path + task_conf = task.Config(**task_obj) + task_conf.path = task_toml_path + result_model = convert_conf(repo_conf, task_conf) + result_dict = result_model.model_dump(by_alias=True, exclude_none=True) + with result_json_path.open("w") as result_file: + json.dump(result_dict, result_file, ensure_ascii=False, indent=4) + result_file.write("\n") diff --git a/joj3_config_generator/models/repo.py b/joj3_config_generator/models/repo.py index 4befab6..288096f 100644 --- a/joj3_config_generator/models/repo.py +++ b/joj3_config_generator/models/repo.py @@ -1,3 +1,4 @@ +from pathlib import Path from typing import List, Optional from pydantic import BaseModel, Field @@ -11,6 +12,7 @@ class Files(BaseModel): class Config(BaseModel): + path: Path = Path(".") teaching_team: List[str] max_size: float = Field(..., ge=0) release_tags: List[str] diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index 09a37fc..ab15bf7 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -1,4 +1,5 @@ from datetime import datetime +from pathlib import Path from typing import List, Optional from pydantic import BaseModel, Field @@ -32,6 +33,7 @@ class Release(BaseModel): class Config(BaseModel): + path: Path = Path(".") task: str # Task name (e.g., hw3 ex5) release: Release # Release configuration stages: List[Stage] # list of stage configurations From c4639aef767269e1fff6ced63500dcca1bd2f8de Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 27 Feb 2025 00:24:37 -0500 Subject: [PATCH 08/13] fix: store relative path to root --- joj3_config_generator/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index 91d879a..271954f 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -72,9 +72,9 @@ def convert( ) task_obj = rtoml.loads(task_toml_path.read_text()) repo_conf = repo.Config(**repo_obj) - repo_conf.path = repo_toml_path + repo_conf.path = repo_toml_path.relative_to(root) task_conf = task.Config(**task_obj) - task_conf.path = task_toml_path + task_conf.path = task_toml_path.relative_to(root) result_model = convert_conf(repo_conf, task_conf) result_dict = result_model.model_dump(by_alias=True, exclude_none=True) with result_json_path.open("w") as result_file: From 4c12bf9a342231658b9274053279568cc40c5436 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 27 Feb 2025 01:14:40 -0500 Subject: [PATCH 09/13] fix: store root path --- joj3_config_generator/main.py | 2 +- joj3_config_generator/models/repo.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index 271954f..eaf9c04 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -72,9 +72,9 @@ def convert( ) task_obj = rtoml.loads(task_toml_path.read_text()) repo_conf = repo.Config(**repo_obj) + repo_conf.root = root repo_conf.path = repo_toml_path.relative_to(root) task_conf = task.Config(**task_obj) - task_conf.path = task_toml_path.relative_to(root) result_model = convert_conf(repo_conf, task_conf) result_dict = result_model.model_dump(by_alias=True, exclude_none=True) with result_json_path.open("w") as result_file: diff --git a/joj3_config_generator/models/repo.py b/joj3_config_generator/models/repo.py index 288096f..5149e18 100644 --- a/joj3_config_generator/models/repo.py +++ b/joj3_config_generator/models/repo.py @@ -12,7 +12,8 @@ class Files(BaseModel): class Config(BaseModel): - path: Path = Path(".") + root: Path = Path(".") + path: Path = Path("repo.toml") teaching_team: List[str] max_size: float = Field(..., ge=0) release_tags: List[str] From 723705b94fc8c8006c6c1ca73181efdf70850033 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 27 Feb 2025 01:17:14 -0500 Subject: [PATCH 10/13] fix: store path for both conf objs --- joj3_config_generator/main.py | 2 ++ joj3_config_generator/models/task.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index eaf9c04..cf522c9 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -75,6 +75,8 @@ def convert( repo_conf.root = root repo_conf.path = repo_toml_path.relative_to(root) task_conf = task.Config(**task_obj) + task_conf.root = root + task_conf.path = task_toml_path.relative_to(root) result_model = convert_conf(repo_conf, task_conf) result_dict = result_model.model_dump(by_alias=True, exclude_none=True) with result_json_path.open("w") as result_file: diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index ab15bf7..6672d02 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -33,7 +33,8 @@ class Release(BaseModel): class Config(BaseModel): - path: Path = Path(".") + root: Path = Path(".") + path: Path = Path("conf.toml") task: str # Task name (e.g., hw3 ex5) release: Release # Release configuration stages: List[Stage] # list of stage configurations From 870f20dde98d1fe32076ff9129ab76cb5237aeb8 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 27 Feb 2025 01:25:56 -0500 Subject: [PATCH 11/13] chore: remove unused utils --- tests/utils.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 tests/utils.py diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index 7eda07e..0000000 --- a/tests/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import Any - - -def safe_id(x: Any) -> str: - if not x or not isinstance(x, (tuple, list)) or len(x) == 0: - return "no_test_cases" - return str(x[0]) From ae77fa23b5cf395743317e3744edff7af405c6f2 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 27 Feb 2025 01:26:21 -0500 Subject: [PATCH 12/13] fix: default path for tests --- joj3_config_generator/models/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index 6672d02..a09a069 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -34,7 +34,7 @@ class Release(BaseModel): class Config(BaseModel): root: Path = Path(".") - path: Path = Path("conf.toml") + path: Path = Path("task.toml") task: str # Task name (e.g., hw3 ex5) release: Release # Release configuration stages: List[Stage] # list of stage configurations From aee123601f809bda0216f27aad893dd0e86e9197 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 27 Feb 2025 01:28:43 -0500 Subject: [PATCH 13/13] fix: root path for tests --- tests/convert/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/convert/utils.py b/tests/convert/utils.py index f51f073..0a458a1 100644 --- a/tests/convert/utils.py +++ b/tests/convert/utils.py @@ -11,15 +11,15 @@ from joj3_config_generator.models import repo, task def read_convert_files( case_name: str, ) -> Tuple[repo.Config, task.Config, Dict[str, Any]]: - root = Path(__file__).resolve().parent - repo_toml_path = root / case_name / "repo.toml" + root = Path(__file__).resolve().parent / case_name + repo_toml_path = root / "repo.toml" repo_toml = repo_toml_path.read_text() if repo_toml_path.exists() else "" - task_toml_path = root / case_name / "task.toml" + task_toml_path = root / "task.toml" task_toml = task_toml_path.read_text() if task_toml_path.exists() else "" - result = json.loads((root / case_name / "task.json").read_text()) + result = json.loads((root / "task.json").read_text()) return ( - repo.Config(**rtoml.loads(repo_toml)), - task.Config(**rtoml.loads(task_toml)), + repo.Config(root=root, **rtoml.loads(repo_toml)), + task.Config(root=root, **rtoml.loads(task_toml)), result, )