From 8bf0f7cfff198febdbcd8715639cad6fa1092ad7 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sun, 20 Oct 2024 04:46:41 -0400 Subject: [PATCH] feat: LLM generated convert function --- joj3_config_generator/convert.py | 61 ++++++++ joj3_config_generator/main.py | 18 ++- joj3_config_generator/model.py | 101 ------------- joj3_config_generator/models/__init__.py | 11 ++ joj3_config_generator/models/repo.py | 18 +++ joj3_config_generator/models/result.py | 135 +++++++++++++++++ joj3_config_generator/models/task.py | 37 +++++ tests/basic/repo.toml | 10 ++ tests/basic/task.json | 181 +++++++++++++++++++++++ tests/basic/task.toml | 30 ++++ tests/conftest.py | 23 +++ tests/example_test.py | 17 --- tests/test_all_cases.py | 10 ++ tests/utils.py | 22 +++ 14 files changed, 552 insertions(+), 122 deletions(-) create mode 100644 joj3_config_generator/convert.py delete mode 100644 joj3_config_generator/model.py create mode 100644 joj3_config_generator/models/__init__.py create mode 100644 joj3_config_generator/models/repo.py create mode 100644 joj3_config_generator/models/result.py create mode 100644 joj3_config_generator/models/task.py create mode 100644 tests/basic/repo.toml create mode 100644 tests/basic/task.json create mode 100644 tests/basic/task.toml create mode 100644 tests/conftest.py delete mode 100644 tests/example_test.py create mode 100644 tests/test_all_cases.py create mode 100644 tests/utils.py diff --git a/joj3_config_generator/convert.py b/joj3_config_generator/convert.py new file mode 100644 index 0000000..719c445 --- /dev/null +++ b/joj3_config_generator/convert.py @@ -0,0 +1,61 @@ +from joj3_config_generator.models import ( + Cmd, + CmdFile, + ExecutorConfig, + ExecutorWithConfig, + ParserConfig, + Repo, + ResultConfig, + Stage, + StageConfig, + Task, + TeapotConfig, +) + + +# FIXME: LLM generated convert function, only for demostration +def convert(repo_conf: Repo, task_conf: Task) -> ResultConfig: + # Create the base ResultConf object + result_conf = ResultConfig( + name=task_conf.task, + log_path=f"{task_conf.task.replace(' ', '_')}.log", + expire_unix_timestamp=( + int(task_conf.release.deadline.timestamp()) + if task_conf.release.deadline + else -1 + ), + stage=StageConfig(stages=[], sandbox_token=repo_conf.sandbox_token), + teapot=TeapotConfig(), + ) + + # Convert each stage in the task configuration + for task_stage in task_conf.stages: + executor_with_config = ExecutorWithConfig( + default=Cmd( + args=task_stage.command.split(), + copy_in={file: CmdFile(src=file) for file in task_stage.files.import_}, + copy_out_cached=task_stage.files.export, + ), + cases=[], # You can add cases if needed + ) + conf_stage = Stage( + name=task_stage.name, + group=task_conf.task, + executor=ExecutorConfig( + name="sandbox", + with_=executor_with_config, + ), + parsers=[ + ParserConfig(name=parser, with_={}) for parser in task_stage.parsers + ], + ) + + if "result-detail" in task_stage.parsers: + result_detail_parser = next( + p for p in conf_stage.parsers if p.name == "result-detail" + ) + result_detail_parser.with_.update(task_stage.result_detail) + + result_conf.stage.stages.append(conf_stage) + + return result_conf diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index 9cb842d..89be721 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -1,3 +1,5 @@ +from pathlib import Path + import inquirer import typer @@ -7,9 +9,9 @@ app = typer.Typer(add_completion=False) @app.command() -def create() -> None: +def create(toml: typer.FileTextWrite) -> None: """ - Create a new JOJ3 config file + Create a new JOJ3 toml config file """ logger.info("Creating") questions = [ @@ -24,8 +26,16 @@ def create() -> None: @app.command() -def convert() -> None: +def convert_joj1(yaml: typer.FileText, toml: typer.FileTextWrite) -> None: """ - Convert a JOJ1 config file to JOJ3 config file + Convert a JOJ1 yaml config file to JOJ3 toml config file """ logger.info("Converting") + + +@app.command() +def convert(root_path: Path = Path(".")) -> None: + """ + Convert given dir of JOJ3 toml config files to JOJ3 json config files + """ + logger.info(f"Converting {root_path.absolute()}") diff --git a/joj3_config_generator/model.py b/joj3_config_generator/model.py deleted file mode 100644 index 133f81d..0000000 --- a/joj3_config_generator/model.py +++ /dev/null @@ -1,101 +0,0 @@ -from datetime import datetime -from typing import List, Optional - -import rtoml -from pydantic import BaseModel, Field - - -class RepoFiles(BaseModel): - whitelist_patterns: List[str] - whitelist_file: Optional[str] - required: List[str] - immutable: List[str] - - -class Repo(BaseModel): - teaching_team: List[str] - max_size: float = Field(..., ge=0) - release_tags: List[str] - files: RepoFiles - - -class ParserResultDetail(BaseModel): - time: bool = True # Display run time - mem: bool = True # Display memory usage - stderr: bool = False # Display stderr messages - - -class Files(BaseModel): - import_: List[str] = Field(alias="import") - export: List[str] - - -class Stage(BaseModel): - name: str # Stage name - command: str # Command to run - files: Files # Files to import and export - score: int # Score for the task - parsers: List[str] # List of parsers - result_detail: ParserResultDetail = ( - ParserResultDetail() - ) # for result-detail parser - - -class Release(BaseModel): - deadline: Optional[datetime] # RFC 3339 formatted date-time with offset - - -class Task(BaseModel): - task: str # Task name (e.g., hw3 ex5) - release: Release # Release configuration - stages: List[Stage] # List of stage configurations - - -if __name__ == "__main__": - repo_toml = """ -teaching_team = [ "prof_john", "ta_alice", "ta_bob" ] -max_size = 50.5 -release_tags = [ "v1.0", "v2.0", "final" ] - -[files] -whitelist_patterns = [ "*.py", "*.txt", "*.md" ] -whitelist_file = ".whitelist" -required = [ "main.py", "README.md" ] -immutable = [ "config.yaml", "setup.py" ] -""" - task_toml = """ -task = "hw3 ex5" - -[release] -deadline = "2024-10-18T23:59:00+08:00" - -[[stages]] -name = "judge_base" -command = "./matlab-joj ./h3/ex5.m" -score = 100 -parsers = [ "diff", "result-detail" ] - -files.import = [ "tools/matlab-joj", "tools/matlab_formatter.py" ] -files.export = [ "output/ex5_results.txt", "output/ex5_logs.txt" ] - -result_detail.time = false -result_detail.mem = false -result_detail.stderr = true - -[[stages]] -name = "judge_base2" -command = "./matlab-joj ./h3/ex5.m" -score = 80 -parsers = [ "diff", "result-detail" ] - -files.import = [ "tools/matlab-joj", "tools/matlab_formatter.py" ] -files.export = [ "output/ex5_results2.txt", "output/ex5_logs2.txt" ] - -result_detail.time = true -result_detail.mem = true -result_detail.stderr = false -""" - repo_obj = rtoml.loads(repo_toml) - task_obj = rtoml.loads(task_toml) - print(Repo(**repo_obj)) - print(Task(**task_obj)) diff --git a/joj3_config_generator/models/__init__.py b/joj3_config_generator/models/__init__.py new file mode 100644 index 0000000..68802d4 --- /dev/null +++ b/joj3_config_generator/models/__init__.py @@ -0,0 +1,11 @@ +from joj3_config_generator.models.repo import Repo as Repo +from joj3_config_generator.models.result import Cmd as Cmd +from joj3_config_generator.models.result import CmdFile as CmdFile +from joj3_config_generator.models.result import ExecutorConfig as ExecutorConfig +from joj3_config_generator.models.result import ExecutorWithConfig as ExecutorWithConfig +from joj3_config_generator.models.result import ParserConfig as ParserConfig +from joj3_config_generator.models.result import ResultConfig as ResultConfig +from joj3_config_generator.models.result import Stage as Stage +from joj3_config_generator.models.result import StageConfig as StageConfig +from joj3_config_generator.models.result import TeapotConfig as TeapotConfig +from joj3_config_generator.models.task import Task as Task diff --git a/joj3_config_generator/models/repo.py b/joj3_config_generator/models/repo.py new file mode 100644 index 0000000..0783468 --- /dev/null +++ b/joj3_config_generator/models/repo.py @@ -0,0 +1,18 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class RepoFiles(BaseModel): + whitelist_patterns: list[str] + whitelist_file: Optional[str] + required: list[str] + immutable: list[str] + + +class Repo(BaseModel): + teaching_team: list[str] + max_size: float = Field(..., ge=0) + release_tags: list[str] + files: RepoFiles + sandbox_token: str diff --git a/joj3_config_generator/models/result.py b/joj3_config_generator/models/result.py new file mode 100644 index 0000000..40306ee --- /dev/null +++ b/joj3_config_generator/models/result.py @@ -0,0 +1,135 @@ +from typing import Any, Optional + +import rtoml +from pydantic import BaseModel, Field + + +class CmdFile(BaseModel): + src: Optional[str] = None + content: Optional[str] = None + file_id: Optional[str] = Field(None, serialization_alias="fileId") + name: Optional[str] = None + max: Optional[int] = None + symlink: Optional[str] = None + stream_in: bool = Field(False, serialization_alias="streamIn") + stream_out: bool = Field(False, serialization_alias="streamOut") + pipe: bool = False + + +class Cmd(BaseModel): + args: list[str] + env: list[str] = [] + stdin: Optional[CmdFile] = None + stdout: Optional[CmdFile] = None + stderr: Optional[CmdFile] = None + cpu_limit: int = Field(0, serialization_alias="cpuLimit") + real_cpu_limit: int = Field(0, serialization_alias="realCpuLimit") + clock_limit: int = Field(0, serialization_alias="clockLimit") + memory_limit: int = Field(0, serialization_alias="memoryLimit") + stack_limit: int = Field(0, serialization_alias="stackLimit") + proc_limit: int = Field(0, serialization_alias="procLimit") + cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit") + cpu_set_limit: str = Field("", serialization_alias="cpuSetLimit") + copy_in: dict[str, CmdFile] = Field({}, serialization_alias="copyIn") + copy_in_cached: dict[str, str] = Field({}, serialization_alias="copyInCached") + copy_in_dir: str = Field("", serialization_alias="copyInDir") + copy_out: list[str] = Field([], serialization_alias="copyOut") + copy_out_cached: list[str] = Field([], serialization_alias="copyOutCached") + copy_out_max: int = Field(0, serialization_alias="copyOutMax") + copy_out_dir: str = Field("", serialization_alias="copyOutDir") + tty: bool = False + strict_memory_limit: bool = Field(False, serialization_alias="strictMemoryLimit") + data_segment_limit: bool = Field(False, serialization_alias="dataSegmentLimit") + address_space_limit: bool = Field(False, serialization_alias="addressSpaceLimit") + + +class OptionalCmd(BaseModel): + args: Optional[list[str]] = None + env: Optional[list[str]] = None + stdin: Optional[CmdFile] = None + stdout: Optional[CmdFile] = None + stderr: Optional[CmdFile] = None + cpu_limit: Optional[int] = Field(None, serialization_alias="cpuLimit") + real_cpu_limit: Optional[int] = Field(None, serialization_alias="realCpuLimit") + clock_limit: Optional[int] = Field(None, serialization_alias="clockLimit") + memory_limit: Optional[int] = Field(None, serialization_alias="memoryLimit") + stack_limit: Optional[int] = Field(None, serialization_alias="stackLimit") + proc_limit: Optional[int] = Field(None, serialization_alias="procLimit") + cpu_rate_limit: Optional[int] = Field(None, serialization_alias="cpuRateLimit") + cpu_set_limit: Optional[str] = Field(None, serialization_alias="cpuSetLimit") + copy_in: Optional[dict[str, CmdFile]] = Field(None, serialization_alias="copyIn") + copy_in_cached: Optional[dict[str, str]] = Field( + None, serialization_alias="copyInCached" + ) + copy_in_dir: Optional[str] = Field(None, serialization_alias="copyInDir") + copy_out: Optional[list[str]] = Field(None, serialization_alias="copyOut") + copy_out_cached: Optional[list[str]] = Field( + None, serialization_alias="copyOutCached" + ) + copy_out_max: Optional[int] = Field(None, serialization_alias="copyOutMax") + copy_out_dir: Optional[str] = Field(None, serialization_alias="copyOutDir") + tty: Optional[bool] = None + strict_memory_limit: Optional[bool] = Field( + None, serialization_alias="strictMemoryLimit" + ) + data_segment_limit: Optional[bool] = Field( + None, serialization_alias="dataSegmentLimit" + ) + address_space_limit: Optional[bool] = Field( + None, serialization_alias="addressSpaceLimit" + ) + + +class Stage(BaseModel): + name: str + group: str + executor: "ExecutorConfig" + parsers: list["ParserConfig"] + + +class ExecutorWithConfig(BaseModel): + default: Cmd + cases: list[OptionalCmd] + + +class ExecutorConfig(BaseModel): + name: str + with_: ExecutorWithConfig = Field(..., serialization_alias="with") + + +class ParserConfig(BaseModel): + name: str + with_: dict[str, Any] = Field(..., serialization_alias="with") + + +class StageConfig(BaseModel): + sandbox_exec_server: str = Field( + "172.17.0.1:5051", serialization_alias="sandboxExecServer" + ) + sandbox_token: str = Field("", serialization_alias="sandboxToken") + output_path: str = Field( + "/tmp/joj3_result.json", serialization_alias="outputPath" + ) # nosec: B108 + stages: list[Stage] + + +class TeapotConfig(BaseModel): + log_path: str = Field( + "/home/tt/.cache/joint-teapot-debug.log", serialization_alias="logPath" + ) + scoreboard_path: str = Field("scoreboard.csv", serialization_alias="scoreboardPath") + failed_table_path: str = Field( + "failed-table.md", serialization_alias="failedTablePath" + ) + grading_repo_name: str = Field("", serialization_alias="gradingRepoName") + skip_issue: bool = Field(False, serialization_alias="skipIssue") + skip_scoreboard: bool = Field(False, serialization_alias="skipScoreboard") + skip_failed_table: bool = Field(False, serialization_alias="skipFailedTable") + + +class ResultConfig(BaseModel): + name: str = "unknown" + log_path: str = Field("", serialization_alias="logPath") + expire_unix_timestamp: int = Field(-1, serialization_alias="expireUnixTimestamp") + stage: StageConfig + teapot: TeapotConfig diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py new file mode 100644 index 0000000..802555b --- /dev/null +++ b/joj3_config_generator/models/task.py @@ -0,0 +1,37 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, Field + + +class ParserResultDetail(BaseModel): + time: bool = True # Display run time + mem: bool = True # Display memory usage + stdout: bool = False # Display stdout messages + stderr: bool = False # Display stderr messages + + +class Files(BaseModel): + import_: list[str] = Field(alias="import") + export: list[str] + + +class Stage(BaseModel): + name: str # Stage name + command: str # Command to run + files: Files # Files to import and export + score: int # Score for the task + parsers: list[str] # list of parsers + result_detail: ParserResultDetail = ( + ParserResultDetail() + ) # for result-detail parser + + +class Release(BaseModel): + deadline: Optional[datetime] # RFC 3339 formatted date-time with offset + + +class Task(BaseModel): + task: str # Task name (e.g., hw3 ex5) + release: Release # Release configuration + stages: list[Stage] # list of stage configurations diff --git a/tests/basic/repo.toml b/tests/basic/repo.toml new file mode 100644 index 0000000..f9012cc --- /dev/null +++ b/tests/basic/repo.toml @@ -0,0 +1,10 @@ +teaching_team = ["prof_john", "ta_alice", "ta_bob"] +max_size = 50.5 +release_tags = ["v1.0", "v2.0", "final"] +sandbox_token = "test" + +[files] +whitelist_patterns = ["*.py", "*.txt", "*.md"] +whitelist_file = ".whitelist" +required = ["main.py", "README.md"] +immutable = ["config.yaml", "setup.py"] diff --git a/tests/basic/task.json b/tests/basic/task.json new file mode 100644 index 0000000..e7da0eb --- /dev/null +++ b/tests/basic/task.json @@ -0,0 +1,181 @@ +{ + "name": "hw3 ex5", + "logPath": "hw3_ex5.log", + "expireUnixTimestamp": 1729267140, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "test", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "judge_base", + "group": "hw3 ex5", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./matlab-joj", + "./h3/ex5.m" + ], + "env": [], + "stdin": null, + "stdout": null, + "stderr": null, + "cpuLimit": 0, + "realCpuLimit": 0, + "clockLimit": 0, + "memoryLimit": 0, + "stackLimit": 0, + "procLimit": 0, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tools/matlab-joj": { + "src": "tools/matlab-joj", + "content": null, + "fileId": null, + "name": null, + "max": null, + "symlink": null, + "streamIn": false, + "streamOut": false, + "pipe": false + }, + "tools/matlab_formatter.py": { + "src": "tools/matlab_formatter.py", + "content": null, + "fileId": null, + "name": null, + "max": null, + "symlink": null, + "streamIn": false, + "streamOut": false, + "pipe": false + } + }, + "copyInCached": {}, + "copyInDir": "", + "copyOut": [], + "copyOutCached": [ + "output/ex5_results.txt", + "output/ex5_logs.txt" + ], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "diff", + "with": {} + }, + { + "name": "result-detail", + "with": { + "time": false, + "mem": false, + "stdout": false, + "stderr": true + } + } + ] + }, + { + "name": "judge_base2", + "group": "hw3 ex5", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./matlab-joj", + "./h3/ex5.m" + ], + "env": [], + "stdin": null, + "stdout": null, + "stderr": null, + "cpuLimit": 0, + "realCpuLimit": 0, + "clockLimit": 0, + "memoryLimit": 0, + "stackLimit": 0, + "procLimit": 0, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tools/matlab-joj": { + "src": "tools/matlab-joj", + "content": null, + "fileId": null, + "name": null, + "max": null, + "symlink": null, + "streamIn": false, + "streamOut": false, + "pipe": false + }, + "tools/matlab_formatter.py": { + "src": "tools/matlab_formatter.py", + "content": null, + "fileId": null, + "name": null, + "max": null, + "symlink": null, + "streamIn": false, + "streamOut": false, + "pipe": false + } + }, + "copyInCached": {}, + "copyInDir": "", + "copyOut": [], + "copyOutCached": [ + "output/ex5_results2.txt", + "output/ex5_logs2.txt" + ], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "diff", + "with": {} + }, + { + "name": "result-detail", + "with": { + "time": true, + "mem": true, + "stdout": false, + "stderr": false + } + } + ] + } + ] + }, + "teapot": { + "logPath": "/home/tt/.cache/joint-teapot-debug.log", + "scoreboardPath": "scoreboard.csv", + "failedTablePath": "failed-table.md", + "gradingRepoName": "", + "skipIssue": false, + "skipScoreboard": false, + "skipFailedTable": false + } +} diff --git a/tests/basic/task.toml b/tests/basic/task.toml new file mode 100644 index 0000000..0872079 --- /dev/null +++ b/tests/basic/task.toml @@ -0,0 +1,30 @@ +task = "hw3 ex5" + +[release] +deadline = "2024-10-18T23:59:00+08:00" + +[[stages]] +name = "judge_base" +command = "./matlab-joj ./h3/ex5.m" +score = 100 +parsers = ["diff", "result-detail"] + +files.import = ["tools/matlab-joj", "tools/matlab_formatter.py"] +files.export = ["output/ex5_results.txt", "output/ex5_logs.txt"] + +result_detail.time = false +result_detail.mem = false +result_detail.stderr = true + +[[stages]] +name = "judge_base2" +command = "./matlab-joj ./h3/ex5.m" +score = 80 +parsers = ["diff", "result-detail"] + +files.import = ["tools/matlab-joj", "tools/matlab_formatter.py"] +files.export = ["output/ex5_results2.txt", "output/ex5_logs2.txt"] + +result_detail.time = true +result_detail.mem = true +result_detail.stderr = false diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6d41010 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +import os +from typing import Any + +import pytest + +from joj3_config_generator.models import Repo, Task +from tests.utils import read_convert_files + + +def get_test_cases() -> list[tuple[str, Repo, Task, dict[str, Any]]]: + test_cases = [] + tests_dir = os.path.dirname(os.path.realpath(__file__)) + for dir_name in os.listdir(tests_dir): + dir_path = os.path.join(tests_dir, dir_name) + if os.path.isdir(dir_path) and dir_name != "__pycache__": + repo, task, expected_result = read_convert_files(dir_path) + test_cases.append((dir_name, repo, task, expected_result)) + return test_cases + + +@pytest.fixture(params=get_test_cases(), ids=lambda x: x[0]) +def test_case(request: pytest.FixtureRequest) -> tuple[Repo, Task, dict[str, Any]]: + return request.param[1:] # return repo, task, expected_result diff --git a/tests/example_test.py b/tests/example_test.py deleted file mode 100644 index 0058043..0000000 --- a/tests/example_test.py +++ /dev/null @@ -1,17 +0,0 @@ -# from xxx import generate - - -from typing import Any - -import pytest - - -@pytest.mark.xfail(strict=True) -def test_generate() -> None: - generate = lambda x: x # TODO: real generate function imported - data_input: dict[Any, Any] = {} # TODO: load real input from some file - data_output: dict[Any, Any] = generate(data_input) - expected_output: dict[Any, Any] = { - "a": "b" - } # TODO: load real output from some file - assert data_output == expected_output diff --git a/tests/test_all_cases.py b/tests/test_all_cases.py new file mode 100644 index 0000000..ca0ddb2 --- /dev/null +++ b/tests/test_all_cases.py @@ -0,0 +1,10 @@ +from typing import Any + +from joj3_config_generator.convert import convert +from joj3_config_generator.models import Repo, Task + + +def test_convert(test_case: tuple[Repo, Task, dict[str, Any]]) -> None: + repo, task, expected_result = test_case + result = convert(repo, task).model_dump(by_alias=True) + assert result == expected_result diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..b733db7 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,22 @@ +import json +import os +from typing import Any + +import rtoml + +from joj3_config_generator.models import Repo, Task + + +def read_convert_files(root: str) -> tuple[Repo, Task, dict[str, Any]]: + repo_toml_path = os.path.join(root, "repo.toml") + task_toml_path = os.path.join(root, "task.toml") + result_json_path = os.path.join(root, "task.json") + with open(repo_toml_path) as repo_file: + repo_toml = repo_file.read() + with open(task_toml_path) as task_file: + task_toml = task_file.read() + with open(result_json_path) as result_file: + expected_result: dict[str, Any] = json.load(result_file) + repo_obj = rtoml.loads(repo_toml) + task_obj = rtoml.loads(task_toml) + return Repo(**repo_obj), Task(**task_obj), expected_result