refactor: re-organize tests & models

.pre-commit-config.yaml

@@ -15,6 +15,7 @@ repos:
     rev: v0.13.0
     hooks:
       - id: yamlfmt
+        exclude: '^tests/'
   - repo: https://github.com/pdm-project/pdm
     rev: 2.19.2
     hooks:

joj3_config_generator/convert.py

@@ -1,22 +1,10 @@
-from joj3_config_generator.models import (
+from joj3_config_generator.models import joj1, repo, result, task
-    Cmd,
-    CmdFile,
-    ExecutorConfig,
-    ExecutorWithConfig,
-    ParserConfig,
-    RepoConfig,
-    ResultConfig,
-    Stage,
-    StageConfig,
-    TaskConfig,
-    TeapotConfig,
-)
 
 
 # FIXME: LLM generated convert function, only for demostration
-def convert(repo_conf: RepoConfig, task_conf: TaskConfig) -> ResultConfig:
+def convert(repo_conf: repo.Config, task_conf: task.Config) -> result.Config:
     # Create the base ResultConf object
-    result_conf = ResultConfig(
+    result_conf = result.Config(
         name=task_conf.task,
         log_path=f"{task_conf.task.replace(' ', '_')}.log",
         expire_unix_timestamp=(
@@ -24,29 +12,31 @@ def convert(repo_conf: RepoConfig, task_conf: TaskConfig) -> ResultConfig:
             if task_conf.release.deadline
             else -1
         ),
-        stage=StageConfig(stages=[], sandbox_token=repo_conf.sandbox_token),
+        stage=result.Stage(stages=[], sandbox_token=repo_conf.sandbox_token),
-        teapot=TeapotConfig(),
+        teapot=result.Teapot(),
     )
 
     # Convert each stage in the task configuration
     for task_stage in task_conf.stages:
-        executor_with_config = ExecutorWithConfig(
+        executor_with_config = result.ExecutorWith(
-            default=Cmd(
+            default=result.Cmd(
                 args=task_stage.command.split(),
-                copy_in={file: CmdFile(src=file) for file in task_stage.files.import_},
+                copy_in={
+                    file: result.CmdFile(src=file) for file in task_stage.files.import_
+                },
                 copy_out_cached=task_stage.files.export,
             ),
             cases=[],  # You can add cases if needed
         )
-        conf_stage = Stage(
+        conf_stage = result.StageDetail(
             name=task_stage.name,
             group=task_conf.task,
-            executor=ExecutorConfig(
+            executor=result.Executor(
                 name="sandbox",
                 with_=executor_with_config,
             ),
             parsers=[
-                ParserConfig(name=parser, with_={}) for parser in task_stage.parsers
+                result.Parser(name=parser, with_={}) for parser in task_stage.parsers
             ],
         )
 
@@ -59,3 +49,8 @@ def convert(repo_conf: RepoConfig, task_conf: TaskConfig) -> ResultConfig:
         result_conf.stage.stages.append(conf_stage)
 
     return result_conf
+
+
+# TODO: implement me
+def convert_joj1(joj1_conf: joj1.Config) -> task.Config:
+    return task.Config(task="", release=task.Release(deadline=None), stages=[])

joj3_config_generator/models/__init__.py

@@ -1,15 +0,0 @@
-from joj3_config_generator.models.joj1 import Case as Case
-from joj3_config_generator.models.joj1 import JOJ1Config as JOJ1Config
-from joj3_config_generator.models.joj1 import Language as Language
-from joj3_config_generator.models.repo import RepoConfig as RepoConfig
-from joj3_config_generator.models.repo import RepoFiles as RepoFiles
-from joj3_config_generator.models.result import Cmd as Cmd
-from joj3_config_generator.models.result import CmdFile as CmdFile
-from joj3_config_generator.models.result import ExecutorConfig as ExecutorConfig
-from joj3_config_generator.models.result import ExecutorWithConfig as ExecutorWithConfig
-from joj3_config_generator.models.result import ParserConfig as ParserConfig
-from joj3_config_generator.models.result import ResultConfig as ResultConfig
-from joj3_config_generator.models.result import Stage as Stage
-from joj3_config_generator.models.result import StageConfig as StageConfig
-from joj3_config_generator.models.result import TeapotConfig as TeapotConfig
-from joj3_config_generator.models.task import TaskConfig as TaskConfig

joj3_config_generator/models/joj1.py

@@ -23,6 +23,6 @@ class Case(BaseModel):
     category: Optional[str] = None
 
 
-class JOJ1Config(BaseModel):
+class Config(BaseModel):
     languages: List[Language]
     cases: List[Case]

joj3_config_generator/models/repo.py

@@ -3,16 +3,16 @@ from typing import List, Optional
 from pydantic import BaseModel, Field
 
 
-class RepoFiles(BaseModel):
+class Files(BaseModel):
     whitelist_patterns: List[str]
     whitelist_file: Optional[str]
     required: List[str]
     immutable: List[str]
 
 
-class RepoConfig(BaseModel):
+class Config(BaseModel):
     teaching_team: List[str]
     max_size: float = Field(..., ge=0)
     release_tags: List[str]
-    files: RepoFiles
+    files: Files
     sandbox_token: str

joj3_config_generator/models/result.py

@@ -80,29 +80,29 @@ class OptionalCmd(BaseModel):
     )
 
 
-class Stage(BaseModel):
+class ExecutorWith(BaseModel):
-    name: str
-    group: str
-    executor: "ExecutorConfig"
-    parsers: List["ParserConfig"]
-
-
-class ExecutorWithConfig(BaseModel):
     default: Cmd
     cases: List[OptionalCmd]
 
 
-class ExecutorConfig(BaseModel):
+class Executor(BaseModel):
     name: str
-    with_: ExecutorWithConfig = Field(..., serialization_alias="with")
+    with_: ExecutorWith = Field(..., serialization_alias="with")
 
 
-class ParserConfig(BaseModel):
+class Parser(BaseModel):
     name: str
     with_: Dict[str, Any] = Field(..., serialization_alias="with")
 
 
-class StageConfig(BaseModel):
+class StageDetail(BaseModel):
+    name: str
+    group: str
+    executor: Executor
+    parsers: List[Parser]
+
+
+class Stage(BaseModel):
     sandbox_exec_server: str = Field(
         "172.17.0.1:5051", serialization_alias="sandboxExecServer"
     )
@@ -110,10 +110,10 @@ class StageConfig(BaseModel):
     output_path: str = Field(
         "/tmp/joj3_result.json", serialization_alias="outputPath"
     )  # nosec: B108
-    stages: List[Stage]
+    stages: List[StageDetail]
 
 
-class TeapotConfig(BaseModel):
+class Teapot(BaseModel):
     log_path: str = Field(
         "/home/tt/.cache/joint-teapot-debug.log", serialization_alias="logPath"
     )
@@ -127,9 +127,9 @@ class TeapotConfig(BaseModel):
     skip_failed_table: bool = Field(False, serialization_alias="skipFailedTable")
 
 
-class ResultConfig(BaseModel):
+class Config(BaseModel):
     name: str = "unknown"
     log_path: str = Field("", serialization_alias="logPath")
     expire_unix_timestamp: int = Field(-1, serialization_alias="expireUnixTimestamp")
-    stage: StageConfig
+    stage: Stage
-    teapot: TeapotConfig
+    teapot: Teapot

joj3_config_generator/models/task.py

@@ -31,7 +31,7 @@ class Release(BaseModel):
     deadline: Optional[datetime]  # RFC 3339 formatted date-time with offset
 
 
-class TaskConfig(BaseModel):
+class Config(BaseModel):
     task: str  # Task name (e.g., hw3 ex5)
     release: Release  # Release configuration
     stages: List[Stage]  # list of stage configurations

pyproject.toml

@@ -47,6 +47,7 @@ excludes = ["tests"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+xfail_strict=true
 
 [tool.mypy]
 plugins = ["pydantic.mypy"]

tests/convert/conftest.py

@@ -1,42 +0,0 @@
-import json
-import os
-from typing import Any, Dict, List, Tuple
-
-import pytest
-import rtoml
-
-from joj3_config_generator.models import RepoConfig, TaskConfig
-from tests.utils import safe_id
-
-
-def read_convert_files(root: str) -> Tuple[RepoConfig, TaskConfig, Dict[str, Any]]:
-    repo_toml_path = os.path.join(root, "repo.toml")
-    task_toml_path = os.path.join(root, "task.toml")
-    result_json_path = os.path.join(root, "task.json")
-    with open(repo_toml_path) as repo_file:
-        repo_toml = repo_file.read()
-    with open(task_toml_path) as task_file:
-        task_toml = task_file.read()
-    with open(result_json_path) as result_file:
-        expected_result: Dict[str, Any] = json.load(result_file)
-    repo_obj = rtoml.loads(repo_toml)
-    task_obj = rtoml.loads(task_toml)
-    return RepoConfig(**repo_obj), TaskConfig(**task_obj), expected_result
-
-
-def get_test_cases() -> List[Tuple[str, RepoConfig, TaskConfig, Dict[str, Any]]]:
-    test_cases = []
-    tests_dir = os.path.dirname(os.path.realpath(__file__))
-    for dir_name in os.listdir(tests_dir):
-        dir_path = os.path.join(tests_dir, dir_name)
-        if os.path.isdir(dir_path) and dir_name != "__pycache__":
-            repo, task, expected_result = read_convert_files(dir_path)
-            test_cases.append((dir_name, repo, task, expected_result))
-    return test_cases
-
-
-@pytest.fixture(params=get_test_cases(), ids=safe_id)
-def test_case(
-    request: pytest.FixtureRequest,
-) -> Tuple[RepoConfig, TaskConfig, Dict[str, Any]]:
-    return request.param[1:]

tests/convert/test_convert_cases.py

@@ -1,10 +1,8 @@
-from typing import Any, Dict, Tuple
-
 from joj3_config_generator.convert import convert
-from joj3_config_generator.models import RepoConfig, TaskConfig
+from tests.convert.utils import read_convert_files
 
 
-def test_convert(test_case: Tuple[RepoConfig, TaskConfig, Dict[str, Any]]) -> None:
+def test_basic() -> None:
-    repo, task, expected_result = test_case
+    repo, task, expected_result = read_convert_files("basic")
     result = convert(repo, task).model_dump(by_alias=True)
     assert result == expected_result

tests/convert/utils.py

@@ -0,0 +1,25 @@
+import json
+import os
+from typing import Any, Dict, Tuple
+
+import rtoml
+
+from joj3_config_generator.models import repo, task
+
+
+def read_convert_files(
+    case_name: str,
+) -> Tuple[repo.Config, task.Config, Dict[str, Any]]:
+    root = os.path.dirname(os.path.realpath(__file__))
+    repo_toml_path = os.path.join(root, case_name, "repo.toml")
+    task_toml_path = os.path.join(root, case_name, "task.toml")
+    result_json_path = os.path.join(root, case_name, "task.json")
+    with open(repo_toml_path) as repo_file:
+        repo_toml = repo_file.read()
+    with open(task_toml_path) as task_file:
+        task_toml = task_file.read()
+    with open(result_json_path) as result_file:
+        result: Dict[str, Any] = json.load(result_file)
+    repo_obj = rtoml.loads(repo_toml)
+    task_obj = rtoml.loads(task_toml)
+    return repo.Config(**repo_obj), task.Config(**task_obj), result

tests/convert_joj1/basic/task.yaml

@@ -0,0 +1,50 @@
+languages:
+  - language: cc
+    compiler_args: g++ -O2 -Wall -std=c++11 -o /out/main /in/main.cpp -lm
+    code_file: main.cpp # not necessary in tarball mode
+    execute_file: main # no need to set for an interpreter (will use config in langs.yaml)
+    execute_args: main # the execute args for all test cases
+  - language: c
+    compiler_args: gcc -O2 -Wall -std=c11 -o /out/main /in/main.c -lm
+    code_file: main.c # not necessary in tarball mode
+    execute_file: main # no need to set for an interpreter (will use config in langs.yaml)
+    execute_args: main # the execute args for all test cases
+default: &default
+  time: 1s
+  memory: 32m
+  score: 10
+cases:
+  - <<: *default
+    input: case0.in
+    output: case0.out
+    execute_args: -abcd --aaaa bbbb
+  - <<: *default
+    input: case1.in
+    output: case1.out
+  - <<: *default
+    input: case2.in
+    output: case2.out
+  - <<: *default
+    input: case3.in
+    output: case3.out
+  - <<: *default
+    input: case4.in
+    output: case4.out
+  - <<: *default
+    input: case5.in
+    output: case5.out
+  - <<: *default
+    input: case6.in
+    output: case6.out
+  - <<: *default
+    input: case7.in
+    output: case7.out
+    category: sentence
+  - <<: *default
+    input: case8.in
+    output: case8.out
+    category: sentence
+  - <<: *default
+    input: case9.in
+    output: case9.out
+    category: sentence

tests/convert_joj1/conftest.py

@@ -1,43 +0,0 @@
-import json
-import os
-from typing import Any, Dict, List, Tuple
-
-import pytest
-import rtoml
-import yaml
-
-from joj3_config_generator.models import JOJ1Config, TaskConfig
-from tests.utils import safe_id
-
-
-def read_convert_joj1_files(root: str) -> Tuple[JOJ1Config, TaskConfig, Dict[str, Any]]:
-    task_yaml_path = os.path.join(root, "task.yaml")
-    task_toml_path = os.path.join(root, "task.toml")
-    expected_json_path = os.path.join(root, "task.json")
-    with open(task_yaml_path) as repo_file:
-        task_yaml = repo_file.read()
-    with open(task_toml_path) as task_file:
-        task_toml = task_file.read()
-    with open(expected_json_path) as result_file:
-        expected_result: Dict[str, Any] = json.load(result_file)
-    joj1_obj = yaml.safe_load(task_yaml)
-    task_obj = rtoml.loads(task_toml)
-    return JOJ1Config(**joj1_obj), TaskConfig(**task_obj), expected_result
-
-
-def get_test_cases() -> List[Tuple[str, JOJ1Config, TaskConfig, Dict[str, Any]]]:
-    test_cases = []
-    tests_dir = os.path.dirname(os.path.realpath(__file__))
-    for dir_name in os.listdir(tests_dir):
-        dir_path = os.path.join(tests_dir, dir_name)
-        if os.path.isdir(dir_path) and dir_name != "__pycache__":
-            joj1, task, expected_result = read_convert_joj1_files(dir_path)
-            test_cases.append((dir_name, joj1, task, expected_result))
-    return test_cases
-
-
-@pytest.fixture(params=get_test_cases(), ids=safe_id)
-def test_case(
-    request: pytest.FixtureRequest,
-) -> Tuple[JOJ1Config, TaskConfig, Dict[str, Any]]:
-    return request.param[1:]

tests/convert_joj1/test_convert_joj1_cases.py

@@ -1,9 +1,11 @@
-from typing import Any, Dict, Tuple
+import pytest
 
-from joj3_config_generator.models import JOJ1Config, TaskConfig
+from joj3_config_generator.convert import convert_joj1
+from tests.convert_joj1.utils import read_convert_joj1_files
 
 
-def test_convert_joj1(test_case: Tuple[JOJ1Config, TaskConfig, Dict[str, Any]]) -> None:
+@pytest.mark.xfail
-    joj1, task, expected_result = test_case
+def test_basic() -> None:
-    result: Dict[str, Any] = {}
+    joj1, expected_result = read_convert_joj1_files("basic")
+    result = convert_joj1(joj1).model_dump(by_alias=True)
     assert result == expected_result

tests/convert_joj1/utils.py

@@ -0,0 +1,20 @@
+import os
+from typing import Any, Dict, Tuple
+
+import rtoml
+import yaml
+
+from joj3_config_generator.models import joj1
+
+
+def read_convert_joj1_files(case_name: str) -> Tuple[joj1.Config, Dict[str, Any]]:
+    root = os.path.dirname(os.path.realpath(__file__))
+    task_yaml_path = os.path.join(root, case_name, "task.yaml")
+    task_toml_path = os.path.join(root, case_name, "task.toml")
+    with open(task_yaml_path) as repo_file:
+        task_yaml = repo_file.read()
+    with open(task_toml_path) as task_file:
+        task_toml = task_file.read()
+    joj1_obj = yaml.safe_load(task_yaml)
+    task_obj = rtoml.loads(task_toml)
+    return joj1.Config(**joj1_obj), task_obj