From e4246ecf45e1d29e30208f192a6c5b9d24a057a6 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Tue, 18 Mar 2025 15:46:01 +0800 Subject: [PATCH 1/8] feat(diff): add auto detect testcases feature --- joj3_config_generator/transformers/task.py | 45 ++- tests/convert/clang-tidy/task.json | 155 ++++++++++- tests/convert/cppcheck/task.json | 155 ++++++++++- tests/convert/cpplint/task.json | 155 ++++++++++- tests/convert/diff/case0.in | 0 tests/convert/diff/case1.in | 0 tests/convert/diff/case2.in | 0 tests/convert/diff/case3.in | 0 tests/convert/diff/task.json | 291 +++++++++++++++++++- tests/convert/diff/task1/subtask1/task5.in | 0 tests/convert/diff/task1/subtask1/task5.out | 0 tests/convert/diff/task1/subtask1/task6.in | 0 tests/convert/diff/task1/subtask1/task6.out | 0 tests/convert/diff/task1/task1.in | 0 tests/convert/diff/task1/task1.out | 0 tests/convert/diff/task1/task2.in | 0 tests/convert/diff/task1/task2.out | 0 tests/convert/diff/task2/task3.in | 0 tests/convert/diff/task2/task3.out | 0 tests/convert/diff/task2/task4.in | 0 tests/convert/diff/task2/task4.out | 0 tests/convert/keyword/task.json | 155 ++++++++++- tests/convert/result-detail/task.json | 155 ++++++++++- 23 files changed, 1100 insertions(+), 11 deletions(-) create mode 100644 tests/convert/diff/case0.in create mode 100644 tests/convert/diff/case1.in create mode 100644 tests/convert/diff/case2.in create mode 100644 tests/convert/diff/case3.in create mode 100644 tests/convert/diff/task1/subtask1/task5.in create mode 100644 tests/convert/diff/task1/subtask1/task5.out create mode 100644 tests/convert/diff/task1/subtask1/task6.in create mode 100644 tests/convert/diff/task1/subtask1/task6.out create mode 100644 tests/convert/diff/task1/task1.in create mode 100644 tests/convert/diff/task1/task1.out create mode 100644 tests/convert/diff/task1/task2.in create mode 100644 tests/convert/diff/task1/task2.out create mode 100644 tests/convert/diff/task2/task3.in create mode 100644 tests/convert/diff/task2/task3.out create mode 100644 tests/convert/diff/task2/task4.in create mode 100644 tests/convert/diff/task2/task4.out diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 04158b1..066b82a 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -2,7 +2,7 @@ import re import shlex from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple +from typing import Any, Callable, Dict, List, Set, Tuple from joj3_config_generator.models import result, task from joj3_config_generator.models.common import Memory, Time @@ -32,7 +32,8 @@ def get_conf_stage( parser_handler_map = get_parser_handler_map( task_stage, conf_stage.executor, - JOJ3_CONFIG_ROOT / task_conf.path.parent, + task_conf.root, + task_conf.path, ) for idx, parser in enumerate(task_stage.parsers): if parser not in parser_handler_map: @@ -45,7 +46,8 @@ def get_conf_stage( def get_parser_handler_map( task_stage: task.Stage, diff_executor_config: result.Executor, - base_dir: Path, + task_root: Path, + task_path: Path, ) -> Dict[ParserEnum, Tuple[Callable[[Any, result.Parser], None], Any]]: return { ParserEnum.CLANG_TIDY: (fix_keyword, task_stage.clangtidy), @@ -61,7 +63,8 @@ def get_parser_handler_map( fix_diff, task_stage=task_stage, diff_executor_config=diff_executor_config, - base_dir=base_dir, + task_root=task_root, + task_path=task_path, ), task_stage.diff, ), @@ -159,13 +162,18 @@ def fix_diff( diff_parser_config: result.Parser, task_stage: task.Stage, diff_executor_config: result.Executor, - base_dir: Path, + task_root: Path, + task_path: Path, ) -> None: + base_dir = JOJ3_CONFIG_ROOT / task_path.parent valid_cases = ( (case, task_stage.cases[case]) for case in task_stage.cases if case not in task_stage.skip and case in task_stage.cases ) + testcases = get_testcases(task_root, task_path) + # TODO: better filter strategy + default_cases = testcases.difference(task_stage.cases) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -195,5 +203,32 @@ def fix_diff( ] ) ) + for case in default_cases: + stage_cases.append( + result.OptionalCmd(stdin=result.LocalFile(src=str(base_dir / f"{case}.in"))) + ) + parser_cases.append( + result.DiffCasesConfig( + outputs=[ + result.DiffOutputConfig( + # TODO: how to balance a good score strategy + score=5, # default score + file_name="stdout", + answer_path=str(base_dir / f"{case}.out"), + ) + ] + ) + ) diff_executor_config.with_.cases = stage_cases diff_parser_config.with_ = result.DiffConfig(name="diff", cases=parser_cases) + + +def get_testcases( + task_root: Path, task_path: Path +) -> Set[str]: # basedir here should be task_conf.root / task_conf.path + testcases = set() + for testcases_path in (task_root / task_path).parent.glob("**/*.in"): + testcases.add( + str(testcases_path.relative_to(task_path.parent)).removesuffix(".in") + ) + return testcases diff --git a/tests/convert/clang-tidy/task.json b/tests/convert/clang-tidy/task.json index 5c9b30e..2c207ef 100644 --- a/tests/convert/clang-tidy/task.json +++ b/tests/convert/clang-tidy/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Clang-tidy", "group": "cq", @@ -135,6 +221,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/cppcheck/task.json b/tests/convert/cppcheck/task.json index bc2d317..144796d 100644 --- a/tests/convert/cppcheck/task.json +++ b/tests/convert/cppcheck/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Cppcheck", "group": "cq", @@ -105,6 +191,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/cpplint/task.json b/tests/convert/cpplint/task.json index 8fdb56f..6ffe678 100644 --- a/tests/convert/cpplint/task.json +++ b/tests/convert/cpplint/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Cpplint", "group": "cq", @@ -107,6 +193,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/diff/case0.in b/tests/convert/diff/case0.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case1.in b/tests/convert/diff/case1.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case2.in b/tests/convert/diff/case2.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case3.in b/tests/convert/diff/case3.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index d50060f..c4bb518 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[joj] ex2-asan", "group": "joj", @@ -79,6 +165,46 @@ "clockLimit": 4000000000, "memoryLimit": 4194304, "procLimit": 50 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/task1.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/task3.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case3.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/task4.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case2.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/task2.in" + } } ] } @@ -112,6 +238,102 @@ "compareSpace": false } ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/task1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/task3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] } ] } @@ -133,6 +355,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/diff/task1/subtask1/task5.in b/tests/convert/diff/task1/subtask1/task5.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/task5.out b/tests/convert/diff/task1/subtask1/task5.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/task6.in b/tests/convert/diff/task1/subtask1/task6.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/task6.out b/tests/convert/diff/task1/subtask1/task6.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task1.in b/tests/convert/diff/task1/task1.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task1.out b/tests/convert/diff/task1/task1.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task2.in b/tests/convert/diff/task1/task2.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task2.out b/tests/convert/diff/task1/task2.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task3.in b/tests/convert/diff/task2/task3.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task3.out b/tests/convert/diff/task2/task3.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task4.in b/tests/convert/diff/task2/task4.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task4.out b/tests/convert/diff/task2/task4.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/keyword/task.json b/tests/convert/keyword/task.json index 355db46..88f6a61 100644 --- a/tests/convert/keyword/task.json +++ b/tests/convert/keyword/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Filelength", "group": "cq", @@ -104,6 +190,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/result-detail/task.json b/tests/convert/result-detail/task.json index cd060e1..8daa140 100644 --- a/tests/convert/result-detail/task.json +++ b/tests/convert/result-detail/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Filelength", "group": "cq", @@ -86,6 +172,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } -- 2.30.2 From 890120c4796675df91e8b878703be8d7f457a9ff Mon Sep 17 00:00:00 2001 From: jon-lee Date: Tue, 18 Mar 2025 15:59:26 +0800 Subject: [PATCH 2/8] fix(diff): use list instead of set to past tests --- joj3_config_generator/transformers/task.py | 12 +- tests/convert/clang-tidy/task.json | 155 +----------- tests/convert/cppcheck/task.json | 155 +----------- tests/convert/cpplint/task.json | 155 +----------- tests/convert/diff/task.json | 265 +++++---------------- tests/convert/keyword/task.json | 155 +----------- tests/convert/result-detail/task.json | 155 +----------- 7 files changed, 68 insertions(+), 984 deletions(-) diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 066b82a..4c91a1d 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -173,7 +173,7 @@ def fix_diff( ) testcases = get_testcases(task_root, task_path) # TODO: better filter strategy - default_cases = testcases.difference(task_stage.cases) + default_cases = list(filter(lambda x: x not in task_stage.cases, testcases)) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -225,10 +225,12 @@ def fix_diff( def get_testcases( task_root: Path, task_path: Path -) -> Set[str]: # basedir here should be task_conf.root / task_conf.path - testcases = set() +) -> List[str]: # basedir here should be task_conf.root / task_conf.path + testcases = [] for testcases_path in (task_root / task_path).parent.glob("**/*.in"): - testcases.add( - str(testcases_path.relative_to(task_path.parent)).removesuffix(".in") + testcases.append( + str( + testcases_path.relative_to((task_root / task_path).parent) + ).removesuffix(".in") ) return testcases diff --git a/tests/convert/clang-tidy/task.json b/tests/convert/clang-tidy/task.json index 2c207ef..5c9b30e 100644 --- a/tests/convert/clang-tidy/task.json +++ b/tests/convert/clang-tidy/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Clang-tidy", "group": "cq", @@ -221,73 +135,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/cppcheck/task.json b/tests/convert/cppcheck/task.json index 144796d..bc2d317 100644 --- a/tests/convert/cppcheck/task.json +++ b/tests/convert/cppcheck/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Cppcheck", "group": "cq", @@ -191,73 +105,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/cpplint/task.json b/tests/convert/cpplint/task.json index 6ffe678..8fdb56f 100644 --- a/tests/convert/cpplint/task.json +++ b/tests/convert/cpplint/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Cpplint", "group": "cq", @@ -193,73 +107,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index c4bb518..4e93004 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[joj] ex2-asan", "group": "joj", @@ -166,6 +80,16 @@ "memoryLimit": 4194304, "procLimit": 50 }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case2.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case3.in" + } + }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/task1.in" @@ -173,7 +97,12 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + "src": "/home/tt/.config/joj/diff/task1/task2.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/task4.in" } }, { @@ -188,22 +117,7 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case3.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task4.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case2.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task1/task2.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" } } ] @@ -239,6 +153,30 @@ } ] }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -256,7 +194,19 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -292,43 +242,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case3.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case2.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -355,73 +269,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/keyword/task.json b/tests/convert/keyword/task.json index 88f6a61..355db46 100644 --- a/tests/convert/keyword/task.json +++ b/tests/convert/keyword/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Filelength", "group": "cq", @@ -190,73 +104,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/result-detail/task.json b/tests/convert/result-detail/task.json index 8daa140..cd060e1 100644 --- a/tests/convert/result-detail/task.json +++ b/tests/convert/result-detail/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Filelength", "group": "cq", @@ -172,73 +86,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } -- 2.30.2 From a4717268ec48973373d9e6fb767d73d4aa528e9f Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 19 Mar 2025 12:56:07 +0800 Subject: [PATCH 3/8] fix: pytest --- tests/convert/diff/task.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 9312c65..fde2dad 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -74,9 +74,7 @@ }, "cpuLimit": 2000000000, "clockLimit": 4000000000, -<<<<<<< HEAD - "memoryLimit": 4194304, - "procLimit": 50 + "memoryLimit": 4194304 }, { "stdin": { @@ -117,9 +115,6 @@ "stdin": { "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" } -======= - "memoryLimit": 4194304 ->>>>>>> master } ] } -- 2.30.2 From e775992e5da4a53c131599638cd16b2d174afef5 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Wed, 19 Mar 2025 02:13:46 -0400 Subject: [PATCH 4/8] fix: sort default cases --- joj3_config_generator/transformers/task.py | 8 +-- tests/convert/diff/task.json | 72 +++++++++++----------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 421bc90..6eebe69 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -173,7 +173,7 @@ def fix_diff( ) testcases = get_testcases(task_root, task_path) # TODO: better filter strategy - default_cases = list(filter(lambda x: x not in task_stage.cases, testcases)) + default_cases = sorted(testcases.difference(task_stage.cases)) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -233,10 +233,10 @@ def fix_diff( def get_testcases( task_root: Path, task_path: Path -) -> List[str]: # basedir here should be task_conf.root / task_conf.path - testcases = [] +) -> Set[str]: # basedir here should be task_conf.root / task_conf.path + testcases = set() for testcases_path in (task_root / task_path).parent.glob("**/*.in"): - testcases.append( + testcases.add( str( testcases_path.relative_to((task_root / task_path).parent) ).removesuffix(".in") diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index fde2dad..4bfc14b 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -86,6 +86,16 @@ "src": "/home/tt/.config/joj/diff/case3.in" } }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" + } + }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/task1.in" @@ -96,11 +106,6 @@ "src": "/home/tt/.config/joj/diff/task1/task2.in" } }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task4.in" - } - }, { "stdin": { "src": "/home/tt/.config/joj/diff/task2/task3.in" @@ -108,12 +113,7 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + "src": "/home/tt/.config/joj/diff/task2/task4.in" } } ] @@ -173,6 +173,30 @@ } ] }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -197,18 +221,6 @@ } ] }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, { "outputs": [ { @@ -226,19 +238,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false -- 2.30.2 From a8a7fd47a095395d4e5943f8107712992eb41943 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 14 May 2025 21:27:33 +0800 Subject: [PATCH 5/8] feat: support for subfolder change param --- joj3_config_generator/transformers/task.py | 13 +- .../{task1/subtask1/task5.in => case9.in} | 0 .../{task1/subtask1/task5.out => case9.out} | 0 tests/convert/diff/task.json | 174 +++++++++++------- tests/convert/diff/task.toml | 10 +- .../task1/{subtask1/task6.in => case4.in} | 0 .../task1/{subtask1/task6.out => case4.out} | 0 .../convert/diff/task1/{task1.in => case5.in} | 0 .../diff/task1/{task1.out => case5.out} | 0 .../task1/{task2.in => subtask1/case10.in} | 0 .../task1/{task2.out => subtask1/case10.out} | 0 .../task3.in => task1/subtask1/case11.in} | 0 .../task3.out => task1/subtask1/case11.out} | 0 .../convert/diff/task2/{task4.in => case6.in} | 0 .../diff/task2/{task4.out => case6.out} | 0 tests/convert/diff/task2/case7.in | 0 tests/convert/diff/task2/case8.out | 0 17 files changed, 124 insertions(+), 73 deletions(-) rename tests/convert/diff/{task1/subtask1/task5.in => case9.in} (100%) rename tests/convert/diff/{task1/subtask1/task5.out => case9.out} (100%) rename tests/convert/diff/task1/{subtask1/task6.in => case4.in} (100%) rename tests/convert/diff/task1/{subtask1/task6.out => case4.out} (100%) rename tests/convert/diff/task1/{task1.in => case5.in} (100%) rename tests/convert/diff/task1/{task1.out => case5.out} (100%) rename tests/convert/diff/task1/{task2.in => subtask1/case10.in} (100%) rename tests/convert/diff/task1/{task2.out => subtask1/case10.out} (100%) rename tests/convert/diff/{task2/task3.in => task1/subtask1/case11.in} (100%) rename tests/convert/diff/{task2/task3.out => task1/subtask1/case11.out} (100%) rename tests/convert/diff/task2/{task4.in => case6.in} (100%) rename tests/convert/diff/task2/{task4.out => case6.out} (100%) create mode 100644 tests/convert/diff/task2/case7.in create mode 100644 tests/convert/diff/task2/case8.out diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 6eebe69..0a04947 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -169,11 +169,17 @@ def fix_diff( valid_cases = ( (case, task_stage.cases[case]) for case in task_stage.cases - if case not in task_stage.skip and case in task_stage.cases + if case not in task_stage.skip ) testcases = get_testcases(task_root, task_path) - # TODO: better filter strategy - default_cases = sorted(testcases.difference(task_stage.cases)) + default_cases = sorted( + [ + case + for case in testcases + if any(case.endswith(other) for other in task_stage.cases) + ] + ) + print(default_cases) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -219,7 +225,6 @@ def fix_diff( parser_case = result.DiffCasesConfig( outputs=[ result.DiffOutputConfig( - # TODO: how to balance a good score strategy score=5, # default score file_name="stdout", answer_path=str(base_dir / f"{case}.out"), diff --git a/tests/convert/diff/task1/subtask1/task5.in b/tests/convert/diff/case9.in similarity index 100% rename from tests/convert/diff/task1/subtask1/task5.in rename to tests/convert/diff/case9.in diff --git a/tests/convert/diff/task1/subtask1/task5.out b/tests/convert/diff/case9.out similarity index 100% rename from tests/convert/diff/task1/subtask1/task5.out rename to tests/convert/diff/case9.out diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 4bfc14b..a7d25c7 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -78,42 +78,56 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case2.in" + "src": "/home/tt/.config/joj/diff/case9.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case11.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case10.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case5.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case0.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case3.in" + "src": "/home/tt/.config/joj/diff/case1.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + "src": "/home/tt/.config/joj/diff/case9.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" + "src": "/home/tt/.config/joj/diff/task1/case5.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/task1.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/task2.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task3.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task4.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" } } ] @@ -125,6 +139,78 @@ "with": { "name": "diff", "cases": [ + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case0.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 123214122421, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 1232131, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case9.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 92321, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case11.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 823131, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case10.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 2590, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -154,7 +240,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case2.out", + "answerPath": "/home/tt/.config/joj/diff/case9.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -166,7 +252,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case3.out", + "answerPath": "/home/tt/.config/joj/diff/task1/case5.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -178,7 +264,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -190,55 +276,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/task1.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task3.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index abdd10c..f4f283d 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -23,10 +23,18 @@ case0.diff.output.ignore_spaces = true #case0.command = "./h7/build/ex2" case0.in = "case0.in" -case1.diff.output.score = 5 +case1.diff.output.score = 123214122421 case1.limit.cpu = "2s" case1.limit.mem = "4m" case1.diff.output.ignore_spaces = true #case1.limit.stdout = 8 #case1.command = "./h7/build/ex2" case1.in = "case1.in" + +case9.diff.output.score = 1232131 + +case11.diff.output.score = 92321 + +case10.diff.output.score = 823131 + +case5.diff.output.score = 2590 diff --git a/tests/convert/diff/task1/subtask1/task6.in b/tests/convert/diff/task1/case4.in similarity index 100% rename from tests/convert/diff/task1/subtask1/task6.in rename to tests/convert/diff/task1/case4.in diff --git a/tests/convert/diff/task1/subtask1/task6.out b/tests/convert/diff/task1/case4.out similarity index 100% rename from tests/convert/diff/task1/subtask1/task6.out rename to tests/convert/diff/task1/case4.out diff --git a/tests/convert/diff/task1/task1.in b/tests/convert/diff/task1/case5.in similarity index 100% rename from tests/convert/diff/task1/task1.in rename to tests/convert/diff/task1/case5.in diff --git a/tests/convert/diff/task1/task1.out b/tests/convert/diff/task1/case5.out similarity index 100% rename from tests/convert/diff/task1/task1.out rename to tests/convert/diff/task1/case5.out diff --git a/tests/convert/diff/task1/task2.in b/tests/convert/diff/task1/subtask1/case10.in similarity index 100% rename from tests/convert/diff/task1/task2.in rename to tests/convert/diff/task1/subtask1/case10.in diff --git a/tests/convert/diff/task1/task2.out b/tests/convert/diff/task1/subtask1/case10.out similarity index 100% rename from tests/convert/diff/task1/task2.out rename to tests/convert/diff/task1/subtask1/case10.out diff --git a/tests/convert/diff/task2/task3.in b/tests/convert/diff/task1/subtask1/case11.in similarity index 100% rename from tests/convert/diff/task2/task3.in rename to tests/convert/diff/task1/subtask1/case11.in diff --git a/tests/convert/diff/task2/task3.out b/tests/convert/diff/task1/subtask1/case11.out similarity index 100% rename from tests/convert/diff/task2/task3.out rename to tests/convert/diff/task1/subtask1/case11.out diff --git a/tests/convert/diff/task2/task4.in b/tests/convert/diff/task2/case6.in similarity index 100% rename from tests/convert/diff/task2/task4.in rename to tests/convert/diff/task2/case6.in diff --git a/tests/convert/diff/task2/task4.out b/tests/convert/diff/task2/case6.out similarity index 100% rename from tests/convert/diff/task2/task4.out rename to tests/convert/diff/task2/case6.out diff --git a/tests/convert/diff/task2/case7.in b/tests/convert/diff/task2/case7.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case8.out b/tests/convert/diff/task2/case8.out new file mode 100644 index 0000000..e69de29 -- 2.30.2 From 391d61d00638703c83b57982aac7ad7227deb44a Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 15 May 2025 17:08:55 +0800 Subject: [PATCH 6/8] style: use const --- joj3_config_generator/models/const.py | 1 + joj3_config_generator/transformers/task.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py index 95a6706..33de35b 100644 --- a/joj3_config_generator/models/const.py +++ b/joj3_config_generator/models/const.py @@ -5,6 +5,7 @@ from joj3_config_generator.models.common import Memory, Time DEFAULT_CPU_LIMIT = Time("1s") DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_FILE_LIMIT = Memory("32m") +DEFAULT_CASE_SCORE = 5 JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 0a04947..b947247 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -4,7 +4,7 @@ from functools import partial from pathlib import Path from typing import Any, Callable, Dict, List, Set, Tuple -from joj3_config_generator.models import result, task +from joj3_config_generator.models import const, result, task from joj3_config_generator.models.common import Memory, Time from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT from joj3_config_generator.models.task import Parser as ParserEnum @@ -179,7 +179,6 @@ def fix_diff( if any(case.endswith(other) for other in task_stage.cases) ] ) - print(default_cases) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -225,7 +224,7 @@ def fix_diff( parser_case = result.DiffCasesConfig( outputs=[ result.DiffOutputConfig( - score=5, # default score + score=const.DEFAULT_CASE_SCORE, file_name="stdout", answer_path=str(base_dir / f"{case}.out"), ) -- 2.30.2 From 26df677b3688db16cd825f01c0b1a46139b531b9 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 21 May 2025 22:36:12 +0800 Subject: [PATCH 7/8] fix(diff): cases not properly detected --- joj3_config_generator/transformers/task.py | 27 ++-- tests/convert/basic/task.json | 44 +----- tests/convert/diff/task.json | 172 ++++++++++----------- tests/convert/diff/task2/case7.out | 0 tests/convert/diff/task2/case8.in | 0 5 files changed, 105 insertions(+), 138 deletions(-) create mode 100644 tests/convert/diff/task2/case7.out create mode 100644 tests/convert/diff/task2/case8.in diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index b947247..4e9bf7e 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -166,19 +166,25 @@ def fix_diff( task_path: Path, ) -> None: base_dir = JOJ3_CONFIG_ROOT / task_path.parent - valid_cases = ( - (case, task_stage.cases[case]) - for case in task_stage.cases - if case not in task_stage.skip - ) + # all intended testcases that is detected testcases = get_testcases(task_root, task_path) + # all testcases that is not specified in the toml config default_cases = sorted( - [ - case - for case in testcases - if any(case.endswith(other) for other in task_stage.cases) - ] + testcases.difference( + [ + casei + for casei in testcases + if any(casei.endswith(casej) for casej in task_stage.cases) + ] + ) ) + # those in toml config that is not skiped + valid_cases = [ + (casej, task_stage.cases[casei]) + for casei in task_stage.cases + for casej in testcases + if (casei not in task_stage.skip and casej.endswith(casei)) + ] stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -216,6 +222,7 @@ def fix_diff( ] ) parser_cases.append(parser_case) + for case in default_cases: cmd = result.OptionalCmd( stdin=result.LocalFile(src=str(base_dir / f"{case}.in")) diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 035c7f7..9f7929d 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -705,22 +705,7 @@ "dataSegmentLimit": false, "addressSpaceLimit": false }, - "cases": [ - { - "stdin": { - "src": "/home/tt/.config/joj/basic/case0.in" - }, - "cpuLimit": 500000000, - "clockLimit": 1000000000, - "memoryLimit": 5242880 - }, - { - "stdin": { - "src": "/home/tt/.config/joj/basic/case1.in" - }, - "memoryLimit": 5242880 - } - ] + "cases": [] } }, "parsers": [ @@ -728,32 +713,7 @@ "name": "diff", "with": { "name": "diff", - "cases": [ - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/basic/case0.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/basic/case1.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - } - ] + "cases": [] } }, { diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index a7d25c7..9c15abd 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -84,50 +84,50 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case11.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" }, "memoryLimit": 268435456 }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case10.in" - }, - "memoryLimit": 268435456 - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case5.in" - }, - "memoryLimit": 268435456 - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case0.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case1.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case9.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task1/case5.in" - } - }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/case5.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case2.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" + "src": "/home/tt/.config/joj/diff/case3.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/case4.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/case6.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/case7.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/case8.in" } } ] @@ -180,7 +180,7 @@ { "score": 92321, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case11.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -192,7 +192,7 @@ { "score": 823131, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case10.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -204,54 +204,6 @@ { "score": 2590, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case5.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case0.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case1.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case9.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/task1/case5.out", "forceQuitOnDiff": false, "alwaysHide": false, @@ -264,7 +216,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", + "answerPath": "/home/tt/.config/joj/diff/case2.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -276,7 +228,55 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", + "answerPath": "/home/tt/.config/joj/diff/case3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/case4.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/case6.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/case7.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/case8.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false diff --git a/tests/convert/diff/task2/case7.out b/tests/convert/diff/task2/case7.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case8.in b/tests/convert/diff/task2/case8.in new file mode 100644 index 0000000..e69de29 -- 2.30.2 From 2f1686c1e32c8e9b5f7e7eda2d9ced02f0a4e528 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 21 May 2025 22:43:00 +0800 Subject: [PATCH 8/8] test: case.command --- tests/convert/diff/task.json | 6 ++++++ tests/convert/diff/task.toml | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 11d1c34..63a4607 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -65,12 +65,18 @@ }, "cases": [ { + "args": [ + "./h7/build/ex2" + ], "stdin": { "src": "/home/tt/.config/joj/diff/case0.in" }, "memoryLimit": 2097152 }, { + "args": [ + "./h7/build/ex2" + ], "stdin": { "src": "/home/tt/.config/joj/diff/case1.in" }, diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index f4f283d..4272f97 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -19,17 +19,14 @@ case0.diff.output.score = 5 case0.limit.cpu = "1s" case0.limit.mem = "2m" case0.diff.output.ignore_spaces = true -#case0.limit.stdout = 8 -#case0.command = "./h7/build/ex2" +case0.command = "./h7/build/ex2" case0.in = "case0.in" case1.diff.output.score = 123214122421 case1.limit.cpu = "2s" case1.limit.mem = "4m" case1.diff.output.ignore_spaces = true -#case1.limit.stdout = 8 -#case1.command = "./h7/build/ex2" -case1.in = "case1.in" +case1.command = "./h7/build/ex2" case9.diff.output.score = 1232131 -- 2.30.2