From e4246ecf45e1d29e30208f192a6c5b9d24a057a6 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Tue, 18 Mar 2025 15:46:01 +0800 Subject: [PATCH 01/18] feat(diff): add auto detect testcases feature --- joj3_config_generator/transformers/task.py | 45 ++- tests/convert/clang-tidy/task.json | 155 ++++++++++- tests/convert/cppcheck/task.json | 155 ++++++++++- tests/convert/cpplint/task.json | 155 ++++++++++- tests/convert/diff/case0.in | 0 tests/convert/diff/case1.in | 0 tests/convert/diff/case2.in | 0 tests/convert/diff/case3.in | 0 tests/convert/diff/task.json | 291 +++++++++++++++++++- tests/convert/diff/task1/subtask1/task5.in | 0 tests/convert/diff/task1/subtask1/task5.out | 0 tests/convert/diff/task1/subtask1/task6.in | 0 tests/convert/diff/task1/subtask1/task6.out | 0 tests/convert/diff/task1/task1.in | 0 tests/convert/diff/task1/task1.out | 0 tests/convert/diff/task1/task2.in | 0 tests/convert/diff/task1/task2.out | 0 tests/convert/diff/task2/task3.in | 0 tests/convert/diff/task2/task3.out | 0 tests/convert/diff/task2/task4.in | 0 tests/convert/diff/task2/task4.out | 0 tests/convert/keyword/task.json | 155 ++++++++++- tests/convert/result-detail/task.json | 155 ++++++++++- 23 files changed, 1100 insertions(+), 11 deletions(-) create mode 100644 tests/convert/diff/case0.in create mode 100644 tests/convert/diff/case1.in create mode 100644 tests/convert/diff/case2.in create mode 100644 tests/convert/diff/case3.in create mode 100644 tests/convert/diff/task1/subtask1/task5.in create mode 100644 tests/convert/diff/task1/subtask1/task5.out create mode 100644 tests/convert/diff/task1/subtask1/task6.in create mode 100644 tests/convert/diff/task1/subtask1/task6.out create mode 100644 tests/convert/diff/task1/task1.in create mode 100644 tests/convert/diff/task1/task1.out create mode 100644 tests/convert/diff/task1/task2.in create mode 100644 tests/convert/diff/task1/task2.out create mode 100644 tests/convert/diff/task2/task3.in create mode 100644 tests/convert/diff/task2/task3.out create mode 100644 tests/convert/diff/task2/task4.in create mode 100644 tests/convert/diff/task2/task4.out diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 04158b1..066b82a 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -2,7 +2,7 @@ import re import shlex from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple +from typing import Any, Callable, Dict, List, Set, Tuple from joj3_config_generator.models import result, task from joj3_config_generator.models.common import Memory, Time @@ -32,7 +32,8 @@ def get_conf_stage( parser_handler_map = get_parser_handler_map( task_stage, conf_stage.executor, - JOJ3_CONFIG_ROOT / task_conf.path.parent, + task_conf.root, + task_conf.path, ) for idx, parser in enumerate(task_stage.parsers): if parser not in parser_handler_map: @@ -45,7 +46,8 @@ def get_conf_stage( def get_parser_handler_map( task_stage: task.Stage, diff_executor_config: result.Executor, - base_dir: Path, + task_root: Path, + task_path: Path, ) -> Dict[ParserEnum, Tuple[Callable[[Any, result.Parser], None], Any]]: return { ParserEnum.CLANG_TIDY: (fix_keyword, task_stage.clangtidy), @@ -61,7 +63,8 @@ def get_parser_handler_map( fix_diff, task_stage=task_stage, diff_executor_config=diff_executor_config, - base_dir=base_dir, + task_root=task_root, + task_path=task_path, ), task_stage.diff, ), @@ -159,13 +162,18 @@ def fix_diff( diff_parser_config: result.Parser, task_stage: task.Stage, diff_executor_config: result.Executor, - base_dir: Path, + task_root: Path, + task_path: Path, ) -> None: + base_dir = JOJ3_CONFIG_ROOT / task_path.parent valid_cases = ( (case, task_stage.cases[case]) for case in task_stage.cases if case not in task_stage.skip and case in task_stage.cases ) + testcases = get_testcases(task_root, task_path) + # TODO: better filter strategy + default_cases = testcases.difference(task_stage.cases) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -195,5 +203,32 @@ def fix_diff( ] ) ) + for case in default_cases: + stage_cases.append( + result.OptionalCmd(stdin=result.LocalFile(src=str(base_dir / f"{case}.in"))) + ) + parser_cases.append( + result.DiffCasesConfig( + outputs=[ + result.DiffOutputConfig( + # TODO: how to balance a good score strategy + score=5, # default score + file_name="stdout", + answer_path=str(base_dir / f"{case}.out"), + ) + ] + ) + ) diff_executor_config.with_.cases = stage_cases diff_parser_config.with_ = result.DiffConfig(name="diff", cases=parser_cases) + + +def get_testcases( + task_root: Path, task_path: Path +) -> Set[str]: # basedir here should be task_conf.root / task_conf.path + testcases = set() + for testcases_path in (task_root / task_path).parent.glob("**/*.in"): + testcases.add( + str(testcases_path.relative_to(task_path.parent)).removesuffix(".in") + ) + return testcases diff --git a/tests/convert/clang-tidy/task.json b/tests/convert/clang-tidy/task.json index 5c9b30e..2c207ef 100644 --- a/tests/convert/clang-tidy/task.json +++ b/tests/convert/clang-tidy/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Clang-tidy", "group": "cq", @@ -135,6 +221,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/cppcheck/task.json b/tests/convert/cppcheck/task.json index bc2d317..144796d 100644 --- a/tests/convert/cppcheck/task.json +++ b/tests/convert/cppcheck/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Cppcheck", "group": "cq", @@ -105,6 +191,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/cpplint/task.json b/tests/convert/cpplint/task.json index 8fdb56f..6ffe678 100644 --- a/tests/convert/cpplint/task.json +++ b/tests/convert/cpplint/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Cpplint", "group": "cq", @@ -107,6 +193,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/diff/case0.in b/tests/convert/diff/case0.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case1.in b/tests/convert/diff/case1.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case2.in b/tests/convert/diff/case2.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case3.in b/tests/convert/diff/case3.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index d50060f..c4bb518 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[joj] ex2-asan", "group": "joj", @@ -79,6 +165,46 @@ "clockLimit": 4000000000, "memoryLimit": 4194304, "procLimit": 50 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/task1.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/task3.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case3.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/task4.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case2.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/task2.in" + } } ] } @@ -112,6 +238,102 @@ "compareSpace": false } ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/task1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/task3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] } ] } @@ -133,6 +355,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/diff/task1/subtask1/task5.in b/tests/convert/diff/task1/subtask1/task5.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/task5.out b/tests/convert/diff/task1/subtask1/task5.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/task6.in b/tests/convert/diff/task1/subtask1/task6.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/task6.out b/tests/convert/diff/task1/subtask1/task6.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task1.in b/tests/convert/diff/task1/task1.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task1.out b/tests/convert/diff/task1/task1.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task2.in b/tests/convert/diff/task1/task2.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/task2.out b/tests/convert/diff/task1/task2.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task3.in b/tests/convert/diff/task2/task3.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task3.out b/tests/convert/diff/task2/task3.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task4.in b/tests/convert/diff/task2/task4.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/task4.out b/tests/convert/diff/task2/task4.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/keyword/task.json b/tests/convert/keyword/task.json index 355db46..88f6a61 100644 --- a/tests/convert/keyword/task.json +++ b/tests/convert/keyword/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Filelength", "group": "cq", @@ -104,6 +190,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } diff --git a/tests/convert/result-detail/task.json b/tests/convert/result-detail/task.json index cd060e1..8daa140 100644 --- a/tests/convert/result-detail/task.json +++ b/tests/convert/result-detail/task.json @@ -10,6 +10,92 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ + { + "name": "Health Check", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--group-config", + "" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 1 + } + }, + { + "name": "debug", + "with": { + "score": 0 + } + } + ] + }, { "name": "[cq] Filelength", "group": "cq", @@ -86,6 +172,73 @@ } ], "preStages": [], - "postStages": [] + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "Nuvole-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" + } + } + ] + } + ] } } -- 2.30.2 From 890120c4796675df91e8b878703be8d7f457a9ff Mon Sep 17 00:00:00 2001 From: jon-lee Date: Tue, 18 Mar 2025 15:59:26 +0800 Subject: [PATCH 02/18] fix(diff): use list instead of set to past tests --- joj3_config_generator/transformers/task.py | 12 +- tests/convert/clang-tidy/task.json | 155 +----------- tests/convert/cppcheck/task.json | 155 +----------- tests/convert/cpplint/task.json | 155 +----------- tests/convert/diff/task.json | 265 +++++---------------- tests/convert/keyword/task.json | 155 +----------- tests/convert/result-detail/task.json | 155 +----------- 7 files changed, 68 insertions(+), 984 deletions(-) diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 066b82a..4c91a1d 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -173,7 +173,7 @@ def fix_diff( ) testcases = get_testcases(task_root, task_path) # TODO: better filter strategy - default_cases = testcases.difference(task_stage.cases) + default_cases = list(filter(lambda x: x not in task_stage.cases, testcases)) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -225,10 +225,12 @@ def fix_diff( def get_testcases( task_root: Path, task_path: Path -) -> Set[str]: # basedir here should be task_conf.root / task_conf.path - testcases = set() +) -> List[str]: # basedir here should be task_conf.root / task_conf.path + testcases = [] for testcases_path in (task_root / task_path).parent.glob("**/*.in"): - testcases.add( - str(testcases_path.relative_to(task_path.parent)).removesuffix(".in") + testcases.append( + str( + testcases_path.relative_to((task_root / task_path).parent) + ).removesuffix(".in") ) return testcases diff --git a/tests/convert/clang-tidy/task.json b/tests/convert/clang-tidy/task.json index 2c207ef..5c9b30e 100644 --- a/tests/convert/clang-tidy/task.json +++ b/tests/convert/clang-tidy/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Clang-tidy", "group": "cq", @@ -221,73 +135,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/cppcheck/task.json b/tests/convert/cppcheck/task.json index 144796d..bc2d317 100644 --- a/tests/convert/cppcheck/task.json +++ b/tests/convert/cppcheck/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Cppcheck", "group": "cq", @@ -191,73 +105,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/cpplint/task.json b/tests/convert/cpplint/task.json index 6ffe678..8fdb56f 100644 --- a/tests/convert/cpplint/task.json +++ b/tests/convert/cpplint/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Cpplint", "group": "cq", @@ -193,73 +107,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index c4bb518..4e93004 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[joj] ex2-asan", "group": "joj", @@ -166,6 +80,16 @@ "memoryLimit": 4194304, "procLimit": 50 }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case2.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case3.in" + } + }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/task1.in" @@ -173,7 +97,12 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + "src": "/home/tt/.config/joj/diff/task1/task2.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/task4.in" } }, { @@ -188,22 +117,7 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case3.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task4.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case2.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task1/task2.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" } } ] @@ -239,6 +153,30 @@ } ] }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -256,7 +194,19 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -292,43 +242,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case3.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case2.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -355,73 +269,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/keyword/task.json b/tests/convert/keyword/task.json index 88f6a61..355db46 100644 --- a/tests/convert/keyword/task.json +++ b/tests/convert/keyword/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Filelength", "group": "cq", @@ -190,73 +104,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } diff --git a/tests/convert/result-detail/task.json b/tests/convert/result-detail/task.json index 8daa140..cd060e1 100644 --- a/tests/convert/result-detail/task.json +++ b/tests/convert/result-detail/task.json @@ -10,92 +10,6 @@ "sandboxToken": "", "outputPath": "/tmp/joj3_result.json", "stages": [ - { - "name": "Health Check", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [], - "env": [], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [ - { - "args": [ - "/usr/local/bin/repo-health-checker", - "-root=.", - "-repoSize=10", - "-checkFileSumList=", - "-checkFileNameList=" - ] - }, - { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-check-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--group-config", - "" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ] - } - ] - } - }, - "parsers": [ - { - "name": "healthcheck", - "with": { - "score": 1 - } - }, - { - "name": "debug", - "with": { - "score": 0 - } - } - ] - }, { "name": "[cq] Filelength", "group": "cq", @@ -172,73 +86,6 @@ } ], "preStages": [], - "postStages": [ - { - "name": "teapot", - "group": "", - "executor": { - "name": "local", - "with": { - "default": { - "args": [ - "/usr/local/bin/joint-teapot", - "joj3-all-env", - "/home/tt/.config/teapot/teapot.env", - "--grading-repo-name", - "Nuvole-joj", - "--max-total-score", - "100" - ], - "env": [ - "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "stdin": { - "content": "" - }, - "stdout": { - "name": "stdout", - "max": 33554432, - "pipe": true - }, - "stderr": { - "name": "stderr", - "max": 33554432, - "pipe": true - }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 134217728, - "stackLimit": 0, - "procLimit": 50, - "cpuRateLimit": 0, - "cpuSetLimit": "", - "copyIn": {}, - "copyInCached": {}, - "copyInDir": ".", - "copyOut": [ - "stdout", - "stderr" - ], - "copyOutCached": [], - "copyOutMax": 0, - "copyOutDir": "", - "tty": false, - "strictMemoryLimit": false, - "dataSegmentLimit": false, - "addressSpaceLimit": false - }, - "cases": [] - } - }, - "parsers": [ - { - "name": "log", - "with": { - "msg": "joj3 summary" - } - } - ] - } - ] + "postStages": [] } } -- 2.30.2 From a4717268ec48973373d9e6fb767d73d4aa528e9f Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 19 Mar 2025 12:56:07 +0800 Subject: [PATCH 03/18] fix: pytest --- tests/convert/diff/task.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 9312c65..fde2dad 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -74,9 +74,7 @@ }, "cpuLimit": 2000000000, "clockLimit": 4000000000, -<<<<<<< HEAD - "memoryLimit": 4194304, - "procLimit": 50 + "memoryLimit": 4194304 }, { "stdin": { @@ -117,9 +115,6 @@ "stdin": { "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" } -======= - "memoryLimit": 4194304 ->>>>>>> master } ] } -- 2.30.2 From e775992e5da4a53c131599638cd16b2d174afef5 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Wed, 19 Mar 2025 02:13:46 -0400 Subject: [PATCH 04/18] fix: sort default cases --- joj3_config_generator/transformers/task.py | 8 +-- tests/convert/diff/task.json | 72 +++++++++++----------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 421bc90..6eebe69 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -173,7 +173,7 @@ def fix_diff( ) testcases = get_testcases(task_root, task_path) # TODO: better filter strategy - default_cases = list(filter(lambda x: x not in task_stage.cases, testcases)) + default_cases = sorted(testcases.difference(task_stage.cases)) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -233,10 +233,10 @@ def fix_diff( def get_testcases( task_root: Path, task_path: Path -) -> List[str]: # basedir here should be task_conf.root / task_conf.path - testcases = [] +) -> Set[str]: # basedir here should be task_conf.root / task_conf.path + testcases = set() for testcases_path in (task_root / task_path).parent.glob("**/*.in"): - testcases.append( + testcases.add( str( testcases_path.relative_to((task_root / task_path).parent) ).removesuffix(".in") diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index fde2dad..4bfc14b 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -86,6 +86,16 @@ "src": "/home/tt/.config/joj/diff/case3.in" } }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" + } + }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/task1.in" @@ -96,11 +106,6 @@ "src": "/home/tt/.config/joj/diff/task1/task2.in" } }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task4.in" - } - }, { "stdin": { "src": "/home/tt/.config/joj/diff/task2/task3.in" @@ -108,12 +113,7 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + "src": "/home/tt/.config/joj/diff/task2/task4.in" } } ] @@ -173,6 +173,30 @@ } ] }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -197,18 +221,6 @@ } ] }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, { "outputs": [ { @@ -226,19 +238,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false -- 2.30.2 From a8a7fd47a095395d4e5943f8107712992eb41943 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 14 May 2025 21:27:33 +0800 Subject: [PATCH 05/18] feat: support for subfolder change param --- joj3_config_generator/transformers/task.py | 13 +- .../{task1/subtask1/task5.in => case9.in} | 0 .../{task1/subtask1/task5.out => case9.out} | 0 tests/convert/diff/task.json | 174 +++++++++++------- tests/convert/diff/task.toml | 10 +- .../task1/{subtask1/task6.in => case4.in} | 0 .../task1/{subtask1/task6.out => case4.out} | 0 .../convert/diff/task1/{task1.in => case5.in} | 0 .../diff/task1/{task1.out => case5.out} | 0 .../task1/{task2.in => subtask1/case10.in} | 0 .../task1/{task2.out => subtask1/case10.out} | 0 .../task3.in => task1/subtask1/case11.in} | 0 .../task3.out => task1/subtask1/case11.out} | 0 .../convert/diff/task2/{task4.in => case6.in} | 0 .../diff/task2/{task4.out => case6.out} | 0 tests/convert/diff/task2/case7.in | 0 tests/convert/diff/task2/case8.out | 0 17 files changed, 124 insertions(+), 73 deletions(-) rename tests/convert/diff/{task1/subtask1/task5.in => case9.in} (100%) rename tests/convert/diff/{task1/subtask1/task5.out => case9.out} (100%) rename tests/convert/diff/task1/{subtask1/task6.in => case4.in} (100%) rename tests/convert/diff/task1/{subtask1/task6.out => case4.out} (100%) rename tests/convert/diff/task1/{task1.in => case5.in} (100%) rename tests/convert/diff/task1/{task1.out => case5.out} (100%) rename tests/convert/diff/task1/{task2.in => subtask1/case10.in} (100%) rename tests/convert/diff/task1/{task2.out => subtask1/case10.out} (100%) rename tests/convert/diff/{task2/task3.in => task1/subtask1/case11.in} (100%) rename tests/convert/diff/{task2/task3.out => task1/subtask1/case11.out} (100%) rename tests/convert/diff/task2/{task4.in => case6.in} (100%) rename tests/convert/diff/task2/{task4.out => case6.out} (100%) create mode 100644 tests/convert/diff/task2/case7.in create mode 100644 tests/convert/diff/task2/case8.out diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 6eebe69..0a04947 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -169,11 +169,17 @@ def fix_diff( valid_cases = ( (case, task_stage.cases[case]) for case in task_stage.cases - if case not in task_stage.skip and case in task_stage.cases + if case not in task_stage.skip ) testcases = get_testcases(task_root, task_path) - # TODO: better filter strategy - default_cases = sorted(testcases.difference(task_stage.cases)) + default_cases = sorted( + [ + case + for case in testcases + if any(case.endswith(other) for other in task_stage.cases) + ] + ) + print(default_cases) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -219,7 +225,6 @@ def fix_diff( parser_case = result.DiffCasesConfig( outputs=[ result.DiffOutputConfig( - # TODO: how to balance a good score strategy score=5, # default score file_name="stdout", answer_path=str(base_dir / f"{case}.out"), diff --git a/tests/convert/diff/task1/subtask1/task5.in b/tests/convert/diff/case9.in similarity index 100% rename from tests/convert/diff/task1/subtask1/task5.in rename to tests/convert/diff/case9.in diff --git a/tests/convert/diff/task1/subtask1/task5.out b/tests/convert/diff/case9.out similarity index 100% rename from tests/convert/diff/task1/subtask1/task5.out rename to tests/convert/diff/case9.out diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 4bfc14b..a7d25c7 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -78,42 +78,56 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case2.in" + "src": "/home/tt/.config/joj/diff/case9.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case11.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case10.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case5.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case0.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case3.in" + "src": "/home/tt/.config/joj/diff/case1.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task5.in" + "src": "/home/tt/.config/joj/diff/case9.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/task6.in" + "src": "/home/tt/.config/joj/diff/task1/case5.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/task1.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/task2.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task3.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task2/task4.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" } } ] @@ -125,6 +139,78 @@ "with": { "name": "diff", "cases": [ + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case0.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 123214122421, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 1232131, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case9.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 92321, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case11.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 823131, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case10.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 2590, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -154,7 +240,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case2.out", + "answerPath": "/home/tt/.config/joj/diff/case9.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -166,7 +252,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case3.out", + "answerPath": "/home/tt/.config/joj/diff/task1/case5.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -178,7 +264,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task5.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -190,55 +276,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/task6.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/task1.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/task2.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task3.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task2/task4.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index abdd10c..f4f283d 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -23,10 +23,18 @@ case0.diff.output.ignore_spaces = true #case0.command = "./h7/build/ex2" case0.in = "case0.in" -case1.diff.output.score = 5 +case1.diff.output.score = 123214122421 case1.limit.cpu = "2s" case1.limit.mem = "4m" case1.diff.output.ignore_spaces = true #case1.limit.stdout = 8 #case1.command = "./h7/build/ex2" case1.in = "case1.in" + +case9.diff.output.score = 1232131 + +case11.diff.output.score = 92321 + +case10.diff.output.score = 823131 + +case5.diff.output.score = 2590 diff --git a/tests/convert/diff/task1/subtask1/task6.in b/tests/convert/diff/task1/case4.in similarity index 100% rename from tests/convert/diff/task1/subtask1/task6.in rename to tests/convert/diff/task1/case4.in diff --git a/tests/convert/diff/task1/subtask1/task6.out b/tests/convert/diff/task1/case4.out similarity index 100% rename from tests/convert/diff/task1/subtask1/task6.out rename to tests/convert/diff/task1/case4.out diff --git a/tests/convert/diff/task1/task1.in b/tests/convert/diff/task1/case5.in similarity index 100% rename from tests/convert/diff/task1/task1.in rename to tests/convert/diff/task1/case5.in diff --git a/tests/convert/diff/task1/task1.out b/tests/convert/diff/task1/case5.out similarity index 100% rename from tests/convert/diff/task1/task1.out rename to tests/convert/diff/task1/case5.out diff --git a/tests/convert/diff/task1/task2.in b/tests/convert/diff/task1/subtask1/case10.in similarity index 100% rename from tests/convert/diff/task1/task2.in rename to tests/convert/diff/task1/subtask1/case10.in diff --git a/tests/convert/diff/task1/task2.out b/tests/convert/diff/task1/subtask1/case10.out similarity index 100% rename from tests/convert/diff/task1/task2.out rename to tests/convert/diff/task1/subtask1/case10.out diff --git a/tests/convert/diff/task2/task3.in b/tests/convert/diff/task1/subtask1/case11.in similarity index 100% rename from tests/convert/diff/task2/task3.in rename to tests/convert/diff/task1/subtask1/case11.in diff --git a/tests/convert/diff/task2/task3.out b/tests/convert/diff/task1/subtask1/case11.out similarity index 100% rename from tests/convert/diff/task2/task3.out rename to tests/convert/diff/task1/subtask1/case11.out diff --git a/tests/convert/diff/task2/task4.in b/tests/convert/diff/task2/case6.in similarity index 100% rename from tests/convert/diff/task2/task4.in rename to tests/convert/diff/task2/case6.in diff --git a/tests/convert/diff/task2/task4.out b/tests/convert/diff/task2/case6.out similarity index 100% rename from tests/convert/diff/task2/task4.out rename to tests/convert/diff/task2/case6.out diff --git a/tests/convert/diff/task2/case7.in b/tests/convert/diff/task2/case7.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case8.out b/tests/convert/diff/task2/case8.out new file mode 100644 index 0000000..e69de29 -- 2.30.2 From 391d61d00638703c83b57982aac7ad7227deb44a Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 15 May 2025 17:08:55 +0800 Subject: [PATCH 06/18] style: use const --- joj3_config_generator/models/const.py | 1 + joj3_config_generator/transformers/task.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py index 95a6706..33de35b 100644 --- a/joj3_config_generator/models/const.py +++ b/joj3_config_generator/models/const.py @@ -5,6 +5,7 @@ from joj3_config_generator.models.common import Memory, Time DEFAULT_CPU_LIMIT = Time("1s") DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_FILE_LIMIT = Memory("32m") +DEFAULT_CASE_SCORE = 5 JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 0a04947..b947247 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -4,7 +4,7 @@ from functools import partial from pathlib import Path from typing import Any, Callable, Dict, List, Set, Tuple -from joj3_config_generator.models import result, task +from joj3_config_generator.models import const, result, task from joj3_config_generator.models.common import Memory, Time from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT from joj3_config_generator.models.task import Parser as ParserEnum @@ -179,7 +179,6 @@ def fix_diff( if any(case.endswith(other) for other in task_stage.cases) ] ) - print(default_cases) stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -225,7 +224,7 @@ def fix_diff( parser_case = result.DiffCasesConfig( outputs=[ result.DiffOutputConfig( - score=5, # default score + score=const.DEFAULT_CASE_SCORE, file_name="stdout", answer_path=str(base_dir / f"{case}.out"), ) -- 2.30.2 From 26df677b3688db16cd825f01c0b1a46139b531b9 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 21 May 2025 22:36:12 +0800 Subject: [PATCH 07/18] fix(diff): cases not properly detected --- joj3_config_generator/transformers/task.py | 27 ++-- tests/convert/basic/task.json | 44 +----- tests/convert/diff/task.json | 172 ++++++++++----------- tests/convert/diff/task2/case7.out | 0 tests/convert/diff/task2/case8.in | 0 5 files changed, 105 insertions(+), 138 deletions(-) create mode 100644 tests/convert/diff/task2/case7.out create mode 100644 tests/convert/diff/task2/case8.in diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index b947247..4e9bf7e 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -166,19 +166,25 @@ def fix_diff( task_path: Path, ) -> None: base_dir = JOJ3_CONFIG_ROOT / task_path.parent - valid_cases = ( - (case, task_stage.cases[case]) - for case in task_stage.cases - if case not in task_stage.skip - ) + # all intended testcases that is detected testcases = get_testcases(task_root, task_path) + # all testcases that is not specified in the toml config default_cases = sorted( - [ - case - for case in testcases - if any(case.endswith(other) for other in task_stage.cases) - ] + testcases.difference( + [ + casei + for casei in testcases + if any(casei.endswith(casej) for casej in task_stage.cases) + ] + ) ) + # those in toml config that is not skiped + valid_cases = [ + (casej, task_stage.cases[casei]) + for casei in task_stage.cases + for casej in testcases + if (casei not in task_stage.skip and casej.endswith(casei)) + ] stage_cases = [] parser_cases = [] for case, case_stage in valid_cases: @@ -216,6 +222,7 @@ def fix_diff( ] ) parser_cases.append(parser_case) + for case in default_cases: cmd = result.OptionalCmd( stdin=result.LocalFile(src=str(base_dir / f"{case}.in")) diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 035c7f7..9f7929d 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -705,22 +705,7 @@ "dataSegmentLimit": false, "addressSpaceLimit": false }, - "cases": [ - { - "stdin": { - "src": "/home/tt/.config/joj/basic/case0.in" - }, - "cpuLimit": 500000000, - "clockLimit": 1000000000, - "memoryLimit": 5242880 - }, - { - "stdin": { - "src": "/home/tt/.config/joj/basic/case1.in" - }, - "memoryLimit": 5242880 - } - ] + "cases": [] } }, "parsers": [ @@ -728,32 +713,7 @@ "name": "diff", "with": { "name": "diff", - "cases": [ - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/basic/case0.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/basic/case1.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - } - ] + "cases": [] } }, { diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index a7d25c7..9c15abd 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -84,50 +84,50 @@ }, { "stdin": { - "src": "/home/tt/.config/joj/diff/case11.in" + "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" }, "memoryLimit": 268435456 }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case10.in" - }, - "memoryLimit": 268435456 - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case5.in" - }, - "memoryLimit": 268435456 - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case0.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case1.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case9.in" - } - }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/task1/case5.in" - } - }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/case5.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case2.in" } }, { "stdin": { - "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" + "src": "/home/tt/.config/joj/diff/case3.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/case4.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/case6.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/case7.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task2/case8.in" } } ] @@ -180,7 +180,7 @@ { "score": 92321, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case11.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -192,7 +192,7 @@ { "score": 823131, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case10.out", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -204,54 +204,6 @@ { "score": 2590, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case5.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case0.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case1.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case9.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, - { - "outputs": [ - { - "score": 5, - "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/task1/case5.out", "forceQuitOnDiff": false, "alwaysHide": false, @@ -264,7 +216,7 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", + "answerPath": "/home/tt/.config/joj/diff/case2.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false @@ -276,7 +228,55 @@ { "score": 5, "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", + "answerPath": "/home/tt/.config/joj/diff/case3.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/case4.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/case6.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/case7.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task2/case8.out", "forceQuitOnDiff": false, "alwaysHide": false, "compareSpace": false diff --git a/tests/convert/diff/task2/case7.out b/tests/convert/diff/task2/case7.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case8.in b/tests/convert/diff/task2/case8.in new file mode 100644 index 0000000..e69de29 -- 2.30.2 From 2f1686c1e32c8e9b5f7e7eda2d9ced02f0a4e528 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Wed, 21 May 2025 22:43:00 +0800 Subject: [PATCH 08/18] test: case.command --- tests/convert/diff/task.json | 6 ++++++ tests/convert/diff/task.toml | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 11d1c34..63a4607 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -65,12 +65,18 @@ }, "cases": [ { + "args": [ + "./h7/build/ex2" + ], "stdin": { "src": "/home/tt/.config/joj/diff/case0.in" }, "memoryLimit": 2097152 }, { + "args": [ + "./h7/build/ex2" + ], "stdin": { "src": "/home/tt/.config/joj/diff/case1.in" }, diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index f4f283d..4272f97 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -19,17 +19,14 @@ case0.diff.output.score = 5 case0.limit.cpu = "1s" case0.limit.mem = "2m" case0.diff.output.ignore_spaces = true -#case0.limit.stdout = 8 -#case0.command = "./h7/build/ex2" +case0.command = "./h7/build/ex2" case0.in = "case0.in" case1.diff.output.score = 123214122421 case1.limit.cpu = "2s" case1.limit.mem = "4m" case1.diff.output.ignore_spaces = true -#case1.limit.stdout = 8 -#case1.command = "./h7/build/ex2" -case1.in = "case1.in" +case1.command = "./h7/build/ex2" case9.diff.output.score = 1232131 -- 2.30.2 From 5acb2ca201effdf44e0964d039281f99b1399506 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 22 May 2025 14:08:39 +0800 Subject: [PATCH 09/18] fix: remove stdin for diff stage --- joj3_config_generator/models/result.py | 2 +- joj3_config_generator/transformers/task.py | 1 + tests/convert/basic/task.json | 6 ------ tests/convert/diff/task.json | 3 --- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/joj3_config_generator/models/result.py b/joj3_config_generator/models/result.py index 4f964b8..308c73a 100644 --- a/joj3_config_generator/models/result.py +++ b/joj3_config_generator/models/result.py @@ -39,7 +39,7 @@ class StreamOut(BaseModel): stream_out: bool = Field(..., alias="streamOut") -InputFile = Union[LocalFile, MemoryFile, PreparedFile, Symlink] +InputFile = Union[LocalFile, MemoryFile, PreparedFile, Symlink, None] class Cmd(BaseModel): diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index e1b98ca..8f070cc 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -238,6 +238,7 @@ def fix_diff( ) parser_cases.append(parser_case) executor.with_.cases = stage_cases + executor.with_.default.stdin = None diff_parser.with_ = result.DiffConfig(name="diff", cases=parser_cases) diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 82e35e4..6fa7a38 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -432,9 +432,6 @@ "env": [ "PATH=/usr/bin:/bin:/usr/local/bin" ], - "stdin": { - "content": "" - }, "stdout": { "name": "stdout", "max": 33554432, @@ -678,9 +675,6 @@ "env": [ "PATH=/usr/bin:/bin:/usr/local/bin" ], - "stdin": { - "content": "" - }, "stdout": { "name": "stdout", "max": 33554432, diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 63a4607..91ec29c 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -24,9 +24,6 @@ "env": [ "PATH=/usr/bin:/bin:/usr/local/bin" ], - "stdin": { - "content": "" - }, "stdout": { "name": "stdout", "max": 33554432, -- 2.30.2 From ca63a94238850af2346cd85d3500340de269d1bf Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 22 May 2025 14:28:35 +0800 Subject: [PATCH 10/18] feat: add default score for diff --- joj3_config_generator/models/const.py | 1 - joj3_config_generator/models/task.py | 1 + joj3_config_generator/transformers/task.py | 12 +++++++++++- tests/convert/diff/task.json | 12 ++++++------ tests/convert/diff/task.toml | 2 ++ 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py index 33de35b..95a6706 100644 --- a/joj3_config_generator/models/const.py +++ b/joj3_config_generator/models/const.py @@ -5,7 +5,6 @@ from joj3_config_generator.models.common import Memory, Time DEFAULT_CPU_LIMIT = Time("1s") DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_FILE_LIMIT = Memory("32m") -DEFAULT_CASE_SCORE = 5 JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index 4e99358..1a3d332 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -51,6 +51,7 @@ class Outputs(BaseModel): class ParserDiff(BaseModel): output: Outputs = Outputs() + default_score: int = 5 class Files(BaseModel): diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 8f070cc..096d035 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -226,11 +226,21 @@ def fix_diff( cmd = result.OptionalCmd( stdin=result.LocalFile(src=str(base_dir / f"{case}.in")) ) + if cmd.args == executor.with_.default.args: + cmd.args = None + if cmd.cpu_limit == executor.with_.default.cpu_limit: + cmd.cpu_limit = None + if cmd.clock_limit == executor.with_.default.clock_limit: + cmd.clock_limit = None + if cmd.memory_limit == executor.with_.default.memory_limit: + cmd.memory_limit = None + if cmd.proc_limit == executor.with_.default.proc_limit: + cmd.proc_limit = None stage_cases.append(cmd) parser_case = result.DiffCasesConfig( outputs=[ result.DiffOutputConfig( - score=const.DEFAULT_CASE_SCORE, + score=task_stage.diff.default_score, file_name="stdout", answer_path=str(base_dir / f"{case}.out"), ) diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 91ec29c..e463c02 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -219,7 +219,7 @@ { "outputs": [ { - "score": 5, + "score": 100, "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/case2.out", "forceQuitOnDiff": false, @@ -231,7 +231,7 @@ { "outputs": [ { - "score": 5, + "score": 100, "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/case3.out", "forceQuitOnDiff": false, @@ -243,7 +243,7 @@ { "outputs": [ { - "score": 5, + "score": 100, "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/task1/case4.out", "forceQuitOnDiff": false, @@ -255,7 +255,7 @@ { "outputs": [ { - "score": 5, + "score": 100, "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/task2/case6.out", "forceQuitOnDiff": false, @@ -267,7 +267,7 @@ { "outputs": [ { - "score": 5, + "score": 100, "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/task2/case7.out", "forceQuitOnDiff": false, @@ -279,7 +279,7 @@ { "outputs": [ { - "score": 5, + "score": 100, "fileName": "stdout", "answerPath": "/home/tt/.config/joj/diff/task2/case8.out", "forceQuitOnDiff": false, diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index 4272f97..bb18497 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -14,6 +14,8 @@ parsers = [ "diff", "result-detail" ] result-detail.exitstatus = true result-detail.stderr = true +diff.default_score = 100 + # will be removed as long as the name is fixed case0.diff.output.score = 5 case0.limit.cpu = "1s" -- 2.30.2 From 927e5ce95ded48dbf57106e55209b9a5f5fa2ebe Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 22 May 2025 14:59:50 +0800 Subject: [PATCH 11/18] fix: cases specific numerics remove when unecessary --- joj3_config_generator/transformers/task.py | 22 ++++++++-------------- tests/convert/diff/task.json | 22 +++++++++------------- tests/convert/diff/task.toml | 6 ++++-- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 096d035..3dcbcbe 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -200,11 +200,12 @@ def fix_diff( ) if cmd.args == executor.with_.default.args: cmd.args = None - if cmd.cpu_limit == executor.with_.default.cpu_limit: + # duplicate with the fallback case in executor.with_ + if cmd.cpu_limit == const.DEFAULT_CPU_LIMIT: cmd.cpu_limit = None - if cmd.clock_limit == executor.with_.default.clock_limit: + if cmd.clock_limit == 2 * const.DEFAULT_CPU_LIMIT: cmd.clock_limit = None - if cmd.memory_limit == executor.with_.default.memory_limit: + if cmd.memory_limit == const.DEFAULT_MEMORY_LIMIT: cmd.memory_limit = None if cmd.proc_limit == executor.with_.default.proc_limit: cmd.proc_limit = None @@ -224,18 +225,11 @@ def fix_diff( parser_cases.append(parser_case) for case in default_cases: cmd = result.OptionalCmd( - stdin=result.LocalFile(src=str(base_dir / f"{case}.in")) + stdin=result.LocalFile(src=str(base_dir / f"{case}.in")), + cpu_limit=None, + clock_limit=None, + memory_limit=None, ) - if cmd.args == executor.with_.default.args: - cmd.args = None - if cmd.cpu_limit == executor.with_.default.cpu_limit: - cmd.cpu_limit = None - if cmd.clock_limit == executor.with_.default.clock_limit: - cmd.clock_limit = None - if cmd.memory_limit == executor.with_.default.memory_limit: - cmd.memory_limit = None - if cmd.proc_limit == executor.with_.default.proc_limit: - cmd.proc_limit = None stage_cases.append(cmd) parser_case = result.DiffCasesConfig( outputs=[ diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index e463c02..96291fa 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -26,17 +26,17 @@ ], "stdout": { "name": "stdout", - "max": 33554432, + "max": 10485760, "pipe": true }, "stderr": { "name": "stderr", - "max": 33554432, + "max": 10485760, "pipe": true }, - "cpuLimit": 1000000000, - "clockLimit": 2000000000, - "memoryLimit": 68157440, + "cpuLimit": 3000000000, + "clockLimit": 6000000000, + "memoryLimit": 10485760, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -84,26 +84,22 @@ { "stdin": { "src": "/home/tt/.config/joj/diff/case9.in" - }, - "memoryLimit": 268435456 + } }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" - }, - "memoryLimit": 268435456 + } }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in" - }, - "memoryLimit": 268435456 + } }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/case5.in" - }, - "memoryLimit": 268435456 + } }, { "stdin": { diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index bb18497..fc28cd7 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -8,7 +8,10 @@ release.begin_time = 2024-12-29 23:59:59+08:00 name = "[joj] ex2-asan" command="./h7/build/ex2-asan -a" files.import = [ "h7/build/ex2-asan" ] -limit.mem = "65m" +limit.cpu = "3s" +limit.mem = "10m" +limit.stdout = "10m" +limit.stderr = "10m" parsers = [ "diff", "result-detail" ] result-detail.exitstatus = true @@ -16,7 +19,6 @@ result-detail.stderr = true diff.default_score = 100 -# will be removed as long as the name is fixed case0.diff.output.score = 5 case0.limit.cpu = "1s" case0.limit.mem = "2m" -- 2.30.2 From b4a9d0cb62b595e6cb8b9d16fde7b159f494f48c Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 22 May 2025 15:18:49 +0800 Subject: [PATCH 12/18] fix: add back DEFAULT_CASE_SCORE --- joj3_config_generator/models/const.py | 1 + joj3_config_generator/models/task.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py index 95a6706..33de35b 100644 --- a/joj3_config_generator/models/const.py +++ b/joj3_config_generator/models/const.py @@ -5,6 +5,7 @@ from joj3_config_generator.models.common import Memory, Time DEFAULT_CPU_LIMIT = Time("1s") DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_FILE_LIMIT = Memory("32m") +DEFAULT_CASE_SCORE = 5 JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index 1a3d332..fd55d89 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida from joj3_config_generator.models.common import Memory, Time from joj3_config_generator.models.const import ( + DEFAULT_CASE_SCORE, DEFAULT_CPU_LIMIT, DEFAULT_FILE_LIMIT, DEFAULT_MEMORY_LIMIT, @@ -51,7 +52,7 @@ class Outputs(BaseModel): class ParserDiff(BaseModel): output: Outputs = Outputs() - default_score: int = 5 + default_score: int = DEFAULT_CASE_SCORE class Files(BaseModel): -- 2.30.2 From 183e6f154597d89ed4a691388d0cf9c9fb6789f2 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 22 May 2025 15:27:50 +0800 Subject: [PATCH 13/18] feat: add DEFAULT_PROC_LIMIT n DEFAULT_CLOCK_LIMIT_MULTIPLIER --- joj3_config_generator/models/const.py | 2 ++ joj3_config_generator/models/result.py | 9 +++++++-- joj3_config_generator/transformers/task.py | 8 ++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py index 33de35b..491371f 100644 --- a/joj3_config_generator/models/const.py +++ b/joj3_config_generator/models/const.py @@ -6,6 +6,8 @@ DEFAULT_CPU_LIMIT = Time("1s") DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_FILE_LIMIT = Memory("32m") DEFAULT_CASE_SCORE = 5 +DEFAULT_CLOCK_LIMIT_MULTIPLIER = 2 +DEFAULT_PROC_LIMIT = 50 JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") diff --git a/joj3_config_generator/models/result.py b/joj3_config_generator/models/result.py index 308c73a..bf171cb 100644 --- a/joj3_config_generator/models/result.py +++ b/joj3_config_generator/models/result.py @@ -3,9 +3,11 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field, field_validator from joj3_config_generator.models.const import ( + DEFAULT_CLOCK_LIMIT_MULTIPLIER, DEFAULT_CPU_LIMIT, DEFAULT_FILE_LIMIT, DEFAULT_MEMORY_LIMIT, + DEFAULT_PROC_LIMIT, ) @@ -49,10 +51,13 @@ class Cmd(BaseModel): stdout: Union[Collector, StreamOut] = Collector(name="stdout") stderr: Union[Collector, StreamOut] = Collector(name="stderr") cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit") - clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit") + clock_limit: int = Field( + DEFAULT_CLOCK_LIMIT_MULTIPLIER * DEFAULT_CPU_LIMIT, + serialization_alias="clockLimit", + ) memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit") stack_limit: int = Field(0, serialization_alias="stackLimit") - proc_limit: int = Field(50, serialization_alias="procLimit") + proc_limit: int = Field(DEFAULT_PROC_LIMIT, serialization_alias="procLimit") cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit") cpu_set_limit: str = Field("", serialization_alias="cpuSetLimit") copy_in: Dict[str, InputFile] = Field({}, serialization_alias="copyIn") diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 3dcbcbe..d907504 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -203,11 +203,14 @@ def fix_diff( # duplicate with the fallback case in executor.with_ if cmd.cpu_limit == const.DEFAULT_CPU_LIMIT: cmd.cpu_limit = None - if cmd.clock_limit == 2 * const.DEFAULT_CPU_LIMIT: + if ( + cmd.clock_limit + == const.DEFAULT_CLOCK_LIMIT_MULTIPLIER * const.DEFAULT_CPU_LIMIT + ): cmd.clock_limit = None if cmd.memory_limit == const.DEFAULT_MEMORY_LIMIT: cmd.memory_limit = None - if cmd.proc_limit == executor.with_.default.proc_limit: + if cmd.proc_limit == const.DEFAULT_PROC_LIMIT: cmd.proc_limit = None stage_cases.append(cmd) parser_case = result.DiffCasesConfig( @@ -229,6 +232,7 @@ def fix_diff( cpu_limit=None, clock_limit=None, memory_limit=None, + proc_limit=None, ) stage_cases.append(cmd) parser_case = result.DiffCasesConfig( -- 2.30.2 From dcebf8640860cf6448ae4384c2a820d4d04180aa Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 22 May 2025 16:06:03 +0800 Subject: [PATCH 14/18] fix: remove none stdin for diff --- joj3_config_generator/models/result.py | 2 +- joj3_config_generator/transformers/task.py | 1 - tests/convert/basic/task.json | 6 ++++++ tests/convert/diff/task.json | 3 +++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/joj3_config_generator/models/result.py b/joj3_config_generator/models/result.py index bf171cb..733a6a6 100644 --- a/joj3_config_generator/models/result.py +++ b/joj3_config_generator/models/result.py @@ -41,7 +41,7 @@ class StreamOut(BaseModel): stream_out: bool = Field(..., alias="streamOut") -InputFile = Union[LocalFile, MemoryFile, PreparedFile, Symlink, None] +InputFile = Union[LocalFile, MemoryFile, PreparedFile, Symlink] class Cmd(BaseModel): diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index d907504..e5849d6 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -246,7 +246,6 @@ def fix_diff( ) parser_cases.append(parser_case) executor.with_.cases = stage_cases - executor.with_.default.stdin = None diff_parser.with_ = result.DiffConfig(name="diff", cases=parser_cases) diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 6fa7a38..82e35e4 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -432,6 +432,9 @@ "env": [ "PATH=/usr/bin:/bin:/usr/local/bin" ], + "stdin": { + "content": "" + }, "stdout": { "name": "stdout", "max": 33554432, @@ -675,6 +678,9 @@ "env": [ "PATH=/usr/bin:/bin:/usr/local/bin" ], + "stdin": { + "content": "" + }, "stdout": { "name": "stdout", "max": 33554432, diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 96291fa..3c9c6d6 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -24,6 +24,9 @@ "env": [ "PATH=/usr/bin:/bin:/usr/local/bin" ], + "stdin": { + "content": "" + }, "stdout": { "name": "stdout", "max": 10485760, -- 2.30.2 From c3f2b217327f5978923ab0a77ad898d6bfd250eb Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 22 May 2025 20:46:01 -0400 Subject: [PATCH 15/18] fix: default limit --- joj3_config_generator/models/task.py | 9 ++++++-- joj3_config_generator/transformers/task.py | 24 +++++++++++----------- tests/convert/diff/task.json | 2 ++ tests/convert/diff/task.toml | 1 + 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index fd55d89..a946551 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -127,9 +127,14 @@ class Stage(BaseModel): @classmethod def gather_cases(cls: Type["Stage"], values: Dict[str, Any]) -> Dict[str, Any]: cases = {k: v for k, v in values.items() if k.startswith("case")} - for key in cases: + limit = values.get("limit", {}) + parsed_cases = {} + for key, case in cases.items(): + case_with_limit = {**limit, **case.get("limit", {})} + case_for_parsing = {**case, "limit": case_with_limit} + parsed_cases[key] = case_for_parsing values.pop(key) - values["cases"] = {k: v for k, v in cases.items()} + values["cases"] = parsed_cases return values diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index e5849d6..249ff2e 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -4,9 +4,13 @@ from functools import partial from pathlib import Path from typing import Any, Callable, Dict, List, Set, Tuple -from joj3_config_generator.models import const, result, task +from joj3_config_generator.models import result, task from joj3_config_generator.models.common import Memory, Time -from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT +from joj3_config_generator.models.const import ( + DEFAULT_CLOCK_LIMIT_MULTIPLIER, + DEFAULT_PROC_LIMIT, + JOJ3_CONFIG_ROOT, +) from joj3_config_generator.models.task import Parser as ParserEnum @@ -194,23 +198,19 @@ def fix_diff( ), args=shlex.split(case_stage.command) if case_stage.command else None, cpu_limit=case_stage.limit.cpu, - clock_limit=2 * case_stage.limit.cpu, + clock_limit=DEFAULT_CLOCK_LIMIT_MULTIPLIER * case_stage.limit.cpu, memory_limit=case_stage.limit.mem, - proc_limit=50, + proc_limit=DEFAULT_PROC_LIMIT, ) if cmd.args == executor.with_.default.args: cmd.args = None - # duplicate with the fallback case in executor.with_ - if cmd.cpu_limit == const.DEFAULT_CPU_LIMIT: + if cmd.cpu_limit == executor.with_.default.cpu_limit: cmd.cpu_limit = None - if ( - cmd.clock_limit - == const.DEFAULT_CLOCK_LIMIT_MULTIPLIER * const.DEFAULT_CPU_LIMIT - ): + if cmd.clock_limit == executor.with_.default.clock_limit: cmd.clock_limit = None - if cmd.memory_limit == const.DEFAULT_MEMORY_LIMIT: + if cmd.memory_limit == executor.with_.default.memory_limit: cmd.memory_limit = None - if cmd.proc_limit == const.DEFAULT_PROC_LIMIT: + if cmd.proc_limit == executor.with_.default.proc_limit: cmd.proc_limit = None stage_cases.append(cmd) parser_case = result.DiffCasesConfig( diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 3c9c6d6..399f9bf 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -71,6 +71,8 @@ "stdin": { "src": "/home/tt/.config/joj/diff/case0.in" }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, "memoryLimit": 2097152 }, { diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index fc28cd7..2f50c35 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -33,6 +33,7 @@ case1.diff.output.ignore_spaces = true case1.command = "./h7/build/ex2" case9.diff.output.score = 1232131 +case9.limit.mem = "10m" case11.diff.output.score = 92321 -- 2.30.2 From 2dbfa986facb4f8540318f98b9b557f1290a260a Mon Sep 17 00:00:00 2001 From: jon-lee Date: Fri, 23 May 2025 20:26:36 +0800 Subject: [PATCH 16/18] feat: add verification for path *.out --- joj3_config_generator/transformers/task.py | 4 ++++ tests/convert/diff/case0.out | 0 tests/convert/diff/case1.out | 0 tests/convert/diff/case2.out | 0 tests/convert/diff/case3.out | 0 5 files changed, 4 insertions(+) create mode 100644 tests/convert/diff/case0.out create mode 100644 tests/convert/diff/case1.out create mode 100644 tests/convert/diff/case2.out create mode 100644 tests/convert/diff/case3.out diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 249ff2e..6055412 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -1,3 +1,4 @@ +import os import re import shlex from functools import partial @@ -259,4 +260,7 @@ def get_testcases( testcases_path.relative_to((task_root / task_path).parent) ).removesuffix(".in") ) + assert os.path.exists( + str(testcases_path).removesuffix(".in") + ".out" + ), f"{testcases_path} does not have a corresponding *.out file!" return testcases diff --git a/tests/convert/diff/case0.out b/tests/convert/diff/case0.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case1.out b/tests/convert/diff/case1.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case2.out b/tests/convert/diff/case2.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case3.out b/tests/convert/diff/case3.out new file mode 100644 index 0000000..e69de29 -- 2.30.2 From 7265411abe9d1dd4b5d2a23db95ac032dbe4628d Mon Sep 17 00:00:00 2001 From: jon-lee Date: Fri, 23 May 2025 20:39:12 +0800 Subject: [PATCH 17/18] fix: use logger instead of direct assert --- joj3_config_generator/transformers/task.py | 7 ++++--- tests/convert/diff/case3.out | 0 tests/convert/diff/task.json | 17 ----------------- 3 files changed, 4 insertions(+), 20 deletions(-) delete mode 100644 tests/convert/diff/case3.out diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 6055412..6fcaa0e 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -13,6 +13,7 @@ from joj3_config_generator.models.const import ( JOJ3_CONFIG_ROOT, ) from joj3_config_generator.models.task import Parser as ParserEnum +from joj3_config_generator.utils.logger import logger def get_conf_stage( @@ -255,12 +256,12 @@ def get_testcases( ) -> Set[str]: # basedir here should be task_conf.root / task_conf.path testcases = set() for testcases_path in (task_root / task_path).parent.glob("**/*.in"): + if not os.path.exists(str(testcases_path).removesuffix(".in") + ".out"): + logger.warning(f"Testcase {testcases_path} has no corresponding .out file") + continue testcases.add( str( testcases_path.relative_to((task_root / task_path).parent) ).removesuffix(".in") ) - assert os.path.exists( - str(testcases_path).removesuffix(".in") + ".out" - ), f"{testcases_path} does not have a corresponding *.out file!" return testcases diff --git a/tests/convert/diff/case3.out b/tests/convert/diff/case3.out deleted file mode 100644 index e69de29..0000000 diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 399f9bf..3b902b5 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -111,11 +111,6 @@ "src": "/home/tt/.config/joj/diff/case2.in" } }, - { - "stdin": { - "src": "/home/tt/.config/joj/diff/case3.in" - } - }, { "stdin": { "src": "/home/tt/.config/joj/diff/task1/case4.in" @@ -229,18 +224,6 @@ } ] }, - { - "outputs": [ - { - "score": 100, - "fileName": "stdout", - "answerPath": "/home/tt/.config/joj/diff/case3.out", - "forceQuitOnDiff": false, - "alwaysHide": false, - "compareSpace": false - } - ] - }, { "outputs": [ { -- 2.30.2 From 044d0f0d417ed037688b1cabaa86e0f997110ac8 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Fri, 23 May 2025 20:43:21 +0800 Subject: [PATCH 18/18] fix: use logger instead of direct assert --- joj3_config_generator/transformers/task.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 6fcaa0e..9ecb7a5 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -1,4 +1,3 @@ -import os import re import shlex from functools import partial @@ -256,7 +255,7 @@ def get_testcases( ) -> Set[str]: # basedir here should be task_conf.root / task_conf.path testcases = set() for testcases_path in (task_root / task_path).parent.glob("**/*.in"): - if not os.path.exists(str(testcases_path).removesuffix(".in") + ".out"): + if not testcases_path.with_suffix(".out").exists(): logger.warning(f"Testcase {testcases_path} has no corresponding .out file") continue testcases.add( -- 2.30.2