From 8a6c34f3e913c1ea8941453a4c13ca2b10dd8291 Mon Sep 17 00:00:00 2001 From: jon-lee Date: Thu, 15 May 2025 17:47:08 +0800 Subject: [PATCH] feat(diff): auto detect testcases (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - added support for subfolders - complete the duplication for `*.toml` file, now ta only need to specify the param for the specific one. default values are hardcoded - all testcases should be named as `casei.in/out`, with no interactions in their name in each of the subfolders, (for hidden cases and non-hidden cases, just change position and run again) Co-authored-by: Boming Zhang Reviewed-on: https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3-config-generator/pulls/13 Reviewed-by: 张泊明518370910136 Co-authored-by: jon-lee Co-committed-by: jon-lee --- joj3_config_generator/models/const.py | 1 + joj3_config_generator/transformers/task.py | 55 +++++- tests/convert/diff/case0.in | 0 tests/convert/diff/case1.in | 0 tests/convert/diff/case2.in | 0 tests/convert/diff/case3.in | 0 tests/convert/diff/case9.in | 0 tests/convert/diff/case9.out | 0 tests/convert/diff/task.json | 174 +++++++++++++++++++ tests/convert/diff/task.toml | 10 +- tests/convert/diff/task1/case4.in | 0 tests/convert/diff/task1/case4.out | 0 tests/convert/diff/task1/case5.in | 0 tests/convert/diff/task1/case5.out | 0 tests/convert/diff/task1/subtask1/case10.in | 0 tests/convert/diff/task1/subtask1/case10.out | 0 tests/convert/diff/task1/subtask1/case11.in | 0 tests/convert/diff/task1/subtask1/case11.out | 0 tests/convert/diff/task2/case6.in | 0 tests/convert/diff/task2/case6.out | 0 tests/convert/diff/task2/case7.in | 0 tests/convert/diff/task2/case8.out | 0 22 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 tests/convert/diff/case0.in create mode 100644 tests/convert/diff/case1.in create mode 100644 tests/convert/diff/case2.in create mode 100644 tests/convert/diff/case3.in create mode 100644 tests/convert/diff/case9.in create mode 100644 tests/convert/diff/case9.out create mode 100644 tests/convert/diff/task1/case4.in create mode 100644 tests/convert/diff/task1/case4.out create mode 100644 tests/convert/diff/task1/case5.in create mode 100644 tests/convert/diff/task1/case5.out create mode 100644 tests/convert/diff/task1/subtask1/case10.in create mode 100644 tests/convert/diff/task1/subtask1/case10.out create mode 100644 tests/convert/diff/task1/subtask1/case11.in create mode 100644 tests/convert/diff/task1/subtask1/case11.out create mode 100644 tests/convert/diff/task2/case6.in create mode 100644 tests/convert/diff/task2/case6.out create mode 100644 tests/convert/diff/task2/case7.in create mode 100644 tests/convert/diff/task2/case8.out diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py index 95a6706..33de35b 100644 --- a/joj3_config_generator/models/const.py +++ b/joj3_config_generator/models/const.py @@ -5,6 +5,7 @@ from joj3_config_generator.models.common import Memory, Time DEFAULT_CPU_LIMIT = Time("1s") DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_FILE_LIMIT = Memory("32m") +DEFAULT_CASE_SCORE = 5 JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") diff --git a/joj3_config_generator/transformers/task.py b/joj3_config_generator/transformers/task.py index 49f9a07..b947247 100644 --- a/joj3_config_generator/transformers/task.py +++ b/joj3_config_generator/transformers/task.py @@ -2,9 +2,9 @@ import re import shlex from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, List, Tuple +from typing import Any, Callable, Dict, List, Set, Tuple -from joj3_config_generator.models import result, task +from joj3_config_generator.models import const, result, task from joj3_config_generator.models.common import Memory, Time from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT from joj3_config_generator.models.task import Parser as ParserEnum @@ -32,7 +32,8 @@ def get_conf_stage( parser_handler_map = get_parser_handler_map( task_stage, conf_stage.executor, - JOJ3_CONFIG_ROOT / task_conf.path.parent, + task_conf.root, + task_conf.path, ) for idx, parser in enumerate(task_stage.parsers): if parser not in parser_handler_map: @@ -45,7 +46,8 @@ def get_conf_stage( def get_parser_handler_map( task_stage: task.Stage, executor: result.Executor, - base_dir: Path, + task_root: Path, + task_path: Path, ) -> Dict[ParserEnum, Tuple[Callable[[Any, result.Parser], None], Any]]: return { ParserEnum.CLANG_TIDY: (fix_keyword, task_stage.clangtidy), @@ -61,7 +63,8 @@ def get_parser_handler_map( fix_diff, task_stage=task_stage, executor=executor, - base_dir=base_dir, + task_root=task_root, + task_path=task_path, ), task_stage.diff, ), @@ -159,12 +162,22 @@ def fix_diff( diff_parser: result.Parser, task_stage: task.Stage, executor: result.Executor, - base_dir: Path, + task_root: Path, + task_path: Path, ) -> None: + base_dir = JOJ3_CONFIG_ROOT / task_path.parent valid_cases = ( (case, task_stage.cases[case]) for case in task_stage.cases - if case not in task_stage.skip and case in task_stage.cases + if case not in task_stage.skip + ) + testcases = get_testcases(task_root, task_path) + default_cases = sorted( + [ + case + for case in testcases + if any(case.endswith(other) for other in task_stage.cases) + ] ) stage_cases = [] parser_cases = [] @@ -203,5 +216,33 @@ def fix_diff( ] ) parser_cases.append(parser_case) + for case in default_cases: + cmd = result.OptionalCmd( + stdin=result.LocalFile(src=str(base_dir / f"{case}.in")) + ) + stage_cases.append(cmd) + parser_case = result.DiffCasesConfig( + outputs=[ + result.DiffOutputConfig( + score=const.DEFAULT_CASE_SCORE, + file_name="stdout", + answer_path=str(base_dir / f"{case}.out"), + ) + ] + ) + parser_cases.append(parser_case) executor.with_.cases = stage_cases diff_parser.with_ = result.DiffConfig(name="diff", cases=parser_cases) + + +def get_testcases( + task_root: Path, task_path: Path +) -> Set[str]: # basedir here should be task_conf.root / task_conf.path + testcases = set() + for testcases_path in (task_root / task_path).parent.glob("**/*.in"): + testcases.add( + str( + testcases_path.relative_to((task_root / task_path).parent) + ).removesuffix(".in") + ) + return testcases diff --git a/tests/convert/diff/case0.in b/tests/convert/diff/case0.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case1.in b/tests/convert/diff/case1.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case2.in b/tests/convert/diff/case2.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case3.in b/tests/convert/diff/case3.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case9.in b/tests/convert/diff/case9.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/case9.out b/tests/convert/diff/case9.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index 42c141c..a7d25c7 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -75,6 +75,60 @@ "cpuLimit": 2000000000, "clockLimit": 4000000000, "memoryLimit": 4194304 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case9.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case11.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case10.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case5.in" + }, + "memoryLimit": 268435456 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case0.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case1.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case9.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/case5.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in" + } + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in" + } } ] } @@ -85,6 +139,78 @@ "with": { "name": "diff", "cases": [ + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case0.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 123214122421, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 1232131, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case9.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 92321, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case11.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 823131, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case10.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 2590, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, { "outputs": [ { @@ -108,6 +234,54 @@ "compareSpace": false } ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case9.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/case5.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case10.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/task1/subtask1/case11.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] } ] } diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml index abdd10c..f4f283d 100644 --- a/tests/convert/diff/task.toml +++ b/tests/convert/diff/task.toml @@ -23,10 +23,18 @@ case0.diff.output.ignore_spaces = true #case0.command = "./h7/build/ex2" case0.in = "case0.in" -case1.diff.output.score = 5 +case1.diff.output.score = 123214122421 case1.limit.cpu = "2s" case1.limit.mem = "4m" case1.diff.output.ignore_spaces = true #case1.limit.stdout = 8 #case1.command = "./h7/build/ex2" case1.in = "case1.in" + +case9.diff.output.score = 1232131 + +case11.diff.output.score = 92321 + +case10.diff.output.score = 823131 + +case5.diff.output.score = 2590 diff --git a/tests/convert/diff/task1/case4.in b/tests/convert/diff/task1/case4.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/case4.out b/tests/convert/diff/task1/case4.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/case5.in b/tests/convert/diff/task1/case5.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/case5.out b/tests/convert/diff/task1/case5.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/case10.in b/tests/convert/diff/task1/subtask1/case10.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/case10.out b/tests/convert/diff/task1/subtask1/case10.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/case11.in b/tests/convert/diff/task1/subtask1/case11.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task1/subtask1/case11.out b/tests/convert/diff/task1/subtask1/case11.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case6.in b/tests/convert/diff/task2/case6.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case6.out b/tests/convert/diff/task2/case6.out new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case7.in b/tests/convert/diff/task2/case7.in new file mode 100644 index 0000000..e69de29 diff --git a/tests/convert/diff/task2/case8.out b/tests/convert/diff/task2/case8.out new file mode 100644 index 0000000..e69de29