JOJ/JOJ3-config-generator
27
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(diff): bugs on diff stdin and numerics #16

Merged
张泊明518370910136 merged 22 commits from fix/diff into master 2025-05-24 02:45:39 +08:00
Conversation 1 Commits 22 Files Changed 9 +71 -59
9 changed files with 71 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
joj3_config_generator/models/const.py
View File

@ -6,6 +6,8 @@ DEFAULT_CPU_LIMIT = Time("1s")
DEFAULT_MEMORY_LIMIT = Memory("256m") DEFAULT_MEMORY_LIMIT = Memory("256m")
DEFAULT_FILE_LIMIT = Memory("32m") DEFAULT_FILE_LIMIT = Memory("32m")
DEFAULT_CASE_SCORE = 5 DEFAULT_CASE_SCORE = 5
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-05-22 15:12:26 +08:00
Outdated
Review
Copy Link

why is it removed?

why is it removed?
李衍志523370910113 commented 2025-05-22 15:17:04 +08:00
Outdated
Review
Copy Link

added a field in toml named diff.default_score and this 5 is now directly written numerically here:

class ParserDiff(BaseModel):
    output: Outputs = Outputs()
    default_score: int = 5

added back now.

added a field in toml named `diff.default_score` and this 5 is now directly written numerically here: ``` class ParserDiff(BaseModel): output: Outputs = Outputs() default_score: int = 5 ``` --- added back now.
张泊明518370910136 commented 2025-05-22 15:18:27 +08:00
Outdated
Review
Copy Link

why not

class ParserDiff(BaseModel):
    output: Outputs = Outputs()
    default_score: int = DEFAULT_CASE_SCORE
why not ``` class ParserDiff(BaseModel): output: Outputs = Outputs() default_score: int = DEFAULT_CASE_SCORE ```
李衍志523370910113 commented 2025-05-22 15:19:51 +08:00
Review
Copy Link

yes, this is the case now, sorry :)

yes, this is the case now, sorry :)
李衍志523370910113 commented 2025-05-22 15:20:25 +08:00
Review
Copy Link

do we need to also create a DEFAULT_PROC_LIMIT?

do we need to also create a `DEFAULT_PROC_LIMIT`?
张泊明518370910136 commented 2025-05-22 15:21:49 +08:00
Review
Copy Link

good idea, maybe we can also have DEFAULT_CLOCK_LIMIT_MULTIPLIER

good idea, maybe we can also have `DEFAULT_CLOCK_LIMIT_MULTIPLIER`
李衍志523370910113 commented 2025-05-22 15:28:10 +08:00
Outdated
Review
Copy Link

both added now.

both added now.
DEFAULT_CLOCK_LIMIT_MULTIPLIER = 2
DEFAULT_PROC_LIMIT = 50
JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj")
TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot")

9
joj3_config_generator/models/result.py
View File

@ -3,9 +3,11 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic import BaseModel, ConfigDict, Field, field_validator
from joj3_config_generator.models.const import ( from joj3_config_generator.models.const import (
DEFAULT_CLOCK_LIMIT_MULTIPLIER,
bomingzh marked this conversation as resolved
张泊明518370910136 commented 2025-05-23 07:45:05 +08:00
Review
Copy Link

Should be applied to other locations in transformers/task.py

Should be applied to other locations in `transformers/task.py`
DEFAULT_CPU_LIMIT, DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT, DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT, DEFAULT_MEMORY_LIMIT,
DEFAULT_PROC_LIMIT,
) )
@ -49,10 +51,13 @@ class Cmd(BaseModel):
stdout: Union[Collector, StreamOut] = Collector(name="stdout") stdout: Union[Collector, StreamOut] = Collector(name="stdout")
stderr: Union[Collector, StreamOut] = Collector(name="stderr") stderr: Union[Collector, StreamOut] = Collector(name="stderr")
cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit") cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit")
clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit") clock_limit: int = Field(
DEFAULT_CLOCK_LIMIT_MULTIPLIER * DEFAULT_CPU_LIMIT,
serialization_alias="clockLimit",
)
memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit") memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit")
stack_limit: int = Field(0, serialization_alias="stackLimit") stack_limit: int = Field(0, serialization_alias="stackLimit")
proc_limit: int = Field(50, serialization_alias="procLimit") proc_limit: int = Field(DEFAULT_PROC_LIMIT, serialization_alias="procLimit")
cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit") cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit")
cpu_set_limit: str = Field("", serialization_alias="cpuSetLimit") cpu_set_limit: str = Field("", serialization_alias="cpuSetLimit")
copy_in: Dict[str, InputFile] = Field({}, serialization_alias="copyIn") copy_in: Dict[str, InputFile] = Field({}, serialization_alias="copyIn")

11
joj3_config_generator/models/task.py
View File

@ -7,6 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
from joj3_config_generator.models.common import Memory, Time from joj3_config_generator.models.common import Memory, Time
from joj3_config_generator.models.const import ( from joj3_config_generator.models.const import (
DEFAULT_CASE_SCORE,
DEFAULT_CPU_LIMIT, DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT, DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT, DEFAULT_MEMORY_LIMIT,
@ -51,6 +52,7 @@ class Outputs(BaseModel):
class ParserDiff(BaseModel): class ParserDiff(BaseModel):
output: Outputs = Outputs() output: Outputs = Outputs()
default_score: int = DEFAULT_CASE_SCORE
class Files(BaseModel): class Files(BaseModel):
@ -125,9 +127,14 @@ class Stage(BaseModel):
@classmethod @classmethod
def gather_cases(cls: Type["Stage"], values: Dict[str, Any]) -> Dict[str, Any]: def gather_cases(cls: Type["Stage"], values: Dict[str, Any]) -> Dict[str, Any]:
cases = {k: v for k, v in values.items() if k.startswith("case")} cases = {k: v for k, v in values.items() if k.startswith("case")}
for key in cases: limit = values.get("limit", {})
parsed_cases = {}
for key, case in cases.items():
case_with_limit = {**limit, **case.get("limit", {})}
case_for_parsing = {**case, "limit": case_with_limit}
parsed_cases[key] = case_for_parsing
values.pop(key) values.pop(key)
values["cases"] = {k: v for k, v in cases.items()} values["cases"] = parsed_cases
return values return values

24
joj3_config_generator/transformers/task.py
View File

@ -4,10 +4,15 @@ from functools import partial
from pathlib import Path from pathlib import Path
from typing import Any, Callable, Dict, List, Set, Tuple from typing import Any, Callable, Dict, List, Set, Tuple
from joj3_config_generator.models import const, result, task from joj3_config_generator.models import result, task
from joj3_config_generator.models.common import Memory, Time from joj3_config_generator.models.common import Memory, Time
from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT from joj3_config_generator.models.const import (
DEFAULT_CLOCK_LIMIT_MULTIPLIER,
DEFAULT_PROC_LIMIT,
JOJ3_CONFIG_ROOT,
)
from joj3_config_generator.models.task import Parser as ParserEnum from joj3_config_generator.models.task import Parser as ParserEnum
from joj3_config_generator.utils.logger import logger
def get_conf_stage( def get_conf_stage(
@ -194,9 +199,9 @@ def fix_diff(
), ),
args=shlex.split(case_stage.command) if case_stage.command else None, args=shlex.split(case_stage.command) if case_stage.command else None,
cpu_limit=case_stage.limit.cpu, cpu_limit=case_stage.limit.cpu,
clock_limit=2 * case_stage.limit.cpu, clock_limit=DEFAULT_CLOCK_LIMIT_MULTIPLIER * case_stage.limit.cpu,
memory_limit=case_stage.limit.mem, memory_limit=case_stage.limit.mem,
proc_limit=50, proc_limit=DEFAULT_PROC_LIMIT,
bomingzh marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-05-22 16:06:21 +08:00
Outdated
Review
Copy Link

What if the with_.default.cpu_limit is not the same as DEFAULT_CPU_LIMIT? Why do we need to set these fields to none?

What if the `with_.default.cpu_limit` is not the same as `DEFAULT_CPU_LIMIT`? Why do we need to set these fields to none?
李衍志523370910113 commented 2025-05-22 17:20:27 +08:00
Outdated
Review
Copy Link

if the with_.default.cpu_limit is not the same as DEFAULT_CPU_LIMIT it means its already been input before, and it is considered as the new default value for all cases (ta might want to control it). If I dont set these field to none, it will use DEFAULT_CPU_LIMIT instead of those ta input, which is not intended. It solve the second problem in #15

if the `with_.default.cpu_limit` is not the same as `DEFAULT_CPU_LIMIT` it means its already been input before, and it is considered as the new default value for all cases (ta might want to control it). If I dont set these field to none, it will use `DEFAULT_CPU_LIMIT` instead of those ta input, which is not intended. It solve the second problem in https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3-config-generator/issues/15
张泊明518370910136 commented 2025-05-23 07:44:02 +08:00
Outdated
Review
Copy Link

Which test case will show this problem?

Which test case will show this problem?
张泊明518370910136 commented 2025-05-23 08:07:15 +08:00
Outdated
Review
Copy Link

We need another pydantic model for auto detected cases. Fields in these cases can be none, which means they are not set and should use with_.default values.

We need another pydantic model for auto detected cases. Fields in these cases can be none, which means they are not set and should use `with_.default` values.
) )
if cmd.args == executor.with_.default.args: if cmd.args == executor.with_.default.args:
cmd.args = None cmd.args = None
@ -224,13 +229,17 @@ def fix_diff(
parser_cases.append(parser_case) parser_cases.append(parser_case)
for case in default_cases: for case in default_cases:
cmd = result.OptionalCmd( cmd = result.OptionalCmd(
stdin=result.LocalFile(src=str(base_dir / f"{case}.in")) stdin=result.LocalFile(src=str(base_dir / f"{case}.in")),
cpu_limit=None,
clock_limit=None,
memory_limit=None,
proc_limit=None,
) )
stage_cases.append(cmd) stage_cases.append(cmd)
parser_case = result.DiffCasesConfig( parser_case = result.DiffCasesConfig(
outputs=[ outputs=[
result.DiffOutputConfig( result.DiffOutputConfig(
score=const.DEFAULT_CASE_SCORE, score=task_stage.diff.default_score,
file_name="stdout", file_name="stdout",
answer_path=str(base_dir / f"{case}.out"), answer_path=str(base_dir / f"{case}.out"),
) )
@ -246,6 +255,9 @@ def get_testcases(
) -> Set[str]: # basedir here should be task_conf.root / task_conf.path ) -> Set[str]: # basedir here should be task_conf.root / task_conf.path
testcases = set() testcases = set()
for testcases_path in (task_root / task_path).parent.glob("**/*.in"): for testcases_path in (task_root / task_path).parent.glob("**/*.in"):
if not testcases_path.with_suffix(".out").exists():
logger.warning(f"Testcase {testcases_path} has no corresponding .out file")
jon-lee marked this conversation as resolved
张泊明518370910136 commented 2025-05-23 20:42:10 +08:00
Review
Copy Link

if not testcases_path.with_suffix(".out").exists():

`if not testcases_path.with_suffix(".out").exists():`
李衍志523370910113 commented 2025-05-23 20:57:21 +08:00
Review
Copy Link

done

done
continue
testcases.add( testcases.add(
str( str(
testcases_path.relative_to((task_root / task_path).parent) testcases_path.relative_to((task_root / task_path).parent)
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-05-23 20:30:58 +08:00
Outdated
Review
Copy Link

No. Just do not append it to the return value and log a warning.

No. Just do not append it to the return value and log a warning.
李衍志523370910113 commented 2025-05-23 20:40:57 +08:00
Outdated
Review
Copy Link

we can probably move some redundant functions like get_testcaes into a utils.py as file llines already bit large

we can probably move some redundant functions like `get_testcaes` into a `utils.py` as file llines already bit large
张泊明518370910136 commented 2025-05-23 20:43:59 +08:00
Outdated
Review
Copy Link

leave it here now. we have not reused the logic.

leave it here now. we have not reused the logic.
👍 1

0
tests/convert/diff/case0.out Normal file
View File

0
tests/convert/diff/case1.out Normal file
View File

0
tests/convert/diff/case2.out Normal file
View File

75
tests/convert/diff/task.json
View File

@ -29,17 +29,17 @@
}, },
"stdout": { "stdout": {
"name": "stdout", "name": "stdout",
"max": 33554432, "max": 10485760,
"pipe": true "pipe": true
}, },
"stderr": { "stderr": {
"name": "stderr", "name": "stderr",
"max": 33554432, "max": 10485760,
"pipe": true "pipe": true
}, },
"cpuLimit": 1000000000, "cpuLimit": 3000000000,
"clockLimit": 2000000000, "clockLimit": 6000000000,
"memoryLimit": 68157440, "memoryLimit": 10485760,
"stackLimit": 0, "stackLimit": 0,
"procLimit": 50, "procLimit": 50,
"cpuRateLimit": 0, "cpuRateLimit": 0,
@ -71,6 +71,8 @@
"stdin": { "stdin": {
"src": "/home/tt/.config/joj/diff/case0.in" "src": "/home/tt/.config/joj/diff/case0.in"
}, },
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 2097152 "memoryLimit": 2097152
}, },
{ {
@ -87,35 +89,26 @@
{ {
"stdin": { "stdin": {
"src": "/home/tt/.config/joj/diff/case9.in" "src": "/home/tt/.config/joj/diff/case9.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/case5.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/case2.in"
} }
}, },
{ {
"stdin": { "stdin": {
"src": "/home/tt/.config/joj/diff/case3.in" "src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/case5.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/case2.in"
} }
}, },
{ {
@ -222,7 +215,7 @@
{ {
"outputs": [ "outputs": [
{ {
"score": 5, "score": 100,
"fileName": "stdout", "fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/case2.out", "answerPath": "/home/tt/.config/joj/diff/case2.out",
"forceQuitOnDiff": false, "forceQuitOnDiff": false,
@ -234,19 +227,7 @@
{ {
"outputs": [ "outputs": [
{ {
"score": 5, "score": 100,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/case3.out",
"forceQuitOnDiff": false,
"alwaysHide": false,
"compareSpace": false
}
]
},
{
"outputs": [
{
"score": 5,
"fileName": "stdout", "fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task1/case4.out", "answerPath": "/home/tt/.config/joj/diff/task1/case4.out",
"forceQuitOnDiff": false, "forceQuitOnDiff": false,
@ -258,7 +239,7 @@
{ {
"outputs": [ "outputs": [
{ {
"score": 5, "score": 100,
"fileName": "stdout", "fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task2/case6.out", "answerPath": "/home/tt/.config/joj/diff/task2/case6.out",
"forceQuitOnDiff": false, "forceQuitOnDiff": false,
@ -270,7 +251,7 @@
{ {
"outputs": [ "outputs": [
{ {
"score": 5, "score": 100,
"fileName": "stdout", "fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task2/case7.out", "answerPath": "/home/tt/.config/joj/diff/task2/case7.out",
"forceQuitOnDiff": false, "forceQuitOnDiff": false,
@ -282,7 +263,7 @@
{ {
"outputs": [ "outputs": [
{ {
"score": 5, "score": 100,
"fileName": "stdout", "fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task2/case8.out", "answerPath": "/home/tt/.config/joj/diff/task2/case8.out",
"forceQuitOnDiff": false, "forceQuitOnDiff": false,

9
tests/convert/diff/task.toml
View File

@ -8,13 +8,17 @@ release.begin_time = 2024-12-29 23:59:59+08:00
name = "[joj] ex2-asan" name = "[joj] ex2-asan"
command="./h7/build/ex2-asan -a" command="./h7/build/ex2-asan -a"
files.import = [ "h7/build/ex2-asan" ] files.import = [ "h7/build/ex2-asan" ]
limit.mem = "65m" limit.cpu = "3s"
limit.mem = "10m"
limit.stdout = "10m"
limit.stderr = "10m"
parsers = [ "diff", "result-detail" ] parsers = [ "diff", "result-detail" ]
result-detail.exitstatus = true result-detail.exitstatus = true
result-detail.stderr = true result-detail.stderr = true
# will be removed as long as the name is fixed diff.default_score = 100
case0.diff.output.score = 5 case0.diff.output.score = 5
case0.limit.cpu = "1s" case0.limit.cpu = "1s"
case0.limit.mem = "2m" case0.limit.mem = "2m"
@ -29,6 +33,7 @@ case1.diff.output.ignore_spaces = true
case1.command = "./h7/build/ex2" case1.command = "./h7/build/ex2"
case9.diff.output.score = 1232131 case9.diff.output.score = 1232131
case9.limit.mem = "10m"
case11.diff.output.score = 92321 case11.diff.output.score = 92321