JOJ/JOJ3-config-generator
27
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(diff): bugs on diff stdin and numerics #16

Merged
张泊明518370910136 merged 22 commits from fix/diff into master 2025-05-24 02:45:39 +08:00
Conversation 1 Commits 22 Files Changed 9 +71 -59
9 changed files with 71 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
joj3_config_generator/models/const.py
View File

@ -6,6 +6,8 @@ DEFAULT_CPU_LIMIT = Time("1s")
DEFAULT_MEMORY_LIMIT = Memory("256m")
DEFAULT_FILE_LIMIT = Memory("32m")
DEFAULT_CASE_SCORE = 5
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-05-22 15:12:26 +08:00
Outdated
Review
Copy Link

why is it removed?

why is it removed?
李衍志523370910113 commented 2025-05-22 15:17:04 +08:00
Outdated
Review
Copy Link

added a field in toml named diff.default_score and this 5 is now directly written numerically here:

class ParserDiff(BaseModel):
    output: Outputs = Outputs()
    default_score: int = 5

added back now.

added a field in toml named `diff.default_score` and this 5 is now directly written numerically here: ``` class ParserDiff(BaseModel): output: Outputs = Outputs() default_score: int = 5 ``` --- added back now.
张泊明518370910136 commented 2025-05-22 15:18:27 +08:00
Outdated
Review
Copy Link

why not

class ParserDiff(BaseModel):
    output: Outputs = Outputs()
    default_score: int = DEFAULT_CASE_SCORE
why not ``` class ParserDiff(BaseModel): output: Outputs = Outputs() default_score: int = DEFAULT_CASE_SCORE ```
李衍志523370910113 commented 2025-05-22 15:19:51 +08:00
Review
Copy Link

yes, this is the case now, sorry :)

yes, this is the case now, sorry :)
李衍志523370910113 commented 2025-05-22 15:20:25 +08:00
Review
Copy Link

do we need to also create a DEFAULT_PROC_LIMIT?

do we need to also create a `DEFAULT_PROC_LIMIT`?
张泊明518370910136 commented 2025-05-22 15:21:49 +08:00
Review
Copy Link

good idea, maybe we can also have DEFAULT_CLOCK_LIMIT_MULTIPLIER

good idea, maybe we can also have `DEFAULT_CLOCK_LIMIT_MULTIPLIER`
李衍志523370910113 commented 2025-05-22 15:28:10 +08:00
Outdated
Review
Copy Link

both added now.

both added now.
DEFAULT_CLOCK_LIMIT_MULTIPLIER = 2
DEFAULT_PROC_LIMIT = 50
JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj")
TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot")

9
joj3_config_generator/models/result.py
View File

@ -3,9 +3,11 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from pydantic import BaseModel, ConfigDict, Field, field_validator
from joj3_config_generator.models.const import (
DEFAULT_CLOCK_LIMIT_MULTIPLIER,
bomingzh marked this conversation as resolved
张泊明518370910136 commented 2025-05-23 07:45:05 +08:00
Review
Copy Link

Should be applied to other locations in transformers/task.py

Should be applied to other locations in `transformers/task.py`
DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT,
DEFAULT_PROC_LIMIT,
)
@ -49,10 +51,13 @@ class Cmd(BaseModel):
stdout: Union[Collector, StreamOut] = Collector(name="stdout")
stderr: Union[Collector, StreamOut] = Collector(name="stderr")
cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit")
clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit")
clock_limit: int = Field(
DEFAULT_CLOCK_LIMIT_MULTIPLIER * DEFAULT_CPU_LIMIT,
serialization_alias="clockLimit",
)
memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit")
stack_limit: int = Field(0, serialization_alias="stackLimit")
proc_limit: int = Field(50, serialization_alias="procLimit")
proc_limit: int = Field(DEFAULT_PROC_LIMIT, serialization_alias="procLimit")
cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit")
cpu_set_limit: str = Field("", serialization_alias="cpuSetLimit")
copy_in: Dict[str, InputFile] = Field({}, serialization_alias="copyIn")

11
joj3_config_generator/models/task.py
View File

@ -7,6 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
from joj3_config_generator.models.common import Memory, Time
from joj3_config_generator.models.const import (
DEFAULT_CASE_SCORE,
DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT,
@ -51,6 +52,7 @@ class Outputs(BaseModel):
class ParserDiff(BaseModel):
output: Outputs = Outputs()
default_score: int = DEFAULT_CASE_SCORE
class Files(BaseModel):
@ -125,9 +127,14 @@ class Stage(BaseModel):
@classmethod
def gather_cases(cls: Type["Stage"], values: Dict[str, Any]) -> Dict[str, Any]:
cases = {k: v for k, v in values.items() if k.startswith("case")}
for key in cases:
limit = values.get("limit", {})
parsed_cases = {}
for key, case in cases.items():
case_with_limit = {**limit, **case.get("limit", {})}
case_for_parsing = {**case, "limit": case_with_limit}
parsed_cases[key] = case_for_parsing
values.pop(key)
values["cases"] = {k: v for k, v in cases.items()}
values["cases"] = parsed_cases
return values

24
joj3_config_generator/transformers/task.py
View File

@ -4,10 +4,15 @@ from functools import partial
from pathlib import Path
from typing import Any, Callable, Dict, List, Set, Tuple
from joj3_config_generator.models import const, result, task
from joj3_config_generator.models import result, task
from joj3_config_generator.models.common import Memory, Time
from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT
from joj3_config_generator.models.const import (
DEFAULT_CLOCK_LIMIT_MULTIPLIER,
DEFAULT_PROC_LIMIT,
JOJ3_CONFIG_ROOT,
)
from joj3_config_generator.models.task import Parser as ParserEnum
from joj3_config_generator.utils.logger import logger
def get_conf_stage(
@ -194,9 +199,9 @@ def fix_diff(
),
args=shlex.split(case_stage.command) if case_stage.command else None,
cpu_limit=case_stage.limit.cpu,
clock_limit=2 * case_stage.limit.cpu,
clock_limit=DEFAULT_CLOCK_LIMIT_MULTIPLIER * case_stage.limit.cpu,
memory_limit=case_stage.limit.mem,
proc_limit=50,
proc_limit=DEFAULT_PROC_LIMIT,
bomingzh marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-05-22 16:06:21 +08:00
Outdated
Review
Copy Link

What if the with_.default.cpu_limit is not the same as DEFAULT_CPU_LIMIT? Why do we need to set these fields to none?

What if the `with_.default.cpu_limit` is not the same as `DEFAULT_CPU_LIMIT`? Why do we need to set these fields to none?
李衍志523370910113 commented 2025-05-22 17:20:27 +08:00
Outdated
Review
Copy Link

if the with_.default.cpu_limit is not the same as DEFAULT_CPU_LIMIT it means its already been input before, and it is considered as the new default value for all cases (ta might want to control it). If I dont set these field to none, it will use DEFAULT_CPU_LIMIT instead of those ta input, which is not intended. It solve the second problem in #15

if the `with_.default.cpu_limit` is not the same as `DEFAULT_CPU_LIMIT` it means its already been input before, and it is considered as the new default value for all cases (ta might want to control it). If I dont set these field to none, it will use `DEFAULT_CPU_LIMIT` instead of those ta input, which is not intended. It solve the second problem in https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3-config-generator/issues/15
张泊明518370910136 commented 2025-05-23 07:44:02 +08:00
Outdated
Review
Copy Link

Which test case will show this problem?

Which test case will show this problem?
张泊明518370910136 commented 2025-05-23 08:07:15 +08:00
Outdated
Review
Copy Link

We need another pydantic model for auto detected cases. Fields in these cases can be none, which means they are not set and should use with_.default values.

We need another pydantic model for auto detected cases. Fields in these cases can be none, which means they are not set and should use `with_.default` values.
)
if cmd.args == executor.with_.default.args:
cmd.args = None
@ -224,13 +229,17 @@ def fix_diff(
parser_cases.append(parser_case)
for case in default_cases:
cmd = result.OptionalCmd(
stdin=result.LocalFile(src=str(base_dir / f"{case}.in"))
stdin=result.LocalFile(src=str(base_dir / f"{case}.in")),
cpu_limit=None,
clock_limit=None,
memory_limit=None,
proc_limit=None,
)
stage_cases.append(cmd)
parser_case = result.DiffCasesConfig(
outputs=[
result.DiffOutputConfig(
score=const.DEFAULT_CASE_SCORE,
score=task_stage.diff.default_score,
file_name="stdout",
answer_path=str(base_dir / f"{case}.out"),
)
@ -246,6 +255,9 @@ def get_testcases(
) -> Set[str]: # basedir here should be task_conf.root / task_conf.path
testcases = set()
for testcases_path in (task_root / task_path).parent.glob("**/*.in"):
if not testcases_path.with_suffix(".out").exists():
logger.warning(f"Testcase {testcases_path} has no corresponding .out file")
jon-lee marked this conversation as resolved
张泊明518370910136 commented 2025-05-23 20:42:10 +08:00
Review
Copy Link

if not testcases_path.with_suffix(".out").exists():

`if not testcases_path.with_suffix(".out").exists():`
李衍志523370910113 commented 2025-05-23 20:57:21 +08:00
Review
Copy Link

done

done
continue
testcases.add(
str(
testcases_path.relative_to((task_root / task_path).parent)
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-05-23 20:30:58 +08:00
Outdated
Review
Copy Link

No. Just do not append it to the return value and log a warning.

No. Just do not append it to the return value and log a warning.
李衍志523370910113 commented 2025-05-23 20:40:57 +08:00
Outdated
Review
Copy Link

we can probably move some redundant functions like get_testcaes into a utils.py as file llines already bit large

we can probably move some redundant functions like `get_testcaes` into a `utils.py` as file llines already bit large
张泊明518370910136 commented 2025-05-23 20:43:59 +08:00
Outdated
Review
Copy Link

leave it here now. we have not reused the logic.

leave it here now. we have not reused the logic.
👍 1

0
tests/convert/diff/case0.out Normal file
View File

0
tests/convert/diff/case1.out Normal file
View File

0
tests/convert/diff/case2.out Normal file
View File

75
tests/convert/diff/task.json
View File

@ -29,17 +29,17 @@
},
"stdout": {
"name": "stdout",
"max": 33554432,
"max": 10485760,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 33554432,
"max": 10485760,
"pipe": true
},
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 68157440,
"cpuLimit": 3000000000,
"clockLimit": 6000000000,
"memoryLimit": 10485760,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -71,6 +71,8 @@
"stdin": {
"src": "/home/tt/.config/joj/diff/case0.in"
},
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 2097152
},
{
@ -87,35 +89,26 @@
{
"stdin": {
"src": "/home/tt/.config/joj/diff/case9.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/case5.in"
},
"memoryLimit": 268435456
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/case2.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/case3.in"
"src": "/home/tt/.config/joj/diff/task1/subtask1/case11.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/subtask1/case10.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/task1/case5.in"
}
},
{
"stdin": {
"src": "/home/tt/.config/joj/diff/case2.in"
}
},
{
@ -222,7 +215,7 @@
{
"outputs": [
{
"score": 5,
"score": 100,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/case2.out",
"forceQuitOnDiff": false,
@ -234,19 +227,7 @@
{
"outputs": [
{
"score": 5,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/case3.out",
"forceQuitOnDiff": false,
"alwaysHide": false,
"compareSpace": false
}
]
},
{
"outputs": [
{
"score": 5,
"score": 100,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task1/case4.out",
"forceQuitOnDiff": false,
@ -258,7 +239,7 @@
{
"outputs": [
{
"score": 5,
"score": 100,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task2/case6.out",
"forceQuitOnDiff": false,
@ -270,7 +251,7 @@
{
"outputs": [
{
"score": 5,
"score": 100,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task2/case7.out",
"forceQuitOnDiff": false,
@ -282,7 +263,7 @@
{
"outputs": [
{
"score": 5,
"score": 100,
"fileName": "stdout",
"answerPath": "/home/tt/.config/joj/diff/task2/case8.out",
"forceQuitOnDiff": false,

9
tests/convert/diff/task.toml
View File

@ -8,13 +8,17 @@ release.begin_time = 2024-12-29 23:59:59+08:00
name = "[joj] ex2-asan"
command="./h7/build/ex2-asan -a"
files.import = [ "h7/build/ex2-asan" ]
limit.mem = "65m"
limit.cpu = "3s"
limit.mem = "10m"
limit.stdout = "10m"
limit.stderr = "10m"
parsers = [ "diff", "result-detail" ]
result-detail.exitstatus = true
result-detail.stderr = true
# will be removed as long as the name is fixed
diff.default_score = 100
case0.diff.output.score = 5
case0.limit.cpu = "1s"
case0.limit.mem = "2m"
@ -29,6 +33,7 @@ case1.diff.output.ignore_spaces = true
case1.command = "./h7/build/ex2"
case9.diff.output.score = 1232131
case9.limit.mem = "10m"
case11.diff.output.score = 92321