From 8164e750581ba0e7a2954171584c2f98fb5017dd Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sun, 2 Mar 2025 03:37:30 -0500 Subject: [PATCH] feat: time & mem consts and defaults --- joj3_config_generator/models/common.py | 19 ++++ joj3_config_generator/models/const.py | 5 + joj3_config_generator/models/result.py | 41 ++++---- joj3_config_generator/models/task.py | 31 +++--- joj3_config_generator/processers/joj1.py | 23 ++-- joj3_config_generator/processers/task.py | 50 ++------- pyproject.toml | 1 - tests/convert/basic/task.json | 128 +++++++++++------------ tests/convert/clang-tidy/task.json | 11 +- tests/convert/cppcheck/task.json | 11 +- tests/convert/cpplint/task.json | 11 +- tests/convert/diff/task.json | 26 ++--- tests/convert/keyword/task.json | 11 +- 13 files changed, 168 insertions(+), 200 deletions(-) create mode 100644 joj3_config_generator/models/common.py create mode 100644 joj3_config_generator/models/const.py diff --git a/joj3_config_generator/models/common.py b/joj3_config_generator/models/common.py new file mode 100644 index 0000000..438f77d --- /dev/null +++ b/joj3_config_generator/models/common.py @@ -0,0 +1,19 @@ +from typing import Union + +import humanfriendly + + +class Memory(int): + def __new__(cls, value: Union[str, int]) -> "Memory": + if isinstance(value, str): + parsed = humanfriendly.parse_size(value, binary=True) + return super().__new__(cls, parsed) + return super().__new__(cls, value) + + +class Time(int): + def __new__(cls, value: Union[str, int]) -> "Time": + if isinstance(value, str): + parsed = humanfriendly.parse_timespan(value) * 1_000_000_000 # ns + return super().__new__(cls, round(parsed)) + return super().__new__(cls, value) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py new file mode 100644 index 0000000..87e9a5a --- /dev/null +++ b/joj3_config_generator/models/const.py @@ -0,0 +1,5 @@ +from joj3_config_generator.models.common import Memory, Time + +DEFAULT_CPU_LIMIT = Time("1s") +DEFAULT_MEMORY_LIMIT = Memory("128m") +DEFAULT_FILE_LIMIT = Memory("32m") diff --git a/joj3_config_generator/models/result.py b/joj3_config_generator/models/result.py index b61f989..9c1f3d4 100644 --- a/joj3_config_generator/models/result.py +++ b/joj3_config_generator/models/result.py @@ -1,8 +1,13 @@ from typing import Any, Dict, List, Optional, Union -import humanfriendly from pydantic import BaseModel, Field +from joj3_config_generator.models.const import ( + DEFAULT_CPU_LIMIT, + DEFAULT_FILE_LIMIT, + DEFAULT_MEMORY_LIMIT, +) + class LocalFile(BaseModel): src: str @@ -18,7 +23,7 @@ class PreparedFile(BaseModel): class Collector(BaseModel): name: str - max: int + max: int = DEFAULT_FILE_LIMIT pipe: bool = True @@ -38,17 +43,14 @@ InputFile = Union[LocalFile | MemoryFile | PreparedFile | Symlink] class Cmd(BaseModel): - args: Optional[List[str]] = None + args: List[str] = [] env: List[str] = [] - stdin: Optional[Union[InputFile | StreamIn]] = None - stdout: Optional[Union[Collector | StreamOut]] = None - stderr: Optional[Union[Collector | StreamOut]] = None - cpu_limit: int = Field(1_000_000_000, serialization_alias="cpuLimit") - real_cpu_limit: int = Field(1_000_000_000, serialization_alias="realCpuLimit") - clock_limit: int = Field(2 * 1_000_000_000, serialization_alias="clockLimit") - memory_limit: int = Field( - humanfriendly.parse_size("128m"), serialization_alias="memoryLimit" - ) + stdin: Union[InputFile | StreamIn] = MemoryFile(content="") + stdout: Union[Collector | StreamOut] = Collector(name="stdout") + stderr: Union[Collector | StreamOut] = Collector(name="stderr") + cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit") + clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit") + memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit") stack_limit: int = Field(0, serialization_alias="stackLimit") proc_limit: int = Field(50, serialization_alias="procLimit") cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit") @@ -74,15 +76,10 @@ class OptionalCmd(BaseModel): stdout: Optional[Union[Collector | StreamOut]] = None stderr: Optional[Union[Collector | StreamOut]] = None cpu_limit: Optional[int] = Field(None, serialization_alias="cpuLimit") - real_cpu_limit: Optional[int] = Field(None, serialization_alias="realCpuLimit") - clock_limit: Optional[int] = Field( - 2 * 1_000_000_000, serialization_alias="clockLimit" - ) - memory_limit: Optional[int] = Field( - humanfriendly.parse_size("128m"), serialization_alias="memoryLimit" - ) + clock_limit: Optional[int] = Field(None, serialization_alias="clockLimit") + memory_limit: Optional[int] = Field(None, serialization_alias="memoryLimit") stack_limit: Optional[int] = Field(None, serialization_alias="stackLimit") - proc_limit: Optional[int] = Field(50, serialization_alias="procLimit") + proc_limit: Optional[int] = Field(None, serialization_alias="procLimit") cpu_rate_limit: Optional[int] = Field(None, serialization_alias="cpuRateLimit") cpu_set_limit: Optional[str] = Field(None, serialization_alias="cpuSetLimit") copy_in: Optional[Dict[str, InputFile]] = Field(None, serialization_alias="copyIn") @@ -90,9 +87,7 @@ class OptionalCmd(BaseModel): None, serialization_alias="copyInCached" ) copy_in_dir: Optional[str] = Field(None, serialization_alias="copyInDir") - copy_out: Optional[List[str]] = Field( - ["stdout", "stderr"], serialization_alias="copyOut" - ) + copy_out: Optional[List[str]] = Field(None, serialization_alias="copyOut") copy_out_cached: Optional[List[str]] = Field( None, serialization_alias="copyOutCached" ) diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index f1c444f..7d3ffd0 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -2,9 +2,14 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Type -import humanfriendly from pydantic import BaseModel, Field, model_validator +from joj3_config_generator.models.const import ( + DEFAULT_CPU_LIMIT, + DEFAULT_FILE_LIMIT, + DEFAULT_MEMORY_LIMIT, +) + class ParserResultDetail(BaseModel): time: Optional[bool] = True # Display run time @@ -31,8 +36,8 @@ class ParserDummy(BaseModel): class ParserKeyword(BaseModel): - keyword: Optional[list[str]] = [] - weight: Optional[list[int]] = [] + keyword: Optional[List[str]] = [] + weight: Optional[List[int]] = [] class Outputs(BaseModel): @@ -47,27 +52,27 @@ class ParserDiff(BaseModel): class Files(BaseModel): - import_: Optional[list[str]] = Field([], alias="import") - export: Optional[list[str]] = [] + import_: Optional[List[str]] = Field([], alias="import") + export: Optional[List[str]] = [] class Limit(BaseModel): - mem: int = humanfriendly.parse_size("128M") - cpu: int = 1_000_000_000 - stderr: int = humanfriendly.parse_size("128M") - stdout: int = humanfriendly.parse_size("128M") + mem: int = DEFAULT_MEMORY_LIMIT + cpu: int = DEFAULT_CPU_LIMIT + stderr: int = DEFAULT_FILE_LIMIT + stdout: int = DEFAULT_FILE_LIMIT class Stage(BaseModel): name: Optional[str] = None # Stage name - env: Optional[list[str]] = None + env: Optional[List[str]] = None command: Optional[str] = None # Command to run files: Optional[Files] = None in_: Optional[str] = Field(None, alias="in") out_: Optional[str] = Field(None, alias="out") score: Optional[int] = 0 - parsers: Optional[list[str]] = [] # list of parsers - limit: Optional[Limit] = Limit() + parsers: Optional[List[str]] = [] # list of parsers + limit: Limit = Limit() dummy: Optional[ParserDummy] = ParserDummy() result_status: Optional[ParserDummy] = Field(ParserDummy(), alias="result-status") keyword: Optional[ParserKeyword] = ParserKeyword() @@ -78,7 +83,7 @@ class Stage(BaseModel): ParserResultDetail(), alias="result-detail" ) file: Optional[ParserFile] = ParserFile() - skip: Optional[list[str]] = [] + skip: Optional[List[str]] = [] # cases related cases: Optional[Dict[str, "Stage"]] = None diff --git a/joj3_config_generator/processers/joj1.py b/joj3_config_generator/processers/joj1.py index d4ee739..5c21549 100644 --- a/joj3_config_generator/processers/joj1.py +++ b/joj3_config_generator/processers/joj1.py @@ -1,12 +1,9 @@ -import humanfriendly -from pytimeparse.timeparse import timeparse - from joj3_config_generator.models import joj1, task +from joj3_config_generator.models.common import Memory, Time +from joj3_config_generator.models.const import DEFAULT_CPU_LIMIT, DEFAULT_MEMORY_LIMIT def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage: - default_cpu = timeparse("1s") - default_mem = humanfriendly.parse_size("32m") cases_conf = [] for i, case in enumerate(joj1_config.cases): cases_conf.append( @@ -14,12 +11,8 @@ def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage: score=case.score, command=case.execute_args if case.execute_args else None, limit=task.Limit( - cpu=timeparse(case.time) if case.time else default_cpu, - mem=( - humanfriendly.parse_size(case.memory) - if case.memory - else default_mem - ), + cpu=Time(case.time) if case.time else DEFAULT_CPU_LIMIT, + mem=(Memory(case.memory) if case.memory else DEFAULT_MEMORY_LIMIT), ), ) ) @@ -32,14 +25,14 @@ def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage: score=100, limit=task.Limit( cpu=( - timeparse(joj1_config.cases[0].time) + Time(joj1_config.cases[0].time) if joj1_config.cases[0].time is not None - else default_cpu + else DEFAULT_CPU_LIMIT ), mem=( - humanfriendly.parse_size(joj1_config.cases[0].memory) + Memory(joj1_config.cases[0].memory) if joj1_config.cases[0].memory is not None - else default_mem + else DEFAULT_MEMORY_LIMIT ), ), cases={f"case{i}": cases_conf[i] for i, _ in enumerate(cases_conf)}, diff --git a/joj3_config_generator/processers/task.py b/joj3_config_generator/processers/task.py index e7a411f..7bef77c 100644 --- a/joj3_config_generator/processers/task.py +++ b/joj3_config_generator/processers/task.py @@ -65,52 +65,14 @@ def get_executor_with_config( # are there any corner cases for file in copy_in_files }, - stdin=( - result.MemoryFile(content="") - if ( - (task_stage.parsers is not None) - and ("diff" not in task_stage.parsers) - ) - else None - ), copy_out=copy_out_files, copy_in_cached={file: file for file in cached}, - copy_out_cached=file_export if file_export is not None else [], - cpu_limit=( - task_stage.limit.cpu * 1_000_000_000 - if task_stage.limit is not None and task_stage.limit.cpu is not None - else 80 * 1_000_000_000 - ), - clock_limit=( - 2 * task_stage.limit.cpu * 1_000_000_000 - if task_stage.limit is not None and task_stage.limit.cpu is not None - else 80 * 1_000_000_000 - ), - memory_limit=( - task_stage.limit.mem * 1_024 * 1_024 - if task_stage.limit is not None and task_stage.limit.mem is not None - else 128 * 1_024 * 1_024 - ), - stderr=result.Collector( - name="stderr", - max=( - task_stage.limit.stderr * 1_000_000_000 - if task_stage.limit is not None - and task_stage.limit.stderr is not None - else 128 * 1_024 * 1_024 - ), - pipe=True, - ), - stdout=result.Collector( - name="stdout", - max=( - task_stage.limit.stdout * 1_000_000_000 - if task_stage.limit is not None - and task_stage.limit.stdout is not None - else 128 * 1_024 * 1_024 - ), - pipe=True, - ), + copy_out_cached=file_export or [], + cpu_limit=task_stage.limit.cpu, + clock_limit=2 * task_stage.limit.cpu, + memory_limit=task_stage.limit.mem, + stderr=result.Collector(name="stderr"), + stdout=result.Collector(name="stdout"), ), cases=[], ) diff --git a/pyproject.toml b/pyproject.toml index 091c982..88219fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ dependencies = [ "inquirer>=3.4.0", "rtoml>=0.11.0", "humanfriendly>=10.0", - "pytimeparse>=1.1.8", ] requires-python = ">=3.9" authors = [{ name = "JOJ3-dev", email = "joj3@focs.ji.sjtu.edu.cn" }] diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 0eb1814..4495071 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -17,11 +17,24 @@ "name": "local", "with": { "default": { + "args": [], "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, "cpuLimit": 1000000000, - "realCpuLimit": 1000000000, "clockLimit": 2000000000, - "memoryLimit": 128000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -51,13 +64,6 @@ "-meta=Changelog.md", "-checkFileSumList=a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc,b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,f6740081487ca34963a005209e2e9adfdf6f3561719af082d40fe80145e0cceb,bbeca1491c2f8364821a328a6677c0c5d59ccd60250abac3cec0887eeb9bde3e", "-checkFileNameList=.gitignore,.gitattributes,.gitea/workflows/push.yaml,.gitea/workflows/release.yaml" - ], - "clockLimit": 2000000000, - "memoryLimit": 128000000, - "procLimit": 50, - "copyOut": [ - "stdout", - "stderr" ] }, { @@ -72,13 +78,6 @@ ], "env": [ "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" - ], - "clockLimit": 2000000000, - "memoryLimit": 128000000, - "procLimit": 50, - "copyOut": [ - "stdout", - "stderr" ] } ] @@ -115,18 +114,17 @@ }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -203,18 +201,17 @@ }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -304,18 +301,17 @@ }, "stdout": { "name": "stdout", - "max": 65000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -431,18 +427,17 @@ }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 65000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -530,18 +525,17 @@ }, "stdout": { "name": "stdout", - "max": 65000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -623,20 +617,22 @@ "-a" ], "env": [], + "stdin": { + "content": "" + }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 95656304705536, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 91224961, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -670,11 +666,7 @@ "cpuLimit": 1000000000, "clockLimit": 2000000000, "memoryLimit": 95656304705536, - "procLimit": 50, - "copyOut": [ - "stdout", - "stderr" - ] + "procLimit": 50 }, { "stdin": { @@ -683,11 +675,7 @@ "cpuLimit": 1000000000, "clockLimit": 2000000000, "memoryLimit": 95656304705536, - "procLimit": 50, - "copyOut": [ - "stdout", - "stderr" - ] + "procLimit": 50 } ] } @@ -762,10 +750,22 @@ "env": [ "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, "cpuLimit": 1000000000, - "realCpuLimit": 1000000000, "clockLimit": 2000000000, - "memoryLimit": 128000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, diff --git a/tests/convert/clang-tidy/task.json b/tests/convert/clang-tidy/task.json index 6197bc0..71b22f4 100644 --- a/tests/convert/clang-tidy/task.json +++ b/tests/convert/clang-tidy/task.json @@ -32,18 +32,17 @@ }, "stdout": { "name": "stdout", - "max": 65000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, diff --git a/tests/convert/cppcheck/task.json b/tests/convert/cppcheck/task.json index 690bac2..9a08ae9 100644 --- a/tests/convert/cppcheck/task.json +++ b/tests/convert/cppcheck/task.json @@ -32,18 +32,17 @@ }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 65000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, diff --git a/tests/convert/cpplint/task.json b/tests/convert/cpplint/task.json index e09c8cb..51139c0 100644 --- a/tests/convert/cpplint/task.json +++ b/tests/convert/cpplint/task.json @@ -31,18 +31,17 @@ }, "stdout": { "name": "stdout", - "max": 65000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json index eac19ea..17a4a59 100644 --- a/tests/convert/diff/task.json +++ b/tests/convert/diff/task.json @@ -22,20 +22,22 @@ "-a" ], "env": [], + "stdin": { + "content": "" + }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 95656304705536, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 91224961, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0, @@ -67,11 +69,7 @@ "cpuLimit": 1000000000, "clockLimit": 2000000000, "memoryLimit": 95656304705536, - "procLimit": 50, - "copyOut": [ - "stdout", - "stderr" - ] + "procLimit": 50 }, { "stdin": { @@ -80,11 +78,7 @@ "cpuLimit": 1000000000, "clockLimit": 2000000000, "memoryLimit": 95656304705536, - "procLimit": 50, - "copyOut": [ - "stdout", - "stderr" - ] + "procLimit": 50 } ] } diff --git a/tests/convert/keyword/task.json b/tests/convert/keyword/task.json index c7b212d..6c5725e 100644 --- a/tests/convert/keyword/task.json +++ b/tests/convert/keyword/task.json @@ -30,18 +30,17 @@ }, "stdout": { "name": "stdout", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, "stderr": { "name": "stderr", - "max": 128000000000000000, + "max": 33554432, "pipe": true }, - "cpuLimit": 1000000000000000000, - "realCpuLimit": 1000000000, - "clockLimit": 2000000000000000000, - "memoryLimit": 134217728000000, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, "procLimit": 50, "cpuRateLimit": 0,