JOJ/JOJ3-config-generator
27
1
Fork 0
Code Issues4 Pull Requests1 Actions Packages Projects Releases Wiki Activity

feat: time & mem consts and defaults

Browse Source
This commit is contained in:
张泊明518370910136 2025-03-02 03:37:30 -05:00
parent 46cafc0ed6
commit 8164e75058
Signed by: 张泊明518370910136
GPG Key ID: D47306D7062CDA9D
13 changed files with 168 additions and 200 deletions
joj3_config_generator
models
common.pyconst.pyresult.pytask.py
processers
joj1.pytask.py
pyproject.toml
tests/convert
basic
task.json
clang-tidy
task.json
cppcheck
task.json
cpplint
task.json
diff
task.json
keyword
task.json

19
joj3_config_generator/models/common.py Normal file
View File

@ -0,0 +1,19 @@
from typing import Union
import humanfriendly
class Memory(int):
def __new__(cls, value: Union[str, int]) -> "Memory":
if isinstance(value, str):
parsed = humanfriendly.parse_size(value, binary=True)
return super().__new__(cls, parsed)
return super().__new__(cls, value)
class Time(int):
def __new__(cls, value: Union[str, int]) -> "Time":
if isinstance(value, str):
parsed = humanfriendly.parse_timespan(value) * 1_000_000_000 # ns
return super().__new__(cls, round(parsed))
return super().__new__(cls, value)

5
joj3_config_generator/models/const.py Normal file
View File

@ -0,0 +1,5 @@
from joj3_config_generator.models.common import Memory, Time
DEFAULT_CPU_LIMIT = Time("1s")
DEFAULT_MEMORY_LIMIT = Memory("128m")
DEFAULT_FILE_LIMIT = Memory("32m")

41
joj3_config_generator/models/result.py
View File

@ -1,8 +1,13 @@
from typing import Any, Dict, List, Optional, Union
import humanfriendly
from pydantic import BaseModel, Field
from joj3_config_generator.models.const import (
DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT,
)
class LocalFile(BaseModel):
src: str
@ -18,7 +23,7 @@ class PreparedFile(BaseModel):
class Collector(BaseModel):
name: str
max: int
max: int = DEFAULT_FILE_LIMIT
pipe: bool = True
@ -38,17 +43,14 @@ InputFile = Union[LocalFile | MemoryFile | PreparedFile | Symlink]
class Cmd(BaseModel):
args: Optional[List[str]] = None
args: List[str] = []
env: List[str] = []
stdin: Optional[Union[InputFile | StreamIn]] = None
stdout: Optional[Union[Collector | StreamOut]] = None
stderr: Optional[Union[Collector | StreamOut]] = None
cpu_limit: int = Field(1_000_000_000, serialization_alias="cpuLimit")
real_cpu_limit: int = Field(1_000_000_000, serialization_alias="realCpuLimit")
clock_limit: int = Field(2 * 1_000_000_000, serialization_alias="clockLimit")
memory_limit: int = Field(
humanfriendly.parse_size("128m"), serialization_alias="memoryLimit"
)
stdin: Union[InputFile | StreamIn] = MemoryFile(content="")
stdout: Union[Collector | StreamOut] = Collector(name="stdout")
stderr: Union[Collector | StreamOut] = Collector(name="stderr")
cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit")
clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit")
memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit")
stack_limit: int = Field(0, serialization_alias="stackLimit")
proc_limit: int = Field(50, serialization_alias="procLimit")
cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit")
@ -74,15 +76,10 @@ class OptionalCmd(BaseModel):
stdout: Optional[Union[Collector | StreamOut]] = None
stderr: Optional[Union[Collector | StreamOut]] = None
cpu_limit: Optional[int] = Field(None, serialization_alias="cpuLimit")
real_cpu_limit: Optional[int] = Field(None, serialization_alias="realCpuLimit")
clock_limit: Optional[int] = Field(
2 * 1_000_000_000, serialization_alias="clockLimit"
)
memory_limit: Optional[int] = Field(
humanfriendly.parse_size("128m"), serialization_alias="memoryLimit"
)
clock_limit: Optional[int] = Field(None, serialization_alias="clockLimit")
memory_limit: Optional[int] = Field(None, serialization_alias="memoryLimit")
stack_limit: Optional[int] = Field(None, serialization_alias="stackLimit")
proc_limit: Optional[int] = Field(50, serialization_alias="procLimit")
proc_limit: Optional[int] = Field(None, serialization_alias="procLimit")
cpu_rate_limit: Optional[int] = Field(None, serialization_alias="cpuRateLimit")
cpu_set_limit: Optional[str] = Field(None, serialization_alias="cpuSetLimit")
copy_in: Optional[Dict[str, InputFile]] = Field(None, serialization_alias="copyIn")
@ -90,9 +87,7 @@ class OptionalCmd(BaseModel):
None, serialization_alias="copyInCached"
)
copy_in_dir: Optional[str] = Field(None, serialization_alias="copyInDir")
copy_out: Optional[List[str]] = Field(
["stdout", "stderr"], serialization_alias="copyOut"
)
copy_out: Optional[List[str]] = Field(None, serialization_alias="copyOut")
copy_out_cached: Optional[List[str]] = Field(
None, serialization_alias="copyOutCached"
)

31
joj3_config_generator/models/task.py
View File

@ -2,9 +2,14 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Type
import humanfriendly
from pydantic import BaseModel, Field, model_validator
from joj3_config_generator.models.const import (
DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT,
)
class ParserResultDetail(BaseModel):
time: Optional[bool] = True # Display run time
@ -31,8 +36,8 @@ class ParserDummy(BaseModel):
class ParserKeyword(BaseModel):
keyword: Optional[list[str]] = []
weight: Optional[list[int]] = []
keyword: Optional[List[str]] = []
weight: Optional[List[int]] = []
class Outputs(BaseModel):
@ -47,27 +52,27 @@ class ParserDiff(BaseModel):
class Files(BaseModel):
import_: Optional[list[str]] = Field([], alias="import")
export: Optional[list[str]] = []
import_: Optional[List[str]] = Field([], alias="import")
export: Optional[List[str]] = []
class Limit(BaseModel):
mem: int = humanfriendly.parse_size("128M")
cpu: int = 1_000_000_000
stderr: int = humanfriendly.parse_size("128M")
stdout: int = humanfriendly.parse_size("128M")
mem: int = DEFAULT_MEMORY_LIMIT
cpu: int = DEFAULT_CPU_LIMIT
stderr: int = DEFAULT_FILE_LIMIT
stdout: int = DEFAULT_FILE_LIMIT
class Stage(BaseModel):
name: Optional[str] = None # Stage name
env: Optional[list[str]] = None
env: Optional[List[str]] = None
command: Optional[str] = None # Command to run
files: Optional[Files] = None
in_: Optional[str] = Field(None, alias="in")
out_: Optional[str] = Field(None, alias="out")
score: Optional[int] = 0
parsers: Optional[list[str]] = [] # list of parsers
limit: Optional[Limit] = Limit()
parsers: Optional[List[str]] = [] # list of parsers
limit: Limit = Limit()
dummy: Optional[ParserDummy] = ParserDummy()
result_status: Optional[ParserDummy] = Field(ParserDummy(), alias="result-status")
keyword: Optional[ParserKeyword] = ParserKeyword()
@ -78,7 +83,7 @@ class Stage(BaseModel):
ParserResultDetail(), alias="result-detail"
)
file: Optional[ParserFile] = ParserFile()
skip: Optional[list[str]] = []
skip: Optional[List[str]] = []
# cases related
cases: Optional[Dict[str, "Stage"]] = None

23
joj3_config_generator/processers/joj1.py
View File

@ -1,12 +1,9 @@
import humanfriendly
from pytimeparse.timeparse import timeparse
from joj3_config_generator.models import joj1, task
from joj3_config_generator.models.common import Memory, Time
from joj3_config_generator.models.const import DEFAULT_CPU_LIMIT, DEFAULT_MEMORY_LIMIT
def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
default_cpu = timeparse("1s")
default_mem = humanfriendly.parse_size("32m")
cases_conf = []
for i, case in enumerate(joj1_config.cases):
cases_conf.append(
@ -14,12 +11,8 @@ def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
score=case.score,
command=case.execute_args if case.execute_args else None,
limit=task.Limit(
cpu=timeparse(case.time) if case.time else default_cpu,
mem=(
humanfriendly.parse_size(case.memory)
if case.memory
else default_mem
),
cpu=Time(case.time) if case.time else DEFAULT_CPU_LIMIT,
mem=(Memory(case.memory) if case.memory else DEFAULT_MEMORY_LIMIT),
),
)
)
@ -32,14 +25,14 @@ def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
score=100,
limit=task.Limit(
cpu=(
timeparse(joj1_config.cases[0].time)
Time(joj1_config.cases[0].time)
if joj1_config.cases[0].time is not None
else default_cpu
else DEFAULT_CPU_LIMIT
),
mem=(
humanfriendly.parse_size(joj1_config.cases[0].memory)
Memory(joj1_config.cases[0].memory)
if joj1_config.cases[0].memory is not None
else default_mem
else DEFAULT_MEMORY_LIMIT
),
),
cases={f"case{i}": cases_conf[i] for i, _ in enumerate(cases_conf)},

50
joj3_config_generator/processers/task.py
View File

@ -65,52 +65,14 @@ def get_executor_with_config(
# are there any corner cases
for file in copy_in_files
},
stdin=(
result.MemoryFile(content="")
if (
(task_stage.parsers is not None)
and ("diff" not in task_stage.parsers)
)
else None
),
copy_out=copy_out_files,
copy_in_cached={file: file for file in cached},
copy_out_cached=file_export if file_export is not None else [],
cpu_limit=(
task_stage.limit.cpu * 1_000_000_000
if task_stage.limit is not None and task_stage.limit.cpu is not None
else 80 * 1_000_000_000
),
clock_limit=(
2 * task_stage.limit.cpu * 1_000_000_000
if task_stage.limit is not None and task_stage.limit.cpu is not None
else 80 * 1_000_000_000
),
memory_limit=(
task_stage.limit.mem * 1_024 * 1_024
if task_stage.limit is not None and task_stage.limit.mem is not None
else 128 * 1_024 * 1_024
),
stderr=result.Collector(
name="stderr",
max=(
task_stage.limit.stderr * 1_000_000_000
if task_stage.limit is not None
and task_stage.limit.stderr is not None
else 128 * 1_024 * 1_024
),
pipe=True,
),
stdout=result.Collector(
name="stdout",
max=(
task_stage.limit.stdout * 1_000_000_000
if task_stage.limit is not None
and task_stage.limit.stdout is not None
else 128 * 1_024 * 1_024
),
pipe=True,
),
copy_out_cached=file_export or [],
cpu_limit=task_stage.limit.cpu,
clock_limit=2 * task_stage.limit.cpu,
memory_limit=task_stage.limit.mem,
stderr=result.Collector(name="stderr"),
stdout=result.Collector(name="stdout"),
),
cases=[],
)

1
pyproject.toml
View File

@ -13,7 +13,6 @@ dependencies = [
"inquirer>=3.4.0",
"rtoml>=0.11.0",
"humanfriendly>=10.0",
"pytimeparse>=1.1.8",
]
requires-python = ">=3.9"
authors = [{ name = "JOJ3-dev", email = "joj3@focs.ji.sjtu.edu.cn" }]

128
tests/convert/basic/task.json
View File

@ -17,11 +17,24 @@
"name": "local",
"with": {
"default": {
"args": [],
"env": [],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -51,13 +64,6 @@
"-meta=Changelog.md",
"-checkFileSumList=a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc,b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,f6740081487ca34963a005209e2e9adfdf6f3561719af082d40fe80145e0cceb,bbeca1491c2f8364821a328a6677c0c5d59ccd60250abac3cec0887eeb9bde3e",
"-checkFileNameList=.gitignore,.gitattributes,.gitea/workflows/push.yaml,.gitea/workflows/release.yaml"
],
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
},
{
@ -72,13 +78,6 @@
],
"env": [
"LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
],
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
}
]
@ -115,18 +114,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -203,18 +201,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -304,18 +301,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -431,18 +427,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -530,18 +525,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -623,20 +617,22 @@
"-a"
],
"env": [],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 95656304705536,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 91224961,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -670,11 +666,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
},
{
"stdin": {
@ -683,11 +675,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
}
]
}
@ -762,10 +750,22 @@
"env": [
"LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

11
tests/convert/clang-tidy/task.json
View File

@ -32,18 +32,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

11
tests/convert/cppcheck/task.json
View File

@ -32,18 +32,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

11
tests/convert/cpplint/task.json
View File

@ -31,18 +31,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

26
tests/convert/diff/task.json
View File

@ -22,20 +22,22 @@
"-a"
],
"env": [],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 95656304705536,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 91224961,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -67,11 +69,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
},
{
"stdin": {
@ -80,11 +78,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
}
]
}

11
tests/convert/keyword/task.json
View File

@ -30,18 +30,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,