dev #10

Merged
李衍志523370910113 merged 238 commits from dev into master 2025-03-05 16:20:39 +08:00
13 changed files with 168 additions and 200 deletions
Showing only changes of commit 8164e75058 - Show all commits

View File

@ -0,0 +1,19 @@
from typing import Union
import humanfriendly
class Memory(int):
jon-lee marked this conversation as resolved Outdated

what compact?

what compact?

no int input, should be done already, so it can be removed. I added that several commit before before you removed that 😇

no int input, should be done already, so it can be removed. I added that several commit before before you removed that 😇
def __new__(cls, value: Union[str, int]) -> "Memory":
if isinstance(value, str):
parsed = humanfriendly.parse_size(value, binary=True)
return super().__new__(cls, parsed)
return super().__new__(cls, value)
class Time(int):
def __new__(cls, value: Union[str, int]) -> "Time":
if isinstance(value, str):
parsed = humanfriendly.parse_timespan(value) * 1_000_000_000 # ns
return super().__new__(cls, round(parsed))
return super().__new__(cls, value)

View File

@ -0,0 +1,5 @@
from joj3_config_generator.models.common import Memory, Time
DEFAULT_CPU_LIMIT = Time("1s")
DEFAULT_MEMORY_LIMIT = Memory("128m")
DEFAULT_FILE_LIMIT = Memory("32m")

View File

@ -1,8 +1,13 @@
from typing import Any, Dict, List, Optional, Union
import humanfriendly
from pydantic import BaseModel, Field
from joj3_config_generator.models.const import (
DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT,
)
class LocalFile(BaseModel):
src: str
@ -18,7 +23,7 @@ class PreparedFile(BaseModel):
class Collector(BaseModel):
jon-lee marked this conversation as resolved Outdated

default value too large, should be 1s 128MB

default value too large, should be 1s 128MB

will fix later, we need to hold some test I reckon?

will fix later, we need to hold some test I reckon?

@bomingzh fixed

@bomingzh fixed
name: str
max: int
max: int = DEFAULT_FILE_LIMIT
pipe: bool = True
@ -38,17 +43,14 @@ InputFile = Union[LocalFile | MemoryFile | PreparedFile | Symlink]
class Cmd(BaseModel):
args: Optional[List[str]] = None
args: List[str] = []
env: List[str] = []
stdin: Optional[Union[InputFile | StreamIn]] = None
stdout: Optional[Union[Collector | StreamOut]] = None
stderr: Optional[Union[Collector | StreamOut]] = None
cpu_limit: int = Field(1_000_000_000, serialization_alias="cpuLimit")
real_cpu_limit: int = Field(1_000_000_000, serialization_alias="realCpuLimit")
clock_limit: int = Field(2 * 1_000_000_000, serialization_alias="clockLimit")
memory_limit: int = Field(
humanfriendly.parse_size("128m"), serialization_alias="memoryLimit"
)
stdin: Union[InputFile | StreamIn] = MemoryFile(content="")
stdout: Union[Collector | StreamOut] = Collector(name="stdout")
stderr: Union[Collector | StreamOut] = Collector(name="stderr")
cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit")
clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit")
memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit")
stack_limit: int = Field(0, serialization_alias="stackLimit")
proc_limit: int = Field(50, serialization_alias="procLimit")
cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit")
@ -74,15 +76,10 @@ class OptionalCmd(BaseModel):
stdout: Optional[Union[Collector | StreamOut]] = None
stderr: Optional[Union[Collector | StreamOut]] = None
cpu_limit: Optional[int] = Field(None, serialization_alias="cpuLimit")
real_cpu_limit: Optional[int] = Field(None, serialization_alias="realCpuLimit")
clock_limit: Optional[int] = Field(
2 * 1_000_000_000, serialization_alias="clockLimit"
)
memory_limit: Optional[int] = Field(
humanfriendly.parse_size("128m"), serialization_alias="memoryLimit"
)
clock_limit: Optional[int] = Field(None, serialization_alias="clockLimit")
memory_limit: Optional[int] = Field(None, serialization_alias="memoryLimit")
stack_limit: Optional[int] = Field(None, serialization_alias="stackLimit")
proc_limit: Optional[int] = Field(50, serialization_alias="procLimit")
proc_limit: Optional[int] = Field(None, serialization_alias="procLimit")
cpu_rate_limit: Optional[int] = Field(None, serialization_alias="cpuRateLimit")
cpu_set_limit: Optional[str] = Field(None, serialization_alias="cpuSetLimit")
copy_in: Optional[Dict[str, InputFile]] = Field(None, serialization_alias="copyIn")
@ -90,9 +87,7 @@ class OptionalCmd(BaseModel):
None, serialization_alias="copyInCached"
)
copy_in_dir: Optional[str] = Field(None, serialization_alias="copyInDir")
copy_out: Optional[List[str]] = Field(
["stdout", "stderr"], serialization_alias="copyOut"
)
copy_out: Optional[List[str]] = Field(None, serialization_alias="copyOut")
copy_out_cached: Optional[List[str]] = Field(
None, serialization_alias="copyOutCached"
)

View File

@ -2,9 +2,14 @@ from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Type
import humanfriendly
from pydantic import BaseModel, Field, model_validator
from joj3_config_generator.models.const import (
DEFAULT_CPU_LIMIT,
DEFAULT_FILE_LIMIT,
DEFAULT_MEMORY_LIMIT,
)
class ParserResultDetail(BaseModel):
time: Optional[bool] = True # Display run time
@ -31,8 +36,8 @@ class ParserDummy(BaseModel):
class ParserKeyword(BaseModel):
keyword: Optional[list[str]] = []
weight: Optional[list[int]] = []
keyword: Optional[List[str]] = []
weight: Optional[List[int]] = []
class Outputs(BaseModel):
@ -47,27 +52,27 @@ class ParserDiff(BaseModel):
class Files(BaseModel):
import_: Optional[list[str]] = Field([], alias="import")
export: Optional[list[str]] = []
import_: Optional[List[str]] = Field([], alias="import")
export: Optional[List[str]] = []
class Limit(BaseModel):
mem: int = humanfriendly.parse_size("128M")
cpu: int = 1_000_000_000
stderr: int = humanfriendly.parse_size("128M")
stdout: int = humanfriendly.parse_size("128M")
mem: int = DEFAULT_MEMORY_LIMIT
cpu: int = DEFAULT_CPU_LIMIT
stderr: int = DEFAULT_FILE_LIMIT
stdout: int = DEFAULT_FILE_LIMIT
class Stage(BaseModel):
name: Optional[str] = None # Stage name
env: Optional[list[str]] = None
env: Optional[List[str]] = None
command: Optional[str] = None # Command to run
files: Optional[Files] = None
in_: Optional[str] = Field(None, alias="in")
out_: Optional[str] = Field(None, alias="out")
score: Optional[int] = 0
parsers: Optional[list[str]] = [] # list of parsers
limit: Optional[Limit] = Limit()
parsers: Optional[List[str]] = [] # list of parsers
limit: Limit = Limit()
dummy: Optional[ParserDummy] = ParserDummy()
result_status: Optional[ParserDummy] = Field(ParserDummy(), alias="result-status")
keyword: Optional[ParserKeyword] = ParserKeyword()
@ -78,7 +83,7 @@ class Stage(BaseModel):
ParserResultDetail(), alias="result-detail"
)
file: Optional[ParserFile] = ParserFile()
jon-lee marked this conversation as resolved Outdated

deprecated

deprecated

@bomingzh any suggestions on the structure?

@bomingzh any suggestions on the structure?
PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.9/migration/
``` PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.9/migration/ ```

fixed with model_config

fixed with `model_config`

str here need to be a StrEnum now.

str here need to be a `StrEnum` now.

But I guess we don't know the set of case in advance, making it dynamic StrEnum is meaningless

But I guess we don't know the set of case in advance, making it dynamic `StrEnum` is meaningless

line changed, the comment is for parsers

line changed, the comment is for `parsers`
skip: Optional[list[str]] = []
skip: Optional[List[str]] = []
jon-lee marked this conversation as resolved Outdated

this should be the StrEnum

this should be the `StrEnum`

It is supported now.

It is supported now.
# cases related
cases: Optional[Dict[str, "Stage"]] = None

View File

@ -1,12 +1,9 @@
import humanfriendly
from pytimeparse.timeparse import timeparse
from joj3_config_generator.models import joj1, task
from joj3_config_generator.models.common import Memory, Time
from joj3_config_generator.models.const import DEFAULT_CPU_LIMIT, DEFAULT_MEMORY_LIMIT
jon-lee marked this conversation as resolved Outdated

is it added to pyproject.toml?

is it added to pyproject.toml?

now added

now added
def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
default_cpu = timeparse("1s")
default_mem = humanfriendly.parse_size("32m")
cases_conf = []
for i, case in enumerate(joj1_config.cases):
jon-lee marked this conversation as resolved Outdated

just use humanfriendly.parse_timespan

just use `humanfriendly.parse_timespan`

seems outdated, mark as resolved.

seems outdated, mark as resolved.
cases_conf.append(
@ -14,12 +11,8 @@ def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
score=case.score,
command=case.execute_args if case.execute_args else None,
limit=task.Limit(
cpu=timeparse(case.time) if case.time else default_cpu,
mem=(
humanfriendly.parse_size(case.memory)
if case.memory
else default_mem
),
cpu=Time(case.time) if case.time else DEFAULT_CPU_LIMIT,
mem=(Memory(case.memory) if case.memory else DEFAULT_MEMORY_LIMIT),
),
)
)
@ -32,14 +25,14 @@ def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
score=100,
limit=task.Limit(
cpu=(
timeparse(joj1_config.cases[0].time)
Time(joj1_config.cases[0].time)
if joj1_config.cases[0].time is not None
else default_cpu
else DEFAULT_CPU_LIMIT
),
mem=(
humanfriendly.parse_size(joj1_config.cases[0].memory)
Memory(joj1_config.cases[0].memory)
if joj1_config.cases[0].memory is not None
else default_mem
else DEFAULT_MEMORY_LIMIT
),
),
cases={f"case{i}": cases_conf[i] for i, _ in enumerate(cases_conf)},

View File

@ -65,52 +65,14 @@ def get_executor_with_config(
# are there any corner cases
for file in copy_in_files
},
stdin=(
result.MemoryFile(content="")
if (
(task_stage.parsers is not None)
and ("diff" not in task_stage.parsers)
)
else None
),
copy_out=copy_out_files,
copy_in_cached={file: file for file in cached},
copy_out_cached=file_export if file_export is not None else [],
cpu_limit=(
task_stage.limit.cpu * 1_000_000_000
if task_stage.limit is not None and task_stage.limit.cpu is not None
else 80 * 1_000_000_000
),
clock_limit=(
2 * task_stage.limit.cpu * 1_000_000_000
if task_stage.limit is not None and task_stage.limit.cpu is not None
else 80 * 1_000_000_000
),
memory_limit=(
task_stage.limit.mem * 1_024 * 1_024
if task_stage.limit is not None and task_stage.limit.mem is not None
else 128 * 1_024 * 1_024
),
stderr=result.Collector(
name="stderr",
max=(
task_stage.limit.stderr * 1_000_000_000
if task_stage.limit is not None
and task_stage.limit.stderr is not None
else 128 * 1_024 * 1_024
),
pipe=True,
),
stdout=result.Collector(
name="stdout",
max=(
task_stage.limit.stdout * 1_000_000_000
if task_stage.limit is not None
and task_stage.limit.stdout is not None
else 128 * 1_024 * 1_024
),
pipe=True,
),
copy_out_cached=file_export or [],
cpu_limit=task_stage.limit.cpu,
clock_limit=2 * task_stage.limit.cpu,
memory_limit=task_stage.limit.mem,
stderr=result.Collector(name="stderr"),
stdout=result.Collector(name="stdout"),
),
jon-lee marked this conversation as resolved Outdated

do not use getattr, visit the field explictly

do not use `getattr`, visit the field explictly

resolved.

resolved.
cases=[],
)

View File

@ -13,7 +13,6 @@ dependencies = [
"inquirer>=3.4.0",
"rtoml>=0.11.0",
"humanfriendly>=10.0",
"pytimeparse>=1.1.8",
]
requires-python = ">=3.9"
authors = [{ name = "JOJ3-dev", email = "joj3@focs.ji.sjtu.edu.cn" }]

View File

@ -17,11 +17,24 @@
"name": "local",
"with": {
"default": {
"args": [],
"env": [],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -51,13 +64,6 @@
"-meta=Changelog.md",
"-checkFileSumList=a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc,b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,f6740081487ca34963a005209e2e9adfdf6f3561719af082d40fe80145e0cceb,bbeca1491c2f8364821a328a6677c0c5d59ccd60250abac3cec0887eeb9bde3e",
"-checkFileNameList=.gitignore,.gitattributes,.gitea/workflows/push.yaml,.gitea/workflows/release.yaml"
],
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
},
{
@ -72,13 +78,6 @@
],
"env": [
"LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
],
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
}
]
@ -115,18 +114,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -203,18 +201,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -304,18 +301,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -431,18 +427,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -530,18 +525,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -623,20 +617,22 @@
"-a"
],
"env": [],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 95656304705536,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 91224961,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -670,11 +666,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
},
{
"stdin": {
@ -683,11 +675,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
}
]
}
@ -762,10 +750,22 @@
"env": [
"LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 128000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

View File

@ -32,18 +32,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

View File

@ -32,18 +32,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

View File

@ -31,18 +31,17 @@
},
"stdout": {
"name": "stdout",
"max": 65000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,

View File

@ -22,20 +22,22 @@
"-a"
],
"env": [],
"stdin": {
"content": ""
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 95656304705536,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 91224961,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,
@ -67,11 +69,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
},
{
"stdin": {
@ -80,11 +78,7 @@
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 95656304705536,
"procLimit": 50,
"copyOut": [
"stdout",
"stderr"
]
"procLimit": 50
}
]
}

View File

@ -30,18 +30,17 @@
},
"stdout": {
"name": "stdout",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"stderr": {
"name": "stderr",
"max": 128000000000000000,
"max": 33554432,
"pipe": true
},
"cpuLimit": 1000000000000000000,
"realCpuLimit": 1000000000,
"clockLimit": 2000000000000000000,
"memoryLimit": 134217728000000,
"cpuLimit": 1000000000,
"clockLimit": 2000000000,
"memoryLimit": 134217728,
"stackLimit": 0,
"procLimit": 50,
"cpuRateLimit": 0,