JOJ/JOJ3-config-generator
27
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: finish keyword related

Browse Source
This commit is contained in:
李衍志523370910113 2024-10-23 22:37:52 +08:00
parent b45d65250d
commit d638e20fb0
5 changed files with 205 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
joj3_config_generator/convert.py
View File

@ -20,6 +20,7 @@ from joj3_config_generator.lib.task import (
) )
from joj3_config_generator.models import joj1, repo, result, task from joj3_config_generator.models import joj1, repo, result, task
from joj3_config_generator.lib.repo import getHealthcheckConfig, getTeapotConfig from joj3_config_generator.lib.repo import getHealthcheckConfig, getTeapotConfig
from joj3_config_generator.lib.task import fix_keyword
from joj3_config_generator.models import ( from joj3_config_generator.models import (
Cmd, Cmd,
CmdFile, CmdFile,
@ -82,6 +83,7 @@ def convert(repo_conf: repo.Config, task_conf: task.Config) -> result.Config:
result.Parser(name=parser, with_={}) for parser in task_stage.parsers result.Parser(name=parser, with_={}) for parser in task_stage.parsers
], ],
) )
# TODO: fix all parser here
if "result-detail" in task_stage.parsers: if "result-detail" in task_stage.parsers:
result_detail_parser = next( result_detail_parser = next(
p for p in conf_stage.parsers if p.name == "result-detail" p for p in conf_stage.parsers if p.name == "result-detail"
@ -89,6 +91,20 @@ def convert(repo_conf: repo.Config, task_conf: task.Config) -> result.Config:
if task_stage.result_detail is not None: if task_stage.result_detail is not None:
result_detail_parser.with_.update(task_stage.result_detail) result_detail_parser.with_.update(task_stage.result_detail)
if "dummy" in task_stage.parsers:
dummy_parser = next(p for p in conf_stage.parsers if p.name == "dummy")
if task_stage.dummy is not None:
dummy_parser.with_.update(task_stage.dummy)
if "result-status" in task_stage.parsers:
result_status_parser = next(
p for p in conf_stage.parsers if p.name == "result-status"
)
if task_stage.result_status is not None:
result_status_parser.with_.update(task_stage.result_status)
conf_stage = fix_keyword(task_stage, conf_stage)
result_conf.stage.stages.append(conf_stage) result_conf.stage.stages.append(conf_stage)
return result_conf return result_conf

419
joj3_config_generator/lib/task.py
View File

@ -1,407 +1,20 @@
import shlex from joj3_config_generator.models.result import Stage as ResultStage
from typing import Tuple from joj3_config_generator.models.task import Stage as TaskStage
import rtoml
from joj3_config_generator.models import joj1, repo, result, task
def remove_nulls(d: result.Config) -> result.Config: def fix_keyword(task_stage: TaskStage, conf_stage: ResultStage) -> ResultStage:
if isinstance(d, dict): keyword_parser = ["clangtidy", "keyword", "cppcheck"] # TODO: may add cpplint
return {k: remove_nulls(v) for k, v in d.items() if v is not None} for parser in task_stage.parsers:
elif isinstance(d, list): if parser in keyword_parser:
return [remove_nulls(item) for item in d] keyword_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
else: keyword_weight = []
return d if getattr(task_stage, parser, None) is not None:
for _, keyword in enumerate(getattr(task_stage, parser).keyword):
keyword_weight.append({"keyword": [keyword], "score": 0})
def get_conf_stage( for idx, weight in enumerate(getattr(task_stage, parser).weight):
task_stage: task.Stage, executor_with_config: result.ExecutorWith keyword_weight[idx]["score"] = weight
) -> result.StageDetail:
conf_stage = result.StageDetail(
name=task_stage.name if task_stage.name is not None else "",
# TODO: we may have cq in future
group=(
"joj"
if (task_stage.name is not None) and ("judge" in task_stage.name)
else None
),
executor=result.Executor(
name="sandbox",
with_=executor_with_config,
),
parsers=(
[result.Parser(name=parser, with_={}) for parser in task_stage.parsers]
if task_stage.parsers is not None
else []
),
)
return conf_stage
def get_executorWithConfig(
task_stage: task.Stage, cached: list[str]
) -> Tuple[result.ExecutorWith, list[str]]:
file_import = (
task_stage.files.import_
if hasattr(task_stage, "files")
and hasattr(task_stage.files, "import_")
and (task_stage.files is not None)
and (task_stage.files.import_ is not None)
else []
)
copy_in_files = [file for file in file_import if (file not in cached)]
file_export = (
task_stage.files.export
if hasattr(task_stage, "files")
and hasattr(task_stage.files, "export")
and (task_stage.files is not None)
else []
)
executor_with_config = result.ExecutorWith(
default=result.Cmd(
args=(
shlex.split(task_stage.command)
if task_stage.command is not None
else []
),
copy_in={
file: result.CmdFile(src=f"/home/tt/.config/joj/{file}")
for file in copy_in_files
},
copy_in_cached={file: file for file in copy_in_files},
copy_out_cached=file_export if file_export is not None else [],
cpu_limit=(
task_stage.limit.cpu * 1_000_000_000
if task_stage.limit is not None and task_stage.limit.cpu is not None
else 4 * 1_000_000_000
),
clock_limit=(
2 * task_stage.limit.cpu * 1_000_000_000
if task_stage.limit is not None and task_stage.limit.cpu is not None
else 8 * 1_000_000_000
),
memory_limit=(
task_stage.limit.mem * 1_024 * 1_024
if task_stage.limit is not None and task_stage.limit.mem is not None
else 4 * 1_024 * 1_024
),
stderr=result.CmdFile(
name="stderr",
max=(
task_stage.limit.stderr * 1_000_000_000
if task_stage.limit is not None
and task_stage.limit.stderr is not None
else 4 * 1_024 * 1_024
),
),
stdout=result.CmdFile(
name="stdout",
max=(
task_stage.limit.stdout * 1_000_000_000
if task_stage.limit is not None
and task_stage.limit.stdout is not None
else 4 * 1_024 * 1_024
),
),
),
cases=[], # You can add cases if needed
)
if file_export is not None:
for file in file_export:
if file not in cached:
cached.append(file)
return (executor_with_config, cached)
def fix_keyword(
task_stage: task.Stage, conf_stage: result.StageDetail
) -> result.StageDetail:
keyword_parser = ["clangtidy", "keyword", "cppcheck", "cpplint"]
if task_stage.parsers is not None:
for parser in task_stage.parsers:
if parser in keyword_parser:
keyword_parser_ = next(
p for p in conf_stage.parsers if p.name == parser
)
keyword_weight = []
if getattr(task_stage, parser, None) is not None:
unique_weight = list(set(getattr(task_stage, parser).weight))
for score in unique_weight:
keyword_weight.append({"keywords": [], "score": score})
for idx, score in enumerate(unique_weight):
for idx_, score_ in enumerate(
getattr(task_stage, parser).weight
):
if score == score_:
keyword_weight[idx]["keywords"].append(
getattr(task_stage, parser).keyword[idx_]
)
else:
continue
keyword_parser_.with_.update({"matches": keyword_weight})
else:
continue
return conf_stage
def fix_result_detail(
task_stage: task.Stage, conf_stage: result.StageDetail
) -> result.StageDetail:
if (task_stage.parsers is not None) and ("result-detail" in task_stage.parsers):
result_detail_parser = next(
p for p in conf_stage.parsers if p.name == "result-detail"
)
if task_stage.result_detail is not None:
show_files = []
if (
task_stage.result_detail.stdout
and task_stage.result_detail.stdout is not None
):
show_files.append("stdout")
if (
task_stage.result_detail.stderr
and task_stage.result_detail.stderr is not None
):
show_files.append("stderr")
result_detail_parser.with_.update(
{
"score": 0,
"comment": "",
"showFiles": show_files,
"showExitStatus": task_stage.result_detail.exitstatus,
"showRuntime": task_stage.result_detail.time,
"showMemory": task_stage.result_detail.mem,
}
)
return conf_stage
def fix_dummy(
task_stage: task.Stage, conf_stage: result.StageDetail
) -> result.StageDetail:
dummy_parser = [
"dummy",
"result-status",
"cpplint",
]
if task_stage.parsers is not None:
for parser in task_stage.parsers:
if parser in dummy_parser:
dummy_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
if (
getattr(task_stage, parser.replace("-", "_"), None) is not None
) and (task_stage.result_status is not None):
dummy_parser_.with_.update(
{
"score": task_stage.result_status.score,
"comment": task_stage.result_status.comment,
"forceQuitOnNotAccepted": task_stage.result_status.forcequit,
}
)
else:
continue
return conf_stage
def fix_diff(
task_stage: task.Stage, conf_stage: result.StageDetail
) -> result.StageDetail:
if task_stage.parsers is not None and "diff" in task_stage.parsers:
diff_parser = next((p for p in conf_stage.parsers if p.name == "diff"), None)
skip = task_stage.skip or []
cases = task_stage.cases or {}
finalized_cases = [case for case in cases if case not in skip]
stage_cases = []
parser_cases = []
for case in finalized_cases:
case_stage = task_stage.cases.get(case) if task_stage.cases else None
if not case_stage:
continue
# Ensure case_stage.limit is defined before accessing .cpu and .mem
cpu_limit = (
case_stage.limit.cpu * 1_000_000_000
if case_stage.limit and case_stage.limit.cpu is not None
else 0
)
clock_limit = (
2 * case_stage.limit.cpu * 1_000_000_000
if case_stage.limit and case_stage.limit.cpu is not None
else 0
)
memory_limit = (
case_stage.limit.mem * 1_024 * 1_024
if case_stage.limit and case_stage.limit.mem is not None
else 0
)
stage_cases.append(
result.OptionalCmd(
stdin=result.CmdFile(
src=f"/home/tt/.config/joj/{conf_stage.name}/{case}.in"
),
cpu_limit=cpu_limit,
clock_limit=clock_limit,
memory_limit=memory_limit,
proc_limit=50,
)
)
# Ensure case_stage.diff and case_stage.diff.output are defined
diff_output = (
case_stage.diff.output
if case_stage.diff and case_stage.diff.output
else None
)
if diff_output:
parser_cases.append(
{
"outputs": [
{
"score": diff_output.score,
"fileName": "stdout",
"answerPath": f"/home/tt/.config/joj/{conf_stage.name}/{case}.out",
"forceQuitOnDiff": diff_output.forcequit,
"alwaysHide": diff_output.hide,
"compareSpace": not diff_output.ignorespaces,
}
]
}
)
if diff_parser and task_stage.diff is not None:
diff_parser.with_.update({"name": "diff", "cases": parser_cases})
conf_stage.executor.with_.cases = stage_cases
return conf_stage
def fix_result_detail(task_stage: TaskStage, conf_stage: ResultStage) -> ResultStage:
if (task_stage.parsers is not None) and ("result-detail" in task_stage.parsers):
result_detail_parser = next(
p for p in conf_stage.parsers if p.name == "result-detail"
)
if task_stage.result_detail is not None:
show_files = []
if (
task_stage.result_detail.stdout
and task_stage.result_detail.stdout is not None
):
show_files.append("stdout")
if (
task_stage.result_detail.stderr
and task_stage.result_detail.stderr is not None
):
show_files.append("stderr")
result_detail_parser.with_.update(
{
"score": 0,
"comment": "",
"showFiles": show_files,
"showExitStatus": task_stage.result_detail.exitstatus,
"showRuntime": task_stage.result_detail.time,
"showMemory": task_stage.result_detail.mem,
}
)
return conf_stage
def fix_comment(task_stage: TaskStage, conf_stage: ResultStage) -> ResultStage:
comment_parser = [
"dummy",
"result-status",
"cpplint",
] # FIXME: determine where cpplint should be
if task_stage.parsers is not None:
for parser in task_stage.parsers:
if parser in comment_parser:
comment_parser_ = next(
p for p in conf_stage.parsers if p.name == parser
)
if getattr(task_stage, parser.replace("-", "_"), None) is not None:
comment_parser_.with_.update(
getattr(task_stage, parser.replace("-", "_"))
)
else:
continue
return conf_stage
def fix_diff(task_stage: TaskStage, conf_stage: ResultStage) -> ResultStage:
if task_stage.parsers is not None and "diff" in task_stage.parsers:
diff_parser = next((p for p in conf_stage.parsers if p.name == "diff"), None)
skip = task_stage.skip or []
cases = task_stage.cases or {}
finalized_cases = [case for case in cases if case not in skip]
stage_cases = []
parser_cases = []
for case in finalized_cases:
case_stage = task_stage.cases.get(case) if task_stage.cases else None
if not case_stage:
continue
# Ensure case_stage.limit is defined before accessing .cpu and .mem
cpu_limit = (
case_stage.limit.cpu * 1_000_000_000
if case_stage.limit and case_stage.limit.cpu is not None
else 0
)
clock_limit = (
2 * case_stage.limit.cpu * 1_000_000_000
if case_stage.limit and case_stage.limit.cpu is not None
else 0
)
memory_limit = (
case_stage.limit.mem * 1_024 * 1_024
if case_stage.limit and case_stage.limit.mem is not None
else 0
)
stage_cases.append(
OptionalCmd(
stdin=CmdFile(
src=f"/home/tt/.config/joj/{conf_stage.name}/{case}.in"
),
cpu_limit=cpu_limit,
clock_limit=clock_limit,
memory_limit=memory_limit,
proc_limit=50,
)
)
# Ensure case_stage.diff and case_stage.diff.output are defined
diff_output = (
case_stage.diff.output
if case_stage.diff and case_stage.diff.output
else None
)
if diff_output:
parser_cases.append(
{
"outputs": [
{
"score": diff_output.score,
"fileName": "stdout",
"answerPath": f"/home/tt/.config/joj/{conf_stage.name}/{case}.out",
"forceQuitOnDiff": diff_output.forcequit,
"alwaysHide": diff_output.hide,
"compareSpace": not diff_output.ignorespaces,
}
]
}
)
if diff_parser and task_stage.diff is not None:
diff_parser.with_.update({"name": "diff", "cases": parser_cases})
conf_stage.executor.with_.cases = stage_cases
keyword_parser_.with_.update({"match": keyword_weight})
else:
continue
return conf_stage return conf_stage

5
joj3_config_generator/models/task.py
View File

@ -17,8 +17,8 @@ class ParserDummy(BaseModel):
class ParserKeyword(BaseModel): class ParserKeyword(BaseModel):
keyword: Optional[list[str]] = None keyword: Optional[list[str]] = []
weight: Optional[list[int]] = None weight: Optional[list[int]] = []
class Files(BaseModel): class Files(BaseModel):
@ -42,6 +42,7 @@ class Stage(BaseModel):
parsers: list[str] # list of parsers parsers: list[str] # list of parsers
limit: Optional[Limit] = None limit: Optional[Limit] = None
dummy: Optional[ParserDummy] = ParserDummy() dummy: Optional[ParserDummy] = ParserDummy()
result_status: Optional[ParserDummy] = Field(ParserDummy(), alias="result-status")
keyword: Optional[ParserKeyword] = ParserKeyword() keyword: Optional[ParserKeyword] = ParserKeyword()
clangtidy: Optional[ParserKeyword] = ParserKeyword() clangtidy: Optional[ParserKeyword] = ParserKeyword()
cppcheck: Optional[ParserKeyword] = ParserKeyword() cppcheck: Optional[ParserKeyword] = ParserKeyword()

184
tests/convert/basic/task.json
View File

@ -15,12 +15,15 @@
"with": { "with": {
"default": { "default": {
"args": [ "args": [
"/tmp/repo-health-checker", "/<function",
"get_temp_directory",
"at",
"0x7f1db69b71a0>/repo-health-checker",
"-root=.", "-root=.",
"-repoSize=50.5", "-repoSize=50.5",
"-meta=main.py", "-meta=main.py",
"-meta=README.md", "-meta=README.md",
"-checkFileSumList=a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc,b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,8d1229900c6fc6711b5cc141d1ab5ea7f5b7b7a4b921d9cfa3957408b43ae723,eb857bcd94857cedc4045cb2d6ba04cb5bbb3daf188abc95fb9478db823ef47e", "-checkFileSumList=a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc,b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,1965adff52af61da8b9e089ff580d60f7e4c294a2930b9809c5cbdf76528de4d,c8bd62bf5297bac738b3845612fd595d677884093070904375463ab7953fce28",
"-checkFileNameList=.gitignore,.gitattributes,push.yaml,release.yaml" "-checkFileNameList=.gitignore,.gitattributes,push.yaml,release.yaml"
], ],
"env": [ "env": [
@ -68,8 +71,8 @@
"cpuRateLimit": 0, "cpuRateLimit": 0,
"cpuSetLimit": "", "cpuSetLimit": "",
"copyIn": { "copyIn": {
"/tmp/repo-health-checker": { "//tmp/repo-checker-90ztqsoq/repo-health-checker": {
"src": "/tmp/repo-health-checker", "src": "//tmp/repo-checker-41mcx5_x/repo-health-checker",
"content": null, "content": null,
"fileId": null, "fileId": null,
"name": null, "name": null,
@ -244,11 +247,15 @@
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": "\n\n### Details\n"
}
}, },
{ {
"name": "result-status", "name": "result-status",
"with": {} "with": {
"comment": "Congratulations! Your code compiled successfully."
}
} }
] ]
}, },
@ -345,11 +352,28 @@
"parsers": [ "parsers": [
{ {
"name": "keyword", "name": "keyword",
"with": {} "with": {
"match": [
{
"keyword": [
"max"
],
"score": 50
},
{
"keyword": [
"recommend"
],
"score": 20
}
]
}
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": ""
}
}, },
{ {
"name": "result-detail", "name": "result-detail",
@ -440,11 +464,100 @@
"parsers": [ "parsers": [
{ {
"name": "clangtidy", "name": "clangtidy",
"with": {} "with": {
"match": [
{
"keyword": [
"codequality-no-global-variables"
],
"score": 10
},
{
"keyword": [
"codequality-no-header-guard"
],
"score": 10
},
{
"keyword": [
"readability-function-size"
],
"score": 50
},
{
"keyword": [
"readability-duplicate-include"
],
"score": 10
},
{
"keyword": [
"readability-identifier-naming"
],
"score": 5
},
{
"keyword": [
"readability-redundant"
],
"score": 5
},
{
"keyword": [
"readability-misleading-indentation"
],
"score": 10
},
{
"keyword": [
"readability-misplaced-array-index"
],
"score": 5
},
{
"keyword": [
"cppcoreguidelines-init-variables"
],
"score": 5
},
{
"keyword": [
"bugprone-suspicious-string-compare"
],
"score": 8
},
{
"keyword": [
"google-global-names-in-headers"
],
"score": 5
},
{
"keyword": [
"clang-diagnostic"
],
"score": 5
},
{
"keyword": [
"clang-analyzer"
],
"score": 5
},
{
"keyword": [
"misc performance"
],
"score": 5
}
]
}
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": "\n\n### Details\n"
}
}, },
{ {
"name": "result-detail", "name": "result-detail",
@ -539,11 +652,46 @@
"parsers": [ "parsers": [
{ {
"name": "cppcheck", "name": "cppcheck",
"with": {} "with": {
"match": [
{
"keyword": [
"error"
],
"score": 20
},
{
"keyword": [
"warning"
],
"score": 10
},
{
"keyword": [
"portability"
],
"score": 15
},
{
"keyword": [
"performance"
],
"score": 15
},
{
"keyword": [
"style"
],
"score": 10
}
]
}
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": "\n\n### Details\n"
}
}, },
{ {
"name": "result-detail", "name": "result-detail",
@ -638,7 +786,9 @@
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": "\n\n### Details\n"
}
}, },
{ {
"name": "result-detail", "name": "result-detail",
@ -729,7 +879,9 @@
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": "\n\n### Details\n"
}
}, },
{ {
"name": "result-detail", "name": "result-detail",
@ -820,7 +972,9 @@
}, },
{ {
"name": "dummy", "name": "dummy",
"with": {} "with": {
"comment": "\n\n### Details\n"
}
}, },
{ {
"name": "result-detail", "name": "result-detail",

2
tests/convert/basic/task.toml
View File

@ -41,7 +41,7 @@ limit.stdout = 65
parsers = [ "clangtidy", "dummy", "result-detail" ] parsers = [ "clangtidy", "dummy", "result-detail" ]
clangtidy.keyword = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ] clangtidy.keyword = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ]
clangtidy.weight = [10, 10, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5, 8] clangtidy.weight = [10, 10, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5]
dummy.comment = "\n\n### Details\n" dummy.comment = "\n\n### Details\n"
result-detail.exitstatus = true result-detail.exitstatus = true
result-detail.stdout = true result-detail.stdout = true