JOJ/JOJ3-config-generator
27
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

dev #10

Merged
李衍志523370910113 merged 238 commits from dev into master 2025-03-05 16:20:39 +08:00
Conversation 16 Commits 238 Files Changed 42 +128 -87
3 changed files with 128 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit df76f63a43 - Show all commits

128
joj3_config_generator/processers/task.py
View File

@ -31,11 +31,21 @@ def get_conf_stage(
]
),
)
fix_result_detail(task_stage, conf_stage)
fix_dummy(task_stage, conf_stage)
fix_keyword(task_stage, conf_stage)
fix_file(task_stage, conf_stage)
fix_diff(task_stage, task_conf, conf_stage)
keyword_parser = ["clangtidy", "keyword", "cppcheck", "cpplint"]
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-03 11:54:43 +08:00
Outdated
Review
Copy Link

should loop through conf_stage.parsers here and update the with field according to the parser name.

should loop through `conf_stage.parsers` here and update the `with` field according to the parser name.
李衍志523370910113 commented 2025-03-03 13:36:23 +08:00
Outdated
Review
Copy Link

I think its already implemented in each of the fix_parsers functions

I think its already implemented in each of the `fix_parsers` functions
张泊明518370910136 commented 2025-03-03 13:43:12 +08:00
Outdated
Review
Copy Link

No, do not find the parser in the fix_xxx function. Instead, iterate through the parsers here and decide how to fill in the with.

No, do not find the parser in the `fix_xxx` function. Instead, iterate through the parsers here and decide how to fill in the `with`.
李衍志523370910113 commented 2025-03-03 16:17:53 +08:00
Outdated
Review
Copy Link

resolved.

resolved.
张泊明518370910136 commented 2025-03-04 03:02:45 +08:00
Outdated
Review
Copy Link

Use a dict to store parser name, field, function to process.

    process_dict: Dict[
        str, Tuple[Callable[[result.ParserConfig, BaseModel], None], BaseModel]
    ] = {
        "clangtidy": (fix_keyword, task_stage.clangtidy),
        "keyword": (fix_keyword, task_stage.keyword),
        "diff": (fix_diff, task_stage.diff),
    }
    for i, parser in enumerate(task_stage.parsers):
        if parser in process_dict:
            func, parser_model = process_dict[parser]
            func(conf_stage.parsers[i], parser_model)
Use a dict to store parser name, field, function to process. ``` process_dict: Dict[ str, Tuple[Callable[[result.ParserConfig, BaseModel], None], BaseModel] ] = { "clangtidy": (fix_keyword, task_stage.clangtidy), "keyword": (fix_keyword, task_stage.keyword), "diff": (fix_diff, task_stage.diff), } for i, parser in enumerate(task_stage.parsers): if parser in process_dict: func, parser_model = process_dict[parser] func(conf_stage.parsers[i], parser_model) ```
李衍志523370910113 commented 2025-03-04 14:46:40 +08:00
Outdated
Review
Copy Link

resolved.

resolved.
dummy_parser = ["dummy", "result-status"]
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-05 03:51:41 +08:00
Outdated
Review
Copy Link

Do we need to support both kinds of names?

Do we need to support both kinds of names?
李衍志523370910113 commented 2025-03-05 08:42:25 +08:00
Outdated
Review
Copy Link

probably yes, since it is easy for new ta to type it wrong

probably yes, since it is easy for new ta to type it wrong
张泊明518370910136 commented 2025-03-05 09:00:28 +08:00
Outdated
Review
Copy Link

parsers name should be a str enum, force them to use the correct names

parsers name should be a str enum, force them to use the correct names
李衍志523370910113 commented 2025-03-05 09:09:46 +08:00
Outdated
Review
Copy Link

ok, then removed.

ok, then removed.
for parser in task_stage.parsers:
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-02-23 22:32:35 +08:00
Outdated
Review
Copy Link

underscore

underscore
李衍志523370910113 commented 2025-02-25 11:02:24 +08:00
Outdated
Review
Copy Link

fixed

fixed
if parser in keyword_parser:
fix_keyword(task_stage, conf_stage, parser)
elif parser in dummy_parser:
fix_dummy(task_stage, conf_stage, parser)
elif parser == "result-detail":
fix_result_detail(task_stage, conf_stage, parser)
elif parser == "file":
fix_file(task_stage, conf_stage, parser)
elif parser == "diff":
fix_diff(task_stage, task_conf, conf_stage, parser)
else:
continue
return conf_stage
@ -70,42 +80,36 @@ def get_executor_with(task_stage: task.Stage, cached: Set[str]) -> result.Execut
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-02 02:20:52 +08:00
Outdated
Review
Copy Link

is it in the correct unit?
Give all the fields in Limit a default value and make it non optional in task_stage.limit

is it in the correct unit? Give all the fields in `Limit` a default value and make it non optional in task_stage.limit
李衍志523370910113 commented 2025-03-02 11:47:43 +08:00
Outdated
Review
Copy Link

resolved

resolved
def fix_keyword(
task_stage: task.Stage, conf_stage: result.StageDetail
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
) -> result.StageDetail:
keyword_parser = ["clangtidy", "keyword", "cppcheck", "cpplint"]
for parser in task_stage.parsers:
if parser in keyword_parser:
keyword_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
keyword_weight: List[result.KeywordConfig] = []
if parser in task_stage.__dict__:
unique_weight = list(set(task_stage.__dict__[parser].weight))
for score in unique_weight:
keyword_weight.append(
result.KeywordConfig(keywords=[], score=score)
keyword_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
keyword_weight: List[result.KeywordConfig] = []
if parser in task_stage.__dict__:
unique_weight = list(set(task_stage.__dict__[parser].weight))
for score in unique_weight:
keyword_weight.append(result.KeywordConfig(keywords=[], score=score))
for idx, score in enumerate(unique_weight):
for idx_, score_ in enumerate(task_stage.__dict__[parser].weight):
if score == score_:
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-05 03:58:08 +08:00
Outdated
Review
Copy Link

The reason for the suffix in keyword_parser_?

The reason for the suffix in `keyword_parser_`?
李衍志523370910113 commented 2025-03-05 08:42:46 +08:00
Outdated
Review
Copy Link

just forgot to remove, sorry

just forgot to remove, sorry
keyword_weight[idx].keywords.append(
task_stage.__dict__[parser].keyword[idx_]
)
else:
continue
for idx, score in enumerate(unique_weight):
for idx_, score_ in enumerate(task_stage.__dict__[parser].weight):
if score == score_:
keyword_weight[idx].keywords.append(
task_stage.__dict__[parser].keyword[idx_]
)
else:
continue
else:
continue
keyword_parser_.with_.update(
result.KeywordMatchConfig(
matches=keyword_weight,
).model_dump(by_alias=True)
)
keyword_parser_.with_.update(
result.KeywordMatchConfig(
matches=keyword_weight,
).model_dump(by_alias=True)
)
return conf_stage
def fix_result_detail(task_stage: task.Stage, conf_stage: result.StageDetail) -> None:
if "result-detail" not in task_stage.parsers:
return
def fix_result_detail(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
) -> None:
result_detail_parser = next(
p for p in conf_stage.parsers if p.name == "result-detail"
)
@ -126,48 +130,40 @@ def fix_result_detail(task_stage: task.Stage, conf_stage: result.StageDetail) ->
)
def fix_dummy(task_stage: task.Stage, conf_stage: result.StageDetail) -> None:
dummy_parser = [
"dummy",
"result-status",
]
for parser in task_stage.parsers:
if parser not in dummy_parser:
continue
dummy_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
if parser.replace("-", "_") not in task_stage.__dict__:
continue
if task_stage.result_status is None:
continue
dummy_parser_.with_.update(
result.DummyConfig(
score=task_stage.result_status.score,
comment=task_stage.result_status.comment,
force_quit_on_not_accepted=task_stage.result_status.force_quit,
).model_dump(by_alias=True)
)
def fix_dummy(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
) -> None:
dummy_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
if parser.replace("-", "_") not in task_stage.__dict__:
return
if task_stage.result_status is None:
return
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-05 04:10:22 +08:00
Outdated
Review
Copy Link

When will it be None?

When will it be None?
dummy_parser_.with_.update(
result.DummyConfig(
score=task_stage.result_status.score,
comment=task_stage.result_status.comment,
force_quit_on_not_accepted=task_stage.result_status.force_quit,
).model_dump(by_alias=True)
)
return
def fix_file(task_stage: task.Stage, conf_stage: result.StageDetail) -> None:
file_parser = ["file"]
for parser in task_stage.parsers:
if parser not in file_parser:
continue
file_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
file_parser_.with_.update(
result.FileConfig(name=task_stage.file.name).model_dump(by_alias=True)
)
def fix_file(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
) -> None:
file_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
file_parser_.with_.update(
bomingzh marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-02 02:23:09 +08:00
Outdated
Review
Copy Link

these fields do not exist now

these fields do not exist now
李衍志523370910113 commented 2025-03-02 11:09:34 +08:00
Outdated
Review
Copy Link

resolved

resolved
张泊明518370910136 commented 2025-03-02 12:15:34 +08:00
Outdated
Review
Copy Link
No description provided.
result.FileConfig(name=task_stage.file.name).model_dump(by_alias=True)
)
def fix_diff(
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-02 02:36:32 +08:00
Outdated
Review
Copy Link

move continue to the other branch to reduce nesting

move `continue` to the other branch to reduce nesting
张泊明518370910136 commented 2025-03-02 12:15:54 +08:00
Outdated
Review
Copy Link

I mean

if parser not in keyword_parser:
    continue

if getattr(task_stage, parser, None) is None:
    continue

if score != score_:
    continue
I mean ``` if parser not in keyword_parser: continue ``` ``` if getattr(task_stage, parser, None) is None: continue ```` ``` if score != score_: continue ````
李衍志523370910113 commented 2025-03-02 17:38:50 +08:00
Outdated
Review
Copy Link

fixed.

fixed.
task_stage: task.Stage,
task_conf: task.Config,
conf_stage: result.StageDetail,
parser: str,
) -> None:
if "diff" not in task_stage.parsers:
return
diff_parser = next((p for p in conf_stage.parsers if p.name == "diff"), None)
diff_parser = next((p for p in conf_stage.parsers if p.name == parser), None)
jon-lee marked this conversation as resolved Outdated
张泊明518370910136 commented 2025-03-05 03:58:39 +08:00
Outdated
Review
Copy Link

Is it necessary to rename?

Is it necessary to rename?
skip = task_stage.skip
cases = task_stage.cases
finalized_cases = [case for case in cases if case not in skip]

85
tests/convert/basic/task.json
View File

@ -222,11 +222,11 @@
}
},
"copyInCached": {
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan"
},
"copyInDir": ".",
"copyOut": [
@ -322,11 +322,11 @@
}
},
"copyInCached": {
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan"
},
"copyInDir": ".",
"copyOut": [
@ -444,11 +444,11 @@
"cpuSetLimit": "",
"copyIn": {},
"copyInCached": {
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan"
},
"copyInDir": ".",
"copyOut": [
@ -467,6 +467,12 @@
}
},
"parsers": [
{
"name": "keyword",
"with": {
"matches": []
}
},
{
"name": "cppcheck",
"with": {
@ -489,6 +495,10 @@
]
}
},
{
"name": "clang-tidy",
"with": {}
},
{
"name": "result-detail",
"with": {
@ -501,6 +511,41 @@
"showRuntime": false,
"showMemory": false
}
},
{
"name": "cpplint",
"with": {
"matches": []
}
},
{
"name": "result-status",
"with": {
"score": 0,
"comment": "",
"forceQuitOnNotAccepted": false
}
},
{
"name": "file",
"with": {
"name": ""
}
},
{
"name": "dummy",
"with": {
"score": 0,
"comment": "",
"forceQuitOnNotAccepted": false
}
},
{
"name": "diff",
"with": {
"name": "diff",
"cases": []
}
}
]
},
@ -542,11 +587,11 @@
"cpuSetLimit": "",
"copyIn": {},
"copyInCached": {
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan"
},
"copyInDir": ".",
"copyOut": [
@ -639,11 +684,11 @@
"cpuSetLimit": "",
"copyIn": {},
"copyInCached": {
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan"
},
"copyInDir": ".",
"copyOut": [

2
tests/convert/basic/task.toml
View File

@ -53,7 +53,7 @@ name = "[cq] Cppcheck"
command = "cppcheck --template='{\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}' --force --enable=all --suppress=missingIncludeSystem --quiet h7/ex2.cpp"
limit.stderr = "8m"
parsers = [ "cppcheck", "result-detail" ]
parsers = [ "keyword", "cppcheck", "clang-tidy", "result-detail", "cpplint", "result-status", "file", "dummy", "diff" ]
cppcheck.keyword = ["error", "warning", "portability", "performance", "style"]
cppcheck.weight = [15, 5, 5, 5, 5]
result-detail.exitstatus = true