dev #10

Merged
李衍志523370910113 merged 238 commits from dev into master 2025-03-05 16:20:39 +08:00
2 changed files with 68 additions and 82 deletions
Showing only changes of commit 6bfefd4f7b - Show all commits
joj3_config_generator/processers
tests/convert/basic

View File

@ -1,8 +1,6 @@
import re
jon-lee marked this conversation as resolved

Path should not be relative to JOJ3_CONFIG_ROOT in this file, should be relative to task.toml dir

Path should not be relative to `JOJ3_CONFIG_ROOT` in this file, should be relative to `task.toml` dir

I reckon you said things is relative to JOJ3_CONFIG_ROOT in JTC before. we have a task.type in task.toml to mend the path

I reckon you said things is relative to `JOJ3_CONFIG_ROOT` in JTC before. we have a `task.type` in `task.toml` to mend the path

config.path is relative to JOJ3_CONFIG_ROOT.

`config.path` is relative to `JOJ3_CONFIG_ROOT`.

could you explain further? I m not quite sure my understanding is clear.

could you explain further? I m not quite sure my understanding is clear.

In joj3_config_generator/models/task.py, Config.path is relative to JOJ3_CONFIG_ROOT, so task.toml will located at JOJ3_CONFIG_ROOT / task_conf.path in JTC.

In `joj3_config_generator/models/task.py`, `Config.path` is relative to `JOJ3_CONFIG_ROOT`, so `task.toml` will located at `JOJ3_CONFIG_ROOT / task_conf.path` in JTC.
import shlex
jon-lee marked this conversation as resolved

Some with_.update is still using raw dict, not model with model_dump.

Some `with_.update` is still using raw dict, not model with `model_dump`.
from typing import Callable, Dict, List, Set, Tuple
from pydantic import BaseModel
from typing import Any, Callable, Dict, List, Set, Tuple
from joj3_config_generator.models import result, task
from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT
@ -32,37 +30,34 @@ def get_conf_stage(
]
),
)
keyword_parser = ["clangtidy", "keyword", "cppcheck", "cpplint"]
dummy_parser = ["dummy", "result-status"]
for parser in task_stage.parsers:
if parser in keyword_parser:
fix_keyword(task_stage, conf_stage, parser)
elif parser in dummy_parser:
fix_dummy(task_stage, conf_stage, parser)
elif parser == "result-detail":
fix_result_detail(task_stage, conf_stage, parser)
elif parser == "file":
fix_file(task_stage, conf_stage, parser)
processed_dict = get_processed_dict(task_stage)
for idx, parser in enumerate(task_stage.parsers):
jon-lee marked this conversation as resolved Outdated

should loop through conf_stage.parsers here and update the with field according to the parser name.

should loop through `conf_stage.parsers` here and update the `with` field according to the parser name.

I think its already implemented in each of the fix_parsers functions

I think its already implemented in each of the `fix_parsers` functions

No, do not find the parser in the fix_xxx function. Instead, iterate through the parsers here and decide how to fill in the with.

No, do not find the parser in the `fix_xxx` function. Instead, iterate through the parsers here and decide how to fill in the `with`.

resolved.

resolved.

Use a dict to store parser name, field, function to process.

    process_dict: Dict[
        str, Tuple[Callable[[result.ParserConfig, BaseModel], None], BaseModel]
    ] = {
        "clangtidy": (fix_keyword, task_stage.clangtidy),
        "keyword": (fix_keyword, task_stage.keyword),
        "diff": (fix_diff, task_stage.diff),
    }
    for i, parser in enumerate(task_stage.parsers):
        if parser in process_dict:
            func, parser_model = process_dict[parser]
            func(conf_stage.parsers[i], parser_model)
Use a dict to store parser name, field, function to process. ``` process_dict: Dict[ str, Tuple[Callable[[result.ParserConfig, BaseModel], None], BaseModel] ] = { "clangtidy": (fix_keyword, task_stage.clangtidy), "keyword": (fix_keyword, task_stage.keyword), "diff": (fix_diff, task_stage.diff), } for i, parser in enumerate(task_stage.parsers): if parser in process_dict: func, parser_model = process_dict[parser] func(conf_stage.parsers[i], parser_model) ```

resolved.

resolved.
if parser in processed_dict or parser.replace("-", "_") in processed_dict:
jon-lee marked this conversation as resolved Outdated

Do we need to support both kinds of names?

Do we need to support both kinds of names?

probably yes, since it is easy for new ta to type it wrong

probably yes, since it is easy for new ta to type it wrong

parsers name should be a str enum, force them to use the correct names

parsers name should be a str enum, force them to use the correct names

ok, then removed.

ok, then removed.
fn, parser_model = processed_dict[parser]
jon-lee marked this conversation as resolved Outdated

underscore

underscore

fixed

fixed
fn(parser_model, conf_stage.parsers[idx])
elif parser == "diff":
fix_diff(task_stage, task_conf, conf_stage, parser)
fix_diff(task_stage, task_conf, conf_stage.parsers[idx], conf_stage)
else:
continue
return conf_stage
# def get_processed_dict(task_stage: task.Stagew) -> Dict[str, Tuple[Callable[[task.Stage, BaseModel], None], BaseModel]]:
# processed_dict: Dict[str, Tuple[Callable[[task.Stage, task.StageDetail], None], BaseModel]] = {
# "clang-tidy": (fix_keyword, result.clang-tidy),
# "keyword": (fix_keyword, result.KeywordConfig),
# "cppcheck": (fix_keyword, result.KeywordConfig),
# "cpplint": (fix_keyword, result.KeywordConfig),
# "result-detail": (fix_result_detail, result.ResultDetailConfig),
# "dummy": (fix_dummy, result.DummyConfig),
# "result-status": (fix_dummy, result.DummyConfig),
# "file": (fix_file, result.FileConfig),
# "diff": (fix_diff, result.DiffConfig),
# }
# return processed_dict
def get_processed_dict(
task_stage: task.Stage,
) -> Dict[str, Tuple[Callable[[Any, result.ParserConfig], None], Any]]:
processed_dict: Dict[
str, Tuple[Callable[[Any, result.ParserConfig], None], Any]
] = {
"clangtidy": (fix_keyword, task_stage.clangtidy),
"keyword": (fix_keyword, task_stage.keyword),
"cppcheck": (fix_keyword, task_stage.cppcheck),
"cpplint": (fix_keyword, task_stage.cpplint),
"result-detail": (fix_result_detail, task_stage.result_detail),
"dummy": (fix_dummy, task_stage.dummy),
"result-status": (fix_dummy, task_stage.result_status),
"file": (fix_file, task_stage.file),
}
return processed_dict
def get_executor_with(task_stage: task.Stage, cached: Set[str]) -> result.ExecutorWith:
jon-lee marked this conversation as resolved Outdated

not necessary

not necessary

resolved.

resolved.
@ -96,23 +91,19 @@ def get_executor_with(task_stage: task.Stage, cached: Set[str]) -> result.Execut
def fix_keyword(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
) -> result.StageDetail:
keyword_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
keyword_config: task.ParserKeyword, keyword_parser_: result.ParserConfig
jon-lee marked this conversation as resolved Outdated

The reason for the suffix in keyword_parser_?

The reason for the suffix in `keyword_parser_`?

just forgot to remove, sorry

just forgot to remove, sorry
) -> None:
keyword_weight: List[result.KeywordConfig] = []
if parser in task_stage.__dict__:
unique_weight = list(set(task_stage.__dict__[parser].weight))
for score in unique_weight:
keyword_weight.append(result.KeywordConfig(keywords=[], score=score))
unique_weight = list(set(keyword_config.weight))
for score in unique_weight:
keyword_weight.append(result.KeywordConfig(keywords=[], score=score))
for idx, score in enumerate(unique_weight):
for idx_, score_ in enumerate(task_stage.__dict__[parser].weight):
if score == score_:
keyword_weight[idx].keywords.append(
task_stage.__dict__[parser].keyword[idx_]
)
else:
continue
for idx, score in enumerate(unique_weight):
for idx_, score_ in enumerate(keyword_config.weight):
if score == score_:
keyword_weight[idx].keywords.append(keyword_config.keyword[idx_])
else:
continue
keyword_parser_.with_.update(
result.KeywordMatchConfig(
@ -120,64 +111,59 @@ def fix_keyword(
).model_dump(by_alias=True)
)
return conf_stage
def fix_result_detail(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
result_detail_parser_config: task.ParserResultDetail,
result_detail_parser: result.ParserConfig,
) -> None:
result_detail_parser = next(p for p in conf_stage.parsers if p.name == parser)
show_files = []
if task_stage.result_detail.stdout:
if result_detail_parser_config.stdout:
show_files.append("stdout")
if task_stage.result_detail.stderr:
if result_detail_parser_config.stderr:
show_files.append("stderr")
result_detail_parser.with_.update(
result.ResultDetailConfig(
score=0,
comment="",
show_files=show_files,
show_exit_status=task_stage.result_detail.exitstatus,
show_runtime=task_stage.result_detail.time,
show_memory=task_stage.result_detail.mem,
show_exit_status=result_detail_parser_config.exitstatus,
jon-lee marked this conversation as resolved Outdated
    if task_stage.parsers is not None:
        for parser in task_stage.parsers:

->

for parser in task_stage.parsers or []:
``` if task_stage.parsers is not None: for parser in task_stage.parsers: ``` -> ``` for parser in task_stage.parsers or []: ```

resolved.

resolved.
show_runtime=result_detail_parser_config.time,
show_memory=result_detail_parser_config.mem,
).model_dump(by_alias=True)
)
def fix_dummy(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
dummy_parser_config: task.ParserDummy, dummy_parser: result.ParserConfig
) -> None:
dummy_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
if parser.replace("-", "_") not in task_stage.__dict__:
# we don't use dummy parser in real application
if dummy_parser_config is None:
jon-lee marked this conversation as resolved Outdated

When will it be None?

When will it be None?
return
if task_stage.result_status is None:
return
dummy_parser_.with_.update(
dummy_parser.with_.update(
result.DummyConfig(
score=task_stage.result_status.score,
comment=task_stage.result_status.comment,
force_quit_on_not_accepted=task_stage.result_status.force_quit,
score=dummy_parser_config.score,
comment=dummy_parser_config.comment,
force_quit_on_not_accepted=dummy_parser_config.force_quit,
).model_dump(by_alias=True)
)
return
def fix_file(
task_stage: task.Stage, conf_stage: result.StageDetail, parser: str
file_parser_config: task.ParserFile, file_parser: result.ParserConfig
) -> None:
file_parser_ = next(p for p in conf_stage.parsers if p.name == parser)
file_parser_.with_.update(
result.FileConfig(name=task_stage.file.name).model_dump(by_alias=True)
file_parser.with_.update(
bomingzh marked this conversation as resolved Outdated

these fields do not exist now

these fields do not exist now

resolved

resolved
No description provided.
result.FileConfig(name=file_parser_config.name).model_dump(by_alias=True)
)
def fix_diff(
jon-lee marked this conversation as resolved Outdated

move continue to the other branch to reduce nesting

move `continue` to the other branch to reduce nesting

I mean

if parser not in keyword_parser:
    continue
if getattr(task_stage, parser, None) is None:
    continue
if score != score_:
    continue
I mean ``` if parser not in keyword_parser: continue ``` ``` if getattr(task_stage, parser, None) is None: continue ```` ``` if score != score_: continue ````

fixed.

fixed.
task_stage: task.Stage,
task_conf: task.Config,
diff_parser_config: result.ParserConfig,
conf_stage: result.StageDetail,
parser: str,
) -> None:
diff_parser = next((p for p in conf_stage.parsers if p.name == parser), None)
diff_parser = diff_parser_config
jon-lee marked this conversation as resolved Outdated

Is it necessary to rename?

Is it necessary to rename?
skip = task_stage.skip
cases = task_stage.cases
finalized_cases = [case for case in cases if case not in skip]

View File

@ -222,11 +222,11 @@
}
},
"copyInCached": {
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/ex2": "h7/build/ex2"
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
},
"copyInDir": ".",
"copyOut": [
@ -322,11 +322,11 @@
}
},
"copyInCached": {
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/ex2": "h7/build/ex2"
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
},
"copyInDir": ".",
"copyOut": [
@ -444,11 +444,11 @@
"cpuSetLimit": "",
"copyIn": {},
"copyInCached": {
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/ex2": "h7/build/ex2"
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
},
"copyInDir": ".",
"copyOut": [
@ -587,11 +587,11 @@
"cpuSetLimit": "",
"copyIn": {},
"copyInCached": {
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/ex2": "h7/build/ex2"
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
},
"copyInDir": ".",
"copyOut": [
@ -684,11 +684,11 @@
"cpuSetLimit": "",
"copyIn": {},
"copyInCached": {
"h7/build/compile_commands.json": "h7/build/compile_commands.json",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2": "h7/build/ex2",
"h7/build/ex2-ubsan": "h7/build/ex2-ubsan",
"h7/build/ex2-asan": "h7/build/ex2-asan",
"h7/build/ex2-msan": "h7/build/ex2-msan",
"h7/build/ex2": "h7/build/ex2"
"h7/build/compile_commands.json": "h7/build/compile_commands.json"
},
"copyInDir": ".",
"copyOut": [