diff --git a/README.md b/README.md index e3b98db..605851a 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,25 @@ 4. Install deps by `pdm install && pdm run pre-commit install` 5. Run the cli app by `pdm run app --help` 6. Check other commands or scripts with `pdm run --list` + +## How to use? + +- `joj3-config-generator convert` function is now supported, currently support one argument as input, it indicates the **convert root** + - default value on the server can be given as `/home/tt/.config/joj` + - **NOTE:** the user should ensure that the ideal `repo.toml` file is in the sub-directory of the **convert root** + - the intended immutable files should be placed at a sub-directory named `immutable_files` at same position as the `repo.toml` file + +```shell +$ tree . +. +|- immutable_files +| |-- push.yaml +| |-- release.yaml +|-- repo.toml +``` + +- sample command on the server + +```shell +joj3-config-generator convert /home/tt/.config/joj +``` diff --git a/joj3_config_generator/convert.py b/joj3_config_generator/convert.py index c689c35..28791b0 100644 --- a/joj3_config_generator/convert.py +++ b/joj3_config_generator/convert.py @@ -1,93 +1,41 @@ -from typing import List +import os +from typing import Dict from joj3_config_generator.models import joj1, repo, result, task +from joj3_config_generator.models.const import CACHE_ROOT, JOJ3_CONFIG_ROOT +from joj3_config_generator.processers.repo import ( + get_health_check_stage, + get_teapot_stage, +) +from joj3_config_generator.processers.task import get_conf_stage -# FIXME: LLM generated convert function, only for demostration def convert(repo_conf: repo.Config, task_conf: task.Config) -> result.Config: # Create the base ResultConf object result_conf = result.Config( - name=task_conf.task, - log_path=f"{task_conf.task.replace(' ', '_')}.log", - expire_unix_timestamp=( - int(task_conf.release.deadline.timestamp()) - if task_conf.release.deadline - else -1 - ), - stage=result.Stage(stages=[], sandbox_token=repo_conf.sandbox_token), - teapot=result.Teapot(), + name=task_conf.task.name, + # exact folder difference specified by type + log_path=str(CACHE_ROOT / "joj3" / f"{task_conf.task.type_}.log"), + expire_unix_timestamp=int(task_conf.release.end_time.timestamp()), + effective_unix_timestamp=int(task_conf.release.begin_time.timestamp()), + actor_csv_path=str(JOJ3_CONFIG_ROOT / "students.csv"), # students.csv position + max_total_score=repo_conf.max_total_score, + stage=result.Stage(sandbox_token=repo_conf.sandbox_token), ) + current_test = os.environ.get("PYTEST_CURRENT_TEST") is not None + # Construct health check stage + if not repo_conf.force_skip_health_check_on_test or not current_test: + result_conf.stage.stages.append(get_health_check_stage(repo_conf)) + cached: Dict[str, None] = {} # Convert each stage in the task configuration for task_stage in task_conf.stages: - executor_with_config = result.ExecutorWith( - default=result.Cmd( - args=task_stage.command.split(), - copy_in={ - file: result.LocalFile(src=file) - for file in task_stage.files.import_ - }, - copy_out_cached=task_stage.files.export, - ), - cases=[], # You can add cases if needed - ) - conf_stage = result.StageDetail( - name=task_stage.name, - group=task_conf.task, - executor=result.Executor( - name="sandbox", - with_=executor_with_config, - ), - parsers=[ - result.Parser(name=parser, with_={}) for parser in task_stage.parsers - ], - ) - - if "result-detail" in task_stage.parsers: - result_detail_parser = next( - p for p in conf_stage.parsers if p.name == "result-detail" - ) - result_detail_parser.with_.update(task_stage.result_detail) - - result_conf.stage.stages.append(conf_stage) + result_conf.stage.stages.append(get_conf_stage(task_conf, task_stage, cached)) + if not repo_conf.force_skip_teapot_on_test or not current_test: + result_conf.stage.post_stages.append(get_teapot_stage(repo_conf)) return result_conf -# FIXME: LLM generated convert function, only for demostration def convert_joj1(joj1_conf: joj1.Config) -> task.Config: - stages = [] - for language in joj1_conf.languages: - # Here you might want to create a stage for each language - # You can define a command based on language properties - command = f"run {language.language}" - # Assuming we don't have explicit files, we will set empty ones or default behavior - files = task.Files(import_=[], export=[]) - # Score can be derived from the first case or set to a default - score = 0 - parsers: List[str] = [] # Define parsers if applicable - if joj1_conf.cases: - score = sum( - case.score for case in joj1_conf.cases - ) # Sum scores for all cases - # Creating a stage for each language - stages.append( - task.Stage( - name=language.language, - command=command, - files=files, - score=score, - parsers=parsers, - result_detail=task.ParserResultDetail(), # You can customize this further if needed - ) - ) - # Assuming no deadline is provided in `joj1`, you can set it accordingly - release_deadline = ( - None # Placeholder for future implementation if deadlines are defined - ) - - return task.Config( - task=joj1_conf.languages[0].language if joj1_conf.languages else "Unnamed Task", - release=task.Release(deadline=release_deadline), - stages=stages, - ) + return task.Config() diff --git a/joj3_config_generator/load.py b/joj3_config_generator/load.py new file mode 100644 index 0000000..0139ff7 --- /dev/null +++ b/joj3_config_generator/load.py @@ -0,0 +1,39 @@ +from pathlib import Path +from typing import Dict, Tuple + +import inquirer +import rtoml +import yaml + +from joj3_config_generator.models import joj1, repo, task + + +def load_joj3_toml_answers() -> Dict[str, str]: + questions = [ + inquirer.List( + "size", + message="What size do you need?", + choices=["Jumbo", "Large", "Standard", "Medium", "Small", "Micro"], + ), + ] + answers = inquirer.prompt(questions) + return answers + + +def load_joj1_yaml(yaml_path: Path) -> joj1.Config: + joj1_obj = yaml.safe_load(yaml_path.read_text()) + return joj1.Config(**joj1_obj) + + +def load_joj3_toml( + root_path: Path, repo_toml_path: Path, task_toml_path: Path +) -> Tuple[repo.Config, task.Config]: + repo_obj = rtoml.loads(repo_toml_path.read_text()) + task_obj = rtoml.loads(task_toml_path.read_text()) + repo_conf = repo.Config(**repo_obj) + repo_conf.root = root_path + repo_conf.path = repo_toml_path.relative_to(root_path) + task_conf = task.Config(**task_obj) + task_conf.root = root_path + task_conf.path = task_toml_path.relative_to(root_path) + return repo_conf, task_conf diff --git a/joj3_config_generator/main.py b/joj3_config_generator/main.py index cf522c9..519b4e2 100644 --- a/joj3_config_generator/main.py +++ b/joj3_config_generator/main.py @@ -1,48 +1,44 @@ import json from pathlib import Path -import inquirer import rtoml import typer -import yaml from typing_extensions import Annotated from joj3_config_generator.convert import convert as convert_conf from joj3_config_generator.convert import convert_joj1 as convert_joj1_conf -from joj3_config_generator.models import joj1, repo, task +from joj3_config_generator.load import ( + load_joj1_yaml, + load_joj3_toml, + load_joj3_toml_answers, +) +from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT from joj3_config_generator.utils.logger import logger app = typer.Typer(add_completion=False) @app.command() -def create(toml: typer.FileTextWrite) -> None: +def create(toml_path: Path) -> None: """ Create a new JOJ3 toml config file """ - logger.info("Creating") - questions = [ - inquirer.List( - "size", - message="What size do you need?", - choices=["Jumbo", "Large", "Standard", "Medium", "Small", "Micro"], - ), - ] - answers = inquirer.prompt(questions) - logger.info(answers) + logger.info(f"Creating toml file {toml_path}") + answers = load_joj3_toml_answers() + logger.debug(f"Got answers: {answers}") + toml_path.write_text(rtoml.dumps({})) @app.command() -def convert_joj1(yaml_file: typer.FileText, toml_file: typer.FileTextWrite) -> None: +def convert_joj1(yaml_path: Path, toml_path: Path) -> None: """ Convert a JOJ1 yaml config file to JOJ3 toml config file """ - logger.info(f"Converting yaml file {yaml_file}") - joj1_obj = yaml.safe_load(yaml_file.read()) - joj1_model = joj1.Config(**joj1_obj) + logger.info(f"Converting yaml file {yaml_path}") + joj1_model = load_joj1_yaml(yaml_path) task_model = convert_joj1_conf(joj1_model) - result_dict = task_model.model_dump(by_alias=True) - toml_file.write(rtoml.dumps(result_dict)) + result_dict = task_model.model_dump(mode="json", by_alias=True, exclude_none=True) + toml_path.write_text(rtoml.dumps(result_dict)) @app.command() @@ -50,8 +46,7 @@ def convert( root: Annotated[ Path, typer.Argument( - help="root directory of config files, " - "located at /home/tt/.config/joj in JTC" + help=f"root directory of config files, located at {JOJ3_CONFIG_ROOT} in JTC" ), ] = Path(".") ) -> None: @@ -60,9 +55,7 @@ def convert( """ logger.info(f"Converting files in {root.absolute()}") for repo_toml_path in root.glob("**/repo.toml"): - repo_path = repo_toml_path.parent - repo_obj = rtoml.loads(repo_toml_path.read_text()) - for task_toml_path in repo_path.glob("**/*.toml"): + for task_toml_path in repo_toml_path.parent.glob("**/*.toml"): if repo_toml_path == task_toml_path: continue toml_name = task_toml_path.name.removesuffix(".toml") @@ -70,15 +63,11 @@ def convert( logger.info( f"Converting {repo_toml_path} & {task_toml_path} to {result_json_path}" ) - task_obj = rtoml.loads(task_toml_path.read_text()) - repo_conf = repo.Config(**repo_obj) - repo_conf.root = root - repo_conf.path = repo_toml_path.relative_to(root) - task_conf = task.Config(**task_obj) - task_conf.root = root - task_conf.path = task_toml_path.relative_to(root) + repo_conf, task_conf = load_joj3_toml(root, repo_toml_path, task_toml_path) result_model = convert_conf(repo_conf, task_conf) - result_dict = result_model.model_dump(by_alias=True, exclude_none=True) + result_dict = result_model.model_dump( + mode="json", by_alias=True, exclude_none=True + ) with result_json_path.open("w") as result_file: json.dump(result_dict, result_file, ensure_ascii=False, indent=4) result_file.write("\n") diff --git a/joj3_config_generator/models/common.py b/joj3_config_generator/models/common.py new file mode 100644 index 0000000..438f77d --- /dev/null +++ b/joj3_config_generator/models/common.py @@ -0,0 +1,19 @@ +from typing import Union + +import humanfriendly + + +class Memory(int): + def __new__(cls, value: Union[str, int]) -> "Memory": + if isinstance(value, str): + parsed = humanfriendly.parse_size(value, binary=True) + return super().__new__(cls, parsed) + return super().__new__(cls, value) + + +class Time(int): + def __new__(cls, value: Union[str, int]) -> "Time": + if isinstance(value, str): + parsed = humanfriendly.parse_timespan(value) * 1_000_000_000 # ns + return super().__new__(cls, round(parsed)) + return super().__new__(cls, value) diff --git a/joj3_config_generator/models/const.py b/joj3_config_generator/models/const.py new file mode 100644 index 0000000..8bea45b --- /dev/null +++ b/joj3_config_generator/models/const.py @@ -0,0 +1,11 @@ +from pathlib import Path + +from joj3_config_generator.models.common import Memory, Time + +DEFAULT_CPU_LIMIT = Time("1s") +DEFAULT_MEMORY_LIMIT = Memory("128m") +DEFAULT_FILE_LIMIT = Memory("32m") + +JOJ3_CONFIG_ROOT = Path("/home/tt/.config/joj") +TEAPOT_CONFIG_ROOT = Path("/home/tt/.config/teapot") +CACHE_ROOT = Path("/home/tt/.cache") diff --git a/joj3_config_generator/models/repo.py b/joj3_config_generator/models/repo.py index 5149e18..41b7d94 100644 --- a/joj3_config_generator/models/repo.py +++ b/joj3_config_generator/models/repo.py @@ -1,21 +1,29 @@ +import socket from pathlib import Path -from typing import List, Optional +from typing import List from pydantic import BaseModel, Field class Files(BaseModel): - whitelist_patterns: List[str] - whitelist_file: Optional[str] - required: List[str] - immutable: List[str] + required: List[str] = [] + immutable: List[str] = [] + + +class Groups(BaseModel): + name: List[str] = [] + max_count: List[int] = [] + time_period_hour: List[int] = [] class Config(BaseModel): + max_size: float = Field(10, ge=0) + files: Files = Files() + sandbox_token: str = Field("") + max_total_score: int = Field(100) + force_skip_health_check_on_test: bool = False + force_skip_teapot_on_test: bool = False + groups: Groups = Groups() root: Path = Path(".") path: Path = Path("repo.toml") - teaching_team: List[str] - max_size: float = Field(..., ge=0) - release_tags: List[str] - files: Files - sandbox_token: str + grading_repo_name: str = f"{socket.gethostname().split('-')[0]}-joj" diff --git a/joj3_config_generator/models/result.py b/joj3_config_generator/models/result.py index c7a81dd..430f936 100644 --- a/joj3_config_generator/models/result.py +++ b/joj3_config_generator/models/result.py @@ -1,6 +1,12 @@ -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from joj3_config_generator.models.const import ( + DEFAULT_CPU_LIMIT, + DEFAULT_FILE_LIMIT, + DEFAULT_MEMORY_LIMIT, +) class LocalFile(BaseModel): @@ -17,7 +23,7 @@ class PreparedFile(BaseModel): class Collector(BaseModel): name: str - max: int + max: int = DEFAULT_FILE_LIMIT pipe: bool = True @@ -33,27 +39,26 @@ class StreamOut(BaseModel): stream_out: bool = Field(..., alias="streamOut") -InputFile = Union[LocalFile | MemoryFile | PreparedFile | Symlink] +InputFile = Union[LocalFile, MemoryFile, PreparedFile, Symlink] class Cmd(BaseModel): - args: List[str] + args: List[str] = [] env: List[str] = [] - stdin: Optional[Union[InputFile | StreamIn]] = None - stdout: Optional[Union[Collector | StreamOut]] = None - stderr: Optional[Union[Collector | StreamOut]] = None - cpu_limit: int = Field(0, serialization_alias="cpuLimit") - real_cpu_limit: int = Field(0, serialization_alias="realCpuLimit") - clock_limit: int = Field(0, serialization_alias="clockLimit") - memory_limit: int = Field(0, serialization_alias="memoryLimit") + stdin: Union[InputFile, StreamIn] = MemoryFile(content="") + stdout: Union[Collector, StreamOut] = Collector(name="stdout") + stderr: Union[Collector, StreamOut] = Collector(name="stderr") + cpu_limit: int = Field(DEFAULT_CPU_LIMIT, serialization_alias="cpuLimit") + clock_limit: int = Field(2 * DEFAULT_CPU_LIMIT, serialization_alias="clockLimit") + memory_limit: int = Field(DEFAULT_MEMORY_LIMIT, serialization_alias="memoryLimit") stack_limit: int = Field(0, serialization_alias="stackLimit") - proc_limit: int = Field(0, serialization_alias="procLimit") + proc_limit: int = Field(50, serialization_alias="procLimit") cpu_rate_limit: int = Field(0, serialization_alias="cpuRateLimit") cpu_set_limit: str = Field("", serialization_alias="cpuSetLimit") copy_in: Dict[str, InputFile] = Field({}, serialization_alias="copyIn") copy_in_cached: Dict[str, str] = Field({}, serialization_alias="copyInCached") copy_in_dir: str = Field(".", serialization_alias="copyInDir") - copy_out: List[str] = Field([], serialization_alias="copyOut") + copy_out: List[str] = Field(["stdout", "stderr"], serialization_alias="copyOut") copy_out_cached: List[str] = Field([], serialization_alias="copyOutCached") copy_out_max: int = Field(0, serialization_alias="copyOutMax") copy_out_dir: str = Field("", serialization_alias="copyOutDir") @@ -66,11 +71,10 @@ class Cmd(BaseModel): class OptionalCmd(BaseModel): args: Optional[List[str]] = None env: Optional[List[str]] = None - stdin: Optional[Union[InputFile | StreamIn]] = None - stdout: Optional[Union[Collector | StreamOut]] = None - stderr: Optional[Union[Collector | StreamOut]] = None + stdin: Optional[Union[InputFile, StreamIn]] = None + stdout: Optional[Union[Collector, StreamOut]] = None + stderr: Optional[Union[Collector, StreamOut]] = None cpu_limit: Optional[int] = Field(None, serialization_alias="cpuLimit") - real_cpu_limit: Optional[int] = Field(None, serialization_alias="realCpuLimit") clock_limit: Optional[int] = Field(None, serialization_alias="clockLimit") memory_limit: Optional[int] = Field(None, serialization_alias="memoryLimit") stack_limit: Optional[int] = Field(None, serialization_alias="stackLimit") @@ -101,23 +105,39 @@ class OptionalCmd(BaseModel): class ExecutorWith(BaseModel): - default: Cmd - cases: List[OptionalCmd] + default: Cmd = Cmd() + cases: List[OptionalCmd] = [] class Executor(BaseModel): name: str - with_: ExecutorWith = Field(..., serialization_alias="with") + with_: ExecutorWith = Field(ExecutorWith(), serialization_alias="with") class Parser(BaseModel): name: str - with_: Dict[str, Any] = Field(..., serialization_alias="with") + if TYPE_CHECKING: + + class Empty(BaseModel): + pass + + with_: BaseModel = Field(Empty(), serialization_alias="with") + else: + with_: Dict[str, Any] = Field({}, serialization_alias="with") + + model_config = ConfigDict(validate_assignment=True) + + @field_validator("with_", mode="before") + @classmethod + def validate_with(cls, v: Any) -> Dict[str, Any]: + if isinstance(v, BaseModel): + return v.model_dump(by_alias=True) + raise ValueError("Must be a BaseModel instance") class StageDetail(BaseModel): name: str - group: str + group: str = "" executor: Executor parsers: List[Parser] @@ -130,26 +150,74 @@ class Stage(BaseModel): output_path: str = Field( "/tmp/joj3_result.json", serialization_alias="outputPath" ) # nosec: B108 - stages: List[StageDetail] - - -class Teapot(BaseModel): - log_path: str = Field( - "/home/tt/.cache/joint-teapot-debug.log", serialization_alias="logPath" - ) - scoreboard_path: str = Field("scoreboard.csv", serialization_alias="scoreboardPath") - failed_table_path: str = Field( - "failed-table.md", serialization_alias="failedTablePath" - ) - grading_repo_name: str = Field("", serialization_alias="gradingRepoName") - skip_issue: bool = Field(False, serialization_alias="skipIssue") - skip_scoreboard: bool = Field(False, serialization_alias="skipScoreboard") - skip_failed_table: bool = Field(False, serialization_alias="skipFailedTable") + stages: List[StageDetail] = [] + pre_stages: List[StageDetail] = Field([], serialization_alias="preStages") + post_stages: List[StageDetail] = Field([], serialization_alias="postStages") class Config(BaseModel): - name: str = "unknown" + name: str = "" log_path: str = Field("", serialization_alias="logPath") expire_unix_timestamp: int = Field(-1, serialization_alias="expireUnixTimestamp") + effective_unix_timestamp: int = Field( + -1, serialization_alias="effectiveUnixTimestamp" + ) + actor_csv_path: str = Field("", serialization_alias="actorCsvPath") + max_total_score: int = Field(100, serialization_alias="maxTotalScore") stage: Stage - teapot: Teapot + + +class DummyConfig(BaseModel): + score: int = 0 + comment: Optional[str] = None + force_quit_on_not_accepted: Optional[bool] = Field( + False, serialization_alias="forceQuitOnNotAccepted" + ) + + +class DiffOutputConfig(BaseModel): + score: int = 100 + file_name: str = Field("", serialization_alias="fileName") + answer_path: str = Field("", serialization_alias="answerPath") + force_quit_on_diff: bool = Field(False, serialization_alias="forceQuitOnDiff") + always_hide: bool = Field(False, serialization_alias="alwaysHide") + compare_space: bool = Field(False, serialization_alias="compareSpace") + + +class ResultDetailConfig(BaseModel): + score: int = 0 + comment: str = "" + show_files: List[str] = Field([], serialization_alias="showFiles") + show_exit_status: bool = Field(True, serialization_alias="showExitStatus") + show_runtime: bool = Field(True, serialization_alias="showRuntime") + show_memory: bool = Field(False, serialization_alias="showMemory") + + +class KeywordConfig(BaseModel): + keywords: List[str] = [] + score: int = 0 + + +class KeywordMatchConfig(BaseModel): + matches: List[KeywordConfig] = [] + + +class FileConfig(BaseModel): + name: str = "" + + +class DiffCasesConfig(BaseModel): + outputs: List[DiffOutputConfig] = [] + + +class DiffConfig(BaseModel): + name: str = "diff" + cases: List[DiffCasesConfig] = [] + + +class MsgConfig(BaseModel): + msg: str = "" + + +class ScoreConfig(BaseModel): + score: int = 0 diff --git a/joj3_config_generator/models/task.py b/joj3_config_generator/models/task.py index a09a069..7e790a3 100644 --- a/joj3_config_generator/models/task.py +++ b/joj3_config_generator/models/task.py @@ -1,8 +1,16 @@ -from datetime import datetime +from datetime import datetime, timedelta +from enum import Enum from pathlib import Path -from typing import List, Optional +from typing import Any, Dict, List, Type -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from joj3_config_generator.models.common import Memory, Time +from joj3_config_generator.models.const import ( + DEFAULT_CPU_LIMIT, + DEFAULT_FILE_LIMIT, + DEFAULT_MEMORY_LIMIT, +) class ParserResultDetail(BaseModel): @@ -10,31 +18,136 @@ class ParserResultDetail(BaseModel): mem: bool = True # Display memory usage stdout: bool = False # Display stdout messages stderr: bool = False # Display stderr messages + exit_status: bool = True + + +class ParserFile(BaseModel): + name: str = "" + + +class ParserLog(BaseModel): + filename: str + msg: str = "" + level: str = "" + + +class ParserDummy(BaseModel): + comment: str = "" + score: int = 0 + force_quit: bool = False + + +class ParserKeyword(BaseModel): + keyword: List[str] = [] + weight: List[int] = [] + + +class Outputs(BaseModel): + score: int = 0 + ignore_spaces: bool = True + hide: bool = False + force_quit: bool = False + + +class ParserDiff(BaseModel): + output: Outputs = Outputs() class Files(BaseModel): - import_: List[str] = Field(serialization_alias="import", validation_alias="import") - export: List[str] + import_: List[str] = Field([], alias="import") + export: List[str] = [] + + +class Limit(BaseModel): + mem: int = DEFAULT_MEMORY_LIMIT + cpu: int = DEFAULT_CPU_LIMIT + stdout: int = DEFAULT_FILE_LIMIT + stderr: int = DEFAULT_FILE_LIMIT + + model_config = ConfigDict(validate_assignment=True) + + @field_validator("cpu", mode="before") + @classmethod + def ensure_time(cls, v: Any) -> Time: + if isinstance(v, str): + return Time(v) + raise ValueError("Must be a string") + + @field_validator("mem", "stdout", "stderr", mode="before") + @classmethod + def ensure_mem(cls, v: Any) -> Memory: + if isinstance(v, str): + return Memory(v) + raise ValueError("Must be a string") + + +class Parser(str, Enum): + CLANG_TIDY = "clangtidy" + CPPCHECK = "cppcheck" + CPPLINT = "cpplint" + KEYWORD = "keyword" + RESULT_STATUS = "result-status" + RESULT_DETAIL = "result-detail" + DUMMY = "dummy" + FILE = "file" + DIFF = "diff" class Stage(BaseModel): - name: str # Stage name - command: str # Command to run - files: Files # Files to import and export - score: int # Score for the task - parsers: List[str] # list of parsers - result_detail: ParserResultDetail = ( - ParserResultDetail() - ) # for result-detail parser + name: str = "" # Stage name + env: List[str] = [] + command: str = "" # Command to run + files: Files = Files() + in_: str = Field("", alias="in") + out_: str = Field("", alias="out") + score: int = 0 + parsers: List[Parser] = [] # list of parsers + limit: Limit = Limit() + dummy: ParserDummy = ParserDummy() + result_status: ParserDummy = Field(ParserDummy(), alias="result-status") + keyword: ParserKeyword = ParserKeyword() + clangtidy: ParserKeyword = ParserKeyword() + cppcheck: ParserKeyword = ParserKeyword() + cpplint: ParserKeyword = ParserKeyword() + result_detail: ParserResultDetail = Field( + ParserResultDetail(), alias="result-detail" + ) + file: ParserFile = ParserFile() + skip: List[str] = [] + + # cases related + cases: Dict[str, "Stage"] = {} + diff: ParserDiff = ParserDiff() + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="before") + @classmethod + def gather_cases(cls: Type["Stage"], values: Dict[str, Any]) -> Dict[str, Any]: + cases = {k: v for k, v in values.items() if k.startswith("case")} + for key in cases: + values.pop(key) + values["cases"] = {k: v for k, v in cases.items()} + return values class Release(BaseModel): - deadline: Optional[datetime] # RFC 3339 formatted date-time with offset + end_time: datetime = datetime.now() + timedelta( + days=365 + ) # RFC 3339 formatted date-time with offset + begin_time: datetime = datetime.fromtimestamp( + 0 + ) # RFC 3339 formatted date-time with offset + + +class Task(BaseModel): + type_: str = Field("unknown", serialization_alias="type", validation_alias="type") + name: str = "unknown" class Config(BaseModel): root: Path = Path(".") path: Path = Path("task.toml") - task: str # Task name (e.g., hw3 ex5) - release: Release # Release configuration - stages: List[Stage] # list of stage configurations + task: Task = Task() # Task name (e.g., hw3 ex5) + release: Release = Release() # Release configuration + stages: List[Stage] = [] # list of stage configurations diff --git a/joj3_config_generator/processers/__init__.py b/joj3_config_generator/processers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/joj3_config_generator/processers/repo.py b/joj3_config_generator/processers/repo.py new file mode 100644 index 0000000..ed29d0e --- /dev/null +++ b/joj3_config_generator/processers/repo.py @@ -0,0 +1,108 @@ +import hashlib +from pathlib import Path +from typing import List + +from joj3_config_generator.models import repo, result +from joj3_config_generator.models.const import CACHE_ROOT, TEAPOT_CONFIG_ROOT + + +def get_teapot_stage(repo_conf: repo.Config) -> result.StageDetail: + args = [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + str(TEAPOT_CONFIG_ROOT / "teapot.env"), + "--grading-repo-name", + repo_conf.grading_repo_name, + "--max-total-score", + str(repo_conf.max_total_score), + ] + + stage_conf = result.StageDetail( + name="teapot", + executor=result.Executor( + name="local", + with_=result.ExecutorWith( + default=result.Cmd( + args=args, + env=[f"LOG_FILE_PATH={CACHE_ROOT}/joint-teapot-debug.log"], + ), + cases=[], + ), + ), + parsers=[result.Parser(name="log", with_=result.MsgConfig(msg="joj3 summary"))], + ) + return stage_conf + + +def get_health_check_args(repo_conf: repo.Config) -> List[str]: + return [ + "/usr/local/bin/repo-health-checker", + "-root=.", + f"-repoSize={str(repo_conf.max_size)}", + *[f"-meta={meta}" for meta in repo_conf.files.required], + f"-checkFileSumList={','.join(get_hashs(repo_conf))}", + f"-checkFileNameList={','.join(repo_conf.files.immutable)}", + ] + + +def get_teapot_check_args(repo_conf: repo.Config) -> List[str]: + return [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + str(TEAPOT_CONFIG_ROOT / "teapot.env"), + "--grading-repo-name", + repo_conf.grading_repo_name, + "--group-config", + ",".join( + f"{name}={max_count}:{time_period}" + for name, max_count, time_period in zip( + repo_conf.groups.name, + repo_conf.groups.max_count, + repo_conf.groups.time_period_hour, + ) + ), + ] + + +def get_health_check_stage(repo_conf: repo.Config) -> result.StageDetail: + health_check_stage = result.StageDetail( + name="Health Check", + group="", + executor=result.Executor( + name="local", + with_=result.ExecutorWith( + default=result.Cmd(), + cases=[ + result.OptionalCmd( + args=get_health_check_args(repo_conf), + ), + result.OptionalCmd( + args=get_teapot_check_args(repo_conf), + env=[f"LOG_FILE_PATH={CACHE_ROOT}/joint-teapot-debug.log"], + ), + ], + ), + ), + parsers=[ + result.Parser(name="healthcheck", with_=result.ScoreConfig(score=1)), + result.Parser(name="debug", with_=result.ScoreConfig(score=0)), + ], + ) + return health_check_stage + + +def calc_sha256sum(file_path: Path) -> str: + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + for byte_block in iter(lambda: f.read(64 * 1024), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + + +def get_hashs(repo_conf: repo.Config) -> List[str]: + base_dir = (repo_conf.root / repo_conf.path).parent + immutable_dir = base_dir / "immutable_files" + immutable_files = [ + immutable_dir / Path(file).name for file in repo_conf.files.immutable + ] + return [calc_sha256sum(file) for file in immutable_files] diff --git a/joj3_config_generator/processers/task.py b/joj3_config_generator/processers/task.py new file mode 100644 index 0000000..e21b208 --- /dev/null +++ b/joj3_config_generator/processers/task.py @@ -0,0 +1,206 @@ +import re +import shlex +from pathlib import Path +from typing import Any, Callable, Dict, List, Tuple + +from joj3_config_generator.models import result, task +from joj3_config_generator.models.common import Memory, Time +from joj3_config_generator.models.const import JOJ3_CONFIG_ROOT +from joj3_config_generator.models.task import Parser as ParserEnum + + +def get_conf_stage( + task_conf: task.Config, + task_stage: task.Stage, + cached: Dict[str, None], +) -> result.StageDetail: + conf_stage = result.StageDetail( + name=task_stage.name, + # group is determined by adding between "[]" in the name of the task + group=( + match.group(1) + if (match := re.search(r"\[([^\[\]]+)\]", task_stage.name or "")) + else "" + ), + executor=result.Executor( + name="sandbox", + with_=get_executor_with(task_stage, cached), + ), + parsers=([result.Parser(name=parser) for parser in task_stage.parsers]), + ) + processed_dict = get_processed_dict(task_stage) + for idx, parser in enumerate(task_stage.parsers): + if parser in processed_dict: + fn, parser_model = processed_dict[parser] + fn(parser_model, conf_stage.parsers[idx]) + elif parser == ParserEnum.DIFF: + fix_diff( + task_stage, + conf_stage.parsers[idx], + conf_stage.executor, + JOJ3_CONFIG_ROOT / task_conf.path.parent, + ) + else: + continue + return conf_stage + + +def get_processed_dict( + task_stage: task.Stage, +) -> Dict[ParserEnum, Tuple[Callable[[Any, result.Parser], None], Any]]: + processed_dict: Dict[ + ParserEnum, Tuple[Callable[[Any, result.Parser], None], Any] + ] = { + ParserEnum.CLANG_TIDY: (fix_keyword, task_stage.clangtidy), + ParserEnum.KEYWORD: (fix_keyword, task_stage.keyword), + ParserEnum.CPPCHECK: (fix_keyword, task_stage.cppcheck), + ParserEnum.CPPLINT: (fix_keyword, task_stage.cpplint), + ParserEnum.RESULT_DETAIL: (fix_result_detail, task_stage.result_detail), + ParserEnum.DUMMY: (fix_dummy, task_stage.dummy), + ParserEnum.RESULT_STATUS: (fix_dummy, task_stage.result_status), + ParserEnum.FILE: (fix_file, task_stage.file), + } + return processed_dict + + +def get_executor_with( + task_stage: task.Stage, cached: Dict[str, None] +) -> result.ExecutorWith: + file_import = task_stage.files.import_ + copy_in_files = (file for file in file_import if file not in cached) + file_export = task_stage.files.export + copy_out_files = ["stdout", "stderr"] + executor_with_config = result.ExecutorWith( + default=result.Cmd( + args=shlex.split(task_stage.command), + copy_in={ + file: result.LocalFile(src=str(JOJ3_CONFIG_ROOT / file)) + # all copyin files store in this tools folder + # TODO: are there any corner cases? + for file in copy_in_files + }, + copy_out=copy_out_files, + copy_in_cached={file: file for file in cached}, + copy_out_cached=file_export, + cpu_limit=Time(task_stage.limit.cpu), + clock_limit=2 * Time(task_stage.limit.cpu), + memory_limit=Memory(task_stage.limit.mem), + stderr=result.Collector( + name="stderr", pipe=True, max=Memory(task_stage.limit.stderr) + ), + stdout=result.Collector( + name="stdout", pipe=True, max=Memory(task_stage.limit.stdout) + ), + ), + cases=[], + ) + for file in file_export: + cached[file] = None + return executor_with_config + + +def fix_keyword( + keyword_config: task.ParserKeyword, keyword_parser: result.Parser +) -> None: + if len(keyword_config.keyword) != len(keyword_config.weight): + raise ValueError("Keywords and weights must have the same length") + score_groups: Dict[int, List[str]] = {} + for keyword, score in zip(keyword_config.keyword, keyword_config.weight): + score_groups.setdefault(score, []).append(keyword) + keyword_parser.with_ = result.KeywordMatchConfig( + matches=[ + result.KeywordConfig(keywords=keywords, score=score) + for score, keywords in score_groups.items() + ] + ) + + +def fix_result_detail( + result_detail_parser_config: task.ParserResultDetail, + result_detail_parser: result.Parser, +) -> None: + show_files = [] + if result_detail_parser_config.stdout: + show_files.append("stdout") + if result_detail_parser_config.stderr: + show_files.append("stderr") + result_detail_parser.with_ = result.ResultDetailConfig( + score=0, + comment="", + show_files=show_files, + show_exit_status=result_detail_parser_config.exit_status, + show_runtime=result_detail_parser_config.time, + show_memory=result_detail_parser_config.mem, + ) + + +def fix_dummy( + dummy_parser_config: task.ParserDummy, dummy_parser: result.Parser +) -> None: + # we don't use dummy parser in real application + dummy_parser.with_ = result.DummyConfig( + score=dummy_parser_config.score, + comment=dummy_parser_config.comment, + force_quit_on_not_accepted=dummy_parser_config.force_quit, + ) + + +def fix_file(file_parser_config: task.ParserFile, file_parser: result.Parser) -> None: + file_parser.with_ = result.FileConfig(name=file_parser_config.name) + + +def fix_diff( + task_stage: task.Stage, + diff_parser_config: result.Parser, + diff_executor: result.Executor, + base_dir: Path, +) -> None: + skip = task_stage.skip + cases = task_stage.cases + finalized_cases = [case for case in cases if case not in skip] + + stage_cases = [] + parser_cases = [] + + for case in finalized_cases: + case_stage = task_stage.cases.get(case) if task_stage.cases else None + if not case_stage: + continue + + cpu_limit = case_stage.limit.cpu + clock_limit = 2 * case_stage.limit.cpu + memory_limit = case_stage.limit.mem + command = case_stage.command + stdin = case_stage.in_ if case_stage.in_ != "" else f"{case}.in" + stdout = case_stage.out_ if case_stage.out_ != "" else f"{case}.out" + + stage_cases.append( + result.OptionalCmd( + stdin=result.LocalFile( + src=str(base_dir / stdin), + ), + args=shlex.split(command) if command else None, + cpu_limit=cpu_limit, + clock_limit=clock_limit, + memory_limit=memory_limit, + proc_limit=50, + ) + ) + diff_output = case_stage.diff.output + parser_cases.append( + result.DiffCasesConfig( + outputs=[ + result.DiffOutputConfig( + score=diff_output.score, + file_name="stdout", + answer_path=str(base_dir / stdout), + force_quit_on_diff=diff_output.force_quit, + always_hide=diff_output.hide, + compare_space=not diff_output.ignore_spaces, + ) + ] + ) + ) + + diff_executor.with_.cases = stage_cases + diff_parser_config.with_ = result.DiffConfig(name="diff", cases=parser_cases) diff --git a/joj3_config_generator/utils/logger.py b/joj3_config_generator/utils/logger.py index 4781b36..6410fa6 100644 --- a/joj3_config_generator/utils/logger.py +++ b/joj3_config_generator/utils/logger.py @@ -2,7 +2,7 @@ import logging import sys from sys import stderr from types import FrameType -from typing import Optional +from typing import Optional, Union from loguru import logger as logger @@ -10,7 +10,7 @@ from loguru import logger as logger # recipe from https://loguru.readthedocs.io/en/stable/overview.html#entirely-compatible-with-standard-logging class InterceptHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: - level: str | int + level: Union[str, int] try: level = logger.level(record.levelname).name except ValueError: diff --git a/pyproject.toml b/pyproject.toml index 3cf5b3c..88219fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pydantic>=2.9.2", "inquirer>=3.4.0", "rtoml>=0.11.0", + "humanfriendly>=10.0", ] requires-python = ">=3.9" authors = [{ name = "JOJ3-dev", email = "joj3@focs.ji.sjtu.edu.cn" }] diff --git a/tests/convert/basic/immutable_files/.gitattributes b/tests/convert/basic/immutable_files/.gitattributes new file mode 100644 index 0000000..b910c4a --- /dev/null +++ b/tests/convert/basic/immutable_files/.gitattributes @@ -0,0 +1,33 @@ +*.avi filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.djvu filter=lfs diff=lfs merge=lfs -text +*.doc filter=lfs diff=lfs merge=lfs -text +*.docx filter=lfs diff=lfs merge=lfs -text +*.epub filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.ipynb filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.JPEG filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.JPG filter=lfs diff=lfs merge=lfs -text +*.mkv filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.ods filter=lfs diff=lfs merge=lfs -text +*.odt filter=lfs diff=lfs merge=lfs -text +*.otf filter=lfs diff=lfs merge=lfs -text +*.pdf filter=lfs diff=lfs merge=lfs -text +*.PDF filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.PNG filter=lfs diff=lfs merge=lfs -text +*.ppt filter=lfs diff=lfs merge=lfs -text +*.pptx filter=lfs diff=lfs merge=lfs -text +*.ps filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.ttf filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text +*.xls filter=lfs diff=lfs merge=lfs -text +*.xlsx filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text diff --git a/tests/convert/basic/immutable_files/.gitignore b/tests/convert/basic/immutable_files/.gitignore new file mode 100644 index 0000000..754f776 --- /dev/null +++ b/tests/convert/basic/immutable_files/.gitignore @@ -0,0 +1,23 @@ +################################ +## White list based gitignore ## +################################ + +# forbidden +* +.* + +# allowed +!.gitignore +!.gitattributes +!.gitea/ +!.gitea/issue_template/ +!.gitea/workflows/ +!*.yaml +!Makefile +!CMakeLists.txt +!h[0-8]/ +!*.m +!*.c +!*.cpp +!*.h +!*.md diff --git a/tests/convert/basic/immutable_files/push.yaml b/tests/convert/basic/immutable_files/push.yaml new file mode 100644 index 0000000..1541fc1 --- /dev/null +++ b/tests/convert/basic/immutable_files/push.yaml @@ -0,0 +1,19 @@ +name: Run JOJ3 on Push +on: [push] + +jobs: + run: + container: + runs-on: focs-latest-slim + volumes: + - /home/tt/.config:/home/tt/.config + - /home/tt/.cache:/home/tt/.cache + - /home/tt/.ssh:/home/tt/.ssh + steps: + - name: Check out repository code + uses: actions/checkout@focs + with: + fetch-depth: 0 + - name: Run joj3 + run: | + sudo -E -u tt joj3 -conf-root /home/tt/.config/joj/tests/homework diff --git a/tests/convert/basic/immutable_files/release.yaml b/tests/convert/basic/immutable_files/release.yaml new file mode 100644 index 0000000..be3f9ac --- /dev/null +++ b/tests/convert/basic/immutable_files/release.yaml @@ -0,0 +1,21 @@ +name: Run JOJ3 on Release +on: + release: + types: [published] + +jobs: + run: + runs-on: focs-latest-slim + container: + volumes: + - /home/tt/.config:/home/tt/.config + - /home/tt/.cache:/home/tt/.cache + - /home/tt/.ssh:/home/tt/.ssh + steps: + - name: Check out repository code + uses: actions/checkout@focs + with: + fetch-depth: 0 + - name: Run joj3 + run: | + sudo -E -u tt joj3 -conf-root "/home/tt/.config/joj/tests/homework" -conf-name "conf-release.json" -tag "${{ github.ref_name }}" diff --git a/tests/convert/basic/repo.toml b/tests/convert/basic/repo.toml index f9012cc..3d13343 100644 --- a/tests/convert/basic/repo.toml +++ b/tests/convert/basic/repo.toml @@ -1,10 +1,16 @@ -teaching_team = ["prof_john", "ta_alice", "ta_bob"] -max_size = 50.5 -release_tags = ["v1.0", "v2.0", "final"] +grading_repo_name = "ece280-joj" + sandbox_token = "test" +# reconfigure later +max_total_score = 100 +max_size = 50.5 + +# for tests +[groups] +name = ["joj", "run"] +max_count = [1000, 1000] +time_period_hour = [24, 24] [files] -whitelist_patterns = ["*.py", "*.txt", "*.md"] -whitelist_file = ".whitelist" -required = ["main.py", "README.md"] -immutable = ["config.yaml", "setup.py"] +required = ["README.md", "Changelog.md"] +immutable = [".gitignore", ".gitattributes",".gitea/workflows/push.yaml", ".gitea/workflows/release.yaml"] diff --git a/tests/convert/basic/task.json b/tests/convert/basic/task.json index 6abcc7a..bc3483b 100644 --- a/tests/convert/basic/task.json +++ b/tests/convert/basic/task.json @@ -1,47 +1,52 @@ { - "name": "hw3 ex5", - "logPath": "hw3_ex5.log", - "expireUnixTimestamp": 1729267140, + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, "stage": { "sandboxExecServer": "172.17.0.1:5051", "sandboxToken": "test", "outputPath": "/tmp/joj3_result.json", "stages": [ { - "name": "judge_base", - "group": "hw3 ex5", + "name": "Health Check", + "group": "", "executor": { - "name": "sandbox", + "name": "local", "with": { "default": { - "args": [ - "./matlab-joj", - "./h3/ex5.m" - ], + "args": [], "env": [], - "cpuLimit": 0, - "realCpuLimit": 0, - "clockLimit": 0, - "memoryLimit": 0, + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, - "procLimit": 0, + "procLimit": 50, "cpuRateLimit": 0, "cpuSetLimit": "", - "copyIn": { - "tools/matlab-joj": { - "src": "tools/matlab-joj" - }, - "tools/matlab_formatter.py": { - "src": "tools/matlab_formatter.py" - } - }, + "copyIn": {}, "copyInCached": {}, "copyInDir": ".", - "copyOut": [], - "copyOutCached": [ - "output/ex5_results.txt", - "output/ex5_logs.txt" + "copyOut": [ + "stdout", + "stderr" ], + "copyOutCached": [], "copyOutMax": 0, "copyOutDir": "", "tty": false, @@ -49,59 +54,98 @@ "dataSegmentLimit": false, "addressSpaceLimit": false }, - "cases": [] + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=50.5", + "-meta=README.md", + "-meta=Changelog.md", + "-checkFileSumList=a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc,b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,2ba059f3977e2e3dee6cacbfbf0ba2578baa1b8e04b4977aec400868b6e49856,bf7d181362affdcc72aac33f3520e4e6371adc48ea62a6f24524b4a3e76724c5", + "-checkFileNameList=.gitignore,.gitattributes,.gitea/workflows/push.yaml,.gitea/workflows/release.yaml" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "ece280-joj", + "--group-config", + "joj=1000:24,run=1000:24" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] } }, "parsers": [ { - "name": "diff", - "with": {} + "name": "healthcheck", + "with": { + "score": 1 + } }, { - "name": "result-detail", + "name": "debug", "with": { - "time": false, - "mem": false, - "stdout": false, - "stderr": true + "score": 0 } } ] }, { - "name": "judge_base2", - "group": "hw3 ex5", + "name": "Compilation", + "group": "", "executor": { "name": "sandbox", "with": { "default": { "args": [ - "./matlab-joj", - "./h3/ex5.m" + "./tools/compile" ], "env": [], - "cpuLimit": 0, - "realCpuLimit": 0, - "clockLimit": 0, - "memoryLimit": 0, + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, "stackLimit": 0, - "procLimit": 0, + "procLimit": 50, "cpuRateLimit": 0, "cpuSetLimit": "", "copyIn": { - "tools/matlab-joj": { - "src": "tools/matlab-joj" - }, - "tools/matlab_formatter.py": { - "src": "tools/matlab_formatter.py" + "tools/compile": { + "src": "/home/tt/.config/joj/tools/compile" } }, "copyInCached": {}, "copyInDir": ".", - "copyOut": [], + "copyOut": [ + "stdout", + "stderr" + ], "copyOutCached": [ - "output/ex5_results2.txt", - "output/ex5_logs2.txt" + "h7/build/ex2", + "h7/build/ex2-asan", + "h7/build/ex2-ubsan", + "h7/build/ex2-msan", + "h7/build/compile_commands.json" ], "copyOutMax": 0, "copyOutDir": "", @@ -115,29 +159,691 @@ }, "parsers": [ { - "name": "diff", - "with": {} + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + }, + { + "name": "result-status", + "with": { + "score": 0, + "comment": "Congratulations! Your code compiled successfully.", + "forceQuitOnNotAccepted": true + } + } + ] + }, + { + "name": "[cq] Filelength", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./tools/filelength", + "400", + "300", + "*.cpp", + "*.h" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tools/filelength": { + "src": "/home/tt/.config/joj/tools/filelength" + } + }, + "copyInCached": { + "h7/build/ex2": "h7/build/ex2", + "h7/build/ex2-asan": "h7/build/ex2-asan", + "h7/build/ex2-ubsan": "h7/build/ex2-ubsan", + "h7/build/ex2-msan": "h7/build/ex2-msan", + "h7/build/compile_commands.json": "h7/build/compile_commands.json" + }, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "keyword", + "with": { + "matches": [ + { + "keywords": [ + "max" + ], + "score": 20 + }, + { + "keywords": [ + "recommended" + ], + "score": 10 + } + ] + } }, { "name": "result-detail", "with": { - "time": true, - "mem": true, - "stdout": false, - "stderr": false + "score": 0, + "comment": "", + "showFiles": [ + "stdout" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + }, + { + "name": "[cq] Clang-tidy", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "run-clang-tidy-18", + "-header-filter=.*", + "-quiet", + "-load=/usr/local/lib/libcodequality.so", + "-p", + "h7/build", + "h7/ex2.cpp" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 4194304, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tests/homework/h7/.clang-tidy": { + "src": "/home/tt/.config/joj/tests/homework/h7/.clang-tidy" + } + }, + "copyInCached": { + "h7/build/ex2": "h7/build/ex2", + "h7/build/ex2-asan": "h7/build/ex2-asan", + "h7/build/ex2-ubsan": "h7/build/ex2-ubsan", + "h7/build/ex2-msan": "h7/build/ex2-msan", + "h7/build/compile_commands.json": "h7/build/compile_commands.json" + }, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "clangtidy", + "with": { + "matches": [ + { + "keywords": [ + "codequality-unchecked-malloc-result", + "readability-duplicate-include", + "readability-identifier-naming", + "readability-redundant", + "readability-misplaced-array-index", + "cppcoreguidelines-init-variables", + "bugprone-suspicious-string-compare", + "google-global-names-in-headers", + "clang-diagnostic", + "clang-analyzer", + "misc", + "performance", + "portability" + ], + "score": 5 + }, + { + "keywords": [ + "codequality-no-global-variables", + "codequality-no-header-guard", + "codequality-no-fflush-stdin" + ], + "score": 20 + }, + { + "keywords": [ + "readability-function-size" + ], + "score": 10 + }, + { + "keywords": [ + "readability-misleading-indentation" + ], + "score": 15 + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stdout" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + }, + { + "name": "[cq] Cppcheck", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "cppcheck", + "--template={\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}", + "--force", + "--enable=all", + "--suppress=missingIncludeSystem", + "--quiet", + "h7/ex2.cpp" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 8388608, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": { + "h7/build/ex2": "h7/build/ex2", + "h7/build/ex2-asan": "h7/build/ex2-asan", + "h7/build/ex2-ubsan": "h7/build/ex2-ubsan", + "h7/build/ex2-msan": "h7/build/ex2-msan", + "h7/build/compile_commands.json": "h7/build/compile_commands.json" + }, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "keyword", + "with": { + "matches": [] + } + }, + { + "name": "cppcheck", + "with": { + "matches": [ + { + "keywords": [ + "error" + ], + "score": 15 + }, + { + "keywords": [ + "warning", + "portability", + "performance", + "style" + ], + "score": 5 + } + ] + } + }, + { + "name": "clangtidy", + "with": { + "matches": [] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + }, + { + "name": "cpplint", + "with": { + "matches": [] + } + }, + { + "name": "result-status", + "with": { + "score": 0, + "comment": "", + "forceQuitOnNotAccepted": false + } + }, + { + "name": "file", + "with": { + "name": "" + } + }, + { + "name": "dummy", + "with": { + "score": 0, + "comment": "", + "forceQuitOnNotAccepted": false + } + }, + { + "name": "diff", + "with": { + "name": "diff", + "cases": [] + } + } + ] + }, + { + "name": "[cq] Cpplint", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "cpplint", + "--linelength=120", + "--filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-runtime/int,-readability/todo,-build/include_subdir,-build/header_guard,-build/include_what_you_use", + "--recursive", + "--exclude=build", + "h7/ex2.cpp" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 68157440, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": { + "h7/build/ex2": "h7/build/ex2", + "h7/build/ex2-asan": "h7/build/ex2-asan", + "h7/build/ex2-ubsan": "h7/build/ex2-ubsan", + "h7/build/ex2-msan": "h7/build/ex2-msan", + "h7/build/compile_commands.json": "h7/build/compile_commands.json" + }, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "cpplint", + "with": { + "matches": [ + { + "keywords": [ + "runtime" + ], + "score": 5 + }, + { + "keywords": [ + "readability" + ], + "score": 20 + }, + { + "keywords": [ + "build" + ], + "score": 10 + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + }, + { + "name": "[joj] ex2-asan", + "group": "joj", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./h7/build/ex2-asan", + "-a" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": { + "h7/build/ex2": "h7/build/ex2", + "h7/build/ex2-asan": "h7/build/ex2-asan", + "h7/build/ex2-ubsan": "h7/build/ex2-ubsan", + "h7/build/ex2-msan": "h7/build/ex2-msan", + "h7/build/compile_commands.json": "h7/build/compile_commands.json" + }, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "stdin": { + "src": "/home/tt/.config/joj/basic/case0.in" + }, + "cpuLimit": 500000000, + "clockLimit": 1000000000, + "memoryLimit": 5242880, + "procLimit": 50 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/basic/case1.in" + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 5242880, + "procLimit": 50 + } + ] + } + }, + "parsers": [ + { + "name": "diff", + "with": { + "name": "diff", + "cases": [ + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/basic/case0.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/basic/case1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": true, + "showMemory": true + } + } + ] + } + ], + "preStages": [], + "postStages": [ + { + "name": "teapot", + "group": "", + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "ece280-joj", + "--max-total-score", + "100" + ], + "env": [ + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "msg": "joj3 summary" } } ] } ] - }, - "teapot": { - "logPath": "/home/tt/.cache/joint-teapot-debug.log", - "scoreboardPath": "scoreboard.csv", - "failedTablePath": "failed-table.md", - "gradingRepoName": "", - "skipIssue": false, - "skipScoreboard": false, - "skipFailedTable": false } } diff --git a/tests/convert/basic/task.toml b/tests/convert/basic/task.toml index 0872079..46b8744 100644 --- a/tests/convert/basic/task.toml +++ b/tests/convert/basic/task.toml @@ -1,30 +1,102 @@ -task = "hw3 ex5" +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later -[release] -deadline = "2024-10-18T23:59:00+08:00" +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 [[stages]] -name = "judge_base" -command = "./matlab-joj ./h3/ex5.m" -score = 100 -parsers = ["diff", "result-detail"] +name = "Compilation" +command = "./tools/compile" # eg. script running cmake commands +files.import = [ "tools/compile" ] +files.export = [ "h7/build/ex2", "h7/build/ex2-asan", "h7/build/ex2-ubsan", "h7/build/ex2-msan", "h7/build/compile_commands.json" ] +score = 1 -files.import = ["tools/matlab-joj", "tools/matlab_formatter.py"] -files.export = ["output/ex5_results.txt", "output/ex5_logs.txt"] - -result_detail.time = false -result_detail.mem = false -result_detail.stderr = true +# compile parsers +parsers = [ "result-detail", "result-status" ] +result-status.comment = "Congratulations! Your code compiled successfully." +result-detail.exitstatus = true +result-detail.stderr = true +result-detail.time = false +result-detail.mem = false +result-status.force_quit = true [[stages]] -name = "judge_base2" -command = "./matlab-joj ./h3/ex5.m" -score = 80 -parsers = ["diff", "result-detail"] +name = "[cq] Filelength" +command = "./tools/filelength 400 300 *.cpp *.h" +files.import = [ "tools/filelength" ] -files.import = ["tools/matlab-joj", "tools/matlab_formatter.py"] -files.export = ["output/ex5_results2.txt", "output/ex5_logs2.txt"] +parsers = [ "keyword", "result-detail" ] +keyword.keyword = [ "max", "recommended"] +keyword.weight = [ 20, 10 ] +result-detail.exitstatus = true +result-detail.stdout = true +result-detail.time = false +result-detail.mem = false -result_detail.time = true -result_detail.mem = true -result_detail.stderr = false +[[stages]] +name = "[cq] Clang-tidy" +command = "run-clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p h7/build h7/ex2.cpp" +files.import = [ "tests/homework/h7/.clang-tidy", "h7/build/compile_commands.json" ] +limit.stdout = "4m" + +parsers = [ "clangtidy", "result-detail" ] +clangtidy.keyword = [ "codequality-unchecked-malloc-result", "codequality-no-global-variables", "codequality-no-header-guard", "codequality-no-fflush-stdin", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc", "performance", "portability" ] +clangtidy.weight = [ 5, 20, 20, 20, 10, 5, 5, 5, 15, 5, 5, 5, 5, 5, 5, 5, 5, 5] +result-detail.exitstatus = true +result-detail.stdout = true +result-detail.time = false +result-detail.mem = false + +[[stages]] +name = "[cq] Cppcheck" +command = "cppcheck --template='{\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}' --force --enable=all --suppress=missingIncludeSystem --quiet h7/ex2.cpp" +limit.stderr = "8m" + +parsers = [ "keyword", "cppcheck", "clangtidy", "result-detail", "cpplint", "result-status", "file", "dummy", "diff" ] +cppcheck.keyword = ["error", "warning", "portability", "performance", "style"] +cppcheck.weight = [15, 5, 5, 5, 5] +result-detail.exitstatus = true +result-detail.stderr = true +result-detail.time = false +result-detail.mem = false + +[[stages]] +name = "[cq] Cpplint" +command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-runtime/int,-readability/todo,-build/include_subdir,-build/header_guard,-build/include_what_you_use --recursive --exclude=build h7/ex2.cpp" +limit.stdout = "65m" + +parsers = [ "cpplint", "result-detail" ] +cpplint.keyword = [ "runtime", "readability", "build" ] +cpplint.weight = [ 5, 20, 10] +result-detail.exitstatus = true +result-detail.stderr = true +result-detail.time = false +result-detail.mem = false + +[[stages]] +name = "[joj] ex2-asan" +command="./h7/build/ex2-asan -a" +files.import = [ "h7/build/ex2-asan" ] +limit.mem = "128m" + +parsers = [ "diff", "result-detail" ] +result-detail.exitstatus = true +result-detail.stderr = true + +# will be removed as long as the name is fixed +case0.diff.output.score = 5 +case0.limit.cpu = "0.5s" +case0.limit.mem = "5m" +case0.diff.output.ignore_spaces = true +#case0.limit.stdout = 8 +#case0.command = "./h7/build/ex2" +case0.in = "case0.in" + +case1.diff.output.score = 5 +case1.limit.cpu = "1s" +case1.limit.mem = "5m" +case1.diff.output.ignore_spaces = true +#case1.limit.stdout = 8 +#case1.command = "./h7/build/ex2" +case1.in = "case1.in" diff --git a/tests/convert/clang-tidy/repo.toml b/tests/convert/clang-tidy/repo.toml new file mode 100644 index 0000000..2858724 --- /dev/null +++ b/tests/convert/clang-tidy/repo.toml @@ -0,0 +1,2 @@ +force_skip_health_check_on_test = true +force_skip_teapot_on_test = true diff --git a/tests/convert/clang-tidy/task.json b/tests/convert/clang-tidy/task.json new file mode 100644 index 0000000..8603816 --- /dev/null +++ b/tests/convert/clang-tidy/task.json @@ -0,0 +1,140 @@ +{ + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "[cq] Clang-tidy", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "run-clang-tidy-18", + "-header-filter=.*", + "-quiet", + "-load=/usr/local/lib/libcodequality.so", + "-p", + "h7/build", + "h7/ex2.cpp" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 68157440, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tests/homework/h7/.clang-tidy": { + "src": "/home/tt/.config/joj/tests/homework/h7/.clang-tidy" + }, + "h7/build/compile_commands.json": { + "src": "/home/tt/.config/joj/h7/build/compile_commands.json" + } + }, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "clangtidy", + "with": { + "matches": [ + { + "keywords": [ + "codequality-unchecked-malloc-result", + "readability-duplicate-include", + "readability-identifier-naming", + "readability-redundant", + "readability-misplaced-array-index", + "cppcoreguidelines-init-variables", + "bugprone-suspicious-string-compare", + "google-global-names-in-headers", + "clang-diagnostic", + "clang-analyzer", + "misc", + "performance", + "portability" + ], + "score": 5 + }, + { + "keywords": [ + "codequality-no-global-variables", + "codequality-no-header-guard", + "codequality-no-fflush-stdin" + ], + "score": 20 + }, + { + "keywords": [ + "readability-function-size" + ], + "score": 10 + }, + { + "keywords": [ + "readability-misleading-indentation" + ], + "score": 15 + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stdout" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + } + ], + "preStages": [], + "postStages": [] + } +} diff --git a/tests/convert/clang-tidy/task.toml b/tests/convert/clang-tidy/task.toml new file mode 100644 index 0000000..cc4f91b --- /dev/null +++ b/tests/convert/clang-tidy/task.toml @@ -0,0 +1,20 @@ +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later + +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 + +[[stages]] +name = "[cq] Clang-tidy" +command = "run-clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p h7/build h7/ex2.cpp" +files.import = [ "tests/homework/h7/.clang-tidy", "h7/build/compile_commands.json" ] +limit.stdout = "65m" + +parsers = [ "clangtidy", "result-detail" ] +clangtidy.keyword = [ "codequality-unchecked-malloc-result", "codequality-no-global-variables", "codequality-no-header-guard", "codequality-no-fflush-stdin", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc", "performance", "portability" ] +clangtidy.weight = [ 5, 20, 20, 20, 10, 5, 5, 5, 15, 5, 5, 5, 5, 5, 5, 5, 5, 5] +result-detail.exitstatus = true +result-detail.stdout = true +result-detail.time = false +result-detail.mem = false diff --git a/tests/convert/cppcheck/repo.toml b/tests/convert/cppcheck/repo.toml new file mode 100644 index 0000000..2858724 --- /dev/null +++ b/tests/convert/cppcheck/repo.toml @@ -0,0 +1,2 @@ +force_skip_health_check_on_test = true +force_skip_teapot_on_test = true diff --git a/tests/convert/cppcheck/task.json b/tests/convert/cppcheck/task.json new file mode 100644 index 0000000..f6e3f9e --- /dev/null +++ b/tests/convert/cppcheck/task.json @@ -0,0 +1,110 @@ +{ + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "[cq] Cppcheck", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "cppcheck", + "--template={\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}", + "--force", + "--enable=all", + "--suppress=missingIncludeSystem", + "--quiet", + "h7/ex2.cpp" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 68157440, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "cppcheck", + "with": { + "matches": [ + { + "keywords": [ + "error" + ], + "score": 15 + }, + { + "keywords": [ + "warning", + "portability", + "performance", + "style" + ], + "score": 5 + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + } + ], + "preStages": [], + "postStages": [] + } +} diff --git a/tests/convert/cppcheck/task.toml b/tests/convert/cppcheck/task.toml new file mode 100644 index 0000000..50c8aa7 --- /dev/null +++ b/tests/convert/cppcheck/task.toml @@ -0,0 +1,19 @@ +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later + +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 + +[[stages]] +name = "[cq] Cppcheck" +command = "cppcheck --template='{\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}' --force --enable=all --suppress=missingIncludeSystem --quiet h7/ex2.cpp" +limit.stderr = "65m" + +parsers = [ "cppcheck", "result-detail" ] +cppcheck.keyword = ["error", "warning", "portability", "performance", "style"] +cppcheck.weight = [15, 5, 5, 5, 5] +result-detail.exitstatus = true +result-detail.stderr = true +result-detail.time = false +result-detail.mem = false diff --git a/tests/convert/cpplint/repo.toml b/tests/convert/cpplint/repo.toml new file mode 100644 index 0000000..2858724 --- /dev/null +++ b/tests/convert/cpplint/repo.toml @@ -0,0 +1,2 @@ +force_skip_health_check_on_test = true +force_skip_teapot_on_test = true diff --git a/tests/convert/cpplint/task.json b/tests/convert/cpplint/task.json new file mode 100644 index 0000000..142343b --- /dev/null +++ b/tests/convert/cpplint/task.json @@ -0,0 +1,112 @@ +{ + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "[cq] Cpplint", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "cpplint", + "--linelength=120", + "--filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-runtime/int,-readability/todo,-build/include_subdir,-build/header_guard,-build/include_what_you_use", + "--recursive", + "--exclude=build", + "h7/ex2.cpp" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 68157440, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "cpplint", + "with": { + "matches": [ + { + "keywords": [ + "runtime" + ], + "score": 5 + }, + { + "keywords": [ + "readability" + ], + "score": 20 + }, + { + "keywords": [ + "build" + ], + "score": 10 + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + } + ], + "preStages": [], + "postStages": [] + } +} diff --git a/tests/convert/cpplint/task.toml b/tests/convert/cpplint/task.toml new file mode 100644 index 0000000..018413d --- /dev/null +++ b/tests/convert/cpplint/task.toml @@ -0,0 +1,19 @@ +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later + +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 + +[[stages]] +name = "[cq] Cpplint" +command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-runtime/int,-readability/todo,-build/include_subdir,-build/header_guard,-build/include_what_you_use --recursive --exclude=build h7/ex2.cpp" +limit.stdout = "65m" + +parsers = [ "cpplint", "result-detail" ] +cpplint.keyword = [ "runtime", "readability", "build" ] +cpplint.weight = [ 5, 20, 10] +result-detail.exitstatus = true +result-detail.stderr = true +result-detail.time = false +result-detail.mem = false diff --git a/tests/convert/diff/repo.toml b/tests/convert/diff/repo.toml new file mode 100644 index 0000000..2858724 --- /dev/null +++ b/tests/convert/diff/repo.toml @@ -0,0 +1,2 @@ +force_skip_health_check_on_test = true +force_skip_teapot_on_test = true diff --git a/tests/convert/diff/task.json b/tests/convert/diff/task.json new file mode 100644 index 0000000..9636feb --- /dev/null +++ b/tests/convert/diff/task.json @@ -0,0 +1,138 @@ +{ + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "[joj] ex2-asan", + "group": "joj", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./h7/build/ex2-asan", + "-a" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 68157440, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "h7/build/ex2-asan": { + "src": "/home/tt/.config/joj/h7/build/ex2-asan" + } + }, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case0.in" + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 2097152, + "procLimit": 50 + }, + { + "stdin": { + "src": "/home/tt/.config/joj/diff/case1.in" + }, + "cpuLimit": 2000000000, + "clockLimit": 4000000000, + "memoryLimit": 4194304, + "procLimit": 50 + } + ] + } + }, + "parsers": [ + { + "name": "diff", + "with": { + "name": "diff", + "cases": [ + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case0.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + }, + { + "outputs": [ + { + "score": 5, + "fileName": "stdout", + "answerPath": "/home/tt/.config/joj/diff/case1.out", + "forceQuitOnDiff": false, + "alwaysHide": false, + "compareSpace": false + } + ] + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stderr" + ], + "showExitStatus": true, + "showRuntime": true, + "showMemory": true + } + } + ] + } + ], + "preStages": [], + "postStages": [] + } +} diff --git a/tests/convert/diff/task.toml b/tests/convert/diff/task.toml new file mode 100644 index 0000000..bfd4f76 --- /dev/null +++ b/tests/convert/diff/task.toml @@ -0,0 +1,33 @@ +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later + +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 + +[[stages]] +name = "[joj] ex2-asan" +command="./h7/build/ex2-asan -a" +files.import = [ "h7/build/ex2-asan" ] +limit.mem = "65m" + +parsers = [ "diff", "result-detail" ] +result-detail.exitstatus = true +result-detail.stderr = true + +# will be removed as long as the name is fixed +case0.diff.output.score = 5 +case0.limit.cpu = "1s" +case0.limit.mem = "2m" +case0.diff.output.ignore_spaces = true +#case0.limit.stdout = 8 +#case0.command = "./h7/build/ex2" +case0.in = "case0.in" + +case1.diff.output.score = 5 +case1.limit.cpu = "2s" +case1.limit.mem = "4m" +case1.diff.output.ignore_spaces = true +#case1.limit.stdout = 8 +#case1.command = "./h7/build/ex2" +case1.in = "case1.in" diff --git a/tests/convert/keyword/repo.toml b/tests/convert/keyword/repo.toml new file mode 100644 index 0000000..2858724 --- /dev/null +++ b/tests/convert/keyword/repo.toml @@ -0,0 +1,2 @@ +force_skip_health_check_on_test = true +force_skip_teapot_on_test = true diff --git a/tests/convert/keyword/task.json b/tests/convert/keyword/task.json new file mode 100644 index 0000000..6265365 --- /dev/null +++ b/tests/convert/keyword/task.json @@ -0,0 +1,109 @@ +{ + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "[cq] Filelength", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./tools/filelength", + "400", + "300", + "*.cpp", + "*.h" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tools/filelength": { + "src": "/home/tt/.config/joj/tools/filelength" + } + }, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "keyword", + "with": { + "matches": [ + { + "keywords": [ + "max" + ], + "score": 20 + }, + { + "keywords": [ + "recommended" + ], + "score": 10 + } + ] + } + }, + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stdout" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + } + ], + "preStages": [], + "postStages": [] + } +} diff --git a/tests/convert/keyword/task.toml b/tests/convert/keyword/task.toml new file mode 100644 index 0000000..12302b1 --- /dev/null +++ b/tests/convert/keyword/task.toml @@ -0,0 +1,19 @@ +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later + +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 + +[[stages]] +name = "[cq] Filelength" +command = "./tools/filelength 400 300 *.cpp *.h" +files.import = [ "tools/filelength" ] + +parsers = [ "keyword", "result-detail" ] +keyword.keyword = [ "max", "recommended"] +keyword.weight = [ 20, 10 ] +result-detail.exitstatus = true +result-detail.stdout = true +result-detail.time = false +result-detail.mem = false diff --git a/tests/convert/result-detail/repo.toml b/tests/convert/result-detail/repo.toml new file mode 100644 index 0000000..2858724 --- /dev/null +++ b/tests/convert/result-detail/repo.toml @@ -0,0 +1,2 @@ +force_skip_health_check_on_test = true +force_skip_teapot_on_test = true diff --git a/tests/convert/result-detail/task.json b/tests/convert/result-detail/task.json new file mode 100644 index 0000000..a9c2768 --- /dev/null +++ b/tests/convert/result-detail/task.json @@ -0,0 +1,91 @@ +{ + "name": "hw7 ex2", + "logPath": "/home/tt/.cache/joj3/homework/h7/e2.log", + "expireUnixTimestamp": 1735574399, + "effectiveUnixTimestamp": 1735487999, + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "maxTotalScore": 100, + "stage": { + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "stages": [ + { + "name": "[cq] Filelength", + "group": "cq", + "executor": { + "name": "sandbox", + "with": { + "default": { + "args": [ + "./tools/filelength", + "400", + "300", + "*.cpp", + "*.h" + ], + "env": [], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 1000000000, + "clockLimit": 2000000000, + "memoryLimit": 134217728, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": { + "tools/filelength": { + "src": "/home/tt/.config/joj/tools/filelength" + } + }, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "result-detail", + "with": { + "score": 0, + "comment": "", + "showFiles": [ + "stdout", + "stderr" + ], + "showExitStatus": true, + "showRuntime": false, + "showMemory": false + } + } + ] + } + ], + "preStages": [], + "postStages": [] + } +} diff --git a/tests/convert/result-detail/task.toml b/tests/convert/result-detail/task.toml new file mode 100644 index 0000000..f6eb28c --- /dev/null +++ b/tests/convert/result-detail/task.toml @@ -0,0 +1,18 @@ +# general task configuration +task.name = "hw7 ex2" # task name +task.type = "homework/h7/e2" # remove this task type later + +release.end_time = 2024-12-30 23:59:59+08:00 +release.begin_time = 2024-12-29 23:59:59+08:00 + +[[stages]] +name = "[cq] Filelength" +command = "./tools/filelength 400 300 *.cpp *.h" +files.import = [ "tools/filelength" ] + +parsers = [ "result-detail" ] +result-detail.exitstatus = true +result-detail.stdout = true +result-detail.stderr = true +result-detail.time = false +result-detail.mem = false diff --git a/tests/convert/test_convert_cases.py b/tests/convert/test_convert_cases.py index 1a3392a..514395a 100644 --- a/tests/convert/test_convert_cases.py +++ b/tests/convert/test_convert_cases.py @@ -3,3 +3,27 @@ from tests.convert.utils import load_case def test_basic() -> None: load_case("basic") + + +def test_clang_tidy() -> None: + load_case("clang-tidy") + + +def test_cppcheck() -> None: + load_case("cppcheck") + + +def test_cpplint() -> None: + load_case("cpplint") + + +def test_diff() -> None: + load_case("diff") + + +def test_keyword() -> None: + load_case("keyword") + + +def test_result_detail() -> None: + load_case("result-detail") diff --git a/tests/convert/utils.py b/tests/convert/utils.py index 0a458a1..79d6723 100644 --- a/tests/convert/utils.py +++ b/tests/convert/utils.py @@ -1,32 +1,18 @@ import json from pathlib import Path -from typing import Any, Dict, Tuple - -import rtoml from joj3_config_generator.convert import convert -from joj3_config_generator.models import repo, task - - -def read_convert_files( - case_name: str, -) -> Tuple[repo.Config, task.Config, Dict[str, Any]]: - root = Path(__file__).resolve().parent / case_name - repo_toml_path = root / "repo.toml" - repo_toml = repo_toml_path.read_text() if repo_toml_path.exists() else "" - task_toml_path = root / "task.toml" - task_toml = task_toml_path.read_text() if task_toml_path.exists() else "" - result = json.loads((root / "task.json").read_text()) - return ( - repo.Config(root=root, **rtoml.loads(repo_toml)), - task.Config(root=root, **rtoml.loads(task_toml)), - result, - ) +from joj3_config_generator.load import load_joj3_toml def load_case(case_name: str) -> None: - repo, task, expected_result = read_convert_files(case_name) - result = convert(repo, task).model_dump( + root = Path(__file__).resolve().parent + repo_toml_path = root / case_name / "repo.toml" + task_toml_path = root / case_name / "task.toml" + repo_conf, task_conf = load_joj3_toml(root, repo_toml_path, task_toml_path) + result_json_path = root / case_name / "task.json" + expected_result = json.loads(result_json_path.read_text()) + result = convert(repo_conf, task_conf).model_dump( mode="json", by_alias=True, exclude_none=True ) assert result == expected_result diff --git a/tests/convert_joj1/utils.py b/tests/convert_joj1/utils.py index 74aa1b5..5352a84 100644 --- a/tests/convert_joj1/utils.py +++ b/tests/convert_joj1/utils.py @@ -1,23 +1,19 @@ from pathlib import Path -from typing import Any, Dict, Tuple import rtoml -import yaml from joj3_config_generator.convert import convert_joj1 -from joj3_config_generator.models import joj1 - - -def read_convert_joj1_files(case_name: str) -> Tuple[joj1.Config, Dict[str, Any]]: - root = Path(__file__).resolve().parent - task_yaml_path = root / case_name / "task.yaml" - task_yaml = task_yaml_path.read_text() - task_toml_path = root / case_name / "task.toml" - task_toml = task_toml_path.read_text() - return joj1.Config(**yaml.safe_load(task_yaml)), rtoml.loads(task_toml) +from joj3_config_generator.load import load_joj1_yaml def load_case(case_name: str) -> None: - joj1, expected_result = read_convert_joj1_files(case_name) - result = convert_joj1(joj1).model_dump(by_alias=True, exclude_none=True) + root = Path(__file__).resolve().parent + task_yaml_path = root / case_name / "task.yaml" + task_yaml = load_joj1_yaml(task_yaml_path) + task_toml_path = root / case_name / "task.toml" + task_toml = task_toml_path.read_text() + expected_result = rtoml.loads(task_toml) + result = convert_joj1(task_yaml).model_dump( + mode="json", by_alias=True, exclude_none=True + ) assert result == expected_result