feat: joj1 convert init
Some checks failed
build / build (pull_request) Failing after 2s
build / build (push) Failing after 0s

This commit is contained in:
李衍志523370910113 2025-02-15 11:40:21 +01:00
parent 84e6f67f83
commit 80b33324e5
9 changed files with 638 additions and 1389 deletions

View File

@ -1,10 +1,12 @@
import json
import os
from pathlib import Path
from typing import Any, List
import rtoml
from joj3_config_generator.models import joj1, repo, result, task
from joj3_config_generator.processers.joj1 import get_joj1_run_stage
from joj3_config_generator.processers.repo import ( # get_teapotcheck_config,
get_healthcheck_config,
get_teapot_stage,
@ -25,14 +27,14 @@ def convert(repo_conf: repo.Config, task_conf: task.Config) -> result.Config:
result_conf = result.Config(
name=task_conf.task.name,
# TODO: specify the exact folder difference
log_path=f"/home/tt/.cache/joj3/{task_conf.task.type_}.log",
log_path=f"{Path.home()}/.cache/joj3/{task_conf.task.type_}.log",
expire_unix_timestamp=(
int(task_conf.release.deadline.timestamp())
if task_conf.release.deadline
else -1
),
# FIXME: don't hardcode
actor_csv_path="/home/tt/.config/joj/students.csv",
actor_csv_path=f"{Path.home()}/.config/joj/students.csv",
stage=result.Stage(
stages=[],
sandbox_token=repo_conf.sandbox_token,
@ -59,47 +61,13 @@ def convert(repo_conf: repo.Config, task_conf: task.Config) -> result.Config:
return result_conf
# FIXME: LLM generated convert function, only for demostration
def convert_joj1(joj1_conf: joj1.Config) -> task.Config:
stages = []
for language in joj1_conf.languages:
# Here you might want to create a stage for each language
# You can define a command based on language properties
command = f"run {language.language}"
# Assuming we don't have explicit files, we will set empty ones or default behavior
files = task.Files(import_=[], export=[]) # type: ignore
# Score can be derived from the first case or set to a default
score = 0
parsers: List[str] = [] # Define parsers if applicable
if joj1_conf.cases:
score = sum(
case.score for case in joj1_conf.cases
) # Sum scores for all cases
# Creating a stage for each language
stages.append(
task.Stage(
name=language.language,
command=command,
files=files,
score=score,
parsers=parsers,
)
)
# Assuming no deadline is provided in `joj1`, you can set it accordingly
release_deadline = (
None # Placeholder for future implementation if deadlines are defined
)
stages = [get_joj1_run_stage(joj1_conf)]
return task.Config(
task=task.Task(
name=(
joj1_conf.languages[0].language
if joj1_conf.languages
else "Unnamed Task"
name=("Blank Task"),
),
type_="",
), # FIXME: fix this type later
release=task.Release(deadline=release_deadline),
release=task.Release(deadline=None),
stages=stages,
)

View File

@ -3,7 +3,6 @@ import os
from pathlib import Path
from typing import Any, Dict
import inquirer
import rtoml
import typer
import yaml
@ -16,23 +15,6 @@ from joj3_config_generator.utils.logger import logger
app = typer.Typer(add_completion=False)
@app.command()
def create(toml: typer.FileTextWrite) -> None:
"""
Create a new JOJ3 toml config file
"""
logger.info("Creating")
questions = [
inquirer.List(
"size",
message="What size do you need?",
choices=["Jumbo", "Large", "Standard", "Medium", "Small", "Micro"],
),
]
answers = inquirer.prompt(questions)
logger.info(answers)
@app.command()
def convert_joj1(yaml_file: typer.FileText, toml_file: typer.FileTextWrite) -> None:
"""
@ -42,15 +24,22 @@ def convert_joj1(yaml_file: typer.FileText, toml_file: typer.FileTextWrite) -> N
joj1_obj = yaml.safe_load(yaml_file.read())
joj1_model = joj1.Config(**joj1_obj)
task_model = convert_joj1_conf(joj1_model)
result_dict = task_model.model_dump(by_alias=True)
result_dict = task_model.model_dump(by_alias=True, exclude_none=True)
toml_file.write(rtoml.dumps(result_dict))
@app.command()
def convert(root: Path = Path(".")) -> Dict[str, Any]:
"""
Convert given dir of JOJ3 toml config files to JOJ3 json config files
"""
def convert(
root: Path = typer.Option(
Path("."),
"--conf-root",
"-c",
help="This should be consistent with the root of how you run JOJ3",
),
debug: bool = typer.Option(
False, "--debug", "-d", help="Enable debug mode for more verbose output"
),
) -> Dict[str, Any]:
logger.info(f"Converting files in {root.absolute()}")
repo_toml_path = os.path.join(root.absolute(), "basic", "repo.toml")
# TODO: loop through all dirs to find all task.toml
@ -69,16 +58,7 @@ def convert(root: Path = Path(".")) -> Dict[str, Any]:
json.dump(result_dict, result_file, ensure_ascii=False, indent=4)
result_file.write("\n")
# FIXME: change the path to the server
# homework_name = "h8"
# folder_path = f"/mnt/c/Users/Nuvole/Desktop/engr151-joj/home/tt/.config/joj/tests/homework/{homework_name}"
# folder_path = (
# "/mnt/c/Users/Nuvole/Desktop/engr151-joj/home/tt/.config/joj/homework/h8"
# )
# folder_path = "/mnt/c/Users/Nuvole/Desktop/engr151-joj/home/tt/.config/joj/homework/h7"
# for projects
# folder_path = "/mnt/c/Users/Nuvole/Desktop/engr151-joj/home/tt/.config/joj/tests/projects/p3/p3m3"
# folder_path = "/mnt/c/Users/Nuvole/Desktop/engr151-joj/home/tt/.config/joj/projects/p3/p3m1"
# assert os.path.exists(folder_path), f"there exists no {folder_path}"
# distribution on json
# need a get folder path function
# distribute_json(folder_path, repo_obj)
return result_dict

View File

@ -86,8 +86,7 @@ class Stage(BaseModel):
class Config:
extra = "allow"
@root_validator(pre=True)
def gather_cases(cls: Type["Stage"], values: Dict[str, Any]) -> Dict[str, Any]:
def gather_cases(self, values: Dict[str, Any]) -> Dict[str, Any]:
cases = {k: v for k, v in values.items() if k.startswith("case")}
for key in cases:
values.pop(key)
@ -108,5 +107,5 @@ class Task(BaseModel):
class Config(BaseModel):
task: Task
release: Release # Release configuration
release: Release
stages: List[Stage] # list of stage configurations

View File

@ -0,0 +1,50 @@
from typing import List
import humanfriendly
from pytimeparse.timeparse import timeparse
from joj3_config_generator.models import joj1, result, task
def get_joj1_run_stage(joj1_config: joj1.Config) -> task.Stage:
default_cpu = timeparse("1s")
default_mem = humanfriendly.parse_size("32m")
cases_conf = []
for i, case in enumerate(joj1_config.cases):
cases_conf.append(
task.Stage(
score=case.score,
command=case.execute_args if case.execute_args else None,
limit=task.Limit(
cpu=timeparse(case.time) if case.time else default_cpu,
mem=(
humanfriendly.parse_size(case.memory)
if case.memory
else default_mem
),
),
)
)
for i, case in enumerate(joj1_config.cases):
cases_conf[i].in_ = case.input
cases_conf[i].out_ = case.output
run_config = task.Stage(
name="This is the converted joj1 run stage",
group="joj",
parsers=["diff", "result-status"],
score=100,
limit=task.Limit(
cpu=(
timeparse(joj1_config.cases[0].time)
if joj1_config.cases[0].time is not None
else default_cpu
),
mem=(
humanfriendly.parse_size(joj1_config.cases[0].memory)
if joj1_config.cases[0].memory is not None
else default_mem
),
),
cases={f"case{i}": cases_conf[i] for i, case in enumerate(joj1_config.cases)},
)
return run_config

View File

@ -28,7 +28,7 @@ def get_teapot_stage(repo_conf: repo.Config) -> result.StageDetail:
default=result.Cmd(
args=shlex.split(args_),
env=[
"LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
f"LOG_FILE_PATH={Path.home()}/.cache/joint-teapot-debug.log"
], # TODO: fix it according to the task name
),
cases=[],
@ -72,7 +72,7 @@ def get_debug_args(repo_conf: repo.Config) -> str:
args = ""
args = (
args
+ f"/usr/local/bin/joint-teapot joj3-check-env /home/tt/.config/teapot/teapot.env --grading-repo-name {get_grading_repo_name()} --group-config"
+ f"/usr/local/bin/joint-teapot joj3-check-env {Path.home()}/.config/teapot/teapot.env --grading-repo-name {get_grading_repo_name()} --group-config "
)
group_config = ""
for i, name in enumerate(repo_conf.groups.name):
@ -100,14 +100,16 @@ def get_healthcheck_config(repo_conf: repo.Config) -> result.StageDetail:
),
result.OptionalCmd(
args=shlex.split(get_debug_args(repo_conf)),
env=["LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"],
env=[
f"LOG_FILE_PATH={Path.home()}/.cache/joint-teapot-debug.log"
],
),
],
),
),
parsers=[
result.ParserConfig(name="healthcheck", with_={"score": 1}),
result.ParserConfig(name="debug", with_={"score": 1}),
result.ParserConfig(name="debug", with_={"score": 0}),
],
)
return healthcheck_stage
@ -126,6 +128,8 @@ def get_hash(immutable_files: list[str]) -> str: # input should be a list
current_file_path = Path(__file__).resolve()
project_root = current_file_path.parents[2]
file_path = f"{project_root}/tests/immutable_p3-test/"
# default value as hardcoded
# file_path = "{Path.home()}/.cache/immutable"
immutable_hash = []
for i, file in enumerate(immutable_files):
immutable_files[i] = file_path + file.rsplit("/", 1)[-1]

View File

@ -239,6 +239,7 @@ def fix_file(
return conf_stage
# TODO: add the logic of looping through all the files in the conf-root and generated conf.toml accordingly, while also get the path of the json file.
def fix_diff(
task_stage: task.Stage, conf_stage: result.StageDetail, task_conf: task.Config
) -> result.StageDetail:

File diff suppressed because it is too large Load Diff

View File

@ -1,26 +1,10 @@
# general task configuration
task.name = "hw7 ex2" # task name
task.type = "homework/h7/e2"
task.type = "homework/h7/e2" # remove this task type later
release.deadline = 2024-12-30 23:59:59+08:00
release.stages = [ "compile" ]
[[stages]]
name = "healthcheck"
score = 1
# healthcheck parsers
parsers = ["healthcheck", "debug"]
cases0.command = "/usr/local/bin/repo-health-checker -repoSize=100"
case1.command = """/usr/local/bin/joint-teapot
joj3-check-env
/home/tt/.config/teapot/teapot.env"
--grading-repo-name
JOJ3-actions-examples
--group-config
joj=50:24,=100:24"""
case1.env = ["LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"]
[[stages]]
name = "Compilation"
command = "./tools/compile" # eg. script running cmake commands

View File

@ -1,31 +1,25 @@
task = "cc"
[[stages]]
name = "cc"
command = "run cc"
name = "This is the converted joj1 run stage"
group = "joj"
score = 100
parsers = []
parsers = ["diff", "result-status"]
skip = []
[stages.files]
import = []
export = []
[stages.limit]
mem = 4
cpu = 4
stderr = 4
stdout = 4
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.dummy]
comment = ""
score = 0
forcequit = true
forcequit = false
[stages.result-status]
comment = ""
score = 0
forcequit = true
forcequit = false
[stages.keyword]
keyword = []
@ -49,70 +43,566 @@ mem = true
stdout = false
stderr = false
exitstatus = false
[stages.diff.output]
score = 0
ignorespaces = false
hide = false
forcequit = true
[stages.cases]
[[stages]]
name = "c"
command = "run c"
score = 100
[stages.file]
[stages.cases.case0]
command = "-abcd --aaaa bbbb"
in = "case0.in"
out = "case0.out"
score = 10
parsers = []
skip = []
[stages.files]
import = []
export = []
[stages.cases.case0.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.limit]
mem = 4
cpu = 4
stderr = 4
stdout = 4
[stages.dummy]
[stages.cases.case0.dummy]
comment = ""
score = 0
forcequit = true
forcequit = false
[stages.result-status]
[stages.cases.case0.result-status]
comment = ""
score = 0
forcequit = true
forcequit = false
[stages.keyword]
[stages.cases.case0.keyword]
keyword = []
weight = []
[stages.clangtidy]
[stages.cases.case0.clangtidy]
keyword = []
weight = []
[stages.cppcheck]
[stages.cases.case0.cppcheck]
keyword = []
weight = []
[stages.cpplint]
[stages.cases.case0.cpplint]
keyword = []
weight = []
[stages.result-detail]
[stages.cases.case0.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case0.file]
[stages.cases.case0.cases]
[stages.cases.case0.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case1]
in = "case1.in"
out = "case1.out"
score = 10
parsers = []
skip = []
[stages.cases.case1.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case1.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case1.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case1.keyword]
keyword = []
weight = []
[stages.cases.case1.clangtidy]
keyword = []
weight = []
[stages.cases.case1.cppcheck]
keyword = []
weight = []
[stages.cases.case1.cpplint]
keyword = []
weight = []
[stages.cases.case1.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case1.file]
[stages.cases.case1.cases]
[stages.cases.case1.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case2]
in = "case2.in"
out = "case2.out"
score = 10
parsers = []
skip = []
[stages.cases.case2.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case2.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case2.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case2.keyword]
keyword = []
weight = []
[stages.cases.case2.clangtidy]
keyword = []
weight = []
[stages.cases.case2.cppcheck]
keyword = []
weight = []
[stages.cases.case2.cpplint]
keyword = []
weight = []
[stages.cases.case2.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case2.file]
[stages.cases.case2.cases]
[stages.cases.case2.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case3]
in = "case3.in"
out = "case3.out"
score = 10
parsers = []
skip = []
[stages.cases.case3.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case3.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case3.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case3.keyword]
keyword = []
weight = []
[stages.cases.case3.clangtidy]
keyword = []
weight = []
[stages.cases.case3.cppcheck]
keyword = []
weight = []
[stages.cases.case3.cpplint]
keyword = []
weight = []
[stages.cases.case3.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case3.file]
[stages.cases.case3.cases]
[stages.cases.case3.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case4]
in = "case4.in"
out = "case4.out"
score = 10
parsers = []
skip = []
[stages.cases.case4.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case4.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case4.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case4.keyword]
keyword = []
weight = []
[stages.cases.case4.clangtidy]
keyword = []
weight = []
[stages.cases.case4.cppcheck]
keyword = []
weight = []
[stages.cases.case4.cpplint]
keyword = []
weight = []
[stages.cases.case4.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case4.file]
[stages.cases.case4.cases]
[stages.cases.case4.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case5]
in = "case5.in"
out = "case5.out"
score = 10
parsers = []
skip = []
[stages.cases.case5.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case5.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case5.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case5.keyword]
keyword = []
weight = []
[stages.cases.case5.clangtidy]
keyword = []
weight = []
[stages.cases.case5.cppcheck]
keyword = []
weight = []
[stages.cases.case5.cpplint]
keyword = []
weight = []
[stages.cases.case5.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case5.file]
[stages.cases.case5.cases]
[stages.cases.case5.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case6]
in = "case6.in"
out = "case6.out"
score = 10
parsers = []
skip = []
[stages.cases.case6.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case6.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case6.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case6.keyword]
keyword = []
weight = []
[stages.cases.case6.clangtidy]
keyword = []
weight = []
[stages.cases.case6.cppcheck]
keyword = []
weight = []
[stages.cases.case6.cpplint]
keyword = []
weight = []
[stages.cases.case6.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case6.file]
[stages.cases.case6.cases]
[stages.cases.case6.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case7]
in = "case7.in"
out = "case7.out"
score = 10
parsers = []
skip = []
[stages.cases.case7.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case7.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case7.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case7.keyword]
keyword = []
weight = []
[stages.cases.case7.clangtidy]
keyword = []
weight = []
[stages.cases.case7.cppcheck]
keyword = []
weight = []
[stages.cases.case7.cpplint]
keyword = []
weight = []
[stages.cases.case7.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case7.file]
[stages.cases.case7.cases]
[stages.cases.case7.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case8]
in = "case8.in"
out = "case8.out"
score = 10
parsers = []
skip = []
[stages.cases.case8.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case8.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case8.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case8.keyword]
keyword = []
weight = []
[stages.cases.case8.clangtidy]
keyword = []
weight = []
[stages.cases.case8.cppcheck]
keyword = []
weight = []
[stages.cases.case8.cpplint]
keyword = []
weight = []
[stages.cases.case8.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case8.file]
[stages.cases.case8.cases]
[stages.cases.case8.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.cases.case9]
in = "case9.in"
out = "case9.out"
score = 10
parsers = []
skip = []
[stages.cases.case9.limit]
mem = 32000000
cpu = 1
stderr = 800
stdout = 800
[stages.cases.case9.dummy]
comment = ""
score = 0
forcequit = false
[stages.cases.case9.result-status]
comment = ""
score = 0
forcequit = false
[stages.cases.case9.keyword]
keyword = []
weight = []
[stages.cases.case9.clangtidy]
keyword = []
weight = []
[stages.cases.case9.cppcheck]
keyword = []
weight = []
[stages.cases.case9.cpplint]
keyword = []
weight = []
[stages.cases.case9.result-detail]
time = true
mem = true
stdout = false
stderr = false
exitstatus = false
[stages.cases.case9.file]
[stages.cases.case9.cases]
[stages.cases.case9.diff.output]
score = 0
ignorespaces = true
hide = false
forcequit = false
[stages.diff.output]
score = 0
ignorespaces = false
ignorespaces = true
hide = false
forcequit = true
forcequit = false
[stages.cases]
[task]
type = ""
name = "Blank Task"
[release]
deadline = "null"