From 2ee5e4f329721fd9de5cccfc19114598e2e907ea Mon Sep 17 00:00:00 2001 From: manuel Date: Wed, 16 Oct 2024 01:12:58 +0800 Subject: [PATCH] feat: more on parsers --- JOJ3-Configuration-Schema.md | 145 +++++++++++++++++++++++------------ 1 file changed, 94 insertions(+), 51 deletions(-) diff --git a/JOJ3-Configuration-Schema.md b/JOJ3-Configuration-Schema.md index bac3306..4fefa16 100644 --- a/JOJ3-Configuration-Schema.md +++ b/JOJ3-Configuration-Schema.md @@ -7,8 +7,7 @@ Levels: - assignment: eg. homework or project - task: eg. exercise or milestone -A task is composed of *stages* which themselves might be composed of one or more *steps*, eg. for -the stage "online-judge" each test-case can be viewed as a step. +A task is composed of *stages* which are composed of one or more *steps*, eg. in stage "online-judge" each test-case can be viewed as a step ## Background @@ -44,7 +43,7 @@ Refer to [Introduction to JOJ3](/Introduction-to-JOJ3.md) for more details on JO ### TOML configuration format -All configurations files at human level must be written in TOML format. They will then be parsed to generate long an dcomplete `.json` files that can be understantood by JOJ3. +All configurations files at human level must be written in TOML format. They will then be parsed to generate long and complete `.json` files that can be understantood by JOJ3. Refer to [TOML reference guide](https://toml.io/en/v1.0.0). After writing a configuration file it is recommended to check its validity, eg. [TOML Lint](https://www.toml-lint.com/). Converting the file into JSON format can help better visualize the structure which can be especially helpful when working with arrays of tables. @@ -54,39 +53,35 @@ Converting the file into JSON format can help better visualize the structure whi The first and most simple file to write is `repo.toml`. The template below can be used as a starter. It contains the part of the configuration that will be used globally for all assignments and tasks. -- `[repo]` table: general configuration parameters - - `max_size [float]`: maximum size allowed for a repo in MB - - `owners [array of string]`: TT members' jaccount - - `release_tags [array of string]`: list of allowed release tags -- `[files]` table: file configuration parameters - - `whitelist.patterns [array of string]`: patterns of files allowed in the repository - - `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary - - `required.files [array of string]`: files that are written by students and must be found in the repository - - `immutable.files [array of string]`: list all files managed by TT that students are forbidden to modify - - `immutable.hash [array of string]`: list all the hash (`sha256sum`) of the immutable files +- `teaching_team [array of string]`: TT members' jaccount +- `max_size [float]`: maximum size allowed for a repo in MB +- `release_tags [array of string]`: list of allowed release tags + +`[files]` +- `whitelist.patterns [array of string]`: patterns of files allowed in the repository +- `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary +- `required [array of string]`: files that are written by students and must be found in the repository +- `immutable [array of string]`: list all files managed by TT that students are forbidden to modify **Important:** -- it should never be possible to disable health check +- it's impossible to disable health check - make `whitelist.patterns` **very strict** or students will push random files - unless you have no way to set or predict students' filename, do not use `whitelist.file` +- put this `repo.toml` file in the "root directory" containing all the configuration for this + repository, eg. `/home/tt/.config/joj/hw` for homework repository
Sample repo.toml ```toml -# repo section -# generic global parameters for the whole repo -[repo] -owners = ["mac-wang", "jon-lee", "allen_wr"] # jaccounts -max_size = 5 # repo max size in MB +teaching_team = ["mac-wang", "jon-lee", "allen_wr"] # jaccounts + +max_size = 5 # 5MB repo max size release_tags = ["h1", "h2", "h3"] # list of valid release tags -# files section -# file related global parameters for the whole repo [files] +immutable = [".gitignore", ".gitattributes", ".gitea/workflows/push.yaml"] # readonly files +required = [ "Changelog.md", "Readme.md" ] # files that must be found whitelist.patterns = ["*.cpp", "*.c", "*.m", "*.md", "Makefile", "CMakelist.txt", ".gitea", "messenger.json"] # files/patterns which are not forbidden -immutable.files = [".gitignore", ".gitattributes", ".gitea/workflows/push.yaml"] # readonly files -immutable.hash = ["hash1", "hash2", "hash3"] # hash of readonly files -required.files = [ "Changelog.md", "Readme.md" ] # files that must be found ```
@@ -99,24 +94,31 @@ share back with students. ### General options -- Global setup - - `task [string]`: name the task (eg. an exercise or project milestone) - - `stages [array of string]`: list all stages run for this task -- `[stage]` table: configuration for `stage` stage - - `command [string]`: command to run for `compile` stage - - `file.input [array of string]`: list of files to copy to ensure command runs as expected (eg. +Global setup: +- `task [string]`: name the task (eg. an exercise or project milestone) +- `release.stages [array of string]`: list all stages to run on a release +- `release.deadline [offset date-time]`: RFC 3339 formatted date-time with offset + +Each stage is configured using a table. All parameters following a table definition belong to it until +the next table is defined. +`[stage1]` table: configuration for stage `stage1` + - `command [string]`: command to run + - `files.import [array of string]`: list of files to import to ensure the command runs as expected (eg. driver and header files needed for compilation) + - `files.export [array of string]`: list of generated files to export to ensure future commands run as expected (eg. + binaries needed for online-judge stages) - `parsers [array of string]`: list of parsers to run on the result of command - `limit.cpu [int]`: maximum running time for the stage or step in sec - `limit.mem [int]`: maximum amount of RAM allowed for stage of step in MB + - ### Parsers Currently the following parsers are available: - Generic: - - `keyword`: catch keywords on any generic text output + - `keyword`: catch keywords on any generic text output and can force quit on a match - `result-detail`: provide basic statistics on memory and CPU usage - - `result-status`: check if the executor exit with status accepted + - `result-status`: check if the executor exit with status accepted, if not quit with error status - `dummy`: output the score and comment in the settings (only use for testing purpose) - Code quality: - `clangtidy`: parse clang-tidy output for specified keywords @@ -126,6 +128,36 @@ Currently the following parsers are available: - Online judge - `diff`: difference between the output and a provided file content (commonly used for judge stage) +diff +- `comment.pass` +- `comment.fail` +- `score` +- `output.hide` +- `output.ignorespaces` +- `forcequit` + +dummy +- `comment` +- `score` + +clangtidy cppcheck cpplint +- `keyword` +- `weight` + +keyword +- quitonmatch + +status +- comment +- score + +details +- time +- mem +- stderr +- stdout +- exit status + Note that parsers can be combined. For instance one might want to show the `diff`, `result-status`, and `result-detail` outputs for the online judge. @@ -135,34 +167,44 @@ Some parsers can also be further configured. ```toml # general task configuration -task="Homework 1 exercise 2" # used for "comment" field in "json" config file -stages = [ "compile", "code-quality", "judge-base", "judge-msan" ] # list of stages, can feature a unique stage -deadline = 2024-10-12T23:59:00+08:00 +task="Homework 1 exercise 2" # task name +release.deadline = 2024-10-12 23:59:00+08:00 +release.stages = [ "compile" ] [compile] command = "cmake" -copyfiles = [ "main.c", "task.h" ] # files to include with repo code when compiling +files.import = [ "main.c", "task.h", "CMakelist.txt" ] # files to include with repo code when compiling +files.export = [ "p1", "p1-msan" ] # limit.cpu = 300 # allow 300s for complex/long compilation -[code-quality] -filelength.name = "File length check" -filelength.command = "file-length" # command to run -filelength.tests = [ "max", "recommend"] # keywords caught by corresponding JOJ plugin -filelenght.weights = [ 50, 20 ] # weight of each keyword -filelength.parsers = ["stdout", "stderr"] +# parsers +result-status.comment = "Congratulations! Your code compiled successfully." +dummy.comment = "\n\n### Details\n" +result-details.status = true +result-details.stderr = true -clangtidy.command = "clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p build" -clangtidy.tests = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ] -clangtidy.weights = [100, 100, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5, 8] +[filelength] +name = "File length check" +command = "file-length" # command to run +tests = [ "max", "recommend"] # keywords caught by corresponding JOJ plugin +weights = [ 50, 20 ] # weight of each keyword +parsers -cppcheck.command = "cppcheck --enable=all --language=c++ --suppress=*:build* --suppress=missingIncludeSystem ./" -cppcheck.tests = ["error", "warning", "portability", "performance", "style"] -cppcheck.weights = [20, 10, 15, 15, 10] +[clangtidy] +command = "clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p build" +tests = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ] +weights = [100, 100, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5, 8] -cpplint.command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-readability/todo,-build/include_subdir,-build/header_guard --recursive --exclude=build ." -cpplint.tests = [ "runtime", "readability", "build" ] -cpplint.weights = [ 10, 20, 15] +[cppcheck] +command = "cppcheck --template='{\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}' --force --enable=all --quiet ./" +tests = ["error", "warning", "portability", "performance", "style"] +weights = [20, 10, 15, 15, 10] + +[cpplint] +command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-readability/todo,-build/include_subdir,-build/header_guard --recursive --exclude=build ." +tests = [ "runtime", "readability", "build" ] +weights = [ 10, 20, 15] [judge-base] command="./driver ./mumsh" @@ -202,6 +244,7 @@ case5.score = 25 case6.score = 25 case6.size.stderr = 32 + ```