diff --git a/JOJ3-Configuration-Schema.md b/JOJ3-Configuration-Schema.md index 3978937..1ae6f0d 100644 --- a/JOJ3-Configuration-Schema.md +++ b/JOJ3-Configuration-Schema.md @@ -51,16 +51,15 @@ Converting the file into JSON format can help better visualize the structure whi The first and most simple file to write is `repo.toml`. The template below can be used as a starter. It contains the part of the configuration that will be used globally for all assignments and tasks. -`[repo]` table: general configuration parameters -- `max_size [float]`: maximum size allowed for a repo in MB -- `owners [array of string]`: TT members' jaccount - -`[files]` table: file configuration parameters: -- `whitelist.patterns [array of string]`: patterns of files allowed in the repository -- `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary -- `required.files [array of string]`: files that are written by students and must be found in the repository -- `immutable.files [array of string]`: list all files managed by TT that students are forbidden to modify -- `immutable.hash [array of string]`: list all the hash (`sha256sum`) of the immutable files +- `[repo]` table: general configuration parameters + - `max_size [float]`: maximum size allowed for a repo in MB + - `owners [array of string]`: TT members' jaccount +- `[files]` table: file configuration parameters + - `whitelist.patterns [array of string]`: patterns of files allowed in the repository + - `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary + - `required.files [array of string]`: files that are written by students and must be found in the repository + - `immutable.files [array of string]`: list all files managed by TT that students are forbidden to modify + - `immutable.hash [array of string]`: list all the hash (`sha256sum`) of the immutable files **Important:** - it should never be possible to disable health check @@ -89,26 +88,56 @@ required.files = [ "Changelog.md", "Readme.md" ] # files that must be found ## Task level configuration -Basic idea: -- list all stages that are to be run -- provide config for each stage -- for compilation we need the command and an optional list of extra files to "import" -- various code quality tools can be used, but they all feature: - - a command - - a list of tests - - a weight for each test -- joj sets a default score and cpu/mem limits -- test cases can be individually configured if necessary +This configuration file will be used to generate the task level configuration of JOJ3. This file +should therefore clearly describe what stages to run, how to run them, and what information to +share back with students. + +### General options + +- global setup + - `task [string]`: name the task (eg. an exercise or project milestone) + - `stages [array of string]`: list all stages run for this task +- `[stage]` table: configuration for `stage` stage + - `command [string]`: command to run for `compile` stage + - `file.input [array of string]`: list of files to copy to ensure command runs as expected (eg. + driver and header files needed for compilation) + - `parsers [array of string]`: list of parser to run on the result of command. +- `[[stages]] [array of table]`: configuration for a list of similar stages (eg. code quality stages) +- `[[steps]] [array of table]`: configuration for list of similar steps (eg. test cases for the online judge) + +### Parsers + +Currently the following parsers are available: +- Generic: + - `keyword`: catch keywords on any generic text output + - `stat`: provide basic statistics on memory and CPU usage + - `stdout`: simple copy of standard output + - `stderr`: simple copy of standard error output +- Code quality: + - `clangtidy`: parse clang-tidy output for specified keywords + - `cppcheck`: parse cppcheck output for specified keywords + - `cpplint`: parse cpplint output for specified keywords + - `elf`: parse elf output for specified keywords +- Online judge + - `diff`: difference between the output and a provided file content (commonly used for judge stage) + +Note that parsers can be combined. For instance one might want to show the `diff`, `stat`, and `stderr` +outputs for the online judge. + +Parser can also be further configured. + +
Sample task.toml ```toml +# general task configuration task="Homework 1 exercise 2" # used for "comment" field in "json" config file -stages = [ "compile", "code_quality", "joj" ] # list of stages, can feature a unique stage +stages = [ "compile", "code_quality", "judge-base", "judge-msan" ] # list of stages, can feature a unique stage [compile] -command = "cmake" +command = "cmake" copyfiles = [ "main.c", "task.h" ] # files to include with repo code when compiling -# limit.cpu = 600 # allow 600s for complex/long compilation - +# limit.cpu = 600 # allow 600s for complex/long compilation + [[code_quality]] command = "file-length" # command to run tests = [ "max", "recommend"] # keywords caught by corresponding JOJ plugin @@ -129,29 +158,43 @@ command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whites tests = [ "runtime", "readability", "build" ] weights = [ 10, 20, 15] -[judge] -score = 10 # default OJ score for each test case +[judge-base] +driver="./driver ./mumsh" limit.cpu = 10 # default cpu limit (in sec) for each test case limit.mem = 50 # set default mem limit (in MB) for all OJ test cases -driver="./driver ./mumsh" +score = 10 # default OJ score for each test case -[judge-memcheck] -cases = [ 1, 2, 4, ,5,9] # FIXME: wrong format -driver="./driver ./mumsh-memory-check" -limit.mem = 500 +[judge-msan] +driver="./driver ./mumsh-msan" +limit.cpu = 10 # default cpu limit (in sec) for each test case +limit.mem = 500 # set default mem limit (in MB) for all OJ test cases +score = 10 [[test_case]] filename = "case4.in" # filename of a test case with non-default setup +stages = [ "judge-base" ] score = 20 # non-default score limit.cpu = 5 limit.mem = 5 +size.stdout = 8 +size.sterr = 8 [[test_case]] -filename = "case10.in" -score = 30 +filename = "case4.in" +stages = [ "judge-msan" ] +score = 20 +limit.cpu = 5 limit.mem = 100 # limit.cpu is kept as default [[test_case]] -filename = "case11.in" +filename = "case5.in" # filename of a test case with non-default setup +score = 20 # non-default score +limit.mem = 20 +hide = true +[[test_case]] +filename = "case9.in" # filename of a test case with non-default setup +stages = [ "judge-base" ] ``` + +