feat: more on parsers
parent
645930f551
commit
2ee5e4f329
|
@ -7,8 +7,7 @@ Levels:
|
||||||
- assignment: eg. homework or project
|
- assignment: eg. homework or project
|
||||||
- task: eg. exercise or milestone
|
- task: eg. exercise or milestone
|
||||||
|
|
||||||
A task is composed of *stages* which themselves might be composed of one or more *steps*, eg. for
|
A task is composed of *stages* which are composed of one or more *steps*, eg. in stage "online-judge" each test-case can be viewed as a step
|
||||||
the stage "online-judge" each test-case can be viewed as a step.
|
|
||||||
|
|
||||||
## Background
|
## Background
|
||||||
|
|
||||||
|
@ -44,7 +43,7 @@ Refer to [Introduction to JOJ3](/Introduction-to-JOJ3.md) for more details on JO
|
||||||
|
|
||||||
### TOML configuration format
|
### TOML configuration format
|
||||||
|
|
||||||
All configurations files at human level must be written in TOML format. They will then be parsed to generate long an dcomplete `.json` files that can be understantood by JOJ3.
|
All configurations files at human level must be written in TOML format. They will then be parsed to generate long and complete `.json` files that can be understantood by JOJ3.
|
||||||
Refer to [TOML reference guide](https://toml.io/en/v1.0.0). After writing a configuration file it is recommended to check its validity, eg. [TOML Lint](https://www.toml-lint.com/).
|
Refer to [TOML reference guide](https://toml.io/en/v1.0.0). After writing a configuration file it is recommended to check its validity, eg. [TOML Lint](https://www.toml-lint.com/).
|
||||||
|
|
||||||
Converting the file into JSON format can help better visualize the structure which can be especially helpful when working with arrays of tables.
|
Converting the file into JSON format can help better visualize the structure which can be especially helpful when working with arrays of tables.
|
||||||
|
@ -54,39 +53,35 @@ Converting the file into JSON format can help better visualize the structure whi
|
||||||
The first and most simple file to write is `repo.toml`. The template below can be used as a starter.
|
The first and most simple file to write is `repo.toml`. The template below can be used as a starter.
|
||||||
It contains the part of the configuration that will be used globally for all assignments and tasks.
|
It contains the part of the configuration that will be used globally for all assignments and tasks.
|
||||||
|
|
||||||
- `[repo]` table: general configuration parameters
|
- `teaching_team [array of string]`: TT members' jaccount
|
||||||
- `max_size [float]`: maximum size allowed for a repo in MB
|
- `max_size [float]`: maximum size allowed for a repo in MB
|
||||||
- `owners [array of string]`: TT members' jaccount
|
- `release_tags [array of string]`: list of allowed release tags
|
||||||
- `release_tags [array of string]`: list of allowed release tags
|
|
||||||
- `[files]` table: file configuration parameters
|
`[files]`
|
||||||
- `whitelist.patterns [array of string]`: patterns of files allowed in the repository
|
- `whitelist.patterns [array of string]`: patterns of files allowed in the repository
|
||||||
- `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary
|
- `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary
|
||||||
- `required.files [array of string]`: files that are written by students and must be found in the repository
|
- `required [array of string]`: files that are written by students and must be found in the repository
|
||||||
- `immutable.files [array of string]`: list all files managed by TT that students are forbidden to modify
|
- `immutable [array of string]`: list all files managed by TT that students are forbidden to modify
|
||||||
- `immutable.hash [array of string]`: list all the hash (`sha256sum`) of the immutable files
|
|
||||||
|
|
||||||
**Important:**
|
**Important:**
|
||||||
- it should never be possible to disable health check
|
- it's impossible to disable health check
|
||||||
- make `whitelist.patterns` **very strict** or students will push random files
|
- make `whitelist.patterns` **very strict** or students will push random files
|
||||||
- unless you have no way to set or predict students' filename, do not use `whitelist.file`
|
- unless you have no way to set or predict students' filename, do not use `whitelist.file`
|
||||||
|
- put this `repo.toml` file in the "root directory" containing all the configuration for this
|
||||||
|
repository, eg. `/home/tt/.config/joj/hw` for homework repository
|
||||||
|
|
||||||
<details><summary>Sample repo.toml</summary>
|
<details><summary>Sample repo.toml</summary>
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
# repo section
|
teaching_team = ["mac-wang", "jon-lee", "allen_wr"] # jaccounts
|
||||||
# generic global parameters for the whole repo
|
|
||||||
[repo]
|
max_size = 5 # 5MB repo max size
|
||||||
owners = ["mac-wang", "jon-lee", "allen_wr"] # jaccounts
|
|
||||||
max_size = 5 # repo max size in MB
|
|
||||||
release_tags = ["h1", "h2", "h3"] # list of valid release tags
|
release_tags = ["h1", "h2", "h3"] # list of valid release tags
|
||||||
|
|
||||||
# files section
|
|
||||||
# file related global parameters for the whole repo
|
|
||||||
[files]
|
[files]
|
||||||
|
immutable = [".gitignore", ".gitattributes", ".gitea/workflows/push.yaml"] # readonly files
|
||||||
|
required = [ "Changelog.md", "Readme.md" ] # files that must be found
|
||||||
whitelist.patterns = ["*.cpp", "*.c", "*.m", "*.md", "Makefile", "CMakelist.txt", ".gitea", "messenger.json"] # files/patterns which are not forbidden
|
whitelist.patterns = ["*.cpp", "*.c", "*.m", "*.md", "Makefile", "CMakelist.txt", ".gitea", "messenger.json"] # files/patterns which are not forbidden
|
||||||
immutable.files = [".gitignore", ".gitattributes", ".gitea/workflows/push.yaml"] # readonly files
|
|
||||||
immutable.hash = ["hash1", "hash2", "hash3"] # hash of readonly files
|
|
||||||
required.files = [ "Changelog.md", "Readme.md" ] # files that must be found
|
|
||||||
```
|
```
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
@ -99,24 +94,31 @@ share back with students.
|
||||||
|
|
||||||
### General options
|
### General options
|
||||||
|
|
||||||
- Global setup
|
Global setup:
|
||||||
- `task [string]`: name the task (eg. an exercise or project milestone)
|
- `task [string]`: name the task (eg. an exercise or project milestone)
|
||||||
- `stages [array of string]`: list all stages run for this task
|
- `release.stages [array of string]`: list all stages to run on a release
|
||||||
- `[stage]` table: configuration for `stage` stage
|
- `release.deadline [offset date-time]`: RFC 3339 formatted date-time with offset
|
||||||
- `command [string]`: command to run for `compile` stage
|
|
||||||
- `file.input [array of string]`: list of files to copy to ensure command runs as expected (eg.
|
Each stage is configured using a table. All parameters following a table definition belong to it until
|
||||||
|
the next table is defined.
|
||||||
|
`[stage1]` table: configuration for stage `stage1`
|
||||||
|
- `command [string]`: command to run
|
||||||
|
- `files.import [array of string]`: list of files to import to ensure the command runs as expected (eg.
|
||||||
driver and header files needed for compilation)
|
driver and header files needed for compilation)
|
||||||
|
- `files.export [array of string]`: list of generated files to export to ensure future commands run as expected (eg.
|
||||||
|
binaries needed for online-judge stages)
|
||||||
- `parsers [array of string]`: list of parsers to run on the result of command
|
- `parsers [array of string]`: list of parsers to run on the result of command
|
||||||
- `limit.cpu [int]`: maximum running time for the stage or step in sec
|
- `limit.cpu [int]`: maximum running time for the stage or step in sec
|
||||||
- `limit.mem [int]`: maximum amount of RAM allowed for stage of step in MB
|
- `limit.mem [int]`: maximum amount of RAM allowed for stage of step in MB
|
||||||
|
-
|
||||||
|
|
||||||
### Parsers
|
### Parsers
|
||||||
|
|
||||||
Currently the following parsers are available:
|
Currently the following parsers are available:
|
||||||
- Generic:
|
- Generic:
|
||||||
- `keyword`: catch keywords on any generic text output
|
- `keyword`: catch keywords on any generic text output and can force quit on a match
|
||||||
- `result-detail`: provide basic statistics on memory and CPU usage
|
- `result-detail`: provide basic statistics on memory and CPU usage
|
||||||
- `result-status`: check if the executor exit with status accepted
|
- `result-status`: check if the executor exit with status accepted, if not quit with error status
|
||||||
- `dummy`: output the score and comment in the settings (only use for testing purpose)
|
- `dummy`: output the score and comment in the settings (only use for testing purpose)
|
||||||
- Code quality:
|
- Code quality:
|
||||||
- `clangtidy`: parse clang-tidy output for specified keywords
|
- `clangtidy`: parse clang-tidy output for specified keywords
|
||||||
|
@ -126,6 +128,36 @@ Currently the following parsers are available:
|
||||||
- Online judge
|
- Online judge
|
||||||
- `diff`: difference between the output and a provided file content (commonly used for judge stage)
|
- `diff`: difference between the output and a provided file content (commonly used for judge stage)
|
||||||
|
|
||||||
|
diff
|
||||||
|
- `comment.pass`
|
||||||
|
- `comment.fail`
|
||||||
|
- `score`
|
||||||
|
- `output.hide`
|
||||||
|
- `output.ignorespaces`
|
||||||
|
- `forcequit`
|
||||||
|
|
||||||
|
dummy
|
||||||
|
- `comment`
|
||||||
|
- `score`
|
||||||
|
|
||||||
|
clangtidy cppcheck cpplint
|
||||||
|
- `keyword`
|
||||||
|
- `weight`
|
||||||
|
|
||||||
|
keyword
|
||||||
|
- quitonmatch
|
||||||
|
|
||||||
|
status
|
||||||
|
- comment
|
||||||
|
- score
|
||||||
|
|
||||||
|
details
|
||||||
|
- time
|
||||||
|
- mem
|
||||||
|
- stderr
|
||||||
|
- stdout
|
||||||
|
- exit status
|
||||||
|
|
||||||
Note that parsers can be combined. For instance one might want to show the `diff`, `result-status`, and `result-detail`
|
Note that parsers can be combined. For instance one might want to show the `diff`, `result-status`, and `result-detail`
|
||||||
outputs for the online judge.
|
outputs for the online judge.
|
||||||
|
|
||||||
|
@ -135,34 +167,44 @@ Some parsers can also be further configured.
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
# general task configuration
|
# general task configuration
|
||||||
task="Homework 1 exercise 2" # used for "comment" field in "json" config file
|
task="Homework 1 exercise 2" # task name
|
||||||
stages = [ "compile", "code-quality", "judge-base", "judge-msan" ] # list of stages, can feature a unique stage
|
|
||||||
deadline = 2024-10-12T23:59:00+08:00
|
|
||||||
|
|
||||||
|
release.deadline = 2024-10-12 23:59:00+08:00
|
||||||
|
release.stages = [ "compile" ]
|
||||||
|
|
||||||
[compile]
|
[compile]
|
||||||
command = "cmake"
|
command = "cmake"
|
||||||
copyfiles = [ "main.c", "task.h" ] # files to include with repo code when compiling
|
files.import = [ "main.c", "task.h", "CMakelist.txt" ] # files to include with repo code when compiling
|
||||||
|
files.export = [ "p1", "p1-msan" ]
|
||||||
# limit.cpu = 300 # allow 300s for complex/long compilation
|
# limit.cpu = 300 # allow 300s for complex/long compilation
|
||||||
|
|
||||||
[code-quality]
|
# parsers
|
||||||
filelength.name = "File length check"
|
result-status.comment = "Congratulations! Your code compiled successfully."
|
||||||
filelength.command = "file-length" # command to run
|
dummy.comment = "\n\n### Details\n"
|
||||||
filelength.tests = [ "max", "recommend"] # keywords caught by corresponding JOJ plugin
|
result-details.status = true
|
||||||
filelenght.weights = [ 50, 20 ] # weight of each keyword
|
result-details.stderr = true
|
||||||
filelength.parsers = ["stdout", "stderr"]
|
|
||||||
|
|
||||||
clangtidy.command = "clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p build"
|
[filelength]
|
||||||
clangtidy.tests = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ]
|
name = "File length check"
|
||||||
clangtidy.weights = [100, 100, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5, 8]
|
command = "file-length" # command to run
|
||||||
|
tests = [ "max", "recommend"] # keywords caught by corresponding JOJ plugin
|
||||||
|
weights = [ 50, 20 ] # weight of each keyword
|
||||||
|
parsers
|
||||||
|
|
||||||
cppcheck.command = "cppcheck --enable=all --language=c++ --suppress=*:build* --suppress=missingIncludeSystem ./"
|
[clangtidy]
|
||||||
cppcheck.tests = ["error", "warning", "portability", "performance", "style"]
|
command = "clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p build"
|
||||||
cppcheck.weights = [20, 10, 15, 15, 10]
|
tests = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ]
|
||||||
|
weights = [100, 100, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5, 8]
|
||||||
|
|
||||||
cpplint.command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-readability/todo,-build/include_subdir,-build/header_guard --recursive --exclude=build ."
|
[cppcheck]
|
||||||
cpplint.tests = [ "runtime", "readability", "build" ]
|
command = "cppcheck --template='{\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}' --force --enable=all --quiet ./"
|
||||||
cpplint.weights = [ 10, 20, 15]
|
tests = ["error", "warning", "portability", "performance", "style"]
|
||||||
|
weights = [20, 10, 15, 15, 10]
|
||||||
|
|
||||||
|
[cpplint]
|
||||||
|
command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-readability/todo,-build/include_subdir,-build/header_guard --recursive --exclude=build ."
|
||||||
|
tests = [ "runtime", "readability", "build" ]
|
||||||
|
weights = [ 10, 20, 15]
|
||||||
|
|
||||||
[judge-base]
|
[judge-base]
|
||||||
command="./driver ./mumsh"
|
command="./driver ./mumsh"
|
||||||
|
@ -202,6 +244,7 @@ case5.score = 25
|
||||||
|
|
||||||
case6.score = 25
|
case6.score = 25
|
||||||
case6.size.stderr = 32
|
case6.size.stderr = 32
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
</details>
|
</details>
|
||||||
|
|
Loading…
Reference in New Issue
Block a user