From 66e3a40d62ee9f9e6a417b7b770ab916127176fd Mon Sep 17 00:00:00 2001 From: manuel Date: Thu, 17 Oct 2024 00:34:48 +0800 Subject: [PATCH] chore: cleanup --- JOJ3-Configuration-Schema.md | 329 ----------------------------------- 1 file changed, 329 deletions(-) delete mode 100644 JOJ3-Configuration-Schema.md diff --git a/JOJ3-Configuration-Schema.md b/JOJ3-Configuration-Schema.md deleted file mode 100644 index 1ab0f84..0000000 --- a/JOJ3-Configuration-Schema.md +++ /dev/null @@ -1,329 +0,0 @@ -# JOJ3 configuration documentation for TAs - -*Goals:* simple configuration files, which can easily be manually edited by TAs. These files are then parsed by `joj-config-generation` script to generate final internal `json` configurations files used by JOJ3. - -Levels: -- repository: global repository configuration -- assignment: eg. homework or project -- task: eg. exercise or milestone - -A task is composed of *stages* which are composed of one or more *steps*, eg. in stage "online-judge" each test-case can be viewed as a step - -## Background - -Brief introduction to the structure of JOJ configuration and how it impacts students' submissions. - -### Configuration repository - -All configuration is done through a `course-joj` repository. TAs connect to the repository, read the -`Readme.md`, and follow instructions to setup JOJ3 for a course. If previous TAs have already -prepared JOJ3, then only testing is required. Otherwise more setup is required. - -The repository follow the server structure, so that deploying is can done easily using s simple -script. To find JOJ configuration files go to `home/tt/.config/joj`. Each folder should correspond -to a different assignment. This is where `.toml` files will have to be pushed in order to generate -the corresponding JOJ3 `conf.json` files. - - -### Commit messages - -When using JOJ3, Commit messages must follow conventional commits format: `type(scope): message`, where `scope` is used to track the exact task which is being worked on. - -Basic rules: -- health check is **always** run first -- all non JOJ stages are run on each push -- JOJ is only triggered if `joj` appears in `message` - -Scope usage: -- `scope` must match JOJ configuration tree (`home/tt/.config/joj` subtree in `course-joj` repo), eg. if tree is `h1/ex2` the `scope` must be `h1/ex2`, if tree is `hw4/1` then scope is also `hw4/1`. -- if a scope is invalid then return a warning containing the list of valid scopes and exit nicely (either this is a typo, or this an exercise that do not require coding) -- as health check must always be run, `scope` should be checked after it - -Refer to [Introduction to JOJ3](/Introduction-to-JOJ3.md) for more details on JOJ3 usage. - -### TOML configuration format - -All configurations files at human level must be written in TOML format. They will then be parsed to generate long and complete `.json` files that can be understantood by JOJ3. -Refer to [TOML reference guide](https://toml.io/en/v1.0.0). After writing a configuration file it is recommended to check its validity, eg. [TOML Lint](https://www.toml-lint.com/). - -Converting the file into JSON format can help better visualize the structure which can be especially helpful when working with arrays of tables. - -All JOJ3 configuration files can be found in `course-joj` repository under `home/tt/.config/joj`. -For each repository that will be used in the course (eg. `hw`, `p1`, and `p2`), create a -corresponding folder. This directory will be the `joj-root` for that repository, eg. -`home/tt/.config/joj/hw` is the `joj-root` for the `hw` repositories, ie. where JOJ3 configurations -for homework tasks will be setup. - -## Repository level configuration - -The first and most simple file to write is `repo.toml`. The template below can be used as a starter. -It contains the part of the configuration that will be used globally for all assignments and tasks. -It should be saved in the `joj-root` configuration for the repository. - -- `teaching_team [array of string]`: TT members' jaccount -- `max_size [float]`: maximum size allowed for a repo in MB -- `release_tags [array of string]`: list of allowed release tags - -`[files]` -- `whitelist.patterns [array of string]`: patterns of files allowed in the repository -- `whitelist.file [string]`: file containing student defined patterns of files. This option should not be enabled unless strictly necessary -- `required [array of string]`: files that are written by students and must be found in the repository -- `immutable [array of string]`: list all files managed by TT that students are forbidden to modify - -**Important:** -- it's impossible to disable health check -- make `whitelist.patterns` **very strict** or students will push random files -- unless you have no way to set or predict students' filename, do not use `whitelist.file` -- put this `repo.toml` file in the "root directory" containing all the configuration for this - repository, eg. `/home/tt/.config/joj/hw` for homework repository - - -## Task level configuration - -This configuration file will be used to generate the task level configuration of JOJ3. This file -should therefore clearly describe what stages to run, how to run them, and what information to -share back with students. - - -### General options - -Global options: -- `task [string]`: name the task (eg. an exercise or project milestone) -- `release.stages [array of string]`: list all stages to run on a release (default: `[ ]`) -- `release.deadline [offset date-time]`: RFC 3339 formatted date-time with offset -- `limit.cpu [int]`: default maximum running time used for all stages in sec (default: `4`) -- `limit.mem [int]`: default maximum amount of RAM allowed for all stages in MB (default: `4`) - -Each stage is configured in a table. All parameters following a table definition belong to it until -the next table is defined. -`[stagename]` table: configuration for stage `stagename` - - `command [string]`: command to run - - `files.import [array of string]`: list of files to import to ensure the command runs as expected (eg. - driver and header files needed for compilation) - - `files.export [array of string]`: list of generated files to export to ensure future commands run as expected (eg. - binaries needed for online-judge stages) - - `parsers [array of string]`: list of parsers to run on the output of `command` - - `limit.cpu [int]`: maximum running time for the stage in sec (default: `4`) - - `limit.mem [int]`: maximum amount of RAM allowed for stage of step in MB (default: `4`) - - -Any online-judge stage *must* feature the keyword `judge`, eg. `judge`, `judge-base`, `asan-judge` are all valid online-judge -stage names while `oj`, `run-base`, and `asan` are not. - -An online-judge stage is configure as any other stage, ie. using the above options, but can feature an -extra parameter: -- `skip [array of string]`: list of test cases to skip for this stage (default: `[ ]`) - -By default all available test-cases are tested, unless some are marked as `skip`. - - -### Parsers - -Currently the following parsers are available: -- Generic: - - `dummy`: output the score and comment - - `keyword`: catch keywords on any generic text output and can force quit on a match - - `result-detail`: provide basic statistics on memory and CPU usage - - `result-status`: check if the executor exit with status accepted, if not quit with error status -- Code quality: - - `clangtidy`: parse clang-tidy output for specified keywords - - `cppcheck`: parse cppcheck output for specified keywords - - `cpplint`: parse cpplint output for specified keywords - - `elf`: parse elf output for specified keywords -- Online judge - - `diff`: difference between the output and a provided file content (commonly used for judge stage) - -Parsers can be combined. For instance one might want to show the `diff`, `result-status`, and `result-detail` -outputs for the online judge. Some parsers can also be further configured. - - -#### Dummy parser options - -- `comment [string]`: display any text -- `score [int]`: score for passing dummy stage (default: `0`) - -*Note.* Adding a dummy stage can be useful to add extra comments or separate parsers output. - -#### Keyword parser - -- `forcequit [boolean]`: quit at the end of this stage if there is a match -- `keyword [array of string]`: list of keywords to catch on `stdout` -- `weight [array of int]`: corresponding weight for each keyword caught on `stdout` - -*Note.* This parser can be used to catch words on the output of any code quality tool which -doesn't already have a parser implement in JOJ3 - -#### Result-detail parser - -- `exitstatus [boolean]`: display exit status (default: `false`) -- `mem [boolean]`: display the memory usage (default: `true`) -- `stderr [boolean]`: display `stderr` messages (default: `false`) -- `stdout [boolean]`: display `stdout` messages (default: `false`) -- `time [boolean]`: display run time (default: `true`) - - -#### Result-status parser - -- `comment [string]`: comment to display on successful run -- `score [int]`: score to assign on a successful run (default: `0`) - -*Note.* The main benefit of this parser is that it quits on failure. It could be used to exit at -the end of a failed compilation stage. - - -#### Clangtidy, cppcheck, and cpplint parsers - -- `keyword [array of string]`: list of keywords to catch on `stdout` -- `weight [array of int]`: corresponding weight for each keyword caught on `stdout` - -*Warning.* Current implementation of cpplint parser is incomplete, so weight and keywords might be -ignored; for now cpplint parser catches all cpplint checks and apply scoring automatically. - -#### elf parser - -Not ready yet. - -#### Diff parser options - -- `comment.pass [string]`: comment to display when passing a test case (default: `"🥳Passed!"`) -- `comment.fail [string]`: comment to display when failing a test case (default: `"🧐Failed..."`) -- `forcequit [boolean]`: quit at the end of the stage if a test case fails (default: `false`) -- `output.hide [boolean]`: hide diff output (default: `false`) -- `output.ignorespaces [boolean]`: ignore white spaces in diff output (default: `true`) -- `score [int]`: score awarded for passing the test case (default: `0`) - -*Note.* This parser can be configured and adjusted for each test-case (others parsers are configured at stage level) - - -## Examples - - -Sample repository configuration file stored in the `joj-root` configuration for the -repository. - -
Sample repo.toml - -```toml -teaching_team = ["mac-wang", "jon-lee", "allen_wr"] # jaccounts - -maxsize = 5 # 5MB repo max size -releasetags = ["h1", "h2", "h3"] # list of valid release tags - -[files] -immutable = [".gitignore", ".gitattributes", ".gitea/workflows/push.yaml"] # readonly files -required = [ "Changelog.md", "Readme.md" ] # files that must be found -whitelist.patterns = ["*.cpp", "*.c", "*.m", "*.md", "Makefile", "CMakelist.txt", ".gitea", "messenger.json"] # files/patterns which are not forbidden -``` - -
- -Sample basic task configuration for MATLAB where most default options are used. - -
Basic sample task.toml - -```toml -task="hw3 ex5" - -release.deadline = 2024-10-18 23:59:00+08:00 - -[judge-base] -command="./matlab-joj ./h3/ex5.m" -files.import = [ "tools/matlab-joj", "tools/matlab_formatter.py" ] -score = 100 - -parsers = ["diff", "result-detail"] -result-detail.time = false -result-detail.mem = false -result-detail.error = true -``` - -
- - -Sample advanced task configuration for C where many default options are overwritten. - -
Advanced sample task.toml - -```toml -# general task configuration -task="Homework 1 exercise 2" # task name - -release.deadline = 2024-10-12 23:59:00+08:00 -release.stages = [ "compile" ] - -[compile] -command = "make.sh" # eg. script running cmake commands -files.import = [ "src/main.c", "src/task.h", "srcCMakelist.txt" ] # -files.export = [ "p1", "p1-msan" ] -# limit.cpu = 300 # allow 300s for complex/long compilation - -# parsers -result-status.comment = "Congratulations! Your code compiled successfully." -dummy.comment = "\n\n### Details\n" -result-details.status = true -result-details.stderr = true - -[filelength] -name = "File length check" -command = "file-length" # command to run -tests = [ "max", "recommend"] # keywords caught by corresponding JOJ plugin -weights = [ 50, 20 ] # weight of each keyword -parsers - -[clangtidy] -command = "run-clang-tidy-18 -header-filter=.* -quiet -load=/usr/local/lib/libcodequality.so -p build" -tests = [ "codequality-no-global-variables", "codequality-no-header-guard", "readability-function-size", "readability-duplicate-include", "readability-identifier-naming", "readability-redundant", "readability-misleading-indentation", "readability-misplaced-array-index", "cppcoreguidelines-init-variables", "bugprone-suspicious-string-compare", "google-global-names-in-headers", "clang-diagnostic", "clang-analyzer", "misc performance" ] -weights = [100, 100, 50, 10, 5, 5, 10, 5, 5, 8, 5, 5, 5, 5, 8] - -[cppcheck] -command = "cppcheck --template='{\"file\":\"{file}\",\"line\":{line}, \"column\":{column}, \"severity\":\"{severity}\", \"message\":\"{message}\", \"id\":\"{id}\"}' --force --enable=all --quiet ./" -tests = ["error", "warning", "portability", "performance", "style"] -weights = [20, 10, 15, 15, 10] - -[cpplint] -command = "cpplint --linelength=120 --filter=-legal,-readability/casting,-whitespace,-runtime/printf,-runtime/threadsafe_fn,-readability/todo,-build/include_subdir,-build/header_guard --recursive --exclude=build ." -tests = [ "runtime", "readability", "build" ] -weights = [ 10, 20, 15] - -[judge-base] -command="./driver ./mumsh" -limit.cpu = 10 # default cpu limit (in sec) for each test case -limit.mem = 50 # set default mem limit (in MB) for all OJ test cases -score = 10 # default OJ score for each test case -parsers = ["diff", "stderr"] -size.diff = 4 -size.stderr = 4 -diff.ignorespaces = true - -case4.score = 15 -case4.limit.cpu = 30 -case4.limit.mem = 10 - -case5.score = 25 - -case6.score = 25 -case6.size.diff = 64 - -case8.size.stderr = 128 - -[judge-msan] -command="./driver ./mumsh-msan" -limit.cpu = 10 # default cpu limit (in sec) for each test case -limit.mem = 500 # set default mem limit (in MB) for all OJ test cases -score = 10 -parsers = ["diff"] -skip = ["case0", "case11"] -diff.ignorespaces = false - -case4.score = 15 -case4.limit.cpu = 30 -case4.limit.mem = 10 - -case5.score = 25 - -case6.score = 25 -case6.size.stderr = 32 - -``` - -