feat: test with submodules

.drone.yml

@@ -25,6 +25,7 @@ steps:
           CONF_GITEATOKEN:
               from_secret: gitea-token
       commands:
+          - make prepare-test
           - make test
     - name: store
       commands:

.gitmodules

@@ -0,0 +1,20 @@
+[submodule "examples/cpplint/sillycode"]
+	path = examples/cpplint/sillycode
+	url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
+	branch = cpplint/sillycode
+[submodule "examples/compile/success"]
+	path = examples/compile/success
+	url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
+	branch = compile/success
+[submodule "examples/compile/error"]
+	path = examples/compile/error
+	url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
+	branch = compile/error
+[submodule "examples/dummy/success"]
+	path = examples/dummy/success
+	url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
+	branch = dummy/success
+[submodule "examples/dummy/error"]
+	path = examples/dummy/error
+	url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
+	branch = dummy/error

Makefile

@@ -11,5 +11,8 @@ clean:
 	rm -rf $(BUILD_DIR)/*
 	rm -rf *.out
 
+prepare-test:
+	git submodule update --init --remote
+
 test:
 	go test -coverprofile cover.out -v ./...

README.md

@@ -4,9 +4,9 @@
 
 1. Make sure you are in a Unix-like OS (Linux, MacOS). For Windows, use [WSL 2](https://learn.microsoft.com/en-us/windows/wsl/install).
 
-2. Install [Go](https://go.dev/doc/install). Also make sure `make` and `git` are installed and all 3 programs are presented in `$PATH`.
+2. Install [Go](https://go.dev/doc/install). Also, make sure `make` and `git` are installed and all 3 programs are presented in `$PATH`.
 
-3. Enable cgroups v2 for your OS. Check [here](https://stackoverflow.com/a/73376219/13724598). So that you do not need root permission to run `go-judge`.
+3. Enable cgroup v2 for your OS. Check [here](https://stackoverflow.com/a/73376219/13724598). So that you do not need root permission to run `go-judge`.
 
 4. Clone [go-judge](https://github.com/criyle/go-judge).
 ```bash
@@ -20,40 +20,44 @@ $ # make sure you are in go-judge directory
 $ ./tmp/go-judge -enable-grpc -enable-debug -enable-metrics
 ```
 
-6. Check the functions of `joj3` with the `make test`, which should pass all the test cases. The cases used here are in `/examples`.
+6. Pull submodules. It might be slow, so only run it when necessary.
+
 ```bash
 $ # make sure you are in JOJ3 directory
+$ make prepare-test
+```
+
+7. Check the functions of `joj3` with the `make test`, which should pass all the test cases. The cases used here are in `/examples`.
+```bash
 $ make test
 go test -coverprofile cover.out -v ./...
         focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/dummy         coverage: 0.0% of statements
 ?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors        [no test files]
 ?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers  [no test files]
 ?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/healthcheck   [no test files]
-        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox                coverage: 0.0% of statements
         focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/dummy          coverage: 0.0% of statements
+        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/cpplint          coverage: 0.0% of statements
+        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox                coverage: 0.0% of statements
+        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/dummy            coverage: 0.0% of statements
+        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage            coverage: 0.0% of statements
         focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/diff             coverage: 0.0% of statements
         focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/dummy         coverage: 0.0% of statements
-        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage            coverage: 0.0% of statements
-        focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/dummy            coverage: 0.0% of statements
         focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/resultstatus             coverage: 0.0% of statements
 === RUN   TestMain
 === RUN   TestMain/compile/success
-    main_test.go:101: stageResults: [{Name:compile Results:[{Score:0 Comment:}]} {Name:run Results:[{Score:100 Comment:executor status: run time: 1867950 ns, memory: 10813440 bytes} {Score:100 Comment:executor status: run time: 1948947 ns, memory: 10813440 bytes}]}]
 === RUN   TestMain/compile/error
-    main_test.go:101: stageResults: [{Name:compile Results:[{Score:0 Comment:Unexpected executor status: Nonzero Exit Status.}]}]
 === RUN   TestMain/dummy/success
-    main_test.go:101: stageResults: [{Name:dummy Results:[{Score:110 Comment:dummy comment + comment from toml conf}]}]
 === RUN   TestMain/dummy/error
-    main_test.go:101: stageResults: [{Name:dummy Results:[{Score:0 Comment:Unexpected executor status: Nonzero Exit Status.
+=== RUN   TestMain/cpplint/sillycode
-        Stderr: dummy negative score: -1}]}]
+--- PASS: TestMain (0.42s)
---- PASS: TestMain (0.39s)
+    --- PASS: TestMain/compile/success (0.28s)
-    --- PASS: TestMain/compile/success (0.36s)
     --- PASS: TestMain/compile/error (0.01s)
-    --- PASS: TestMain/dummy/success (0.02s)
+    --- PASS: TestMain/dummy/success (0.01s)
     --- PASS: TestMain/dummy/error (0.01s)
+    --- PASS: TestMain/cpplint/sillycode (0.11s)
 PASS
-coverage: 68.5% of statements
+coverage: 72.4% of statements
-ok      focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/joj3  0.403s  coverage: 68.5% of statements
+ok      focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/joj3  0.425s  coverage: 72.4% of statements
 ```
 
 ### For developers
@@ -66,7 +70,7 @@ $ pre-commit install
 pre-commit installed at .git/hooks/pre-commit
 ```
 
-3. You only need to run step 5 and 6 in quick start during development.
+3. You only need to run steps 5 and 7 in the quick start during development. If the test cases need to be updated, step 6 is also needed.
 
 ## Models
 

cmd/joj3/conf.go

@@ -89,7 +89,7 @@ func commitMsgToConf() (conf Conf, err error) {
 		return
 	}
 	msg := commit.Message
-	slog.Info("commit msg to conf", "msg", msg)
+	slog.Debug("commit msg to conf", "msg", msg)
 	// TODO: parse msg to conf name
 	conf = parseConfFile("conf.toml")
 	return

cmd/joj3/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"log/slog"
 	"os"
 
@@ -10,7 +9,6 @@ import (
 	_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers"
 	"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
 
-	"github.com/go-git/go-git/v5"
 	"github.com/jinzhu/copier"
 )
 
@@ -75,14 +73,9 @@ func outputResult(conf Conf, results []stage.StageResult) error {
 func main() {
 	conf, err := commitMsgToConf()
 	if err != nil {
-		// FIXME: just for local testing purpose
-		if errors.Is(err, git.ErrRepositoryNotExists) {
-			conf = parseConfFile("conf.toml")
-		} else {
 		slog.Error("no conf found", "error", err)
 		os.Exit(1)
 	}
-	}
 	setupSlog(conf)
 	stages := generateStages(conf)
 	defer stage.Cleanup()

cmd/joj3/main_test.go

@@ -10,72 +10,70 @@ import (
 	"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
 )
 
-func compareStageResults(t *testing.T, actual, want []stage.StageResult) {
+func compareStageResults(t *testing.T, actual, expected []stage.StageResult, regex bool) {
 	t.Helper()
-	if len(actual) != len(want) {
+	if len(actual) != len(expected) {
-		t.Fatalf("len(actual) = %d, want %d", len(actual), len(want))
+		t.Fatalf("len(actual) = %d, expected %d", len(actual), len(expected))
 	}
 	for i := range actual {
-		if actual[i].Name != want[i].Name {
+		if actual[i].Name != expected[i].Name {
-			t.Errorf("actual[%d].Name = %s, want = %s", i, actual[i].Name,
+			t.Errorf("actual[%d].Name = %s, expected = %s", i, actual[i].Name,
-				want[i].Name)
+				expected[i].Name)
 		}
-		if len(actual[i].Results) != len(want[i].Results) {
+		if len(actual[i].Results) != len(expected[i].Results) {
-			t.Fatalf("len(actual[%d].Results) = %d, want = %d", i,
+			t.Fatalf("len(actual[%d].Results) = %d, expected = %d", i,
-				len(actual[i].Results), len(want[i].Results))
+				len(actual[i].Results), len(expected[i].Results))
 		}
 		for j := range actual[i].Results {
-			if actual[i].Results[j].Score != want[i].Results[j].Score {
+			if actual[i].Results[j].Score != expected[i].Results[j].Score {
-				t.Errorf("actual[%d].Results[%d].Score = %d, want = %d", i, j,
+				t.Errorf("actual[%d].Results[%d].Score = %d, expected = %d", i, j,
-					actual[i].Results[j].Score, want[i].Results[j].Score)
+					actual[i].Results[j].Score, expected[i].Results[j].Score)
 			}
-			r := regexp.MustCompile(want[i].Results[j].Comment)
+			if regex {
+				r := regexp.MustCompile(expected[i].Results[j].Comment)
 				if !r.MatchString(actual[i].Results[j].Comment) {
-				t.Errorf("actual[%d].Results[%d].Comment = %s, want RegExp = %s",
+					t.Errorf("actual[%d].Results[%d].Comment = %s, expected RegExp = %s",
 						i, j, actual[i].Results[j].Comment,
-					want[i].Results[j].Comment)
+						expected[i].Results[j].Comment)
+				}
+			} else if actual[i].Results[j].Comment != expected[i].Results[j].Comment {
+				t.Errorf("actual[%d].Results[%d].Comment = %s, expected = %s", i, j,
+					actual[i].Results[j].Comment, expected[i].Results[j].Comment)
 			}
 		}
 	}
 }
 
+func readStageResults(t *testing.T, path string) []stage.StageResult {
+	t.Helper()
+	file, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	var results []stage.StageResult
+	err = json.NewDecoder(file).Decode(&results)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return results
+}
+
 func TestMain(t *testing.T) {
-	tests := []struct {
+	tests := []string{
-		name string
+		"compile/success",
-		want []stage.StageResult
+		"compile/error",
-	}{
+		"dummy/success",
-		{"compile/success", []stage.StageResult{
+		"dummy/error",
-			{Name: "compile", Results: []stage.ParserResult{
+		"cpplint/sillycode",
-				{Score: 0, Comment: ""},
-			}},
-			{Name: "run", Results: []stage.ParserResult{
-				{Score: 100, Comment: "executor status: run time: \\d+ ns, memory: \\d+ bytes"},
-				{Score: 100, Comment: "executor status: run time: \\d+ ns, memory: \\d+ bytes"},
-			}},
-		}},
-		{"compile/error", []stage.StageResult{
-			{Name: "compile", Results: []stage.ParserResult{
-				{Score: 0, Comment: "Unexpected executor status: Nonzero Exit Status\\."},
-			}},
-		}},
-		{"dummy/success", []stage.StageResult{
-			{Name: "dummy", Results: []stage.ParserResult{
-				{Score: 110, Comment: "dummy comment \\+ comment from toml conf"},
-			}},
-		}},
-		{"dummy/error", []stage.StageResult{
-			{Name: "dummy", Results: []stage.ParserResult{
-				{Score: 0, Comment: "Unexpected executor status: Nonzero Exit Status\\.\\s*Stderr: dummy negative score: -1"},
-			}},
-		}},
 	}
 	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
+		t.Run(tt, func(t *testing.T) {
 			origDir, err := os.Getwd()
 			if err != nil {
 				t.Fatal(err)
 			}
-			err = os.Chdir(fmt.Sprintf("../../examples/%s", tt.name))
+			err = os.Chdir(fmt.Sprintf("../../examples/%s", tt))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -86,20 +84,18 @@ func TestMain(t *testing.T) {
 				}
 			}()
 			os.Args = []string{"./joj3"}
-			main()
 			outputFile := "joj3_result.json"
-			data, err := os.ReadFile(outputFile)
-			if err != nil {
-				t.Fatal(err)
-			}
 			defer os.Remove(outputFile)
-			var stageResults []stage.StageResult
+			main()
-			err = json.Unmarshal(data, &stageResults)
+			stageResults := readStageResults(t, outputFile)
-			if err != nil {
+			regex := true
-				t.Fatal(err)
+			expectedFile := "expected_regex.json"
+			if _, err := os.Stat(expectedFile); os.IsNotExist(err) {
+				regex = false
+				expectedFile = "expected.json"
 			}
-			t.Logf("stageResults: %+v", stageResults)
+			expectedStageResults := readStageResults(t, expectedFile)
-			compareStageResults(t, stageResults, tt.want)
+			compareStageResults(t, stageResults, expectedStageResults, regex)
 		})
 	}
 }

examples/compile/error

@@ -0,0 +1 @@
+Subproject commit dc7be30db1fbfd83ecb78f214cff3f42732a1365

examples/compile/error/conf.toml

@@ -1,61 +0,0 @@
-skipGitea = true
-logLevel = 0
-[[stages]]
-name = "compile"
-[stages.executor]
-name = "sandbox"
-[stages.executor.with.default]
-args = ["g++", "b.cc", "-o", "a"]
-env = ["PATH=/usr/bin:/bin"]
-cpuLimit = 10_000_000_000
-memoryLimit = 104_857_600
-procLimit = 50
-copyInCwd = true
-copyOut = ["stdout", "stderr"]
-copyOutCached = ["a"]
-[stages.executor.with.default.stdin]
-content = ""
-[stages.executor.with.default.stdout]
-name = "stdout"
-max = 4_096
-[stages.executor.with.default.stderr]
-name = "stderr"
-max = 4_096
-[stages.parser]
-name = "result-status"
-[stages.parser.with]
-score = 100
-comment = "compile done"
-[[stages]]
-name = "run"
-[stages.executor]
-name = "sandbox"
-[stages.executor.with.default]
-args = ["./a"]
-env = ["PATH=/usr/bin:/bin"]
-cpuLimit = 1_000_000_000
-memoryLimit = 104_857_600
-procLimit = 50
-copyOut = ["stdout", "stderr"]
-[stages.executor.with.default.stdout]
-name = "stdout"
-max = 4_096
-[stages.executor.with.default.stderr]
-name = "stderr"
-max = 4_096
-[stages.executor.with.default.copyInCached]
-a = "a"
-[[stages.executor.with.cases]]
-[stages.executor.with.cases.stdin]
-src = "./cases/1.in"
-[[stages.executor.with.cases]]
-[stages.executor.with.cases.stdin]
-src = "./cases/2.in"
-[stages.parser]
-name = "diff"
-[[stages.parser.with.cases]]
-score = 100
-stdoutPath = "./cases/1.out"
-[[stages.parser.with.cases]]
-score = 100
-stdoutPath = "./cases/2.out"

examples/compile/success

@@ -0,0 +1 @@
+Subproject commit 21f6df70d483f34a99b50335e0a1fbe50d3a82dd

examples/compile/success/a.cc

@@ -1,6 +0,0 @@
-#include <iostream>
-int main() {
-  int a, b;
-  std::cin >> a >> b;
-  std::cout << a + b << '\n';
-}

examples/compile/success/cases/1.in

@@ -1 +0,0 @@
-1 1

examples/compile/success/cases/1.out

@@ -1 +0,0 @@
-2

examples/compile/success/cases/2.in

@@ -1 +0,0 @@
-1024 2048

examples/compile/success/cases/2.out

@@ -1 +0,0 @@
-3072

examples/compile/success/conf.toml

@@ -1,60 +0,0 @@
-skipGitea = true
-[[stages]]
-name = "compile"
-[stages.executor]
-name = "sandbox"
-[stages.executor.with.default]
-args = ["g++", "a.cc", "-o", "a"]
-env = ["PATH=/usr/bin:/bin"]
-cpuLimit = 10_000_000_000
-memoryLimit = 104_857_600
-procLimit = 50
-copyInCwd = true
-copyOut = ["stdout", "stderr"]
-copyOutCached = ["a"]
-[stages.executor.with.default.stdin]
-content = ""
-[stages.executor.with.default.stdout]
-name = "stdout"
-max = 4_096
-[stages.executor.with.default.stderr]
-name = "stderr"
-max = 4_096
-[stages.parser]
-name = "result-status"
-[stages.parser.with]
-score = 100
-comment = "compile done"
-[[stages]]
-name = "run"
-[stages.executor]
-name = "sandbox"
-[stages.executor.with.default]
-args = ["./a"]
-env = ["PATH=/usr/bin:/bin"]
-cpuLimit = 1_000_000_000
-memoryLimit = 104_857_600
-procLimit = 50
-copyOut = ["stdout", "stderr"]
-[stages.executor.with.default.stdout]
-name = "stdout"
-max = 4_096
-[stages.executor.with.default.stderr]
-name = "stderr"
-max = 4_096
-[stages.executor.with.default.copyInCached]
-a = "a"
-[[stages.executor.with.cases]]
-[stages.executor.with.cases.stdin]
-src = "./cases/1.in"
-[[stages.executor.with.cases]]
-[stages.executor.with.cases.stdin]
-src = "./cases/2.in"
-[stages.parser]
-name = "diff"
-[[stages.parser.with.cases]]
-score = 100
-stdoutPath = "./cases/1.out"
-[[stages.parser.with.cases]]
-score = 100
-stdoutPath = "./cases/2.out"

examples/cpplint/sillycode

@@ -0,0 +1 @@
+Subproject commit e6504575379ef49c1495dc635f7aba36f57dddc8

examples/dummy/error

@@ -0,0 +1 @@
+Subproject commit 3ebebbc913534fb4c4598d3ed313d1bb3ff29020

examples/dummy/error/conf.toml

@@ -1,28 +0,0 @@
-skipGitea = true
-[[stages]]
-name = "dummy"
-[stages.executor]
-name = "sandbox"
-[stages.executor.with.default]
-args = ["./dummy", "--score", "-1"]
-env = ["PATH=/usr/bin:/bin"]
-cpuLimit = 10_000_000_000
-memoryLimit = 104_857_600
-procLimit = 50
-copyInCwd = true
-[stages.executor.with.default.copyIn.dummy]
-src = "./../../../build/dummy"
-copyOut = ["stdout", "stderr"]
-[stages.executor.with.default.stdin]
-content = ""
-[stages.executor.with.default.stdout]
-name = "stdout"
-max = 4_096
-[stages.executor.with.default.stderr]
-name = "stderr"
-max = 4_096
-[stages.parser]
-name = "dummy"
-[stages.parser.with]
-score = 10
-comment = " + comment from toml conf"

examples/dummy/success

@@ -0,0 +1 @@
+Subproject commit 4bca6f0e4b5e263532b8410afecca615e90524e9

examples/dummy/success/conf.toml

@@ -1,28 +0,0 @@
-skipGitea = true
-[[stages]]
-name = "dummy"
-[stages.executor]
-name = "sandbox"
-[stages.executor.with.default]
-args = ["./dummy", "--score", "100"]
-env = ["PATH=/usr/bin:/bin"]
-cpuLimit = 10_000_000_000
-memoryLimit = 104_857_600
-procLimit = 50
-copyInCwd = true
-[stages.executor.with.default.copyIn.dummy]
-src = "./../../../build/dummy"
-copyOut = ["stdout", "stderr"]
-[stages.executor.with.default.stdin]
-content = ""
-[stages.executor.with.default.stdout]
-name = "stdout"
-max = 4_096
-[stages.executor.with.default.stderr]
-name = "stderr"
-max = 4_096
-[stages.parser]
-name = "dummy"
-[stages.parser.with]
-score = 10
-comment = " + comment from toml conf"

internal/stage/model.go

@@ -159,11 +159,11 @@ type Stage struct {
 }
 
 type ParserResult struct {
-	Score   int
+	Score   int    `json:"score"`
-	Comment string
+	Comment string `json:"comment"`
 }
 
 type StageResult struct {
-	Name    string
+	Name    string         `json:"name"`
-	Results []ParserResult
+	Results []ParserResult `json:"results"`
 }