feat: go test on examples

.drone.yml

@@ -17,6 +17,5 @@ steps:
           - export PATH=$PATH:/usr/local/go/bin
           - go env -w GOPROXY=https://goproxy.cn,direct
           - make
-          - ./examples/success/run.sh
+          - make test
-          - ./examples/compile_error/run.sh
           - cp build/joj3 /home/drone/.local/bin/joj3

README.md

@@ -5,26 +5,36 @@
 To register the sandbox executor, you need to run go-judge before running this program.
 
 ```bash
-$ export CONF_GITEATOKEN="<YOUR_TOKEN>" && make clean && make && ./examples/success/run.sh && ./examples/compile_error/run.sh
+$ make test
-rm -rf ./build/*
+go test -v ./...
-rm -rf *.out
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/dummy [no test files]
-go build -o ./build/joj3 ./cmd/joj3
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors        [no test files]
-++ dirname -- ./examples/success/run.sh
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/dummy  [no test files]
-+ DIRNAME=./examples/success
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/dummy    [no test files]
-+ cd ./examples/success
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers  [no test files]
-+ ./../../build/joj3
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/diff     [no test files]
-+ cat ./joj3_result.json
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox        [no test files]
-[{"Name":"compile","Results":[{"Score":0,"Comment":""}]},{"Name":"run","Results":[{"Score":100,"Comment":"executor status: run time: 2811900 ns, memory: 16658432 bytes"},{"Score":100,"Comment":"executor status: run time: 2578200 ns, memory: 13094912 bytes"}]}]
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/resultstatus     [no test files]
-+ rm -f ./joj3_result.json
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage    [no test files]
-+ cd -
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/dummy [no test files]
-++ dirname -- ./examples/compile_error/run.sh
+?       focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/healthcheck   [no test files]
-+ DIRNAME=./examples/compile_error
+=== RUN   TestMain
-+ cd ./examples/compile_error
+=== RUN   TestMain/success
-+ ./../../build/joj3
+    main_test.go:96: stageResults: [{Name:compile Results:[{Score:0 Comment:}]} {Name:run Results:[{Score:100 Comment:executor status: run time: 1910200 ns, memory: 13529088 bytes} {Score:100 Comment:executor status: run time: 1703000 ns, memory: 15536128 bytes}]}]
-+ cat ./joj3_result.json
+=== RUN   TestMain/compile_error
-[{"Name":"compile","Results":[{"Score":0,"Comment":"Unexpected executor status: Nonzero Exit Status."}]}]
+    main_test.go:96: stageResults: [{Name:compile Results:[{Score:0 Comment:Unexpected executor status: Nonzero Exit Status.}]}]
-+ rm -f ./joj3_result.json
+=== RUN   TestMain/dummy
-+ cd -
+    main_test.go:96: stageResults: [{Name:dummy Results:[{Score:110 Comment:dummy comment + comment from toml conf}]}]
+=== RUN   TestMain/dummy_error
+    main_test.go:96: stageResults: [{Name:dummy Results:[{Score:0 Comment:Unexpected executor status: Nonzero Exit Status.
+        Stderr: dummy negative score: -1}]}]
+--- PASS: TestMain (0.29s)
+    --- PASS: TestMain/success (0.27s)
+    --- PASS: TestMain/compile_error (0.01s)
+    --- PASS: TestMain/dummy (0.01s)
+    --- PASS: TestMain/dummy_error (0.01s)
+PASS
+ok      focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/joj3  0.295s
 ```
 
 ### For developers

cmd/joj3/conf.go

@@ -12,6 +12,7 @@ import (
 type Conf struct {
 	LogLevel   int    `default:"0"`
 	OutputPath string `default:"joj3_result.json"`
+	SkipGitea  bool
 	GiteaUrl   string `default:"https://focs.ji.sjtu.edu.cn/git"`
 	GiteaToken string `default:""`
 	GiteaOwner string `default:"tests"`

cmd/joj3/main.go

@@ -90,6 +90,7 @@ func main() {
 	if err := outputResult(conf, results); err != nil {
 		slog.Error("output result", "error", err)
 	}
+	if !conf.SkipGitea {
 		if err := stage.Submit(
 			conf.GiteaUrl,
 			conf.GiteaToken,
@@ -100,3 +101,4 @@ func main() {
 			slog.Error("submit result", "error", err)
 		}
 	}
+}

cmd/joj3/main_test.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+
+	"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
+)
+
+func compareStageResults(t *testing.T, actual, want []stage.StageResult) {
+	t.Helper()
+	if len(actual) != len(want) {
+		t.Fatalf("len(actual) = %d, want %d", len(actual), len(want))
+	}
+	for i := range actual {
+		if actual[i].Name != want[i].Name {
+			t.Errorf("actual[%d].Name = %s, want = %s", i, actual[i].Name,
+				want[i].Name)
+		}
+		if len(actual[i].Results) != len(want[i].Results) {
+			t.Fatalf("len(actual[%d].Results) = %d, want = %d", i,
+				len(actual[i].Results), len(want[i].Results))
+		}
+		for j := range actual[i].Results {
+			if actual[i].Results[j].Score != want[i].Results[j].Score {
+				t.Errorf("actual[%d].Results[%d].Score = %d, want = %d", i, j,
+					actual[i].Results[j].Score, want[i].Results[j].Score)
+			}
+			r := regexp.MustCompile(want[i].Results[j].Comment)
+			if !r.MatchString(actual[i].Results[j].Comment) {
+				t.Errorf("actual[%d].Results[%d].Comment = %s, want RegExp = %s",
+					i, j, actual[i].Results[j].Comment,
+					want[i].Results[j].Comment)
+			}
+		}
+	}
+}
+
+func TestMain(t *testing.T) {
+	tests := []struct {
+		name string
+		want []stage.StageResult
+	}{
+		{"success", []stage.StageResult{
+			{Name: "compile", Results: []stage.ParserResult{
+				{Score: 0, Comment: ""},
+			}},
+			{Name: "run", Results: []stage.ParserResult{
+				{Score: 100, Comment: "executor status: run time: \\d+ ns, memory: \\d+ bytes"},
+				{Score: 100, Comment: "executor status: run time: \\d+ ns, memory: \\d+ bytes"},
+			}},
+		}},
+		{"compile_error", []stage.StageResult{
+			{Name: "compile", Results: []stage.ParserResult{
+				{Score: 0, Comment: "Unexpected executor status: Nonzero Exit Status\\."},
+			}},
+		}},
+		{"dummy", []stage.StageResult{
+			{Name: "dummy", Results: []stage.ParserResult{
+				{Score: 110, Comment: "dummy comment \\+ comment from toml conf"},
+			}},
+		}},
+		{"dummy_error", []stage.StageResult{
+			{Name: "dummy", Results: []stage.ParserResult{
+				{Score: 0, Comment: "Unexpected executor status: Nonzero Exit Status\\.\\s*Stderr: dummy negative score: -1"},
+			}},
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			origDir, err := os.Getwd()
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = os.Chdir(fmt.Sprintf("../../examples/%s", tt.name))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer func() {
+				err := os.Chdir(origDir)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}()
+			os.Args = []string{"./joj3"}
+			main()
+			outputFile := "joj3_result.json"
+			data, err := os.ReadFile(outputFile)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer os.Remove(outputFile)
+			var stageResults []stage.StageResult
+			err = json.Unmarshal(data, &stageResults)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("stageResults: %+v", stageResults)
+			compareStageResults(t, stageResults, tt.want)
+		})
+	}
+}

examples/compile_error/conf.toml

@@ -1,3 +1,4 @@
+skipGitea = true
 logLevel = 0
 [[stages]]
 name = "compile"

examples/compile_error/run.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-DIRNAME=`dirname -- "$0"`
-# cd to make CopyInCwd work
-cd $DIRNAME
-./../../build/joj3
-cat ./joj3_result.json
-rm -f ./joj3_result.json
-cd -

examples/dummy/conf.toml

@@ -1,3 +1,4 @@
+skipGitea = true
 [[stages]]
 name = "dummy"
 [stages.executor]

examples/dummy/run.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-DIRNAME=`dirname -- "$0"`
-# cd to make CopyInCwd work
-cd $DIRNAME
-./../../build/joj3
-cat ./joj3_result.json
-rm -f ./joj3_result.json
-cd -

examples/dummy_error/conf.toml

@@ -1,3 +1,4 @@
+skipGitea = true
 [[stages]]
 name = "dummy"
 [stages.executor]

examples/dummy_error/run.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-DIRNAME=`dirname -- "$0"`
-# cd to make CopyInCwd work
-cd $DIRNAME
-./../../build/joj3
-cat ./joj3_result.json
-rm -f ./joj3_result.json
-cd -

examples/success/conf.toml

@@ -1,3 +1,4 @@
+skipGitea = true
 [[stages]]
 name = "compile"
 [stages.executor]

examples/success/run.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -xe
-DIRNAME=`dirname -- "$0"`
-# cd to make CopyInCwd work
-cd $DIRNAME
-./../../build/joj3
-cat ./joj3_result.json
-rm -f ./joj3_result.json
-cd -

internal/executors/sandbox/executor.go

@@ -53,13 +53,14 @@ func (e *Sandbox) Run(cmds []stage.Cmd) ([]stage.ExecutorResult, error) {
 }
 
 func (e *Sandbox) Cleanup() error {
-	for _, fileID := range e.cachedMap {
+	for k, fileID := range e.cachedMap {
 		_, err := e.execClient.FileDelete(context.TODO(), &pb.FileID{
 			FileID: fileID,
 		})
 		if err != nil {
 			slog.Error("sandbox cleanup", "error", err)
 		}
+		delete(e.cachedMap, k)
 	}
 	return nil
 }

internal/stage/run.go

@@ -8,7 +8,7 @@ import (
 )
 
 func Run(stages []Stage) []StageResult {
-	var stageResults []StageResult
+	stageResults := []StageResult{}
 	for _, stage := range stages {
 		slog.Debug("stage start", "name", stage.Name)
 		slog.Debug("executor run start", "cmds", stage.ExecutorCmds)