From b59380ad4ea88ce802a43323a59880dd49f3f717 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 23 Mar 2024 19:55:10 -0400 Subject: [PATCH] feat: `go test` on examples --- .drone.yml | 3 +- README.md | 50 +++++++----- cmd/joj3/conf.go | 1 + cmd/joj3/main.go | 18 +++-- cmd/joj3/main_test.go | 105 +++++++++++++++++++++++++ examples/compile_error/conf.toml | 1 + examples/compile_error/run.sh | 10 --- examples/dummy/conf.toml | 1 + examples/dummy/run.sh | 10 --- examples/dummy_error/conf.toml | 1 + examples/dummy_error/run.sh | 10 --- examples/success/conf.toml | 1 + examples/success/run.sh | 10 --- internal/executors/sandbox/executor.go | 3 +- internal/stage/run.go | 2 +- 15 files changed, 154 insertions(+), 72 deletions(-) create mode 100644 cmd/joj3/main_test.go delete mode 100755 examples/compile_error/run.sh delete mode 100755 examples/dummy/run.sh delete mode 100755 examples/dummy_error/run.sh delete mode 100755 examples/success/run.sh diff --git a/.drone.yml b/.drone.yml index 305d121..cc90312 100644 --- a/.drone.yml +++ b/.drone.yml @@ -17,6 +17,5 @@ steps: - export PATH=$PATH:/usr/local/go/bin - go env -w GOPROXY=https://goproxy.cn,direct - make - - ./examples/success/run.sh - - ./examples/compile_error/run.sh + - make test - cp build/joj3 /home/drone/.local/bin/joj3 diff --git a/README.md b/README.md index c39d6a8..ffcd5b6 100644 --- a/README.md +++ b/README.md @@ -5,26 +5,36 @@ To register the sandbox executor, you need to run go-judge before running this program. ```bash -$ export CONF_GITEATOKEN="" && make clean && make && ./examples/success/run.sh && ./examples/compile_error/run.sh -rm -rf ./build/* -rm -rf *.out -go build -o ./build/joj3 ./cmd/joj3 -++ dirname -- ./examples/success/run.sh -+ DIRNAME=./examples/success -+ cd ./examples/success -+ ./../../build/joj3 -+ cat ./joj3_result.json -[{"Name":"compile","Results":[{"Score":0,"Comment":""}]},{"Name":"run","Results":[{"Score":100,"Comment":"executor status: run time: 2811900 ns, memory: 16658432 bytes"},{"Score":100,"Comment":"executor status: run time: 2578200 ns, memory: 13094912 bytes"}]}] -+ rm -f ./joj3_result.json -+ cd - -++ dirname -- ./examples/compile_error/run.sh -+ DIRNAME=./examples/compile_error -+ cd ./examples/compile_error -+ ./../../build/joj3 -+ cat ./joj3_result.json -[{"Name":"compile","Results":[{"Score":0,"Comment":"Unexpected executor status: Nonzero Exit Status."}]}] -+ rm -f ./joj3_result.json -+ cd - +$ make test +go test -v ./... +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/dummy [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/dummy [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/dummy [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/diff [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/resultstatus [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/dummy [no test files] +? focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/healthcheck [no test files] +=== RUN TestMain +=== RUN TestMain/success + main_test.go:96: stageResults: [{Name:compile Results:[{Score:0 Comment:}]} {Name:run Results:[{Score:100 Comment:executor status: run time: 1910200 ns, memory: 13529088 bytes} {Score:100 Comment:executor status: run time: 1703000 ns, memory: 15536128 bytes}]}] +=== RUN TestMain/compile_error + main_test.go:96: stageResults: [{Name:compile Results:[{Score:0 Comment:Unexpected executor status: Nonzero Exit Status.}]}] +=== RUN TestMain/dummy + main_test.go:96: stageResults: [{Name:dummy Results:[{Score:110 Comment:dummy comment + comment from toml conf}]}] +=== RUN TestMain/dummy_error + main_test.go:96: stageResults: [{Name:dummy Results:[{Score:0 Comment:Unexpected executor status: Nonzero Exit Status. + Stderr: dummy negative score: -1}]}] +--- PASS: TestMain (0.29s) + --- PASS: TestMain/success (0.27s) + --- PASS: TestMain/compile_error (0.01s) + --- PASS: TestMain/dummy (0.01s) + --- PASS: TestMain/dummy_error (0.01s) +PASS +ok focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/cmd/joj3 0.295s ``` ### For developers diff --git a/cmd/joj3/conf.go b/cmd/joj3/conf.go index 1b67274..ec576ad 100644 --- a/cmd/joj3/conf.go +++ b/cmd/joj3/conf.go @@ -12,6 +12,7 @@ import ( type Conf struct { LogLevel int `default:"0"` OutputPath string `default:"joj3_result.json"` + SkipGitea bool GiteaUrl string `default:"https://focs.ji.sjtu.edu.cn/git"` GiteaToken string `default:""` GiteaOwner string `default:"tests"` diff --git a/cmd/joj3/main.go b/cmd/joj3/main.go index 06c3eb6..154d8cf 100644 --- a/cmd/joj3/main.go +++ b/cmd/joj3/main.go @@ -90,13 +90,15 @@ func main() { if err := outputResult(conf, results); err != nil { slog.Error("output result", "error", err) } - if err := stage.Submit( - conf.GiteaUrl, - conf.GiteaToken, - conf.GiteaOwner, - conf.GiteaRepo, - results, - ); err != nil { - slog.Error("submit result", "error", err) + if !conf.SkipGitea { + if err := stage.Submit( + conf.GiteaUrl, + conf.GiteaToken, + conf.GiteaOwner, + conf.GiteaRepo, + results, + ); err != nil { + slog.Error("submit result", "error", err) + } } } diff --git a/cmd/joj3/main_test.go b/cmd/joj3/main_test.go new file mode 100644 index 0000000..73b387d --- /dev/null +++ b/cmd/joj3/main_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "regexp" + "testing" + + "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage" +) + +func compareStageResults(t *testing.T, actual, want []stage.StageResult) { + t.Helper() + if len(actual) != len(want) { + t.Fatalf("len(actual) = %d, want %d", len(actual), len(want)) + } + for i := range actual { + if actual[i].Name != want[i].Name { + t.Errorf("actual[%d].Name = %s, want = %s", i, actual[i].Name, + want[i].Name) + } + if len(actual[i].Results) != len(want[i].Results) { + t.Fatalf("len(actual[%d].Results) = %d, want = %d", i, + len(actual[i].Results), len(want[i].Results)) + } + for j := range actual[i].Results { + if actual[i].Results[j].Score != want[i].Results[j].Score { + t.Errorf("actual[%d].Results[%d].Score = %d, want = %d", i, j, + actual[i].Results[j].Score, want[i].Results[j].Score) + } + r := regexp.MustCompile(want[i].Results[j].Comment) + if !r.MatchString(actual[i].Results[j].Comment) { + t.Errorf("actual[%d].Results[%d].Comment = %s, want RegExp = %s", + i, j, actual[i].Results[j].Comment, + want[i].Results[j].Comment) + } + } + } +} + +func TestMain(t *testing.T) { + tests := []struct { + name string + want []stage.StageResult + }{ + {"success", []stage.StageResult{ + {Name: "compile", Results: []stage.ParserResult{ + {Score: 0, Comment: ""}, + }}, + {Name: "run", Results: []stage.ParserResult{ + {Score: 100, Comment: "executor status: run time: \\d+ ns, memory: \\d+ bytes"}, + {Score: 100, Comment: "executor status: run time: \\d+ ns, memory: \\d+ bytes"}, + }}, + }}, + {"compile_error", []stage.StageResult{ + {Name: "compile", Results: []stage.ParserResult{ + {Score: 0, Comment: "Unexpected executor status: Nonzero Exit Status\\."}, + }}, + }}, + {"dummy", []stage.StageResult{ + {Name: "dummy", Results: []stage.ParserResult{ + {Score: 110, Comment: "dummy comment \\+ comment from toml conf"}, + }}, + }}, + {"dummy_error", []stage.StageResult{ + {Name: "dummy", Results: []stage.ParserResult{ + {Score: 0, Comment: "Unexpected executor status: Nonzero Exit Status\\.\\s*Stderr: dummy negative score: -1"}, + }}, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + origDir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + err = os.Chdir(fmt.Sprintf("../../examples/%s", tt.name)) + if err != nil { + t.Fatal(err) + } + defer func() { + err := os.Chdir(origDir) + if err != nil { + t.Fatal(err) + } + }() + os.Args = []string{"./joj3"} + main() + outputFile := "joj3_result.json" + data, err := os.ReadFile(outputFile) + if err != nil { + t.Fatal(err) + } + defer os.Remove(outputFile) + var stageResults []stage.StageResult + err = json.Unmarshal(data, &stageResults) + if err != nil { + t.Fatal(err) + } + t.Logf("stageResults: %+v", stageResults) + compareStageResults(t, stageResults, tt.want) + }) + } +} diff --git a/examples/compile_error/conf.toml b/examples/compile_error/conf.toml index 4d37141..a52ff88 100644 --- a/examples/compile_error/conf.toml +++ b/examples/compile_error/conf.toml @@ -1,3 +1,4 @@ +skipGitea = true logLevel = 0 [[stages]] name = "compile" diff --git a/examples/compile_error/run.sh b/examples/compile_error/run.sh deleted file mode 100755 index 9088452..0000000 --- a/examples/compile_error/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -xe -DIRNAME=`dirname -- "$0"` -# cd to make CopyInCwd work -cd $DIRNAME -./../../build/joj3 -cat ./joj3_result.json -rm -f ./joj3_result.json -cd - diff --git a/examples/dummy/conf.toml b/examples/dummy/conf.toml index 4c55dc5..80b2d72 100644 --- a/examples/dummy/conf.toml +++ b/examples/dummy/conf.toml @@ -1,3 +1,4 @@ +skipGitea = true [[stages]] name = "dummy" [stages.executor] diff --git a/examples/dummy/run.sh b/examples/dummy/run.sh deleted file mode 100755 index 9088452..0000000 --- a/examples/dummy/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -xe -DIRNAME=`dirname -- "$0"` -# cd to make CopyInCwd work -cd $DIRNAME -./../../build/joj3 -cat ./joj3_result.json -rm -f ./joj3_result.json -cd - diff --git a/examples/dummy_error/conf.toml b/examples/dummy_error/conf.toml index 72576e5..c6b8fa2 100644 --- a/examples/dummy_error/conf.toml +++ b/examples/dummy_error/conf.toml @@ -1,3 +1,4 @@ +skipGitea = true [[stages]] name = "dummy" [stages.executor] diff --git a/examples/dummy_error/run.sh b/examples/dummy_error/run.sh deleted file mode 100755 index 9088452..0000000 --- a/examples/dummy_error/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -xe -DIRNAME=`dirname -- "$0"` -# cd to make CopyInCwd work -cd $DIRNAME -./../../build/joj3 -cat ./joj3_result.json -rm -f ./joj3_result.json -cd - diff --git a/examples/success/conf.toml b/examples/success/conf.toml index 0d767da..e91739a 100644 --- a/examples/success/conf.toml +++ b/examples/success/conf.toml @@ -1,3 +1,4 @@ +skipGitea = true [[stages]] name = "compile" [stages.executor] diff --git a/examples/success/run.sh b/examples/success/run.sh deleted file mode 100755 index 9088452..0000000 --- a/examples/success/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -xe -DIRNAME=`dirname -- "$0"` -# cd to make CopyInCwd work -cd $DIRNAME -./../../build/joj3 -cat ./joj3_result.json -rm -f ./joj3_result.json -cd - diff --git a/internal/executors/sandbox/executor.go b/internal/executors/sandbox/executor.go index 72a1daf..188c4b9 100644 --- a/internal/executors/sandbox/executor.go +++ b/internal/executors/sandbox/executor.go @@ -53,13 +53,14 @@ func (e *Sandbox) Run(cmds []stage.Cmd) ([]stage.ExecutorResult, error) { } func (e *Sandbox) Cleanup() error { - for _, fileID := range e.cachedMap { + for k, fileID := range e.cachedMap { _, err := e.execClient.FileDelete(context.TODO(), &pb.FileID{ FileID: fileID, }) if err != nil { slog.Error("sandbox cleanup", "error", err) } + delete(e.cachedMap, k) } return nil } diff --git a/internal/stage/run.go b/internal/stage/run.go index 92cd215..cd751a3 100644 --- a/internal/stage/run.go +++ b/internal/stage/run.go @@ -8,7 +8,7 @@ import ( ) func Run(stages []Stage) []StageResult { - var stageResults []StageResult + stageResults := []StageResult{} for _, stage := range stages { slog.Debug("stage start", "name", stage.Name) slog.Debug("executor run start", "cmds", stage.ExecutorCmds)