From 2ac4d5f8e944b851b8832183e0b8fcbeb1a14f99 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 15 Jun 2024 04:26:18 -0400 Subject: [PATCH 1/8] chore: update test workflow --- Makefile | 10 +++++++--- scripts/prepare_test_repos.sh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d5ba0a4..cea7f45 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,23 @@ -.PHONY: all clean test +.PHONY: all build clean prepare-test test BUILD_DIR = ./build +TMP_DIR = ./tmp APPS := $(notdir $(wildcard ./cmd/*)) FLAGS := "-s -w" -all: +all: build + +build: $(foreach APP,$(APPS), go build -ldflags=$(FLAGS) -o $(BUILD_DIR)/$(APP) ./cmd/$(APP);) clean: rm -rf $(BUILD_DIR)/* + rm -rf $(TMP_DIR)/* rm -rf *.out prepare-test: git submodule update --init --remote test: - ./scripts/prepare_test_repos.sh + ./scripts/prepare_test_repos.sh $(TMP_DIR) go test -coverprofile cover.out -v ./... diff --git a/scripts/prepare_test_repos.sh b/scripts/prepare_test_repos.sh index fc877b4..de93557 100755 --- a/scripts/prepare_test_repos.sh +++ b/scripts/prepare_test_repos.sh @@ -2,7 +2,7 @@ set -ex declare -A repo_names -tmp_dir="./tmp" +tmp_dir=${1:-./tmp} submodules_dir="$tmp_dir/submodules" rm -rf $submodules_dir mkdir -p $submodules_dir From 60d894f4a8dd9f78d257cd5c34cb86f945047b9d Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Wed, 19 Jun 2024 03:32:37 -0400 Subject: [PATCH 2/8] fix: test submodule reset to remote --- scripts/prepare_test_repos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/prepare_test_repos.sh b/scripts/prepare_test_repos.sh index de93557..21f0558 100755 --- a/scripts/prepare_test_repos.sh +++ b/scripts/prepare_test_repos.sh @@ -24,6 +24,7 @@ for submodule in $submodules; do repo_names[$repo_name]=1 cd $repo_dir git checkout -q $branch + git reset -q --hard origin/$branch cd - submodule_dir="$submodules_dir/$repo_name/$submodule" mkdir -p $submodule_dir From 9445f93e44e3513722aa3b80f4e2562aaf39f909 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 20 Jun 2024 04:18:41 -0400 Subject: [PATCH 3/8] chore: run foreach test repos script --- scripts/run_foreach_test_repos.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 scripts/run_foreach_test_repos.sh diff --git a/scripts/run_foreach_test_repos.sh b/scripts/run_foreach_test_repos.sh new file mode 100755 index 0000000..3c9ce55 --- /dev/null +++ b/scripts/run_foreach_test_repos.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -ex +tmp_dir=${1:-./tmp} +JOJ3=$(git rev-parse --show-toplevel)/build/joj3 +command=${2:-$JOJ3} +submodules_dir="$tmp_dir/submodules" +submodules=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') +for submodule in $submodules; do + url=$(git config --file .gitmodules --get-regexp "submodule.$submodule.url" | awk '{ print $2 }') + repo_name=$(echo $url | rev | cut -d'/' -f 1 | rev | cut -d'.' -f 1) + submodule_dir="$submodules_dir/$repo_name/$submodule" + cd $submodule_dir + eval "$command" + if [[ $command == $JOJ3 ]]; then + if [ -f "./expected.json" ]; then + mv -f "joj3_result.json" "expected.json" + fi + fi + cd - +done From c5d31d9a1a76917e2b5a403feeeb36108f4a0035 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 20 Jun 2024 04:20:14 -0400 Subject: [PATCH 4/8] chore: update submodule --- examples/clangtidy/sillycode | 2 +- examples/compile/error | 2 +- examples/compile/success | 2 +- examples/cppcheck/sillycode | 2 +- examples/cpplint/sillycode | 2 +- examples/dummy/error | 2 +- examples/dummy/success | 2 +- examples/keyword/clangtidy/sillycode | 2 +- examples/keyword/cpplint/sillycode | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/clangtidy/sillycode b/examples/clangtidy/sillycode index 34f1bcd..6084f4d 160000 --- a/examples/clangtidy/sillycode +++ b/examples/clangtidy/sillycode @@ -1 +1 @@ -Subproject commit 34f1bcd9cf3e2905ecfd95284b25eae2568d38df +Subproject commit 6084f4dc39929257f35eb2467b5aa105ceb03e81 diff --git a/examples/compile/error b/examples/compile/error index 9f0d88e..2593e79 160000 --- a/examples/compile/error +++ b/examples/compile/error @@ -1 +1 @@ -Subproject commit 9f0d88eff4e9d7d3ac58f460609272f7d2e527e7 +Subproject commit 2593e79505a93042d308c5fc355dba671dd4fdba diff --git a/examples/compile/success b/examples/compile/success index a900711..638e9f6 160000 --- a/examples/compile/success +++ b/examples/compile/success @@ -1 +1 @@ -Subproject commit a90071148cb7e39cf3e93b18a1e71632d7e974a3 +Subproject commit 638e9f661092d39daaf6e1ffc8ba5998fc56c96a diff --git a/examples/cppcheck/sillycode b/examples/cppcheck/sillycode index d33b07f..40fe2db 160000 --- a/examples/cppcheck/sillycode +++ b/examples/cppcheck/sillycode @@ -1 +1 @@ -Subproject commit d33b07f18bf36c77dcf9a012d265dc7bd45fac9f +Subproject commit 40fe2dbef18ac5188f72fe788426e3d9c8aa88ae diff --git a/examples/cpplint/sillycode b/examples/cpplint/sillycode index 6d47ed4..1ec92f3 160000 --- a/examples/cpplint/sillycode +++ b/examples/cpplint/sillycode @@ -1 +1 @@ -Subproject commit 6d47ed4f6f7b8632784513edd8a206689503fc56 +Subproject commit 1ec92f39ce2fac82356e79ae08c457784769d49c diff --git a/examples/dummy/error b/examples/dummy/error index 9cf0d99..8571a8b 160000 --- a/examples/dummy/error +++ b/examples/dummy/error @@ -1 +1 @@ -Subproject commit 9cf0d994aa613dc38f2bc99f42d0dd7509c3d42c +Subproject commit 8571a8ba30fb2c431d504ed9cfc74d33b9820246 diff --git a/examples/dummy/success b/examples/dummy/success index c2fe51b..a933d18 160000 --- a/examples/dummy/success +++ b/examples/dummy/success @@ -1 +1 @@ -Subproject commit c2fe51b4cd485339fd18924196b01eed6848ba1d +Subproject commit a933d18dda38446d96e36ec5032c6dd48546511b diff --git a/examples/keyword/clangtidy/sillycode b/examples/keyword/clangtidy/sillycode index 9de8a32..9938ef0 160000 --- a/examples/keyword/clangtidy/sillycode +++ b/examples/keyword/clangtidy/sillycode @@ -1 +1 @@ -Subproject commit 9de8a32bc3a55f0652d487421384d0a163bf0aaa +Subproject commit 9938ef006e25c8caea24493172e60a58380c8df4 diff --git a/examples/keyword/cpplint/sillycode b/examples/keyword/cpplint/sillycode index 9be8c3c..96d0234 160000 --- a/examples/keyword/cpplint/sillycode +++ b/examples/keyword/cpplint/sillycode @@ -1 +1 @@ -Subproject commit 9be8c3c329342059f49683a4e810938e8321adbe +Subproject commit 96d02348a20a5a244cd4f82e94a04dccfe3f009c From 2f8604f647f1f6280e5a5d951dd166047b274f4c Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Thu, 20 Jun 2024 05:07:10 -0400 Subject: [PATCH 5/8] chore: rename test names --- cmd/joj3/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/joj3/main_test.go b/cmd/joj3/main_test.go index 22cbb78..0f35e6b 100644 --- a/cmd/joj3/main_test.go +++ b/cmd/joj3/main_test.go @@ -63,7 +63,7 @@ func readStageResults(t *testing.T, path string) []stage.StageResult { func TestMain(t *testing.T) { var tests []string - root := "../../tmp/submodules/JOJ3-examples" + root := "../../tmp/submodules/JOJ3-examples/examples/" err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err From b608a22cfddbb9fc9ee5909f5ee510fa2e4496c9 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sun, 30 Jun 2024 02:42:51 -0400 Subject: [PATCH 6/8] feat: read sandbox conf --- cmd/joj3/conf.go | 8 +++++--- cmd/joj3/main.go | 3 ++- internal/executors/all.go | 7 ++++++- internal/executors/sandbox/meta.go | 10 +++++++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cmd/joj3/conf.go b/cmd/joj3/conf.go index 40dff6b..4f9951f 100644 --- a/cmd/joj3/conf.go +++ b/cmd/joj3/conf.go @@ -10,9 +10,11 @@ import ( ) type Conf struct { - LogLevel int `default:"0"` - OutputPath string `default:"joj3_result.json"` - Stages []struct { + SandboxExecServer string `default:"localhost:5051"` + SandboxToken string `default:""` + LogLevel int `default:"0"` + OutputPath string `default:"joj3_result.json"` + Stages []struct { Name string Executor struct { Name string diff --git a/cmd/joj3/main.go b/cmd/joj3/main.go index e21a52d..b8e0e05 100644 --- a/cmd/joj3/main.go +++ b/cmd/joj3/main.go @@ -5,7 +5,7 @@ import ( "log/slog" "os" - _ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors" + "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors" _ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers" "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage" @@ -77,6 +77,7 @@ func main() { os.Exit(1) } setupSlog(conf) + executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken) stages := generateStages(conf) defer stage.Cleanup() results := stage.Run(stages) diff --git a/internal/executors/all.go b/internal/executors/all.go index bd860e6..76a7f96 100644 --- a/internal/executors/all.go +++ b/internal/executors/all.go @@ -2,8 +2,13 @@ package executors import ( _ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/dummy" - _ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox" + "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox" ) // this file does nothing but imports to ensure all the init() functions // in the subpackages are called + +// overwrite the default registered executors +func InitWithConf(sandboxExecServer, sandboxToken string) { + sandbox.InitWithConf(sandboxExecServer, sandboxToken) +} diff --git a/internal/executors/sandbox/meta.go b/internal/executors/sandbox/meta.go index 2702646..e984184 100644 --- a/internal/executors/sandbox/meta.go +++ b/internal/executors/sandbox/meta.go @@ -8,9 +8,17 @@ var name = "sandbox" func init() { stage.RegisterExecutor(name, &Sandbox{ - // TODO: read from conf execServer: "localhost:5051", token: "", cachedMap: make(map[string]string), }) } + +// overwrite the default registered executor +func InitWithConf(execServer, token string) { + stage.RegisterExecutor(name, &Sandbox{ + execServer: execServer, + token: token, + cachedMap: make(map[string]string), + }) +} From 97f89d210bba38ae7bc445bdda3ee41bf5c19bef Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sun, 30 Jun 2024 07:39:28 -0400 Subject: [PATCH 7/8] chore: update for deployment --- README.md | 2 +- internal/executors/sandbox/executor.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbca578..296d0eb 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ $ go build -o ./tmp/go-judge ./cmd/go-judge 5. Run `go-judge`. ```bash $ # make sure you are in go-judge directory -$ ./tmp/go-judge -enable-grpc -enable-debug -enable-metrics +$ ./tmp/go-judge -http-addr 0.0.0.0:5050 -grpc-addr 0.0.0.0:5051 -monitor-addr 0.0.0.0:5052 -enable-grpc -enable-debug -enable-metrics ``` 6. Pull submodules. It might be slow, so only run it when necessary. diff --git a/internal/executors/sandbox/executor.go b/internal/executors/sandbox/executor.go index 188c4b9..92c0eae 100644 --- a/internal/executors/sandbox/executor.go +++ b/internal/executors/sandbox/executor.go @@ -18,6 +18,7 @@ type Sandbox struct { func (e *Sandbox) Run(cmds []stage.Cmd) ([]stage.ExecutorResult, error) { var err error if e.execClient == nil { + slog.Debug("create exec client", "server", e.execServer) e.execClient, err = createExecClient(e.execServer, e.token) if err != nil { return nil, err From 739069a18f027986f1e05ee67609710bdc20fb89 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sun, 30 Jun 2024 08:45:17 -0400 Subject: [PATCH 8/8] feat: convert local file to memory file --- internal/executors/sandbox/convert.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/executors/sandbox/convert.go b/internal/executors/sandbox/convert.go index e5b786f..827dd15 100644 --- a/internal/executors/sandbox/convert.go +++ b/internal/executors/sandbox/convert.go @@ -1,6 +1,7 @@ package sandbox import ( + "log/slog" "os" "path/filepath" "strings" @@ -113,7 +114,12 @@ func convertPBFile(i stage.CmdFile) *pb.Request_File { i.Src = &absPath } } - return &pb.Request_File{File: &pb.Request_File_Local{Local: &pb.Request_LocalFile{Src: *i.Src}}} + s, err := os.ReadFile(*i.Src) + if err != nil { + s = []byte{} + slog.Error("read file error", "path", *i.Src, "error", err) + } + return &pb.Request_File{File: &pb.Request_File_Memory{Memory: &pb.Request_MemoryFile{Content: s}}} case i.Content != nil: s := strToBytes(*i.Content) return &pb.Request_File{File: &pb.Request_File_Memory{Memory: &pb.Request_MemoryFile{Content: s}}}