From 2d4c7c30a31c1fa208fd4fb6ab26a47c9dfd9817 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 13:21:58 +0800 Subject: [PATCH 01/13] refactor(cmd/joj3): conf (#51) Reviewed-on: https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3/pulls/51 Co-authored-by: Boming Zhang Co-committed-by: Boming Zhang --- Makefile | 2 +- cmd/joj3/conf/conf.go | 57 +++++++++++++++++++++++++- cmd/joj3/stage/main.go | 27 +++++++++---- cmd/joj3/teapot/main.go | 90 +++++++++++++++++++++++------------------ internal/stage/model.go | 17 +++++--- internal/stage/run.go | 69 +++++++++++++++++++------------ 6 files changed, 180 insertions(+), 82 deletions(-) diff --git a/Makefile b/Makefile index 70840d3..67c1186 100644 --- a/Makefile +++ b/Makefile @@ -30,4 +30,4 @@ test: ci-test: ./scripts/prepare_test_repos.sh $(TMP_DIR) ./scripts/run_foreach_test_repos.sh $(TMP_DIR) "sed -i '2i \ \ \"sandboxExecServer\": \"172.17.0.1:5051\",' conf.json" - go test -coverprofile cover.out -v ./... + GITHUB_ACTIONS="test" go test -coverprofile cover.out -v ./... diff --git a/cmd/joj3/conf/conf.go b/cmd/joj3/conf/conf.go index 944e62c..403aa6c 100644 --- a/cmd/joj3/conf/conf.go +++ b/cmd/joj3/conf/conf.go @@ -13,16 +13,48 @@ import ( "github.com/koding/multiconfig" ) +type ConfStage struct { + Name string + Group string + Executor struct { + Name string + With struct { + Default stage.Cmd + Cases []OptionalCmd + } + } + Parsers []struct { + Name string + With interface{} + } +} + type Conf struct { + Name string `default:"unknown"` + LogPath string `default:""` + Stage struct { + SandboxExecServer string `default:"localhost:5051"` + SandboxToken string `default:""` + OutputPath string `default:"joj3_result.json"` + Stages []ConfStage + } + Teapot struct { + LogPath string `default:"/home/tt/.cache/joint-teapot-debug.log"` + ScoreboardPath string `default:"scoreboard.csv"` + FailedTablePath string `default:"failed-table.md"` + GradingRepoName string `default:""` + SkipIssue bool `default:"false"` + SkipScoreboard bool `default:"false"` + SkipFailedTable bool `default:"false"` + } + // TODO: remove the following backward compatibility fields SandboxExecServer string `default:"localhost:5051"` SandboxToken string `default:""` - LogPath string `default:""` OutputPath string `default:"joj3_result.json"` GradingRepoName string `default:""` SkipTeapot bool `default:"true"` ScoreboardPath string `default:"scoreboard.csv"` FailedTablePath string `default:"failed-table.md"` - Name string `default:"unknown"` Stages []struct { Name string Group string @@ -127,6 +159,27 @@ func parseConfFile(path string) (conf Conf, err error) { slog.Error("validate stages conf", "error", err) return } + // TODO: remove the following backward compatibility codes + if len(conf.Stage.Stages) == 0 { + conf.Stage.SandboxExecServer = conf.SandboxExecServer + conf.Stage.SandboxToken = conf.SandboxToken + conf.Stage.OutputPath = conf.OutputPath + conf.Stage.Stages = make([]ConfStage, len(conf.Stages)) + for i, stage := range conf.Stages { + conf.Stage.Stages[i].Name = stage.Name + conf.Stage.Stages[i].Group = stage.Group + conf.Stage.Stages[i].Executor = stage.Executor + conf.Stage.Stages[i].Parsers = []struct { + Name string + With interface{} + }{ + { + Name: stage.Parser.Name, + With: stage.Parser.With, + }, + } + } + } return } diff --git a/cmd/joj3/stage/main.go b/cmd/joj3/stage/main.go index 8d88791..33fd449 100644 --- a/cmd/joj3/stage/main.go +++ b/cmd/joj3/stage/main.go @@ -16,7 +16,7 @@ import ( func generateStages(conf conf.Conf, group string) ([]stage.Stage, error) { stages := []stage.Stage{} existNames := map[string]bool{} - for _, s := range conf.Stages { + for _, s := range conf.Stage.Stages { if s.Group != "" && group != s.Group { continue } @@ -50,12 +50,20 @@ func generateStages(conf conf.Conf, group string) ([]stage.Stage, error) { if len(s.Executor.With.Cases) == 0 { cmds = []stage.Cmd{defaultCmd} } + parsers := []stage.StageParser{} + for _, p := range s.Parsers { + parsers = append(parsers, stage.StageParser{ + Name: p.Name, + Conf: p.With, + }) + } stages = append(stages, stage.Stage{ - Name: s.Name, - ExecutorName: s.Executor.Name, - ExecutorCmds: cmds, - ParserName: s.Parser.Name, - ParserConf: s.Parser.With, + Name: s.Name, + Executor: stage.StageExecutor{ + Name: s.Executor.Name, + Cmds: cmds, + }, + Parsers: parsers, }) } slog.Debug("stages generated", "stages", stages) @@ -74,7 +82,10 @@ func outputResult(outputPath string, results []stage.StageResult) error { } func Run(conf conf.Conf, group string) error { - executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken) + executors.InitWithConf( + conf.Stage.SandboxExecServer, + conf.Stage.SandboxToken, + ) stages, err := generateStages(conf, group) if err != nil { slog.Error("generate stages", "error", err) @@ -86,7 +97,7 @@ func Run(conf conf.Conf, group string) error { slog.Error("run stages", "error", err) return err } - if err := outputResult(conf.OutputPath, results); err != nil { + if err := outputResult(conf.Stage.OutputPath, results); err != nil { slog.Error("output result", "error", err) return err } diff --git a/cmd/joj3/teapot/main.go b/cmd/joj3/teapot/main.go index 74a4a60..3fb6003 100644 --- a/cmd/joj3/teapot/main.go +++ b/cmd/joj3/teapot/main.go @@ -12,12 +12,16 @@ import ( ) func Run(conf conf.Conf) error { - if conf.SkipTeapot { + actions := os.Getenv("GITHUB_ACTIONS") + if actions != "true" { + slog.Info("teapot exit", "GITHUB_ACTIONS", actions) return nil } - os.Setenv("LOG_FILE_PATH", "/home/tt/.cache/joint-teapot-debug.log") + os.Setenv("LOG_FILE_PATH", conf.Teapot.LogPath) os.Setenv("_TYPER_STANDARD_TRACEBACK", "1") envFilePath := "/home/tt/.config/teapot/teapot.env" + // TODO: pass sha to joint-teapot + // sha := os.Getenv("GITHUB_SHA") actor := os.Getenv("GITHUB_ACTOR") repository := os.Getenv("GITHUB_REPOSITORY") runNumber := os.Getenv("GITHUB_RUN_NUMBER") @@ -29,49 +33,55 @@ func Run(conf conf.Conf) error { repoParts := strings.Split(repository, "/") repoName := repoParts[1] re := regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) - cmd := exec.Command("joint-teapot", "joj3-scoreboard", - envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName, - runNumber, conf.ScoreboardPath, conf.Name) // #nosec G204 - outputBytes, err := cmd.CombinedOutput() - output := re.ReplaceAllString(string(outputBytes), "") - for _, line := range strings.Split(output, "\n") { - if line == "" { - continue + if !conf.Teapot.SkipScoreboard { + cmd := exec.Command("joint-teapot", "joj3-scoreboard", + envFilePath, conf.Stage.OutputPath, actor, conf.Teapot.GradingRepoName, + repoName, runNumber, conf.Teapot.ScoreboardPath, conf.Name) // #nosec G204 + outputBytes, err := cmd.CombinedOutput() + output := re.ReplaceAllString(string(outputBytes), "") + for _, line := range strings.Split(output, "\n") { + if line == "" { + continue + } + slog.Info("joint-teapot joj3-scoreboard", "output", line) } - slog.Info("joint-teapot joj3-scoreboard", "output", line) - } - if err != nil { - slog.Error("joint-teapot joj3-scoreboard", "err", err) - return err - } - cmd = exec.Command("joint-teapot", "joj3-failed-table", - envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName, - runNumber, conf.FailedTablePath, conf.Name) // #nosec G204 - outputBytes, err = cmd.CombinedOutput() - output = re.ReplaceAllString(string(outputBytes), "") - for _, line := range strings.Split(output, "\n") { - if line == "" { - continue + if err != nil { + slog.Error("joint-teapot joj3-scoreboard", "err", err) + return err } - slog.Info("joint-teapot joj3-failed-table", "output", line) } - if err != nil { - slog.Error("joint-teapot joj3-failed-table", "err", err) - return err - } - cmd = exec.Command("joint-teapot", "joj3-create-result-issue", - envFilePath, conf.OutputPath, repoName, runNumber, conf.Name) // #nosec G204 - outputBytes, err = cmd.CombinedOutput() - output = re.ReplaceAllString(string(outputBytes), "") - for _, line := range strings.Split(output, "\n") { - if line == "" { - continue + if !conf.Teapot.SkipFailedTable { + cmd := exec.Command("joint-teapot", "joj3-failed-table", + envFilePath, conf.Stage.OutputPath, actor, conf.Teapot.GradingRepoName, + repoName, runNumber, conf.Teapot.FailedTablePath, conf.Name) // #nosec G204 + outputBytes, err := cmd.CombinedOutput() + output := re.ReplaceAllString(string(outputBytes), "") + for _, line := range strings.Split(output, "\n") { + if line == "" { + continue + } + slog.Info("joint-teapot joj3-failed-table", "output", line) + } + if err != nil { + slog.Error("joint-teapot joj3-failed-table", "err", err) + return err } - slog.Info("joint-teapot joj3-create-result-issue", "output", line) } - if err != nil { - slog.Error("joint-teapot joj3-create-result-issue", "err", err) - return err + if !conf.Teapot.SkipIssue { + cmd := exec.Command("joint-teapot", "joj3-create-result-issue", + envFilePath, conf.Stage.OutputPath, repoName, runNumber, conf.Name) // #nosec G204 + outputBytes, err := cmd.CombinedOutput() + output := re.ReplaceAllString(string(outputBytes), "") + for _, line := range strings.Split(output, "\n") { + if line == "" { + continue + } + slog.Info("joint-teapot joj3-create-result-issue", "output", line) + } + if err != nil { + slog.Error("joint-teapot joj3-create-result-issue", "err", err) + return err + } } return nil } diff --git a/internal/stage/model.go b/internal/stage/model.go index bd21dd7..a6c7e79 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -150,12 +150,19 @@ func (r ExecutorResult) String() string { return fmt.Sprintf("%+v", d) } +type StageExecutor struct { + Name string + Cmds []Cmd +} +type StageParser struct { + Name string + Conf any +} + type Stage struct { - Name string - ExecutorName string - ExecutorCmds []Cmd - ParserName string - ParserConf any + Name string + Executor StageExecutor + Parsers []StageParser } type ParserResult struct { diff --git a/internal/stage/run.go b/internal/stage/run.go index 4e346bc..b303e7c 100644 --- a/internal/stage/run.go +++ b/internal/stage/run.go @@ -8,51 +8,68 @@ import ( func Run(stages []Stage) (stageResults []StageResult, err error) { var executorResults []ExecutorResult var parserResults []ParserResult + var tmpParserResults []ParserResult var forceQuit bool slog.Info("stage run start") for _, stage := range stages { slog.Info("stage start", "name", stage.Name) - slog.Info("executor run start", "name", stage.ExecutorName) - slog.Debug("executor run start", "name", stage.ExecutorName, - "cmds", stage.ExecutorCmds) - executor, ok := executorMap[stage.ExecutorName] + slog.Info("executor run start", "name", stage.Executor.Name) + slog.Debug("executor run start", "name", stage.Executor.Name, + "cmds", stage.Executor.Cmds) + executor, ok := executorMap[stage.Executor.Name] if !ok { - slog.Error("executor not found", "name", stage.ExecutorName) - err = fmt.Errorf("executor not found: %s", stage.ExecutorName) + slog.Error("executor not found", "name", stage.Executor.Name) + err = fmt.Errorf("executor not found: %s", stage.Executor.Name) return } - executorResults, err = executor.Run(stage.ExecutorCmds) + executorResults, err = executor.Run(stage.Executor.Cmds) if err != nil { - slog.Error("executor run error", "name", stage.ExecutorName, "error", err) + slog.Error("executor run error", "name", stage.Executor.Name, "error", err) return } slog.Debug("executor run done", "results", executorResults) for _, executorResult := range executorResults { slog.Debug("executor run done", "result.Files", executorResult.Files) } - slog.Info("parser run start", "name", stage.ParserName) - slog.Debug("parser run start", "name", stage.ParserName, - "conf", stage.ParserConf) - parser, ok := parserMap[stage.ParserName] - if !ok { - slog.Error("parser not found", "name", stage.ParserName) - err = fmt.Errorf("parser not found: %s", stage.ParserName) - return + parserResults = []ParserResult{} + stageForceQuit := false + for _, stageParser := range stage.Parsers { + slog.Info("parser run start", "name", stageParser.Name) + slog.Debug("parser run start", "name", stageParser.Name, + "conf", stageParser.Conf) + parser, ok := parserMap[stageParser.Name] + if !ok { + slog.Error("parser not found", "name", stageParser.Name) + err = fmt.Errorf("parser not found: %s", stageParser.Name) + return + } + tmpParserResults, forceQuit, err = parser.Run( + executorResults, stageParser.Conf) + if err != nil { + slog.Error("parser run error", "name", stageParser.Name, "error", err) + return + } + stageForceQuit = stageForceQuit || forceQuit + slog.Debug("parser run done", "results", tmpParserResults) + if len(parserResults) == 0 { + parserResults = tmpParserResults + } else { + for i := range len(parserResults) { + parserResults[i].Score += tmpParserResults[i].Score + parserResults[i].Comment += tmpParserResults[i].Comment + } + } + if forceQuit { + slog.Error("parser force quit", "name", stageParser.Name) + } } - parserResults, forceQuit, err = parser.Run(executorResults, stage.ParserConf) - if err != nil { - slog.Error("parser run error", "name", stage.ParserName, "error", err) - return - } - slog.Debug("parser run done", "results", parserResults) stageResults = append(stageResults, StageResult{ Name: stage.Name, Results: parserResults, - ForceQuit: forceQuit, + ForceQuit: stageForceQuit, }) - if forceQuit { - slog.Error("parser force quit", "name", stage.ParserName) - return + if stageForceQuit { + break } } return From ad0918bbde12f32467bde1c9c360843c342ff83a Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 01:30:06 -0400 Subject: [PATCH 02/13] fix(cmd/joj3): teapot conf --- cmd/joj3/conf/conf.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/joj3/conf/conf.go b/cmd/joj3/conf/conf.go index 403aa6c..87f9878 100644 --- a/cmd/joj3/conf/conf.go +++ b/cmd/joj3/conf/conf.go @@ -179,6 +179,14 @@ func parseConfFile(path string) (conf Conf, err error) { }, } } + conf.Teapot.GradingRepoName = conf.GradingRepoName + conf.Teapot.ScoreboardPath = conf.ScoreboardPath + conf.Teapot.FailedTablePath = conf.FailedTablePath + if conf.SkipTeapot { + conf.Teapot.SkipScoreboard = true + conf.Teapot.SkipFailedTable = true + conf.Teapot.SkipIssue = true + } } return } From 2a2166f1adda9d2a20c317aabf3b584c5f12df2c Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 01:53:08 -0400 Subject: [PATCH 03/13] feat(parser/diff): add pass/fail comment to conf --- internal/parser/diff/parser.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/parser/diff/parser.go b/internal/parser/diff/parser.go index a1800a4..360e91b 100644 --- a/internal/parser/diff/parser.go +++ b/internal/parser/diff/parser.go @@ -21,7 +21,9 @@ const ( ) type Conf struct { - Cases []struct { + PassComment string `default:"🥳Passed!\n"` + FailComment string `default:"🧐Failed...\n"` + Cases []struct { IgnoreResultStatus bool Outputs []struct { Score int @@ -72,9 +74,9 @@ func (*Diff) Run(results []stage.ExecutorResult, confAny any) ( if compareChars(string(answer), result.Files[output.FileName], output.CompareSpace) { score += output.Score - comment += "Pass!\n" + comment += conf.PassComment } else { - comment += "Fail!\n" + comment += conf.FailComment comment += fmt.Sprintf("Difference found in `%s`.\n", output.FileName) if !output.AlwaysHide { From 703ff7d4be157242c55d7d3b366da9a5b8b30797 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 02:03:05 -0400 Subject: [PATCH 04/13] fix(parser/resultdetail): add extra newline when showing files --- internal/parser/resultdetail/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/parser/resultdetail/parser.go b/internal/parser/resultdetail/parser.go index 64bb340..a171169 100644 --- a/internal/parser/resultdetail/parser.go +++ b/internal/parser/resultdetail/parser.go @@ -49,7 +49,7 @@ func (*ResultDetail) Run(results []stage.ExecutorResult, confAny any) ( content, ok := result.Files[file] comment += fmt.Sprintf("File: `%s`.\n", file) if ok { - comment += fmt.Sprintf("```%s```\n", content) + comment += fmt.Sprintf("```%s\n```\n", content) } else { comment += "Not found.\n" } From 901290d263660221616a4e5f9b9afa2a10b79da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E8=B5=B5=E5=98=89=E7=A8=8B521432910016?= Date: Sat, 12 Oct 2024 18:30:54 +0800 Subject: [PATCH 05/13] feat: check repo size with conf (#56) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: zzjc1234 <2359047351@qq.com> Co-authored-by: Boming Zhang Reviewed-on: https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3/pulls/56 Reviewed-by: 张泊明518370910136 Co-authored-by: 周赵嘉程521432910016 Co-committed-by: 周赵嘉程521432910016 --- cmd/repo-health-checker/main.go | 3 ++- pkg/healthcheck/reposize.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/repo-health-checker/main.go b/cmd/repo-health-checker/main.go index 160b587..65b90dd 100644 --- a/cmd/repo-health-checker/main.go +++ b/cmd/repo-health-checker/main.go @@ -44,6 +44,7 @@ func main() { checkRelease := flag.Bool("checkRelease", true, "trigger release check") rootDir := flag.String("root", "", "") repo := flag.String("repo", "", "") + size := flag.Float64("reposize", 2, "size of the repo") localList := flag.String("localList", "", "") droneBranch := flag.String("droneBranch", "", "") checkFileNameList := flag.String("checkFileNameList", "", "Comma-separated list of files to check.") @@ -57,7 +58,7 @@ func main() { } setupSlog() var err error - err = healthcheck.RepoSize() + err = healthcheck.RepoSize(*size) if err != nil { fmt.Printf("### Repo Size Check Failed:\n%s\n", err.Error()) } diff --git a/pkg/healthcheck/reposize.go b/pkg/healthcheck/reposize.go index dd17d06..939adb8 100644 --- a/pkg/healthcheck/reposize.go +++ b/pkg/healthcheck/reposize.go @@ -10,7 +10,7 @@ import ( // RepoSize checks the size of the repository to determine if it is oversized. // It executes the 'git count-objects -v' command to obtain the size information, -func RepoSize() error { +func RepoSize(confSize float64) error { // TODO: reimplement here when go-git is available // https://github.com/go-git/go-git/blob/master/COMPATIBILITY.md cmd := exec.Command("git", "count-objects", "-v") @@ -33,8 +33,8 @@ func RepoSize() error { sum += size } } - if sum > 2048 { - return fmt.Errorf("Repository larger than 2MB. Please clean up or contact the teaching team.") + if sum > int(confSize*1024) { + return fmt.Errorf("Repository larger than %.1f MiB. Please clean up or contact the teaching team.", confSize) } return nil } From f8a7ae6067274f938db70155dc965e4bcebc2e8b Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 06:34:40 -0400 Subject: [PATCH 06/13] docs: update OS restriction --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c4aa581..691a206 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,16 @@ ## Quick Start -1. Make sure you are in a Unix-like OS (Linux, MacOS). For Windows, use [WSL 2](https://learn.microsoft.com/en-us/windows/wsl/install). +1. Clone this repo in a Linux computer. For Windows, use [WSL 2](https://learn.microsoft.com/en-us/windows/wsl/install). + +```bash +$ git clone ssh://git@focs.ji.sjtu.edu.cn:2222/JOJ/JOJ3.git +``` 2. Install [Go](https://go.dev/doc/install). Also, make sure `make` and `git` are installed and all 3 programs are presented in `$PATH`. - If you have problem on connecting to the Go website and Go packages, download Go from [studygolang](https://studygolang.com/dl) and run `go env -w GOPROXY=https://goproxy.io,direct` to set the Go modules mirror proxy after installing Go. -3. Enable cgroup v2 for your OS. Check [here](https://stackoverflow.com/a/73376219/13724598). So that you do not need root permission to run `go-judge`. +3. Enable cgroup v2 for your OS. For WSL2, check [here](https://stackoverflow.com/a/73376219/13724598). So that you do not need root permission to run `go-judge`. 4. Clone [go-judge](https://github.com/criyle/go-judge). From 9cd0e4467855d5e2761e4449bfbf2780c367d546 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 06:57:01 -0400 Subject: [PATCH 07/13] refactor(cmd/joj3): split parse msg & parse conf file --- cmd/joj3/conf/conf.go | 12 +++--------- cmd/joj3/main.go | 9 ++++++++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/joj3/conf/conf.go b/cmd/joj3/conf/conf.go index 87f9878..9c3cd13 100644 --- a/cmd/joj3/conf/conf.go +++ b/cmd/joj3/conf/conf.go @@ -144,7 +144,7 @@ func parseConventionalCommit(commit string) (*ConventionalCommit, error) { return cc, nil } -func parseConfFile(path string) (conf Conf, err error) { +func ParseConfFile(path string) (conf Conf, err error) { d := &multiconfig.DefaultLoader{} d.Loader = multiconfig.MultiLoader( &multiconfig.TagLoader{}, @@ -191,7 +191,7 @@ func parseConfFile(path string) (conf Conf, err error) { return } -func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) { +func ParseMsg(confRoot, confName, msg string) (confPath, group string, err error) { slog.Info("parse msg", "msg", msg) conventionalCommit, err := parseConventionalCommit(msg) if err != nil { @@ -199,7 +199,7 @@ func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err erro } slog.Info("conventional commit", "commit", conventionalCommit) confRoot = filepath.Clean(confRoot) - confPath := filepath.Clean(fmt.Sprintf("%s/%s/%s", + confPath = filepath.Clean(fmt.Sprintf("%s/%s/%s", confRoot, conventionalCommit.Scope, confName)) relPath, err := filepath.Rel(confRoot, confPath) if err != nil { @@ -209,11 +209,6 @@ func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err erro err = fmt.Errorf("invalid scope as path: %s", conventionalCommit.Scope) return } - slog.Info("try to load conf", "path", confPath) - conf, err = parseConfFile(confPath) - if err != nil { - return - } groupKeywords := []string{"joj"} for _, groupKeyword := range groupKeywords { if strings.Contains( @@ -222,7 +217,6 @@ func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err erro break } } - slog.Debug("conf loaded", "conf", conf) return } diff --git a/cmd/joj3/main.go b/cmd/joj3/main.go index ed18f0d..a1d4610 100644 --- a/cmd/joj3/main.go +++ b/cmd/joj3/main.go @@ -45,7 +45,7 @@ func mainImpl() error { return err } } - confObj, group, err := conf.ParseMsg(confRoot, confName, msg) + confPath, group, err := conf.ParseMsg(confRoot, confName, msg) if err != nil { slog.Error("parse msg", "error", err) validScopes, scopeErr := conf.ListValidScopes( @@ -58,6 +58,13 @@ func mainImpl() error { "valid scopes", validScopes) return err } + slog.Info("try to load conf", "path", confPath) + confObj, err := conf.ParseConfFile(confPath) + if err != nil { + slog.Error("parse conf", "error", err) + return err + } + slog.Debug("conf loaded", "conf", confObj) if err := setupSlog(confObj.LogPath); err != nil { // after conf is loaded slog.Error("setup slog", "error", err) return err From 4e3d202fa9a39335b0f64aecaccaf71a3b106e33 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 07:16:13 -0400 Subject: [PATCH 08/13] feat(cmd/joj3): pass `conf.Conf` by pointer --- cmd/joj3/conf/conf.go | 7 ++++--- cmd/joj3/stage/main.go | 4 ++-- cmd/joj3/teapot/main.go | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/joj3/conf/conf.go b/cmd/joj3/conf/conf.go index 9c3cd13..a8fe35f 100644 --- a/cmd/joj3/conf/conf.go +++ b/cmd/joj3/conf/conf.go @@ -144,18 +144,19 @@ func parseConventionalCommit(commit string) (*ConventionalCommit, error) { return cc, nil } -func ParseConfFile(path string) (conf Conf, err error) { +func ParseConfFile(path string) (conf *Conf, err error) { + conf = new(Conf) d := &multiconfig.DefaultLoader{} d.Loader = multiconfig.MultiLoader( &multiconfig.TagLoader{}, &multiconfig.JSONLoader{Path: path}, ) d.Validator = multiconfig.MultiValidator(&multiconfig.RequiredValidator{}) - if err = d.Load(&conf); err != nil { + if err = d.Load(conf); err != nil { slog.Error("parse stages conf", "error", err) return } - if err = d.Validate(&conf); err != nil { + if err = d.Validate(conf); err != nil { slog.Error("validate stages conf", "error", err) return } diff --git a/cmd/joj3/stage/main.go b/cmd/joj3/stage/main.go index 33fd449..ad57132 100644 --- a/cmd/joj3/stage/main.go +++ b/cmd/joj3/stage/main.go @@ -13,7 +13,7 @@ import ( "github.com/jinzhu/copier" ) -func generateStages(conf conf.Conf, group string) ([]stage.Stage, error) { +func generateStages(conf *conf.Conf, group string) ([]stage.Stage, error) { stages := []stage.Stage{} existNames := map[string]bool{} for _, s := range conf.Stage.Stages { @@ -81,7 +81,7 @@ func outputResult(outputPath string, results []stage.StageResult) error { append(content, []byte("\n")...), 0o600) } -func Run(conf conf.Conf, group string) error { +func Run(conf *conf.Conf, group string) error { executors.InitWithConf( conf.Stage.SandboxExecServer, conf.Stage.SandboxToken, diff --git a/cmd/joj3/teapot/main.go b/cmd/joj3/teapot/main.go index 3fb6003..2c032a5 100644 --- a/cmd/joj3/teapot/main.go +++ b/cmd/joj3/teapot/main.go @@ -11,7 +11,7 @@ import ( "github.com/joint-online-judge/JOJ3/cmd/joj3/conf" ) -func Run(conf conf.Conf) error { +func Run(conf *conf.Conf) error { actions := os.Getenv("GITHUB_ACTIONS") if actions != "true" { slog.Info("teapot exit", "GITHUB_ACTIONS", actions) From d934e9a067511515ef2283fdb6ea4ce3d2d8b66e Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 07:40:15 -0400 Subject: [PATCH 09/13] feat(cmd/joj3): run teapot commands in goroutine --- cmd/joj3/teapot/main.go | 79 ++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/cmd/joj3/teapot/main.go b/cmd/joj3/teapot/main.go index 2c032a5..329c221 100644 --- a/cmd/joj3/teapot/main.go +++ b/cmd/joj3/teapot/main.go @@ -7,6 +7,7 @@ import ( "os/exec" "regexp" "strings" + "sync" "github.com/joint-online-judge/JOJ3/cmd/joj3/conf" ) @@ -33,55 +34,61 @@ func Run(conf *conf.Conf) error { repoParts := strings.Split(repository, "/") repoName := repoParts[1] re := regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) - if !conf.Teapot.SkipScoreboard { - cmd := exec.Command("joint-teapot", "joj3-scoreboard", - envFilePath, conf.Stage.OutputPath, actor, conf.Teapot.GradingRepoName, - repoName, runNumber, conf.Teapot.ScoreboardPath, conf.Name) // #nosec G204 + execCommand := func(name string, cmdArgs []string) error { + cmd := exec.Command(name, cmdArgs...) // #nosec G204 outputBytes, err := cmd.CombinedOutput() output := re.ReplaceAllString(string(outputBytes), "") for _, line := range strings.Split(output, "\n") { if line == "" { continue } - slog.Info("joint-teapot joj3-scoreboard", "output", line) - } - if err != nil { - slog.Error("joint-teapot joj3-scoreboard", "err", err) - return err + slog.Info(fmt.Sprintf("%s %s", name, cmdArgs[0]), "output", line) } + return err } - if !conf.Teapot.SkipFailedTable { - cmd := exec.Command("joint-teapot", "joj3-failed-table", - envFilePath, conf.Stage.OutputPath, actor, conf.Teapot.GradingRepoName, - repoName, runNumber, conf.Teapot.FailedTablePath, conf.Name) // #nosec G204 - outputBytes, err := cmd.CombinedOutput() - output := re.ReplaceAllString(string(outputBytes), "") - for _, line := range strings.Split(output, "\n") { - if line == "" { - continue + var wg sync.WaitGroup + var scoreboardErr, failedTableErr, issueErr error + wg.Add(2) + go func() { + defer wg.Done() + if !conf.Teapot.SkipScoreboard { + err := execCommand("joint-teapot", []string{ + "joj3-scoreboard", envFilePath, conf.Stage.OutputPath, actor, + conf.Teapot.GradingRepoName, repoName, runNumber, + conf.Teapot.ScoreboardPath, conf.Name, + }) + if err != nil { + scoreboardErr = err } - slog.Info("joint-teapot joj3-failed-table", "output", line) } - if err != nil { - slog.Error("joint-teapot joj3-failed-table", "err", err) - return err - } - } - if !conf.Teapot.SkipIssue { - cmd := exec.Command("joint-teapot", "joj3-create-result-issue", - envFilePath, conf.Stage.OutputPath, repoName, runNumber, conf.Name) // #nosec G204 - outputBytes, err := cmd.CombinedOutput() - output := re.ReplaceAllString(string(outputBytes), "") - for _, line := range strings.Split(output, "\n") { - if line == "" { - continue + if !conf.Teapot.SkipFailedTable { + err := execCommand("joint-teapot", []string{ + "joj3-failed-table", envFilePath, conf.Stage.OutputPath, actor, + conf.Teapot.GradingRepoName, repoName, runNumber, + conf.Teapot.FailedTablePath, conf.Name, + }) + if err != nil { + failedTableErr = err } - slog.Info("joint-teapot joj3-create-result-issue", "output", line) } - if err != nil { - slog.Error("joint-teapot joj3-create-result-issue", "err", err) - return err + }() + go func() { + defer wg.Done() + if !conf.Teapot.SkipIssue { + err := execCommand("joint-teapot", []string{ + "joj3-create-result-issue", envFilePath, conf.Stage.OutputPath, + repoName, runNumber, conf.Name, + }) + if err != nil { + issueErr = err + } } + }() + wg.Wait() + if scoreboardErr != nil || failedTableErr != nil || issueErr != nil { + slog.Error("teapot exit", "scoreboardErr", scoreboardErr, + "failedTableErr", failedTableErr, "issueErr", issueErr) + return fmt.Errorf("teapot exit") } return nil } From 5f633016567a4968350a873456a577b53e0736f3 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 17:24:14 -0400 Subject: [PATCH 10/13] feat(parser/resultdetail): human readable units --- internal/parser/resultdetail/parser.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/parser/resultdetail/parser.go b/internal/parser/resultdetail/parser.go index a171169..a8a24e0 100644 --- a/internal/parser/resultdetail/parser.go +++ b/internal/parser/resultdetail/parser.go @@ -29,21 +29,21 @@ func (*ResultDetail) Run(results []stage.ExecutorResult, confAny any) ( var res []stage.ParserResult for _, result := range results { comment := "" - // TODO: more human readable units if conf.ShowExitStatus { - comment += fmt.Sprintf("Exit Status: %d\n", result.ExitStatus) + comment += fmt.Sprintf("Exit Status: `%d`\n", result.ExitStatus) } if conf.ShowError { - comment += fmt.Sprintf("Error: %s\n", result.Error) + comment += fmt.Sprintf("Error: `%s`\n", result.Error) } if conf.ShowTime { - comment += fmt.Sprintf("Time: %d\n", result.Time) + comment += fmt.Sprintf("Time: `%d ms`\n", result.Time/1e9) } if conf.ShowMemory { - comment += fmt.Sprintf("Memory: %d\n", result.Memory) + comment += fmt.Sprintf("Memory: `%.2f MiB`\n", + float64(result.Memory)/(1024*1024)) } if conf.ShowRunTime { - comment += fmt.Sprintf("RunTime: %d\n", result.RunTime) + comment += fmt.Sprintf("RunTime: `%d ms`\n", result.RunTime/1e9) } for _, file := range conf.ShowFiles { content, ok := result.Files[file] From 7004c5368e0faa5af266a5f349669fa8639feedf Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 17:34:26 -0400 Subject: [PATCH 11/13] fix(parser/resultdetail): file format --- internal/parser/resultdetail/parser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/parser/resultdetail/parser.go b/internal/parser/resultdetail/parser.go index a8a24e0..8bdb026 100644 --- a/internal/parser/resultdetail/parser.go +++ b/internal/parser/resultdetail/parser.go @@ -47,9 +47,9 @@ func (*ResultDetail) Run(results []stage.ExecutorResult, confAny any) ( } for _, file := range conf.ShowFiles { content, ok := result.Files[file] - comment += fmt.Sprintf("File: `%s`.\n", file) + comment += fmt.Sprintf("File: `%s`:\n", file) if ok { - comment += fmt.Sprintf("```%s\n```\n", content) + comment += fmt.Sprintf("```\n%s\n```\n", content) } else { comment += "Not found.\n" } From f4662eb123f0837235637d51ca98fc7233243d60 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 17:41:08 -0400 Subject: [PATCH 12/13] fix(parser/resultdetail): file format --- internal/parser/resultdetail/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/parser/resultdetail/parser.go b/internal/parser/resultdetail/parser.go index 8bdb026..5222d28 100644 --- a/internal/parser/resultdetail/parser.go +++ b/internal/parser/resultdetail/parser.go @@ -47,7 +47,7 @@ func (*ResultDetail) Run(results []stage.ExecutorResult, confAny any) ( } for _, file := range conf.ShowFiles { content, ok := result.Files[file] - comment += fmt.Sprintf("File: `%s`:\n", file) + comment += fmt.Sprintf("File `%s`:\n", file) if ok { comment += fmt.Sprintf("```\n%s\n```\n", content) } else { From cdab5ab4c81abb660872b4127511b04f776ee49a Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Sat, 12 Oct 2024 17:44:16 -0400 Subject: [PATCH 13/13] chore: add date & time to version --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 67c1186..eb94023 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,9 @@ BUILD_DIR = ./build TMP_DIR = ./tmp APPS := $(notdir $(wildcard ./cmd/*)) -VERSION := $(shell git rev-parse --short HEAD) +COMMIT_HASH := $(shell git rev-parse --short HEAD) +DATE := $(shell date +"%Y%m%d-%H%M%S") +VERSION := $(COMMIT_HASH)-$(DATE) FLAGS := "-s -w -X main.Version=$(VERSION)" all: build