From c3faf8ed558464c19231bf3415c635043262cfe0 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Mon, 7 Oct 2024 13:27:20 +0800 Subject: [PATCH] feat: run stages and teapot in joj3 (#47) Reviewed-on: https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3/pulls/47 Co-authored-by: Boming Zhang Co-committed-by: Boming Zhang --- .gitea/workflows/build.yaml | 3 + Makefile | 1 + cmd/joj3/{ => conf}/conf.go | 26 +++- cmd/joj3/{ => conf}/conf_test.go | 2 +- cmd/joj3/main.go | 140 +++--------------- cmd/joj3/main_test.go | 2 +- cmd/joj3/stage/main.go | 94 ++++++++++++ cmd/joj3/teapot/main.go | 57 +++++++ .../main.go | 0 9 files changed, 202 insertions(+), 123 deletions(-) rename cmd/joj3/{ => conf}/conf.go (88%) rename cmd/joj3/{ => conf}/conf_test.go (99%) create mode 100644 cmd/joj3/stage/main.go create mode 100644 cmd/joj3/teapot/main.go rename cmd/{healthcheck => repo-health-checker}/main.go (100%) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 340acfc..192ddda 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -34,6 +34,9 @@ jobs: rm -rf golangci-lint-1.61.0-linux-amd64.tar.gz mkdir -p /root/go/bin mv /tmp/golangci-lint-1.61.0-linux-amd64/golangci-lint /root/go/bin + - name: Setup Joint-Teapot + run: | + pip install git+https://ghp.ci/https://github.com/BoYanZh/Joint-Teapot - name: Lint run: make lint - name: Build diff --git a/Makefile b/Makefile index 3a56bac..f7123b6 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ all: build build: $(foreach APP,$(APPS), go build -ldflags=$(FLAGS) -o $(BUILD_DIR)/$(APP) ./cmd/$(APP);) + cp ./build/repo-health-checker ./build/healthcheck clean: rm -rf $(BUILD_DIR)/* diff --git a/cmd/joj3/conf.go b/cmd/joj3/conf/conf.go similarity index 88% rename from cmd/joj3/conf.go rename to cmd/joj3/conf/conf.go index 6e63847..c385e4e 100644 --- a/cmd/joj3/conf.go +++ b/cmd/joj3/conf/conf.go @@ -1,4 +1,4 @@ -package main +package conf import ( "fmt" @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "github.com/go-git/go-git/v5" "github.com/joint-online-judge/JOJ3/internal/stage" "github.com/koding/multiconfig" ) @@ -17,6 +18,8 @@ type Conf struct { SandboxToken string `default:""` LogPath string `default:""` OutputPath string `default:"joj3_result.json"` + GradingRepoName string `default:""` + SkipTeapot bool `default:"true"` Stages []struct { Name string Group string @@ -73,6 +76,23 @@ type ConventionalCommit struct { Footer string } +func GetCommitMsg() (msg string, err error) { + r, err := git.PlainOpen(".") + if err != nil { + return + } + ref, err := r.Head() + if err != nil { + return + } + commit, err := r.CommitObject(ref.Hash()) + if err != nil { + return + } + msg = commit.Message + return +} + func parseConventionalCommit(commit string) (*ConventionalCommit, error) { re := regexp.MustCompile(`(?s)^(\w+)(\(([^)]+)\))?!?: (.+?)(\n\n(.+?))?(\n\n(.+))?$`) matches := re.FindStringSubmatch(strings.TrimSpace(commit)) @@ -107,7 +127,7 @@ func parseConfFile(path string) (conf Conf, err error) { return } -func parseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) { +func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) { slog.Info("parse msg", "msg", msg) conventionalCommit, err := parseConventionalCommit(msg) if err != nil { @@ -142,7 +162,7 @@ func parseMsg(confRoot, confName, msg string) (conf Conf, group string, err erro return } -func listValidScopes(confRoot, confName, msg string) ([]string, error) { +func ListValidScopes(confRoot, confName, msg string) ([]string, error) { conventionalCommit, err := parseConventionalCommit(msg) if err != nil { return []string{}, err diff --git a/cmd/joj3/conf_test.go b/cmd/joj3/conf/conf_test.go similarity index 99% rename from cmd/joj3/conf_test.go rename to cmd/joj3/conf/conf_test.go index 0906cc3..ccab160 100644 --- a/cmd/joj3/conf_test.go +++ b/cmd/joj3/conf/conf_test.go @@ -1,4 +1,4 @@ -package main +package conf import ( "reflect" diff --git a/cmd/joj3/main.go b/cmd/joj3/main.go index a211105..257e108 100644 --- a/cmd/joj3/main.go +++ b/cmd/joj3/main.go @@ -1,97 +1,15 @@ package main import ( - "encoding/json" "flag" "fmt" "log/slog" - "os" - "github.com/joint-online-judge/JOJ3/internal/executors" - _ "github.com/joint-online-judge/JOJ3/internal/parsers" - "github.com/joint-online-judge/JOJ3/internal/stage" - - "github.com/go-git/go-git/v5" - "github.com/jinzhu/copier" + "github.com/joint-online-judge/JOJ3/cmd/joj3/conf" + "github.com/joint-online-judge/JOJ3/cmd/joj3/stage" + "github.com/joint-online-judge/JOJ3/cmd/joj3/teapot" ) -func getCommitMsg() (msg string, err error) { - r, err := git.PlainOpen(".") - if err != nil { - return - } - ref, err := r.Head() - if err != nil { - return - } - commit, err := r.CommitObject(ref.Hash()) - if err != nil { - return - } - msg = commit.Message - return -} - -func generateStages(conf Conf, group string) ([]stage.Stage, error) { - stages := []stage.Stage{} - existNames := map[string]bool{} - for _, s := range conf.Stages { - if s.Group != "" && group != s.Group { - continue - } - _, ok := existNames[s.Name] // check for existence - if ok { - continue - } - existNames[s.Name] = true - var cmds []stage.Cmd - defaultCmd := s.Executor.With.Default - for _, optionalCmd := range s.Executor.With.Cases { - cmd := s.Executor.With.Default - err := copier.Copy(&cmd, &optionalCmd) - if err != nil { - slog.Error("generate stages", "error", err) - return stages, err - } - // since these 3 values are pointers, copier will always copy - // them, so we need to check them manually - if defaultCmd.Stdin != nil && optionalCmd.Stdin == nil { - cmd.Stdin = defaultCmd.Stdin - } - if defaultCmd.Stdout != nil && optionalCmd.Stdout == nil { - cmd.Stdout = defaultCmd.Stdout - } - if defaultCmd.Stderr != nil && optionalCmd.Stderr == nil { - cmd.Stderr = defaultCmd.Stderr - } - cmds = append(cmds, cmd) - } - if len(s.Executor.With.Cases) == 0 { - cmds = []stage.Cmd{defaultCmd} - } - stages = append(stages, stage.Stage{ - Name: s.Name, - ExecutorName: s.Executor.Name, - ExecutorCmds: cmds, - ParserName: s.Parser.Name, - ParserConf: s.Parser.With, - }) - } - slog.Debug("stages generated", "stages", stages) - return stages, nil -} - -func outputResult(outputPath string, results []stage.StageResult) error { - slog.Info("output result start", "path", outputPath) - slog.Debug("output result start", "path", outputPath, "results", results) - content, err := json.Marshal(results) - if err != nil { - return err - } - return os.WriteFile(outputPath, - append(content, []byte("\n")...), 0o600) -} - var ( confRoot string confName string @@ -107,61 +25,47 @@ func init() { showVersion = flag.Bool("version", false, "print current version") } -func mainImpl() error { +func main() { if err := setupSlog(""); err != nil { // before conf is loaded - return err + slog.Error("setup slog", "error", err) + return } flag.Parse() if *showVersion { fmt.Println(Version) - return nil + return } slog.Info("start joj3", "version", Version) if msg == "" { var err error - msg, err = getCommitMsg() + msg, err = conf.GetCommitMsg() if err != nil { slog.Error("get commit msg", "error", err) - return err + return } } - conf, group, err := parseMsg(confRoot, confName, msg) + confObj, group, err := conf.ParseMsg(confRoot, confName, msg) if err != nil { slog.Error("parse msg", "error", err) - validScopes, scopeErr := listValidScopes( + validScopes, scopeErr := conf.ListValidScopes( confRoot, confName, msg) if scopeErr != nil { slog.Error("list valid scopes", "error", scopeErr) - return scopeErr + return } slog.Info("hint: valid scopes in commit message", "scopes", validScopes) - return err + return } - if err := setupSlog(conf.LogPath); err != nil { // after conf is loaded - return err + if err := setupSlog(confObj.LogPath); err != nil { // after conf is loaded + slog.Error("setup slog", "error", err) + return } - executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken) - stages, err := generateStages(conf, group) - if err != nil { - slog.Error("generate stages", "error", err) - return err + if err := stage.Run(confObj, group); err != nil { + slog.Error("stage run", "error", err) + return } - defer stage.Cleanup() - results, err := stage.Run(stages) - if err != nil { - slog.Error("run stages", "error", err) - return err - } - if err := outputResult(conf.OutputPath, results); err != nil { - slog.Error("output result", "error", err) - return err - } - return nil -} - -func main() { - if err := mainImpl(); err != nil { - slog.Error("main exit", "error", err) - os.Exit(1) + if err := teapot.Run(confObj); err != nil { + slog.Error("teapot run", "error", err) + return } } diff --git a/cmd/joj3/main_test.go b/cmd/joj3/main_test.go index e160176..ea442fc 100644 --- a/cmd/joj3/main_test.go +++ b/cmd/joj3/main_test.go @@ -61,7 +61,7 @@ func readStageResults(t *testing.T, path string) []stage.StageResult { return results } -func TestMain(t *testing.T) { +func TestRun(t *testing.T) { var tests []string root := "../../tmp/submodules/JOJ3-examples/examples/" err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { diff --git a/cmd/joj3/stage/main.go b/cmd/joj3/stage/main.go new file mode 100644 index 0000000..1f86c1b --- /dev/null +++ b/cmd/joj3/stage/main.go @@ -0,0 +1,94 @@ +package stage + +import ( + "encoding/json" + "log/slog" + "os" + + "github.com/joint-online-judge/JOJ3/cmd/joj3/conf" + "github.com/joint-online-judge/JOJ3/internal/executors" + _ "github.com/joint-online-judge/JOJ3/internal/parsers" + "github.com/joint-online-judge/JOJ3/internal/stage" + + "github.com/jinzhu/copier" +) + +func generateStages(conf conf.Conf, group string) ([]stage.Stage, error) { + stages := []stage.Stage{} + existNames := map[string]bool{} + for _, s := range conf.Stages { + if s.Group != "" && group != s.Group { + continue + } + _, ok := existNames[s.Name] // check for existence + if ok { + continue + } + existNames[s.Name] = true + var cmds []stage.Cmd + defaultCmd := s.Executor.With.Default + for _, optionalCmd := range s.Executor.With.Cases { + cmd := s.Executor.With.Default + err := copier.Copy(&cmd, &optionalCmd) + if err != nil { + slog.Error("generate stages", "error", err) + return stages, err + } + // since these 3 values are pointers, copier will always copy + // them, so we need to check them manually + if defaultCmd.Stdin != nil && optionalCmd.Stdin == nil { + cmd.Stdin = defaultCmd.Stdin + } + if defaultCmd.Stdout != nil && optionalCmd.Stdout == nil { + cmd.Stdout = defaultCmd.Stdout + } + if defaultCmd.Stderr != nil && optionalCmd.Stderr == nil { + cmd.Stderr = defaultCmd.Stderr + } + cmds = append(cmds, cmd) + } + if len(s.Executor.With.Cases) == 0 { + cmds = []stage.Cmd{defaultCmd} + } + stages = append(stages, stage.Stage{ + Name: s.Name, + ExecutorName: s.Executor.Name, + ExecutorCmds: cmds, + ParserName: s.Parser.Name, + ParserConf: s.Parser.With, + }) + } + slog.Debug("stages generated", "stages", stages) + return stages, nil +} + +func outputResult(outputPath string, results []stage.StageResult) error { + slog.Info("output result start", "path", outputPath) + slog.Debug("output result start", "path", outputPath, "results", results) + content, err := json.Marshal(results) + if err != nil { + return err + } + return os.WriteFile(outputPath, + append(content, []byte("\n")...), 0o600) +} + +func Run(conf conf.Conf, group string) error { + executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken) + stages, err := generateStages(conf, group) + if err != nil { + slog.Error("generate stages", "error", err) + return err + } + defer stage.Cleanup() + results, err := stage.Run(stages) + if err != nil { + slog.Error("run stages", "error", err) + return err + } + if err := outputResult(conf.OutputPath, results); err != nil { + slog.Error("output result", "error", err) + return err + } + return nil +} diff --git a/cmd/joj3/teapot/main.go b/cmd/joj3/teapot/main.go new file mode 100644 index 0000000..8cbe38f --- /dev/null +++ b/cmd/joj3/teapot/main.go @@ -0,0 +1,57 @@ +package teapot + +import ( + "fmt" + "log/slog" + "os" + "os/exec" + "strings" + + "github.com/joint-online-judge/JOJ3/cmd/joj3/conf" +) + +func Run(conf conf.Conf) error { + if conf.SkipTeapot { + return nil + } + os.Setenv("LOG_FILE_PATH", "/home/tt/.cache/joint-teapot-debug.log") + os.Setenv("_TYPER_STANDARD_TRACEBACK", "1") + envFilePath := "/home/tt/.config/teapot/teapot.env" + actor := os.Getenv("GITHUB_ACTOR") + repository := os.Getenv("GITHUB_REPOSITORY") + runNumber := os.Getenv("GITHUB_RUN_NUMBER") + if actor == "" || repository == "" || strings.Count(repository, "/") != 1 || + runNumber == "" { + slog.Error("teapot env not set") + return fmt.Errorf("teapot env not set") + } + repoParts := strings.Split(repository, "/") + repoName := repoParts[1] + cmd := exec.Command("joint-teapot", "joj3-scoreboard", + envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName, + runNumber) // #nosec G204 + output, err := cmd.CombinedOutput() + if err != nil { + slog.Error("running git command:", "err", err) + return err + } + slog.Info("joint-teapot joj3-scoreboard", "output", string(output)) + cmd = exec.Command("joint-teapot", "joj3-failed-table", + envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName, + runNumber) // #nosec G204 + output, err = cmd.CombinedOutput() + if err != nil { + slog.Error("running git command:", "err", err) + return err + } + slog.Info("joint-teapot joj3-failed-table", "output", string(output)) + cmd = exec.Command("joint-teapot", "joj3-create-result-issue", + envFilePath, conf.OutputPath, repoName, runNumber) // #nosec G204 + output, err = cmd.CombinedOutput() + if err != nil { + slog.Error("running git command:", "err", err) + return err + } + slog.Info("joint-teapot joj3-create-result-issue", "output", string(output)) + return nil +} diff --git a/cmd/healthcheck/main.go b/cmd/repo-health-checker/main.go similarity index 100% rename from cmd/healthcheck/main.go rename to cmd/repo-health-checker/main.go