feat: run stages and teapot in joj3 #47

Merged
张泊明518370910136 merged 8 commits from wrapper into master 2024-10-07 13:27:21 +08:00
9 changed files with 202 additions and 123 deletions

View File

@ -34,6 +34,9 @@ jobs:
rm -rf golangci-lint-1.61.0-linux-amd64.tar.gz rm -rf golangci-lint-1.61.0-linux-amd64.tar.gz
mkdir -p /root/go/bin mkdir -p /root/go/bin
mv /tmp/golangci-lint-1.61.0-linux-amd64/golangci-lint /root/go/bin mv /tmp/golangci-lint-1.61.0-linux-amd64/golangci-lint /root/go/bin
- name: Setup Joint-Teapot
run: |
pip install git+https://ghp.ci/https://github.com/BoYanZh/Joint-Teapot
- name: Lint - name: Lint
run: make lint run: make lint
- name: Build - name: Build

View File

@ -10,6 +10,7 @@ all: build
build: build:
$(foreach APP,$(APPS), go build -ldflags=$(FLAGS) -o $(BUILD_DIR)/$(APP) ./cmd/$(APP);) $(foreach APP,$(APPS), go build -ldflags=$(FLAGS) -o $(BUILD_DIR)/$(APP) ./cmd/$(APP);)
cp ./build/repo-health-checker ./build/healthcheck
clean: clean:
rm -rf $(BUILD_DIR)/* rm -rf $(BUILD_DIR)/*

View File

@ -1,4 +1,4 @@
package main package conf
import ( import (
"fmt" "fmt"
@ -8,6 +8,7 @@ import (
"regexp" "regexp"
"strings" "strings"
"github.com/go-git/go-git/v5"
"github.com/joint-online-judge/JOJ3/internal/stage" "github.com/joint-online-judge/JOJ3/internal/stage"
"github.com/koding/multiconfig" "github.com/koding/multiconfig"
) )
@ -17,6 +18,8 @@ type Conf struct {
SandboxToken string `default:""` SandboxToken string `default:""`
LogPath string `default:""` LogPath string `default:""`
OutputPath string `default:"joj3_result.json"` OutputPath string `default:"joj3_result.json"`
GradingRepoName string `default:""`
SkipTeapot bool `default:"true"`
Stages []struct { Stages []struct {
Name string Name string
Group string Group string
@ -73,6 +76,23 @@ type ConventionalCommit struct {
Footer string Footer string
} }
func GetCommitMsg() (msg string, err error) {
r, err := git.PlainOpen(".")
if err != nil {
return
}
ref, err := r.Head()
if err != nil {
return
}
commit, err := r.CommitObject(ref.Hash())
if err != nil {
return
}
msg = commit.Message
return
}
func parseConventionalCommit(commit string) (*ConventionalCommit, error) { func parseConventionalCommit(commit string) (*ConventionalCommit, error) {
re := regexp.MustCompile(`(?s)^(\w+)(\(([^)]+)\))?!?: (.+?)(\n\n(.+?))?(\n\n(.+))?$`) re := regexp.MustCompile(`(?s)^(\w+)(\(([^)]+)\))?!?: (.+?)(\n\n(.+?))?(\n\n(.+))?$`)
matches := re.FindStringSubmatch(strings.TrimSpace(commit)) matches := re.FindStringSubmatch(strings.TrimSpace(commit))
@ -107,7 +127,7 @@ func parseConfFile(path string) (conf Conf, err error) {
return return
} }
func parseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) { func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) {
slog.Info("parse msg", "msg", msg) slog.Info("parse msg", "msg", msg)
conventionalCommit, err := parseConventionalCommit(msg) conventionalCommit, err := parseConventionalCommit(msg)
if err != nil { if err != nil {
@ -142,7 +162,7 @@ func parseMsg(confRoot, confName, msg string) (conf Conf, group string, err erro
return return
} }
func listValidScopes(confRoot, confName, msg string) ([]string, error) { func ListValidScopes(confRoot, confName, msg string) ([]string, error) {
conventionalCommit, err := parseConventionalCommit(msg) conventionalCommit, err := parseConventionalCommit(msg)
if err != nil { if err != nil {
return []string{}, err return []string{}, err

View File

@ -1,4 +1,4 @@
package main package conf
import ( import (
"reflect" "reflect"

View File

@ -1,97 +1,15 @@
package main package main
import ( import (
"encoding/json"
"flag" "flag"
"fmt" "fmt"
"log/slog" "log/slog"
"os"
"github.com/joint-online-judge/JOJ3/internal/executors" "github.com/joint-online-judge/JOJ3/cmd/joj3/conf"
_ "github.com/joint-online-judge/JOJ3/internal/parsers" "github.com/joint-online-judge/JOJ3/cmd/joj3/stage"
"github.com/joint-online-judge/JOJ3/internal/stage" "github.com/joint-online-judge/JOJ3/cmd/joj3/teapot"
"github.com/go-git/go-git/v5"
"github.com/jinzhu/copier"
) )
func getCommitMsg() (msg string, err error) {
r, err := git.PlainOpen(".")
if err != nil {
return
}
ref, err := r.Head()
if err != nil {
return
}
commit, err := r.CommitObject(ref.Hash())
if err != nil {
return
}
msg = commit.Message
return
}
func generateStages(conf Conf, group string) ([]stage.Stage, error) {
stages := []stage.Stage{}
existNames := map[string]bool{}
for _, s := range conf.Stages {
if s.Group != "" && group != s.Group {
continue
}
_, ok := existNames[s.Name] // check for existence
if ok {
continue
}
existNames[s.Name] = true
var cmds []stage.Cmd
defaultCmd := s.Executor.With.Default
for _, optionalCmd := range s.Executor.With.Cases {
cmd := s.Executor.With.Default
err := copier.Copy(&cmd, &optionalCmd)
if err != nil {
slog.Error("generate stages", "error", err)
return stages, err
}
// since these 3 values are pointers, copier will always copy
// them, so we need to check them manually
if defaultCmd.Stdin != nil && optionalCmd.Stdin == nil {
cmd.Stdin = defaultCmd.Stdin
}
if defaultCmd.Stdout != nil && optionalCmd.Stdout == nil {
cmd.Stdout = defaultCmd.Stdout
}
if defaultCmd.Stderr != nil && optionalCmd.Stderr == nil {
cmd.Stderr = defaultCmd.Stderr
}
cmds = append(cmds, cmd)
}
if len(s.Executor.With.Cases) == 0 {
cmds = []stage.Cmd{defaultCmd}
}
stages = append(stages, stage.Stage{
Name: s.Name,
ExecutorName: s.Executor.Name,
ExecutorCmds: cmds,
ParserName: s.Parser.Name,
ParserConf: s.Parser.With,
})
}
slog.Debug("stages generated", "stages", stages)
return stages, nil
}
func outputResult(outputPath string, results []stage.StageResult) error {
slog.Info("output result start", "path", outputPath)
slog.Debug("output result start", "path", outputPath, "results", results)
content, err := json.Marshal(results)
if err != nil {
return err
}
return os.WriteFile(outputPath,
append(content, []byte("\n")...), 0o600)
}
var ( var (
confRoot string confRoot string
confName string confName string
@ -107,61 +25,47 @@ func init() {
showVersion = flag.Bool("version", false, "print current version") showVersion = flag.Bool("version", false, "print current version")
} }
func mainImpl() error { func main() {
if err := setupSlog(""); err != nil { // before conf is loaded if err := setupSlog(""); err != nil { // before conf is loaded
return err slog.Error("setup slog", "error", err)
return
} }
flag.Parse() flag.Parse()
if *showVersion { if *showVersion {
fmt.Println(Version) fmt.Println(Version)
return nil return
} }
slog.Info("start joj3", "version", Version) slog.Info("start joj3", "version", Version)
if msg == "" { if msg == "" {
var err error var err error
msg, err = getCommitMsg() msg, err = conf.GetCommitMsg()
if err != nil { if err != nil {
slog.Error("get commit msg", "error", err) slog.Error("get commit msg", "error", err)
return err return
} }
} }
conf, group, err := parseMsg(confRoot, confName, msg) confObj, group, err := conf.ParseMsg(confRoot, confName, msg)
if err != nil { if err != nil {
slog.Error("parse msg", "error", err) slog.Error("parse msg", "error", err)
validScopes, scopeErr := listValidScopes( validScopes, scopeErr := conf.ListValidScopes(
confRoot, confName, msg) confRoot, confName, msg)
if scopeErr != nil { if scopeErr != nil {
slog.Error("list valid scopes", "error", scopeErr) slog.Error("list valid scopes", "error", scopeErr)
return scopeErr return
} }
slog.Info("hint: valid scopes in commit message", "scopes", validScopes) slog.Info("hint: valid scopes in commit message", "scopes", validScopes)
return err return
} }
if err := setupSlog(conf.LogPath); err != nil { // after conf is loaded if err := setupSlog(confObj.LogPath); err != nil { // after conf is loaded
return err slog.Error("setup slog", "error", err)
return
} }
executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken) if err := stage.Run(confObj, group); err != nil {
stages, err := generateStages(conf, group) slog.Error("stage run", "error", err)
if err != nil { return
slog.Error("generate stages", "error", err)
return err
} }
defer stage.Cleanup() if err := teapot.Run(confObj); err != nil {
results, err := stage.Run(stages) slog.Error("teapot run", "error", err)
if err != nil { return
slog.Error("run stages", "error", err)
return err
}
if err := outputResult(conf.OutputPath, results); err != nil {
slog.Error("output result", "error", err)
return err
}
return nil
}
func main() {
if err := mainImpl(); err != nil {
slog.Error("main exit", "error", err)
os.Exit(1)
} }
} }

View File

@ -61,7 +61,7 @@ func readStageResults(t *testing.T, path string) []stage.StageResult {
return results return results
} }
func TestMain(t *testing.T) { func TestRun(t *testing.T) {
var tests []string var tests []string
root := "../../tmp/submodules/JOJ3-examples/examples/" root := "../../tmp/submodules/JOJ3-examples/examples/"
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {

94
cmd/joj3/stage/main.go Normal file
View File

@ -0,0 +1,94 @@
package stage
import (
"encoding/json"
"log/slog"
"os"
"github.com/joint-online-judge/JOJ3/cmd/joj3/conf"
"github.com/joint-online-judge/JOJ3/internal/executors"
_ "github.com/joint-online-judge/JOJ3/internal/parsers"
"github.com/joint-online-judge/JOJ3/internal/stage"
"github.com/jinzhu/copier"
)
func generateStages(conf conf.Conf, group string) ([]stage.Stage, error) {
stages := []stage.Stage{}
existNames := map[string]bool{}
for _, s := range conf.Stages {
if s.Group != "" && group != s.Group {
continue
}
_, ok := existNames[s.Name] // check for existence
if ok {
continue
}
existNames[s.Name] = true
var cmds []stage.Cmd
defaultCmd := s.Executor.With.Default
for _, optionalCmd := range s.Executor.With.Cases {
cmd := s.Executor.With.Default
err := copier.Copy(&cmd, &optionalCmd)
if err != nil {
slog.Error("generate stages", "error", err)
return stages, err
}
// since these 3 values are pointers, copier will always copy
// them, so we need to check them manually
if defaultCmd.Stdin != nil && optionalCmd.Stdin == nil {
cmd.Stdin = defaultCmd.Stdin
}
if defaultCmd.Stdout != nil && optionalCmd.Stdout == nil {
cmd.Stdout = defaultCmd.Stdout
}
if defaultCmd.Stderr != nil && optionalCmd.Stderr == nil {
cmd.Stderr = defaultCmd.Stderr
}
cmds = append(cmds, cmd)
}
if len(s.Executor.With.Cases) == 0 {
cmds = []stage.Cmd{defaultCmd}
}
stages = append(stages, stage.Stage{
Name: s.Name,
ExecutorName: s.Executor.Name,
ExecutorCmds: cmds,
ParserName: s.Parser.Name,
ParserConf: s.Parser.With,
})
}
slog.Debug("stages generated", "stages", stages)
return stages, nil
}
func outputResult(outputPath string, results []stage.StageResult) error {
slog.Info("output result start", "path", outputPath)
slog.Debug("output result start", "path", outputPath, "results", results)
content, err := json.Marshal(results)
if err != nil {
return err
}
return os.WriteFile(outputPath,
append(content, []byte("\n")...), 0o600)
}
func Run(conf conf.Conf, group string) error {
executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken)
stages, err := generateStages(conf, group)
if err != nil {
slog.Error("generate stages", "error", err)
return err
}
defer stage.Cleanup()
results, err := stage.Run(stages)
if err != nil {
slog.Error("run stages", "error", err)
return err
}
if err := outputResult(conf.OutputPath, results); err != nil {
slog.Error("output result", "error", err)
return err
}
return nil
}

57
cmd/joj3/teapot/main.go Normal file
View File

@ -0,0 +1,57 @@
package teapot
import (
"fmt"
"log/slog"
"os"
"os/exec"
"strings"
"github.com/joint-online-judge/JOJ3/cmd/joj3/conf"
)
func Run(conf conf.Conf) error {
if conf.SkipTeapot {
return nil
}
os.Setenv("LOG_FILE_PATH", "/home/tt/.cache/joint-teapot-debug.log")
os.Setenv("_TYPER_STANDARD_TRACEBACK", "1")
envFilePath := "/home/tt/.config/teapot/teapot.env"
actor := os.Getenv("GITHUB_ACTOR")
repository := os.Getenv("GITHUB_REPOSITORY")
runNumber := os.Getenv("GITHUB_RUN_NUMBER")
if actor == "" || repository == "" || strings.Count(repository, "/") != 1 ||
runNumber == "" {
slog.Error("teapot env not set")
return fmt.Errorf("teapot env not set")
}
repoParts := strings.Split(repository, "/")
repoName := repoParts[1]
cmd := exec.Command("joint-teapot", "joj3-scoreboard",
envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName,
runNumber) // #nosec G204
output, err := cmd.CombinedOutput()
if err != nil {
slog.Error("running git command:", "err", err)
return err
}
slog.Info("joint-teapot joj3-scoreboard", "output", string(output))
cmd = exec.Command("joint-teapot", "joj3-failed-table",
envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName,
runNumber) // #nosec G204
output, err = cmd.CombinedOutput()
if err != nil {
slog.Error("running git command:", "err", err)
return err
}
slog.Info("joint-teapot joj3-failed-table", "output", string(output))
cmd = exec.Command("joint-teapot", "joj3-create-result-issue",
envFilePath, conf.OutputPath, repoName, runNumber) // #nosec G204
output, err = cmd.CombinedOutput()
if err != nil {
slog.Error("running git command:", "err", err)
return err
}
slog.Info("joint-teapot joj3-create-result-issue", "output", string(output))
return nil
}