JOJ/JOJ3
25
6
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: run stages and teapot in joj3 #47

Merged
张泊明518370910136 merged 8 commits from wrapper into master 2024-10-07 13:27:21 +08:00
Conversation 0 Commits 8 Files Changed 9 +202 -123
9 changed files with 202 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitea/workflows/build.yaml
View File

@ -34,6 +34,9 @@ jobs:
rm -rf golangci-lint-1.61.0-linux-amd64.tar.gz
mkdir -p /root/go/bin
mv /tmp/golangci-lint-1.61.0-linux-amd64/golangci-lint /root/go/bin
- name: Setup Joint-Teapot
run: |
pip install git+https://ghp.ci/https://github.com/BoYanZh/Joint-Teapot
- name: Lint
run: make lint
- name: Build

1
Makefile
View File

@ -10,6 +10,7 @@ all: build
build:
$(foreach APP,$(APPS), go build -ldflags=$(FLAGS) -o $(BUILD_DIR)/$(APP) ./cmd/$(APP);)
cp ./build/repo-health-checker ./build/healthcheck
clean:
rm -rf $(BUILD_DIR)/*

26
cmd/joj3/conf.go → cmd/joj3/conf/conf.go
View File

@ -1,4 +1,4 @@
package main
package conf
import (
"fmt"
@ -8,6 +8,7 @@ import (
"regexp"
"strings"
"github.com/go-git/go-git/v5"
"github.com/joint-online-judge/JOJ3/internal/stage"
"github.com/koding/multiconfig"
)
@ -17,6 +18,8 @@ type Conf struct {
SandboxToken string `default:""`
LogPath string `default:""`
OutputPath string `default:"joj3_result.json"`
GradingRepoName string `default:""`
SkipTeapot bool `default:"true"`
Stages []struct {
Name string
Group string
@ -73,6 +76,23 @@ type ConventionalCommit struct {
Footer string
}
func GetCommitMsg() (msg string, err error) {
r, err := git.PlainOpen(".")
if err != nil {
return
}
ref, err := r.Head()
if err != nil {
return
}
commit, err := r.CommitObject(ref.Hash())
if err != nil {
return
}
msg = commit.Message
return
}
func parseConventionalCommit(commit string) (*ConventionalCommit, error) {
re := regexp.MustCompile(`(?s)^(\w+)(\(([^)]+)\))?!?: (.+?)(\n\n(.+?))?(\n\n(.+))?$`)
matches := re.FindStringSubmatch(strings.TrimSpace(commit))
@ -107,7 +127,7 @@ func parseConfFile(path string) (conf Conf, err error) {
return
}
func parseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) {
func ParseMsg(confRoot, confName, msg string) (conf Conf, group string, err error) {
slog.Info("parse msg", "msg", msg)
conventionalCommit, err := parseConventionalCommit(msg)
if err != nil {
@ -142,7 +162,7 @@ func parseMsg(confRoot, confName, msg string) (conf Conf, group string, err erro
return
}
func listValidScopes(confRoot, confName, msg string) ([]string, error) {
func ListValidScopes(confRoot, confName, msg string) ([]string, error) {
conventionalCommit, err := parseConventionalCommit(msg)
if err != nil {
return []string{}, err

2
cmd/joj3/conf_test.go → cmd/joj3/conf/conf_test.go
View File

@ -1,4 +1,4 @@
package main
package conf
import (
"reflect"

140
cmd/joj3/main.go
View File

@ -1,97 +1,15 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"log/slog"
"os"
"github.com/joint-online-judge/JOJ3/internal/executors"
_ "github.com/joint-online-judge/JOJ3/internal/parsers"
"github.com/joint-online-judge/JOJ3/internal/stage"
"github.com/go-git/go-git/v5"
"github.com/jinzhu/copier"
"github.com/joint-online-judge/JOJ3/cmd/joj3/conf"
"github.com/joint-online-judge/JOJ3/cmd/joj3/stage"
"github.com/joint-online-judge/JOJ3/cmd/joj3/teapot"
)
func getCommitMsg() (msg string, err error) {
r, err := git.PlainOpen(".")
if err != nil {
return
}
ref, err := r.Head()
if err != nil {
return
}
commit, err := r.CommitObject(ref.Hash())
if err != nil {
return
}
msg = commit.Message
return
}
func generateStages(conf Conf, group string) ([]stage.Stage, error) {
stages := []stage.Stage{}
existNames := map[string]bool{}
for _, s := range conf.Stages {
if s.Group != "" && group != s.Group {
continue
}
_, ok := existNames[s.Name] // check for existence
if ok {
continue
}
existNames[s.Name] = true
var cmds []stage.Cmd
defaultCmd := s.Executor.With.Default
for _, optionalCmd := range s.Executor.With.Cases {
cmd := s.Executor.With.Default
err := copier.Copy(&cmd, &optionalCmd)
if err != nil {
slog.Error("generate stages", "error", err)
return stages, err
}
// since these 3 values are pointers, copier will always copy
// them, so we need to check them manually
if defaultCmd.Stdin != nil && optionalCmd.Stdin == nil {
cmd.Stdin = defaultCmd.Stdin
}
if defaultCmd.Stdout != nil && optionalCmd.Stdout == nil {
cmd.Stdout = defaultCmd.Stdout
}
if defaultCmd.Stderr != nil && optionalCmd.Stderr == nil {
cmd.Stderr = defaultCmd.Stderr
}
cmds = append(cmds, cmd)
}
if len(s.Executor.With.Cases) == 0 {
cmds = []stage.Cmd{defaultCmd}
}
stages = append(stages, stage.Stage{
Name: s.Name,
ExecutorName: s.Executor.Name,
ExecutorCmds: cmds,
ParserName: s.Parser.Name,
ParserConf: s.Parser.With,
})
}
slog.Debug("stages generated", "stages", stages)
return stages, nil
}
func outputResult(outputPath string, results []stage.StageResult) error {
slog.Info("output result start", "path", outputPath)
slog.Debug("output result start", "path", outputPath, "results", results)
content, err := json.Marshal(results)
if err != nil {
return err
}
return os.WriteFile(outputPath,
append(content, []byte("\n")...), 0o600)
}
var (
confRoot string
confName string
@ -107,61 +25,47 @@ func init() {
showVersion = flag.Bool("version", false, "print current version")
}
func mainImpl() error {
func main() {
if err := setupSlog(""); err != nil { // before conf is loaded
return err
slog.Error("setup slog", "error", err)
return
}
flag.Parse()
if *showVersion {
fmt.Println(Version)
return nil
return
}
slog.Info("start joj3", "version", Version)
if msg == "" {
var err error
msg, err = getCommitMsg()
msg, err = conf.GetCommitMsg()
if err != nil {
slog.Error("get commit msg", "error", err)
return err
return
}
}
conf, group, err := parseMsg(confRoot, confName, msg)
confObj, group, err := conf.ParseMsg(confRoot, confName, msg)
if err != nil {
slog.Error("parse msg", "error", err)
validScopes, scopeErr := listValidScopes(
validScopes, scopeErr := conf.ListValidScopes(
confRoot, confName, msg)
if scopeErr != nil {
slog.Error("list valid scopes", "error", scopeErr)
return scopeErr
return
}
slog.Info("hint: valid scopes in commit message", "scopes", validScopes)
return err
return
}
if err := setupSlog(conf.LogPath); err != nil { // after conf is loaded
return err
if err := setupSlog(confObj.LogPath); err != nil { // after conf is loaded
slog.Error("setup slog", "error", err)
return
}
executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken)
stages, err := generateStages(conf, group)
if err != nil {
slog.Error("generate stages", "error", err)
return err
if err := stage.Run(confObj, group); err != nil {
slog.Error("stage run", "error", err)
return
}
defer stage.Cleanup()
results, err := stage.Run(stages)
if err != nil {
slog.Error("run stages", "error", err)
return err
}
if err := outputResult(conf.OutputPath, results); err != nil {
slog.Error("output result", "error", err)
return err
}
return nil
}
func main() {
if err := mainImpl(); err != nil {
slog.Error("main exit", "error", err)
os.Exit(1)
if err := teapot.Run(confObj); err != nil {
slog.Error("teapot run", "error", err)
return
}
}

2
cmd/joj3/main_test.go
View File

@ -61,7 +61,7 @@ func readStageResults(t *testing.T, path string) []stage.StageResult {
return results
}
func TestMain(t *testing.T) {
func TestRun(t *testing.T) {
var tests []string
root := "../../tmp/submodules/JOJ3-examples/examples/"
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {

94
cmd/joj3/stage/main.go Normal file
View File

@ -0,0 +1,94 @@
package stage
import (
"encoding/json"
"log/slog"
"os"
"github.com/joint-online-judge/JOJ3/cmd/joj3/conf"
"github.com/joint-online-judge/JOJ3/internal/executors"
_ "github.com/joint-online-judge/JOJ3/internal/parsers"
"github.com/joint-online-judge/JOJ3/internal/stage"
"github.com/jinzhu/copier"
)
func generateStages(conf conf.Conf, group string) ([]stage.Stage, error) {
stages := []stage.Stage{}
existNames := map[string]bool{}
for _, s := range conf.Stages {
if s.Group != "" && group != s.Group {
continue
}
_, ok := existNames[s.Name] // check for existence
if ok {
continue
}
existNames[s.Name] = true
var cmds []stage.Cmd
defaultCmd := s.Executor.With.Default
for _, optionalCmd := range s.Executor.With.Cases {
cmd := s.Executor.With.Default
err := copier.Copy(&cmd, &optionalCmd)
if err != nil {
slog.Error("generate stages", "error", err)
return stages, err
}
// since these 3 values are pointers, copier will always copy
// them, so we need to check them manually
if defaultCmd.Stdin != nil && optionalCmd.Stdin == nil {
cmd.Stdin = defaultCmd.Stdin
}
if defaultCmd.Stdout != nil && optionalCmd.Stdout == nil {
cmd.Stdout = defaultCmd.Stdout
}
if defaultCmd.Stderr != nil && optionalCmd.Stderr == nil {
cmd.Stderr = defaultCmd.Stderr
}
cmds = append(cmds, cmd)
}
if len(s.Executor.With.Cases) == 0 {
cmds = []stage.Cmd{defaultCmd}
}
stages = append(stages, stage.Stage{
Name: s.Name,
ExecutorName: s.Executor.Name,
ExecutorCmds: cmds,
ParserName: s.Parser.Name,
ParserConf: s.Parser.With,
})
}
slog.Debug("stages generated", "stages", stages)
return stages, nil
}
func outputResult(outputPath string, results []stage.StageResult) error {
slog.Info("output result start", "path", outputPath)
slog.Debug("output result start", "path", outputPath, "results", results)
content, err := json.Marshal(results)
if err != nil {
return err
}
return os.WriteFile(outputPath,
append(content, []byte("\n")...), 0o600)
}
func Run(conf conf.Conf, group string) error {
executors.InitWithConf(conf.SandboxExecServer, conf.SandboxToken)
stages, err := generateStages(conf, group)
if err != nil {
slog.Error("generate stages", "error", err)
return err
}
defer stage.Cleanup()
results, err := stage.Run(stages)
if err != nil {
slog.Error("run stages", "error", err)
return err
}
if err := outputResult(conf.OutputPath, results); err != nil {
slog.Error("output result", "error", err)
return err
}
return nil
}

57
cmd/joj3/teapot/main.go Normal file
View File

@ -0,0 +1,57 @@
package teapot
import (
"fmt"
"log/slog"
"os"
"os/exec"
"strings"
"github.com/joint-online-judge/JOJ3/cmd/joj3/conf"
)
func Run(conf conf.Conf) error {
if conf.SkipTeapot {
return nil
}
os.Setenv("LOG_FILE_PATH", "/home/tt/.cache/joint-teapot-debug.log")
os.Setenv("_TYPER_STANDARD_TRACEBACK", "1")
envFilePath := "/home/tt/.config/teapot/teapot.env"
actor := os.Getenv("GITHUB_ACTOR")
repository := os.Getenv("GITHUB_REPOSITORY")
runNumber := os.Getenv("GITHUB_RUN_NUMBER")
if actor == "" || repository == "" || strings.Count(repository, "/") != 1 ||
runNumber == "" {
slog.Error("teapot env not set")
return fmt.Errorf("teapot env not set")
}
repoParts := strings.Split(repository, "/")
repoName := repoParts[1]
cmd := exec.Command("joint-teapot", "joj3-scoreboard",
envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName,
runNumber) // #nosec G204
output, err := cmd.CombinedOutput()
if err != nil {
slog.Error("running git command:", "err", err)
return err
}
slog.Info("joint-teapot joj3-scoreboard", "output", string(output))
cmd = exec.Command("joint-teapot", "joj3-failed-table",
envFilePath, conf.OutputPath, actor, conf.GradingRepoName, repoName,
runNumber) // #nosec G204
output, err = cmd.CombinedOutput()
if err != nil {
slog.Error("running git command:", "err", err)
return err
}
slog.Info("joint-teapot joj3-failed-table", "output", string(output))
cmd = exec.Command("joint-teapot", "joj3-create-result-issue",
envFilePath, conf.OutputPath, repoName, runNumber) // #nosec G204
output, err = cmd.CombinedOutput()
if err != nil {
slog.Error("running git command:", "err", err)
return err
}
slog.Info("joint-teapot joj3-create-result-issue", "output", string(output))
return nil
}

0
cmd/healthcheck/main.go → cmd/repo-health-checker/main.go
View File