feat: multiple cmds each stage

README.md

@@ -14,7 +14,7 @@ go build -o ./build/joj3 ./cmd/joj3
 + cd ./_example/simple
 + ./../../build/joj3
 + cat ./joj3_result.json
-[{"Name":"compile","Score":100,"Comment":"compile done, executor status: run time: 239591301 ns, memory: 57176064 bytes"},{"Name":"run","Score":100,"Comment":"executor status: run time: 1839200 ns, memory: 16826368 bytes"}]
+[{"Name":"compile","ParserResults":[{"Score":100,"Comment":"compile done, executor status: run time: 274901520 ns, memory: 57880576 bytes"}]},{"Name":"run","ParserResults":[{"Score":100,"Comment":"executor status: run time: 2343025 ns, memory: 13225984 bytes"},{"Score":0,"Comment":"executor status: run time: 2071433 ns, memory: 14544896 bytes"}]}]
 + rm -f ./joj3_result.json
 + cd -
 ```

_example/simple/2.stdin

@@ -0,0 +1 @@
+1024 2048

_example/simple/2.stdout

@@ -0,0 +1 @@
+3072

_example/simple/conf.toml

@@ -3,7 +3,7 @@ logLevel = 8
 name = "compile"
 [stages.executor]
 name = "sandbox"
-[stages.executor.with]
+[stages.executor.with.default]
 args = ["/usr/bin/g++", "a.cc", "-o", "a"]
 env = ["PATH=/usr/bin:/bin"]
 cpuLimit = 10_000_000_000
@@ -12,12 +12,12 @@ procLimit = 50
 copyInCwd = true
 copyOut = ["stdout", "stderr"]
 copyOutCached = ["a"]
-[[stages.executor.with.files]]
+[[stages.executor.with.default.files]]
 content = ""
-[[stages.executor.with.files]]
+[[stages.executor.with.default.files]]
 name = "stdout"
 max = 4_096
-[[stages.executor.with.files]]
+[[stages.executor.with.default.files]]
 name = "stderr"
 max = 4_096
 [stages.parser]
@@ -29,25 +29,38 @@ comment = "compile done"
 name = "run"
 [stages.executor]
 name = "sandbox"
-[stages.executor.with]
+[stages.executor.with.default]
 args = ["./a"]
 env = ["PATH=/usr/bin:/bin"]
 cpuLimit = 10_000_000_000
 memoryLimit = 104_857_600
 procLimit = 50
 copyOut = ["stdout", "stderr"]
-[stages.executor.with.copyInCached]
+[stages.executor.with.default.copyInCached]
 a = "a"
-[[stages.executor.with.files]]
+[[stages.executor.with.cases]]
+[[stages.executor.with.cases.files]]
 src = "1.stdin"
-[[stages.executor.with.files]]
+[[stages.executor.with.cases.files]]
 name = "stdout"
 max = 4_096
-[[stages.executor.with.files]]
+[[stages.executor.with.cases.files]]
+name = "stderr"
+max = 4_096
+[[stages.executor.with.cases]]
+[[stages.executor.with.cases.files]]
+src = "1.stdin"
+[[stages.executor.with.cases.files]]
+name = "stdout"
+max = 4_096
+[[stages.executor.with.cases.files]]
 name = "stderr"
 max = 4_096
 [stages.parser]
 name = "diff"
-[stages.parser.with]
+[[stages.parser.with.cases]]
 score = 100
 stdoutPath = "1.stdout"
+[[stages.parser.with.cases]]
+score = 100
+stdoutPath = "2.stdout"

cmd/joj3/conf.go

@@ -9,7 +9,10 @@ type Conf struct {
 		Name     string
 		Executor struct {
 			Name string
-			With stage.Cmd
+			With struct {
+				Default stage.Cmd
+				Cases   []OptionalCmd
+			}
 		}
 		Parser struct {
 			Name string
@@ -17,3 +20,32 @@ type Conf struct {
 		}
 	}
 }
+
+type OptionalCmd struct {
+	Args  *[]string
+	Env   *[]string
+	Files *[]*stage.CmdFile
+
+	CPULimit     *uint64
+	RealCPULimit *uint64
+	ClockLimit   *uint64
+	MemoryLimit  *uint64
+	StackLimit   *uint64
+	ProcLimit    *uint64
+	CPURateLimit *uint64
+	CPUSetLimit  *string
+
+	CopyIn       *map[string]stage.CmdFile
+	CopyInCached *map[string]string
+	CopyInCwd    *bool
+
+	CopyOut       *[]string
+	CopyOutCached *[]string
+	CopyOutMax    *uint64
+	CopyOutDir    *string
+
+	TTY               *bool
+	StrictMemoryLimit *bool
+	DataSegmentLimit  *bool
+	AddressSpaceLimit *bool
+}

cmd/joj3/main.go

@@ -10,15 +10,18 @@ import (
 	_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers"
 	"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
 
-	"github.com/pelletier/go-toml/v2"
+	// "github.com/pelletier/go-toml/v2" may panic on some error
+	"github.com/BurntSushi/toml"
+	"github.com/jinzhu/copier"
 )
 
-func parseConfFile(tomlPath *string) (Conf, []stage.Stage) {
+func parseConfFile(tomlPath *string) Conf {
 	tomlConfig, err := os.ReadFile(*tomlPath)
 	if err != nil {
 		slog.Error("read toml config", "error", err)
 		os.Exit(1)
 	}
+	// fill in default value of config file
 	conf := Conf{
 		LogLevel:   0,
 		OutputPath: "joj3_result.json",
@@ -28,17 +31,7 @@ func parseConfFile(tomlPath *string) (Conf, []stage.Stage) {
 		slog.Error("parse stages config", "error", err)
 		os.Exit(1)
 	}
-	stages := []stage.Stage{}
+	return conf
-	for _, s := range conf.Stages {
-		stages = append(stages, stage.Stage{
-			Name:         s.Name,
-			ExecutorName: s.Executor.Name,
-			ExecutorCmd:  s.Executor.With,
-			ParserName:   s.Parser.Name,
-			ParserConfig: s.Parser.With,
-		})
-	}
-	return conf, stages
 }
 
 func setupSlog(conf Conf) {
@@ -50,6 +43,30 @@ func setupSlog(conf Conf) {
 	slog.SetDefault(logger)
 }
 
+func generateStages(conf Conf) []stage.Stage {
+	stages := []stage.Stage{}
+	for _, s := range conf.Stages {
+		var cmds []stage.Cmd
+		for _, optionalCmd := range s.Executor.With.Cases {
+			cmd := s.Executor.With.Default
+			copier.Copy(&cmd, &optionalCmd)
+			cmds = append(cmds, cmd)
+		}
+		if len(s.Executor.With.Cases) == 0 {
+			cmds = append(cmds, s.Executor.With.Default)
+		}
+		slog.Info("parse stages config", "cmds", cmds)
+		stages = append(stages, stage.Stage{
+			Name:         s.Name,
+			ExecutorName: s.Executor.Name,
+			ExecutorCmds: cmds,
+			ParserName:   s.Parser.Name,
+			ParserConfig: s.Parser.With,
+		})
+	}
+	return stages
+}
+
 func outputResult(conf Conf, results []stage.StageResult) error {
 	content, err := json.Marshal(results)
 	if err != nil {
@@ -62,8 +79,9 @@ func outputResult(conf Conf, results []stage.StageResult) error {
 func main() {
 	tomlPath := flag.String("c", "conf.toml", "file path of the toml config")
 	flag.Parse()
-	conf, stages := parseConfFile(tomlPath)
+	conf := parseConfFile(tomlPath)
 	setupSlog(conf)
+	stages := generateStages(conf)
 	defer stage.Cleanup()
 	results := stage.Run(stages)
 	err := outputResult(conf, results)

go.mod

@@ -3,9 +3,9 @@ module focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3
 go 1.22.0
 
 require (
+	github.com/BurntSushi/toml v1.3.2
 	github.com/criyle/go-judge v1.8.2
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/pelletier/go-toml/v2 v2.1.1
 	google.golang.org/grpc v1.62.0
 )
 
@@ -13,6 +13,7 @@ require (
 	github.com/creack/pty v1.1.21 // indirect
 	github.com/criyle/go-sandbox v0.10.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/jinzhu/copier v0.4.0
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect

go.sum

@@ -1,31 +1,21 @@
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
 github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/criyle/go-judge v1.8.2 h1:dGjLqJRBifqsLVZj1scr23zdM4wPe98HTIVgYzPuRxA=
 github.com/criyle/go-judge v1.8.2/go.mod h1:3RgsMp21D+UvXzkpOGsVFbLe2T2Lwk8jPEmCntQrvHQ=
 github.com/criyle/go-sandbox v0.10.1 h1:z9Il/UXQwKEvIwdr1wVheWWWAqGWtdTItBmEsWqFqT4=
 github.com/criyle/go-sandbox v0.10.1/go.mod h1:ivPw/HEh5unxVRlXJxCgkgTCuy+cxTkQDX7D2XQf/kg=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
@@ -43,7 +33,3 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/executors/dummy/executor.go

@@ -7,8 +7,10 @@ import (
 
 type Dummy struct{}
 
-func (e *Dummy) Run(stage.Cmd) (*stage.ExecutorResult, error) {
+func (e *Dummy) Run(cmds []stage.Cmd) ([]stage.ExecutorResult, error) {
-	return &stage.ExecutorResult{
+	var res []stage.ExecutorResult
+	for range cmds {
+		res = append(res, stage.ExecutorResult{
 			Status:     stage.Status(envexec.StatusInvalid),
 			ExitStatus: 0,
 			Error:      "I'm a dummy",
@@ -17,7 +19,9 @@ func (e *Dummy) Run(stage.Cmd) (*stage.ExecutorResult, error) {
 			RunTime:    0,
 			Files:      map[string]string{},
 			FileIDs:    map[string]string{},
-	}, nil
+		})
+	}
+	return res, nil
 }
 
 func (e *Dummy) Cleanup() error {

internal/executors/sandbox/executor.go

@@ -14,7 +14,10 @@ type Sandbox struct {
 	cachedMap  map[string]string
 }
 
-func (e *Sandbox) Run(cmd stage.Cmd) (*stage.ExecutorResult, error) {
+func (e *Sandbox) Run(cmds []stage.Cmd) ([]stage.ExecutorResult, error) {
+	// cannot use range loop since we need to change the value
+	for i := 0; i < len(cmds); i++ {
+		cmd := &cmds[i]
 		if cmd.CopyIn == nil {
 			cmd.CopyIn = make(map[string]stage.CmdFile)
 		}
@@ -23,7 +26,8 @@ func (e *Sandbox) Run(cmd stage.Cmd) (*stage.ExecutorResult, error) {
 				cmd.CopyIn[k] = stage.CmdFile{FileID: &fileID}
 			}
 		}
-	pbReq := &pb.Request{Cmd: convertPBCmd([]stage.Cmd{cmd})}
+	}
+	pbReq := &pb.Request{Cmd: convertPBCmd(cmds)}
 	pbRet, err := e.execClient.Exec(context.TODO(), pbReq)
 	if err != nil {
 		return nil, err
@@ -31,11 +35,13 @@ func (e *Sandbox) Run(cmd stage.Cmd) (*stage.ExecutorResult, error) {
 	if pbRet.Error != "" {
 		return nil, fmt.Errorf("sandbox execute error: %s", pbRet.Error)
 	}
-	executorRes := &convertPBResult(pbRet.Results)[0]
+	results := convertPBResult(pbRet.Results)
-	for fileName, fileID := range executorRes.FileIDs {
+	for _, result := range results {
+		for fileName, fileID := range result.FileIDs {
 			e.cachedMap[fileName] = fileID
 		}
-	return executorRes, nil
+	}
+	return results, nil
 }
 
 func (e *Sandbox) Cleanup() error {

internal/parsers/diff/parser.go

@@ -8,33 +8,43 @@ import (
 )
 
 type Config struct {
+	Cases []struct {
 		Score      int
 		StdoutPath string
 	}
+}
 
 type Diff struct{}
 
-func (e *Diff) Run(result *stage.ExecutorResult, configAny any) (
+func (e *Diff) Run(results []stage.ExecutorResult, configAny any) (
-	*stage.ParserResult, error,
+	[]stage.ParserResult, error,
 ) {
 	config, err := stage.DecodeConfig[Config](configAny)
 	if err != nil {
 		return nil, err
 	}
+	if len(config.Cases) != len(results) {
+		return nil, fmt.Errorf("cases number not match")
+	}
+	var res []stage.ParserResult
+	for i, caseConfig := range config.Cases {
+		result := results[i]
 		score := 0
-	stdout, err := os.ReadFile(config.StdoutPath)
+		stdout, err := os.ReadFile(caseConfig.StdoutPath)
 		if err != nil {
 			return nil, err
 		}
 		// TODO: more compare strategies
 		if string(stdout) == result.Files["stdout"] {
-		score = config.Score
+			score = caseConfig.Score
 		}
-	return &stage.ParserResult{
+		res = append(res, stage.ParserResult{
 			Score: score,
 			Comment: fmt.Sprintf(
 				"executor status: run time: %d ns, memory: %d bytes",
 				result.RunTime, result.Memory,
 			),
-	}, nil
+		})
+	}
+	return res, nil
 }

internal/parsers/dummy/parser.go

@@ -13,18 +13,22 @@ type Config struct {
 
 type Dummy struct{}
 
-func (e *Dummy) Run(result *stage.ExecutorResult, configAny any) (
+func (e *Dummy) Run(results []stage.ExecutorResult, configAny any) (
-	*stage.ParserResult, error,
+	[]stage.ParserResult, error,
 ) {
 	config, err := stage.DecodeConfig[Config](configAny)
 	if err != nil {
 		return nil, err
 	}
-	return &stage.ParserResult{
+	var res []stage.ParserResult
+	for _, result := range results {
+		res = append(res, stage.ParserResult{
 			Score: config.Score,
 			Comment: fmt.Sprintf(
 				"%s, executor status: run time: %d ns, memory: %d bytes",
 				config.Comment, result.RunTime, result.Memory,
 			),
-	}, nil
+		})
+	}
+	return res, nil
 }

internal/stage/executor.go

@@ -3,7 +3,7 @@ package stage
 var executorMap = map[string]Executor{}
 
 type Executor interface {
-	Run(Cmd) (*ExecutorResult, error)
+	Run([]Cmd) ([]ExecutorResult, error)
 	Cleanup() error
 }
 

internal/stage/model.go

@@ -151,7 +151,7 @@ func (r ExecutorResult) String() string {
 type Stage struct {
 	Name         string
 	ExecutorName string
-	ExecutorCmd  Cmd
+	ExecutorCmds []Cmd
 	ParserName   string
 	ParserConfig any
 }
@@ -163,5 +163,5 @@ type ParserResult struct {
 
 type StageResult struct {
 	Name          string
-	*ParserResult
+	ParserResults []ParserResult
 }

internal/stage/parser.go

@@ -3,7 +3,7 @@ package stage
 var parserMap = map[string]Parser{}
 
 type Parser interface {
-	Run(*ExecutorResult, any) (*ParserResult, error)
+	Run([]ExecutorResult, any) ([]ParserResult, error)
 }
 
 func RegisterParser(name string, parser Parser) {

internal/stage/run.go

@@ -5,31 +5,31 @@ import (
 )
 
 func Run(stages []Stage) []StageResult {
-	var parserResults []StageResult
+	var stageResults []StageResult
 	for _, stage := range stages {
 		slog.Info("stage start", "name", stage.Name)
-		slog.Info("executor run start", "cmd", stage.ExecutorCmd)
+		slog.Info("executor run start", "cmds", stage.ExecutorCmds)
 		executor := executorMap[stage.ExecutorName]
-		executorResult, err := executor.Run(stage.ExecutorCmd)
+		executorResults, err := executor.Run(stage.ExecutorCmds)
 		if err != nil {
 			slog.Error("executor run error", "name", stage.ExecutorName, "error", err)
 			break
 		}
-		slog.Info("executor run done", "result", executorResult)
+		slog.Info("executor run done", "results", executorResults)
 		slog.Info("parser run start", "config", stage.ParserConfig)
 		parser := parserMap[stage.ParserName]
-		parserResult, err := parser.Run(executorResult, stage.ParserConfig)
+		parserResults, err := parser.Run(executorResults, stage.ParserConfig)
 		if err != nil {
 			slog.Error("parser run error", "name", stage.ExecutorName, "error", err)
 			break
 		}
-		slog.Info("parser run done", "result", parserResult)
+		slog.Info("parser run done", "results", parserResults)
-		parserResults = append(parserResults, StageResult{
+		stageResults = append(stageResults, StageResult{
 			Name:          stage.Name,
-			ParserResult: parserResult,
+			ParserResults: parserResults,
 		})
 	}
-	return parserResults
+	return stageResults
 }
 
 func Cleanup() {