From 4d594e1d875864b86de729591b047b912e9f60a0 Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Mon, 4 Mar 2024 00:05:26 -0500 Subject: [PATCH] refactor: copy go-judge cmd model codes --- cmd/joj3/main.go | 58 ++++++---- internal/executors/dummy/executor.go | 8 +- internal/executors/sandbox/convert.go | 59 +++------- internal/executors/sandbox/executor.go | 6 +- internal/parsers/dummy/parser.go | 3 +- internal/stage/executor.go | 6 +- internal/stage/model.go | 149 ++++++++++++++++++++++++- internal/stage/parser.go | 4 +- 8 files changed, 204 insertions(+), 89 deletions(-) diff --git a/cmd/joj3/main.go b/cmd/joj3/main.go index ea3b646..8c67a73 100644 --- a/cmd/joj3/main.go +++ b/cmd/joj3/main.go @@ -10,30 +10,40 @@ import ( func main() { tomlConfig := ` -[[stages]] -name = "stage 0" - [stages.executor] - name = "sandbox" - [stages.executor.with] - args = [ "ls" ] - env = [ "PATH=/usr/bin:/bin" ] - cpuLimit = 10_000_000_000 - memoryLimit = 104_857_600 - procLimit = 50 - copyOut = [ "stdout", "stderr" ] - [[stages.executor.with.files]] - content = "" - [[stages.executor.with.files]] - name = "stdout" - max = 4_096 - [[stages.executor.with.files]] - name = "stderr" - max = 4_096 - [stages.parser] - name = "dummy" - [stages.parser.with] - score = 100 - comment = "dummy comment for stage 0" + [[stages]] + name = "stage 0" + + [stages.executor] + name = "sandbox" + + [stages.executor.with] + args = [ "ls" ] + env = [ "PATH=/usr/bin:/bin" ] + cpuLimit = 10_000_000_000 + memoryLimit = 104_857_600 + procLimit = 50 + copyOut = [ "stdout", "stderr" ] + + [stages.executor.with.copyIn.test] + src = "/home/boyanzh/joint-online-judge/go-judge/go.mod" + + [[stages.executor.with.files]] + content = "" + + [[stages.executor.with.files]] + name = "stdout" + max = 4_096 + + [[stages.executor.with.files]] + name = "stderr" + max = 4_096 + + [stages.parser] + name = "dummy" + + [stages.parser.with] + score = 100 + comment = "dummy comment for stage 0" ` stages := stage.ParseStages(tomlConfig) results := stage.Run(stages) diff --git a/internal/executors/dummy/executor.go b/internal/executors/dummy/executor.go index be0b09a..dba763e 100644 --- a/internal/executors/dummy/executor.go +++ b/internal/executors/dummy/executor.go @@ -1,15 +1,15 @@ package dummy import ( - "github.com/criyle/go-judge/cmd/go-judge/model" + "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage" "github.com/criyle/go-judge/envexec" ) type Dummy struct{} -func (e *Dummy) Run(model.Cmd) (*model.Result, error) { - return &model.Result{ - Status: model.Status(envexec.StatusInvalid), +func (e *Dummy) Run(stage.Cmd) (*stage.Result, error) { + return &stage.Result{ + Status: stage.Status(envexec.StatusInvalid), ExitStatus: 0, Error: "I'm a dummy", Time: 0, diff --git a/internal/executors/sandbox/convert.go b/internal/executors/sandbox/convert.go index 0d147ee..6b9750c 100644 --- a/internal/executors/sandbox/convert.go +++ b/internal/executors/sandbox/convert.go @@ -3,12 +3,12 @@ package sandbox import ( "strings" - "github.com/criyle/go-judge/cmd/go-judge/model" + "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage" "github.com/criyle/go-judge/pb" ) // copied from https://github.com/criyle/go-judge/blob/master/cmd/go-judge-shell/grpc.go -func convertPBCmd(cmd []model.Cmd) []*pb.Request_CmdType { +func convertPBCmd(cmd []stage.Cmd) []*pb.Request_CmdType { var ret []*pb.Request_CmdType for _, c := range cmd { ret = append(ret, &pb.Request_CmdType{ @@ -36,7 +36,7 @@ func convertPBCmd(cmd []model.Cmd) []*pb.Request_CmdType { return ret } -func convertPBCopyIn(copyIn map[string]model.CmdFile) map[string]*pb.Request_File { +func convertPBCopyIn(copyIn map[string]stage.CmdFile) map[string]*pb.Request_File { rt := make(map[string]*pb.Request_File, len(copyIn)) for k, i := range copyIn { if i.Symlink != nil { @@ -63,7 +63,7 @@ func convertPBCopyOut(copyOut []string) []*pb.Request_CmdCopyOutFile { return rt } -func convertSymlink(copyIn map[string]model.CmdFile) map[string]string { +func convertSymlink(copyIn map[string]stage.CmdFile) map[string]string { ret := make(map[string]string) for k, v := range copyIn { if v.Symlink == nil { @@ -74,7 +74,7 @@ func convertSymlink(copyIn map[string]model.CmdFile) map[string]string { return ret } -func convertPBFiles(files []*model.CmdFile) []*pb.Request_File { +func convertPBFiles(files []*stage.CmdFile) []*pb.Request_File { var ret []*pb.Request_File for _, f := range files { if f == nil { @@ -86,7 +86,7 @@ func convertPBFiles(files []*model.CmdFile) []*pb.Request_File { return ret } -func convertPBFile(i model.CmdFile) *pb.Request_File { +func convertPBFile(i stage.CmdFile) *pb.Request_File { switch { case i.Src != nil: return &pb.Request_File{File: &pb.Request_File_Local{Local: &pb.Request_LocalFile{Src: *i.Src}}} @@ -105,29 +105,11 @@ func convertPBFile(i model.CmdFile) *pb.Request_File { return nil } -func convertPBPipeMapping(pm []model.PipeMap) []*pb.Request_PipeMap { - var ret []*pb.Request_PipeMap - for _, p := range pm { - ret = append(ret, &pb.Request_PipeMap{ - In: convertPBPipeIndex(p.In), - Out: convertPBPipeIndex(p.Out), - Name: p.Name, - Proxy: p.Proxy, - Max: uint64(p.Max), - }) - } - return ret -} - -func convertPBPipeIndex(pi model.PipeIndex) *pb.Request_PipeMap_PipeIndex { - return &pb.Request_PipeMap_PipeIndex{Index: int32(pi.Index), Fd: int32(pi.Fd)} -} - -func convertPBResult(res []*pb.Response_Result) []model.Result { - var ret []model.Result +func convertPBResult(res []*pb.Response_Result) []stage.Result { + var ret []stage.Result for _, r := range res { - ret = append(ret, model.Result{ - Status: model.Status(r.Status), + ret = append(ret, stage.Result{ + Status: stage.Status(r.Status), ExitStatus: int(r.ExitStatus), Error: r.Error, Time: r.Time, @@ -150,25 +132,12 @@ func convertFiles(buf map[string][]byte) map[string]string { return ret } -func convertPBRequest(req *model.Request) *pb.StreamRequest { - ret := &pb.StreamRequest{ - Request: &pb.StreamRequest_ExecRequest{ - ExecRequest: &pb.Request{ - RequestID: req.RequestID, - Cmd: convertPBCmd(req.Cmd), - PipeMapping: convertPBPipeMapping(req.PipeMapping), - }, - }, - } - return ret -} - -func convertPBFileError(fe []*pb.Response_FileError) []model.FileError { - var ret []model.FileError +func convertPBFileError(fe []*pb.Response_FileError) []stage.FileError { + var ret []stage.FileError for _, v := range fe { - ret = append(ret, model.FileError{ + ret = append(ret, stage.FileError{ Name: v.Name, - Type: model.FileErrorType(v.Type), + Type: stage.FileErrorType(v.Type), Message: v.Message, }) } diff --git a/internal/executors/sandbox/executor.go b/internal/executors/sandbox/executor.go index 24ee860..7d91066 100644 --- a/internal/executors/sandbox/executor.go +++ b/internal/executors/sandbox/executor.go @@ -5,7 +5,7 @@ import ( "fmt" "log/slog" - "github.com/criyle/go-judge/cmd/go-judge/model" + "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage" "github.com/criyle/go-judge/pb" ) @@ -13,9 +13,9 @@ type Sandbox struct { execClient pb.ExecutorClient } -func (e *Sandbox) Run(cmd model.Cmd) (*model.Result, error) { +func (e *Sandbox) Run(cmd stage.Cmd) (*stage.Result, error) { slog.Info("sandbox run", "cmd", cmd) - req := &pb.Request{Cmd: convertPBCmd([]model.Cmd{cmd})} + req := &pb.Request{Cmd: convertPBCmd([]stage.Cmd{cmd})} ret, err := e.execClient.Exec(context.TODO(), req) if err != nil { return nil, err diff --git a/internal/parsers/dummy/parser.go b/internal/parsers/dummy/parser.go index 5ee2619..e5f98c7 100644 --- a/internal/parsers/dummy/parser.go +++ b/internal/parsers/dummy/parser.go @@ -5,7 +5,6 @@ import ( "log/slog" "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage" - "github.com/criyle/go-judge/cmd/go-judge/model" "github.com/mitchellh/mapstructure" ) @@ -16,7 +15,7 @@ type Config struct { type Dummy struct{} -func (e *Dummy) Run(result *model.Result, configAny any) ( +func (e *Dummy) Run(result *stage.Result, configAny any) ( *stage.ParserResult, error, ) { var config Config diff --git a/internal/stage/executor.go b/internal/stage/executor.go index b1469c7..805246d 100644 --- a/internal/stage/executor.go +++ b/internal/stage/executor.go @@ -1,13 +1,9 @@ package stage -import ( - "github.com/criyle/go-judge/cmd/go-judge/model" -) - var executorMap = map[string]Executor{} type Executor interface { - Run(model.Cmd) (*model.Result, error) + Run(Cmd) (*Result, error) } func RegisterExecutor(name string, executor Executor) { diff --git a/internal/stage/model.go b/internal/stage/model.go index 7815747..61bf99f 100644 --- a/internal/stage/model.go +++ b/internal/stage/model.go @@ -1,14 +1,157 @@ package stage import ( - "github.com/criyle/go-judge/cmd/go-judge/model" + "fmt" + "strconv" + "time" + + "github.com/criyle/go-judge/envexec" ) +// copied from https://github.com/criyle/go-judge/blob/master/cmd/go-judge/model/model.go +// FileError defines the location, file name and the detailed message for a failed file operation +type FileError = envexec.FileError + +// FileErrorType defines the location that file operation fails +type FileErrorType = envexec.FileErrorType + +// CmdFile defines file from multiple source including local / memory / cached or pipe collector +type CmdFile struct { + Src *string `json:"src"` + Content *string `json:"content"` + FileID *string `json:"fileId"` + Name *string `json:"name"` + Max *int64 `json:"max"` + Symlink *string `json:"symlink"` + StreamIn bool `json:"streamIn"` + StreamOut bool `json:"streamOut"` + Pipe bool `json:"pipe"` +} + +// Cmd defines command and limits to start a program using in envexec +type Cmd struct { + Args []string `json:"args"` + Env []string `json:"env,omitempty"` + Files []*CmdFile `json:"files,omitempty"` + + CPULimit uint64 `json:"cpuLimit"` + RealCPULimit uint64 `json:"realCpuLimit"` + ClockLimit uint64 `json:"clockLimit"` + MemoryLimit uint64 `json:"memoryLimit"` + StackLimit uint64 `json:"stackLimit"` + ProcLimit uint64 `json:"procLimit"` + CPURateLimit uint64 `json:"cpuRateLimit"` + CPUSetLimit string `json:"cpuSetLimit"` + + CopyIn map[string]CmdFile `json:"copyIn"` + + CopyOut []string `json:"copyOut"` + CopyOutCached []string `json:"copyOutCached"` + CopyOutMax uint64 `json:"copyOutMax"` + CopyOutDir string `json:"copyOutDir"` + + TTY bool `json:"tty,omitempty"` + StrictMemoryLimit bool `json:"strictMemoryLimit"` + DataSegmentLimit bool `json:"dataSegmentLimit"` + AddressSpaceLimit bool `json:"addressSpaceLimit"` +} + +// PipeIndex defines indexing for a pipe fd +type PipeIndex struct { + Index int `json:"index"` + Fd int `json:"fd"` +} + +// PipeMap defines in / out pipe for multiple program +type PipeMap struct { + In PipeIndex `json:"in"` + Out PipeIndex `json:"out"` + Name string `json:"name"` + Max int64 `json:"max"` + Proxy bool `json:"proxy"` +} + +// Request defines single worker request +type Request struct { + RequestID string `json:"requestId"` + Cmd []Cmd `json:"cmd"` + PipeMapping []PipeMap `json:"pipeMapping"` +} + +// Status offers JSON marshal for envexec.Status +type Status envexec.Status + +// String converts status to string +func (s Status) String() string { + return envexec.Status(s).String() +} + +// MarshalJSON convert status into string +func (s Status) MarshalJSON() ([]byte, error) { + return []byte("\"" + envexec.Status(s).String() + "\""), nil +} + +// UnmarshalJSON convert string into status +func (s *Status) UnmarshalJSON(b []byte) error { + str := string(b) + v, err := envexec.StringToStatus(str) + if err != nil { + return err + } + *s = Status(v) + return nil +} + +// Result defines single command result +type Result struct { + Status Status `json:"status"` + ExitStatus int `json:"exitStatus"` + Error string `json:"error,omitempty"` + Time uint64 `json:"time"` + Memory uint64 `json:"memory"` + RunTime uint64 `json:"runTime"` + Files map[string]string `json:"files,omitempty"` + FileIDs map[string]string `json:"fileIds,omitempty"` + FileError []FileError `json:"fileError,omitempty"` + + files []string + Buffs map[string][]byte `json:"-"` +} + +func (r Result) String() string { + type Result struct { + Status Status + ExitStatus int + Error string + Time time.Duration + RunTime time.Duration + Memory envexec.Size + Files map[string]string + FileIDs map[string]string + FileError []FileError + } + d := Result{ + Status: r.Status, + ExitStatus: r.ExitStatus, + Error: r.Error, + Time: time.Duration(r.Time), + RunTime: time.Duration(r.RunTime), + Memory: envexec.Size(r.Memory), + Files: make(map[string]string), + FileIDs: r.FileIDs, + FileError: r.FileError, + } + for k, v := range r.Files { + d.Files[k] = "len:" + strconv.Itoa(len(v)) + } + return fmt.Sprintf("%+v", d) +} + type Stage struct { Name string ExecutorName string Executor Executor - ExecutorCmd model.Cmd + ExecutorCmd Cmd ParserName string Parser Parser ParserConfig any @@ -29,7 +172,7 @@ type StagesConfig struct { Name string Executor struct { Name string - With model.Cmd + With Cmd } Parser struct { Name string diff --git a/internal/stage/parser.go b/internal/stage/parser.go index 6fbd9b3..5cbc45d 100644 --- a/internal/stage/parser.go +++ b/internal/stage/parser.go @@ -1,11 +1,9 @@ package stage -import "github.com/criyle/go-judge/cmd/go-judge/model" - var parserMap = map[string]Parser{} type Parser interface { - Run(*model.Result, any) (*ParserResult, error) + Run(*Result, any) (*ParserResult, error) } func RegisterParser(name string, parser Parser) {