JOJ/JOJ3
24
5
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: run single command in sandbox

Browse Source
This commit is contained in:
张泊明518370910136 2024-03-03 18:02:51 -05:00
parent d1317cde79
commit d5a104e7c4
Signed by: 张泊明518370910136
GPG Key ID: D47306D7062CDA9D
16 changed files with 384 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -1,10 +1,16 @@
# JOJ3
In order to register sandbox executor, you need to run go-judge before running this program.
```bash
$ make clean && make && ./build/joj3
rm -rf ./build/*
rm -rf *.out
go build -o ./build/joj3 ./cmd/joj3
stage 0: score: 100, comment: dummy comment for stage 0
stage 1: score: 101, comment: dummy comment for stage 1
2024/03/03 18:01:11 INFO stage start name="stage 0"
2024/03/03 18:01:11 INFO sandbox run cmd="{Args:[ls] Env:[PATH=/usr/bin:/bin] Files:[0xc0000aa340 0xc0000aa380 0xc0000aa3c0] CPULimit:10000000000 RealCPULimit:0 ClockLimit:0 MemoryLimit:104857600 StackLimit:0 ProcLimit:50 CPURateLimit:0 CPUSetLimit: CopyIn:map[] CopyOut:[stdout stderr] CopyOutCached:[] CopyOutMax:0 CopyOutDir: TTY:false StrictMemoryLimit:false DataSegmentLimit:false AddressSpaceLimit:false}"
2024/03/03 18:01:11 INFO sandbox run ret="results:{status:Accepted time:1162000 runTime:3847400 memory:14929920 files:{key:\"stderr\" value:\"\"} files:{key:\"stdout\" value:\"stderr\\nstdout\\n\"}}"
2024/03/03 18:01:11 INFO executor done result="{Status:Accepted ExitStatus:0 Error: Time:1.162ms RunTime:3.8474ms Memory:14.2 MiB Files:map[stderr:len:0 stdout:len:14] FileIDs:map[] FileError:[]}"
2024/03/03 18:01:11 INFO parser done result="&{Score:100 Comment:dummy comment for stage 0, executor status: run time: 3847400 ns, memory: 14929920 bytes}"
stage 0: score: 100, comment: dummy comment for stage 0, executor status: run time: 3847400 ns, memory: 14929920 bytes
```

46
cmd/joj3/main.go
View File

@ -10,28 +10,30 @@ import (
func main() {
tomlConfig := `
[[stages]]
name = "stage 0"
[stages.executor]
name = "dummy"
[stages.executor.with]
args = [ "/usr/bin/cat", "/dev/null" ]
[stages.parser]
name = "dummy"
[stages.parser.with]
score = 100
comment = "dummy comment for stage 0"
[[stages]]
name = "stage 1"
[stages.executor]
name = "dummy"
[stages.executor.with]
args = [ "/usr/bin/cat", "/dev/null" ]
[stages.parser]
name = "dummy"
[stages.parser.with]
score = 101
comment = "dummy comment for stage 1"
[[stages]]
name = "stage 0"
[stages.executor]
name = "sandbox"
[stages.executor.with]
args = [ "ls" ]
env = [ "PATH=/usr/bin:/bin" ]
cpuLimit = 10_000_000_000
memoryLimit = 104_857_600
procLimit = 50
copyOut = [ "stdout", "stderr" ]
[[stages.executor.with.files]]
content = ""
[[stages.executor.with.files]]
name = "stdout"
max = 4_096
[[stages.executor.with.files]]
name = "stderr"
max = 4_096
[stages.parser]
name = "dummy"
[stages.parser.with]
score = 100
comment = "dummy comment for stage 0"
`
stages := stage.ParseStages(tomlConfig)
results := stage.Run(stages)

6
go.mod
View File

@ -5,9 +5,15 @@ go 1.22.0
require (
github.com/criyle/go-judge v1.8.1
github.com/mitchellh/mapstructure v1.5.0
google.golang.org/grpc v1.61.0
)
require (
github.com/golang/protobuf v1.5.3 // indirect
golang.org/x/net v0.20.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect
google.golang.org/protobuf v1.32.0 // indirect
github.com/creack/pty v1.1.21 // indirect
github.com/criyle/go-sandbox v0.10.1 // indirect
github.com/pelletier/go-toml/v2 v2.1.1

17
go.sum
View File

@ -7,6 +7,10 @@ github.com/criyle/go-sandbox v0.10.1/go.mod h1:ivPw/HEh5unxVRlXJxCgkgTCuy+cxTkQD
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
@ -20,10 +24,23 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe h1:bQnxqljG/wqi4NTXu2+DJ3n7APcEA882QZ1JvhQAq9o=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s=
google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0=
google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

1
internal/executors/all.go
View File

@ -2,6 +2,7 @@ package executors
import (
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/dummy"
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/executors/sandbox"
)
// this file does nothing but imports to ensure all the init() functions

6
internal/executors/dummy/executor.go
View File

@ -7,8 +7,8 @@ import (
type Dummy struct{}
func (e *Dummy) Run(model.Cmd) model.Result {
return model.Result{
func (e *Dummy) Run(model.Cmd) (*model.Result, error) {
return &model.Result{
Status: model.Status(envexec.StatusInvalid),
ExitStatus: 0,
Error: "I'm a dummy",
@ -17,5 +17,5 @@ func (e *Dummy) Run(model.Cmd) model.Result {
RunTime: 0,
Files: map[string]string{},
FileIDs: map[string]string{},
}
}, nil
}

176
internal/executors/sandbox/convert.go Normal file
View File

@ -0,0 +1,176 @@
package sandbox
import (
"strings"
"github.com/criyle/go-judge/cmd/go-judge/model"
"github.com/criyle/go-judge/pb"
)
// copied from https://github.com/criyle/go-judge/blob/master/cmd/go-judge-shell/grpc.go
func convertPBCmd(cmd []model.Cmd) []*pb.Request_CmdType {
var ret []*pb.Request_CmdType
for _, c := range cmd {
ret = append(ret, &pb.Request_CmdType{
Args: c.Args,
Env: c.Env,
Tty: c.TTY,
Files: convertPBFiles(c.Files),
CpuTimeLimit: c.CPULimit,
ClockTimeLimit: c.ClockLimit,
MemoryLimit: c.MemoryLimit,
StackLimit: c.StackLimit,
ProcLimit: c.ProcLimit,
CpuRateLimit: c.CPURateLimit,
CpuSetLimit: c.CPUSetLimit,
DataSegmentLimit: c.DataSegmentLimit,
AddressSpaceLimit: c.AddressSpaceLimit,
CopyIn: convertPBCopyIn(c.CopyIn),
CopyOut: convertPBCopyOut(c.CopyOut),
CopyOutCached: convertPBCopyOut(c.CopyOutCached),
CopyOutMax: c.CopyOutMax,
CopyOutDir: c.CopyOutDir,
Symlinks: convertSymlink(c.CopyIn),
})
}
return ret
}
func convertPBCopyIn(copyIn map[string]model.CmdFile) map[string]*pb.Request_File {
rt := make(map[string]*pb.Request_File, len(copyIn))
for k, i := range copyIn {
if i.Symlink != nil {
continue
}
rt[k] = convertPBFile(i)
}
return rt
}
func convertPBCopyOut(copyOut []string) []*pb.Request_CmdCopyOutFile {
rt := make([]*pb.Request_CmdCopyOutFile, 0)
for _, n := range copyOut {
optional := false
if strings.HasSuffix(n, "?") {
optional = true
n = strings.TrimSuffix(n, "?")
}
rt = append(rt, &pb.Request_CmdCopyOutFile{
Name: n,
Optional: optional,
})
}
return rt
}
func convertSymlink(copyIn map[string]model.CmdFile) map[string]string {
ret := make(map[string]string)
for k, v := range copyIn {
if v.Symlink == nil {
continue
}
ret[k] = *v.Symlink
}
return ret
}
func convertPBFiles(files []*model.CmdFile) []*pb.Request_File {
var ret []*pb.Request_File
for _, f := range files {
if f == nil {
ret = append(ret, nil)
} else {
ret = append(ret, convertPBFile(*f))
}
}
return ret
}
func convertPBFile(i model.CmdFile) *pb.Request_File {
switch {
case i.Src != nil:
return &pb.Request_File{File: &pb.Request_File_Local{Local: &pb.Request_LocalFile{Src: *i.Src}}}
case i.Content != nil:
s := strToBytes(*i.Content)
return &pb.Request_File{File: &pb.Request_File_Memory{Memory: &pb.Request_MemoryFile{Content: s}}}
case i.FileID != nil:
return &pb.Request_File{File: &pb.Request_File_Cached{Cached: &pb.Request_CachedFile{FileID: *i.FileID}}}
case i.Name != nil && i.Max != nil:
return &pb.Request_File{File: &pb.Request_File_Pipe{Pipe: &pb.Request_PipeCollector{Name: *i.Name, Max: *i.Max, Pipe: i.Pipe}}}
case i.StreamIn:
return &pb.Request_File{File: &pb.Request_File_StreamIn{}}
case i.StreamOut:
return &pb.Request_File{File: &pb.Request_File_StreamOut{}}
}
return nil
}
func convertPBPipeMapping(pm []model.PipeMap) []*pb.Request_PipeMap {
var ret []*pb.Request_PipeMap
for _, p := range pm {
ret = append(ret, &pb.Request_PipeMap{
In: convertPBPipeIndex(p.In),
Out: convertPBPipeIndex(p.Out),
Name: p.Name,
Proxy: p.Proxy,
Max: uint64(p.Max),
})
}
return ret
}
func convertPBPipeIndex(pi model.PipeIndex) *pb.Request_PipeMap_PipeIndex {
return &pb.Request_PipeMap_PipeIndex{Index: int32(pi.Index), Fd: int32(pi.Fd)}
}
func convertPBResult(res []*pb.Response_Result) []model.Result {
var ret []model.Result
for _, r := range res {
ret = append(ret, model.Result{
Status: model.Status(r.Status),
ExitStatus: int(r.ExitStatus),
Error: r.Error,
Time: r.Time,
RunTime: r.RunTime,
Memory: r.Memory,
Files: convertFiles(r.Files),
Buffs: r.Files,
FileIDs: r.FileIDs,
FileError: convertPBFileError(r.FileError),
})
}
return ret
}
func convertFiles(buf map[string][]byte) map[string]string {
ret := make(map[string]string, len(buf))
for k, v := range buf {
ret[k] = byteArrayToString(v)
}
return ret
}
func convertPBRequest(req *model.Request) *pb.StreamRequest {
ret := &pb.StreamRequest{
Request: &pb.StreamRequest_ExecRequest{
ExecRequest: &pb.Request{
RequestID: req.RequestID,
Cmd: convertPBCmd(req.Cmd),
PipeMapping: convertPBPipeMapping(req.PipeMapping),
},
},
}
return ret
}
func convertPBFileError(fe []*pb.Response_FileError) []model.FileError {
var ret []model.FileError
for _, v := range fe {
ret = append(ret, model.FileError{
Name: v.Name,
Type: model.FileErrorType(v.Type),
Message: v.Message,
})
}
return ret
}

28
internal/executors/sandbox/executor.go Normal file
View File

@ -0,0 +1,28 @@
package sandbox
import (
"context"
"fmt"
"log/slog"
"github.com/criyle/go-judge/cmd/go-judge/model"
"github.com/criyle/go-judge/pb"
)
type Sandbox struct {
execClient pb.ExecutorClient
}
func (e *Sandbox) Run(cmd model.Cmd) (*model.Result, error) {
slog.Info("sandbox run", "cmd", cmd)
req := &pb.Request{Cmd: convertPBCmd([]model.Cmd{cmd})}
ret, err := e.execClient.Exec(context.TODO(), req)
if err != nil {
return nil, err
}
if ret.Error != "" {
return nil, fmt.Errorf("compile error: %s", ret.Error)
}
slog.Info("sandbox run", "ret", ret)
return &convertPBResult(ret.Results)[0], nil
}

53
internal/executors/sandbox/grpc.go Normal file
View File

@ -0,0 +1,53 @@
package sandbox
import (
"context"
"log/slog"
"os"
"github.com/criyle/go-judge/pb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
)
// copied from https://github.com/criyle/go-judger-demo/blob/master/apigateway/main.go
func createExecClient(execServer, token string) pb.ExecutorClient {
conn, err := createGRPCConnection(execServer, token)
if err != nil {
slog.Error("gRPC connection", "error", err)
os.Exit(1)
}
return pb.NewExecutorClient(conn)
}
func createGRPCConnection(addr, token string) (*grpc.ClientConn, error) {
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
}
if token != "" {
opts = append(opts, grpc.WithPerRPCCredentials(newTokenAuth(token)))
}
return grpc.Dial(addr, opts...)
}
type tokenAuth struct {
token string
}
func newTokenAuth(token string) credentials.PerRPCCredentials {
return &tokenAuth{token: token}
}
// Return value is mapped to request headers.
func (t *tokenAuth) GetRequestMetadata(ctx context.Context, in ...string) (
map[string]string, error,
) {
return map[string]string{
"authorization": "Bearer " + t.token,
}, nil
}
func (*tokenAuth) RequireTransportSecurity() bool {
return false
}

14
internal/executors/sandbox/meta.go Normal file
View File

@ -0,0 +1,14 @@
package sandbox
import (
"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
)
var name = "sandbox"
func init() {
stage.RegisterExecutor(name, &Sandbox{
// TODO: read from config
execClient: createExecClient("localhost:5051", ""),
})
}

12
internal/executors/sandbox/util.go Normal file
View File

@ -0,0 +1,12 @@
package sandbox
import "unsafe"
// faster with no memory copy
func strToBytes(s string) []byte {
return unsafe.Slice(unsafe.StringData(s), len(s))
}
func byteArrayToString(buf []byte) string {
return *(*string)(unsafe.Pointer(&buf))
}

21
internal/parsers/dummy/parser.go
View File

@ -1,6 +1,9 @@
package dummy
import (
"fmt"
"log/slog"
"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
"github.com/criyle/go-judge/cmd/go-judge/model"
"github.com/mitchellh/mapstructure"
@ -13,14 +16,20 @@ type Config struct {
type Dummy struct{}
func (e *Dummy) Run(result model.Result, configAny any) stage.ParserResult {
func (e *Dummy) Run(result *model.Result, configAny any) (
*stage.ParserResult, error,
) {
var config Config
err := mapstructure.Decode(configAny, &config)
if err != nil {
panic(err)
}
return stage.ParserResult{
Score: config.Score,
Comment: config.Comment,
slog.Error("failed to decode config", "err", err)
return nil, err
}
return &stage.ParserResult{
Score: config.Score,
Comment: fmt.Sprintf(
"%s, executor status: run time: %d ns, memory: %d bytes",
config.Comment, result.RunTime, result.Memory,
),
}, nil
}

2
internal/stage/executor.go
View File

@ -7,7 +7,7 @@ import (
var executorMap = map[string]Executor{}
type Executor interface {
Run(model.Cmd) model.Result
Run(model.Cmd) (*model.Result, error)
}
func RegisterExecutor(name string, executor Executor) {

6
internal/stage/model.go
View File

@ -6,8 +6,10 @@ import (
type Stage struct {
Name string
ExecutorName string
Executor Executor
ExecutorCmd model.Cmd
ParserName string
Parser Parser
ParserConfig any
}
@ -19,7 +21,7 @@ type ParserResult struct {
type StageResult struct {
Name string
ParserResult
*ParserResult
}
type StagesConfig struct {
@ -27,7 +29,7 @@ type StagesConfig struct {
Name string
Executor struct {
Name string
With interface{}
With model.Cmd
}
Parser struct {
Name string

2
internal/stage/parser.go
View File

@ -5,7 +5,7 @@ import "github.com/criyle/go-judge/cmd/go-judge/model"
var parserMap = map[string]Parser{}
type Parser interface {
Run(model.Result, any) ParserResult
Run(*model.Result, any) (*ParserResult, error)
}
func RegisterParser(name string, parser Parser) {

26
internal/stage/run.go
View File

@ -1,7 +1,9 @@
package stage
import (
"github.com/criyle/go-judge/cmd/go-judge/model"
"log/slog"
"os"
"github.com/pelletier/go-toml/v2"
)
@ -9,14 +11,17 @@ func ParseStages(tomlConfig string) []Stage {
var stagesConfig StagesConfig
err := toml.Unmarshal([]byte(tomlConfig), &stagesConfig)
if err != nil {
panic(err)
slog.Error("parse stages config", "error", err)
os.Exit(1)
}
stages := []Stage{}
for _, stage := range stagesConfig.Stages {
stages = append(stages, Stage{
Name: stage.Name,
ExecutorName: stage.Executor.Name,
Executor: executorMap[stage.Executor.Name],
ExecutorCmd: model.Cmd{},
ExecutorCmd: stage.Executor.With,
ParserName: stage.Parser.Name,
Parser: parserMap[stage.Parser.Name],
ParserConfig: stage.Parser.With,
})
@ -27,8 +32,19 @@ func ParseStages(tomlConfig string) []Stage {
func Run(stages []Stage) []StageResult {
var parserResults []StageResult
for _, stage := range stages {
executorResult := stage.Executor.Run(stage.ExecutorCmd)
parserResult := stage.Parser.Run(executorResult, stage.ParserConfig)
slog.Info("stage start", "name", stage.Name)
executorResult, err := stage.Executor.Run(stage.ExecutorCmd)
if err != nil {
slog.Error("executor error", "name", stage.ExecutorName, "error", err)
break
}
slog.Info("executor done", "result", executorResult)
parserResult, err := stage.Parser.Run(executorResult, stage.ParserConfig)
if err != nil {
slog.Error("parser error", "name", stage.ExecutorName, "error", err)
break
}
slog.Info("parser done", "result", parserResult)
parserResults = append(parserResults, StageResult{
Name: stage.Name,
ParserResult: parserResult,