JOJ/JOJ3
24
5
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: copy in cached & cleanup

Browse Source
This commit is contained in:
张泊明518370910136 2024-03-04 01:02:44 -05:00
parent 4d594e1d87
commit 5d6bf8c5cd
Signed by: 张泊明518370910136
GPG Key ID: D47306D7062CDA9D
8 changed files with 111 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -7,10 +7,17 @@ $ make clean && make && ./build/joj3
rm -rf ./build/*
rm -rf *.out
go build -o ./build/joj3 ./cmd/joj3
2024/03/03 18:01:11 INFO stage start name="stage 0"
2024/03/03 18:01:11 INFO sandbox run cmd="{Args:[ls] Env:[PATH=/usr/bin:/bin] Files:[0xc0000aa340 0xc0000aa380 0xc0000aa3c0] CPULimit:10000000000 RealCPULimit:0 ClockLimit:0 MemoryLimit:104857600 StackLimit:0 ProcLimit:50 CPURateLimit:0 CPUSetLimit: CopyIn:map[] CopyOut:[stdout stderr] CopyOutCached:[] CopyOutMax:0 CopyOutDir: TTY:false StrictMemoryLimit:false DataSegmentLimit:false AddressSpaceLimit:false}"
2024/03/03 18:01:11 INFO sandbox run ret="results:{status:Accepted time:1162000 runTime:3847400 memory:14929920 files:{key:\"stderr\" value:\"\"} files:{key:\"stdout\" value:\"stderr\\nstdout\\n\"}}"
2024/03/03 18:01:11 INFO executor done result="{Status:Accepted ExitStatus:0 Error: Time:1.162ms RunTime:3.8474ms Memory:14.2 MiB Files:map[stderr:len:0 stdout:len:14] FileIDs:map[] FileError:[]}"
2024/03/03 18:01:11 INFO parser done result="&{Score:100 Comment:dummy comment for stage 0, executor status: run time: 3847400 ns, memory: 14929920 bytes}"
stage 0: score: 100, comment: dummy comment for stage 0, executor status: run time: 3847400 ns, memory: 14929920 bytes
2024/03/04 01:00:33 INFO stage start name=compile
2024/03/04 01:00:33 INFO sandbox run cmd="{Args:[/usr/bin/g++ a.cc -o a] Env:[PATH=/usr/bin:/bin] Files:[0xc00007e380 0xc00007e3c0 0xc00007e400] CPULimit:10000000000 RealCPULimit:0 ClockLimit:0 MemoryLimit:104857600 StackLimit:0 ProcLimit:50 CPURateLimit:0 CPUSetLimit: CopyIn:map[a.cc:{Src:<nil> Content:0xc0000245f0 FileID:<nil> Name:<nil> Max:<nil> Symlink:<nil> StreamIn:false StreamOut:false Pipe:false}] CopyInCached:map[] CopyOut:[stdout stderr] CopyOutCached:[a] CopyOutMax:0 CopyOutDir: TTY:false StrictMemoryLimit:false DataSegmentLimit:false AddressSpaceLimit:false}"
2024/03/04 01:00:33 INFO sandbox run ret="results:{status:Accepted time:327939000 runTime:328796901 memory:57540608 files:{key:\"stderr\" value:\"\"} files:{key:\"stdout\" value:\"\"} fileIDs:{key:\"a\" value:\"YCYTGTCQ\"}}"
2024/03/04 01:00:33 INFO executor done result="{Status:Accepted ExitStatus:0 Error: Time:327.939ms RunTime:328.796901ms Memory:54.9 MiB Files:map[stderr:len:0 stdout:len:0] FileIDs:map[a:YCYTGTCQ] FileError:[]}"
2024/03/04 01:00:33 INFO parser done result="&{Score:100 Comment:compile done, executor status: run time: 328796901 ns, memory: 57540608 bytes}"
2024/03/04 01:00:33 INFO stage start name=run
2024/03/04 01:00:33 INFO sandbox run cmd="{Args:[a] Env:[PATH=/usr/bin:/bin] Files:[0xc00007e440 0xc00007e480 0xc00007e4c0] CPULimit:10000000000 RealCPULimit:0 ClockLimit:0 MemoryLimit:104857600 StackLimit:0 ProcLimit:50 CPURateLimit:0 CPUSetLimit: CopyIn:map[] CopyInCached:map[a:a] CopyOut:[stdout stderr] CopyOutCached:[] CopyOutMax:0 CopyOutDir: TTY:false StrictMemoryLimit:false DataSegmentLimit:false AddressSpaceLimit:false}"
2024/03/04 01:00:33 INFO sandbox run ret="results:{status:Accepted time:1334000 runTime:2083023 memory:15384576 files:{key:\"stderr\" value:\"\"} files:{key:\"stdout\" value:\"2\\n\"}}"
2024/03/04 01:00:33 INFO executor done result="{Status:Accepted ExitStatus:0 Error: Time:1.334ms RunTime:2.083023ms Memory:14.7 MiB Files:map[stderr:len:0 stdout:len:2] FileIDs:map[] FileError:[]}"
2024/03/04 01:00:33 INFO parser done result="&{Score:100 Comment:run done, executor status: run time: 2083023 ns, memory: 15384576 bytes}"
compile: score: 100, comment: compile done, executor status: run time: 328796901 ns, memory: 57540608 bytes
run: score: 100, comment: run done, executor status: run time: 2083023 ns, memory: 15384576 bytes
2024/03/04 01:00:33 INFO sandbox cleanup
```

94
cmd/joj3/main.go
View File

@ -10,41 +10,67 @@ import (
func main() {
tomlConfig := `
[[stages]]
name = "stage 0"
[stages.executor]
name = "sandbox"
[stages.executor.with]
args = [ "ls" ]
env = [ "PATH=/usr/bin:/bin" ]
cpuLimit = 10_000_000_000
memoryLimit = 104_857_600
procLimit = 50
copyOut = [ "stdout", "stderr" ]
[stages.executor.with.copyIn.test]
src = "/home/boyanzh/joint-online-judge/go-judge/go.mod"
[[stages.executor.with.files]]
content = ""
[[stages.executor.with.files]]
name = "stdout"
max = 4_096
[[stages.executor.with.files]]
name = "stderr"
max = 4_096
[stages.parser]
name = "dummy"
[stages.parser.with]
score = 100
comment = "dummy comment for stage 0"
[[stages]]
name = "compile"
[stages.executor]
name = "sandbox"
[stages.executor.with]
args = [ "/usr/bin/g++", "a.cc", "-o", "a" ]
env = [ "PATH=/usr/bin:/bin" ]
cpuLimit = 10_000_000_000
memoryLimit = 104_857_600
procLimit = 50
copyOut = [ "stdout", "stderr" ]
copyOutCached = [ "a" ]
[stages.executor.with.copyIn."a.cc"]
content = """
#include <iostream>
int main() {
int a, b;
std::cin >> a >> b;
std::cout << a + b << '\\n';
}"""
[[stages.executor.with.files]]
content = ""
[[stages.executor.with.files]]
name = "stdout"
max = 4_096
[[stages.executor.with.files]]
name = "stderr"
max = 4_096
[stages.parser]
name = "dummy"
[stages.parser.with]
score = 100
comment = "compile done"
[[stages]]
name = "run"
[stages.executor]
name = "sandbox"
[stages.executor.with]
args = [ "a" ]
env = [ "PATH=/usr/bin:/bin" ]
cpuLimit = 10_000_000_000
memoryLimit = 104_857_600
procLimit = 50
copyOut = [ "stdout", "stderr" ]
[stages.executor.with.copyInCached]
a = "a"
[[stages.executor.with.files]]
content = "1 1"
[[stages.executor.with.files]]
name = "stdout"
max = 4_096
[[stages.executor.with.files]]
name = "stderr"
max = 4_096
[stages.parser]
name = "dummy"
[stages.parser.with]
score = 100
comment = "run done"
`
defer stage.Cleanup()
stages := stage.ParseStages(tomlConfig)
results := stage.Run(stages)
for _, result := range results {

4
internal/executors/dummy/executor.go
View File

@ -19,3 +19,7 @@ func (e *Dummy) Run(stage.Cmd) (*stage.Result, error) {
FileIDs: map[string]string{},
}, nil
}
func (e *Dummy) Cleanup() error {
return nil
}

25
internal/executors/sandbox/executor.go
View File

@ -11,10 +11,19 @@ import (
type Sandbox struct {
execClient pb.ExecutorClient
cachedMap map[string]string
}
func (e *Sandbox) Run(cmd stage.Cmd) (*stage.Result, error) {
slog.Info("sandbox run", "cmd", cmd)
if cmd.CopyIn == nil {
cmd.CopyIn = make(map[string]stage.CmdFile)
}
for k, v := range cmd.CopyInCached {
if fileID, ok := e.cachedMap[v]; ok {
cmd.CopyIn[k] = stage.CmdFile{FileID: &fileID}
}
}
req := &pb.Request{Cmd: convertPBCmd([]stage.Cmd{cmd})}
ret, err := e.execClient.Exec(context.TODO(), req)
if err != nil {
@ -24,5 +33,19 @@ func (e *Sandbox) Run(cmd stage.Cmd) (*stage.Result, error) {
return nil, fmt.Errorf("compile error: %s", ret.Error)
}
slog.Info("sandbox run", "ret", ret)
return &convertPBResult(ret.Results)[0], nil
res := &convertPBResult(ret.Results)[0]
for fileName, fileID := range res.FileIDs {
e.cachedMap[fileName] = fileID
}
return res, nil
}
func (e *Sandbox) Cleanup() error {
slog.Info("sandbox cleanup")
for _, fileID := range e.cachedMap {
e.execClient.FileDelete(context.TODO(), &pb.FileID{
FileID: fileID,
})
}
return nil
}

1
internal/executors/sandbox/meta.go
View File

@ -10,5 +10,6 @@ func init() {
stage.RegisterExecutor(name, &Sandbox{
// TODO: read from config
execClient: createExecClient("localhost:5051", ""),
cachedMap: make(map[string]string),
})
}

1
internal/stage/executor.go
View File

@ -4,6 +4,7 @@ var executorMap = map[string]Executor{}
type Executor interface {
Run(Cmd) (*Result, error)
Cleanup() error
}
func RegisterExecutor(name string, executor Executor) {

3
internal/stage/model.go
View File

@ -43,7 +43,8 @@ type Cmd struct {
CPURateLimit uint64 `json:"cpuRateLimit"`
CPUSetLimit string `json:"cpuSetLimit"`
CopyIn map[string]CmdFile `json:"copyIn"`
CopyIn map[string]CmdFile `json:"copyIn"`
CopyInCached map[string]string `json:"copyInCached"`
CopyOut []string `json:"copyOut"`
CopyOutCached []string `json:"copyOutCached"`

6
internal/stage/run.go
View File

@ -52,3 +52,9 @@ func Run(stages []Stage) []StageResult {
}
return parserResults
}
func Cleanup() {
for _, executor := range executorMap {
executor.Cleanup()
}
}