feat: repo health check (#16) #17
28
.gitmodules
vendored
|
@ -38,6 +38,34 @@
|
|||
path = examples/keyword/clangtidy/sillycode
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = keyword/clangtidy/sillycode
|
||||
[submodule "examples/healthcheck/asciifile"]
|
||||
path = examples/healthcheck/asciifile
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/asciifile
|
||||
[submodule "examples/healthcheck/asciimsg"]
|
||||
path = examples/healthcheck/asciimsg
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/asciimsg
|
||||
[submodule "examples/healthcheck/forbiddenfile"]
|
||||
path = examples/healthcheck/forbiddenfile
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/forbiddenfile
|
||||
[submodule "examples/healthcheck/meta"]
|
||||
path = examples/healthcheck/meta
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/meta
|
||||
[submodule "examples/healthcheck/release"]
|
||||
path = examples/healthcheck/release
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/release
|
||||
[submodule "examples/healthcheck/reposize"]
|
||||
path = examples/healthcheck/reposize
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/reposize
|
||||
[submodule "examples/healthcheck/repoverify"]
|
||||
path = examples/healthcheck/repoverify
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
branch = healthcheck/repoverify
|
||||
[submodule "examples/cppcheck/sillycode"]
|
||||
path = examples/cppcheck/sillycode
|
||||
url = ssh://git@focs.ji.sjtu.edu.cn:2222/FOCS-dev/JOJ3-examples.git
|
||||
|
|
|
@ -84,3 +84,7 @@ Check the `Result` at <https://github.com/criyle/go-judge#rest-api-interface>.
|
|||
|
||||
- `Score int`: score of the stage.
|
||||
- `Comment string`: comment on the stage.
|
||||
|
||||
### HealthCheck
|
||||
|
||||
The repohealth check will return a json list to for check result. The structure of json file is in `pkg/healthcheck/util.go`
|
||||
|
|
78
cmd/healthcheck/main.go
Normal file
|
@ -0,0 +1,78 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/healthcheck"
|
||||
)
|
||||
|
||||
// parseMultiValueFlag parses a multi-value command-line flag and appends its values to the provided slice.
|
||||
// It registers a flag with the specified name and description, associating it with a multiStringValue receiver.
|
||||
func parseMultiValueFlag(values *[]string, flagName, description string) {
|
||||
flag.Var((*multiStringValue)(values), flagName, description)
|
||||
}
|
||||
|
||||
type multiStringValue []string
|
||||
|
||||
// Set appends a new value to the multiStringValue slice.
|
||||
// It satisfies the flag.Value interface, allowing multiStringValue to be used as a flag value.
|
||||
func (m *multiStringValue) Set(value string) error {
|
||||
*m = append(*m, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *multiStringValue) String() string {
|
||||
return fmt.Sprintf("%v", *m)
|
||||
}
|
||||
|
||||
// Generally, err is used for runtime errors, and checkRes is used for the result of the checks.
|
||||
func main() {
|
||||
var info []healthcheck.CheckStage
|
||||
var gitWhitelist, metaFile, releaseTags []string
|
||||
var tmp healthcheck.CheckStage
|
||||
|
||||
rootDir := flag.String("root", "", "")
|
||||
repo := flag.String("repo", "", "")
|
||||
droneBranch := flag.String("droneBranch", "", "")
|
||||
releaseCategories := flag.String("releaseCategories", "", "")
|
||||
releaseNumber := flag.Int("releaseNumber", 0, "")
|
||||
// FIXME: for drone usage
|
||||
adminDir := flag.String("admin", "", "") // adminDir is for config files
|
||||
parseMultiValueFlag(&gitWhitelist, "whitelist", "")
|
||||
parseMultiValueFlag(&metaFile, "meta", "")
|
||||
parseMultiValueFlag(&releaseTags, "releaseTags", "")
|
||||
flag.Parse()
|
||||
|
||||
tmp = healthcheck.RepoSize()
|
||||
info = append(info, tmp)
|
||||
|
||||
tmp = healthcheck.ForbiddenCheck(*rootDir, gitWhitelist, *repo, *droneBranch)
|
||||
info = append(info, tmp)
|
||||
|
||||
tmp = healthcheck.MetaCheck(*rootDir, metaFile)
|
||||
info = append(info, tmp)
|
||||
|
||||
tmp = healthcheck.NonAsciiFiles(*rootDir)
|
||||
info = append(info, tmp)
|
||||
|
||||
tmp = healthcheck.NonAsciiMsg(*rootDir)
|
||||
info = append(info, tmp)
|
||||
|
||||
// TODO: find a way to test the release tag
|
||||
bomingzh marked this conversation as resolved
Outdated
|
||||
tmp = healthcheck.CheckReleases(*rootDir, *releaseCategories, *releaseNumber)
|
||||
info = append(info, tmp)
|
||||
|
||||
// FIXME: for drone usage
|
||||
tmp = healthcheck.Verify(*rootDir, *adminDir)
|
||||
info = append(info, tmp)
|
||||
|
||||
jsonData, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
fmt.Fprint(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
bomingzh marked this conversation as resolved
Outdated
张泊明518370910136
commented
check releases need to use Gitea API, just check tags is enough check releases need to use Gitea API, just check tags is enough
汪睿522370910169
commented
Yep, only check tags Yep, only check tags
张泊明518370910136
commented
Where is it tested? Where is it tested?
|
||||
}
|
||||
fmt.Printf("%s", jsonData)
|
||||
}
|
|
@ -14,6 +14,18 @@ import (
|
|||
|
||||
func compareStageResults(t *testing.T, actual, expected []stage.StageResult, regex bool) {
|
||||
t.Helper()
|
||||
|
||||
// For Test
|
||||
fmt.Println("Actual:")
|
||||
for _, result := range actual {
|
||||
fmt.Println(result)
|
||||
}
|
||||
|
||||
fmt.Println("Expected:")
|
||||
for _, result := range expected {
|
||||
fmt.Println(result)
|
||||
}
|
||||
|
||||
if len(actual) != len(expected) {
|
||||
t.Fatalf("len(actual) = %d, expected %d", len(actual), len(expected))
|
||||
}
|
||||
|
|
1
examples/healthcheck/asciifile
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit 46804afd2ebe8787a9d43711b191e424134ce25b
|
1
examples/healthcheck/asciimsg
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit e9ed6a464a507730ef956bb8ea6dbe84fffea7ad
|
1
examples/healthcheck/forbiddenfile
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit 245d036af0cbfe3745ef303d239a2d7225067d0b
|
1
examples/healthcheck/meta
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit 4f5e444940d2c383a0b069405e2ef42b01878bc5
|
1
examples/healthcheck/release
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit a4cedea002a198c2dae373f7ee6f6dc67753fae6
|
1
examples/healthcheck/reposize
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit d78308bcaaaeda9ead86069652288ae911503e33
|
1
examples/healthcheck/repoverify
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit a7a3bd0894c2e727e3ab3f9ddda3d438fbb86b30
|
|
@ -5,6 +5,7 @@ import (
|
|||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/cppcheck"
|
||||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/cpplint"
|
||||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/diff"
|
||||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/healthcheck"
|
||||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/keyword"
|
||||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/resultstatus"
|
||||
_ "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/parsers/sample"
|
||||
|
|
9
internal/parsers/healthcheck/meta.go
Normal file
|
@ -0,0 +1,9 @@
|
|||
package healthcheck
|
||||
|
||||
import "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
|
||||
|
||||
var name = "healthcheck"
|
||||
|
||||
func init() {
|
||||
stage.RegisterParser(name, &Healthcheck{})
|
||||
}
|
49
internal/parsers/healthcheck/parser.go
Normal file
|
@ -0,0 +1,49 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
// "encoding/json"
|
||||
"fmt"
|
||||
|
||||
"focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/internal/stage"
|
||||
// "focs.ji.sjtu.edu.cn/git/FOCS-dev/JOJ3/pkg/healthcheck"
|
||||
"github.com/criyle/go-judge/envexec"
|
||||
)
|
||||
|
||||
type Conf struct {
|
||||
Score int
|
||||
Comment string
|
||||
}
|
||||
|
||||
type Healthcheck struct{}
|
||||
|
||||
func Parse(executorResult stage.ExecutorResult, conf Conf) stage.ParserResult {
|
||||
stdout := executorResult.Files["stdout"]
|
||||
stderr := executorResult.Files["stderr"]
|
||||
if executorResult.Status != stage.Status(envexec.StatusAccepted) {
|
||||
return stage.ParserResult{
|
||||
Score: 0,
|
||||
Comment: fmt.Sprintf(
|
||||
"Unexpected executor status: %s.\nStderr: %s",
|
||||
executorResult.Status, stderr,
|
||||
),
|
||||
}
|
||||
}
|
||||
return stage.ParserResult{
|
||||
Score: 0,
|
||||
Comment: stdout,
|
||||
zzjc123 marked this conversation as resolved
Outdated
张泊明518370910136
commented
Comments need to be human-readable. You can use markdown format here. I suggest only keep the stderr field. You can get exitcode directly from executor if needed. Use logger to log any details for student to debug as they can check the log info from drone. A comment of health check should look like this: Repo SizePass Forbidden FileThe following forbidden files were found: README.md, conf.toml, expected.json, healthcheck, stderr, stdin.sh, stdout
Check name...check detail... Comments need to be human-readable. You can use markdown format here. I suggest only keep the stderr field. You can get exitcode directly from executor if needed. Use logger to log any details for student to debug as they can check the log info from drone.
A comment of health check should look like this:
### Repo Size
Pass
### Forbidden File
The following forbidden files were found: README.md, conf.toml, expected.json, healthcheck, stderr, stdin.sh, stdout
To fix it, first make a backup of your repository and then run the following commands:
```
...
```
### Check name...
check detail...
|
||||
}
|
||||
}
|
||||
|
||||
func (*Healthcheck) Run(results []stage.ExecutorResult, confAny any) (
|
||||
[]stage.ParserResult, bool, error,
|
||||
) {
|
||||
conf, err := stage.DecodeConf[Conf](confAny)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
var res []stage.ParserResult
|
||||
for _, result := range results {
|
||||
res = append(res, Parse(result, *conf))
|
||||
}
|
||||
return res, false, nil
|
||||
}
|
|
@ -57,13 +57,13 @@ func (*Keyword) Run(results []stage.ExecutorResult, confAny any) (
|
|||
return nil, true, err
|
||||
}
|
||||
var res []stage.ParserResult
|
||||
forceQuit := false
|
||||
end := false
|
||||
for _, result := range results {
|
||||
tmp, matched := Parse(result, *conf)
|
||||
if matched && conf.EndOnMatch {
|
||||
forceQuit = true
|
||||
end = true
|
||||
}
|
||||
res = append(res, tmp)
|
||||
}
|
||||
return res, forceQuit, nil
|
||||
return res, end, nil
|
||||
}
|
||||
|
|
|
@ -19,12 +19,12 @@ func (*ResultStatus) Run(results []stage.ExecutorResult, confAny any) (
|
|||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
forceQuit := false
|
||||
end := false
|
||||
var res []stage.ParserResult
|
||||
for _, result := range results {
|
||||
comment := ""
|
||||
if result.Status != stage.Status(envexec.StatusAccepted) {
|
||||
forceQuit = true
|
||||
end = true
|
||||
comment = fmt.Sprintf(
|
||||
"Unexpected executor status: %s.", result.Status,
|
||||
)
|
||||
|
@ -34,5 +34,5 @@ func (*ResultStatus) Run(results []stage.ExecutorResult, confAny any) (
|
|||
Comment: comment,
|
||||
})
|
||||
}
|
||||
return res, forceQuit, nil
|
||||
return res, end, nil
|
||||
}
|
||||
|
|
|
@ -164,7 +164,6 @@ type ParserResult struct {
|
|||
}
|
||||
|
||||
type StageResult struct {
|
||||
Name string `json:"name"`
|
||||
Results []ParserResult `json:"results"`
|
||||
ForceQuit bool `json:"force_quit"`
|
||||
Name string `json:"name"`
|
||||
Results []ParserResult `json:"results"`
|
||||
}
|
||||
|
|
|
@ -26,18 +26,17 @@ func Run(stages []Stage) []StageResult {
|
|||
slog.Error("parser not found", "name", stage.ParserName)
|
||||
break
|
||||
}
|
||||
parserResults, forceQuit, err := parser.Run(executorResults, stage.ParserConf)
|
||||
parserResults, end, err := parser.Run(executorResults, stage.ParserConf)
|
||||
if err != nil {
|
||||
slog.Error("parser run error", "name", stage.ExecutorName, "error", err)
|
||||
break
|
||||
}
|
||||
slog.Debug("parser run done", "results", parserResults)
|
||||
stageResults = append(stageResults, StageResult{
|
||||
Name: stage.Name,
|
||||
Results: parserResults,
|
||||
ForceQuit: forceQuit,
|
||||
Name: stage.Name,
|
||||
Results: parserResults,
|
||||
})
|
||||
if forceQuit {
|
||||
if end {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
83
pkg/healthcheck/commit.go
Normal file
|
@ -0,0 +1,83 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
)
|
||||
|
||||
// nonAsciiMsg checks for non-ASCII characters in the commit message.
|
||||
// If the message starts with "Merge pull request", it skips the non-ASCII characters check.
|
||||
// Otherwise, it iterates over each character in the message and checks if it is a non-ASCII character.
|
||||
// If a non-ASCII character is found, it returns an error indicating not to use non-ASCII characters in commit messages.
|
||||
// Otherwise, it returns nil indicating that the commit message is valid.
|
||||
func NonAsciiMsg(root string) (jsonOut CheckStage) {
|
||||
jsonOut = CheckStage{
|
||||
Name: "NonAsciiMsg",
|
||||
StdOut: "Checking for non-ASCII characters in commit message: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
|
||||
zzjc123 marked this conversation as resolved
Outdated
张泊明518370910136
commented
https://github.com/go-git/go-git#in-memory-example Try to avoid https://github.com/go-git/go-git#in-memory-example
Try to avoid `exec.Command`.
|
||||
// cmd := exec.Command("git", "log", "--encoding=UTF-8", "--format=%B")
|
||||
repo, err := git.PlainOpen(root)
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = fmt.Errorf("Error openning git repo%w", err)
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
ref, err := repo.Head()
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = fmt.Errorf("Error getting reference%w", err)
|
||||
return jsonOut
|
||||
}
|
||||
commits, err := repo.Log(&git.LogOptions{From: ref.Hash()})
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = fmt.Errorf("Error getting commits%w", err)
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
var msgs []string
|
||||
err = commits.ForEach(func(c *object.Commit) error {
|
||||
msgs = append(msgs, c.Message)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = fmt.Errorf("Error converting log to string%w", err)
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
var nonAsciiMsgs []string
|
||||
for _, msg := range msgs {
|
||||
if msg == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(msg, "Merge pull request") {
|
||||
continue
|
||||
}
|
||||
for _, c := range msg {
|
||||
if c > unicode.MaxASCII {
|
||||
nonAsciiMsgs = append(nonAsciiMsgs, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(nonAsciiMsgs) > 0 {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 105
|
||||
jsonOut.StdErr = "Non-ASCII characters in commit messages:\n" + strings.Join(nonAsciiMsgs, "\n")
|
||||
return jsonOut
|
||||
}
|
||||
jsonOut.StdOut += "OK"
|
||||
return jsonOut
|
||||
}
|
94
pkg/healthcheck/forbidden.go
Normal file
|
@ -0,0 +1,94 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// getForbiddens retrieves a list of forbidden files in the specified root directory.
|
||||
// It searches for files that do not match the specified regex patterns in the given file list.
|
||||
func getForbiddens(root string, fileList []string) ([]string, error) {
|
||||
var matches []string
|
||||
|
||||
var regexList []*regexp.Regexp
|
||||
regexList, err := getRegex(fileList)
|
||||
if err != nil {
|
||||
fmt.Println("Error compiling regex:", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
if info.Name() == ".git" || info.Name() == ".gitea" || info.Name() == "ci" {
|
||||
return filepath.SkipDir
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
match := false
|
||||
for _, regex := range regexList {
|
||||
if regex.MatchString(info.Name()) {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !match {
|
||||
matches = append(matches, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return matches, err
|
||||
}
|
||||
|
||||
// forbiddenCheck checks for forbidden files in the specified root directory.
|
||||
// It prints the list of forbidden files found, along with instructions on how to fix them.
|
||||
func ForbiddenCheck(rootDir string, regexList []string, repo string, droneBranch string) (jsonOut CheckStage) {
|
||||
jsonOut = CheckStage{
|
||||
Name: "forbiddenFile",
|
||||
StdOut: "Checking forbidden files: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
|
||||
// INFO: test case
|
||||
// regexList := []string{`\.$`, `\.git`, `\.drone.yml`, `Makefile`, `CMakeLists.txt`,`.*\.go`,`.*\.toml`, `.*\.c`, `.*\.cc`, `.*\.cpp`, `.*\.h`, `.*\.md`}
|
||||
// rootDir = "/Users/zhouzhaojiacheng/Desktop/STUDENT_ORG/TechJI/Dev/joj/JOJ3"
|
||||
|
||||
forbids, err := getForbiddens(rootDir, regexList)
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = err
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
var message string
|
||||
|
||||
if len(forbids) > 0 {
|
||||
message += fmt.Sprint(103, "the following forbidden files were found: ")
|
||||
for _, file := range forbids {
|
||||
message += fmt.Sprint(file, ", ")
|
||||
}
|
||||
message += "\n\nTo fix it, first make a backup of your repository and then run the following commands:\nfor i in "
|
||||
for _, file := range forbids {
|
||||
message += fmt.Sprint(file, " ")
|
||||
}
|
||||
message += fmt.Sprint("; do git filter-repo --force --invert-paths --path \\\"\\$i\\\"; done\ngit remote add origin ", repo, "\ngit push --set-upstream origin ", droneBranch, " --force")
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 103
|
||||
jsonOut.StdErr = message
|
||||
return jsonOut
|
||||
} else {
|
||||
jsonOut.StdOut += "OK"
|
||||
return jsonOut
|
||||
}
|
||||
}
|
|
@ -1 +0,0 @@
|
|||
package healthcheck
|
86
pkg/healthcheck/meta.go
Normal file
|
@ -0,0 +1,86 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// getMetas retrieves a list of metadata files that are expected to exist in the specified root directory.
|
||||
// It checks for the existence of each file in the fileList and provides instructions if any file is missing.
|
||||
func getMetas(rootDir string, fileList []string) ([]string, string, error) {
|
||||
addExt(fileList, "\\.*")
|
||||
regexList, err := getRegex(fileList)
|
||||
var unmatchedList []string
|
||||
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
files, err := os.ReadDir(rootDir)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("Error reading directory:%w", err)
|
||||
}
|
||||
|
||||
matched := false
|
||||
umatchedRes := ""
|
||||
|
||||
// TODO: it seems that there is no good find subsitution now
|
||||
// modify current code if exist a better solution
|
||||
for i, regex := range regexList {
|
||||
for _, file := range files {
|
||||
if file.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
if regex.MatchString(file.Name()) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
unmatchedList = append(unmatchedList, fileList[i])
|
||||
str := fmt.Sprint("\tno ", fileList[i], " file found")
|
||||
switch fileList[i] {
|
||||
case "readme\\.*":
|
||||
str += ", please refer to https://www.makeareadme.com/ for more information"
|
||||
case "changelog\\.*":
|
||||
str += ", please refer to https://keepachangelog.com/en/1.1.0/ for more information"
|
||||
default:
|
||||
str += ""
|
||||
}
|
||||
str += "\n"
|
||||
|
||||
umatchedRes += str
|
||||
}
|
||||
}
|
||||
|
||||
return unmatchedList, umatchedRes, nil
|
||||
}
|
||||
|
||||
// metaCheck performs a check for metadata files in the specified root directory.
|
||||
// It prints a message if any required metadata files are missing.
|
||||
func MetaCheck(rootDir string, fileList []string) (jsonOut CheckStage) {
|
||||
unmatchedList, umatchedRes, err := getMetas(rootDir, fileList)
|
||||
jsonOut = CheckStage{
|
||||
Name: "metaFile",
|
||||
StdOut: "Checking the existence of meta file: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = err
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
if len(unmatchedList) == 0 {
|
||||
jsonOut.StdOut += "OK"
|
||||
return jsonOut
|
||||
} else {
|
||||
jsonOut.ExitCode = 104
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.StdErr = fmt.Sprintf("%d important project files missing\n"+umatchedRes, len(unmatchedList))
|
||||
return jsonOut
|
||||
}
|
||||
}
|
90
pkg/healthcheck/nonascii.go
Normal file
|
@ -0,0 +1,90 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// getNonAscii retrieves a list of files in the specified root directory that contain non-ASCII characters.
|
||||
// It searches for non-ASCII characters in each file's content and returns a list of paths to files containing non-ASCII characters.
|
||||
func getNonAscii(root string) ([]string, error) {
|
||||
var nonAscii []string
|
||||
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
if info.Name() == ".git" || info.Name() == ".gitea" || info.Name() == "ci" {
|
||||
return filepath.SkipDir
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if info.Name() == "healthcheck" {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
cont := true
|
||||
for _, c := range scanner.Text() {
|
||||
if c > unicode.MaxASCII {
|
||||
nonAscii = append(nonAscii, "\t"+path)
|
||||
cont = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !cont {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return nonAscii, err
|
||||
}
|
||||
|
||||
// nonAsciiFiles checks for non-ASCII characters in files within the specified root directory.
|
||||
// It prints a message with the paths to files containing non-ASCII characters, if any.
|
||||
zzjc123 marked this conversation as resolved
Outdated
张泊明518370910136
commented
And do not just And do not just `panic` on these errors. These errors are not unrecoverable. You can just record these errors and let the program continue working on the other parts. Or just return the error to the upper level and let that function decides what is the next step.
|
||||
func NonAsciiFiles(root string) (jsonOut CheckStage) {
|
||||
jsonOut = CheckStage{
|
||||
Name: "NonAsciiFiles",
|
||||
StdOut: "Checking for non-ascii files: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
nonAscii, err := getNonAscii(root)
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = err
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
var nonAsciiRes string
|
||||
if len(nonAscii) > 0 {
|
||||
nonAsciiRes = fmt.Sprintf("Non-ASCII characters found in the following files:\n" + strings.Join(nonAscii, "\n"))
|
||||
jsonOut.ExitCode = 105
|
||||
jsonOut.StdOut += "Failed"
|
||||
} else {
|
||||
jsonOut.StdOut += "OK"
|
||||
}
|
||||
|
||||
jsonOut.StdErr = nonAsciiRes
|
||||
|
||||
return jsonOut
|
||||
}
|
88
pkg/healthcheck/release.go
Normal file
|
@ -0,0 +1,88 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
)
|
||||
|
||||
func catTags(all []string) (out string) {
|
||||
out = ""
|
||||
for _, str := range all {
|
||||
out += str + " "
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func getTagsFromRepo(repoPath string) ([]string, error) {
|
||||
repo, err := git.PlainOpen(repoPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Cannot open repo: %v", err)
|
||||
}
|
||||
|
||||
refs, err := repo.Tags()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Cannot get tags: %v", err)
|
||||
}
|
||||
|
||||
var tags []string
|
||||
err = refs.ForEach(func(ref *plumbing.Reference) error {
|
||||
tags = append(tags, ref.Name().Short())
|
||||
return nil
|
||||
})
|
||||
zzjc123 marked this conversation as resolved
Outdated
张泊明518370910136
commented
why not just let the parameter fully determine the target tag? e.g. why not just let the parameter fully determine the target tag? e.g. `func CheckReleases(repoPath, target string)`
汪睿522370910169
commented
Because we cannot decide how many hw or project release need to be checked yes. The int n is used to specif thr number Because we cannot decide how many hw or project release need to be checked yes. The int n is used to specif thr number
张泊明518370910136
commented
So since we need to pass So since we need to pass `category` and `n` to this function, we still need to decide the count of hw&proj when generating `conf.toml`. I do not see much difference.
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error while iterating tags: %v", err)
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func CheckReleases(repoPath string, category string, n int) (jsonOut CheckStage) {
|
||||
jsonOut = CheckStage{
|
||||
Name: "ReleaseCheck",
|
||||
StdOut: "Checking release tag: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
|
||||
tags, err := getTagsFromRepo(repoPath)
|
||||
if err != nil {
|
||||
log.Fatalf("Error in getting tags: %v", err)
|
||||
}
|
||||
|
||||
var prefix string
|
||||
|
||||
switch category {
|
||||
case "exam":
|
||||
prefix = "e"
|
||||
张泊明518370910136
commented
Why is the Why is the `target` release tag wrong?
汪睿522370910169
commented
I intended to mean release tag is missing or wrong. I intended to mean release tag is missing or wrong.
汪睿522370910169
commented
Do we need to specify missing or wrong? I think only we need to inform students that their release is wrong. Do we need to specify missing or wrong? I think only we need to inform students that their release is wrong.
张泊明518370910136
commented
Do we need to test if they submit extra tags? Do we need to test if they submit extra tags?
汪睿522370910169
commented
I think extra is fine, as long as they don't miss any tags. I think extra is fine, as long as they don't miss any tags.
张泊明518370910136
commented
Then just print the error on missing tags. Then just print the error on missing tags.
|
||||
case "project":
|
||||
prefix = "p"
|
||||
case "homework":
|
||||
prefix = "h"
|
||||
default:
|
||||
prefix = "a"
|
||||
}
|
||||
|
||||
target := prefix + fmt.Sprintf("%d", n)
|
||||
found := false
|
||||
for _, tag := range tags {
|
||||
if tag == target {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
tagList := catTags(tags)
|
||||
jsonOut.ExitCode = 107
|
||||
jsonOut.StdOut = "Failed"
|
||||
jsonOut.StdErr = fmt.Sprintf("wrong release tag '%s', please use one of %s aborting", target, tagList)
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
jsonOut.StdOut += "OK"
|
||||
jsonOut.ExitCode = 0
|
||||
jsonOut.StdErr = "Fine"
|
||||
return jsonOut
|
||||
}
|
56
pkg/healthcheck/reposize.go
Normal file
|
@ -0,0 +1,56 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// RepoSize checks the size of the repository to determine if it is oversized.
|
||||
// It executes the 'git count-objects -v' command to obtain the size information,
|
||||
func RepoSize() (jsonOut CheckStage) {
|
||||
jsonOut = CheckStage{
|
||||
Name: "RepoSize",
|
||||
StdOut: "Checking repository size: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
|
||||
// TODO: reimplement here when go-git is available
|
||||
bomingzh marked this conversation as resolved
Outdated
张泊明518370910136
commented
add a comment about link to https://github.com/go-git/go-git/blob/master/COMPATIBILITY.md#plumbing-commands , we just can not use go-git to implement it. add a comment about link to https://github.com/go-git/go-git/blob/master/COMPATIBILITY.md#plumbing-commands , we just can not use go-git to implement it.
张泊明518370910136
commented
@manuel :
@manuel :
add a "TODO comment" in the source code with
- a note to reimplement with go-git when available
- link to feature/compatibility page of go-git
|
||||
// https://github.com/go-git/go-git/blob/master/COMPATIBILITY.md
|
||||
cmd := exec.Command("git", "count-objects", "-v")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = fmt.Errorf("Error running git command:%w", err)
|
||||
return jsonOut
|
||||
}
|
||||
|
||||
lines := strings.Split(string(output), "\n")
|
||||
var sum int
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "size") {
|
||||
fields := strings.Fields(line)
|
||||
sizeStr := fields[1]
|
||||
size, err := strconv.Atoi(sizeStr)
|
||||
if err != nil {
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 1
|
||||
jsonOut.ErrorLog = fmt.Errorf("Error running git command:%w", err)
|
||||
return jsonOut
|
||||
}
|
||||
sum += size
|
||||
}
|
||||
}
|
||||
|
||||
if sum <= 2048 {
|
||||
jsonOut.StdOut += "OK"
|
||||
return jsonOut
|
||||
}
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 100
|
||||
jsonOut.StdErr = "repository larger than 2MB, please clean up or contact the teaching team."
|
||||
return jsonOut
|
||||
}
|
39
pkg/healthcheck/utils.go
Normal file
|
@ -0,0 +1,39 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// For ExitCode, see https://focs.ji.sjtu.edu.cn/git/TAs/resources/src/branch/drone/dronelib.checks
|
||||
// 1 for unrecoverable error and 0 for succeses
|
||||
type CheckStage struct {
|
||||
Name string `json:"name of check"`
|
||||
StdOut string `json:"stdout"`
|
||||
ExitCode int `json:"exit code"`
|
||||
StdErr string `json:"stderr"`
|
||||
ErrorLog error `json:"errorLog"`
|
||||
}
|
||||
|
||||
// addExt appends the specified extension to each file name in the given fileList.
|
||||
// It modifies the original fileList in place.
|
||||
func addExt(fileList []string, ext string) {
|
||||
for i, file := range fileList {
|
||||
fileList[i] = file + ext
|
||||
}
|
||||
}
|
||||
|
||||
// getRegex compiles each regex pattern in the fileList into a []*regexp.Regexp slice.
|
||||
// It returns a slice containing compiled regular expressions.
|
||||
func getRegex(fileList []string) ([]*regexp.Regexp, error) {
|
||||
var regexList []*regexp.Regexp
|
||||
for _, pattern := range fileList {
|
||||
regex, err := regexp.Compile("(?i)" + pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error compiling regex:%w", err)
|
||||
}
|
||||
regexList = append(regexList, regex)
|
||||
}
|
||||
|
||||
return regexList, nil
|
||||
}
|
114
pkg/healthcheck/verify.go
Normal file
|
@ -0,0 +1,114 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// fileExists checks if a file exists at the specified path.
|
||||
func fileExists(filePath string) bool {
|
||||
_, err := os.Stat(filePath)
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// filesMatch checks if two files are identical.
|
||||
func filesMatch(file1, file2 string) (bool, error) {
|
||||
f1, err := os.Open(file1)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f1.Close()
|
||||
|
||||
f2, err := os.Open(file2)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer f2.Close()
|
||||
|
||||
scanner1 := bufio.NewScanner(f1)
|
||||
scanner2 := bufio.NewScanner(f2)
|
||||
|
||||
for scanner1.Scan() && scanner2.Scan() {
|
||||
line1 := scanner1.Text()
|
||||
line2 := scanner2.Text()
|
||||
|
||||
if line1 != line2 {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
bomingzh marked this conversation as resolved
Outdated
张泊明518370910136
commented
remove the comment remove the comment
|
||||
|
||||
if scanner1.Scan() || scanner2.Scan() {
|
||||
// One file has more lines than the other
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
bomingzh marked this conversation as resolved
Outdated
张泊明518370910136
commented
ditto ditto
|
||||
}
|
||||
|
||||
// compareDirectories compares the contents of two directories.
|
||||
func compareDirectories(dir1, dir2 string, jsonOut *CheckStage) error {
|
||||
allMatch := true
|
||||
var message string
|
||||
err := filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
bomingzh marked this conversation as resolved
Outdated
张泊明518370910136
commented
remove it remove it
|
||||
return err
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
relPath, _ := filepath.Rel(dir1, path)
|
||||
file2 := filepath.Join(dir2, relPath)
|
||||
|
||||
if !fileExists(file2) {
|
||||
// fmt.Printf("File %s in %s is missing in %s\n, please immediately revert your changes!\n", relPath, dir1, dir2)
|
||||
张泊明518370910136
commented
remove it remove it
张泊明518370910136
commented
I mean remove the I mean remove the `os.Exit(1)`, and also the one on line 55.
|
||||
message += "File missing"
|
||||
allMatch = false
|
||||
jsonOut.StdOut += "Failed"
|
||||
jsonOut.ExitCode = 101
|
||||
jsonOut.StdErr = message
|
||||
张泊明518370910136
commented
why not just return error on not all passed? why not just return error on not all passed?
|
||||
return nil
|
||||
}
|
||||
|
||||
// fmt.Printf("Checking integrity of %s:\n", path)
|
||||
match, err := filesMatch(path, file2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !match {
|
||||
// fmt.Printf("File %s in %s is not identical to %s\nPlease revert your changes or contact the teaching team if you have a valid reason for adjusting them.\n", relPath, dir1, dir2)
|
||||
message += "File is not identical"
|
||||
allMatch = false
|
||||
jsonOut.StdOut += "Failed"
|
||||
张泊明518370910136
commented
ditto ditto
|
||||
jsonOut.ExitCode = 101
|
||||
jsonOut.StdErr = message
|
||||
张泊明518370910136
commented
Why so many returns? Why so many returns?
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if allMatch {
|
||||
jsonOut.StdOut += "OK!"
|
||||
} else {
|
||||
jsonOut.StdOut += "Failed!"
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Verify checks if the contents of two directories are identical.
|
||||
func Verify(rootDir, compareDir string) CheckStage {
|
||||
jsonOut := CheckStage{
|
||||
Name: "verifyFile",
|
||||
StdOut: "Checking files to be verified: ",
|
||||
ExitCode: 0,
|
||||
StdErr: "",
|
||||
}
|
||||
err := compareDirectories(rootDir, compareDir, &jsonOut)
|
||||
// fmt.Println("Comparison finished ")
|
||||
if err != nil {
|
||||
fmt.Println("Error:", err)
|
||||
}
|
||||
return jsonOut
|
||||
}
|
is it still TODO?
Done yet