diff --git a/tt/teabag.sh b/tt/teabag.sh new file mode 100755 index 0000000..6171c73 --- /dev/null +++ b/tt/teabag.sh @@ -0,0 +1,556 @@ +#!/bin/bash + +GITEA="https://focs.ji.sjtu.edu.cn/git" +GITEAAPI="https://focs.ji.sjtu.edu.cn/git/api/v1" +GITURL="https://git@focs.ji.sjtu.edu.cn/git" + +# defaults +CONF=repos.conf +DEPS="jq curl" +RECOMS="lowdown" + +#shopt -s extglob + +echoerr() { ERR=$1; shift; echo "Error: $@" 1>&2; exit $ERR; } + +# curl + jq needed for creating repo.list +systemcheck() { + + for i in $DEPS; do + which $i >/dev/null || echoerr -2 "$i not found, please install it first" + done + + for i in $RECOMS; do + which $i >/dev/null || echo "Warning: $i not found, some features such as displaying reports might not work well" + done + +} + +# create a list of repos +setup() { + +# rm -f "$CONF" + + if [ -e "$CONF" ]; then + echo "WARNING: $CONF already exists!" + read -n1 -p "Delete and continue, edit and exit, or exit? (d/e/X) " confwarn + + echo + + case "$confwarn" in + d) echo rm -f "$CONF" + ;; + e) sensible-editor "$CONF" + return + ;; + *) exit + ;; + esac + fi + + l=50 + let i=1 + + while [ $l = 50 ]; do + + repos="$(curl -s "$GITEAAPI/orgs/$ORG/repos?limit=50&page=$i" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | jq -r '.[] | .full_name')" + + l=$(echo "$repos" | wc -l) + REPOS="$REPOS$(echo -e "$repos" | sed '/'$PATTERN'/I !d; s/^/\t/; s/$/ \\/g')" >> "$CONF" + + let i++ + + done + + REPOS=$(sort <<< $REPOS) + + cat << EOF > "$CONF" +# Edit and ensure teams are in increasing order +# notde: this is important if reports are posted on gitea +REPOS=( \\ +$REPOS +) + +# For JOJ scores, specify the scoreboard filename, eg. scoreboard-p1.csv +SCOREBOARD= + +# For contribution statistics, specify the file entensions of the source code +# eg. SRC=('***.cc' '***.cpp' '***.h' '***.hh') +SRC=() + +# Patterns of commits to ignore in contributions, eg. auto-generated code +# comma separated list of patterns +# eg. SKIP="chore: messenger, build(Visuals)" +SKIP= + +# To post a report in an issue dedicated to each team specify: +# - the repo where to post the issue (eg. teaching-team) +# - the issue number of the 1st team, eg. 24 +# note: issues must already exist and be consecutive, eg. team01->24, team02->25, etc. +REPORTREPO= +REPORTISSUE= +EOF + + echo "IMPORTANT: make sure to manually edit $CONF and apply necessary changes" + +} + +# clone all repos +init() { + + # for i in ${REPOS[@]}; do + # git clone "$GITURL/$i" + # done + + # echo "Set up: email, name, merge strategy, lfs" + +# # for bots use a different email address + # read -p "Input the jaccount or bot account to use: " jaccount + # if [ -n "${jaccount%bot-*}" ]; then + # email=$jaccount@sjtu.edu.cn + # else + # email=$jaccount@focs.ji.sjtu.edu.cn + # fi + + # read -p "Input your name: " name + + # for j in ${REPOS[@]/*\/}; do + # cd $j + # git config user.email $email + # git config user.name "$name" + # git config pull.rebase true + # git lfs install > /dev/null + # cd .. + # done + + echo "Cloning JOJ repo" + if [ -d "${REPOS[0]%/*}-joj" ]; then + echo "WARNING: ${REPOS[0]%/*}-joj directory already exists, trying to pull..." + cd "${REPOS[0]%/*}-joj" + git pull + else + git clone -b grading $GITURL/${REPOS[0]%/*}/${REPOS[0]%/*}-joj + fi + +} + +# copy files to all repos listed in repos.list +copy() { + + RSYNC=$(which rsync) + [ -z $RSYNC ] && RSYNC="cp -r" && echo Warning: rsync not found, using cp instead + + for j in ${REPOS[@]/*\/}; do + + echo "Processing $j..." + + cd $j; git switch $BRANCH 2>/dev/null; git pull; cd .. + + sync_files + + done + + echo "To complete the process, rerun with the push option." + +} + +copyall() { + + # cleanup + rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md + + echo "Warning: copying to all branches of all repos. When completed DO NOT run again with the push option." + read -n 1 -p "Please y to confirm: " confirm + + [ "$confirm" != "y" ] && echo "Canceled" && exit + + RSYNC=$(which rsync) + [ -z $RSYNC ] && RSYNC="cp -r" && echo Warning: rsync not found, using cp instead + + read -p "Commit message: " commit + + for j in ${REPOS[@]/*\/}; do + + echo "Processing $j..." + +# many cd to ensure sync_files called from same dir as repos.list + cd $j; + branches=$(git branch -a | sed '/\// !d; s,.*origin/,,g' | sort -u); + cd .. + + for k in $branches; do + echo "Processing branch $k..." + cd $j + git switch $k 2>/dev/null + git pull + cd .. + sync_files + cd $j + git add . + git commit -m"$commit" + git push + cd .. + done + + done + +} + +# file path relative or absolute +# should be called from same dir as repos.list +sync_files() { + + for i in "${FILES[@]}"; do + if [ -e "$i" ]; then + $RSYNC -a "${i%/}" "$j" + else + echo "Warning: file \"$i\" not found, skipping" + fi + done +} + +# add commit pull push +push() { + + # cleanup + rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md + + read -p "Commit message: " commit + + for j in ${REPOS[@]/*\/}; do + cd $j + echo "Processing $j..." + git add . + git commit -m"$commit" + git pull + git push + cd .. + done + +} + +report() { + +# update grading + cd ${REPOS[0]%/*}-joj + git pull + cd .. + + for t in ${REPOS[@]}; do + + trepo=${t/*\/} + TEAM=($(sed '/'$trepo'/ !d; s/ /+/g; s/^\([^,]*\),[^,]*,\([^@]*\)@.*/\1 \2/g;' "${CONF%.conf}.csv")) + + echo "Processing $trepo..." + + [ -d "$trepo" ] || echoerr 10 "$trepo no found, make sure to first run $0 with init parameter" + cd $trepo + + # cleanup + rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md + + git stash -q + git switch -q $BRANCH + if ! git pull --ff-only -q; then trepofail="$trepo $trepofail"; fi + + for i in ${RANGE//\./ }; do git tag -l | grep -q $i || tagfail="$tagfail- $i ($trepo)\n"; done + + echo -n "## Git usage report for $trepo " >> report.md + [ -n "$RANGE" ] && echo "($RANGE)" >> report.md + echo >> report.md + + report_release + report_contrib + report_commits + report_joj + + if [ x$TISSUE = x1 ]; then + curl -s -X POST "$GITEAAPI/repos/${t%/*}/$REPORTREPO/issues/$REPORTISSUE/comments" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" -H 'accept: application/json' -H 'Content-Type: application/json' -d "{\"body\": \"$(cat report.md)\"}" > /dev/null + let REPORTISSUE++ + else + echo + lowdown -Tterm report.md + echo + fi + +# students with no commit + NC="$(sed '/[^,]*,[[:space:]]*0/ !d; s/[[:space:]]*\([^,]*\),.*/- \1('$trepo')/' commits.csv)" + [ -n "$NC" ] && NOCOM="$NC\n$NOCOM" + + cd .. + + done + + if [ -n "$trepofail" ]; then + echo -e "\n## Repo pull failure" + echo -e "\n$trepofail" + fi + + if [ -n "$tagfail" ]; then + echo -e "\n## Missing tags" + echo -e "\n$tagfail" + fi + + echo -e "\n## Students with no commit for $RANGE" + echo -e "\n$NOCOM" + +} + +report_release() { + + echo -e "### Releases\n" >> report.md + + R=($(curl -s "$GITEAAPI/repos/$t/releases" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | jq -r '.[] | .tag_name, .published_at, .target_commitish, .name' | sed 's/ /_/g')) + + let i=0 + while [ -n "${R[i]}" ]; do + RS="**failed**" + curl -s "$GITEAAPI/repos/$t/actions/workflows/release.yaml/badge?tag=${R[i]}&event=release" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | grep -q success && RS="**successful**" + echo "- ${R[i+3]//_/ } released on $(date -d ${R[i+1]} '+%B %d at %H:%M') from ${R[i+2]} with tag ${R[i]}: $RS" >> report.md + let i+=4 + done + +} + +report_contrib() { + +# all commits + git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE 2>/dev/null | tr "\n" " " | tr "@" "\n" | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-all.csv + +# code commits only +# SRC (array) and SKIP (string) defined in CONF file +# SRC=('***.c' '***.h') +# SKIP="chore:.*messenger, build(visuals)" + + if [ -n "$SKIP" ]; then + git log --no-merges --pretty="@%aL^%s^%h^" --shortstat -i --invert-grep --grep="${SKIP//,/\\|}" $RANGE -- "${SRC[@]}" 2>/dev/null | tr "\n" " " | tr "@" "\n" | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv + else + git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE -- "${SRC[@]}" | tr "\n" " " | tr "@" "\n" 2>/dev/null | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv + fi + + n=${#TEAM[@]} + +# loc + totadd=$(awk -F , '{s+=$5}END{print s}' commits-code.csv) + + for((i=1; i loc.csv + +# indiv commits + for((i=1; i commits.csv + +# indiv commits + loc + echo -e "\n### Individual contribution\n" >> report.md + echo "| Author | Commits | Files | Additions | Deletions | Delta (A-D) | Ratio (A/D) | Workload |" >> report.md + echo "|---------------------------|---------|-------|-----------|-----------|-------------|-------------|----------|" >> report.md + join -t, commits.csv loc.csv | sed 's/,/|/g; s/^\|$/|/g' >> report.md + +} + +report_commits() { + +# alignment: space + GITEA + repo + commit + space + l=$((1+32+${#t}+15+1)) + +# non-atomic + echo -e "\n### Non-atomic commits\n" >> report.md + printf "| | Commits %$((l-10))s | Files | Additions | Deletions | Author |\n" " " >> report.md + printf "|----| %${l}s|-------|-----------|-----------|----------------------|\n" " " | tr ' ' - >> report.md + + if [ x$TISSUE = x1 ]; then + awk -v gitea=$GITEA -v cnt=0 -v repo=$t -F, '{if($5>80 || $6>80) { cnt++; printf "| %2s | [%s](%s/%s/commit/%s) | %s | %s | %s | %s |\n", cnt, $2, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md + else + awk -v gitea=$GITEA -v cnt=0 -v repo=$t -F, '{if($5>80 || $6>80) { cnt++; printf "| %2s | %s/%s/commit/%s | %5s | %9s | %9s | %20s |\n", cnt, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md + fi + +# non-conventional + echo -e "\n### Non-conventional commits\n" >> report.md + awk -v cnt=0 -F , '$2 !~ /chore|feat|docs|test|ci|refactor|perf|build|revert|fix|style/ { cnt++; printf "%s. %s (%s)\n", cnt, $2, $1 }' commits-all.csv >> report.md + +} + +report_joj() { + + echo -e "\n### JOJ scores\n" >> report.md + echo -e "**WARNING.** For more precise and detailed scores refer to [joj-mon](https://focs.ji.sjtu.edu.cn/joj-mon/d/${REPOS[0]%/*})\n" >> report.md + + dos2unix < "../${REPOS[0]%/*}-joj/$SCOREBOARD" | sed 's/^\|$/|/g; 1 s/\(.*\)/\1\n\1/' | column -s , -t -o \| | sed -n '2 s/[^|]/-/g; 1,2 p; /'$trepo'/ p' >> report.md + +} + +contrib() { + + t=$(head -1 $CONF | cut -d / -f 2) + LIST="${CONF%.conf}.csv" + INFO=($(grep -i "$STUDENT" "$LIST" | sed 's/[^,]*,[^,]*,\([^@]*\)@.*,\(.*\)/\2 \1/g')) + + for((i=0;i<${#INFO[@]};i=i+2)); do + + cd ${INFO[i]}/ + + [ -e commits-code.csv ] || echoerr 25 "No data found, first generate a report" + + if [ x$DUMP = x1 ]; then + + STUDENT=$(grep ${INFO[$((i+1))]} ../$LIST | sed 's/,/ /g') + echo -e "Commits for $STUDENT dumped to ${INFO/[i]}/${INFO[$((i+1))]}.diff" + + rm -f "${INFO[$((i+1))]}".diff + commits=$(grep -i "${INFO[$((i+1))]}" commits-code.csv | awk -F, '{print $3}') + for c in $commits; do + git show --diff-merges=off -t $c -- $SRC 2>/dev/null >> "${INFO[$((i+1))]}".diff + done + [ -e "${INFO[$((i+1))]}".diff ] || echo "Warning: no commits found for $STUDENT" + + + else + + echo -e "Commits for $(grep ${INFO[$((i+1))]} ../$LIST | sed 's/,/ /g')\n" + + while true; do + grep -i "${INFO[$((i+1))]}" commits-code.csv | column -s ',' -t + echo + read -p "Show commit (hash or empty to exit): " show + [ -z "$show" ] && break + git show --diff-merges=off $show -- $SRC 2>/dev/null + echo + done + + fi + + cd .. + + done + +} + +# delete all repos with all also delete repos.list +cleanup(){ + + for j in ${REPOS[@]/*\/}; do + rm -rf "$j" + done + + [ "$ALL" = "1" ] && rm -f "$CONF" + +} + + +USAGE="$0\n +\t-c|--config repos-hw.conf,Use repos-hw.conf\n +\tsetup organisation-name [pattern],Generate a repos.conf file and optionally limit it to pattern\n +\tinit,Clone all repos listed in repos.conf\n +\tcopy br file1 [file2] [file3] ...,Copy files to branch br of all repos listed in repos.conf. If br is @all then copy to all branches of all repos\n +\tpush,Add commit pull push\n +\treport [-i] [-dev] [range],Create a report.md for each team.\n,-i to post comments in gitea issues\n,-dev to pull dev branch\n,range: empty (all commit) | tag (all commits until tag) | tag1..tag2 (all commits between tag1 and tag2)\n +\tcontrib [-d] [pattern],List all commits by all students or team matching pattern. If -d is specified all commits of each student are dumped to .diff files\n +\thelp,Display this help\n +\tcleanup [all],Delete all repos and also repos.conf when all is specified" + +[ -z "$1" ] && echo -e "$USAGE" |column -t -s , && exit -1 + +while [ "$1" ]; do + + case "$1" in + + cleanup) + shift + ALL=0 + [ "$1" = "all" ] && ALL=1 && shift + CLEANUP=1; + ;; + + copy) + shift + BRANCH="$1"; shift + let i=0 + while [ "$1" ] ; do + FILES[$i]="$1"; shift + let i++ + done + if [ "$BRANCH" = "@all" ]; then + COPYALL=1 + else + COPY=1 + fi + ;; + + report) + BRANCH=master; shift + while [ "$1" ]; do + case "$1" in + -i) TISSUE=1; shift;; + -dev) BRANCH=dev; shift;; + *) RANGE="$1"; shift;; + esac + done + STATS=1 + ;; + + contrib) + shift + while [ "$1" ]; do + [ "$1" = "-d" ] && DUMP=1 && shift + [ -z "$STUDENT" ] && STUDENT="$1" && shift + done + QUERY=1 + ;; + + init) + shift + INIT=1 + PROTO=$1 + shift + ;; + + push) + shift + PUSH=1 + ;; + + setup) + shift; + [ "$1" ] || echoerr -1 "You must specify an organisation" + ORG="$1"; shift; PATTERN=.; [ "$1" ] && PATTERN="$1" && shift + SETUP=1 + ;; + + -c|--config) + shift + CONF=$1 + shift + ;; + + *) + echo -e "$USAGE" |column -t -s , + exit -1 + ;; + + esac + +done + + +if ! [ -e .env ]; then + echo "Warning: .env unset, setting it up now." + read -p "Gitea token: " GITEA_ACCESS_TOKEN + echo "# Ensure the token has proper permissions (repository (r) + issue (rw))" > .env + echo "GITEA_ACCESS_TOKEN=${GITEA_ACCESS_TOKEN}" >> .env + . .env +else + . .env +fi + +[ x$SETUP = x1 ] && systemcheck && setup && exit + +[ ! -e "$CONF" ] && echoerr -2 "no $CONF found, please first run: $0 setup organisation-name" + +. $CONF + +[ x$CLEANUP = x1 ] && cleanup && exit +[ x$COPY = x1 ] && copy && exit +[ x$COPYALL = x1 ] && copyall && exit +[ x$STATS = x1 ] && report && exit +[ x$QUERY = x1 ] && contrib && exit +[ x$PUSH = x1 ] && push && exit +[ x$INIT = x1 ] && init && exit +