chore: teabag import

Readme.md

@@ -0,0 +1,7 @@
+# JOJ3 helper scripts
+
+Collection of short shell scripts to simplify work with JOJ3. 
+
+- `admin`: scripts and configuration files used to deploy and setup JOJ3
+- `tt`: scripts used by TT to simplify routine tasks such as copying files to students repositories or collecting contribution statistics for group projects
+

tt/Readme.md

@@ -0,0 +1,49 @@
+# JOJ helper scripts for TT
+
+## `teabag.sh`
+
+This script contains simple shell recipes to *complement* [joint-teapot](https://focs.ji.sjtu.edu.cn/git/JOJ/Joint-Teapot) features. It has two main purposes: (i) copy files to students repositories and (ii) analyze contribution for group works.
+
+### Usage
+
+Run `teabag.sh -h` for detailed help. Common workflow: 
+
+1. Preparation: 
+  - run with `setup` 
+  - run with `init`
+  - use joint-teapot to generate a CSV file for groups (optional: used for contribution analysis)
+2. Copy files feature: 
+  - run with `copy` 
+  - run with `push`
+  - inform students new files were pushed and they should `pull` the changes
+3. Contribution analysis feature:
+  - run with `report`
+  - run with `contrib` (optional)
+
+Note. TT should create template repositories and only use `teabag.sh` for adjustments along the
+semester.
+
+### FAQ
+
+**Q: Why is teabag using HTTP to clone repositories instead of SSH?**
+
+A: As SJTU firewall very quickly blocks SSH requests, processing repositories (eg. cloning) is
+extremely slow. The firewall being less strict with HTTP, all repositories can be handled 
+smoothly by `teabag.sh`.
+
+**Q: I input my Jaccount password when prompted for a password but it fails, why?**
+
+A: Input your Gitea token at the password prompt.
+
+**Q: What permissions should I give to my token?**
+
+A: For security reasons, always keep permissions as minimal as possible. This script only requires 
+`write` on repository and optionally `write` on issues if using the contribution analysis feature
+to post the results in Gitea issues.
+
+**Q: I'm prompted for my token for each repository, is there a way to input it only once?**
+
+A: Yes, `man git-credential`. While you can directly add your Gitea token to `$HOME/.gitconfig`, it
+is more secure to use a *password manager* such as [pass](https://www.passwordstore.org/) to store
+your credentials. Once properly setup you will not be prompted again for your token!
+

tt/teabag.sh

@@ -0,0 +1,556 @@
+#!/bin/bash
+
+GITEA="https://focs.ji.sjtu.edu.cn/git"
+GITEAAPI="https://focs.ji.sjtu.edu.cn/git/api/v1"
+GITURL="https://git@focs.ji.sjtu.edu.cn/git"
+
+# defaults
+CONF=repos.conf
+DEPS="jq curl"
+RECOMS="lowdown"
+
+#shopt -s extglob
+
+echoerr() { ERR=$1; shift; echo "Error: $@" 1>&2; exit $ERR; }
+
+# curl + jq needed for creating repo.list
+systemcheck() {
+
+  for i in $DEPS; do 
+		which $i >/dev/null || echoerr -2 "$i not found, please install it first" 
+	done
+
+  for i in $RECOMS; do 
+		which $i >/dev/null || echo "Warning: $i not found, some features such as displaying reports might not work well" 
+  done
+
+}
+
+# create a list of repos 
+setup() {
+
+#	rm -f "$CONF"
+
+  if [ -e "$CONF" ]; then 
+    echo "WARNING: $CONF already exists!"
+    read -n1 -p "Delete and continue, edit and exit, or exit? (d/e/X) " confwarn
+
+    echo
+
+    case "$confwarn" in
+      d) echo rm -f "$CONF"
+        ;;
+      e) sensible-editor "$CONF"
+         return
+        ;;
+      *) exit
+        ;;
+    esac
+  fi
+
+	l=50 
+	let i=1 
+
+  while [ $l = 50 ]; do
+
+		repos="$(curl -s "$GITEAAPI/orgs/$ORG/repos?limit=50&page=$i" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | jq -r '.[] | .full_name')"
+
+		l=$(echo "$repos" | wc -l)
+    REPOS="$REPOS$(echo -e "$repos" | sed '/'$PATTERN'/I !d; s/^/\t/; s/$/ \\/g')" >> "$CONF"
+
+		let i++
+
+	done
+
+  REPOS=$(sort <<< $REPOS)
+
+  cat << EOF > "$CONF"
+# Edit and ensure teams are in increasing order
+# notde: this is important if reports are posted on gitea
+REPOS=( \\
+$REPOS
+)
+
+# For JOJ scores, specify the scoreboard filename, eg. scoreboard-p1.csv
+SCOREBOARD=
+
+# For contribution statistics, specify the file entensions of the source code
+# eg. SRC=('***.cc' '***.cpp' '***.h' '***.hh')
+SRC=() 
+
+# Patterns of commits to ignore in contributions, eg. auto-generated code
+# comma separated list of patterns
+# eg. SKIP="chore: messenger, build(Visuals)"
+SKIP=
+
+# To post a report in an issue dedicated to each team specify:
+# - the repo where to post the issue (eg. teaching-team)
+# - the issue number of the 1st team, eg. 24
+# note: issues must already exist and be consecutive, eg. team01->24, team02->25, etc.
+REPORTREPO=
+REPORTISSUE=
+EOF
+
+  echo "IMPORTANT: make sure to manually edit $CONF and apply necessary changes"
+
+}
+
+# clone all repos 
+init() {
+
+	# for i in ${REPOS[@]}; do 
+	# 	git clone "$GITURL/$i" 
+	# done 
+
+	# echo "Set up: email, name, merge strategy, lfs"
+	
+# # for bots use a different email address
+	# read -p "Input the jaccount or bot account to use: " jaccount
+  # if [ -n "${jaccount%bot-*}" ]; then
+    # email=$jaccount@sjtu.edu.cn
+  # else
+    # email=$jaccount@focs.ji.sjtu.edu.cn
+  # fi
+
+	# read -p "Input your name: " name 
+
+	# for j in ${REPOS[@]/*\/}; do
+	# 	cd $j
+    # git config user.email $email
+	# 	git config user.name  "$name"
+	#   git config pull.rebase true 
+	# 	git lfs install > /dev/null
+	# 	cd ..
+	# done
+
+  echo "Cloning JOJ repo"
+  if [ -d "${REPOS[0]%/*}-joj" ]; then
+    echo "WARNING: ${REPOS[0]%/*}-joj directory already exists, trying to pull..."
+    cd "${REPOS[0]%/*}-joj"
+    git pull
+  else
+    git clone -b grading $GITURL/${REPOS[0]%/*}/${REPOS[0]%/*}-joj
+  fi
+
+}
+
+# copy files to all repos listed in repos.list
+copy() {
+
+	RSYNC=$(which rsync)
+	[ -z $RSYNC ] && RSYNC="cp -r" && echo Warning: rsync not found, using cp instead
+
+	for j in ${REPOS[@]/*\/}; do
+
+		echo "Processing $j..."
+
+			cd $j; git switch $BRANCH 2>/dev/null; git pull; cd ..
+
+			sync_files
+
+	done 
+
+	echo "To complete the process, rerun with the push option."
+
+}
+		
+copyall() {
+
+  # cleanup
+	rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md
+  
+	echo "Warning: copying to all branches of all repos. When completed DO NOT run again with the push option."
+  read -n 1 -p "Please y to confirm: " confirm
+
+  [ "$confirm" != "y" ] && echo "Canceled" && exit 
+
+	RSYNC=$(which rsync)
+	[ -z $RSYNC ] && RSYNC="cp -r" && echo Warning: rsync not found, using cp instead
+
+	read -p "Commit message: " commit
+
+	for j in ${REPOS[@]/*\/}; do
+
+		echo "Processing $j..."
+
+# many cd to ensure sync_files called from same dir as repos.list
+			cd $j; 
+			branches=$(git branch -a | sed '/\// !d; s,.*origin/,,g' | sort -u);
+			cd ..
+
+			for k in $branches; do 
+				echo "Processing branch $k..."
+				cd $j
+				git switch $k  2>/dev/null
+				git pull
+				cd ..
+				sync_files
+				cd $j
+				git add .
+				git commit -m"$commit"
+				git push
+				cd ..
+			done 
+
+		done
+
+}
+
+# file path relative or absolute
+# should be called from same dir as repos.list
+sync_files() {
+
+	for i in "${FILES[@]}"; do 
+			if [ -e "$i" ]; then
+				$RSYNC -a "${i%/}" "$j"
+			else
+			 echo "Warning: file \"$i\" not found, skipping" 	
+			fi
+	done
+}
+
+# add commit pull push 
+push() {
+
+  # cleanup
+	rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md
+
+	read -p "Commit message: " commit
+
+	for j in ${REPOS[@]/*\/}; do
+		cd $j
+		echo "Processing $j..."
+		git add .
+		git commit -m"$commit"
+		git pull
+		git push
+		cd ..
+	done 
+
+}
+
+report() {
+
+# update grading
+  cd ${REPOS[0]%/*}-joj
+  git pull
+  cd ..
+
+	for t in ${REPOS[@]}; do
+
+    trepo=${t/*\/}
+    TEAM=($(sed '/'$trepo'/ !d; s/ /+/g; s/^\([^,]*\),[^,]*,\([^@]*\)@.*/\1 \2/g;' "${CONF%.conf}.csv"))
+
+		echo "Processing $trepo..."
+
+    [ -d "$trepo" ] || echoerr 10 "$trepo no found, make sure to first run $0 with init parameter"
+		cd $trepo
+
+    # cleanup
+		rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md
+
+    git stash -q
+    git switch -q $BRANCH
+    if ! git pull --ff-only -q; then trepofail="$trepo $trepofail"; fi
+
+    for i in ${RANGE//\./ }; do git tag -l | grep -q $i || tagfail="$tagfail- $i ($trepo)\n"; done
+
+    echo -n "## Git usage report for $trepo " >> report.md
+    [ -n "$RANGE" ] && echo "($RANGE)" >> report.md
+    echo >> report.md
+
+		report_release 
+		report_contrib
+		report_commits
+    report_joj
+			
+    if [ x$TISSUE = x1 ]; then
+      curl -s -X POST "$GITEAAPI/repos/${t%/*}/$REPORTREPO/issues/$REPORTISSUE/comments" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" -H 'accept: application/json' -H 'Content-Type: application/json' -d "{\"body\": \"$(cat report.md)\"}" > /dev/null
+      let REPORTISSUE++
+    else
+      echo
+  		lowdown -Tterm report.md
+      echo
+    fi
+
+# students with no commit    
+    NC="$(sed '/[^,]*,[[:space:]]*0/ !d; s/[[:space:]]*\([^,]*\),.*/- \1('$trepo')/' commits.csv)"
+    [ -n "$NC" ] && NOCOM="$NC\n$NOCOM" 
+
+    cd ..
+
+	done
+
+  if [ -n "$trepofail" ]; then
+    echo -e "\n## Repo pull failure"
+    echo -e "\n$trepofail"
+  fi
+
+  if [ -n "$tagfail" ]; then
+    echo -e "\n## Missing tags"
+    echo -e "\n$tagfail"
+  fi
+
+  echo -e "\n## Students with no commit for $RANGE"
+  echo -e "\n$NOCOM"
+
+}
+
+report_release() {
+
+    echo -e "### Releases\n" >> report.md
+
+    R=($(curl -s "$GITEAAPI/repos/$t/releases" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | jq -r '.[] | .tag_name, .published_at, .target_commitish, .name' | sed 's/ /_/g'))
+
+    let i=0
+    while [ -n "${R[i]}" ]; do
+      RS="**failed**"
+      curl -s "$GITEAAPI/repos/$t/actions/workflows/release.yaml/badge?tag=${R[i]}&event=release" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | grep -q success && RS="**successful**"
+      echo "- ${R[i+3]//_/ } released on $(date -d ${R[i+1]} '+%B %d at %H:%M') from ${R[i+2]} with tag ${R[i]}: $RS" >> report.md
+      let i+=4
+    done
+
+}
+
+report_contrib() {
+
+# all commits
+  git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE 2>/dev/null | tr "\n" " " | tr "@" "\n" | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-all.csv
+
+# code commits only 
+# SRC (array) and SKIP (string) defined in CONF file
+# SRC=('***.c' '***.h')
+# SKIP="chore:.*messenger, build(visuals)"
+
+  if [ -n "$SKIP" ]; then
+    git log --no-merges --pretty="@%aL^%s^%h^" --shortstat -i --invert-grep --grep="${SKIP//,/\\|}" $RANGE -- "${SRC[@]}" 2>/dev/null | tr "\n" " " | tr "@" "\n" | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv 
+  else
+    git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE -- "${SRC[@]}" | tr "\n" " " | tr "@" "\n" 2>/dev/null | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv
+  fi
+
+  n=${#TEAM[@]}
+
+# loc
+  totadd=$(awk -F , '{s+=$5}END{print s}' commits-code.csv)
+
+  for((i=1; i<n; i+=2)); do j=$((i-1)); grep -i "${TEAM[i]}" commits-code.csv | awk -F, -v author="${TEAM[j]}" -v totadd=$totadd '{files+=$4; inserted+=$5; deleted+=$6; delta+=$5-$6; ratio=inserted/deleted; contrib=inserted/totadd} END {printf "%26s ,%6s ,%10s ,%10s ,%12s ,%12s , %8s \n", author, files, inserted, deleted, delta, ratio,contrib}'; done | sort | sed 's/+/ /' > loc.csv
+
+# indiv commits
+  for((i=1; i<n; i+=2)); do j=$((i-1)); printf "%26s ," "${TEAM[j]}"; c=$(grep -ci ${TEAM[i]} commits-code.csv); printf "%8s \n" "$c"; done | sort | sed 's/+/ /' > commits.csv
+
+# indiv commits + loc
+  echo -e "\n### Individual contribution\n" >> report.md
+  echo "| Author                    | Commits | Files | Additions | Deletions | Delta (A-D) | Ratio (A/D) | Workload |" >> report.md
+  echo "|---------------------------|---------|-------|-----------|-----------|-------------|-------------|----------|" >> report.md
+  join -t, commits.csv loc.csv | sed 's/,/|/g; s/^\|$/|/g' >> report.md	
+
+}
+
+report_commits() {
+
+# alignment: space + GITEA + repo + commit + space
+  l=$((1+32+${#t}+15+1))
+
+# non-atomic
+  echo -e "\n### Non-atomic commits\n" >> report.md
+  printf "|    | Commits %$((l-10))s | Files | Additions | Deletions | Author               |\n" " " >> report.md
+  printf "|----| %${l}s|-------|-----------|-----------|----------------------|\n" " " | tr ' ' - >> report.md
+  
+  if [ x$TISSUE = x1 ]; then
+    awk -v gitea=$GITEA -v cnt=0 -v repo=$t -F, '{if($5>80 || $6>80) { cnt++; printf "| %2s | [%s](%s/%s/commit/%s) | %s | %s | %s | %s |\n", cnt, $2, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md
+  else
+    awk -v gitea=$GITEA -v cnt=0 -v repo=$t -F, '{if($5>80 || $6>80) { cnt++; printf "| %2s | %s/%s/commit/%s | %5s | %9s | %9s | %20s |\n", cnt, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md
+  fi
+
+# non-conventional
+  echo -e "\n### Non-conventional commits\n" >> report.md
+  awk -v cnt=0 -F , '$2 !~ /chore|feat|docs|test|ci|refactor|perf|build|revert|fix|style/ { cnt++; printf "%s. %s (%s)\n", cnt, $2, $1 }' commits-all.csv >> report.md
+
+}
+
+report_joj() {
+
+  echo -e "\n### JOJ scores\n" >> report.md
+  echo -e "**WARNING.** For more precise and detailed scores refer to [joj-mon](https://focs.ji.sjtu.edu.cn/joj-mon/d/${REPOS[0]%/*})\n" >> report.md
+
+  dos2unix < "../${REPOS[0]%/*}-joj/$SCOREBOARD" | sed 's/^\|$/|/g; 1 s/\(.*\)/\1\n\1/' | column -s , -t -o \| | sed -n '2 s/[^|]/-/g; 1,2 p; /'$trepo'/ p' >> report.md
+
+}
+
+contrib() {
+
+  t=$(head -1 $CONF | cut -d / -f 2)
+  LIST="${CONF%.conf}.csv"
+  INFO=($(grep -i "$STUDENT" "$LIST" | sed 's/[^,]*,[^,]*,\([^@]*\)@.*,\(.*\)/\2 \1/g'))
+
+  for((i=0;i<${#INFO[@]};i=i+2)); do
+    
+    cd ${INFO[i]}/
+    
+    [ -e commits-code.csv ] || echoerr 25 "No data found, first generate a report"
+
+    if [ x$DUMP = x1 ]; then
+      
+      STUDENT=$(grep ${INFO[$((i+1))]} ../$LIST | sed 's/,/ /g')
+      echo -e "Commits for $STUDENT dumped to ${INFO/[i]}/${INFO[$((i+1))]}.diff"
+
+      rm -f "${INFO[$((i+1))]}".diff
+      commits=$(grep -i "${INFO[$((i+1))]}" commits-code.csv | awk -F, '{print $3}')
+      for c in $commits; do 
+        git show --diff-merges=off -t $c -- $SRC 2>/dev/null >> "${INFO[$((i+1))]}".diff
+      done
+      [ -e  "${INFO[$((i+1))]}".diff ] || echo "Warning: no commits found for $STUDENT"
+
+
+    else
+
+    echo -e "Commits for $(grep ${INFO[$((i+1))]} ../$LIST | sed 's/,/ /g')\n"
+
+      while true; do
+        grep -i "${INFO[$((i+1))]}" commits-code.csv |  column -s ',' -t
+        echo
+        read -p "Show commit (hash or empty to exit): " show
+        [ -z "$show" ] && break
+        git show --diff-merges=off $show -- $SRC 2>/dev/null
+        echo
+      done
+
+    fi
+
+    cd ..
+
+  done
+
+}
+
+# delete all repos with all also delete repos.list
+cleanup(){
+
+	for j in ${REPOS[@]/*\/}; do
+		rm -rf "$j"
+	done
+
+	[ "$ALL" = "1" ] && rm -f "$CONF"
+
+}
+
+
+USAGE="$0\n
+\t-c|--config repos-hw.conf,Use repos-hw.conf\n
+\tsetup organisation-name [pattern],Generate a repos.conf file and optionally limit it to pattern\n
+\tinit,Clone all repos listed in repos.conf\n
+\tcopy br file1 [file2] [file3] ...,Copy files to branch br of all repos listed in repos.conf. If br is @all then copy to all branches of all repos\n
+\tpush,Add commit pull push\n
+\treport [-i] [-dev] [range],Create a report.md for each team.\n,-i to post comments in gitea issues\n,-dev to pull dev branch\n,range: empty (all commit) | tag (all commits until tag) | tag1..tag2 (all commits between tag1 and tag2)\n
+\tcontrib [-d] [pattern],List all commits by all students or team matching pattern. If -d is specified all commits of each student are dumped to .diff files\n
+\thelp,Display this help\n
+\tcleanup [all],Delete all repos and also repos.conf when all is specified"
+
+[ -z "$1" ] && echo -e "$USAGE" |column -t -s , && exit -1
+
+while [ "$1" ]; do
+
+	case "$1" in
+
+		cleanup)
+			shift
+			ALL=0
+			[ "$1" = "all" ] && ALL=1 && shift
+			CLEANUP=1;
+			;;
+
+		copy)
+			shift 
+			BRANCH="$1"; shift
+			let i=0
+			while [ "$1" ] ; do 
+				FILES[$i]="$1"; shift
+				let i++
+			done
+			if [ "$BRANCH" = "@all" ]; then 
+				COPYALL=1
+			else
+				COPY=1
+			fi
+			;;
+
+		report)
+      BRANCH=master; shift
+      while [ "$1" ]; do
+        case "$1" in 
+          -i) TISSUE=1; shift;;
+          -dev) BRANCH=dev; shift;;
+          *) RANGE="$1"; shift;;
+        esac
+      done
+			STATS=1
+			;;
+
+    contrib)
+      shift
+      while [ "$1" ]; do
+        [ "$1" = "-d" ] && DUMP=1 && shift
+        [ -z "$STUDENT" ] && STUDENT="$1" && shift
+      done
+      QUERY=1
+      ;;
+
+		init)
+			shift
+			INIT=1
+			PROTO=$1 
+			shift
+			;;
+
+		push)
+			shift
+			PUSH=1
+			;;
+
+		setup)	
+			shift; 
+			[ "$1" ] || echoerr -1 "You must specify an organisation" 
+			ORG="$1"; shift; PATTERN=.; [ "$1" ] && PATTERN="$1" && shift
+			SETUP=1 
+			;;
+
+		-c|--config)
+			shift 
+			CONF=$1
+			shift
+			;;
+
+		*)
+			echo -e "$USAGE" |column -t -s ,
+			exit -1
+			;;
+
+	esac
+
+done
+
+
+if ! [ -e .env ]; then
+	echo "Warning: .env unset, setting it up now."
+	read -p "Gitea token: " GITEA_ACCESS_TOKEN
+  echo "# Ensure the token has proper permissions (repository (r) + issue (rw))" > .env 
+	echo "GITEA_ACCESS_TOKEN=${GITEA_ACCESS_TOKEN}" >> .env 
+	. .env
+else
+	. .env
+fi
+
+[ x$SETUP = x1 ] && systemcheck && setup && exit
+
+[ ! -e "$CONF" ] && echoerr -2 "no $CONF found, please first run: $0 setup organisation-name"
+
+. $CONF
+
+[ x$CLEANUP = x1 ] && cleanup && exit
+[ x$COPY = x1 ]    && copy    && exit
+[ x$COPYALL = x1 ] && copyall && exit
+[ x$STATS = x1 ]   && report  && exit
+[ x$QUERY = x1 ]   && contrib && exit
+[ x$PUSH = x1 ]    && push    && exit
+[ x$INIT = x1 ]    && init    && exit
+