feat: plugin support and clearner report

tt/teabag.sh

@@ -72,6 +72,9 @@ $REPOS
 # For JOJ scores, specify the scoreboard filename, eg. scoreboard-p1.csv
 SCOREBOARD=
 
+# Max acceptable commit length (atomic commits expected)
+MAXCOML=80
+
 # For contribution statistics, specify the file entensions of the source code
 # eg. SRC=('***.cc' '***.cpp' '***.h' '***.hh')
 SRC=() 
@@ -81,10 +84,12 @@ SRC=()
 # eg. SKIP="chore: messenger, build(Visuals)"
 SKIP=
 
-# To post a report in an issue dedicated to each team specify:
+# To post a report in an issue/comment for each team specify:
 # - the repo where to post the issue (eg. teaching-team)
-# - the issue number of the 1st team, eg. 24
+# - the issue number, eg. 24 (issue must already exist)
-# note: issues must already exist and be consecutive, eg. team01->24, team02->25, etc.
+# - add a + in front of issue number to post in a different issue for each team
+# - with a + issues must exist and be consecutive, eg. team01->24, team02->25, etc.
+# eg. REPORTISSUE=24 to post all reports in issue 24 or REPORTISSUE=+24 to post in consecutive issues
 REPORTREPO=
 REPORTISSUE=
 EOF
@@ -155,7 +160,7 @@ copy() {
 copyall() {
 
   # cleanup
-	rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md
+	rm -f commits-code.csv commits-all.csv commits-loc.csv commits.csv report.md
   
 	echo "Warning: copying to all branches of all repos. When completed DO NOT run again with the push option."
   read -n 1 -p "Please y to confirm: " confirm
@@ -211,7 +216,7 @@ sync_files() {
 push() {
 
   # cleanup
-	rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md
+	rm -f commits-code.csv commits-all.csv commit-loc.csv commits.csv report.md
 
 	read -p "Commit message: " commit
 
@@ -229,58 +234,91 @@ push() {
 
 report() {
 
+  TEAMS=0
+  if [ -e "${CONF%.conf}.csv" ]; then 
+    TEAMS=1
+  else
+    TEAMS=0
+    echo "WARNING. File "${CONF%.conf}.csv" not found, assuming individual work."
+    read -p "Press Enter to continue"
+  fi
+
 # update grading
   cd ${REPOS[0]%/*}-joj
   git pull
   cd ..
 
+  let nb=1
 	for t in ${REPOS[@]}; do
 
+    unset TEAM
+    declare -A TEAM
     trepo=${t/*\/}
-    TEAM=($(sed '/'$trepo'/ !d; s/ /+/g; s/^\([^,]*\),[^,]*,\([^@]*\)@.*/\1 \2/g;' "${CONF%.conf}.csv"))
 
 		echo "Processing $trepo..."
 
-    [ -d "$trepo" ] || echoerr 10 "$trepo no found, make sure to first run $0 with init parameter"
+    [ -d "$trepo" ] || echoerr 10 "$trepo not found, make sure to first run $0 with init parameter"
 		cd $trepo
 
     # cleanup
-		rm -f commits-code.csv commits-all.csv loc.csv commits.csv report.md
+		rm -f commits-code.csv commits-all.csv commits-loc.csv commits.csv report.md
-
     git stash -q
-    git switch -q $BRANCH
+
+    git switch master
+    git tag -d $(git tag -l)
+    git fetch --tags
+
     if ! git pull --ff-only -q; then trepofail="$trepo $trepofail"; fi
+    
+    echo -n "## Git usage report for [$trepo]($GITEA/$t) " >> report.md
+
+    report_range
+    if [ -n "$RANGE" ]; then 
+      if ! git checkout -q ${RANGE//*./}; then trepofail="$trepo $trepofail"; fi
+      task=" ($RANGE)"
+    elif [ -n "$BRANCH" ]; then
+      git switch -q $BRANCH
+      task=" ($BRANCH branch)"
+    fi
+
+    printf "$task [%.2d/%.2d]" $nb ${#REPOS[@]} >> report.md
 
     for i in ${RANGE//\./ }; do git tag -l | grep -q $i || tagfail="$tagfail- $i ($trepo)\n"; done
 
-    echo -n "## Git usage report for $trepo " >> report.md
-    [ -n "$RANGE" ] && echo "($RANGE)" >> report.md
     echo >> report.md
 
-		report_release 
+		report_progress
-		report_contrib
+ 		report_db
+		[ $TEAMS = 1 ] && report_contrib
 		report_commits
-    report_joj
+    [ -n "$SCOREBOARD" ] && report_joj
 			
+# back to default    
+    git switch master
+
     if [ x$TISSUE = x1 ]; then
       curl -s -X POST "$GITEAAPI/repos/${t%/*}/$REPORTREPO/issues/$REPORTISSUE/comments" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" -H 'accept: application/json' -H 'Content-Type: application/json' -d "{\"body\": \"$(cat report.md)\"}" > /dev/null
-      let REPORTISSUE++
+      [[ "$REPORTISSUE" =~ "+" ]] && let REPORTISSUE++
     else
       echo
   		lowdown -Tterm report.md
       echo
     fi
 
-# students with no commit    
+# students with no commit
-    NC="$(sed '/[^,]*,[[:space:]]*0/ !d; s/[[:space:]]*\([^,]*\),.*/- \1('$trepo')/' commits.csv)"
+    if [ $TEAMS = 1 ]; then
-    [ -n "$NC" ] && NOCOM="$NC\n$NOCOM" 
+      NC="$(sed '/[^,]*,[[:space:]]*0/ !d; s/[[:space:]]*\([^,]*\),.*/- \1('$trepo')/' commits.csv)"
+      [ -n "$NC" ] && NOCOM="$NC\n$NOCOM" 
+    fi
 
     cd ..
 
+    let nb++
+
 	done
 
   if [ -n "$trepofail" ]; then
-    echo -e "\n## Repo pull failure"
+    echo -e "\n## Repo failure$task"
     echo -e "\n$trepofail"
   fi
 
@@ -289,31 +327,64 @@ report() {
     echo -e "\n$tagfail"
   fi
 
-  echo -e "\n## Students with no commit for $RANGE"
+  if [ $TEAMS = 1 ]; then
-  echo -e "\n$NOCOM"
+    echo -e "\n## Students with no commit for $RANGE"
+    echo -e "\n$NOCOM"
+  fi
 
 }
 
-report_release() {
+report_range() {
+    RELEASES=( skip )
+    R=( \
+      $(curl -s "$GITEAAPI/repos/$t/releases" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | \
+      jq -r '.[] | .tag_name, .published_at, .target_commitish, .name' | \
+      sed 's/ /_/g') \
+    )
+    for((i=${#R[@]}-4; i>=0; i-=4)); do 
+      RELEASES+=( ${R[i]} )
+    done
 
-    echo -e "### Releases\n" >> report.md
+    if [[ "$RRANGE" =~ ".." ]]; then
+      RANGE="${RELEASES[${RRANGE%%..*}]}..${RELEASES[${RRANGE//*..}]}"
+    elif [ -n "$RRANGE" ]; then 
+      RANGE=${RELEASES[${RRANGE}]}; 
+    else
+      RANGE=""
+    fi
+}
 
-    R=($(curl -s "$GITEAAPI/repos/$t/releases" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | jq -r '.[] | .tag_name, .published_at, .target_commitish, .name' | sed 's/ /_/g'))
+report_progress() {
+
+    echo -e "### Progress\n" >> report.md
+
+    echo -e "\nReleases:\n" >> report.md
 
     let i=0
     while [ -n "${R[i]}" ]; do
       RS="**failed**"
-      curl -s "$GITEAAPI/repos/$t/actions/workflows/release.yaml/badge?tag=${R[i]}&event=release" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | grep -q success && RS="**successful**"
+      curl -s "$GITEAAPI/repos/$t/actions/workflows/release.yaml/badge?tag=${R[i]}&event=release" -H "Authorization: Bearer ${GITEA_ACCESS_TOKEN}" | \
+        grep -q success && RS="**successful**"
       echo "- ${R[i+3]//_/ } released on $(date -d ${R[i+1]} '+%B %d at %H:%M') from ${R[i+2]} with tag ${R[i]}: $RS" >> report.md
       let i+=4
     done
 
+    loc=0
+    src="${SRC[@]//*./}"; src=".*\.\(${src// /\\|}\)$"
+    src=$(find . -regex ${src} -print)
+    [ -n "$src" ] && loc=$(wc -l --total=only -- ${src})
+    echo -e "\nTotal lines of code: $loc\n" >> report.md
+
+    echo -e "\nRepository size: $(du -sh | cut -f 1)\n" >> report.md
+
 }
 
-report_contrib() {
+report_db() {
 
 # all commits
-  git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE 2>/dev/null | tr "\n" " " | tr "@" "\n" | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-all.csv
+  git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE 2>/dev/null | \
+    tr "\n" " " | tr "@" "\n" | \
+    sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; s/"//g; /^$/ d' > commits-all.csv
 
 # code commits only 
 # SRC (array) and SKIP (string) defined in CONF file
@@ -321,26 +392,49 @@ report_contrib() {
 # SKIP="chore:.*messenger, build(visuals)"
 
   if [ -n "$SKIP" ]; then
-    git log --no-merges --pretty="@%aL^%s^%h^" --shortstat -i --invert-grep --grep="${SKIP//,/\\|}" $RANGE -- "${SRC[@]}" 2>/dev/null | tr "\n" " " | tr "@" "\n" | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv 
+    git log --no-merges --pretty="@%aL^%s^%h^" --shortstat -i --invert-grep --grep="${SKIP//,/\\|}" $RANGE -- "${SRC[@]}" 2>/dev/null | \
+      tr "\n" " " | tr "@" "\n" | \
+      sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv 
   else
-    git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE -- "${SRC[@]}" | tr "\n" " " | tr "@" "\n" 2>/dev/null | sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv
+    git log --no-merges --pretty="@%aL^%s^%h^" --shortstat $RANGE -- "${SRC[@]}" | tr "\n" " " | tr "@" "\n" 2>/dev/null | \
+      sed 's/changed, \([0-9]*\) d/changed, 0 insertion(+), \1 d/g; s/ file[s]* changed,\|insertion[s]*(+)[,]*\|deletion[s]*(-)/^/g; s/,/ -/g; s/\^[[:space:],]*/,/g; /^$/ d' > commits-code.csv
   fi
 
-  n=${#TEAM[@]}
+}
+
+report_contrib() {
+
+  while read email student; do TEAM[${email%@*}]="$student"; done <<< $(awk -F, '$4 ~ /'$trepo'/ {printf "%s %s\n", $3, $1}' "../${CONF%.conf}.csv")
+  
+  # jaccount to name
+  for i in "${!TEAM[@]}"; do
+    sed -i "s/$i/${TEAM[$i]}/g" commits-all.csv commits-code.csv
+  done
 
 # loc
   totadd=$(awk -F , '{s+=$5}END{print s}' commits-code.csv)
 
-  for((i=1; i<n; i+=2)); do j=$((i-1)); grep -i "${TEAM[i]}" commits-code.csv | awk -F, -v author="${TEAM[j]}" -v totadd=$totadd '{files+=$4; inserted+=$5; deleted+=$6; delta+=$5-$6; ratio=inserted/deleted; contrib=inserted/totadd} END {printf "%26s ,%6s ,%10s ,%10s ,%12s ,%12s , %8s \n", author, files, inserted, deleted, delta, ratio,contrib}'; done | sort | sed 's/+/ /' > loc.csv
+  for i in "${!TEAM[@]}"; do
+    author="${TEAM[$i]}"
+    grep -i "$author" commits-code.csv | \
+      awk -F, -v author="$author" -v totadd=$totadd '{files+=$4; inserted+=$5; deleted+=$6; delta+=$5-$6; ratio=inserted/deleted; contrib=inserted/totadd} END {printf "%26s ,%6s ,%10s ,%10s ,%12s ,%12s , %8s \n", author, files, inserted, deleted, delta, ratio,contrib}'
+  done | \
+    sort > commits-loc.csv
 
 # indiv commits
-  for((i=1; i<n; i+=2)); do j=$((i-1)); printf "%26s ," "${TEAM[j]}"; c=$(grep -ci ${TEAM[i]} commits-code.csv); printf "%8s \n" "$c"; done | sort | sed 's/+/ /' > commits.csv
+  for i in "${!TEAM[@]}"; do
+    author="${TEAM[$i]}"
+    printf "%26s ," "${TEAM[$i]}"; 
+    c=$(grep -ci "$author" commits-code.csv); 
+    printf "%8s \n" "$c"; 
+  done | \
+    sort > commits.csv
 
 # indiv commits + loc
   echo -e "\n### Individual contribution\n" >> report.md
   echo "| Author                    | Commits | Files | Additions | Deletions | Delta (A-D) | Ratio (A/D) | Workload |" >> report.md
   echo "|---------------------------|---------|-------|-----------|-----------|-------------|-------------|----------|" >> report.md
-  join -t, commits.csv loc.csv | sed 's/,/|/g; s/^\|$/|/g' >> report.md	
+  join -t, commits.csv commits-loc.csv | tee commits-stats.csv | sed 's/,/|/g; s/^\|$/|/g' >> report.md	
 
 }
 
@@ -351,26 +445,28 @@ report_commits() {
 
 # non-atomic
   echo -e "\n### Non-atomic commits\n" >> report.md
+
+  echo -e "Commits larger than $MAXCOML lines.\n" >> report.md
+
   printf "|    | Commits %$((l-10))s | Files | Additions | Deletions | Author               |\n" " " >> report.md
   printf "|----| %${l}s|-------|-----------|-----------|----------------------|\n" " " | tr ' ' - >> report.md
   
   if [ x$TISSUE = x1 ]; then
-    awk -v gitea=$GITEA -v cnt=0 -v repo=$t -F, '{if($5>80 || $6>80) { cnt++; printf "| %2s | [%s](%s/%s/commit/%s) | %s | %s | %s | %s |\n", cnt, $2, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md
+    awk -v gitea=$GITEA -v ml=$MAXCOML -v cnt=0 -v repo=$t -F, '{if($5>ml || $6>ml) { cnt++; printf "| %2s | [%s](%s/%s/commit/%s) | %s | %s | %s | %s |\n", cnt, $2, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md
   else
-    awk -v gitea=$GITEA -v cnt=0 -v repo=$t -F, '{if($5>80 || $6>80) { cnt++; printf "| %2s | %s/%s/commit/%s | %5s | %9s | %9s | %20s |\n", cnt, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md
+    awk -v gitea=$GITEA -v ml=$MAXCOML -v cnt=0 -v repo=$t -F, '{if($5>ml || $6>ml) { cnt++; printf "| %2s | %s/%s/commit/%s | %5s | %9s | %9s | %20s |\n", cnt, gitea, repo, $3, $4, $5, $6, $1}}' commits-code.csv >> report.md
   fi
 
-# non-conventional
+  # non-conventional
   echo -e "\n### Non-conventional commits\n" >> report.md
-  awk -v cnt=0 -F , '$2 !~ /chore|feat|docs|test|ci|refactor|perf|build|revert|fix|style/ { cnt++; printf "%s. %s (%s)\n", cnt, $2, $1 }' commits-all.csv >> report.md
+  awk -v cnt=0 -F , '$2 !~ /^chore|^feat|^docs|^test|^ci|^refactor|^perf|^build|^revert|^fix|^style/ { cnt++; printf "%s. %s (%s)\n", cnt, $2, $1 }' commits-all.csv >> report.md
 
 }
 
 report_joj() {
 
   echo -e "\n### JOJ scores\n" >> report.md
-  echo -e "**WARNING.** For more precise and detailed scores refer to [joj-mon](https://focs.ji.sjtu.edu.cn/joj-mon/d/${REPOS[0]%/*})\n" >> report.md
+  echo -e "**WARNING.** For more precise and detailed scores refer to [joj-mon](https://focs.ji.sjtu.edu.cn/joj-mon/d/${REPOS[0]%/*}/${REPOS[0]%/*}?var-Filters=Repository%7C%3D%7C$t&from=now-30d)\n" >> report.md
-
   dos2unix < "../${REPOS[0]%/*}-joj/$SCOREBOARD" | sed 's/^\|$/|/g; 1 s/\(.*\)/\1\n\1/' | column -s , -t -o \| | sed -n '2 s/[^|]/-/g; 1,2 p; /'$trepo'/ p' >> report.md
 
 }
@@ -435,11 +531,12 @@ cleanup(){
 
 USAGE="$0\n
 \t-c|--config repos-hw.conf,Use repos-hw.conf\n
+\t-p|--plugin plug \"args\",Use plugin plug with arguments \"args\". Make sure arguments are quoted\n
 \tsetup organisation-name [pattern],Generate a repos.conf file and optionally limit it to pattern\n
 \tinit,Clone all repos listed in repos.conf\n
 \tcopy br file1 [file2] [file3] ...,Copy files to branch br of all repos listed in repos.conf. If br is @all then copy to all branches of all repos\n
 \tpush,Add commit pull push\n
-\treport [-i] [-dev] [range],Create a report.md for each team.\n,-i to post comments in gitea issues\n,-dev to pull dev branch\n,range: empty (all commit) | tag (all commits until tag) | tag1..tag2 (all commits between tag1 and tag2)\n
+\treport [-i] [-dev] [-r range],Create a report.md for each team.\n,-i to post comments in gitea issues\n,-dev to pull dev branch\n,-r range: missing means all commits | -r n (all commits until nth tag) | n..m (all commits between nth and mth tags)\n
 \tcontrib [-d] [pattern],List all commits by all students or team matching pattern. If -d is specified all commits of each student are dumped to .diff files\n
 \thelp,Display this help\n
 \tcleanup [all],Delete all repos and also repos.conf when all is specified"
@@ -473,12 +570,16 @@ while [ "$1" ]; do
 			;;
 
 		report)
-      BRANCH=master; shift
+      shift
       while [ "$1" ]; do
         case "$1" in 
           -i) TISSUE=1; shift;;
           -dev) BRANCH=dev; shift;;
-          *) RANGE="$1"; shift;;
+          -r) shift; RRANGE="$1"; shift;;
+          *) 
+            echo -e "$USAGE" |column -t -s ,
+            exit -1
+            ;;
         esac
       done
 			STATS=1
@@ -518,8 +619,24 @@ while [ "$1" ]; do
 			shift
 			;;
 
+    -p|--plugin)
+      shift
+      # check teabag.sh directory and current directory for teabag-plug.sh file where plug is the plugin name
+      tdir=$(dirname $0)
+      if [ -e $tdir/teabag-$1.sh ]; then 
+        . $tdir/teabag-$1.sh
+      elif [ -e ./teabag-$1.sh ]; then
+        . ./teabag-$1.sh
+      else
+        echoerr 231 "plugin \"$1\" not found, make sure teabag-$1.sh exists either in the current dir or in the same dir a teabag.sh"
+      fi
+      PLUGIN="$1"
+      shift
+      PLUGINARGS="$@"
+      shift
+      ;;
 		*)
-			echo -e "$USAGE" |column -t -s ,
+			echo -e "$USAGE" | column -t -s ,
 			exit -1
 			;;
 
@@ -551,4 +668,5 @@ fi
 [ x$QUERY = x1 ]   && contrib && exit
 [ x$PUSH = x1 ]    && push    && exit
 [ x$INIT = x1 ]    && init    && exit
+[ -n "$PLUGIN" ]   && $PLUGIN $PLUGINARGS && exit