toolbox/reports/project-fsck-status.sh

   1 #!/bin/sh
   2
   3 # Report on project repository status.
   4 # Output can be sent as email to admin with -m
   5 # Automatically runs with nice and ionice (if available)
   6
   7 set -e
   8
   9 datefmt='%Y-%m-%d %H:%M:%S %z'
  10 startdate="$(date "+$datefmt")"
  11
  12 . @basedir@/shlib.sh
  13 PATH="$cfg_basedir/bin:${PATH:-$(/usr/bin/getconf PATH)}"
  14 export PATH
  15
  16 USAGE="${0##*/} [-hmEW] [--full] [<project>...]"
  17 HELP="
  18 NAME
  19        ${0##*/} - report on \"health\" of Girocco repositories
  20
  21 SYNOPSIS
  22        $USAGE
  23
  24 DESCRIPTION
  25        The ${0##*/} script checks all Girocco repositories for
  26        consistency and includes a count of empty repositories as well.
  27
  28        The results can automatically be mailed to the Girocco admin or
  29        simply displayed on STDOUT or optionally suppressed if there are
  30        no errors or warnings.
  31
  32        Note that this utility currently provides absolutely no progress
  33        reporting even when run on a terminal so there will be no output
  34        whatsoever until all projects have been checked.
  35
  36        The \"checking\" runs using nice and, if available, ionice.
  37
  38        Projects not listed in \$chroot/etc/group will not be checked.
  39
  40 OPTIONS
  41        -h     show this help
  42
  43        -m     e-mail results to the \$Girocco::Config::admin address.
  44               This option suppresses output to STDOUT.
  45
  46        -E     Suppress output (i.e. no e-mail will be sent if -m has been used)
  47               unless at least one error has been detected.  If this flag is
  48               used WITHOUT -W and warnings occur, output will still be
  49               suppressed.
  50
  51        -W     Suppress output unless at least one error or warning has been
  52               detected.  This flag overrides -E and will always produce output
  53               when only warnings are detected.
  54
  55        --full Run a full \"git fsck\" check on each repository.  Without this
  56               option a much, much, much (5x or more) faster check is done on
  57               each repository using \"git rev-list --objects --all\" that will
  58               verify all objects reachable from any ref are present but does
  59               not perform all the validation checks nor does it verify the
  60               integrity of 'blob' objects (just that they are present).
  61
  62               If the default Girocco 'transfer.fsckObjects=true' option has
  63               been left intact, use of the \"--full\" option should not
  64               normally be necessary.
  65
  66        --no-full
  67               Disables --full.  Present for completeness.
  68
  69        --reflog
  70               Check ref logs too.  Implied by \"--full\" but not by \"--no-full\".
  71
  72        --indexed-objects
  73               Check indexed objects too.  Implied by \"--full\" but not by
  74               \"--no-full\".  Requires Git version 2.2.0 or later if not using
  75               the \"--full\" option to do it.
  76
  77        --list Show a list of all projects checked in the order checked.
  78               This is the default if any specific projects are given.
  79
  80        --no-list
  81               Do NOT show a list of each project name checked.
  82               This is the default when all projects are being checked.
  83
  84        <project>
  85               Optionally one or more Girocco project names may be given to
  86               check only those projects.
  87
  88 TIPS
  89        Leave Girocco's default 'transfer.fsckObjects=true' setting alone and
  90        have a cron job run this utility in the default --no-full mode using the
  91        -m and -E options once a week to detect repository corruption.
  92
  93        The default --no-full mode will detect any fatal errors with all
  94        reachable 'tag', 'commit' and 'tree' objects.  The --no-full mode will
  95        detect missing 'blob' objects but not corrupt 'blob' objects.
  96
  97        The --no-full mode is an order of magnitude faster than --full mode and
  98        does not complain about non-fatal object errors.
  99 "
 100
 101 usage() {
 102         if [ "${1:-1}" = "1" ]; then
 103                 printf 'usage: %s\n' "$USAGE" >&2
 104                 return 0
 105         fi
 106         printf '%s\n' "$HELP"
 107 }
 108
 109 mailresult=
 110 minoutput=0 # 0 => always output, 1 => if warn or err, 2 => if err
 111 fullcheck=
 112 onlylist=
 113 shownames=
 114 reflog=
 115 indexedobj=
 116
 117 while [ $# -gt 0 ]; do
 118         if [ "${1#-[!-]?}" != "$1" ]; then
 119                 _rest="${1#-?}"
 120                 _next="${1%"$_rest"}"
 121                 shift
 122                 set -- "$_next" "-$_rest" "$@"
 123         fi
 124         case "$1" in
 125         "-?")
 126                 usage 1 2>&1
 127                 exit 0
 128                 ;;
 129         -h|--help)
 130                 usage 2
 131                 exit 0
 132                 ;;
 133         -m|--mail)
 134                 mailresult=1
 135                 ;;
 136         -E)
 137                 [ "${minoutput:-0}" != "0" ] || minoutput=2
 138                 ;;
 139         -W)
 140                 minoutput=1
 141                 ;;
 142         --full)
 143                 fullcheck=1
 144                 ;;
 145         --no-full)
 146                 fullcheck=
 147                 ;;
 148         --reflog)
 149                 reflog=1
 150                 ;;
 151         --indexed-objects)
 152                 indexedobj=1
 153                 ;;
 154         --list)
 155                 shownames=1
 156                 ;;
 157         --no-list)
 158                 shownames=0
 159                 ;;
 160         --)
 161                 shift
 162                 break
 163                 ;;
 164         -?*)
 165                 echo "${0##*/}: unknown option: $1" >&2
 166                 usage 1
 167                 exit 1
 168                 ;;
 169         *)
 170                 break
 171                 ;;
 172         esac
 173         shift
 174 done
 175
 176 [ -n "$shownames" ] || [ $# -eq 0 ] || shownames=1
 177
 178 while [ $# -gt 0 ]; do
 179         aproj="${1%.git}"
 180         shift
 181         if ! [ -d "$cfg_reporoot/$aproj.git" ]; then
 182                 echo "${0##*/}: fatal: no such dir: $cfg_reporoot/$aproj.git" >&2
 183                 exit 1
 184         fi
 185         if ! is_git_dir "$cfg_reporoot/$aproj.git"; then
 186                 echo "${0##*/}: fatal: not a git dir: $cfg_reporoot/$aproj.git" >&2
 187                 exit 1
 188         fi
 189         onlylist="${onlylist:+$onlylist }$aproj.git"
 190 done
 191
 192 hasnice=
 193 ! command -v nice >/dev/null || hasnice=1
 194 hasionice=
 195 ! command -v ionice >/dev/null || hasionice=1
 196
 197 nl='
 198 '
 199
 200 projlist="$(cut -d : -f 1 <"$cfg_chroot/etc/group")"
 201
 202 is_listed_proj() {
 203         echo "$projlist" | grep -q -e "^$1$"
 204 }
 205
 206 is_empty_proj() {
 207         # if packed-refs is empty and no files in refs then empty
 208         # we do NOT want to run any git command in case the repo is bad
 209         _pd="$cfg_reporoot/$1.git"
 210         if [ -f "$_pd/packed-refs" ] && [ -s "$_pd/packed-refs" ]; then
 211                 if [ $(LC_ALL=C sed -n '/^#/!p' <"$_pd/packed-refs" | LC_ALL=C wc -l) -gt 0 ]; then
 212                         return 1
 213                 fi
 214         fi
 215         test $(find -L "$_pd/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) -eq 0
 216 }
 217
 218 get_check_proj() (
 219         cd "$cfg_reporoot/$1.git" || {
 220                 echo "no such directory: $cfg_reporoot/$1.git"
 221                 return 1
 222         }
 223         if [ -n "$fullcheck" ]; then
 224                 # use git fsck
 225                 # using --strict changes "zero-padded file modes" from a warning into an error
 226                 # which we do NOT want so we do NOT use --strict
 227                 cmd="git fsck"
 228                 [ -z "$var_have_git_1710" ] || cmd="$cmd --no-dangling"
 229                 # no need for --no-progress (v1.7.9+) since stderr will not be a tty
 230                 cmd="$cmd 2>&1"
 231         else
 232                 # use git rev-list --objects --all
 233                 cmd="git rev-list --objects --all${reflog:+ --reflog}${indexedobj:+ --indexed-objects}"
 234                 # but we only want stderr output not any of the objects list
 235                 cmd="$cmd 2>&1 >/dev/null"
 236         fi
 237         [ -z "$hasionice" ] || cmd="ionice -c 3 $cmd"
 238         [ -z "$hasnice" ] || cmd="nice -n 19 $cmd"
 239         fsckresult=0
 240         fsckoutput="$(eval "$cmd")" || fsckresult=$?
 241         if [ -n "$fullcheck" ] && [ -z "$var_have_git_1710" ]; then
 242                 # strip lines starting with "dangling" since --no-dangling is not supported
 243                 # note that "dangling" is NOT translated
 244                 fsckoutput="$(printf '%s\n' "$fsckoutput" | LC_ALL=C sed -n '/^dangling/!p')"
 245         fi
 246         [ -z "$fsckoutput" ] || printf '%s\n' "$fsckoutput"
 247         return $fsckresult
 248 )
 249
 250 cd "$cfg_reporoot"
 251 absroot="$(pwd -P)"
 252 # howmany is how many fsck was run on plus how many were empty
 253 howmany=0
 254 # mtcount is how many were empty
 255 mtcount=0
 256 # okcount is how many fsck returned 0 status for
 257 okcount=0
 258 # warncount is how many fsck returned 0 status but non-empty output
 259 warncount=0
 260 # errresults are results from fsck non-0 status fsck runs
 261 errresults=
 262 # warnresults are non-empty results from 0 status fsck runs
 263 warnresults=
 264 # if non-empty, warning results may be generated
 265 haswarn=1
 266 [ -n "$fullcheck" ] || haswarn=
 267 # list of all projects checked in order if --list is active
 268 allprojs=
 269
 270 while IFS='' read -r proj; do
 271         if [ -L "$proj" ]; then
 272                 # symlinks to elsewhere under $cfg_reporoot are ignored
 273                 # symlinks to outside there are reported on unless orphaned
 274                 absproj="$(cd "$proj" && pwd -P)" && [ -n "$absproj" ] &&
 275                         [ -d "$absproj" ] || continue
 276                 case "$absproj" in "$absroot"/*)
 277                         continue
 278                 esac
 279         fi
 280         proj="${proj#./}"
 281         proj="${proj%.git}"
 282         is_listed_proj "$proj" && is_git_dir "$cfg_reporoot/$proj.git" || continue
 283         [ -d "$proj.git/objects" ] || continue
 284         [ "${shownames:-0}" = "0" ] || allprojs="${allprojs:+$allprojs }$proj"
 285         howmany="$(( $howmany + 1 ))"
 286         if is_empty_proj "$proj"; then
 287                 mtcount="$(( $mtcount + 1 ))"
 288                 continue
 289         fi
 290         ok=1
 291         output="$(get_check_proj "$proj")" || ok=
 292         [ -z "$ok" ] || okcount="$(( $okcount + 1 ))"
 293         [ -n "$ok" ] || [ -n "$output" ] || output="git fsck failed with no output"
 294         if [ -n "$output" ]; then
 295                 output="$(printf '%s\n' "$output" | LC_ALL=C sed 's/^/    /')$nl"
 296                 if [ -n "$ok" ]; then
 297                         # warning
 298                         warncount="$(( $warncount + 1 ))"
 299                         [ -z "$warnresults" ] || warnresults="$warnresults$nl"
 300                         warnresults="$warnresults$proj: (warnings only)$nl$output"
 301                 else
 302                         [ -z "$errresults" ] || errresults="$errresults$nl"
 303                         errresults="$errresults$proj: (errors found)$nl$output"
 304                 fi
 305         fi
 306 done <<EOT
 307 $(
 308         if [ -n "$onlylist" ]; then
 309                 printf '%s\n' $onlylist
 310         else
 311                 find -L . -type d \( -path ./_recyclebin -o -path ./_global -o -name '*.git' -print \) -prune 2>/dev/null |
 312                 LC_ALL=C sort -f
 313         fi
 314 )
 315 EOT
 316
 317 enddate="$(date "+$datefmt")"
 318 [ "$minoutput" != "1" ] || [ "$howmany" != "$(( $okcount + $mtcount ))" ] || [ "$warncount" != "0" ] || exit 0
 319 [ "$minoutput" != "2" ] || [ "$howmany" != "$(( $okcount + $mtcount ))" ] || exit 0
 320 domail=cat
 321 [ -z "$mailresult" ] || domail='mailref "fsck@$cfg_gitweburl" -s "[$cfg_name] Project Fsck Status Report" "$cfg_admin"'
 322 {
 323         cat <<EOT
 324 Project Fsck Status Report
 325 ==========================
 326
 327        Start Time: $startdate
 328          End Time: $enddate
 329
 330  Projects Checked: $howmany
 331     Projects Okay: $(( $okcount + $mtcount )) (passed + ${haswarn:+warned + }empty)
 332
 333   Projects Passed: $(( $okcount - $warncount ))
 334 EOT
 335         [ -z "$haswarn" ] || cat <<EOT
 336   Projects Warned: $warncount
 337 EOT
 338         cat <<EOT
 339    Projects Empty: $mtcount
 340   Projects Failed: $(( $howmany - $mtcount - $okcount ))
 341 EOT
 342         if [ "${shownames:-0}" != "0" ] && [ -n "$allprojs" ]; then
 343                 prefix="\
 344  Projects Checked: "
 345                 for aproj in $allprojs; do
 346                         printf '%s%s\n' "$prefix" "$aproj"
 347                         prefix="\
 348                    "
 349                 done
 350         fi
 351
 352         echo ""
 353
 354         if [ -n "$errresults" ] || [ -n "$warnresults" ]; then
 355                 echo ""
 356                 echo "Fsck Output"
 357                 echo "-----------"
 358                 echo ""
 359                 printf '%s' "$errresults"
 360                 [ -z "$errresults" ] || [ -z "$warnresults" ] || echo ""
 361                 printf '%s' "$warnresults"
 362                 echo ""
 363         fi
 364 } | eval "$domail"