toolbox/reports/project-fsck-status.sh

   1 #!/bin/sh
   2
   3 # Report on project repository status.
   4 # Output can be sent as email to admin with -m
   5 # Automatically runs with nice and ionice (if available)
   6
   7 set -e
   8
   9 datefmt='%Y-%m-%d %H:%M:%S %z'
  10 startdate="$(date "+$datefmt")"
  11
  12 . @basedir@/shlib.sh
  13 PATH="$cfg_basedir/bin:${PATH:-$(/usr/bin/getconf PATH)}"
  14 export PATH
  15
  16 USAGE="${0##*/} [-hmEW] [--full] [<project>...]"
  17 HELP="
  18 NAME
  19        ${0##*/} - report on \"health\" of Girocco repositories
  20
  21 SYNOPSIS
  22        $USAGE
  23
  24 DESCRIPTION
  25        The ${0##*/} script checks all Girocco repositories for
  26        consistency and includes a count of empty repositories as well.
  27
  28        The results can automatically be mailed to the Girocco admin or
  29        simply displayed on STDOUT or optionally suppressed if there are
  30        no errors or warnings.
  31
  32        Note that this utility currently provides absolutely no progress
  33        reporting even when run on a terminal so there will be no output
  34        whatsoever until all projects have been checked.
  35
  36        The \"checking\" runs using nice and, if available, ionice.
  37
  38        Projects not listed in \$chroot/etc/group will not be checked.
  39
  40 OPTIONS
  41        -h     show this help
  42
  43        -m     e-mail results to the \$Girocco::Config::admin address.
  44               This option suppresses output to STDOUT.
  45
  46        -E     Suppress output (i.e. no e-mail will be sent if -m has been used)
  47               unless at least one error has been detected.  If this flag is
  48               used WITHOUT -W and warnings occur, output will still be
  49               suppressed.
  50
  51        -W     Suppress output unless at least one error or warning has been
  52               detected.  This flag overrides -E and will always produce output
  53               when only warnings are detected.
  54
  55        --full Run a full \"git fsck\" check on each repository.  Without this
  56               option a much, much, much (5x or more) faster check is done on
  57               each repository using \"git rev-list --objects --all\" that will
  58               verify all objects reachable from any ref are present but does
  59               not perform all the validation checks nor does it verify the
  60               integrity of 'blob' objects (just that they are present).
  61
  62               If the default Girocco 'transfer.fsckObjects=true' option has
  63               been left intact, use of the \"--full\" option should not
  64               normally be necessary.
  65
  66        --no-full
  67               Disables --full.  Present for completeness.
  68
  69        --list Show a list of all projects checked in the order checked.
  70               This is the default if any specific projects are given.
  71
  72        --no-list
  73               Do NOT show a list of each project name checked.
  74               This is the default when all projects are being checked.
  75
  76        <project>
  77               Optionally one or more Girocco project names may be given to
  78               check only those projects.
  79
  80 TIPS
  81        Leave Girocco's default 'transfer.fsckObjects=true' setting alone and
  82        have a cron job run this utility in the default --no-full mode using the
  83        -m and -E options once a week to detect repository corruption.
  84
  85        The default --no-full mode will detect any fatal errors with all
  86        reachable 'tag', 'commit' and 'tree' objects.  The --no-full mode will
  87        detect missing 'blob' objects but not corrupt 'blob' objects.
  88
  89        The --no-full mode is an order of magnitude faster than --full mode and
  90        does not complain about non-fatal object errors.
  91 "
  92
  93 usage() {
  94         if [ "${1:-1}" = "1" ]; then
  95                 printf 'usage: %s\n' "$USAGE" >&2
  96                 return 0
  97         fi
  98         printf '%s\n' "$HELP"
  99 }
 100
 101 mailresult=
 102 minoutput=0 # 0 => always output, 1 => if warn or err, 2 => if err
 103 fullcheck=
 104 onlylist=
 105 shownames=
 106
 107 while [ $# -gt 0 ]; do
 108         if [ "${1#-[!-]?}" != "$1" ]; then
 109                 _rest="${1#-?}"
 110                 _next="${1%"$_rest"}"
 111                 shift
 112                 set -- "$_next" "-$_rest" "$@"
 113         fi
 114         case "$1" in
 115         "-?")
 116                 usage 1 2>&1
 117                 exit 0
 118                 ;;
 119         -h|--help)
 120                 usage 2
 121                 exit 0
 122                 ;;
 123         -m|--mail)
 124                 mailresult=1
 125                 ;;
 126         -E)
 127                 [ "${minoutput:-0}" != "0" ] || minoutput=2
 128                 ;;
 129         -W)
 130                 minoutput=1
 131                 ;;
 132         --full)
 133                 fullcheck=1
 134                 ;;
 135         --no-full)
 136                 fullcheck=
 137                 ;;
 138         --list)
 139                 shownames=1
 140                 ;;
 141         --no-list)
 142                 shownames=0
 143                 ;;
 144         --)
 145                 shift
 146                 break
 147                 ;;
 148         -?*)
 149                 echo "${0##*/}: unknown option: $1" >&2
 150                 usage 1
 151                 exit 1
 152                 ;;
 153         *)
 154                 break
 155                 ;;
 156         esac
 157         shift
 158 done
 159
 160 [ -n "$shownames" ] || [ $# -eq 0 ] || shownames=1
 161
 162 while [ $# -gt 0 ]; do
 163         aproj="${1%.git}"
 164         shift
 165         if ! [ -d "$cfg_reporoot/$aproj.git" ]; then
 166                 echo "${0##*/}: fatal: no such dir: $cfg_reporoot/$aproj.git" >&2
 167                 exit 1
 168         fi
 169         if ! is_git_dir "$cfg_reporoot/$aproj.git"; then
 170                 echo "${0##*/}: fatal: not a git dir: $cfg_reporoot/$aproj.git" >&2
 171                 exit 1
 172         fi
 173         onlylist="${onlylist:+$onlylist }$aproj.git"
 174 done
 175
 176 hasnice=
 177 ! command -v nice >/dev/null || hasnice=1
 178 hasionice=
 179 ! command -v ionice >/dev/null || hasionice=1
 180
 181 nl='
 182 '
 183
 184 projlist="$(cut -d : -f 1 <"$cfg_chroot/etc/group")"
 185
 186 is_listed_proj() {
 187         echo "$projlist" | grep -q -e "^$1$"
 188 }
 189
 190 is_empty_proj() {
 191         # if packed-refs is empty and no files in refs then empty
 192         # we do NOT want to run any git command in case the repo is bad
 193         _pd="$cfg_reporoot/$1.git"
 194         if [ -f "$_pd/packed-refs" ] && [ -s "$_pd/packed-refs" ]; then
 195                 if [ $(LC_ALL=C sed -n '/^#/!p' <"$_pd/packed-refs" | LC_ALL=C wc -l) -gt 0 ]; then
 196                         return 1
 197                 fi
 198         fi
 199         test $(find -L "$_pd/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) -eq 0
 200 }
 201
 202 get_check_proj() (
 203         cd "$cfg_reporoot/$1.git" || {
 204                 echo "no such directory: $cfg_reporoot/$1.git"
 205                 return 1
 206         }
 207         if [ -n "$fullcheck" ]; then
 208                 # use git fsck
 209                 # using --strict changes "zero-padded file modes" from a warning into an error
 210                 # which we do NOT want so we do NOT use --strict
 211                 cmd="git fsck"
 212                 [ -z "$var_have_git_1710" ] || cmd="$cmd --no-dangling"
 213                 # no need for --no-progress (v1.7.9+) since stderr will not be a tty
 214                 cmd="$cmd 2>&1"
 215         else
 216                 # use git rev-list --objects --all
 217                 cmd="git rev-list --objects --all"
 218                 # but we only want stderr output not any of the objects list
 219                 cmd="$cmd 2>&1 >/dev/null"
 220         fi
 221         [ -z "$hasionice" ] || cmd="ionice -c 3 $cmd"
 222         [ -z "$hasnice" ] || cmd="nice -n 19 $cmd"
 223         fsckresult=0
 224         fsckoutput="$(eval "$cmd")" || fsckresult=$?
 225         if [ -n "$fullcheck" ] && [ -z "$var_have_git_1710" ]; then
 226                 # strip lines starting with "dangling" since --no-dangling is not supported
 227                 # note that "dangling" is NOT translated
 228                 fsckoutput="$(printf '%s\n' "$fsckoutput" | LC_ALL=C sed -n '/^dangling/!p')"
 229         fi
 230         [ -z "$fsckoutput" ] || printf '%s\n' "$fsckoutput"
 231         return $fsckresult
 232 )
 233
 234 cd "$cfg_reporoot"
 235 absroot="$(pwd -P)"
 236 # howmany is how many fsck was run on plus how many were empty
 237 howmany=0
 238 # mtcount is how many were empty
 239 mtcount=0
 240 # okcount is how many fsck returned 0 status for
 241 okcount=0
 242 # warncount is how many fsck returned 0 status but non-empty output
 243 warncount=0
 244 # errresults are results from fsck non-0 status fsck runs
 245 errresults=
 246 # warnresults are non-empty results from 0 status fsck runs
 247 warnresults=
 248 # if non-empty, warning results may be generated
 249 haswarn=1
 250 [ -n "$fullcheck" ] || haswarn=
 251 # list of all projects checked in order if --list is active
 252 allprojs=
 253
 254 while IFS='' read -r proj; do
 255         if [ -L "$proj" ]; then
 256                 # symlinks to elsewhere under $cfg_reporoot are ignored
 257                 # symlinks to outside there are reported on unless orphaned
 258                 absproj="$(cd "$proj" && pwd -P)" && [ -n "$absproj" ] &&
 259                         [ -d "$absproj" ] || continue
 260                 case "$absproj" in "$absroot"/*)
 261                         continue
 262                 esac
 263         fi
 264         proj="${proj#./}"
 265         proj="${proj%.git}"
 266         is_listed_proj "$proj" && is_git_dir "$cfg_reporoot/$proj.git" || continue
 267         [ -d "$proj.git/objects" ] || continue
 268         [ "${shownames:-0}" = "0" ] || allprojs="${allprojs:+$allprojs }$proj"
 269         howmany="$(( $howmany + 1 ))"
 270         if is_empty_proj "$proj"; then
 271                 mtcount="$(( $mtcount + 1 ))"
 272                 continue
 273         fi
 274         ok=1
 275         output="$(get_check_proj "$proj")" || ok=
 276         [ -z "$ok" ] || okcount="$(( $okcount + 1 ))"
 277         [ -n "$ok" ] || [ -n "$output" ] || output="git fsck failed with no output"
 278         if [ -n "$output" ]; then
 279                 output="$(printf '%s\n' "$output" | LC_ALL=C sed 's/^/    /')$nl"
 280                 if [ -n "$ok" ]; then
 281                         # warning
 282                         warncount="$(( $warncount + 1 ))"
 283                         [ -z "$warnresults" ] || warnresults="$warnresults$nl"
 284                         warnresults="$warnresults$proj: (warnings only)$nl$output"
 285                 else
 286                         [ -z "$errresults" ] || errresults="$errresults$nl"
 287                         errresults="$errresults$proj: (errors found)$nl$output"
 288                 fi
 289         fi
 290 done <<EOT
 291 $(
 292         if [ -n "$onlylist" ]; then
 293                 printf '%s\n' $onlylist
 294         else
 295                 find -L . -type d \( -path ./_recyclebin -o -path ./_global -o -name '*.git' -print \) -prune 2>/dev/null |
 296                 LC_ALL=C sort -f
 297         fi
 298 )
 299 EOT
 300
 301 enddate="$(date "+$datefmt")"
 302 [ "$minoutput" != "1" ] || [ "$howmany" != "$(( $okcount + $mtcount ))" ] || [ "$warncount" != "0" ] || exit 0
 303 [ "$minoutput" != "2" ] || [ "$howmany" != "$(( $okcount + $mtcount ))" ] || exit 0
 304 domail=cat
 305 [ -z "$mailresult" ] || domail='mailref "fsck@$cfg_gitweburl" -s "[$cfg_name] Project Fsck Status Report" "$cfg_admin"'
 306 {
 307         cat <<EOT
 308 Project Fsck Status Report
 309 ==========================
 310
 311        Start Time: $startdate
 312          End Time: $enddate
 313
 314  Projects Checked: $howmany
 315     Projects Okay: $(( $okcount + $mtcount )) (passed + ${haswarn:+warned + }empty)
 316
 317   Projects Passed: $(( $okcount - $warncount ))
 318 EOT
 319         [ -z "$haswarn" ] || cat <<EOT
 320   Projects Warned: $warncount
 321 EOT
 322         cat <<EOT
 323    Projects Empty: $mtcount
 324   Projects Failed: $(( $howmany - $mtcount - $okcount ))
 325 EOT
 326         if [ "${shownames:-0}" != "0" ] && [ -n "$allprojs" ]; then
 327                 prefix="\
 328  Projects Checked: "
 329                 for aproj in $allprojs; do
 330                         printf '%s%s\n' "$prefix" "$aproj"
 331                         prefix="\
 332                    "
 333                 done
 334         fi
 335
 336         echo ""
 337
 338         if [ -n "$errresults" ] || [ -n "$warnresults" ]; then
 339                 echo ""
 340                 echo "Fsck Output"
 341                 echo "-----------"
 342                 echo ""
 343                 printf '%s' "$errresults"
 344                 [ -z "$errresults" ] || [ -z "$warnresults" ] || echo ""
 345                 printf '%s' "$warnresults"
 346                 echo ""
 347         fi
 348 } | eval "$domail"