shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         inc_basedir=@basedir@
  54         [ "@basedir@" != '@'basedir'@' ] || inc_basedir="$PWD"
  55         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  56         perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf -le \
  57                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  58                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  59                         $val =~ s/([\\"\$\`])/\\$1/gos;
  60                         $val =~ s/(?:\r\n|\r|\n)$//os;
  61                         print "cfg_$_=\"$val\"";
  62                         print "defined_cfg_$_=",
  63                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  64                 }'
  65 )
  66
  67 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  68 _fcp() {
  69         if _fp="$(command -v "$1" 2>/dev/null)"; then
  70                 printf '%s\n' "$_fp"
  71         else
  72                 printf '%s\n' "$1"
  73         fi
  74 }
  75
  76 get_girocco_config_var_list() (
  77         # Same as get_girocco_config_pm_var_list except that
  78         # the following variables (all starting with var_) are added:
  79         # var_group             cfg_owning_group if defined otherwise `id -gn`
  80         # var_git_ver           The version number part from `git version`
  81         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  82         # var_sh_bin            Full path to the posix sh interpreter to use
  83         # var_perl_bin          Full path to the perl interpreter to use
  84         # var_gzip_bin          Full path to the gzip executable to use
  85         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  86         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  87         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  88         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  89         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  90         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  91         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  92         # var_have_git_235      Set to 1 if git version >= 2.3.5 otherwise ''
  93         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  94         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  95         # var_window_memory     Value to use for repack --window-memory=
  96         # var_big_file_threshold Value to use for core.bigFileThreshold
  97         # var_redelta_threshold Recompute deltas if no more than this many objs
  98         # var_upload_window     If not "", pack.window to use for upload-pack
  99         # var_log_window_size   Value to use for git-svn --log-window-size=
 100         # var_utf8_locale       Value to use for a UTF-8 locale if available
 101         # var_xargs_r           A "-r" if xargs needs it to behave correctly
 102         # var_du_exclude        Option to exclude PATTERN from du if available
 103         # var_du_follow         Option to follow command line sym links if available
 104         _cfg_vars="$(get_girocco_config_pm_var_list)"
 105         eval "$_cfg_vars"
 106         printf '%s\n' "$_cfg_vars"
 107         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 108         _gver="$("$cfg_git_bin" version 2>/dev/null |
 109                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 110         printf 'var_git_ver=%s\n' "$_gver"
 111         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 112         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 113         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 114         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 115         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 116         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 121         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 122         printf 'var_have_git_235=%s\n' "$([ $(vcmp "$_gver" 2.3.5) -ge 0 ] && echo 1)"
 123         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 124         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 125         __girocco_conf="$GIROCCO_CONF"
 126         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 127         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 128         inc_basedir=@basedir@
 129         [ "@basedir@" != '@'basedir'@' ] || inc_basedir="$PWD"
 130         printf "var_window_memory=%s\n" \
 131                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_windowmemory')"
 133         printf "var_big_file_threshold=%s\n" \
 134                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 135                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 136         printf "var_redelta_threshold=%s\n" \
 137                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 138                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 139         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 140            [ "$cfg_upload_pack_window" -le 50 ]; then
 141                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 142         else
 143                 printf "var_upload_window=%s\n" ""
 144         fi
 145         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 146         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 147         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 148                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 149                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 150                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 151         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 152         # On some broken platforms running xargs without -r and empty input runs the command
 153         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 154         # The disk usage report produces better numbers if du has an exclude option
 155         _x0="${0##*/}"
 156         _x0="${_x0%?}?*"
 157         for _duopt in --exclude -I; do
 158                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 159                         printf 'var_du_exclude=%s\n' "$_duopt"
 160                         break
 161                 fi
 162         done
 163         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 164                 printf 'var_du_follow=%s\n' "-H"
 165                 break
 166         fi
 167 )
 168
 169 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 170 # definitions from it rather than running Perl.
 171 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 172         # Import all the variables from Girocco::Config to the local environment,
 173         eval "$(get_girocco_config_var_list)"
 174 else
 175         # Import the variables from shlib_vars.sh which avoids needlessly
 176         # running another copy of Perl
 177         . "@basedir@/shlib_vars.sh"
 178 fi
 179
 180 # git_add_config "some.var=value"
 181 # every ' in value must be replaced with the 4-character sequence '\'' before
 182 # calling this function or Git will barf.  Will not be effective unless running
 183 # Git version 1.7.3 or later.
 184 git_add_config() {
 185         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 186         export GIT_CONFIG_PARAMETERS
 187 }
 188
 189 # Make sure we have a reproducible environment by using a controlled HOME dir
 190 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 191 HOME="$cfg_chroot/etc/girocco"
 192 TMPDIR="/tmp"
 193 GIT_CONFIG_NOSYSTEM=1
 194 GIT_ATTR_NOSYSTEM=1
 195 GIT_NO_REPLACE_OBJECTS=1
 196 GIT_TERMINAL_PROMPT=0
 197 GIT_PAGER="cat"
 198 PAGER="cat"
 199 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 200 GIT_SVN_NOTTY=1
 201 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 202 GIT_SSH="$cfg_basedir/bin/git-ssh"
 203 SVN_SSH="$cfg_basedir/bin/git-ssh"
 204 export XDG_CONFIG_HOME
 205 export HOME
 206 export TMPDIR
 207 export GIT_CONFIG_NOSYSTEM
 208 export GIT_ATTR_NOSYSTEM
 209 export GIT_NO_REPLACE_OBJECTS
 210 export GIT_TERMINAL_PROMPT
 211 export GIT_PAGER
 212 export PAGER
 213 export GIT_ASKPASS
 214 export GIT_SVN_NOTTY
 215 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 216 export GIT_SSH
 217 export SVN_SSH
 218 unset GIT_USER_AGENT
 219 unset GIT_HTTP_USER_AGENT
 220 if [ -n "$defined_cfg_git_client_ua" ]; then
 221         GIT_USER_AGENT="$cfg_git_client_ua"
 222         export GIT_USER_AGENT
 223 fi
 224 unset GIT_CONFIG_PARAMETERS
 225
 226 #
 227 ## IMPORTANT!
 228 ##
 229 ## Keep gitweb/gitweb_config.perl in sync with these git_add_config calls
 230 ## Keep bin/git-shell-verify in sync with these git_add_config calls
 231 #
 232 git_add_config "core.ignoreCase=false"
 233 git_add_config "core.pager=cat"
 234 if [ -n "$cfg_git_no_mmap" ]; then
 235         # Just like compiling with NO_MMAP
 236         git_add_config "core.packedGitWindowSize=1m"
 237 else
 238         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 239         git_add_config "core.packedGitWindowSize=32m"
 240 fi
 241 # Always use the 32-bit default (256m) even on 64-bit to avoid memory blowout
 242 git_add_config "core.packedGitLimit=256m"
 243 [ -z "$var_big_file_threshold" ] ||
 244         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 245 git_add_config "gc.auto=0"
 246
 247 # Make sure any sendmail.pl config is always available
 248 unset SENDMAIL_PL_HOST
 249 unset SENDMAIL_PL_PORT
 250 unset SENDMAIL_PL_NCBIN
 251 unset SENDMAIL_PL_NCOPT
 252 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 253 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 254 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 255 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 256
 257 # Set PATH and PYTHON to the values set by Config.pm, if any
 258 unset PYTHON
 259 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 260 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 261
 262 # Extra GIT variables that generally ought to be cleared, but whose clearing
 263 # could potentially interfere with the correct operation of hook scripts so
 264 # they are segregated into a separate function for use as appropriate
 265 clean_git_env() {
 266         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 267         unset GIT_CONFIG
 268         unset GIT_DIR
 269         unset GIT_GRAFT_FILE
 270         unset GIT_INDEX_FILE
 271         unset GIT_OBJECT_DIRECTORY
 272         unset GIT_NAMESPACE
 273 }
 274
 275 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 276 # and nc_openbsd to the desired executables because when using
 277 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 278 # different and unexpected ways:
 279 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 280 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 281 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 282 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 283 # setting only while running func which none of the /bin/sh's do.
 284 #
 285 # Instead we'd like to use an alias that provides the desired behavior without
 286 # any of the bad (a), (b) or (c) effects.
 287 #
 288 # However, unfortunately, some of the crazy /bin/sh implementations do not
 289 # recognize alias expansions when preceded by variable assignments!
 290 #
 291 # So we are left with git() {} and nc_openbsd() {} functions and in the
 292 # case of git() {} we can compensate for (b) and (c) failing to export
 293 # but not (a) and (b) persisting the values so the caller will simply
 294 # have to beware and explicitly unset any variables that should not persist
 295 # beyond the function call itself.
 296
 297 git() (
 298         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 299         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 300         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 301         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 302         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 303         exec "$cfg_git_bin" "$@"
 304 )
 305
 306 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 307 # that allows us to use git update-ref --stdin where supported and the slow shell
 308 # script where not, but only the "delete" operation is currently supported.
 309 git_updateref_stdin() {
 310         if [ -n "$var_have_git_185" ]; then
 311                 git update-ref --stdin
 312         else
 313                 while read -r _op _ref; do
 314                         case "$_op" in
 315                                 delete)
 316                                         git update-ref -d "$_ref"
 317                                         ;;
 318                                 *)
 319                                         echo "bad git_updateref_stdin op: $_op" >&2
 320                                         exit 1
 321                                         ;;
 322                         esac
 323                 done
 324         fi
 325 }
 326
 327 # see comments for git() -- callers must explicitly export all variables
 328 # intended for the commands these functions run before calling them
 329 perl() { command "${var_perl_bin:-perl}" "$@"; }
 330 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 331
 332 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 333
 334 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 335
 336 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 337
 338 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 339
 340 # Some platforms' broken xargs runs the command always at least once even if
 341 # there's no input unless given a special option.  Automatically supply the
 342 # option on those platforms by providing an xargs function.
 343 xargs() { command xargs $var_xargs_r "$@"; }
 344
 345 _addrlist() {
 346         _list=
 347         for _addr in "$@"; do
 348                 [ -z "$_list" ] || _list="$_list, "
 349                 _list="$_list$_addr"
 350         done
 351         echo "$_list"
 352 }
 353
 354 _sendmail() {
 355         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 356         if [ -n "$cfg_sender" ]; then
 357                 "$_mailer" -i -f "$cfg_sender" "$@"
 358         else
 359                 "$_mailer" -i "$@"
 360         fi
 361 }
 362
 363 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 364 # "References:" header.  It may be "" to suppress the "References" header.
 365 # Following arguments are just like mail function
 366 mailref() {
 367         _references=
 368         if [ $# -ge 1 ]; then
 369                 _references="$1"
 370                 shift
 371         fi
 372         _subject=
 373         if [ "$1" = "-s" ]; then
 374                 shift
 375                 _subject="$1"
 376                 shift
 377         fi
 378         {
 379                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 380                 echo "To: $(_addrlist "$@")"
 381                 [ -z "$_subject" ] || echo "Subject: $_subject"
 382                 echo "MIME-Version: 1.0"
 383                 echo "Content-Type: text/plain; charset=utf-8; format=fixed"
 384                 echo "Content-Transfer-Encoding: 8bit"
 385                 [ -z "$_references" ] || echo "References: <$_references>"
 386                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 387                 echo "Auto-Submitted: auto-generated"
 388                 echo ""
 389                 cat
 390         } | _sendmail "$@"
 391 }
 392
 393 # Usage: mail [-s <subject>] <addr> [<addr>...]
 394 mail() {
 395         mailref "" "$@"
 396 }
 397
 398 # bang CMD... will execute the command with well-defined failure mode;
 399 # set bang_action to string of the failed action ('clone', 'update', ...);
 400 # re-define the bang_trap() function to do custom cleanup before bailing out
 401 bang() {
 402         bang_errcode=
 403         bang_catch "$@"
 404         [ "${bang_errcode:-0}" = "0" ] || bang_failed
 405 }
 406
 407 bang_catch() {
 408         bang_active=1
 409         bang_cmd="$*"
 410         bang_errcode=0
 411         if [ "${show_progress:-0}" != "0" ]; then
 412                 exec 3>&1
 413                 read -r bang_errcode <<-EOT || :
 414                 $(
 415                         exec 4>&3 3>&1 1>&4 4>&-
 416                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 417                 )
 418                 EOT
 419                 exec 3>&-
 420                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 421                         # All right. Cool.
 422                         bang_active=
 423                         bang_cmd=
 424                         return;
 425                 fi
 426         else
 427                 if "$@" >>"$bang_log" 2>&1; then
 428                         # All right. Cool.
 429                         bang_active=
 430                         bang_cmd=
 431                         return;
 432                 else
 433                         bang_errcode="$?"
 434                 fi
 435         fi
 436 }
 437
 438 bang_failed() {
 439         bang_active=
 440         unset GIT_DIR
 441         >.banged
 442         cat "$bang_log" >.banglog
 443         echo "" >>.banglog
 444         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 445         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 446         if [ "${show_progress:-0}" != "0" ]; then
 447                 echo ""
 448                 echo "$bang_cmd failed with error code $bang_errcode"
 449         fi
 450         if [ -e .bangagain ]; then
 451                 git config --remove-section girocco.bang 2>/dev/null || :
 452                 rm -f .bangagain
 453         fi
 454         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 455         bangcount=$(( ${bangcount:-0} + 1 ))
 456         git config --int girocco.bang.count $bangcount
 457         if [ $bangcount -eq 1 ]; then
 458                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 459         fi
 460         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 461            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 462            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 463                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 464                 bangaddrs=
 465                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 466                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 467                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 468                 rsubj=
 469                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 470                 [ -z "$bangaddrs" ] ||
 471                 {
 472                         echo "$bang_cmd failed with error code $bang_errcode"
 473                         echo ""
 474                         rsubj=
 475                         if [ $bangcount -gt 1 ]; then
 476                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 477                                 echo ""
 478                         fi
 479                         echo "you will not receive any more notifications until recovery"
 480                         echo "this status message may be disabled on the project admin page"
 481                         echo ""
 482                         echo "Log follows:"
 483                         echo ""
 484                         cat "$bang_log"
 485                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 486                 git config --bool girocco.bang.messagesent true
 487         fi
 488         bangthrottle=
 489         [ $bangcount -lt 15 ] ||
 490                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 491                 bangthrottle=1
 492         bang_trap $bangthrottle
 493         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 494         exit $bang_errcode
 495 }
 496
 497 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 498 # Identical to bang CMD... except the command is eval'd instead of executed.
 499 bang_eval() {
 500         bang eval "$*"
 501 }
 502
 503 # Default bang settings:
 504 bang_setup() {
 505         bang_active=
 506         bang_action="lame_programmer"
 507         bang_trap() { :; }
 508         bang_tmpdir="${TMPDIR:-/tmp}"
 509         bang_tmpdir="${bang_tmpdir%/}"
 510         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 511         is_git_dir . || {
 512                 echo "bang_setup called with current directory not a git directory" >&2
 513                 exit 1
 514         }
 515         trap 'rm -f "$bang_log"' EXIT
 516         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 517         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 518 }
 519
 520 # Remove banged status
 521 bang_reset() {
 522         rm -f .banged .bangagain .banglog
 523         git config --remove-section girocco.bang 2>/dev/null || :
 524 }
 525
 526 # Check to see if banged status
 527 is_banged() {
 528         [ -e .banged ]
 529 }
 530
 531 # Check to see if banged message was sent
 532 was_banged_message_sent() {
 533         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 534 }
 535
 536 # Progress report - if show_progress is set, shows the given message.
 537 progress() {
 538         [ "${show_progress:-0}" = "0" ] || echo "$*"
 539 }
 540
 541 # Project config accessors; must be run in project directory
 542 config_get() {
 543         case "$1" in
 544         *.*)
 545                 git config "$1";;
 546         *)
 547                 git config "gitweb.$1";;
 548         esac
 549 }
 550
 551 config_set() {
 552         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 553 }
 554
 555 config_set_raw() {
 556         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 557 }
 558
 559 config_get_date_seconds() {
 560         _dt="$(config_get "$1")" || :
 561         [ -n "$_dt" ] || return 1
 562         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 563         [ -n "$_ds" ] || return 1
 564         echo "$_ds"
 565 }
 566
 567 # Tool for checking whether given number of seconds has not passed yet
 568 check_interval() {
 569         os="$(config_get_date_seconds "$1")" || return 1
 570         ns="$(date +%s)"
 571         [ $ns -lt $(($os+$2)) ]
 572 }
 573
 574 # Check if we are running with effective root permissions
 575 is_root() {
 576         [ "$(id -u 2>/dev/null)" = "0" ]
 577 }
 578
 579 # Check to see if the single argument (default ".") is a Git directory
 580 is_git_dir() {
 581         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 582         # And we are slightly more picky (must be refs/.+ not refs/.*)
 583         [ $# -ne 0 ] || set -- "."
 584         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 585         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 586         if [ -L "$1/HEAD" ]; then
 587                 _hr="$(readlink "$1/HEAD")"
 588                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 589         fi
 590         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 591         read -r _hr <"$1/HEAD" || return 1
 592         case "$_hr" in
 593                 $octet20*)
 594                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 595                         return 0;;
 596                 ref:refs/?*)
 597                         return 0;;
 598                 ref:*)
 599                         _hr="${_hr##ref:*[ $tab]}"
 600                         case "$_hr" in "refs/"?*) return 0;; esac
 601         esac
 602         return 1
 603 }
 604
 605 # Check to see if the single argument (default ".") is a directory with no refs
 606 is_empty_refs_dir() {
 607         [ $# -ne 0 ] || set -- "."
 608         if [ -s "$1/packed-refs" ]; then
 609                 # could be a packed-refs file with just a '# pack-refs ..." line
 610                 # null hash lines and peel lines do not count either
 611                 _refcnt="$(( $(LC_ALL=C sed <"$1/packed-refs" \
 612                         -e "/^00* /d" \
 613                         -e "/^$octet20$hexdig* refs\/[^ $tab]*\$/!d" | wc -l) ))"
 614                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 615         fi
 616         if [ -d "$1/refs" ]; then
 617                 # quick and dirty check, doesn't try to validate contents
 618                 # or ignore embedded symbolic refs
 619                 _refcnt="$(( $(find -L "$1/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) ))"
 620                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 621         fi
 622         # last chance a detached HEAD (we ignore any linked working trees though)
 623         [ -s "$1/HEAD" ] && read -r _hr <"$1/HEAD" && [ -n "$_hr" ] || return 0
 624         [ "${_hr#*[!0-9a-f]}" != "$_hr" ] || [ "${_hr#*[!0]}" = "$_hr" ] || [ "${#_hr}" -lt 40 ] || return 1
 625         return 0
 626 }
 627
 628 # List all Git repositories, with given prefix if specified, one-per-line
 629 # All project names starting with _ are always excluded from the result
 630 get_repo_list() {
 631         if [ -n "$1" ]; then
 632                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 633         else
 634                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 635         fi | while IFS=: read name id; do
 636                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 637         done
 638 }
 639
 640 # set the variable named by the first argument to the project part (i.e. WITH
 641 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 642 # specified by the second argument.
 643 # This function cannot be fooled by symbolic links.
 644 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 645 # The directory specified by the second argument must exist.
 646 v_get_proj_from_dir() {
 647         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 648         [ -d "$2" ] || return 1
 649         case "$2" in
 650         "$cfg_reporoot/"?*)
 651                 # Simple case that does not need any fancy footwork
 652                 _projpart="${2#$cfg_reporoot/}"
 653                 ;;
 654         *)
 655                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 656                 _abspd="$(cd "$2" && pwd -P)"
 657                 case "$_abspd" in
 658                 "$_absrr/"?*)
 659                         # The normal case
 660                         _projpart="${_abspd#$_absrr/}"
 661                         ;;
 662                 *)
 663                         # Must have been reached via a symbolic link, but
 664                         # we have no way to know the source, so use a
 665                         # generic "_external" leader combined with just the
 666                         # trailing directory name
 667                         _abspd="${_abspd%/}"
 668                         _abspd="${_abspd%/.git}"
 669                         _projpart="_external/${_abspd##*/}"
 670                         ;;
 671                 esac
 672         esac
 673         eval "$1="'"$_projpart"'
 674 }
 675
 676 # Returns success if "$1" does not exist or contains only blank lines and comments
 677 # The parsing rules are in Git's sha1-file.c parse_alt_odb_entry function;
 678 # the format for blank lines and comments has been the same since Git v0.99.5
 679 is_empty_alternates_file() {
 680         [ -n "$1" ] || return 0
 681         [ -e "$1" ] && [ -f "$1" ] && [ -s "$1" ] || return 0
 682         [ -r "$1" ] || return 1
 683         LC_ALL=C awk <"$1" '!/^$/ && !/^#/ {exit 1}'
 684 }
 685
 686 # Return success if the given project name has at least one immediate child fork
 687 # that has a non-empty alternates file
 688 has_forks_with_alternates() {
 689         _prj="${1%.git}"
 690         [ -n "$_prj" ] || return 1
 691         [ -d "$cfg_reporoot/$_prj" ] || return 1
 692         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 693         if
 694                 get_repo_list "$_prj/[^/:][^/:]*:" |
 695                 while read -r _prjname && [ -n "$_prjname" ]; do
 696                         is_empty_alternates_file "$cfg_reporoot/$_prjname.git/objects/info/alternates" ||
 697                         exit 1 # will only exit implicit subshell created by '|'
 698                 done
 699         then
 700                 return 1
 701         fi
 702         return 0
 703 }
 704
 705 # returns empty string and error for empty string otherwise one of
 706 #   m  =>  normal Git mirror
 707 #   s  =>  mirror from svn source
 708 #   d  =>  mirror from darcs source
 709 #   b  =>  mirror from bzr source
 710 #   h  =>  mirror from hg source
 711 #   w  =>  mirror from mediawiki source
 712 #   f  =>  mirror from other fast-import source
 713 # note that if the string is non-empty and none of s, d, b or h match the
 714 # return will always be type m regardless of whether it's a valid Git URL
 715 get_url_mirror_type() {
 716         case "$1" in
 717                 "")
 718                         return 1
 719                         ;;
 720                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 721                         echo 's'
 722                         ;;
 723                 darcs://* | darcs+http://* | darcs+https://*)
 724                         echo 'd'
 725                         ;;
 726                 bzr://*)
 727                         echo 'b'
 728                         ;;
 729                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 730                         echo 'h'
 731                         ;;
 732                 mediawiki::*)
 733                         echo 'w'
 734                         ;;
 735                 *)
 736                         echo 'm'
 737                         ;;
 738         esac
 739         return 0
 740 }
 741
 742 # returns false for empty string
 743 # returns true if the passed in url is a mirror using git fast-import
 744 is_gfi_mirror_url() {
 745         [ -n "$1" ] || return 1
 746         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 747                 d|b|h|w|f)
 748                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 749                         # and so do generic "f" fast-import mirrors
 750                         return 0
 751                         ;;
 752                 *)
 753                         # Don't think git-svn currently uses git fast-import
 754                         # And Git mirrors certainly do not
 755                         return 1
 756                         ;;
 757         esac
 758         # assume it does not use git fast-import
 759         return 1
 760 }
 761
 762 # returns false for empty string
 763 # returns true if the passed in url is a mirror using git-svn
 764 is_svn_mirror_url() {
 765         [ -n "$1" ] || return 1
 766         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 767 }
 768
 769 # returns mirror url for gitweb.baseurl of git directory
 770 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 771 # will fail if the directory does not have .nofetch and gitweb.baseurl
 772 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 773 # automatically strips any leading "disabled " prefix before returning result
 774 get_mirror_url() {
 775         _gitdir="${1:-.}"
 776         # always return empty for non-mirrors
 777         ! [ -e "$_gitdir/.nofetch" ] || return 0
 778         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 779         _url="${_url##* }"
 780         [ -n "$_url" ] || return 1
 781         printf '%s\n' "$_url"
 782         return 0
 783 }
 784
 785 # returns get_url_mirror_type for gitweb.baseurl of git directory
 786 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 787 # will fail if the directory does not have .nofetch and gitweb.baseurl
 788 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 789 # automatically strips any leading "disabled " prefix before testing
 790 get_mirror_type() {
 791         _url="$(get_mirror_url "$@")" || return 1
 792         [ -n "$_url" ] || return 0
 793         get_url_mirror_type "$_url"
 794 }
 795
 796 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 797 is_gfi_mirror() {
 798         _url="$(get_mirror_url "$@")" || return 1
 799         is_gfi_mirror_url "$_url"
 800 }
 801
 802 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 803 is_svn_mirror() {
 804         _url="$(get_mirror_url "$@")" || return 1
 805         is_svn_mirror_url "$_url"
 806 }
 807
 808 # current directory must already be set to Git repository
 809 # if girocco.headok is already true succeeds without doing anything
 810 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 811 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 812 # then refs/heads/trunk and finally the first top-level head from
 813 # refs/heads/* (i.e. only two slashes in the name) and finally any
 814 # existing refs/heads.  The first one to succeed wins and sets headok=true
 815 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 816 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 817 # when the repository is being set up -- if the HEAD is later deleted (through
 818 # a push or fetch --prune) that's no longer our responsibility to fix
 819 check_and_set_head() {
 820         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 821         if git rev-parse --verify --quiet HEAD >/dev/null; then
 822                 git config --bool girocco.headok true
 823                 return 0
 824         fi
 825         for _hr in refs/heads/master refs/heads/trunk; do
 826                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 827                         _update_head_symref "$_hr"
 828                         return 0
 829                 fi
 830         done
 831         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 832         while read -r _hr; do
 833                 case "${_hr#refs/heads/}" in */*) :;; *)
 834                         _update_head_symref "$_hr"
 835                         exit 1 # exit subshell created by "|"
 836                 esac
 837         done || return 0
 838         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 839         if [ -n "$_hr" ]; then
 840                 _update_head_symref "$_hr"
 841                 return 0
 842         fi
 843         return 1
 844 }
 845 _update_head_symref() {
 846         git symbolic-ref HEAD "$1"
 847         git config --bool girocco.headok true
 848         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 849 }
 850
 851 # current directory must already be set to Git repository
 852 # if the directory needs to have gc run and .needsgc is not already set
 853 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 854 # Girocco shouldn't generate any loose objects but we check for that anyway
 855 check_and_set_needsgc() {
 856         # If there's a .needspack file and ANY loose objects with a newer timestamp
 857         # then also set .needsgc otherwise remove it.  The only caller that may set
 858         # .needspack is a mirror therefore we don't have to worry about removing a
 859         # .needspack out from under a simultaneous creator.  We always do this and
 860         # do it first to try and avoid leaving a stale .needspack lying around.
 861         if [ -e .needspack ]; then
 862                 _objfiles=
 863                 _objfiles="$(( $(find -L objects/$octet -maxdepth 1 -newer .needspack -name "$octet19*" -type f -print 2>/dev/null |
 864                          head -n 1 | LC_ALL=C wc -l) +0 ))"
 865                 if [ "${_objfiles:-0}" = "0" ]; then
 866                         rm -f .needspack
 867                 else
 868                         [ -e .needsgc ] || >.needsgc
 869                 fi
 870         fi
 871         ! [ -e .needsgc ] || return 0
 872         _packs=
 873         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 874         if [ "${_packs:-0}" -ge 20 ]; then
 875                 >.needsgc
 876                 return 0
 877         fi
 878         _logfiles=
 879         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 880         if [ "${_logfiles:-0}" -ge 50 ]; then
 881                 >.needsgc
 882                 return 0
 883         fi
 884         # Truly git gc only checks the number of objects in the objects/17 directory
 885         # We check for -ge 10 which should make the probability of having more than
 886         # 5120 (20*256) loose objects present when there are less than 10 in
 887         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 888         _objfiles=
 889         ! [ -d objects/17 ] ||
 890         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 891         if [ "${_objfiles:-0}" -ge 10 ]; then
 892                 >.needsgc
 893                 return 0
 894         fi
 895 }
 896
 897 # A well-known UTF-8 locale is required for some of the fast-import providers
 898 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 899 # but that is not reliably UTF-8 but rather usually US-ASCII.
 900 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 901 # install time and store that in $var_utf8_locale if one is found.
 902 # If we cannot find one in the `locale -a` output then we just use a well-known
 903 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 904 # it.  We only set this temporarily when running the fast-import providers.
 905 set_utf8_locale() {
 906         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 907         export LC_ALL
 908 }
 909
 910 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 911 git_hg_fetch() (
 912         set_utf8_locale
 913         _python="${PYTHON:-python}"
 914         rm -f hg2git-marks.old hg2git-marks.new
 915         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 916                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 917                         if [ -n "$var_have_git_185" ]; then
 918                                 git cat-file --batch-check=':%(rest) %(objectname)'
 919                         else
 920                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 921                         fi
 922                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 923                 if [ -n "$var_have_git_171" ] &&
 924                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 925                         if [ -z "$var_have_git_185" ] ||
 926                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 927                                 _nm='hg-fast-export'
 928                                 GIT_AUTHOR_NAME="$_nm"
 929                                 GIT_COMMITTER_NAME="$_nm"
 930                                 GIT_AUTHOR_EMAIL="$_nm"
 931                                 GIT_COMMITTER_EMAIL="$_nm"
 932                                 export GIT_AUTHOR_NAME
 933                                 export GIT_COMMITTER_NAME
 934                                 export GIT_AUTHOR_EMAIL
 935                                 export GIT_COMMITTER_EMAIL
 936                                 git notes --ref=refs/notes/hg prune
 937                                 unset GIT_AUTHOR_NAME
 938                                 unset GIT_COMMITTER_NAME
 939                                 unset GIT_AUTHOR_EMAIL
 940                                 unset GIT_COMMITTER_EMAIL
 941                         fi
 942                 fi
 943         else
 944                 >hg2git-marks.old
 945         fi
 946         _err1=
 947         _err2=
 948         exec 3>&1
 949         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 950         $(
 951                 exec 4>&3 3>&1 1>&4 4>&-
 952                 {
 953                         _e1=0
 954                         _af="$(git config hg.authorsfile)" || :
 955                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 956                                 --repo    "$(pwd)/repo.hg" \
 957                                 --marks   "$(pwd)/hg2git-marks.old" \
 958                                 --mapping "$(pwd)/hg2git-mapping" \
 959                                 --heads   "$(pwd)/hg2git-heads" \
 960                                 --status  "$(pwd)/hg2git-state" \
 961                                 -U unknown --force --flatten --hg-hash'
 962                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 963                         eval "$_cmd" 3>&- || _e1=$?
 964                         echo $_e1 >&3
 965                 } |
 966                 {
 967                         _e2=0
 968                         git fast-import \
 969                                 --import-marks="$(pwd)/hg2git-marks.old" \
 970                                 --export-marks="$(pwd)/hg2git-marks.new" \
 971                                 --export-pack-edges="$(pwd)/gfi-packs" \
 972                                 --force 3>&- || _e2=$?
 973                         echo $_e2 >&3
 974                 }
 975         )
 976         EOT
 977         exec 3>&-
 978         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 979         mv -f hg2git-marks.new hg2git-marks
 980         rm -f hg2git-marks.old
 981         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 982         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 983 )