shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  91         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  92         # var_window_memory     Value to use for repack --window-memory=
  93         # var_big_file_threshold Value to use for core.bigFileThreshold
  94         # var_redelta_threshold Recompute deltas if no more than this many objs
  95         # var_upload_window     If not "", pack.window to use for upload-pack
  96         # var_log_window_size   Value to use for git-svn --log-window-size=
  97         # var_utf8_locale       Value to use for a UTF-8 locale if available
  98         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  99         # var_du_exclude        Option to exclude PATTERN from du if available
 100         # var_du_follow         Option to follow command line sym links if available
 101         _cfg_vars="$(get_girocco_config_pm_var_list)"
 102         eval "$_cfg_vars"
 103         printf '%s\n' "$_cfg_vars"
 104         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 105         _gver="$("$cfg_git_bin" version 2>/dev/null |
 106                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 107         printf 'var_git_ver=%s\n' "$_gver"
 108         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 109         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 110         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 111         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 112         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 113         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 114         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 121         __girocco_conf="$GIROCCO_CONF"
 122         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 123         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 124         printf "var_window_memory=%s\n" \
 125                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 126                 -MGirocco::Util -e 'print calc_windowmemory')"
 127         printf "var_big_file_threshold=%s\n" \
 128                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 129                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 130         printf "var_redelta_threshold=%s\n" \
 131                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 133         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 134            [ "$cfg_upload_pack_window" -le 50 ]; then
 135                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 136         else
 137                 printf "var_upload_window=%s\n" ""
 138         fi
 139         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 140         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 141         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 142                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 143                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 144                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 145         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 146         # On some broken platforms running xargs without -r and empty input runs the command
 147         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 148         # The disk usage report produces better numbers if du has an exclude option
 149         _x0="$(basename "$0")"
 150         _x0="${_x0%?}?*"
 151         for _duopt in --exclude -I; do
 152                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 153                         printf 'var_du_exclude=%s\n' "$_duopt"
 154                         break
 155                 fi
 156         done
 157         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 158                 printf 'var_du_follow=%s\n' "-H"
 159                 break
 160         fi
 161 )
 162
 163 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 164 # definitions from it rather than running Perl.
 165 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 166         # Import all the variables from Girocco::Config to the local environment,
 167         eval "$(get_girocco_config_var_list)"
 168 else
 169         # Import the variables from shlib_vars.sh which avoids needlessly
 170         # running another copy of Perl
 171         . "@basedir@/shlib_vars.sh"
 172 fi
 173
 174 # git_add_config "some.var=value"
 175 # every ' in value must be replaced with the 4-character sequence '\'' before
 176 # calling this function or Git will barf.  Will not be effective unless running
 177 # Git version 1.7.3 or later.
 178 git_add_config() {
 179         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 180         export GIT_CONFIG_PARAMETERS
 181 }
 182
 183 # Make sure we have a reproducible environment by using a controlled HOME dir
 184 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 185 HOME="$cfg_chroot/etc/girocco"
 186 TMPDIR="/tmp"
 187 GIT_CONFIG_NOSYSTEM=1
 188 GIT_ATTR_NOSYSTEM=1
 189 GIT_NO_REPLACE_OBJECTS=1
 190 GIT_TERMINAL_PROMPT=0
 191 GIT_PAGER="cat"
 192 PAGER="cat"
 193 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 194 GIT_SVN_NOTTY=1
 195 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 196 export XDG_CONFIG_HOME
 197 export HOME
 198 export TMPDIR
 199 export GIT_CONFIG_NOSYSTEM
 200 export GIT_ATTR_NOSYSTEM
 201 export GIT_NO_REPLACE_OBJECTS
 202 export GIT_TERMINAL_PROMPT
 203 export GIT_PAGER
 204 export PAGER
 205 export GIT_ASKPASS
 206 export GIT_SVN_NOTTY
 207 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 208 unset GIT_USER_AGENT
 209 unset GIT_HTTP_USER_AGENT
 210 if [ -n "$defined_cfg_git_client_ua" ]; then
 211         GIT_USER_AGENT="$cfg_git_client_ua"
 212         export GIT_USER_AGENT
 213         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 214         export GIT_HTTP_USER_AGENT
 215 fi
 216 unset GIT_CONFIG_PARAMETERS
 217 git_add_config "core.ignoreCase=false"
 218 git_add_config "core.pager=cat"
 219 if [ -n "$cfg_git_no_mmap" ]; then
 220         # Just like compiling with NO_MMAP
 221         git_add_config "core.packedGitWindowSize=1m"
 222 else
 223         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 224         git_add_config "core.packedGitWindowSize=32m"
 225 fi
 226 [ -z "$var_big_file_threshold" ] ||
 227         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 228 git_add_config "gc.auto=0"
 229
 230 # Make sure any sendmail.pl config is always available
 231 unset SENDMAIL_PL_HOST
 232 unset SENDMAIL_PL_PORT
 233 unset SENDMAIL_PL_NCBIN
 234 unset SENDMAIL_PL_NCOPT
 235 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 236 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 237 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 238 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 239
 240 # Set PATH and PYTHON to the values set by Config.pm, if any
 241 unset PYTHON
 242 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 243 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 244
 245 # Extra GIT variables that generally ought to be cleared, but whose clearing
 246 # could potentially interfere with the correct operation of hook scripts so
 247 # they are segregated into a separate function for use as appropriate
 248 clean_git_env() {
 249         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 250         unset GIT_CONFIG
 251         unset GIT_DIR
 252         unset GIT_GRAFT_FILE
 253         unset GIT_INDEX_FILE
 254         unset GIT_OBJECT_DIRECTORY
 255         unset GIT_NAMESPACE
 256 }
 257
 258 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 259 # and nc_openbsd to the desired executables because when using
 260 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 261 # different and unexpected ways:
 262 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 263 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 264 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 265 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 266 # setting only while running func which none of the /bin/sh's do.
 267 #
 268 # Instead we'd like to use an alias that provides the desired behavior without
 269 # any of the bad (a), (b) or (c) effects.
 270 #
 271 # However, unfortunately, some of the crazy /bin/sh implementations do not
 272 # recognize alias expansions when preceded by variable assignments!
 273 #
 274 # So we are left with git() {} and nc_openbsd() {} functions and in the
 275 # case of git() {} we can compensate for (b) and (c) failing to export
 276 # but not (a) and (b) persisting the values so the caller will simply
 277 # have to beware and explicitly unset any variables that should not persist
 278 # beyond the function call itself.
 279
 280 git() (
 281         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 282         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 283         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 284         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 285         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 286         exec "$cfg_git_bin" "$@"
 287 )
 288
 289 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 290 # that allows us to use git update-ref --stdin where supported and the slow shell
 291 # script where not, but only the "delete" operation is currently supported.
 292 git_updateref_stdin() {
 293         if [ -n "$var_have_git_185" ]; then
 294                 git update-ref --stdin
 295         else
 296                 while read -r _op _ref; do
 297                         case "$_op" in
 298                                 delete)
 299                                         git update-ref -d "$_ref"
 300                                         ;;
 301                                 *)
 302                                         echo "bad git_updateref_stdin op: $_op" >&2
 303                                         exit 1
 304                                         ;;
 305                         esac
 306                 done
 307         fi
 308 }
 309
 310 # see comments for git() -- callers must explicitly export all variables
 311 # intended for the commands these functions run before calling them
 312 perl() { command "${var_perl_bin:-perl}" "$@"; }
 313 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 314
 315 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 316
 317 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 318
 319 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 320
 321 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 322
 323 # Some platforms' broken xargs runs the command always at least once even if
 324 # there's no input unless given a special option.  Automatically supply the
 325 # option on those platforms by providing an xargs function.
 326 xargs() { command xargs $var_xargs_r "$@"; }
 327
 328 _addrlist() {
 329         _list=
 330         for _addr in "$@"; do
 331                 [ -z "$_list" ] || _list="$_list, "
 332                 _list="$_list$_addr"
 333         done
 334         echo "$_list"
 335 }
 336
 337 _sendmail() {
 338         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 339         if [ -n "$cfg_sender" ]; then
 340                 "$_mailer" -i -f "$cfg_sender" "$@"
 341         else
 342                 "$_mailer" -i "$@"
 343         fi
 344 }
 345
 346 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 347 # "References:" header.  It may be "" to suppress the "References" header.
 348 # Following arguments are just like mail function
 349 mailref() {
 350         _references=
 351         if [ $# -ge 1 ]; then
 352                 _references="$1"
 353                 shift
 354         fi
 355         _subject=
 356         if [ "$1" = "-s" ]; then
 357                 shift
 358                 _subject="$1"
 359                 shift
 360         fi
 361         {
 362                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 363                 echo "To: $(_addrlist "$@")"
 364                 [ -z "$_subject" ] || echo "Subject: $_subject"
 365                 echo "MIME-Version: 1.0"
 366                 echo "Content-Type: text/plain; charset=UTF-8"
 367                 echo "Content-Transfer-Encoding: 8bit"
 368                 [ -z "$_references" ] || echo "References: <$_references>"
 369                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 370                 echo "Auto-Submitted: auto-generated"
 371                 echo ""
 372                 cat
 373         } | _sendmail "$@"
 374 }
 375
 376 # Usage: mail [-s <subject>] <addr> [<addr>...]
 377 mail() {
 378         mailref "" "$@"
 379 }
 380
 381 # bang CMD... will execute the command with well-defined failure mode;
 382 # set bang_action to string of the failed action ('clone', 'update', ...);
 383 # re-define the bang_trap() function to do custom cleanup before bailing out
 384 bang() {
 385         bang_active=1
 386         bang_cmd="$*"
 387         bang_errcode=0
 388         if [ "${show_progress:-0}" != "0" ]; then
 389                 exec 3>&1
 390                 read -r bang_errcode <<-EOT || :
 391                 $(
 392                         exec 4>&3 3>&1 1>&4 4>&-
 393                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 394                 )
 395                 EOT
 396                 exec 3>&-
 397                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 398                         # All right. Cool.
 399                         bang_active=
 400                         bang_cmd=
 401                         return;
 402                 fi
 403         else
 404                 if "$@" >>"$bang_log" 2>&1; then
 405                         # All right. Cool.
 406                         bang_active=
 407                         bang_cmd=
 408                         return;
 409                 else
 410                         bang_errcode="$?"
 411                 fi
 412         fi
 413         bang_failed
 414 }
 415
 416 bang_failed() {
 417         bang_active=
 418         unset GIT_DIR
 419         >.banged
 420         cat "$bang_log" >.banglog
 421         echo "" >>.banglog
 422         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 423         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 424         if [ "${show_progress:-0}" != "0" ]; then
 425                 echo ""
 426                 echo "$bang_cmd failed with error code $bang_errcode"
 427         fi
 428         if [ -e .bangagain ]; then
 429                 git config --remove-section girocco.bang 2>/dev/null || :
 430                 rm -f .bangagain
 431         fi
 432         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 433         bangcount=$(( ${bangcount:-0} + 1 ))
 434         git config --int girocco.bang.count $bangcount
 435         if [ $bangcount -eq 1 ]; then
 436                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 437         fi
 438         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 439            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 440            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 441                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 442                 bangaddrs=
 443                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 444                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 445                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 446                 rsubj=
 447                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 448                 [ -z "$bangaddrs" ] ||
 449                 {
 450                         echo "$bang_cmd failed with error code $bang_errcode"
 451                         echo ""
 452                         rsubj=
 453                         if [ $bangcount -gt 1 ]; then
 454                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 455                                 echo ""
 456                         fi
 457                         echo "you will not receive any more notifications until recovery"
 458                         echo "this status message may be disabled on the project admin page"
 459                         echo ""
 460                         echo "Log follows:"
 461                         echo ""
 462                         cat "$bang_log"
 463                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 464                 git config --bool girocco.bang.messagesent true
 465         fi
 466         bangthrottle=
 467         [ $bangcount -lt 15 ] ||
 468                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 469                 bangthrottle=1
 470         bang_trap $bangthrottle
 471         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 472         exit $bang_errcode
 473 }
 474
 475 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 476 # Identical to bang CMD... except the command is eval'd instead of executed.
 477 bang_eval() {
 478         bang eval "$*"
 479 }
 480
 481 # Default bang settings:
 482 bang_setup() {
 483         bang_active=
 484         bang_action="lame_programmer"
 485         bang_trap() { :; }
 486         bang_tmpdir="${TMPDIR:-/tmp}"
 487         bang_tmpdir="${bang_tmpdir%/}"
 488         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 489         is_git_dir . || {
 490                 echo "bang_setup called with current directory not a git directory" >&2
 491                 exit 1
 492         }
 493         trap 'rm -f "$bang_log"' EXIT
 494         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 495         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 496 }
 497
 498 # Remove banged status
 499 bang_reset() {
 500         rm -f .banged .bangagain .banglog
 501         git config --remove-section girocco.bang 2>/dev/null || :
 502 }
 503
 504 # Check to see if banged status
 505 is_banged() {
 506         [ -e .banged ]
 507 }
 508
 509 # Check to see if banged message was sent
 510 was_banged_message_sent() {
 511         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 512 }
 513
 514 # Progress report - if show_progress is set, shows the given message.
 515 progress() {
 516         [ "${show_progress:-0}" = "0" ] || echo "$*"
 517 }
 518
 519 # Project config accessors; must be run in project directory
 520 config_get() {
 521         case "$1" in
 522         *.*)
 523                 git config "$1";;
 524         *)
 525                 git config "gitweb.$1";;
 526         esac
 527 }
 528
 529 config_set() {
 530         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 531 }
 532
 533 config_set_raw() {
 534         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 535 }
 536
 537 config_get_date_seconds() {
 538         _dt="$(config_get "$1")" || :
 539         [ -n "$_dt" ] || return 1
 540         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 541         [ -n "$_ds" ] || return 1
 542         echo "$_ds"
 543 }
 544
 545 # Tool for checking whether given number of seconds has not passed yet
 546 check_interval() {
 547         os="$(config_get_date_seconds "$1")" || return 1
 548         ns="$(date +%s)"
 549         [ $ns -lt $(($os+$2)) ]
 550 }
 551
 552 # Check if we are running with effective root permissions
 553 is_root() {
 554         [ "$(id -u 2>/dev/null)" = "0" ]
 555 }
 556
 557 # Check to see if the single argument is a Git directory
 558 is_git_dir() {
 559         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 560         # And we are slightly more picky (must be refs/.+ not refs/.*)
 561         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 562         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 563         if [ -L "$1/HEAD" ]; then
 564                 _hr="$(readlink "$1/HEAD")"
 565                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 566         fi
 567         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 568         read -r _hr <"$1/HEAD" || return 1
 569         case "$_hr" in
 570                 $octet20*)
 571                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 572                         return 0;;
 573                 ref:refs/?*)
 574                         return 0;;
 575                 ref:*)
 576                         _hr="${_hr##ref:*[ $tab]}"
 577                         case "$_hr" in "refs/"?*) return 0;; esac
 578         esac
 579         return 1
 580 }
 581
 582 # List all Git repositories, with given prefix if specified, one-per-line
 583 # All project names starting with _ are always excluded from the result
 584 get_repo_list() {
 585         if [ -n "$1" ]; then
 586                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 587         else
 588                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 589         fi | while IFS=: read name id; do
 590                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 591         done
 592 }
 593
 594 # set the variable named by the first argument to the project part (i.e. WITH
 595 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 596 # specified by the second argument.
 597 # This function cannot be fooled by symbolic links.
 598 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 599 # The directory specified by the second argument must exist.
 600 v_get_proj_from_dir() {
 601         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 602         [ -d "$2" ] || return 1
 603         case "$2" in
 604         "$cfg_reporoot/"?*)
 605                 # Simple case that does not need any fancy footwork
 606                 _projpart="${2#$cfg_reporoot/}"
 607                 ;;
 608         *)
 609                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 610                 _abspd="$(cd "$2" && pwd -P)"
 611                 case "$_abspd" in
 612                 "$_absrr/"?*)
 613                         # The normal case
 614                         _projpart="${_abspd#$_absrr/}"
 615                         ;;
 616                 *)
 617                         # Must have been reached via a symbolic link, but
 618                         # we have no way to know the source, so use a
 619                         # generic "_external" leader combined with just the
 620                         # trailing directory name
 621                         _abspd="${_abspd%/}"
 622                         _abspd="${_abspd%/.git}"
 623                         _projpart="_external/${_abspd##*/}"
 624                         ;;
 625                 esac
 626         esac
 627         eval "$1="'"$_projpart"'
 628 }
 629
 630 # Return success if the given project name has at least one immediate child fork
 631 # that has a non-zero length alternates file
 632 has_forks_with_alternates() {
 633         _prj="${1%.git}"
 634         [ -n "$_prj" ] || return 1
 635         [ -d "$cfg_reporoot/$_prj" ] || return 1
 636         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 637         if
 638                 get_repo_list "$_prj/[^/:][^/:]*:" |
 639                 while read -r _prjname && [ -n "$_prjname" ]; do
 640                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 641                         exit 1 # will only exit implicit subshell created by '|'
 642                 done
 643         then
 644                 return 1
 645         fi
 646         return 0
 647 }
 648
 649 # returns empty string and error for empty string otherwise one of
 650 #   m  =>  normal Git mirror
 651 #   s  =>  mirror from svn source
 652 #   d  =>  mirror from darcs source
 653 #   b  =>  mirror from bzr source
 654 #   h  =>  mirror from hg source
 655 #   w  =>  mirror from mediawiki source
 656 #   f  =>  mirror from other fast-import source
 657 # note that if the string is non-empty and none of s, d, b or h match the
 658 # return will always be type m regardless of whether it's a valid Git URL
 659 get_url_mirror_type() {
 660         case "$1" in
 661                 "")
 662                         return 1
 663                         ;;
 664                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 665                         echo 's'
 666                         ;;
 667                 darcs://*)
 668                         echo 'd'
 669                         ;;
 670                 bzr://*)
 671                         echo 'b'
 672                         ;;
 673                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 674                         echo 'h'
 675                         ;;
 676                 mediawiki::*)
 677                         echo 'w'
 678                         ;;
 679                 *)
 680                         echo 'm'
 681                         ;;
 682         esac
 683         return 0
 684 }
 685
 686 # returns false for empty string
 687 # returns true if the passed in url is a mirror using git fast-import
 688 is_gfi_mirror_url() {
 689         [ -n "$1" ] || return 1
 690         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 691                 d|b|h|w|f)
 692                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 693                         # and so do generic "f" fast-import mirrors
 694                         return 0
 695                         ;;
 696                 *)
 697                         # Don't think git-svn currently uses git fast-import
 698                         # And Git mirrors certainly do not
 699                         return 1
 700                         ;;
 701         esac
 702         # assume it does not use git fast-import
 703         return 1
 704 }
 705
 706 # returns false for empty string
 707 # returns true if the passed in url is a mirror using git-svn
 708 is_svn_mirror_url() {
 709         [ -n "$1" ] || return 1
 710         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 711 }
 712
 713 # returns mirror url for gitweb.baseurl of git directory
 714 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 715 # will fail if the directory does not have .nofetch and gitweb.baseurl
 716 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 717 # automatically strips any leading "disabled " prefix before returning result
 718 get_mirror_url() {
 719         _gitdir="${1:-.}"
 720         # always return empty for non-mirrors
 721         ! [ -e "$_gitdir/.nofetch" ] || return 0
 722         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 723         _url="${_url##* }"
 724         [ -n "$_url" ] || return 1
 725         printf '%s\n' "$_url"
 726         return 0
 727 }
 728
 729 # returns get_url_mirror_type for gitweb.baseurl of git directory
 730 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 731 # will fail if the directory does not have .nofetch and gitweb.baseurl
 732 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 733 # automatically strips any leading "disabled " prefix before testing
 734 get_mirror_type() {
 735         _url="$(get_mirror_url "$@")" || return 1
 736         [ -n "$_url" ] || return 0
 737         get_url_mirror_type "$_url"
 738 }
 739
 740 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 741 is_gfi_mirror() {
 742         _url="$(get_mirror_url "$@")" || return 1
 743         is_gfi_mirror_url "$_url"
 744 }
 745
 746 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 747 is_svn_mirror() {
 748         _url="$(get_mirror_url "$@")" || return 1
 749         is_svn_mirror_url "$_url"
 750 }
 751
 752 # current directory must already be set to Git repository
 753 # if girocco.headok is already true succeeds without doing anything
 754 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 755 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 756 # then refs/heads/trunk and finally the first top-level head from
 757 # refs/heads/* (i.e. only two slashes in the name) and finally any
 758 # existing refs/heads.  The first one to succeed wins and sets headok=true
 759 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 760 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 761 # when the repository is being set up -- if the HEAD is later deleted (through
 762 # a push or fetch --prune) that's no longer our responsibility to fix
 763 check_and_set_head() {
 764         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 765         if git rev-parse --verify --quiet HEAD >/dev/null; then
 766                 git config --bool girocco.headok true
 767                 return 0
 768         fi
 769         for _hr in refs/heads/master refs/heads/trunk; do
 770                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 771                         _update_head_symref "$_hr"
 772                         return 0
 773                 fi
 774         done
 775         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 776         while read -r _hr; do
 777                 case "${_hr#refs/heads/}" in */*) :;; *)
 778                         _update_head_symref "$_hr"
 779                         exit 1 # exit subshell created by "|"
 780                 esac
 781         done || return 0
 782         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 783         if [ -n "$_hr" ]; then
 784                 _update_head_symref "$_hr"
 785                 return 0
 786         fi
 787         return 1
 788 }
 789 _update_head_symref() {
 790         git symbolic-ref HEAD "$1"
 791         git config --bool girocco.headok true
 792         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 793 }
 794
 795 # current directory must already be set to Git repository
 796 # if the directory needs to have gc run and .needsgc is not already set
 797 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 798 # Girocco shouldn't generate any loose objects but we check for that anyway
 799 check_and_set_needsgc() {
 800         ! [ -e .needsgc ] || return 0
 801         _packs=
 802         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 803         if [ "${_packs:-0}" -ge 20 ]; then
 804                 >.needsgc
 805                 return 0
 806         fi
 807         _logfiles=
 808         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 809         if [ "${_logfiles:-0}" -ge 50 ]; then
 810                 >.needsgc
 811                 return 0
 812         fi
 813         # Truly git gc only checks the number of objects in the objects/17 directory
 814         # We check for -ge 10 which should make the probability of having more than
 815         # 5120 (20*256) loose objects present when there are less than 10 in
 816         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 817         _objfiles=
 818         ! [ -d objects/17 ] ||
 819         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 820         if [ "${_objfiles:-0}" -ge 10 ]; then
 821                 >.needsgc
 822                 return 0
 823         fi
 824 }
 825
 826 # A well-known UTF-8 locale is required for some of the fast-import providers
 827 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 828 # but that is not reliably UTF-8 but rather usually US-ASCII.
 829 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 830 # install time and store that in $var_utf8_locale if one is found.
 831 # If we cannot find one in the `locale -a` output then we just use a well-known
 832 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 833 # it.  We only set this temporarily when running the fast-import providers.
 834 set_utf8_locale() {
 835         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 836         export LC_ALL
 837 }
 838
 839 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 840 git_hg_fetch() (
 841         set_utf8_locale
 842         _python="${PYTHON:-python}"
 843         rm -f hg2git-marks.old hg2git-marks.new
 844         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 845                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 846                         if [ -n "$var_have_git_185" ]; then
 847                                 git cat-file --batch-check=':%(rest) %(objectname)'
 848                         else
 849                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 850                         fi
 851                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 852                 if [ -n "$var_have_git_171" ] &&
 853                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 854                         if [ -z "$var_have_git_185" ] ||
 855                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 856                                 _nm='hg-fast-export'
 857                                 GIT_AUTHOR_NAME="$_nm"
 858                                 GIT_COMMITTER_NAME="$_nm"
 859                                 GIT_AUTHOR_EMAIL="$_nm"
 860                                 GIT_COMMITTER_EMAIL="$_nm"
 861                                 export GIT_AUTHOR_NAME
 862                                 export GIT_COMMITTER_NAME
 863                                 export GIT_AUTHOR_EMAIL
 864                                 export GIT_COMMITTER_EMAIL
 865                                 git notes --ref=refs/notes/hg prune
 866                                 unset GIT_AUTHOR_NAME
 867                                 unset GIT_COMMITTER_NAME
 868                                 unset GIT_AUTHOR_EMAIL
 869                                 unset GIT_COMMITTER_EMAIL
 870                         fi
 871                 fi
 872         else
 873                 >hg2git-marks.old
 874         fi
 875         _err1=
 876         _err2=
 877         exec 3>&1
 878         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 879         $(
 880                 exec 4>&3 3>&1 1>&4 4>&-
 881                 {
 882                         _e1=0
 883                         _af="$(git config hg.authorsfile)" || :
 884                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 885                                 --repo    "$(pwd)/repo.hg" \
 886                                 --marks   "$(pwd)/hg2git-marks.old" \
 887                                 --mapping "$(pwd)/hg2git-mapping" \
 888                                 --heads   "$(pwd)/hg2git-heads" \
 889                                 --status  "$(pwd)/hg2git-state" \
 890                                 -U unknown --force --flatten --hg-hash'
 891                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 892                         eval "$_cmd" 3>&- || _e1=$?
 893                         echo $_e1 >&3
 894                 } |
 895                 {
 896                         _e2=0
 897                         git fast-import \
 898                                 --import-marks="$(pwd)/hg2git-marks.old" \
 899                                 --export-marks="$(pwd)/hg2git-marks.new" \
 900                                 --export-pack-edges="$(pwd)/gfi-packs" \
 901                                 --force 3>&- || _e2=$?
 902                         echo $_e2 >&3
 903                 }
 904         )
 905         EOT
 906         exec 3>&-
 907         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 908         mv -f hg2git-marks.new hg2git-marks
 909         rm -f hg2git-marks.old
 910         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 911         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 912 )