shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_235      Set to 1 if git version >= 2.3.5 otherwise ''
  91         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  92         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  93         # var_window_memory     Value to use for repack --window-memory=
  94         # var_big_file_threshold Value to use for core.bigFileThreshold
  95         # var_redelta_threshold Recompute deltas if no more than this many objs
  96         # var_upload_window     If not "", pack.window to use for upload-pack
  97         # var_log_window_size   Value to use for git-svn --log-window-size=
  98         # var_utf8_locale       Value to use for a UTF-8 locale if available
  99         # var_xargs_r           A "-r" if xargs needs it to behave correctly
 100         # var_du_exclude        Option to exclude PATTERN from du if available
 101         # var_du_follow         Option to follow command line sym links if available
 102         _cfg_vars="$(get_girocco_config_pm_var_list)"
 103         eval "$_cfg_vars"
 104         printf '%s\n' "$_cfg_vars"
 105         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 106         _gver="$("$cfg_git_bin" version 2>/dev/null |
 107                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 108         printf 'var_git_ver=%s\n' "$_gver"
 109         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 110         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 111         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 112         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 113         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 114         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_235=%s\n' "$([ $(vcmp "$_gver" 2.3.5) -ge 0 ] && echo 1)"
 121         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 122         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 123         __girocco_conf="$GIROCCO_CONF"
 124         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 125         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 126         printf "var_window_memory=%s\n" \
 127                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 128                 -MGirocco::Util -e 'print calc_windowmemory')"
 129         printf "var_big_file_threshold=%s\n" \
 130                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 131                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 132         printf "var_redelta_threshold=%s\n" \
 133                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 134                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 135         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 136            [ "$cfg_upload_pack_window" -le 50 ]; then
 137                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 138         else
 139                 printf "var_upload_window=%s\n" ""
 140         fi
 141         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 142         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 143         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 144                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 145                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 146                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 147         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 148         # On some broken platforms running xargs without -r and empty input runs the command
 149         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 150         # The disk usage report produces better numbers if du has an exclude option
 151         _x0="$(basename "$0")"
 152         _x0="${_x0%?}?*"
 153         for _duopt in --exclude -I; do
 154                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 155                         printf 'var_du_exclude=%s\n' "$_duopt"
 156                         break
 157                 fi
 158         done
 159         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 160                 printf 'var_du_follow=%s\n' "-H"
 161                 break
 162         fi
 163 )
 164
 165 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 166 # definitions from it rather than running Perl.
 167 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 168         # Import all the variables from Girocco::Config to the local environment,
 169         eval "$(get_girocco_config_var_list)"
 170 else
 171         # Import the variables from shlib_vars.sh which avoids needlessly
 172         # running another copy of Perl
 173         . "@basedir@/shlib_vars.sh"
 174 fi
 175
 176 # git_add_config "some.var=value"
 177 # every ' in value must be replaced with the 4-character sequence '\'' before
 178 # calling this function or Git will barf.  Will not be effective unless running
 179 # Git version 1.7.3 or later.
 180 git_add_config() {
 181         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 182         export GIT_CONFIG_PARAMETERS
 183 }
 184
 185 # Make sure we have a reproducible environment by using a controlled HOME dir
 186 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 187 HOME="$cfg_chroot/etc/girocco"
 188 TMPDIR="/tmp"
 189 GIT_CONFIG_NOSYSTEM=1
 190 GIT_ATTR_NOSYSTEM=1
 191 GIT_NO_REPLACE_OBJECTS=1
 192 GIT_TERMINAL_PROMPT=0
 193 GIT_PAGER="cat"
 194 PAGER="cat"
 195 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 196 GIT_SVN_NOTTY=1
 197 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 198 GIT_SSH="$cfg_basedir/bin/git-ssh"
 199 SVN_SSH="$cfg_basedir/bin/git-ssh"
 200 export XDG_CONFIG_HOME
 201 export HOME
 202 export TMPDIR
 203 export GIT_CONFIG_NOSYSTEM
 204 export GIT_ATTR_NOSYSTEM
 205 export GIT_NO_REPLACE_OBJECTS
 206 export GIT_TERMINAL_PROMPT
 207 export GIT_PAGER
 208 export PAGER
 209 export GIT_ASKPASS
 210 export GIT_SVN_NOTTY
 211 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 212 export GIT_SSH
 213 export SVN_SSH
 214 unset GIT_USER_AGENT
 215 unset GIT_HTTP_USER_AGENT
 216 if [ -n "$defined_cfg_git_client_ua" ]; then
 217         GIT_USER_AGENT="$cfg_git_client_ua"
 218         export GIT_USER_AGENT
 219 fi
 220 unset GIT_CONFIG_PARAMETERS
 221 git_add_config "core.ignoreCase=false"
 222 git_add_config "core.pager=cat"
 223 if [ -n "$cfg_git_no_mmap" ]; then
 224         # Just like compiling with NO_MMAP
 225         git_add_config "core.packedGitWindowSize=1m"
 226 else
 227         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 228         git_add_config "core.packedGitWindowSize=32m"
 229 fi
 230 [ -z "$var_big_file_threshold" ] ||
 231         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 232 git_add_config "gc.auto=0"
 233
 234 # Make sure any sendmail.pl config is always available
 235 unset SENDMAIL_PL_HOST
 236 unset SENDMAIL_PL_PORT
 237 unset SENDMAIL_PL_NCBIN
 238 unset SENDMAIL_PL_NCOPT
 239 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 240 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 241 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 242 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 243
 244 # Set PATH and PYTHON to the values set by Config.pm, if any
 245 unset PYTHON
 246 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 247 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 248
 249 # Extra GIT variables that generally ought to be cleared, but whose clearing
 250 # could potentially interfere with the correct operation of hook scripts so
 251 # they are segregated into a separate function for use as appropriate
 252 clean_git_env() {
 253         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 254         unset GIT_CONFIG
 255         unset GIT_DIR
 256         unset GIT_GRAFT_FILE
 257         unset GIT_INDEX_FILE
 258         unset GIT_OBJECT_DIRECTORY
 259         unset GIT_NAMESPACE
 260 }
 261
 262 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 263 # and nc_openbsd to the desired executables because when using
 264 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 265 # different and unexpected ways:
 266 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 267 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 268 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 269 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 270 # setting only while running func which none of the /bin/sh's do.
 271 #
 272 # Instead we'd like to use an alias that provides the desired behavior without
 273 # any of the bad (a), (b) or (c) effects.
 274 #
 275 # However, unfortunately, some of the crazy /bin/sh implementations do not
 276 # recognize alias expansions when preceded by variable assignments!
 277 #
 278 # So we are left with git() {} and nc_openbsd() {} functions and in the
 279 # case of git() {} we can compensate for (b) and (c) failing to export
 280 # but not (a) and (b) persisting the values so the caller will simply
 281 # have to beware and explicitly unset any variables that should not persist
 282 # beyond the function call itself.
 283
 284 git() (
 285         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 286         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 287         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 288         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 289         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 290         exec "$cfg_git_bin" "$@"
 291 )
 292
 293 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 294 # that allows us to use git update-ref --stdin where supported and the slow shell
 295 # script where not, but only the "delete" operation is currently supported.
 296 git_updateref_stdin() {
 297         if [ -n "$var_have_git_185" ]; then
 298                 git update-ref --stdin
 299         else
 300                 while read -r _op _ref; do
 301                         case "$_op" in
 302                                 delete)
 303                                         git update-ref -d "$_ref"
 304                                         ;;
 305                                 *)
 306                                         echo "bad git_updateref_stdin op: $_op" >&2
 307                                         exit 1
 308                                         ;;
 309                         esac
 310                 done
 311         fi
 312 }
 313
 314 # see comments for git() -- callers must explicitly export all variables
 315 # intended for the commands these functions run before calling them
 316 perl() { command "${var_perl_bin:-perl}" "$@"; }
 317 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 318
 319 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 320
 321 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 322
 323 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 324
 325 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 326
 327 # Some platforms' broken xargs runs the command always at least once even if
 328 # there's no input unless given a special option.  Automatically supply the
 329 # option on those platforms by providing an xargs function.
 330 xargs() { command xargs $var_xargs_r "$@"; }
 331
 332 _addrlist() {
 333         _list=
 334         for _addr in "$@"; do
 335                 [ -z "$_list" ] || _list="$_list, "
 336                 _list="$_list$_addr"
 337         done
 338         echo "$_list"
 339 }
 340
 341 _sendmail() {
 342         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 343         if [ -n "$cfg_sender" ]; then
 344                 "$_mailer" -i -f "$cfg_sender" "$@"
 345         else
 346                 "$_mailer" -i "$@"
 347         fi
 348 }
 349
 350 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 351 # "References:" header.  It may be "" to suppress the "References" header.
 352 # Following arguments are just like mail function
 353 mailref() {
 354         _references=
 355         if [ $# -ge 1 ]; then
 356                 _references="$1"
 357                 shift
 358         fi
 359         _subject=
 360         if [ "$1" = "-s" ]; then
 361                 shift
 362                 _subject="$1"
 363                 shift
 364         fi
 365         {
 366                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 367                 echo "To: $(_addrlist "$@")"
 368                 [ -z "$_subject" ] || echo "Subject: $_subject"
 369                 echo "MIME-Version: 1.0"
 370                 echo "Content-Type: text/plain; charset=UTF-8"
 371                 echo "Content-Transfer-Encoding: 8bit"
 372                 [ -z "$_references" ] || echo "References: <$_references>"
 373                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 374                 echo "Auto-Submitted: auto-generated"
 375                 echo ""
 376                 cat
 377         } | _sendmail "$@"
 378 }
 379
 380 # Usage: mail [-s <subject>] <addr> [<addr>...]
 381 mail() {
 382         mailref "" "$@"
 383 }
 384
 385 # bang CMD... will execute the command with well-defined failure mode;
 386 # set bang_action to string of the failed action ('clone', 'update', ...);
 387 # re-define the bang_trap() function to do custom cleanup before bailing out
 388 bang() {
 389         bang_errcode=
 390         bang_catch "$@"
 391         [ "${bang_errcode:-0}" = "0" ] || bang_failed
 392 }
 393
 394 bang_catch() {
 395         bang_active=1
 396         bang_cmd="$*"
 397         bang_errcode=0
 398         if [ "${show_progress:-0}" != "0" ]; then
 399                 exec 3>&1
 400                 read -r bang_errcode <<-EOT || :
 401                 $(
 402                         exec 4>&3 3>&1 1>&4 4>&-
 403                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 404                 )
 405                 EOT
 406                 exec 3>&-
 407                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 408                         # All right. Cool.
 409                         bang_active=
 410                         bang_cmd=
 411                         return;
 412                 fi
 413         else
 414                 if "$@" >>"$bang_log" 2>&1; then
 415                         # All right. Cool.
 416                         bang_active=
 417                         bang_cmd=
 418                         return;
 419                 else
 420                         bang_errcode="$?"
 421                 fi
 422         fi
 423 }
 424
 425 bang_failed() {
 426         bang_active=
 427         unset GIT_DIR
 428         >.banged
 429         cat "$bang_log" >.banglog
 430         echo "" >>.banglog
 431         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 432         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 433         if [ "${show_progress:-0}" != "0" ]; then
 434                 echo ""
 435                 echo "$bang_cmd failed with error code $bang_errcode"
 436         fi
 437         if [ -e .bangagain ]; then
 438                 git config --remove-section girocco.bang 2>/dev/null || :
 439                 rm -f .bangagain
 440         fi
 441         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 442         bangcount=$(( ${bangcount:-0} + 1 ))
 443         git config --int girocco.bang.count $bangcount
 444         if [ $bangcount -eq 1 ]; then
 445                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 446         fi
 447         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 448            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 449            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 450                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 451                 bangaddrs=
 452                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 453                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 454                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 455                 rsubj=
 456                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 457                 [ -z "$bangaddrs" ] ||
 458                 {
 459                         echo "$bang_cmd failed with error code $bang_errcode"
 460                         echo ""
 461                         rsubj=
 462                         if [ $bangcount -gt 1 ]; then
 463                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 464                                 echo ""
 465                         fi
 466                         echo "you will not receive any more notifications until recovery"
 467                         echo "this status message may be disabled on the project admin page"
 468                         echo ""
 469                         echo "Log follows:"
 470                         echo ""
 471                         cat "$bang_log"
 472                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 473                 git config --bool girocco.bang.messagesent true
 474         fi
 475         bangthrottle=
 476         [ $bangcount -lt 15 ] ||
 477                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 478                 bangthrottle=1
 479         bang_trap $bangthrottle
 480         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 481         exit $bang_errcode
 482 }
 483
 484 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 485 # Identical to bang CMD... except the command is eval'd instead of executed.
 486 bang_eval() {
 487         bang eval "$*"
 488 }
 489
 490 # Default bang settings:
 491 bang_setup() {
 492         bang_active=
 493         bang_action="lame_programmer"
 494         bang_trap() { :; }
 495         bang_tmpdir="${TMPDIR:-/tmp}"
 496         bang_tmpdir="${bang_tmpdir%/}"
 497         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 498         is_git_dir . || {
 499                 echo "bang_setup called with current directory not a git directory" >&2
 500                 exit 1
 501         }
 502         trap 'rm -f "$bang_log"' EXIT
 503         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 504         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 505 }
 506
 507 # Remove banged status
 508 bang_reset() {
 509         rm -f .banged .bangagain .banglog
 510         git config --remove-section girocco.bang 2>/dev/null || :
 511 }
 512
 513 # Check to see if banged status
 514 is_banged() {
 515         [ -e .banged ]
 516 }
 517
 518 # Check to see if banged message was sent
 519 was_banged_message_sent() {
 520         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 521 }
 522
 523 # Progress report - if show_progress is set, shows the given message.
 524 progress() {
 525         [ "${show_progress:-0}" = "0" ] || echo "$*"
 526 }
 527
 528 # Project config accessors; must be run in project directory
 529 config_get() {
 530         case "$1" in
 531         *.*)
 532                 git config "$1";;
 533         *)
 534                 git config "gitweb.$1";;
 535         esac
 536 }
 537
 538 config_set() {
 539         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 540 }
 541
 542 config_set_raw() {
 543         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 544 }
 545
 546 config_get_date_seconds() {
 547         _dt="$(config_get "$1")" || :
 548         [ -n "$_dt" ] || return 1
 549         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 550         [ -n "$_ds" ] || return 1
 551         echo "$_ds"
 552 }
 553
 554 # Tool for checking whether given number of seconds has not passed yet
 555 check_interval() {
 556         os="$(config_get_date_seconds "$1")" || return 1
 557         ns="$(date +%s)"
 558         [ $ns -lt $(($os+$2)) ]
 559 }
 560
 561 # Check if we are running with effective root permissions
 562 is_root() {
 563         [ "$(id -u 2>/dev/null)" = "0" ]
 564 }
 565
 566 # Check to see if the single argument (default ".") is a Git directory
 567 is_git_dir() {
 568         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 569         # And we are slightly more picky (must be refs/.+ not refs/.*)
 570         [ $# -ne 0 ] || set -- "."
 571         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 572         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 573         if [ -L "$1/HEAD" ]; then
 574                 _hr="$(readlink "$1/HEAD")"
 575                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 576         fi
 577         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 578         read -r _hr <"$1/HEAD" || return 1
 579         case "$_hr" in
 580                 $octet20*)
 581                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 582                         return 0;;
 583                 ref:refs/?*)
 584                         return 0;;
 585                 ref:*)
 586                         _hr="${_hr##ref:*[ $tab]}"
 587                         case "$_hr" in "refs/"?*) return 0;; esac
 588         esac
 589         return 1
 590 }
 591
 592 # Check to see if the single argument (default ".") is a directory with no refs
 593 is_empty_refs_dir() {
 594         [ $# -ne 0 ] || set -- "."
 595         if [ -s "$1/packed-refs" ]; then
 596                 # could be a packed-refs file with just a '# pack-refs ..." line
 597                 # null hash lines and peel lines do not count either
 598                 _refcnt="$(( $(LC_ALL=C sed <"$1/packed-refs" \
 599                         -e "/^00* /d" \
 600                         -e "/^$octet20$hexdig* refs\/[^ $tab]*\$/!d" | wc -l) ))"
 601                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 602         fi
 603         if [ -d "$1/refs" ]; then
 604                 # quick and dirty check, doesn't try to validate contents
 605                 # or ignore embedded symbolic refs
 606                 _refcnt="$(( $(find -L "$1/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) ))"
 607                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 608         fi
 609         # last chance a detached HEAD (we ignore any linked working trees though)
 610         [ -s "$1/HEAD" ] && read -r _hr <"$1/HEAD" && [ -n "$_hr" ] || return 0
 611         [ "${_hr#*[!0-9a-f]}" != "$_hr" ] || [ "${_hr#*[!0]}" = "$_hr" ] || [ "${#_hr}" -lt 40 ] || return 1
 612         return 0
 613 }
 614
 615 # List all Git repositories, with given prefix if specified, one-per-line
 616 # All project names starting with _ are always excluded from the result
 617 get_repo_list() {
 618         if [ -n "$1" ]; then
 619                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 620         else
 621                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 622         fi | while IFS=: read name id; do
 623                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 624         done
 625 }
 626
 627 # set the variable named by the first argument to the project part (i.e. WITH
 628 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 629 # specified by the second argument.
 630 # This function cannot be fooled by symbolic links.
 631 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 632 # The directory specified by the second argument must exist.
 633 v_get_proj_from_dir() {
 634         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 635         [ -d "$2" ] || return 1
 636         case "$2" in
 637         "$cfg_reporoot/"?*)
 638                 # Simple case that does not need any fancy footwork
 639                 _projpart="${2#$cfg_reporoot/}"
 640                 ;;
 641         *)
 642                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 643                 _abspd="$(cd "$2" && pwd -P)"
 644                 case "$_abspd" in
 645                 "$_absrr/"?*)
 646                         # The normal case
 647                         _projpart="${_abspd#$_absrr/}"
 648                         ;;
 649                 *)
 650                         # Must have been reached via a symbolic link, but
 651                         # we have no way to know the source, so use a
 652                         # generic "_external" leader combined with just the
 653                         # trailing directory name
 654                         _abspd="${_abspd%/}"
 655                         _abspd="${_abspd%/.git}"
 656                         _projpart="_external/${_abspd##*/}"
 657                         ;;
 658                 esac
 659         esac
 660         eval "$1="'"$_projpart"'
 661 }
 662
 663 # Return success if the given project name has at least one immediate child fork
 664 # that has a non-zero length alternates file
 665 has_forks_with_alternates() {
 666         _prj="${1%.git}"
 667         [ -n "$_prj" ] || return 1
 668         [ -d "$cfg_reporoot/$_prj" ] || return 1
 669         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 670         if
 671                 get_repo_list "$_prj/[^/:][^/:]*:" |
 672                 while read -r _prjname && [ -n "$_prjname" ]; do
 673                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 674                         exit 1 # will only exit implicit subshell created by '|'
 675                 done
 676         then
 677                 return 1
 678         fi
 679         return 0
 680 }
 681
 682 # returns empty string and error for empty string otherwise one of
 683 #   m  =>  normal Git mirror
 684 #   s  =>  mirror from svn source
 685 #   d  =>  mirror from darcs source
 686 #   b  =>  mirror from bzr source
 687 #   h  =>  mirror from hg source
 688 #   w  =>  mirror from mediawiki source
 689 #   f  =>  mirror from other fast-import source
 690 # note that if the string is non-empty and none of s, d, b or h match the
 691 # return will always be type m regardless of whether it's a valid Git URL
 692 get_url_mirror_type() {
 693         case "$1" in
 694                 "")
 695                         return 1
 696                         ;;
 697                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 698                         echo 's'
 699                         ;;
 700                 darcs://*)
 701                         echo 'd'
 702                         ;;
 703                 bzr://*)
 704                         echo 'b'
 705                         ;;
 706                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 707                         echo 'h'
 708                         ;;
 709                 mediawiki::*)
 710                         echo 'w'
 711                         ;;
 712                 *)
 713                         echo 'm'
 714                         ;;
 715         esac
 716         return 0
 717 }
 718
 719 # returns false for empty string
 720 # returns true if the passed in url is a mirror using git fast-import
 721 is_gfi_mirror_url() {
 722         [ -n "$1" ] || return 1
 723         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 724                 d|b|h|w|f)
 725                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 726                         # and so do generic "f" fast-import mirrors
 727                         return 0
 728                         ;;
 729                 *)
 730                         # Don't think git-svn currently uses git fast-import
 731                         # And Git mirrors certainly do not
 732                         return 1
 733                         ;;
 734         esac
 735         # assume it does not use git fast-import
 736         return 1
 737 }
 738
 739 # returns false for empty string
 740 # returns true if the passed in url is a mirror using git-svn
 741 is_svn_mirror_url() {
 742         [ -n "$1" ] || return 1
 743         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 744 }
 745
 746 # returns mirror url for gitweb.baseurl of git directory
 747 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 748 # will fail if the directory does not have .nofetch and gitweb.baseurl
 749 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 750 # automatically strips any leading "disabled " prefix before returning result
 751 get_mirror_url() {
 752         _gitdir="${1:-.}"
 753         # always return empty for non-mirrors
 754         ! [ -e "$_gitdir/.nofetch" ] || return 0
 755         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 756         _url="${_url##* }"
 757         [ -n "$_url" ] || return 1
 758         printf '%s\n' "$_url"
 759         return 0
 760 }
 761
 762 # returns get_url_mirror_type for gitweb.baseurl of git directory
 763 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 764 # will fail if the directory does not have .nofetch and gitweb.baseurl
 765 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 766 # automatically strips any leading "disabled " prefix before testing
 767 get_mirror_type() {
 768         _url="$(get_mirror_url "$@")" || return 1
 769         [ -n "$_url" ] || return 0
 770         get_url_mirror_type "$_url"
 771 }
 772
 773 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 774 is_gfi_mirror() {
 775         _url="$(get_mirror_url "$@")" || return 1
 776         is_gfi_mirror_url "$_url"
 777 }
 778
 779 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 780 is_svn_mirror() {
 781         _url="$(get_mirror_url "$@")" || return 1
 782         is_svn_mirror_url "$_url"
 783 }
 784
 785 # current directory must already be set to Git repository
 786 # if girocco.headok is already true succeeds without doing anything
 787 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 788 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 789 # then refs/heads/trunk and finally the first top-level head from
 790 # refs/heads/* (i.e. only two slashes in the name) and finally any
 791 # existing refs/heads.  The first one to succeed wins and sets headok=true
 792 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 793 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 794 # when the repository is being set up -- if the HEAD is later deleted (through
 795 # a push or fetch --prune) that's no longer our responsibility to fix
 796 check_and_set_head() {
 797         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 798         if git rev-parse --verify --quiet HEAD >/dev/null; then
 799                 git config --bool girocco.headok true
 800                 return 0
 801         fi
 802         for _hr in refs/heads/master refs/heads/trunk; do
 803                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 804                         _update_head_symref "$_hr"
 805                         return 0
 806                 fi
 807         done
 808         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 809         while read -r _hr; do
 810                 case "${_hr#refs/heads/}" in */*) :;; *)
 811                         _update_head_symref "$_hr"
 812                         exit 1 # exit subshell created by "|"
 813                 esac
 814         done || return 0
 815         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 816         if [ -n "$_hr" ]; then
 817                 _update_head_symref "$_hr"
 818                 return 0
 819         fi
 820         return 1
 821 }
 822 _update_head_symref() {
 823         git symbolic-ref HEAD "$1"
 824         git config --bool girocco.headok true
 825         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 826 }
 827
 828 # current directory must already be set to Git repository
 829 # if the directory needs to have gc run and .needsgc is not already set
 830 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 831 # Girocco shouldn't generate any loose objects but we check for that anyway
 832 check_and_set_needsgc() {
 833         # If there's a .needspack file and ANY loose objects with a newer timestamp
 834         # then also set .needsgc otherwise remove it.  The only caller that may set
 835         # .needspack is a mirror therefore we don't have to worry about removing a
 836         # .needspack out from under a simultaneous creator.  We always do this and
 837         # do it first to try and avoid leaving a stale .needspack lying around.
 838         if [ -e .needspack ]; then
 839                 _objfiles=
 840                 _objfiles="$(( $(find -L objects/$octet -maxdepth 1 -newer .needspack -name "$octet19*" -type f -print 2>/dev/null |
 841                          head -n 1 | LC_ALL=C wc -l) +0 ))"
 842                 if [ "${_objfiles:-0}" = "0" ]; then
 843                         rm -f .needspack
 844                 else
 845                         [ -e .needsgc ] || >.needsgc
 846                 fi
 847         fi
 848         ! [ -e .needsgc ] || return 0
 849         _packs=
 850         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 851         if [ "${_packs:-0}" -ge 20 ]; then
 852                 >.needsgc
 853                 return 0
 854         fi
 855         _logfiles=
 856         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 857         if [ "${_logfiles:-0}" -ge 50 ]; then
 858                 >.needsgc
 859                 return 0
 860         fi
 861         # Truly git gc only checks the number of objects in the objects/17 directory
 862         # We check for -ge 10 which should make the probability of having more than
 863         # 5120 (20*256) loose objects present when there are less than 10 in
 864         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 865         _objfiles=
 866         ! [ -d objects/17 ] ||
 867         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 868         if [ "${_objfiles:-0}" -ge 10 ]; then
 869                 >.needsgc
 870                 return 0
 871         fi
 872 }
 873
 874 # A well-known UTF-8 locale is required for some of the fast-import providers
 875 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 876 # but that is not reliably UTF-8 but rather usually US-ASCII.
 877 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 878 # install time and store that in $var_utf8_locale if one is found.
 879 # If we cannot find one in the `locale -a` output then we just use a well-known
 880 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 881 # it.  We only set this temporarily when running the fast-import providers.
 882 set_utf8_locale() {
 883         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 884         export LC_ALL
 885 }
 886
 887 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 888 git_hg_fetch() (
 889         set_utf8_locale
 890         _python="${PYTHON:-python}"
 891         rm -f hg2git-marks.old hg2git-marks.new
 892         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 893                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 894                         if [ -n "$var_have_git_185" ]; then
 895                                 git cat-file --batch-check=':%(rest) %(objectname)'
 896                         else
 897                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 898                         fi
 899                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 900                 if [ -n "$var_have_git_171" ] &&
 901                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 902                         if [ -z "$var_have_git_185" ] ||
 903                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 904                                 _nm='hg-fast-export'
 905                                 GIT_AUTHOR_NAME="$_nm"
 906                                 GIT_COMMITTER_NAME="$_nm"
 907                                 GIT_AUTHOR_EMAIL="$_nm"
 908                                 GIT_COMMITTER_EMAIL="$_nm"
 909                                 export GIT_AUTHOR_NAME
 910                                 export GIT_COMMITTER_NAME
 911                                 export GIT_AUTHOR_EMAIL
 912                                 export GIT_COMMITTER_EMAIL
 913                                 git notes --ref=refs/notes/hg prune
 914                                 unset GIT_AUTHOR_NAME
 915                                 unset GIT_COMMITTER_NAME
 916                                 unset GIT_AUTHOR_EMAIL
 917                                 unset GIT_COMMITTER_EMAIL
 918                         fi
 919                 fi
 920         else
 921                 >hg2git-marks.old
 922         fi
 923         _err1=
 924         _err2=
 925         exec 3>&1
 926         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 927         $(
 928                 exec 4>&3 3>&1 1>&4 4>&-
 929                 {
 930                         _e1=0
 931                         _af="$(git config hg.authorsfile)" || :
 932                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 933                                 --repo    "$(pwd)/repo.hg" \
 934                                 --marks   "$(pwd)/hg2git-marks.old" \
 935                                 --mapping "$(pwd)/hg2git-mapping" \
 936                                 --heads   "$(pwd)/hg2git-heads" \
 937                                 --status  "$(pwd)/hg2git-state" \
 938                                 -U unknown --force --flatten --hg-hash'
 939                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 940                         eval "$_cmd" 3>&- || _e1=$?
 941                         echo $_e1 >&3
 942                 } |
 943                 {
 944                         _e2=0
 945                         git fast-import \
 946                                 --import-marks="$(pwd)/hg2git-marks.old" \
 947                                 --export-marks="$(pwd)/hg2git-marks.new" \
 948                                 --export-pack-edges="$(pwd)/gfi-packs" \
 949                                 --force 3>&- || _e2=$?
 950                         echo $_e2 >&3
 951                 }
 952         )
 953         EOT
 954         exec 3>&-
 955         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 956         mv -f hg2git-marks.new hg2git-marks
 957         rm -f hg2git-marks.old
 958         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 959         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 960 )