shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         inc_basedir=@basedir@
  54         [ "@basedir@" != '@'basedir'@' ] || inc_basedir="$PWD"
  55         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  56         perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf -le \
  57                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  58                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  59                         $val =~ s/([\\"\$\`])/\\$1/gos;
  60                         $val =~ s/(?:\r\n|\r|\n)$//os;
  61                         print "cfg_$_=\"$val\"";
  62                         print "defined_cfg_$_=",
  63                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  64                 }'
  65 )
  66
  67 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  68 _fcp() {
  69         if _fp="$(command -v "$1" 2>/dev/null)"; then
  70                 printf '%s\n' "$_fp"
  71         else
  72                 printf '%s\n' "$1"
  73         fi
  74 }
  75
  76 get_girocco_config_var_list() (
  77         # Same as get_girocco_config_pm_var_list except that
  78         # the following variables (all starting with var_) are added:
  79         # var_group             cfg_owning_group if defined otherwise `id -gn`
  80         # var_git_ver           The version number part from `git version`
  81         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  82         # var_sh_bin            Full path to the posix sh interpreter to use
  83         # var_perl_bin          Full path to the perl interpreter to use
  84         # var_gzip_bin          Full path to the gzip executable to use
  85         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  86         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  87         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  88         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  89         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  90         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  91         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  92         # var_have_git_235      Set to 1 if git version >= 2.3.5 otherwise ''
  93         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  94         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  95         # var_window_memory     Value to use for repack --window-memory=
  96         # var_big_file_threshold Value to use for core.bigFileThreshold
  97         # var_redelta_threshold Recompute deltas if no more than this many objs
  98         # var_upload_window     If not "", pack.window to use for upload-pack
  99         # var_log_window_size   Value to use for git-svn --log-window-size=
 100         # var_utf8_locale       Value to use for a UTF-8 locale if available
 101         # var_xargs_r           A "-r" if xargs needs it to behave correctly
 102         # var_du_exclude        Option to exclude PATTERN from du if available
 103         # var_du_follow         Option to follow command line sym links if available
 104         _cfg_vars="$(get_girocco_config_pm_var_list)"
 105         eval "$_cfg_vars"
 106         printf '%s\n' "$_cfg_vars"
 107         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 108         _gver="$("$cfg_git_bin" version 2>/dev/null |
 109                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 110         printf 'var_git_ver=%s\n' "$_gver"
 111         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 112         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 113         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 114         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 115         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 116         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 121         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 122         printf 'var_have_git_235=%s\n' "$([ $(vcmp "$_gver" 2.3.5) -ge 0 ] && echo 1)"
 123         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 124         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 125         __girocco_conf="$GIROCCO_CONF"
 126         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 127         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 128         inc_basedir=@basedir@
 129         [ "@basedir@" != '@'basedir'@' ] || inc_basedir="$PWD"
 130         printf "var_window_memory=%s\n" \
 131                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_windowmemory')"
 133         printf "var_big_file_threshold=%s\n" \
 134                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 135                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 136         printf "var_redelta_threshold=%s\n" \
 137                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 138                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 139         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 140            [ "$cfg_upload_pack_window" -le 50 ]; then
 141                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 142         else
 143                 printf "var_upload_window=%s\n" ""
 144         fi
 145         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 146         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 147         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 148                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 149                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 150                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 151         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 152         # On some broken platforms running xargs without -r and empty input runs the command
 153         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 154         # The disk usage report produces better numbers if du has an exclude option
 155         _x0="$(basename "$0")"
 156         _x0="${_x0%?}?*"
 157         for _duopt in --exclude -I; do
 158                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 159                         printf 'var_du_exclude=%s\n' "$_duopt"
 160                         break
 161                 fi
 162         done
 163         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 164                 printf 'var_du_follow=%s\n' "-H"
 165                 break
 166         fi
 167 )
 168
 169 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 170 # definitions from it rather than running Perl.
 171 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 172         # Import all the variables from Girocco::Config to the local environment,
 173         eval "$(get_girocco_config_var_list)"
 174 else
 175         # Import the variables from shlib_vars.sh which avoids needlessly
 176         # running another copy of Perl
 177         . "@basedir@/shlib_vars.sh"
 178 fi
 179
 180 # git_add_config "some.var=value"
 181 # every ' in value must be replaced with the 4-character sequence '\'' before
 182 # calling this function or Git will barf.  Will not be effective unless running
 183 # Git version 1.7.3 or later.
 184 git_add_config() {
 185         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 186         export GIT_CONFIG_PARAMETERS
 187 }
 188
 189 # Make sure we have a reproducible environment by using a controlled HOME dir
 190 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 191 HOME="$cfg_chroot/etc/girocco"
 192 TMPDIR="/tmp"
 193 GIT_CONFIG_NOSYSTEM=1
 194 GIT_ATTR_NOSYSTEM=1
 195 GIT_NO_REPLACE_OBJECTS=1
 196 GIT_TERMINAL_PROMPT=0
 197 GIT_PAGER="cat"
 198 PAGER="cat"
 199 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 200 GIT_SVN_NOTTY=1
 201 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 202 GIT_SSH="$cfg_basedir/bin/git-ssh"
 203 SVN_SSH="$cfg_basedir/bin/git-ssh"
 204 export XDG_CONFIG_HOME
 205 export HOME
 206 export TMPDIR
 207 export GIT_CONFIG_NOSYSTEM
 208 export GIT_ATTR_NOSYSTEM
 209 export GIT_NO_REPLACE_OBJECTS
 210 export GIT_TERMINAL_PROMPT
 211 export GIT_PAGER
 212 export PAGER
 213 export GIT_ASKPASS
 214 export GIT_SVN_NOTTY
 215 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 216 export GIT_SSH
 217 export SVN_SSH
 218 unset GIT_USER_AGENT
 219 unset GIT_HTTP_USER_AGENT
 220 if [ -n "$defined_cfg_git_client_ua" ]; then
 221         GIT_USER_AGENT="$cfg_git_client_ua"
 222         export GIT_USER_AGENT
 223 fi
 224 unset GIT_CONFIG_PARAMETERS
 225 git_add_config "core.ignoreCase=false"
 226 git_add_config "core.pager=cat"
 227 if [ -n "$cfg_git_no_mmap" ]; then
 228         # Just like compiling with NO_MMAP
 229         git_add_config "core.packedGitWindowSize=1m"
 230 else
 231         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 232         git_add_config "core.packedGitWindowSize=32m"
 233 fi
 234 [ -z "$var_big_file_threshold" ] ||
 235         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 236 git_add_config "gc.auto=0"
 237
 238 # Make sure any sendmail.pl config is always available
 239 unset SENDMAIL_PL_HOST
 240 unset SENDMAIL_PL_PORT
 241 unset SENDMAIL_PL_NCBIN
 242 unset SENDMAIL_PL_NCOPT
 243 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 244 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 245 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 246 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 247
 248 # Set PATH and PYTHON to the values set by Config.pm, if any
 249 unset PYTHON
 250 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 251 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 252
 253 # Extra GIT variables that generally ought to be cleared, but whose clearing
 254 # could potentially interfere with the correct operation of hook scripts so
 255 # they are segregated into a separate function for use as appropriate
 256 clean_git_env() {
 257         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 258         unset GIT_CONFIG
 259         unset GIT_DIR
 260         unset GIT_GRAFT_FILE
 261         unset GIT_INDEX_FILE
 262         unset GIT_OBJECT_DIRECTORY
 263         unset GIT_NAMESPACE
 264 }
 265
 266 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 267 # and nc_openbsd to the desired executables because when using
 268 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 269 # different and unexpected ways:
 270 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 271 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 272 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 273 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 274 # setting only while running func which none of the /bin/sh's do.
 275 #
 276 # Instead we'd like to use an alias that provides the desired behavior without
 277 # any of the bad (a), (b) or (c) effects.
 278 #
 279 # However, unfortunately, some of the crazy /bin/sh implementations do not
 280 # recognize alias expansions when preceded by variable assignments!
 281 #
 282 # So we are left with git() {} and nc_openbsd() {} functions and in the
 283 # case of git() {} we can compensate for (b) and (c) failing to export
 284 # but not (a) and (b) persisting the values so the caller will simply
 285 # have to beware and explicitly unset any variables that should not persist
 286 # beyond the function call itself.
 287
 288 git() (
 289         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 290         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 291         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 292         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 293         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 294         exec "$cfg_git_bin" "$@"
 295 )
 296
 297 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 298 # that allows us to use git update-ref --stdin where supported and the slow shell
 299 # script where not, but only the "delete" operation is currently supported.
 300 git_updateref_stdin() {
 301         if [ -n "$var_have_git_185" ]; then
 302                 git update-ref --stdin
 303         else
 304                 while read -r _op _ref; do
 305                         case "$_op" in
 306                                 delete)
 307                                         git update-ref -d "$_ref"
 308                                         ;;
 309                                 *)
 310                                         echo "bad git_updateref_stdin op: $_op" >&2
 311                                         exit 1
 312                                         ;;
 313                         esac
 314                 done
 315         fi
 316 }
 317
 318 # see comments for git() -- callers must explicitly export all variables
 319 # intended for the commands these functions run before calling them
 320 perl() { command "${var_perl_bin:-perl}" "$@"; }
 321 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 322
 323 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 324
 325 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 326
 327 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 328
 329 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 330
 331 # Some platforms' broken xargs runs the command always at least once even if
 332 # there's no input unless given a special option.  Automatically supply the
 333 # option on those platforms by providing an xargs function.
 334 xargs() { command xargs $var_xargs_r "$@"; }
 335
 336 _addrlist() {
 337         _list=
 338         for _addr in "$@"; do
 339                 [ -z "$_list" ] || _list="$_list, "
 340                 _list="$_list$_addr"
 341         done
 342         echo "$_list"
 343 }
 344
 345 _sendmail() {
 346         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 347         if [ -n "$cfg_sender" ]; then
 348                 "$_mailer" -i -f "$cfg_sender" "$@"
 349         else
 350                 "$_mailer" -i "$@"
 351         fi
 352 }
 353
 354 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 355 # "References:" header.  It may be "" to suppress the "References" header.
 356 # Following arguments are just like mail function
 357 mailref() {
 358         _references=
 359         if [ $# -ge 1 ]; then
 360                 _references="$1"
 361                 shift
 362         fi
 363         _subject=
 364         if [ "$1" = "-s" ]; then
 365                 shift
 366                 _subject="$1"
 367                 shift
 368         fi
 369         {
 370                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 371                 echo "To: $(_addrlist "$@")"
 372                 [ -z "$_subject" ] || echo "Subject: $_subject"
 373                 echo "MIME-Version: 1.0"
 374                 echo "Content-Type: text/plain; charset=UTF-8"
 375                 echo "Content-Transfer-Encoding: 8bit"
 376                 [ -z "$_references" ] || echo "References: <$_references>"
 377                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 378                 echo "Auto-Submitted: auto-generated"
 379                 echo ""
 380                 cat
 381         } | _sendmail "$@"
 382 }
 383
 384 # Usage: mail [-s <subject>] <addr> [<addr>...]
 385 mail() {
 386         mailref "" "$@"
 387 }
 388
 389 # bang CMD... will execute the command with well-defined failure mode;
 390 # set bang_action to string of the failed action ('clone', 'update', ...);
 391 # re-define the bang_trap() function to do custom cleanup before bailing out
 392 bang() {
 393         bang_errcode=
 394         bang_catch "$@"
 395         [ "${bang_errcode:-0}" = "0" ] || bang_failed
 396 }
 397
 398 bang_catch() {
 399         bang_active=1
 400         bang_cmd="$*"
 401         bang_errcode=0
 402         if [ "${show_progress:-0}" != "0" ]; then
 403                 exec 3>&1
 404                 read -r bang_errcode <<-EOT || :
 405                 $(
 406                         exec 4>&3 3>&1 1>&4 4>&-
 407                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 408                 )
 409                 EOT
 410                 exec 3>&-
 411                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 412                         # All right. Cool.
 413                         bang_active=
 414                         bang_cmd=
 415                         return;
 416                 fi
 417         else
 418                 if "$@" >>"$bang_log" 2>&1; then
 419                         # All right. Cool.
 420                         bang_active=
 421                         bang_cmd=
 422                         return;
 423                 else
 424                         bang_errcode="$?"
 425                 fi
 426         fi
 427 }
 428
 429 bang_failed() {
 430         bang_active=
 431         unset GIT_DIR
 432         >.banged
 433         cat "$bang_log" >.banglog
 434         echo "" >>.banglog
 435         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 436         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 437         if [ "${show_progress:-0}" != "0" ]; then
 438                 echo ""
 439                 echo "$bang_cmd failed with error code $bang_errcode"
 440         fi
 441         if [ -e .bangagain ]; then
 442                 git config --remove-section girocco.bang 2>/dev/null || :
 443                 rm -f .bangagain
 444         fi
 445         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 446         bangcount=$(( ${bangcount:-0} + 1 ))
 447         git config --int girocco.bang.count $bangcount
 448         if [ $bangcount -eq 1 ]; then
 449                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 450         fi
 451         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 452            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 453            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 454                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 455                 bangaddrs=
 456                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 457                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 458                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 459                 rsubj=
 460                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 461                 [ -z "$bangaddrs" ] ||
 462                 {
 463                         echo "$bang_cmd failed with error code $bang_errcode"
 464                         echo ""
 465                         rsubj=
 466                         if [ $bangcount -gt 1 ]; then
 467                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 468                                 echo ""
 469                         fi
 470                         echo "you will not receive any more notifications until recovery"
 471                         echo "this status message may be disabled on the project admin page"
 472                         echo ""
 473                         echo "Log follows:"
 474                         echo ""
 475                         cat "$bang_log"
 476                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 477                 git config --bool girocco.bang.messagesent true
 478         fi
 479         bangthrottle=
 480         [ $bangcount -lt 15 ] ||
 481                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 482                 bangthrottle=1
 483         bang_trap $bangthrottle
 484         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 485         exit $bang_errcode
 486 }
 487
 488 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 489 # Identical to bang CMD... except the command is eval'd instead of executed.
 490 bang_eval() {
 491         bang eval "$*"
 492 }
 493
 494 # Default bang settings:
 495 bang_setup() {
 496         bang_active=
 497         bang_action="lame_programmer"
 498         bang_trap() { :; }
 499         bang_tmpdir="${TMPDIR:-/tmp}"
 500         bang_tmpdir="${bang_tmpdir%/}"
 501         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 502         is_git_dir . || {
 503                 echo "bang_setup called with current directory not a git directory" >&2
 504                 exit 1
 505         }
 506         trap 'rm -f "$bang_log"' EXIT
 507         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 508         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 509 }
 510
 511 # Remove banged status
 512 bang_reset() {
 513         rm -f .banged .bangagain .banglog
 514         git config --remove-section girocco.bang 2>/dev/null || :
 515 }
 516
 517 # Check to see if banged status
 518 is_banged() {
 519         [ -e .banged ]
 520 }
 521
 522 # Check to see if banged message was sent
 523 was_banged_message_sent() {
 524         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 525 }
 526
 527 # Progress report - if show_progress is set, shows the given message.
 528 progress() {
 529         [ "${show_progress:-0}" = "0" ] || echo "$*"
 530 }
 531
 532 # Project config accessors; must be run in project directory
 533 config_get() {
 534         case "$1" in
 535         *.*)
 536                 git config "$1";;
 537         *)
 538                 git config "gitweb.$1";;
 539         esac
 540 }
 541
 542 config_set() {
 543         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 544 }
 545
 546 config_set_raw() {
 547         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 548 }
 549
 550 config_get_date_seconds() {
 551         _dt="$(config_get "$1")" || :
 552         [ -n "$_dt" ] || return 1
 553         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 554         [ -n "$_ds" ] || return 1
 555         echo "$_ds"
 556 }
 557
 558 # Tool for checking whether given number of seconds has not passed yet
 559 check_interval() {
 560         os="$(config_get_date_seconds "$1")" || return 1
 561         ns="$(date +%s)"
 562         [ $ns -lt $(($os+$2)) ]
 563 }
 564
 565 # Check if we are running with effective root permissions
 566 is_root() {
 567         [ "$(id -u 2>/dev/null)" = "0" ]
 568 }
 569
 570 # Check to see if the single argument (default ".") is a Git directory
 571 is_git_dir() {
 572         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 573         # And we are slightly more picky (must be refs/.+ not refs/.*)
 574         [ $# -ne 0 ] || set -- "."
 575         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 576         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 577         if [ -L "$1/HEAD" ]; then
 578                 _hr="$(readlink "$1/HEAD")"
 579                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 580         fi
 581         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 582         read -r _hr <"$1/HEAD" || return 1
 583         case "$_hr" in
 584                 $octet20*)
 585                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 586                         return 0;;
 587                 ref:refs/?*)
 588                         return 0;;
 589                 ref:*)
 590                         _hr="${_hr##ref:*[ $tab]}"
 591                         case "$_hr" in "refs/"?*) return 0;; esac
 592         esac
 593         return 1
 594 }
 595
 596 # Check to see if the single argument (default ".") is a directory with no refs
 597 is_empty_refs_dir() {
 598         [ $# -ne 0 ] || set -- "."
 599         if [ -s "$1/packed-refs" ]; then
 600                 # could be a packed-refs file with just a '# pack-refs ..." line
 601                 # null hash lines and peel lines do not count either
 602                 _refcnt="$(( $(LC_ALL=C sed <"$1/packed-refs" \
 603                         -e "/^00* /d" \
 604                         -e "/^$octet20$hexdig* refs\/[^ $tab]*\$/!d" | wc -l) ))"
 605                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 606         fi
 607         if [ -d "$1/refs" ]; then
 608                 # quick and dirty check, doesn't try to validate contents
 609                 # or ignore embedded symbolic refs
 610                 _refcnt="$(( $(find -L "$1/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) ))"
 611                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 612         fi
 613         # last chance a detached HEAD (we ignore any linked working trees though)
 614         [ -s "$1/HEAD" ] && read -r _hr <"$1/HEAD" && [ -n "$_hr" ] || return 0
 615         [ "${_hr#*[!0-9a-f]}" != "$_hr" ] || [ "${_hr#*[!0]}" = "$_hr" ] || [ "${#_hr}" -lt 40 ] || return 1
 616         return 0
 617 }
 618
 619 # List all Git repositories, with given prefix if specified, one-per-line
 620 # All project names starting with _ are always excluded from the result
 621 get_repo_list() {
 622         if [ -n "$1" ]; then
 623                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 624         else
 625                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 626         fi | while IFS=: read name id; do
 627                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 628         done
 629 }
 630
 631 # set the variable named by the first argument to the project part (i.e. WITH
 632 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 633 # specified by the second argument.
 634 # This function cannot be fooled by symbolic links.
 635 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 636 # The directory specified by the second argument must exist.
 637 v_get_proj_from_dir() {
 638         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 639         [ -d "$2" ] || return 1
 640         case "$2" in
 641         "$cfg_reporoot/"?*)
 642                 # Simple case that does not need any fancy footwork
 643                 _projpart="${2#$cfg_reporoot/}"
 644                 ;;
 645         *)
 646                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 647                 _abspd="$(cd "$2" && pwd -P)"
 648                 case "$_abspd" in
 649                 "$_absrr/"?*)
 650                         # The normal case
 651                         _projpart="${_abspd#$_absrr/}"
 652                         ;;
 653                 *)
 654                         # Must have been reached via a symbolic link, but
 655                         # we have no way to know the source, so use a
 656                         # generic "_external" leader combined with just the
 657                         # trailing directory name
 658                         _abspd="${_abspd%/}"
 659                         _abspd="${_abspd%/.git}"
 660                         _projpart="_external/${_abspd##*/}"
 661                         ;;
 662                 esac
 663         esac
 664         eval "$1="'"$_projpart"'
 665 }
 666
 667 # Return success if the given project name has at least one immediate child fork
 668 # that has a non-zero length alternates file
 669 has_forks_with_alternates() {
 670         _prj="${1%.git}"
 671         [ -n "$_prj" ] || return 1
 672         [ -d "$cfg_reporoot/$_prj" ] || return 1
 673         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 674         if
 675                 get_repo_list "$_prj/[^/:][^/:]*:" |
 676                 while read -r _prjname && [ -n "$_prjname" ]; do
 677                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 678                         exit 1 # will only exit implicit subshell created by '|'
 679                 done
 680         then
 681                 return 1
 682         fi
 683         return 0
 684 }
 685
 686 # returns empty string and error for empty string otherwise one of
 687 #   m  =>  normal Git mirror
 688 #   s  =>  mirror from svn source
 689 #   d  =>  mirror from darcs source
 690 #   b  =>  mirror from bzr source
 691 #   h  =>  mirror from hg source
 692 #   w  =>  mirror from mediawiki source
 693 #   f  =>  mirror from other fast-import source
 694 # note that if the string is non-empty and none of s, d, b or h match the
 695 # return will always be type m regardless of whether it's a valid Git URL
 696 get_url_mirror_type() {
 697         case "$1" in
 698                 "")
 699                         return 1
 700                         ;;
 701                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 702                         echo 's'
 703                         ;;
 704                 darcs://*)
 705                         echo 'd'
 706                         ;;
 707                 bzr://*)
 708                         echo 'b'
 709                         ;;
 710                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 711                         echo 'h'
 712                         ;;
 713                 mediawiki::*)
 714                         echo 'w'
 715                         ;;
 716                 *)
 717                         echo 'm'
 718                         ;;
 719         esac
 720         return 0
 721 }
 722
 723 # returns false for empty string
 724 # returns true if the passed in url is a mirror using git fast-import
 725 is_gfi_mirror_url() {
 726         [ -n "$1" ] || return 1
 727         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 728                 d|b|h|w|f)
 729                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 730                         # and so do generic "f" fast-import mirrors
 731                         return 0
 732                         ;;
 733                 *)
 734                         # Don't think git-svn currently uses git fast-import
 735                         # And Git mirrors certainly do not
 736                         return 1
 737                         ;;
 738         esac
 739         # assume it does not use git fast-import
 740         return 1
 741 }
 742
 743 # returns false for empty string
 744 # returns true if the passed in url is a mirror using git-svn
 745 is_svn_mirror_url() {
 746         [ -n "$1" ] || return 1
 747         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 748 }
 749
 750 # returns mirror url for gitweb.baseurl of git directory
 751 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 752 # will fail if the directory does not have .nofetch and gitweb.baseurl
 753 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 754 # automatically strips any leading "disabled " prefix before returning result
 755 get_mirror_url() {
 756         _gitdir="${1:-.}"
 757         # always return empty for non-mirrors
 758         ! [ -e "$_gitdir/.nofetch" ] || return 0
 759         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 760         _url="${_url##* }"
 761         [ -n "$_url" ] || return 1
 762         printf '%s\n' "$_url"
 763         return 0
 764 }
 765
 766 # returns get_url_mirror_type for gitweb.baseurl of git directory
 767 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 768 # will fail if the directory does not have .nofetch and gitweb.baseurl
 769 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 770 # automatically strips any leading "disabled " prefix before testing
 771 get_mirror_type() {
 772         _url="$(get_mirror_url "$@")" || return 1
 773         [ -n "$_url" ] || return 0
 774         get_url_mirror_type "$_url"
 775 }
 776
 777 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 778 is_gfi_mirror() {
 779         _url="$(get_mirror_url "$@")" || return 1
 780         is_gfi_mirror_url "$_url"
 781 }
 782
 783 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 784 is_svn_mirror() {
 785         _url="$(get_mirror_url "$@")" || return 1
 786         is_svn_mirror_url "$_url"
 787 }
 788
 789 # current directory must already be set to Git repository
 790 # if girocco.headok is already true succeeds without doing anything
 791 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 792 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 793 # then refs/heads/trunk and finally the first top-level head from
 794 # refs/heads/* (i.e. only two slashes in the name) and finally any
 795 # existing refs/heads.  The first one to succeed wins and sets headok=true
 796 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 797 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 798 # when the repository is being set up -- if the HEAD is later deleted (through
 799 # a push or fetch --prune) that's no longer our responsibility to fix
 800 check_and_set_head() {
 801         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 802         if git rev-parse --verify --quiet HEAD >/dev/null; then
 803                 git config --bool girocco.headok true
 804                 return 0
 805         fi
 806         for _hr in refs/heads/master refs/heads/trunk; do
 807                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 808                         _update_head_symref "$_hr"
 809                         return 0
 810                 fi
 811         done
 812         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 813         while read -r _hr; do
 814                 case "${_hr#refs/heads/}" in */*) :;; *)
 815                         _update_head_symref "$_hr"
 816                         exit 1 # exit subshell created by "|"
 817                 esac
 818         done || return 0
 819         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 820         if [ -n "$_hr" ]; then
 821                 _update_head_symref "$_hr"
 822                 return 0
 823         fi
 824         return 1
 825 }
 826 _update_head_symref() {
 827         git symbolic-ref HEAD "$1"
 828         git config --bool girocco.headok true
 829         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 830 }
 831
 832 # current directory must already be set to Git repository
 833 # if the directory needs to have gc run and .needsgc is not already set
 834 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 835 # Girocco shouldn't generate any loose objects but we check for that anyway
 836 check_and_set_needsgc() {
 837         # If there's a .needspack file and ANY loose objects with a newer timestamp
 838         # then also set .needsgc otherwise remove it.  The only caller that may set
 839         # .needspack is a mirror therefore we don't have to worry about removing a
 840         # .needspack out from under a simultaneous creator.  We always do this and
 841         # do it first to try and avoid leaving a stale .needspack lying around.
 842         if [ -e .needspack ]; then
 843                 _objfiles=
 844                 _objfiles="$(( $(find -L objects/$octet -maxdepth 1 -newer .needspack -name "$octet19*" -type f -print 2>/dev/null |
 845                          head -n 1 | LC_ALL=C wc -l) +0 ))"
 846                 if [ "${_objfiles:-0}" = "0" ]; then
 847                         rm -f .needspack
 848                 else
 849                         [ -e .needsgc ] || >.needsgc
 850                 fi
 851         fi
 852         ! [ -e .needsgc ] || return 0
 853         _packs=
 854         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 855         if [ "${_packs:-0}" -ge 20 ]; then
 856                 >.needsgc
 857                 return 0
 858         fi
 859         _logfiles=
 860         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 861         if [ "${_logfiles:-0}" -ge 50 ]; then
 862                 >.needsgc
 863                 return 0
 864         fi
 865         # Truly git gc only checks the number of objects in the objects/17 directory
 866         # We check for -ge 10 which should make the probability of having more than
 867         # 5120 (20*256) loose objects present when there are less than 10 in
 868         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 869         _objfiles=
 870         ! [ -d objects/17 ] ||
 871         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 872         if [ "${_objfiles:-0}" -ge 10 ]; then
 873                 >.needsgc
 874                 return 0
 875         fi
 876 }
 877
 878 # A well-known UTF-8 locale is required for some of the fast-import providers
 879 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 880 # but that is not reliably UTF-8 but rather usually US-ASCII.
 881 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 882 # install time and store that in $var_utf8_locale if one is found.
 883 # If we cannot find one in the `locale -a` output then we just use a well-known
 884 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 885 # it.  We only set this temporarily when running the fast-import providers.
 886 set_utf8_locale() {
 887         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 888         export LC_ALL
 889 }
 890
 891 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 892 git_hg_fetch() (
 893         set_utf8_locale
 894         _python="${PYTHON:-python}"
 895         rm -f hg2git-marks.old hg2git-marks.new
 896         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 897                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 898                         if [ -n "$var_have_git_185" ]; then
 899                                 git cat-file --batch-check=':%(rest) %(objectname)'
 900                         else
 901                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 902                         fi
 903                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 904                 if [ -n "$var_have_git_171" ] &&
 905                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 906                         if [ -z "$var_have_git_185" ] ||
 907                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 908                                 _nm='hg-fast-export'
 909                                 GIT_AUTHOR_NAME="$_nm"
 910                                 GIT_COMMITTER_NAME="$_nm"
 911                                 GIT_AUTHOR_EMAIL="$_nm"
 912                                 GIT_COMMITTER_EMAIL="$_nm"
 913                                 export GIT_AUTHOR_NAME
 914                                 export GIT_COMMITTER_NAME
 915                                 export GIT_AUTHOR_EMAIL
 916                                 export GIT_COMMITTER_EMAIL
 917                                 git notes --ref=refs/notes/hg prune
 918                                 unset GIT_AUTHOR_NAME
 919                                 unset GIT_COMMITTER_NAME
 920                                 unset GIT_AUTHOR_EMAIL
 921                                 unset GIT_COMMITTER_EMAIL
 922                         fi
 923                 fi
 924         else
 925                 >hg2git-marks.old
 926         fi
 927         _err1=
 928         _err2=
 929         exec 3>&1
 930         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 931         $(
 932                 exec 4>&3 3>&1 1>&4 4>&-
 933                 {
 934                         _e1=0
 935                         _af="$(git config hg.authorsfile)" || :
 936                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 937                                 --repo    "$(pwd)/repo.hg" \
 938                                 --marks   "$(pwd)/hg2git-marks.old" \
 939                                 --mapping "$(pwd)/hg2git-mapping" \
 940                                 --heads   "$(pwd)/hg2git-heads" \
 941                                 --status  "$(pwd)/hg2git-state" \
 942                                 -U unknown --force --flatten --hg-hash'
 943                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 944                         eval "$_cmd" 3>&- || _e1=$?
 945                         echo $_e1 >&3
 946                 } |
 947                 {
 948                         _e2=0
 949                         git fast-import \
 950                                 --import-marks="$(pwd)/hg2git-marks.old" \
 951                                 --export-marks="$(pwd)/hg2git-marks.new" \
 952                                 --export-pack-edges="$(pwd)/gfi-packs" \
 953                                 --force 3>&- || _e2=$?
 954                         echo $_e2 >&3
 955                 }
 956         )
 957         EOT
 958         exec 3>&-
 959         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 960         mv -f hg2git-marks.new hg2git-marks
 961         rm -f hg2git-marks.old
 962         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 963         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 964 )