shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  91         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  92         # var_window_memory     Value to use for repack --window-memory=
  93         # var_big_file_threshold Value to use for core.bigFileThreshold
  94         # var_redelta_threshold Recompute deltas if no more than this many objs
  95         # var_upload_window     If not "", pack.window to use for upload-pack
  96         # var_log_window_size   Value to use for git-svn --log-window-size=
  97         # var_utf8_locale       Value to use for a UTF-8 locale if available
  98         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  99         # var_du_exclude        Option to exclude PATTERN from du if available
 100         # var_du_follow         Option to follow command line sym links if available
 101         _cfg_vars="$(get_girocco_config_pm_var_list)"
 102         eval "$_cfg_vars"
 103         printf '%s\n' "$_cfg_vars"
 104         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 105         _gver="$("$cfg_git_bin" version 2>/dev/null |
 106                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 107         printf 'var_git_ver=%s\n' "$_gver"
 108         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 109         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 110         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 111         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 112         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 113         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 114         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 121         __girocco_conf="$GIROCCO_CONF"
 122         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 123         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 124         printf "var_window_memory=%s\n" \
 125                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 126                 -MGirocco::Util -e 'print calc_windowmemory')"
 127         printf "var_big_file_threshold=%s\n" \
 128                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 129                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 130         printf "var_redelta_threshold=%s\n" \
 131                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 133         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 134            [ "$cfg_upload_pack_window" -le 50 ]; then
 135                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 136         else
 137                 printf "var_upload_window=%s\n" ""
 138         fi
 139         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 140         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 141         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 142                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 143                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 144                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 145         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 146         # On some broken platforms running xargs without -r and empty input runs the command
 147         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 148         # The disk usage report produces better numbers if du has an exclude option
 149         _x0="$(basename "$0")"
 150         _x0="${_x0%?}?*"
 151         for _duopt in --exclude -I; do
 152                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 153                         printf 'var_du_exclude=%s\n' "$_duopt"
 154                         break
 155                 fi
 156         done
 157         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 158                 printf 'var_du_follow=%s\n' "-H"
 159                 break
 160         fi
 161 )
 162
 163 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 164 # definitions from it rather than running Perl.
 165 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 166         # Import all the variables from Girocco::Config to the local environment,
 167         eval "$(get_girocco_config_var_list)"
 168 else
 169         # Import the variables from shlib_vars.sh which avoids needlessly
 170         # running another copy of Perl
 171         . "@basedir@/shlib_vars.sh"
 172 fi
 173
 174 # git_add_config "some.var=value"
 175 # every ' in value must be replaced with the 4-character sequence '\'' before
 176 # calling this function or Git will barf.  Will not be effective unless running
 177 # Git version 1.7.3 or later.
 178 git_add_config() {
 179         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 180         export GIT_CONFIG_PARAMETERS
 181 }
 182
 183 # Make sure we have a reproducible environment by using a controlled HOME dir
 184 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 185 HOME="$cfg_chroot/etc/girocco"
 186 TMPDIR="/tmp"
 187 GIT_CONFIG_NOSYSTEM=1
 188 GIT_ATTR_NOSYSTEM=1
 189 GIT_NO_REPLACE_OBJECTS=1
 190 GIT_TERMINAL_PROMPT=0
 191 GIT_PAGER="cat"
 192 PAGER="cat"
 193 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 194 GIT_SVN_NOTTY=1
 195 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 196 export XDG_CONFIG_HOME
 197 export HOME
 198 export TMPDIR
 199 export GIT_CONFIG_NOSYSTEM
 200 export GIT_ATTR_NOSYSTEM
 201 export GIT_NO_REPLACE_OBJECTS
 202 export GIT_TERMINAL_PROMPT
 203 export GIT_PAGER
 204 export PAGER
 205 export GIT_ASKPASS
 206 export GIT_SVN_NOTTY
 207 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 208 unset GIT_USER_AGENT
 209 unset GIT_HTTP_USER_AGENT
 210 if [ -n "$defined_cfg_git_client_ua" ]; then
 211         GIT_USER_AGENT="$cfg_git_client_ua"
 212         export GIT_USER_AGENT
 213         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 214         export GIT_HTTP_USER_AGENT
 215 fi
 216 unset GIT_CONFIG_PARAMETERS
 217 git_add_config "core.ignoreCase=false"
 218 git_add_config "core.pager=cat"
 219 if [ -n "$cfg_git_no_mmap" ]; then
 220         # Just like compiling with NO_MMAP
 221         git_add_config "core.packedGitWindowSize=1m"
 222 else
 223         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 224         git_add_config "core.packedGitWindowSize=32m"
 225 fi
 226 [ -z "$var_big_file_threshold" ] ||
 227         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 228
 229 # Make sure any sendmail.pl config is always available
 230 unset SENDMAIL_PL_HOST
 231 unset SENDMAIL_PL_PORT
 232 unset SENDMAIL_PL_NCBIN
 233 unset SENDMAIL_PL_NCOPT
 234 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 235 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 236 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 237 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 238
 239 # Set PATH and PYTHON to the values set by Config.pm, if any
 240 unset PYTHON
 241 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 242 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 243
 244 # Extra GIT variables that generally ought to be cleared, but whose clearing
 245 # could potentially interfere with the correct operation of hook scripts so
 246 # they are segregated into a separate function for use as appropriate
 247 clean_git_env() {
 248         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 249         unset GIT_CONFIG
 250         unset GIT_DIR
 251         unset GIT_GRAFT_FILE
 252         unset GIT_INDEX_FILE
 253         unset GIT_OBJECT_DIRECTORY
 254         unset GIT_NAMESPACE
 255 }
 256
 257 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 258 # and nc_openbsd to the desired executables because when using
 259 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 260 # different and unexpected ways:
 261 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 262 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 263 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 264 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 265 # setting only while running func which none of the /bin/sh's do.
 266 #
 267 # Instead we'd like to use an alias that provides the desired behavior without
 268 # any of the bad (a), (b) or (c) effects.
 269 #
 270 # However, unfortunately, some of the crazy /bin/sh implementations do not
 271 # recognize alias expansions when preceded by variable assignments!
 272 #
 273 # So we are left with git() {} and nc_openbsd() {} functions and in the
 274 # case of git() {} we can compensate for (b) and (c) failing to export
 275 # but not (a) and (b) persisting the values so the caller will simply
 276 # have to beware and explicitly unset any variables that should not persist
 277 # beyond the function call itself.
 278
 279 git() (
 280         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 281         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 282         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 283         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 284         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 285         exec "$cfg_git_bin" "$@"
 286 )
 287
 288 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 289 # that allows us to use git update-ref --stdin where supported and the slow shell
 290 # script where not, but only the "delete" operation is currently supported.
 291 git_updateref_stdin() {
 292         if [ -n "$var_have_git_185" ]; then
 293                 git update-ref --stdin
 294         else
 295                 while read -r _op _ref; do
 296                         case "$_op" in
 297                                 delete)
 298                                         git update-ref -d "$_ref"
 299                                         ;;
 300                                 *)
 301                                         echo "bad git_updateref_stdin op: $_op" >&2
 302                                         exit 1
 303                                         ;;
 304                         esac
 305                 done
 306         fi
 307 }
 308
 309 # see comments for git() -- callers must explicitly export all variables
 310 # intended for the commands these functions run before calling them
 311 perl() { command "${var_perl_bin:-perl}" "$@"; }
 312 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 313
 314 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 315
 316 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 317
 318 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 319
 320 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 321
 322 # Some platforms' broken xargs runs the command always at least once even if
 323 # there's no input unless given a special option.  Automatically supply the
 324 # option on those platforms by providing an xargs function.
 325 xargs() { command xargs $var_xargs_r "$@"; }
 326
 327 _addrlist() {
 328         _list=
 329         for _addr in "$@"; do
 330                 [ -z "$_list" ] || _list="$_list, "
 331                 _list="$_list$_addr"
 332         done
 333         echo "$_list"
 334 }
 335
 336 _sendmail() {
 337         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 338         if [ -n "$cfg_sender" ]; then
 339                 "$_mailer" -i -f "$cfg_sender" "$@"
 340         else
 341                 "$_mailer" -i "$@"
 342         fi
 343 }
 344
 345 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 346 # "References:" header.  It may be "" to suppress the "References" header.
 347 # Following arguments are just like mail function
 348 mailref() {
 349         _references=
 350         if [ $# -ge 1 ]; then
 351                 _references="$1"
 352                 shift
 353         fi
 354         _subject=
 355         if [ "$1" = "-s" ]; then
 356                 shift
 357                 _subject="$1"
 358                 shift
 359         fi
 360         {
 361                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 362                 echo "To: $(_addrlist "$@")"
 363                 [ -z "$_subject" ] || echo "Subject: $_subject"
 364                 echo "MIME-Version: 1.0"
 365                 echo "Content-Type: text/plain; charset=UTF-8"
 366                 echo "Content-Transfer-Encoding: 8bit"
 367                 [ -z "$_references" ] || echo "References: <$_references>"
 368                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 369                 echo "Auto-Submitted: auto-generated"
 370                 echo ""
 371                 cat
 372         } | _sendmail "$@"
 373 }
 374
 375 # Usage: mail [-s <subject>] <addr> [<addr>...]
 376 mail() {
 377         mailref "" "$@"
 378 }
 379
 380 # bang CMD... will execute the command with well-defined failure mode;
 381 # set bang_action to string of the failed action ('clone', 'update', ...);
 382 # re-define the bang_trap() function to do custom cleanup before bailing out
 383 bang() {
 384         bang_active=1
 385         bang_cmd="$*"
 386         bang_errcode=0
 387         if [ "${show_progress:-0}" != "0" ]; then
 388                 exec 3>&1
 389                 read -r bang_errcode <<-EOT || :
 390                 $(
 391                         exec 4>&3 3>&1 1>&4 4>&-
 392                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 393                 )
 394                 EOT
 395                 exec 3>&-
 396                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 397                         # All right. Cool.
 398                         bang_active=
 399                         bang_cmd=
 400                         return;
 401                 fi
 402         else
 403                 if "$@" >>"$bang_log" 2>&1; then
 404                         # All right. Cool.
 405                         bang_active=
 406                         bang_cmd=
 407                         return;
 408                 else
 409                         bang_errcode="$?"
 410                 fi
 411         fi
 412         bang_failed
 413 }
 414
 415 bang_failed() {
 416         bang_active=
 417         unset GIT_DIR
 418         >.banged
 419         cat "$bang_log" >.banglog
 420         echo "" >>.banglog
 421         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 422         if [ "${show_progress:-0}" != "0" ]; then
 423                 echo ""
 424                 echo "$bang_cmd failed with error code $bang_errcode"
 425         fi
 426         if [ -e .bangagain ]; then
 427                 git config --remove-section girocco.bang 2>/dev/null || :
 428                 rm -f .bangagain
 429         fi
 430         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 431         bangcount=$(( ${bangcount:-0} + 1 ))
 432         git config --int girocco.bang.count $bangcount
 433         if [ $bangcount -eq 1 ]; then
 434                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 435         fi
 436         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 437            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 438            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 439                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 440                 bangaddrs=
 441                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 442                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 443                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 444                 rsubj=
 445                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 446                 [ -z "$bangaddrs" ] ||
 447                 {
 448                         echo "$bang_cmd failed with error code $bang_errcode"
 449                         echo ""
 450                         rsubj=
 451                         if [ $bangcount -gt 1 ]; then
 452                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 453                                 echo ""
 454                         fi
 455                         echo "you will not receive any more notifications until recovery"
 456                         echo "this status message may be disabled on the project admin page"
 457                         echo ""
 458                         echo "Log follows:"
 459                         echo ""
 460                         cat "$bang_log"
 461                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 462                 git config --bool girocco.bang.messagesent true
 463         fi
 464         bangthrottle=
 465         [ $bangcount -lt 15 ] ||
 466                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 467                 bangthrottle=1
 468         bang_trap $bangthrottle
 469         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 470         exit $bang_errcode
 471 }
 472
 473 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 474 # Identical to bang CMD... except the command is eval'd instead of executed.
 475 bang_eval() {
 476         bang eval "$*"
 477 }
 478
 479 # Default bang settings:
 480 bang_setup() {
 481         bang_active=
 482         bang_action="lame_programmer"
 483         bang_trap() { :; }
 484         bang_tmpdir="${TMPDIR:-/tmp}"
 485         bang_tmpdir="${bang_tmpdir%/}"
 486         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 487         is_git_dir . || {
 488                 echo "bang_setup called with current directory not a git directory" >&2
 489                 exit 1
 490         }
 491         trap 'rm -f "$bang_log"' EXIT
 492         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 493         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 494 }
 495
 496 # Remove banged status
 497 bang_reset() {
 498         rm -f .banged .bangagain .banglog
 499         git config --remove-section girocco.bang 2>/dev/null || :
 500 }
 501
 502 # Check to see if banged status
 503 is_banged() {
 504         [ -e .banged ]
 505 }
 506
 507 # Check to see if banged message was sent
 508 was_banged_message_sent() {
 509         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 510 }
 511
 512 # Progress report - if show_progress is set, shows the given message.
 513 progress() {
 514         [ "${show_progress:-0}" = "0" ] || echo "$*"
 515 }
 516
 517 # Project config accessors; must be run in project directory
 518 config_get() {
 519         case "$1" in
 520         *.*)
 521                 git config "$1";;
 522         *)
 523                 git config "gitweb.$1";;
 524         esac
 525 }
 526
 527 config_set() {
 528         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 529 }
 530
 531 config_set_raw() {
 532         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 533 }
 534
 535 config_get_date_seconds() {
 536         _dt="$(config_get "$1")" || :
 537         [ -n "$_dt" ] || return 1
 538         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 539         [ -n "$_ds" ] || return 1
 540         echo "$_ds"
 541 }
 542
 543 # Tool for checking whether given number of seconds has not passed yet
 544 check_interval() {
 545         os="$(config_get_date_seconds "$1")" || return 1
 546         ns="$(date +%s)"
 547         [ $ns -lt $(($os+$2)) ]
 548 }
 549
 550 # Check if we are running with effective root permissions
 551 is_root() {
 552         [ "$(id -u 2>/dev/null)" = "0" ]
 553 }
 554
 555 # Check to see if the single argument is a Git directory
 556 is_git_dir() {
 557         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 558         # And we are slightly more picky (must be refs/.+ not refs/.*)
 559         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 560         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 561         if [ -L "$1/HEAD" ]; then
 562                 _hr="$(readlink "$1/HEAD")"
 563                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 564         fi
 565         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 566         read -r _hr <"$1/HEAD" || return 1
 567         case "$_hr" in
 568                 $octet20*)
 569                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 570                         return 0;;
 571                 ref:refs/?*)
 572                         return 0;;
 573                 ref:*)
 574                         _hr="${_hr##ref:*[ $tab]}"
 575                         case "$_hr" in "refs/"?*) return 0;; esac
 576         esac
 577         return 1
 578 }
 579
 580 # List all Git repositories, with given prefix if specified, one-per-line
 581 # All project names starting with _ are always excluded from the result
 582 get_repo_list() {
 583         if [ -n "$1" ]; then
 584                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 585         else
 586                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 587         fi | while IFS=: read name id; do
 588                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 589         done
 590 }
 591
 592 # Return success if the given project name has at least one immediate child fork
 593 # that has a non-zero length alternates file
 594 has_forks_with_alternates() {
 595         _prj="${1%.git}"
 596         [ -n "$_prj" ] || return 1
 597         [ -d "$cfg_reporoot/$_prj" ] || return 1
 598         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 599         if
 600                 get_repo_list "$_prj/[^/:][^/:]*:" |
 601                 while read -r _prjname && [ -n "$_prjname" ]; do
 602                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 603                         exit 1 # will only exit implicit subshell created by '|'
 604                 done
 605         then
 606                 return 1
 607         fi
 608         return 0
 609 }
 610
 611 # returns empty string and error for empty string otherwise one of
 612 #   m  =>  normal Git mirror
 613 #   s  =>  mirror from svn source
 614 #   d  =>  mirror from darcs source
 615 #   b  =>  mirror from bzr source
 616 #   h  =>  mirror from hg source
 617 #   w  =>  mirror from mediawiki source
 618 #   f  =>  mirror from other fast-import source
 619 # note that if the string is non-empty and none of s, d, b or h match the
 620 # return will always be type m regardless of whether it's a valid Git URL
 621 get_url_mirror_type() {
 622         case "$1" in
 623                 "")
 624                         return 1
 625                         ;;
 626                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 627                         echo 's'
 628                         ;;
 629                 darcs://*)
 630                         echo 'd'
 631                         ;;
 632                 bzr://*)
 633                         echo 'b'
 634                         ;;
 635                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 636                         echo 'h'
 637                         ;;
 638                 mediawiki::*)
 639                         echo 'w'
 640                         ;;
 641                 *)
 642                         echo 'm'
 643                         ;;
 644         esac
 645         return 0
 646 }
 647
 648 # returns false for empty string
 649 # returns true if the passed in url is a mirror using git fast-import
 650 is_gfi_mirror_url() {
 651         [ -n "$1" ] || return 1
 652         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 653                 d|b|h|w|f)
 654                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 655                         # and so do generic "f" fast-import mirrors
 656                         return 0
 657                         ;;
 658                 *)
 659                         # Don't think git-svn currently uses git fast-import
 660                         # And Git mirrors certainly do not
 661                         return 1
 662                         ;;
 663         esac
 664         # assume it does not use git fast-import
 665         return 1
 666 }
 667
 668 # returns false for empty string
 669 # returns true if the passed in url is a mirror using git-svn
 670 is_svn_mirror_url() {
 671         [ -n "$1" ] || return 1
 672         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 673 }
 674
 675 # returns mirror url for gitweb.baseurl of git directory
 676 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 677 # will fail if the directory does not have .nofetch and gitweb.baseurl
 678 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 679 # automatically strips any leading "disabled " prefix before returning result
 680 get_mirror_url() {
 681         _gitdir="${1:-.}"
 682         # always return empty for non-mirrors
 683         ! [ -e "$_gitdir/.nofetch" ] || return 0
 684         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 685         _url="${_url##* }"
 686         [ -n "$_url" ] || return 1
 687         printf '%s\n' "$_url"
 688         return 0
 689 }
 690
 691 # returns get_url_mirror_type for gitweb.baseurl of git directory
 692 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 693 # will fail if the directory does not have .nofetch and gitweb.baseurl
 694 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 695 # automatically strips any leading "disabled " prefix before testing
 696 get_mirror_type() {
 697         _url="$(get_mirror_url "$@")" || return 1
 698         [ -n "$_url" ] || return 0
 699         get_url_mirror_type "$_url"
 700 }
 701
 702 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 703 is_gfi_mirror() {
 704         _url="$(get_mirror_url "$@")" || return 1
 705         is_gfi_mirror_url "$_url"
 706 }
 707
 708 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 709 is_svn_mirror() {
 710         _url="$(get_mirror_url "$@")" || return 1
 711         is_svn_mirror_url "$_url"
 712 }
 713
 714 # current directory must already be set to Git repository
 715 # if girocco.headok is already true succeeds without doing anything
 716 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 717 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 718 # then refs/heads/trunk and finally the first top-level head from
 719 # refs/heads/* (i.e. only two slashes in the name) and finally any
 720 # existing refs/heads.  The first one to succeed wins and sets headok=true
 721 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 722 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 723 # when the repository is being set up -- if the HEAD is later deleted (through
 724 # a push or fetch --prune) that's no longer our responsibility to fix
 725 check_and_set_head() {
 726         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 727         if git rev-parse --verify --quiet HEAD >/dev/null; then
 728                 git config --bool girocco.headok true
 729                 return 0
 730         fi
 731         for _hr in refs/heads/master refs/heads/trunk; do
 732                 if git rev-parse --verify --quiet "$_hr"; then
 733                         _update_head_symref "$_hr"
 734                         return 0
 735                 fi
 736         done
 737         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 738         while read -r _hr; do
 739                 case "${_hr#refs/heads/}" in */*) :;; *)
 740                         _update_head_symref "$_hr"
 741                         exit 1 # exit subshell created by "|"
 742                 esac
 743         done || return 0
 744         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 745         if [ -n "$_hr" ]; then
 746                 _update_head_symref "$_hr"
 747                 return 0
 748         fi
 749         return 1
 750 }
 751 _update_head_symref() {
 752         git symbolic-ref HEAD "$1"
 753         git config --bool girocco.headok true
 754         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 755 }
 756
 757 # current directory must already be set to Git repository
 758 # if the directory needs to have gc run and .needsgc is not already set
 759 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 760 # Girocco shouldn't generate any loose objects but we check for that anyway
 761 check_and_set_needsgc() {
 762         ! [ -e .needsgc ] || return 0
 763         _packs=
 764         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 765         if [ "${_packs:-0}" -ge 20 ]; then
 766                 >.needsgc
 767                 return 0
 768         fi
 769         _logfiles=
 770         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 771         if [ "${_logfiles:-0}" -ge 50 ]; then
 772                 >.needsgc
 773                 return 0
 774         fi
 775         # Truly git gc only checks the number of objects in the objects/17 directory
 776         # We check for -ge 10 which should make the probability of having more than
 777         # 5120 (20*256) loose objects present when there are less than 10 in
 778         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 779         _objfiles=
 780         ! [ -d objects/17 ] ||
 781         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 782         if [ "${_objfiles:-0}" -ge 10 ]; then
 783                 >.needsgc
 784                 return 0
 785         fi
 786 }
 787
 788 # A well-known UTF-8 locale is required for some of the fast-import providers
 789 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 790 # but that is not reliably UTF-8 but rather usually US-ASCII.
 791 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 792 # install time and store that in $var_utf8_locale if one is found.
 793 # If we cannot find one in the `locale -a` output then we just use a well-known
 794 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 795 # it.  We only set this temporarily when running the fast-import providers.
 796 set_utf8_locale() {
 797         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 798         export LC_ALL
 799 }
 800
 801 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 802 git_hg_fetch() (
 803         set_utf8_locale
 804         _python="${PYTHON:-python}"
 805         rm -f hg2git-marks.old hg2git-marks.new
 806         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 807                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 808                         if [ -n "$var_have_git_185" ]; then
 809                                 git cat-file --batch-check=':%(rest) %(objectname)'
 810                         else
 811                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 812                         fi
 813                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 814                 if [ -n "$var_have_git_171" ] &&
 815                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 816                         if [ -z "$var_have_git_185" ] ||
 817                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 818                                 _nm='hg-fast-export'
 819                                 GIT_AUTHOR_NAME="$_nm"
 820                                 GIT_COMMITTER_NAME="$_nm"
 821                                 GIT_AUTHOR_EMAIL="$_nm"
 822                                 GIT_COMMITTER_EMAIL="$_nm"
 823                                 export GIT_AUTHOR_NAME
 824                                 export GIT_COMMITTER_NAME
 825                                 export GIT_AUTHOR_EMAIL
 826                                 export GIT_COMMITTER_EMAIL
 827                                 git notes --ref=refs/notes/hg prune
 828                                 unset GIT_AUTHOR_NAME
 829                                 unset GIT_COMMITTER_NAME
 830                                 unset GIT_AUTHOR_EMAIL
 831                                 unset GIT_COMMITTER_EMAIL
 832                         fi
 833                 fi
 834         else
 835                 >hg2git-marks.old
 836         fi
 837         _err1=
 838         _err2=
 839         exec 3>&1
 840         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 841         $(
 842                 exec 4>&3 3>&1 1>&4 4>&-
 843                 {
 844                         _e1=0
 845                         _af="$(git config hg.authorsfile)" || :
 846                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 847                                 --repo    "$(pwd)/repo.hg" \
 848                                 --marks   "$(pwd)/hg2git-marks.old" \
 849                                 --mapping "$(pwd)/hg2git-mapping" \
 850                                 --heads   "$(pwd)/hg2git-heads" \
 851                                 --status  "$(pwd)/hg2git-state" \
 852                                 -U unknown --force --flatten --hg-hash'
 853                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 854                         eval "$_cmd" 3>&- || _e1=$?
 855                         echo $_e1 >&3
 856                 } |
 857                 {
 858                         _e2=0
 859                         git fast-import \
 860                                 --import-marks="$(pwd)/hg2git-marks.old" \
 861                                 --export-marks="$(pwd)/hg2git-marks.new" \
 862                                 --export-pack-edges="$(pwd)/gfi-packs" \
 863                                 --force 3>&- || _e2=$?
 864                         echo $_e2 >&3
 865                 }
 866         )
 867         EOT
 868         exec 3>&-
 869         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 870         mv -f hg2git-marks.new hg2git-marks
 871         rm -f hg2git-marks.old
 872         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 873         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 874 )