shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # SHA-1 patterns
   7 octet='[0-9a-f][0-9a-f]'
   8 octet4="$octet$octet$octet$octet"
   9 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  10 octet20="$octet4$octet4$octet4$octet4$octet4"
  11 nullsha="0000000000000000000000000000000000000000"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  91         # var_window_memory     Value to use for repack --window-memory=
  92         # var_big_file_threshold Value to use for core.bigFileThreshold
  93         # var_redelta_threshold Recompute deltas if no more than this many objs
  94         # var_upload_window     If not "", pack.window to use for upload-pack
  95         # var_log_window_size   Value to use for git-svn --log-window-size=
  96         # var_utf8_locale       Value to use for a UTF-8 locale if available
  97         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  98         # var_du_exclude        Option to exclude PATTERN from du if available
  99         # var_du_follow         Option to follow command line sym links if available
 100         _cfg_vars="$(get_girocco_config_pm_var_list)"
 101         eval "$_cfg_vars"
 102         printf '%s\n' "$_cfg_vars"
 103         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 104         _gver="$("$cfg_git_bin" version 2>/dev/null |
 105                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 106         printf 'var_git_ver=%s\n' "$_gver"
 107         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 108         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 109         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 110         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 111         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 112         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 113         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 114         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 119         __girocco_conf="$GIROCCO_CONF"
 120         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 121         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 122         printf "var_window_memory=%s\n" \
 123                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 124                 -MGirocco::Util -e 'print calc_windowmemory')"
 125         printf "var_big_file_threshold=%s\n" \
 126                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 127                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 128         printf "var_redelta_threshold=%s\n" \
 129                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 130                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 131         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 132            [ "$cfg_upload_pack_window" -le 50 ]; then
 133                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 134         else
 135                 printf "var_upload_window=%s\n" ""
 136         fi
 137         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 138         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 139         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 140                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 141                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 142                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 143         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 144         # On some broken platforms running xargs without -r and empty input runs the command
 145         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 146         # The disk usage report produces better numbers if du has an exclude option
 147         _x0="$(basename "$0")"
 148         _x0="${_x0%?}?*"
 149         for _duopt in --exclude -I; do
 150                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 151                         printf 'var_du_exclude=%s\n' "$_duopt"
 152                         break
 153                 fi
 154         done
 155         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 156                 printf 'var_du_follow=%s\n' "-H"
 157                 break
 158         fi
 159 )
 160
 161 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 162 # definitions from it rather than running Perl.
 163 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 164         # Import all the variables from Girocco::Config to the local environment,
 165         eval "$(get_girocco_config_var_list)"
 166 else
 167         # Import the variables from shlib_vars.sh which avoids needlessly
 168         # running another copy of Perl
 169         . "@basedir@/shlib_vars.sh"
 170 fi
 171
 172 # git_add_config "some.var=value"
 173 # every ' in value must be replaced with the 4-character sequence '\'' before
 174 # calling this function or Git will barf.  Will not be effective unless running
 175 # Git version 1.7.3 or later.
 176 git_add_config() {
 177         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 178         export GIT_CONFIG_PARAMETERS
 179 }
 180
 181 # Make sure we have a reproducible environment by using a controlled HOME dir
 182 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 183 HOME="$cfg_chroot/etc/girocco"
 184 TMPDIR="/tmp"
 185 GIT_CONFIG_NOSYSTEM=1
 186 GIT_ATTR_NOSYSTEM=1
 187 GIT_NO_REPLACE_OBJECTS=1
 188 GIT_TERMINAL_PROMPT=0
 189 GIT_PAGER="cat"
 190 PAGER="cat"
 191 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 192 GIT_SVN_NOTTY=1
 193 export XDG_CONFIG_HOME
 194 export HOME
 195 export TMPDIR
 196 export GIT_CONFIG_NOSYSTEM
 197 export GIT_ATTR_NOSYSTEM
 198 export GIT_NO_REPLACE_OBJECTS
 199 export GIT_TERMINAL_PROMPT
 200 export GIT_PAGER
 201 export PAGER
 202 export GIT_ASKPASS
 203 export GIT_SVN_NOTTY
 204 unset GIT_USER_AGENT
 205 unset GIT_HTTP_USER_AGENT
 206 if [ -n "$defined_cfg_git_client_ua" ]; then
 207         GIT_USER_AGENT="$cfg_git_client_ua"
 208         export GIT_USER_AGENT
 209         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 210         export GIT_HTTP_USER_AGENT
 211 fi
 212 unset GIT_CONFIG_PARAMETERS
 213 git_add_config "core.ignoreCase=false"
 214 git_add_config "core.pager=cat"
 215 if [ -n "$cfg_git_no_mmap" ]; then
 216         # Just like compiling with NO_MMAP
 217         git_add_config "core.packedGitWindowSize=1m"
 218 else
 219         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 220         git_add_config "core.packedGitWindowSize=32m"
 221 fi
 222 [ -z "$var_big_file_threshold" ] ||
 223         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 224
 225 # Make sure any sendmail.pl config is always available
 226 unset SENDMAIL_PL_HOST
 227 unset SENDMAIL_PL_PORT
 228 unset SENDMAIL_PL_NCBIN
 229 unset SENDMAIL_PL_NCOPT
 230 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 231 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 232 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 233 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 234
 235 # Set PATH and PYTHON to the values set by Config.pm, if any
 236 unset PYTHON
 237 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 238 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 239
 240 # Extra GIT variables that generally ought to be cleared, but whose clearing
 241 # could potentially interfere with the correct operation of hook scripts so
 242 # they are segregated into a separate function for use as appropriate
 243 clean_git_env() {
 244         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 245         unset GIT_CONFIG
 246         unset GIT_DIR
 247         unset GIT_GRAFT_FILE
 248         unset GIT_INDEX_FILE
 249         unset GIT_OBJECT_DIRECTORY
 250         unset GIT_NAMESPACE
 251 }
 252
 253 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 254 # and nc_openbsd to the desired executables because when using
 255 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 256 # different and unexpected ways:
 257 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 258 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 259 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 260 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 261 # setting only while running func which none of the /bin/sh's do.
 262 #
 263 # Instead we'd like to use an alias that provides the desired behavior without
 264 # any of the bad (a), (b) or (c) effects.
 265 #
 266 # However, unfortunately, some of the crazy /bin/sh implementations do not
 267 # recognize alias expansions when preceded by variable assignments!
 268 #
 269 # So we are left with git() {} and nc_openbsd() {} functions and in the
 270 # case of git() {} we can compensate for (b) and (c) failing to export
 271 # but not (a) and (b) persisting the values so the caller will simply
 272 # have to beware and explicitly unset any variables that should not persist
 273 # beyond the function call itself.
 274
 275 git() (
 276         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 277         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 278         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 279         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 280         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 281         exec "$cfg_git_bin" "$@"
 282 )
 283
 284 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 285 # that allows us to use git update-ref --stdin where supported and the slow shell
 286 # script where not, but only the "delete" operation is currently supported.
 287 git_updateref_stdin() {
 288         if [ -n "$var_have_git_185" ]; then
 289                 git update-ref --stdin
 290         else
 291                 while read -r _op _ref; do
 292                         case "$_op" in
 293                                 delete)
 294                                         git update-ref -d "$_ref"
 295                                         ;;
 296                                 *)
 297                                         echo "bad git_updateref_stdin op: $_op" >&2
 298                                         exit 1
 299                                         ;;
 300                         esac
 301                 done
 302         fi
 303 }
 304
 305 # see comments for git() -- callers must explicitly export all variables
 306 # intended for the commands these functions run before calling them
 307 perl() { command "${var_perl_bin:-perl}" "$@"; }
 308 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 309
 310 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 311
 312 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 313
 314 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 315
 316 # Some platforms' broken xargs runs the command always at least once even if
 317 # there's no input unless given a special option.  Automatically supply the
 318 # option on those platforms by providing an xargs function.
 319 xargs() { command xargs $var_xargs_r "$@"; }
 320
 321 _addrlist() {
 322         _list=
 323         for _addr in "$@"; do
 324                 [ -z "$_list" ] || _list="$_list, "
 325                 _list="$_list$_addr"
 326         done
 327         echo "$_list"
 328 }
 329
 330 _sendmail() {
 331         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 332         if [ -n "$cfg_sender" ]; then
 333                 "$_mailer" -i -f "$cfg_sender" "$@"
 334         else
 335                 "$_mailer" -i "$@"
 336         fi
 337 }
 338
 339 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 340 # "References:" header.  It may be "" to suppress the "References" header.
 341 # Following arguments are just like mail function
 342 mailref() {
 343         _references=
 344         if [ $# -ge 1 ]; then
 345                 _references="$1"
 346                 shift
 347         fi
 348         _subject=
 349         if [ "$1" = "-s" ]; then
 350                 shift
 351                 _subject="$1"
 352                 shift
 353         fi
 354         {
 355                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 356                 echo "To: $(_addrlist "$@")"
 357                 [ -z "$_subject" ] || echo "Subject: $_subject"
 358                 echo "MIME-Version: 1.0"
 359                 echo "Content-Type: text/plain; charset=UTF-8"
 360                 echo "Content-Transfer-Encoding: 8bit"
 361                 [ -z "$_references" ] || echo "References: <$_references>"
 362                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 363                 echo "Auto-Submitted: auto-generated"
 364                 echo ""
 365                 cat
 366         } | _sendmail "$@"
 367 }
 368
 369 # Usage: mail [-s <subject>] <addr> [<addr>...]
 370 mail() {
 371         mailref "" "$@"
 372 }
 373
 374 # bang CMD... will execute the command with well-defined failure mode;
 375 # set bang_action to string of the failed action ('clone', 'update', ...);
 376 # re-define the bang_trap() function to do custom cleanup before bailing out
 377 bang() {
 378         bang_active=1
 379         bang_cmd="$*"
 380         bang_errcode=0
 381         if [ -n "$show_progress" ]; then
 382                 exec 3>&1
 383                 read -r bang_errcode <<-EOT || :
 384                 $(
 385                         exec 4>&3 3>&1 1>&4 4>&-
 386                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 387                 )
 388                 EOT
 389                 exec 3>&-
 390                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 391                         # All right. Cool.
 392                         bang_active=
 393                         bang_cmd=
 394                         return;
 395                 fi
 396         else
 397                 if "$@" >>"$bang_log" 2>&1; then
 398                         # All right. Cool.
 399                         bang_active=
 400                         bang_cmd=
 401                         return;
 402                 else
 403                         bang_errcode="$?"
 404                 fi
 405         fi
 406         bang_failed
 407 }
 408
 409 bang_failed() {
 410         bang_active=
 411         unset GIT_DIR
 412         >.banged
 413         cat "$bang_log" >.banglog
 414         echo "" >>.banglog
 415         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 416         if [ -n "$show_progress" ]; then
 417                 echo ""
 418                 echo "$bang_cmd failed with error code $bang_errcode"
 419         fi
 420         if [ -e .bangagain ]; then
 421                 git config --remove-section girocco.bang 2>/dev/null || :
 422                 rm -f .bangagain
 423         fi
 424         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 425         bangcount=$(( ${bangcount:-0} + 1 ))
 426         git config --int girocco.bang.count $bangcount
 427         if [ $bangcount -eq 1 ]; then
 428                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 429         fi
 430         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 431            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 432            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 433                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 434                 bangaddrs=
 435                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 436                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 437                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 438                 rsubj=
 439                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 440                 [ -z "$bangaddrs" ] ||
 441                 {
 442                         echo "$bang_cmd failed with error code $bang_errcode"
 443                         echo ""
 444                         rsubj=
 445                         if [ $bangcount -gt 1 ]; then
 446                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 447                                 echo ""
 448                         fi
 449                         echo "you will not receive any more notifications until recovery"
 450                         echo "this status message may be disabled on the project admin page"
 451                         echo ""
 452                         echo "Log follows:"
 453                         echo ""
 454                         cat "$bang_log"
 455                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 456                 git config --bool girocco.bang.messagesent true
 457         fi
 458         bangthrottle=
 459         [ $bangcount -lt 15 ] ||
 460                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 461                 bangthrottle=1
 462         bang_trap $bangthrottle
 463         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 464         exit $bang_errcode
 465 }
 466
 467 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 468 # Identical to bang CMD... except the command is eval'd instead of executed.
 469 bang_eval() {
 470         bang eval "$*"
 471 }
 472
 473 # Default bang settings:
 474 bang_setup() {
 475         bang_active=
 476         bang_action="lame_programmer"
 477         bang_trap() { :; }
 478         bang_tmpdir="${TMPDIR:-/tmp}"
 479         bang_tmpdir="${bang_tmpdir%/}"
 480         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 481         is_git_dir . || {
 482                 echo "bang_setup called with current directory not a git directory" >&2
 483                 exit 1
 484         }
 485         trap 'rm -f "$bang_log"' EXIT
 486         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 487         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 488 }
 489
 490 # Remove banged status
 491 bang_reset() {
 492         rm -f .banged .bangagain .banglog
 493         git config --remove-section girocco.bang 2>/dev/null || :
 494 }
 495
 496 # Check to see if banged status
 497 is_banged() {
 498         [ -e .banged ]
 499 }
 500
 501 # Check to see if banged message was sent
 502 was_banged_message_sent() {
 503         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 504 }
 505
 506 # Progress report - if show_progress is set, shows the given message.
 507 progress() {
 508         [ -z "$show_progress" ] || echo "$*"
 509 }
 510
 511 # Project config accessors; must be run in project directory
 512 config_get() {
 513         case "$1" in
 514         *.*)
 515                 git config "$1";;
 516         *)
 517                 git config "gitweb.$1";;
 518         esac
 519 }
 520
 521 config_set() {
 522         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 523 }
 524
 525 config_set_raw() {
 526         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 527 }
 528
 529 config_get_date_seconds() {
 530         _dt="$(config_get "$1")" || :
 531         [ -n "$_dt" ] || return 1
 532         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 533         [ -n "$_ds" ] || return 1
 534         echo "$_ds"
 535 }
 536
 537 # Tool for checking whether given number of seconds has not passed yet
 538 check_interval() {
 539         os="$(config_get_date_seconds "$1")" || return 1
 540         ns="$(date +%s)"
 541         [ $ns -lt $(($os+$2)) ]
 542 }
 543
 544 # Check if we are running with effective root permissions
 545 is_root() {
 546         [ "$(id -u 2>/dev/null)" = "0" ]
 547 }
 548
 549 # Check to see if the single argument is a Git directory
 550 is_git_dir() {
 551         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 552         # And we are slightly more picky (must be refs/.+ not refs/.*)
 553         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 554         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 555         if [ -L "$1/HEAD" ]; then
 556                 _hr="$(readlink "$1/HEAD")"
 557                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 558         fi
 559         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 560         read -r _hr <"$1/HEAD" || return 1
 561         case "$_hr" in
 562                 $octet20 | ref:refs/?*)
 563                         return 0;;
 564                 ref:*)
 565                         _hr="${_hr##ref:*[ $tab]}"
 566                         case "$_hr" in "refs/"?*) return 0;; esac
 567         esac
 568         return 1
 569 }
 570
 571 # List all Git repositories, with given prefix if specified, one-per-line
 572 # All project names starting with _ are always excluded from the result
 573 get_repo_list() {
 574         if [ -n "$1" ]; then
 575                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 576         else
 577                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 578         fi | while IFS=: read name id; do
 579                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 580         done
 581 }
 582
 583 # Return success if the given project name has any forks
 584 has_forks() {
 585         _prj="${1%.git}"
 586         [ -n "$_prj" ] || return 1
 587         [ -d "$cfg_reporoot/$_prj" ] || return 1
 588         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 589         test $(get_repo_list "$_prj/[^/:][^/:]*:" | LC_ALL=C wc -l) -gt 0
 590 }
 591
 592 # returns empty string and error for empty string otherwise one of
 593 #   m  =>  normal Git mirror
 594 #   s  =>  mirror from svn source
 595 #   d  =>  mirror from darcs source
 596 #   b  =>  mirror from bzr source
 597 #   h  =>  mirror from hg source
 598 #   w  =>  mirror from mediawiki source
 599 #   f  =>  mirror from other fast-import source
 600 # note that if the string is non-empty and none of s, d, b or h match the
 601 # return will always be type m regardless of whether it's a valid Git URL
 602 get_url_mirror_type() {
 603         case "$1" in
 604                 "")
 605                         return 1
 606                         ;;
 607                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 608                         echo 's'
 609                         ;;
 610                 darcs://*)
 611                         echo 'd'
 612                         ;;
 613                 bzr://*)
 614                         echo 'b'
 615                         ;;
 616                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 617                         echo 'h'
 618                         ;;
 619                 mediawiki::*)
 620                         echo 'w'
 621                         ;;
 622                 *)
 623                         echo 'm'
 624                         ;;
 625         esac
 626         return 0
 627 }
 628
 629 # returns false for empty string
 630 # returns true if the passed in url is a mirror using git fast-import
 631 is_gfi_mirror_url() {
 632         [ -n "$1" ] || return 1
 633         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 634                 d|b|h|w|f)
 635                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 636                         # and so do generic "f" fast-import mirrors
 637                         return 0
 638                         ;;
 639                 *)
 640                         # Don't think git-svn currently uses git fast-import
 641                         # And Git mirrors certainly do not
 642                         return 1
 643                         ;;
 644         esac
 645         # assume it does not use git fast-import
 646         return 1
 647 }
 648
 649 # returns false for empty string
 650 # returns true if the passed in url is a mirror using git-svn
 651 is_svn_mirror_url() {
 652         [ -n "$1" ] || return 1
 653         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 654 }
 655
 656 # returns mirror url for gitweb.baseurl of git directory
 657 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 658 # will fail if the directory does not have .nofetch and gitweb.baseurl
 659 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 660 # automatically strips any leading "disabled " prefix before returning result
 661 get_mirror_url() {
 662         _gitdir="${1:-.}"
 663         # always return empty for non-mirrors
 664         ! [ -e "$_gitdir/.nofetch" ] || return 0
 665         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 666         _url="${_url##* }"
 667         [ -n "$_url" ] || return 1
 668         printf '%s\n' "$_url"
 669         return 0
 670 }
 671
 672 # returns get_url_mirror_type for gitweb.baseurl of git directory
 673 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 674 # will fail if the directory does not have .nofetch and gitweb.baseurl
 675 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 676 # automatically strips any leading "disabled " prefix before testing
 677 get_mirror_type() {
 678         _url="$(get_mirror_url "$@")" || return 1
 679         [ -n "$_url" ] || return 0
 680         get_url_mirror_type "$_url"
 681 }
 682
 683 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 684 is_gfi_mirror() {
 685         _url="$(get_mirror_url "$@")" || return 1
 686         is_gfi_mirror_url "$_url"
 687 }
 688
 689 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 690 is_svn_mirror() {
 691         _url="$(get_mirror_url "$@")" || return 1
 692         is_svn_mirror_url "$_url"
 693 }
 694
 695 # current directory must already be set to Git repository
 696 # if girocco.headok is already true succeeds without doing anything
 697 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 698 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 699 # then refs/heads/trunk and finally the first top-level head from
 700 # refs/heads/* (i.e. only two slashes in the name) and finally any
 701 # existing refs/heads.  The first one to succeed wins and sets headok=true
 702 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 703 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 704 # when the repository is being set up -- if the HEAD is later deleted (through
 705 # a push or fetch --prune) that's no longer our responsibility to fix
 706 check_and_set_head() {
 707         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 708         if git rev-parse --verify --quiet HEAD >/dev/null; then
 709                 git config --bool girocco.headok true
 710                 return 0
 711         fi
 712         for _hr in refs/heads/master refs/heads/trunk; do
 713                 if git rev-parse --verify --quiet "$_hr"; then
 714                         _update_head_symref "$_hr"
 715                         return 0
 716                 fi
 717         done
 718         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 719         while read -r _hr; do
 720                 case "${_hr#refs/heads/}" in */*) :;; *)
 721                         _update_head_symref "$_hr"
 722                         exit 1 # exit subshell created by "|"
 723                 esac
 724         done || return 0
 725         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 726         if [ -n "$_hr" ]; then
 727                 _update_head_symref "$_hr"
 728                 return 0
 729         fi
 730         return 1
 731 }
 732 _update_head_symref() {
 733         git symbolic-ref HEAD "$1"
 734         git config --bool girocco.headok true
 735         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 736 }
 737
 738 # A well-known UTF-8 locale is required for some of the fast-import providers
 739 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 740 # but that is not reliably UTF-8 but rather usually US-ASCII.
 741 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 742 # install time and store that in $var_utf8_locale if one is found.
 743 # If we cannot find one in the `locale -a` output then we just use a well-known
 744 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 745 # it.  We only set this temporarily when running the fast-import providers.
 746 set_utf8_locale() {
 747         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 748         export LC_ALL
 749 }
 750
 751 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 752 git_hg_fetch() (
 753         set_utf8_locale
 754         _python="${PYTHON:-python}"
 755         rm -f hg2git-marks.old hg2git-marks.new
 756         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 757                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 758                         if [ -n "$var_have_git_185" ]; then
 759                                 git cat-file --batch-check=':%(rest) %(objectname)'
 760                         else
 761                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 762                         fi
 763                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 764                 if [ -n "$var_have_git_171" ] &&
 765                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 766                         if [ -z "$var_have_git_185" ] ||
 767                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 768                                 _nm='hg-fast-export'
 769                                 GIT_AUTHOR_NAME="$_nm"
 770                                 GIT_COMMITTER_NAME="$_nm"
 771                                 GIT_AUTHOR_EMAIL="$_nm"
 772                                 GIT_COMMITTER_EMAIL="$_nm"
 773                                 export GIT_AUTHOR_NAME
 774                                 export GIT_COMMITTER_NAME
 775                                 export GIT_AUTHOR_EMAIL
 776                                 export GIT_COMMITTER_EMAIL
 777                                 git notes --ref=refs/notes/hg prune
 778                                 unset GIT_AUTHOR_NAME
 779                                 unset GIT_COMMITTER_NAME
 780                                 unset GIT_AUTHOR_EMAIL
 781                                 unset GIT_COMMITTER_EMAIL
 782                         fi
 783                 fi
 784         else
 785                 >hg2git-marks.old
 786         fi
 787         _err1=
 788         _err2=
 789         exec 3>&1
 790         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 791         $(
 792                 exec 4>&3 3>&1 1>&4 4>&-
 793                 {
 794                         _e1=0
 795                         _af="$(git config hg.authorsfile)" || :
 796                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 797                                 --repo    "$(pwd)/repo.hg" \
 798                                 --marks   "$(pwd)/hg2git-marks.old" \
 799                                 --mapping "$(pwd)/hg2git-mapping" \
 800                                 --heads   "$(pwd)/hg2git-heads" \
 801                                 --status  "$(pwd)/hg2git-state" \
 802                                 -U unknown --force --flatten --hg-hash'
 803                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 804                         eval "$_cmd" 3>&- || _e1=$?
 805                         echo $_e1 >&3
 806                 } |
 807                 {
 808                         _e2=0
 809                         git fast-import \
 810                                 --import-marks="$(pwd)/hg2git-marks.old" \
 811                                 --export-marks="$(pwd)/hg2git-marks.new" \
 812                                 --export-pack-edges="$(pwd)/gfi-packs" \
 813                                 --force 3>&- || _e2=$?
 814                         echo $_e2 >&3
 815                 }
 816         )
 817         EOT
 818         exec 3>&-
 819         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 820         mv -f hg2git-marks.new hg2git-marks
 821         rm -f hg2git-marks.old
 822         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 823         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 824 )