shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  91         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  92         # var_window_memory     Value to use for repack --window-memory=
  93         # var_big_file_threshold Value to use for core.bigFileThreshold
  94         # var_redelta_threshold Recompute deltas if no more than this many objs
  95         # var_upload_window     If not "", pack.window to use for upload-pack
  96         # var_log_window_size   Value to use for git-svn --log-window-size=
  97         # var_utf8_locale       Value to use for a UTF-8 locale if available
  98         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  99         # var_du_exclude        Option to exclude PATTERN from du if available
 100         # var_du_follow         Option to follow command line sym links if available
 101         _cfg_vars="$(get_girocco_config_pm_var_list)"
 102         eval "$_cfg_vars"
 103         printf '%s\n' "$_cfg_vars"
 104         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 105         _gver="$("$cfg_git_bin" version 2>/dev/null |
 106                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 107         printf 'var_git_ver=%s\n' "$_gver"
 108         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 109         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 110         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 111         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 112         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 113         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 114         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 121         __girocco_conf="$GIROCCO_CONF"
 122         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 123         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 124         printf "var_window_memory=%s\n" \
 125                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 126                 -MGirocco::Util -e 'print calc_windowmemory')"
 127         printf "var_big_file_threshold=%s\n" \
 128                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 129                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 130         printf "var_redelta_threshold=%s\n" \
 131                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 133         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 134            [ "$cfg_upload_pack_window" -le 50 ]; then
 135                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 136         else
 137                 printf "var_upload_window=%s\n" ""
 138         fi
 139         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 140         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 141         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 142                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 143                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 144                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 145         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 146         # On some broken platforms running xargs without -r and empty input runs the command
 147         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 148         # The disk usage report produces better numbers if du has an exclude option
 149         _x0="$(basename "$0")"
 150         _x0="${_x0%?}?*"
 151         for _duopt in --exclude -I; do
 152                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 153                         printf 'var_du_exclude=%s\n' "$_duopt"
 154                         break
 155                 fi
 156         done
 157         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 158                 printf 'var_du_follow=%s\n' "-H"
 159                 break
 160         fi
 161 )
 162
 163 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 164 # definitions from it rather than running Perl.
 165 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 166         # Import all the variables from Girocco::Config to the local environment,
 167         eval "$(get_girocco_config_var_list)"
 168 else
 169         # Import the variables from shlib_vars.sh which avoids needlessly
 170         # running another copy of Perl
 171         . "@basedir@/shlib_vars.sh"
 172 fi
 173
 174 # git_add_config "some.var=value"
 175 # every ' in value must be replaced with the 4-character sequence '\'' before
 176 # calling this function or Git will barf.  Will not be effective unless running
 177 # Git version 1.7.3 or later.
 178 git_add_config() {
 179         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 180         export GIT_CONFIG_PARAMETERS
 181 }
 182
 183 # Make sure we have a reproducible environment by using a controlled HOME dir
 184 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 185 HOME="$cfg_chroot/etc/girocco"
 186 TMPDIR="/tmp"
 187 GIT_CONFIG_NOSYSTEM=1
 188 GIT_ATTR_NOSYSTEM=1
 189 GIT_NO_REPLACE_OBJECTS=1
 190 GIT_TERMINAL_PROMPT=0
 191 GIT_PAGER="cat"
 192 PAGER="cat"
 193 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 194 GIT_SVN_NOTTY=1
 195 export XDG_CONFIG_HOME
 196 export HOME
 197 export TMPDIR
 198 export GIT_CONFIG_NOSYSTEM
 199 export GIT_ATTR_NOSYSTEM
 200 export GIT_NO_REPLACE_OBJECTS
 201 export GIT_TERMINAL_PROMPT
 202 export GIT_PAGER
 203 export PAGER
 204 export GIT_ASKPASS
 205 export GIT_SVN_NOTTY
 206 unset GIT_USER_AGENT
 207 unset GIT_HTTP_USER_AGENT
 208 if [ -n "$defined_cfg_git_client_ua" ]; then
 209         GIT_USER_AGENT="$cfg_git_client_ua"
 210         export GIT_USER_AGENT
 211         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 212         export GIT_HTTP_USER_AGENT
 213 fi
 214 unset GIT_CONFIG_PARAMETERS
 215 git_add_config "core.ignoreCase=false"
 216 git_add_config "core.pager=cat"
 217 if [ -n "$cfg_git_no_mmap" ]; then
 218         # Just like compiling with NO_MMAP
 219         git_add_config "core.packedGitWindowSize=1m"
 220 else
 221         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 222         git_add_config "core.packedGitWindowSize=32m"
 223 fi
 224 [ -z "$var_big_file_threshold" ] ||
 225         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 226
 227 # Make sure any sendmail.pl config is always available
 228 unset SENDMAIL_PL_HOST
 229 unset SENDMAIL_PL_PORT
 230 unset SENDMAIL_PL_NCBIN
 231 unset SENDMAIL_PL_NCOPT
 232 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 233 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 234 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 235 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 236
 237 # Set PATH and PYTHON to the values set by Config.pm, if any
 238 unset PYTHON
 239 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 240 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 241
 242 # Extra GIT variables that generally ought to be cleared, but whose clearing
 243 # could potentially interfere with the correct operation of hook scripts so
 244 # they are segregated into a separate function for use as appropriate
 245 clean_git_env() {
 246         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 247         unset GIT_CONFIG
 248         unset GIT_DIR
 249         unset GIT_GRAFT_FILE
 250         unset GIT_INDEX_FILE
 251         unset GIT_OBJECT_DIRECTORY
 252         unset GIT_NAMESPACE
 253 }
 254
 255 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 256 # and nc_openbsd to the desired executables because when using
 257 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 258 # different and unexpected ways:
 259 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 260 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 261 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 262 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 263 # setting only while running func which none of the /bin/sh's do.
 264 #
 265 # Instead we'd like to use an alias that provides the desired behavior without
 266 # any of the bad (a), (b) or (c) effects.
 267 #
 268 # However, unfortunately, some of the crazy /bin/sh implementations do not
 269 # recognize alias expansions when preceded by variable assignments!
 270 #
 271 # So we are left with git() {} and nc_openbsd() {} functions and in the
 272 # case of git() {} we can compensate for (b) and (c) failing to export
 273 # but not (a) and (b) persisting the values so the caller will simply
 274 # have to beware and explicitly unset any variables that should not persist
 275 # beyond the function call itself.
 276
 277 git() (
 278         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 279         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 280         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 281         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 282         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 283         exec "$cfg_git_bin" "$@"
 284 )
 285
 286 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 287 # that allows us to use git update-ref --stdin where supported and the slow shell
 288 # script where not, but only the "delete" operation is currently supported.
 289 git_updateref_stdin() {
 290         if [ -n "$var_have_git_185" ]; then
 291                 git update-ref --stdin
 292         else
 293                 while read -r _op _ref; do
 294                         case "$_op" in
 295                                 delete)
 296                                         git update-ref -d "$_ref"
 297                                         ;;
 298                                 *)
 299                                         echo "bad git_updateref_stdin op: $_op" >&2
 300                                         exit 1
 301                                         ;;
 302                         esac
 303                 done
 304         fi
 305 }
 306
 307 # see comments for git() -- callers must explicitly export all variables
 308 # intended for the commands these functions run before calling them
 309 perl() { command "${var_perl_bin:-perl}" "$@"; }
 310 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 311
 312 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 313
 314 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 315
 316 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 317
 318 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 319
 320 # Some platforms' broken xargs runs the command always at least once even if
 321 # there's no input unless given a special option.  Automatically supply the
 322 # option on those platforms by providing an xargs function.
 323 xargs() { command xargs $var_xargs_r "$@"; }
 324
 325 _addrlist() {
 326         _list=
 327         for _addr in "$@"; do
 328                 [ -z "$_list" ] || _list="$_list, "
 329                 _list="$_list$_addr"
 330         done
 331         echo "$_list"
 332 }
 333
 334 _sendmail() {
 335         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 336         if [ -n "$cfg_sender" ]; then
 337                 "$_mailer" -i -f "$cfg_sender" "$@"
 338         else
 339                 "$_mailer" -i "$@"
 340         fi
 341 }
 342
 343 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 344 # "References:" header.  It may be "" to suppress the "References" header.
 345 # Following arguments are just like mail function
 346 mailref() {
 347         _references=
 348         if [ $# -ge 1 ]; then
 349                 _references="$1"
 350                 shift
 351         fi
 352         _subject=
 353         if [ "$1" = "-s" ]; then
 354                 shift
 355                 _subject="$1"
 356                 shift
 357         fi
 358         {
 359                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 360                 echo "To: $(_addrlist "$@")"
 361                 [ -z "$_subject" ] || echo "Subject: $_subject"
 362                 echo "MIME-Version: 1.0"
 363                 echo "Content-Type: text/plain; charset=UTF-8"
 364                 echo "Content-Transfer-Encoding: 8bit"
 365                 [ -z "$_references" ] || echo "References: <$_references>"
 366                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 367                 echo "Auto-Submitted: auto-generated"
 368                 echo ""
 369                 cat
 370         } | _sendmail "$@"
 371 }
 372
 373 # Usage: mail [-s <subject>] <addr> [<addr>...]
 374 mail() {
 375         mailref "" "$@"
 376 }
 377
 378 # bang CMD... will execute the command with well-defined failure mode;
 379 # set bang_action to string of the failed action ('clone', 'update', ...);
 380 # re-define the bang_trap() function to do custom cleanup before bailing out
 381 bang() {
 382         bang_active=1
 383         bang_cmd="$*"
 384         bang_errcode=0
 385         if [ -n "$show_progress" ]; then
 386                 exec 3>&1
 387                 read -r bang_errcode <<-EOT || :
 388                 $(
 389                         exec 4>&3 3>&1 1>&4 4>&-
 390                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 391                 )
 392                 EOT
 393                 exec 3>&-
 394                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 395                         # All right. Cool.
 396                         bang_active=
 397                         bang_cmd=
 398                         return;
 399                 fi
 400         else
 401                 if "$@" >>"$bang_log" 2>&1; then
 402                         # All right. Cool.
 403                         bang_active=
 404                         bang_cmd=
 405                         return;
 406                 else
 407                         bang_errcode="$?"
 408                 fi
 409         fi
 410         bang_failed
 411 }
 412
 413 bang_failed() {
 414         bang_active=
 415         unset GIT_DIR
 416         >.banged
 417         cat "$bang_log" >.banglog
 418         echo "" >>.banglog
 419         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 420         if [ -n "$show_progress" ]; then
 421                 echo ""
 422                 echo "$bang_cmd failed with error code $bang_errcode"
 423         fi
 424         if [ -e .bangagain ]; then
 425                 git config --remove-section girocco.bang 2>/dev/null || :
 426                 rm -f .bangagain
 427         fi
 428         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 429         bangcount=$(( ${bangcount:-0} + 1 ))
 430         git config --int girocco.bang.count $bangcount
 431         if [ $bangcount -eq 1 ]; then
 432                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 433         fi
 434         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 435            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 436            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 437                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 438                 bangaddrs=
 439                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 440                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 441                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 442                 rsubj=
 443                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 444                 [ -z "$bangaddrs" ] ||
 445                 {
 446                         echo "$bang_cmd failed with error code $bang_errcode"
 447                         echo ""
 448                         rsubj=
 449                         if [ $bangcount -gt 1 ]; then
 450                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 451                                 echo ""
 452                         fi
 453                         echo "you will not receive any more notifications until recovery"
 454                         echo "this status message may be disabled on the project admin page"
 455                         echo ""
 456                         echo "Log follows:"
 457                         echo ""
 458                         cat "$bang_log"
 459                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 460                 git config --bool girocco.bang.messagesent true
 461         fi
 462         bangthrottle=
 463         [ $bangcount -lt 15 ] ||
 464                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 465                 bangthrottle=1
 466         bang_trap $bangthrottle
 467         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 468         exit $bang_errcode
 469 }
 470
 471 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 472 # Identical to bang CMD... except the command is eval'd instead of executed.
 473 bang_eval() {
 474         bang eval "$*"
 475 }
 476
 477 # Default bang settings:
 478 bang_setup() {
 479         bang_active=
 480         bang_action="lame_programmer"
 481         bang_trap() { :; }
 482         bang_tmpdir="${TMPDIR:-/tmp}"
 483         bang_tmpdir="${bang_tmpdir%/}"
 484         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 485         is_git_dir . || {
 486                 echo "bang_setup called with current directory not a git directory" >&2
 487                 exit 1
 488         }
 489         trap 'rm -f "$bang_log"' EXIT
 490         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 491         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 492 }
 493
 494 # Remove banged status
 495 bang_reset() {
 496         rm -f .banged .bangagain .banglog
 497         git config --remove-section girocco.bang 2>/dev/null || :
 498 }
 499
 500 # Check to see if banged status
 501 is_banged() {
 502         [ -e .banged ]
 503 }
 504
 505 # Check to see if banged message was sent
 506 was_banged_message_sent() {
 507         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 508 }
 509
 510 # Progress report - if show_progress is set, shows the given message.
 511 progress() {
 512         [ -z "$show_progress" ] || echo "$*"
 513 }
 514
 515 # Project config accessors; must be run in project directory
 516 config_get() {
 517         case "$1" in
 518         *.*)
 519                 git config "$1";;
 520         *)
 521                 git config "gitweb.$1";;
 522         esac
 523 }
 524
 525 config_set() {
 526         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 527 }
 528
 529 config_set_raw() {
 530         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 531 }
 532
 533 config_get_date_seconds() {
 534         _dt="$(config_get "$1")" || :
 535         [ -n "$_dt" ] || return 1
 536         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 537         [ -n "$_ds" ] || return 1
 538         echo "$_ds"
 539 }
 540
 541 # Tool for checking whether given number of seconds has not passed yet
 542 check_interval() {
 543         os="$(config_get_date_seconds "$1")" || return 1
 544         ns="$(date +%s)"
 545         [ $ns -lt $(($os+$2)) ]
 546 }
 547
 548 # Check if we are running with effective root permissions
 549 is_root() {
 550         [ "$(id -u 2>/dev/null)" = "0" ]
 551 }
 552
 553 # Check to see if the single argument is a Git directory
 554 is_git_dir() {
 555         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 556         # And we are slightly more picky (must be refs/.+ not refs/.*)
 557         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 558         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 559         if [ -L "$1/HEAD" ]; then
 560                 _hr="$(readlink "$1/HEAD")"
 561                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 562         fi
 563         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 564         read -r _hr <"$1/HEAD" || return 1
 565         case "$_hr" in
 566                 $octet20*)
 567                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 568                         return 0;;
 569                 ref:refs/?*)
 570                         return 0;;
 571                 ref:*)
 572                         _hr="${_hr##ref:*[ $tab]}"
 573                         case "$_hr" in "refs/"?*) return 0;; esac
 574         esac
 575         return 1
 576 }
 577
 578 # List all Git repositories, with given prefix if specified, one-per-line
 579 # All project names starting with _ are always excluded from the result
 580 get_repo_list() {
 581         if [ -n "$1" ]; then
 582                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 583         else
 584                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 585         fi | while IFS=: read name id; do
 586                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 587         done
 588 }
 589
 590 # Return success if the given project name has at least one immediate child fork
 591 # that has a non-zero length alternates file
 592 has_forks_with_alternates() {
 593         _prj="${1%.git}"
 594         [ -n "$_prj" ] || return 1
 595         [ -d "$cfg_reporoot/$_prj" ] || return 1
 596         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 597         if
 598                 get_repo_list "$_prj/[^/:][^/:]*:" |
 599                 while read -r _prjname && [ -n "$_prjname" ]; do
 600                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 601                         exit 1 # will only exit implicit subshell created by '|'
 602                 done
 603         then
 604                 return 1
 605         fi
 606         return 0
 607 }
 608
 609 # returns empty string and error for empty string otherwise one of
 610 #   m  =>  normal Git mirror
 611 #   s  =>  mirror from svn source
 612 #   d  =>  mirror from darcs source
 613 #   b  =>  mirror from bzr source
 614 #   h  =>  mirror from hg source
 615 #   w  =>  mirror from mediawiki source
 616 #   f  =>  mirror from other fast-import source
 617 # note that if the string is non-empty and none of s, d, b or h match the
 618 # return will always be type m regardless of whether it's a valid Git URL
 619 get_url_mirror_type() {
 620         case "$1" in
 621                 "")
 622                         return 1
 623                         ;;
 624                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 625                         echo 's'
 626                         ;;
 627                 darcs://*)
 628                         echo 'd'
 629                         ;;
 630                 bzr://*)
 631                         echo 'b'
 632                         ;;
 633                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 634                         echo 'h'
 635                         ;;
 636                 mediawiki::*)
 637                         echo 'w'
 638                         ;;
 639                 *)
 640                         echo 'm'
 641                         ;;
 642         esac
 643         return 0
 644 }
 645
 646 # returns false for empty string
 647 # returns true if the passed in url is a mirror using git fast-import
 648 is_gfi_mirror_url() {
 649         [ -n "$1" ] || return 1
 650         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 651                 d|b|h|w|f)
 652                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 653                         # and so do generic "f" fast-import mirrors
 654                         return 0
 655                         ;;
 656                 *)
 657                         # Don't think git-svn currently uses git fast-import
 658                         # And Git mirrors certainly do not
 659                         return 1
 660                         ;;
 661         esac
 662         # assume it does not use git fast-import
 663         return 1
 664 }
 665
 666 # returns false for empty string
 667 # returns true if the passed in url is a mirror using git-svn
 668 is_svn_mirror_url() {
 669         [ -n "$1" ] || return 1
 670         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 671 }
 672
 673 # returns mirror url for gitweb.baseurl of git directory
 674 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 675 # will fail if the directory does not have .nofetch and gitweb.baseurl
 676 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 677 # automatically strips any leading "disabled " prefix before returning result
 678 get_mirror_url() {
 679         _gitdir="${1:-.}"
 680         # always return empty for non-mirrors
 681         ! [ -e "$_gitdir/.nofetch" ] || return 0
 682         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 683         _url="${_url##* }"
 684         [ -n "$_url" ] || return 1
 685         printf '%s\n' "$_url"
 686         return 0
 687 }
 688
 689 # returns get_url_mirror_type for gitweb.baseurl of git directory
 690 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 691 # will fail if the directory does not have .nofetch and gitweb.baseurl
 692 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 693 # automatically strips any leading "disabled " prefix before testing
 694 get_mirror_type() {
 695         _url="$(get_mirror_url "$@")" || return 1
 696         [ -n "$_url" ] || return 0
 697         get_url_mirror_type "$_url"
 698 }
 699
 700 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 701 is_gfi_mirror() {
 702         _url="$(get_mirror_url "$@")" || return 1
 703         is_gfi_mirror_url "$_url"
 704 }
 705
 706 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 707 is_svn_mirror() {
 708         _url="$(get_mirror_url "$@")" || return 1
 709         is_svn_mirror_url "$_url"
 710 }
 711
 712 # current directory must already be set to Git repository
 713 # if girocco.headok is already true succeeds without doing anything
 714 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 715 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 716 # then refs/heads/trunk and finally the first top-level head from
 717 # refs/heads/* (i.e. only two slashes in the name) and finally any
 718 # existing refs/heads.  The first one to succeed wins and sets headok=true
 719 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 720 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 721 # when the repository is being set up -- if the HEAD is later deleted (through
 722 # a push or fetch --prune) that's no longer our responsibility to fix
 723 check_and_set_head() {
 724         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 725         if git rev-parse --verify --quiet HEAD >/dev/null; then
 726                 git config --bool girocco.headok true
 727                 return 0
 728         fi
 729         for _hr in refs/heads/master refs/heads/trunk; do
 730                 if git rev-parse --verify --quiet "$_hr"; then
 731                         _update_head_symref "$_hr"
 732                         return 0
 733                 fi
 734         done
 735         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 736         while read -r _hr; do
 737                 case "${_hr#refs/heads/}" in */*) :;; *)
 738                         _update_head_symref "$_hr"
 739                         exit 1 # exit subshell created by "|"
 740                 esac
 741         done || return 0
 742         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 743         if [ -n "$_hr" ]; then
 744                 _update_head_symref "$_hr"
 745                 return 0
 746         fi
 747         return 1
 748 }
 749 _update_head_symref() {
 750         git symbolic-ref HEAD "$1"
 751         git config --bool girocco.headok true
 752         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 753 }
 754
 755 # A well-known UTF-8 locale is required for some of the fast-import providers
 756 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 757 # but that is not reliably UTF-8 but rather usually US-ASCII.
 758 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 759 # install time and store that in $var_utf8_locale if one is found.
 760 # If we cannot find one in the `locale -a` output then we just use a well-known
 761 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 762 # it.  We only set this temporarily when running the fast-import providers.
 763 set_utf8_locale() {
 764         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 765         export LC_ALL
 766 }
 767
 768 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 769 git_hg_fetch() (
 770         set_utf8_locale
 771         _python="${PYTHON:-python}"
 772         rm -f hg2git-marks.old hg2git-marks.new
 773         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 774                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 775                         if [ -n "$var_have_git_185" ]; then
 776                                 git cat-file --batch-check=':%(rest) %(objectname)'
 777                         else
 778                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 779                         fi
 780                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 781                 if [ -n "$var_have_git_171" ] &&
 782                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 783                         if [ -z "$var_have_git_185" ] ||
 784                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 785                                 _nm='hg-fast-export'
 786                                 GIT_AUTHOR_NAME="$_nm"
 787                                 GIT_COMMITTER_NAME="$_nm"
 788                                 GIT_AUTHOR_EMAIL="$_nm"
 789                                 GIT_COMMITTER_EMAIL="$_nm"
 790                                 export GIT_AUTHOR_NAME
 791                                 export GIT_COMMITTER_NAME
 792                                 export GIT_AUTHOR_EMAIL
 793                                 export GIT_COMMITTER_EMAIL
 794                                 git notes --ref=refs/notes/hg prune
 795                                 unset GIT_AUTHOR_NAME
 796                                 unset GIT_COMMITTER_NAME
 797                                 unset GIT_AUTHOR_EMAIL
 798                                 unset GIT_COMMITTER_EMAIL
 799                         fi
 800                 fi
 801         else
 802                 >hg2git-marks.old
 803         fi
 804         _err1=
 805         _err2=
 806         exec 3>&1
 807         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 808         $(
 809                 exec 4>&3 3>&1 1>&4 4>&-
 810                 {
 811                         _e1=0
 812                         _af="$(git config hg.authorsfile)" || :
 813                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 814                                 --repo    "$(pwd)/repo.hg" \
 815                                 --marks   "$(pwd)/hg2git-marks.old" \
 816                                 --mapping "$(pwd)/hg2git-mapping" \
 817                                 --heads   "$(pwd)/hg2git-heads" \
 818                                 --status  "$(pwd)/hg2git-state" \
 819                                 -U unknown --force --flatten --hg-hash'
 820                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 821                         eval "$_cmd" 3>&- || _e1=$?
 822                         echo $_e1 >&3
 823                 } |
 824                 {
 825                         _e2=0
 826                         git fast-import \
 827                                 --import-marks="$(pwd)/hg2git-marks.old" \
 828                                 --export-marks="$(pwd)/hg2git-marks.new" \
 829                                 --export-pack-edges="$(pwd)/gfi-packs" \
 830                                 --force 3>&- || _e2=$?
 831                         echo $_e2 >&3
 832                 }
 833         )
 834         EOT
 835         exec 3>&-
 836         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 837         mv -f hg2git-marks.new hg2git-marks
 838         rm -f hg2git-marks.old
 839         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 840         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 841 )