shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  91         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  92         # var_window_memory     Value to use for repack --window-memory=
  93         # var_big_file_threshold Value to use for core.bigFileThreshold
  94         # var_redelta_threshold Recompute deltas if no more than this many objs
  95         # var_upload_window     If not "", pack.window to use for upload-pack
  96         # var_log_window_size   Value to use for git-svn --log-window-size=
  97         # var_utf8_locale       Value to use for a UTF-8 locale if available
  98         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  99         # var_du_exclude        Option to exclude PATTERN from du if available
 100         # var_du_follow         Option to follow command line sym links if available
 101         _cfg_vars="$(get_girocco_config_pm_var_list)"
 102         eval "$_cfg_vars"
 103         printf '%s\n' "$_cfg_vars"
 104         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 105         _gver="$("$cfg_git_bin" version 2>/dev/null |
 106                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 107         printf 'var_git_ver=%s\n' "$_gver"
 108         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 109         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 110         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 111         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 112         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 113         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 114         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 121         __girocco_conf="$GIROCCO_CONF"
 122         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 123         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 124         printf "var_window_memory=%s\n" \
 125                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 126                 -MGirocco::Util -e 'print calc_windowmemory')"
 127         printf "var_big_file_threshold=%s\n" \
 128                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 129                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 130         printf "var_redelta_threshold=%s\n" \
 131                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 133         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 134            [ "$cfg_upload_pack_window" -le 50 ]; then
 135                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 136         else
 137                 printf "var_upload_window=%s\n" ""
 138         fi
 139         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 140         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 141         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 142                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 143                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 144                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 145         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 146         # On some broken platforms running xargs without -r and empty input runs the command
 147         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 148         # The disk usage report produces better numbers if du has an exclude option
 149         _x0="$(basename "$0")"
 150         _x0="${_x0%?}?*"
 151         for _duopt in --exclude -I; do
 152                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 153                         printf 'var_du_exclude=%s\n' "$_duopt"
 154                         break
 155                 fi
 156         done
 157         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 158                 printf 'var_du_follow=%s\n' "-H"
 159                 break
 160         fi
 161 )
 162
 163 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 164 # definitions from it rather than running Perl.
 165 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 166         # Import all the variables from Girocco::Config to the local environment,
 167         eval "$(get_girocco_config_var_list)"
 168 else
 169         # Import the variables from shlib_vars.sh which avoids needlessly
 170         # running another copy of Perl
 171         . "@basedir@/shlib_vars.sh"
 172 fi
 173
 174 # git_add_config "some.var=value"
 175 # every ' in value must be replaced with the 4-character sequence '\'' before
 176 # calling this function or Git will barf.  Will not be effective unless running
 177 # Git version 1.7.3 or later.
 178 git_add_config() {
 179         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 180         export GIT_CONFIG_PARAMETERS
 181 }
 182
 183 # Make sure we have a reproducible environment by using a controlled HOME dir
 184 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 185 HOME="$cfg_chroot/etc/girocco"
 186 TMPDIR="/tmp"
 187 GIT_CONFIG_NOSYSTEM=1
 188 GIT_ATTR_NOSYSTEM=1
 189 GIT_NO_REPLACE_OBJECTS=1
 190 GIT_TERMINAL_PROMPT=0
 191 GIT_PAGER="cat"
 192 PAGER="cat"
 193 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 194 GIT_SVN_NOTTY=1
 195 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 196 SVN_SSH="$cfg_basedir/bin/git-svn-ssh"
 197 export XDG_CONFIG_HOME
 198 export HOME
 199 export TMPDIR
 200 export GIT_CONFIG_NOSYSTEM
 201 export GIT_ATTR_NOSYSTEM
 202 export GIT_NO_REPLACE_OBJECTS
 203 export GIT_TERMINAL_PROMPT
 204 export GIT_PAGER
 205 export PAGER
 206 export GIT_ASKPASS
 207 export GIT_SVN_NOTTY
 208 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 209 export SVN_SSH
 210 unset GIT_USER_AGENT
 211 unset GIT_HTTP_USER_AGENT
 212 if [ -n "$defined_cfg_git_client_ua" ]; then
 213         GIT_USER_AGENT="$cfg_git_client_ua"
 214         export GIT_USER_AGENT
 215         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 216         export GIT_HTTP_USER_AGENT
 217 fi
 218 unset GIT_CONFIG_PARAMETERS
 219 git_add_config "core.ignoreCase=false"
 220 git_add_config "core.pager=cat"
 221 if [ -n "$cfg_git_no_mmap" ]; then
 222         # Just like compiling with NO_MMAP
 223         git_add_config "core.packedGitWindowSize=1m"
 224 else
 225         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 226         git_add_config "core.packedGitWindowSize=32m"
 227 fi
 228 [ -z "$var_big_file_threshold" ] ||
 229         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 230 git_add_config "gc.auto=0"
 231
 232 # Make sure any sendmail.pl config is always available
 233 unset SENDMAIL_PL_HOST
 234 unset SENDMAIL_PL_PORT
 235 unset SENDMAIL_PL_NCBIN
 236 unset SENDMAIL_PL_NCOPT
 237 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 238 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 239 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 240 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 241
 242 # Set PATH and PYTHON to the values set by Config.pm, if any
 243 unset PYTHON
 244 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 245 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 246
 247 # Extra GIT variables that generally ought to be cleared, but whose clearing
 248 # could potentially interfere with the correct operation of hook scripts so
 249 # they are segregated into a separate function for use as appropriate
 250 clean_git_env() {
 251         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 252         unset GIT_CONFIG
 253         unset GIT_DIR
 254         unset GIT_GRAFT_FILE
 255         unset GIT_INDEX_FILE
 256         unset GIT_OBJECT_DIRECTORY
 257         unset GIT_NAMESPACE
 258 }
 259
 260 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 261 # and nc_openbsd to the desired executables because when using
 262 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 263 # different and unexpected ways:
 264 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 265 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 266 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 267 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 268 # setting only while running func which none of the /bin/sh's do.
 269 #
 270 # Instead we'd like to use an alias that provides the desired behavior without
 271 # any of the bad (a), (b) or (c) effects.
 272 #
 273 # However, unfortunately, some of the crazy /bin/sh implementations do not
 274 # recognize alias expansions when preceded by variable assignments!
 275 #
 276 # So we are left with git() {} and nc_openbsd() {} functions and in the
 277 # case of git() {} we can compensate for (b) and (c) failing to export
 278 # but not (a) and (b) persisting the values so the caller will simply
 279 # have to beware and explicitly unset any variables that should not persist
 280 # beyond the function call itself.
 281
 282 git() (
 283         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 284         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 285         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 286         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 287         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 288         exec "$cfg_git_bin" "$@"
 289 )
 290
 291 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 292 # that allows us to use git update-ref --stdin where supported and the slow shell
 293 # script where not, but only the "delete" operation is currently supported.
 294 git_updateref_stdin() {
 295         if [ -n "$var_have_git_185" ]; then
 296                 git update-ref --stdin
 297         else
 298                 while read -r _op _ref; do
 299                         case "$_op" in
 300                                 delete)
 301                                         git update-ref -d "$_ref"
 302                                         ;;
 303                                 *)
 304                                         echo "bad git_updateref_stdin op: $_op" >&2
 305                                         exit 1
 306                                         ;;
 307                         esac
 308                 done
 309         fi
 310 }
 311
 312 # see comments for git() -- callers must explicitly export all variables
 313 # intended for the commands these functions run before calling them
 314 perl() { command "${var_perl_bin:-perl}" "$@"; }
 315 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 316
 317 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 318
 319 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 320
 321 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 322
 323 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 324
 325 # Some platforms' broken xargs runs the command always at least once even if
 326 # there's no input unless given a special option.  Automatically supply the
 327 # option on those platforms by providing an xargs function.
 328 xargs() { command xargs $var_xargs_r "$@"; }
 329
 330 _addrlist() {
 331         _list=
 332         for _addr in "$@"; do
 333                 [ -z "$_list" ] || _list="$_list, "
 334                 _list="$_list$_addr"
 335         done
 336         echo "$_list"
 337 }
 338
 339 _sendmail() {
 340         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 341         if [ -n "$cfg_sender" ]; then
 342                 "$_mailer" -i -f "$cfg_sender" "$@"
 343         else
 344                 "$_mailer" -i "$@"
 345         fi
 346 }
 347
 348 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 349 # "References:" header.  It may be "" to suppress the "References" header.
 350 # Following arguments are just like mail function
 351 mailref() {
 352         _references=
 353         if [ $# -ge 1 ]; then
 354                 _references="$1"
 355                 shift
 356         fi
 357         _subject=
 358         if [ "$1" = "-s" ]; then
 359                 shift
 360                 _subject="$1"
 361                 shift
 362         fi
 363         {
 364                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 365                 echo "To: $(_addrlist "$@")"
 366                 [ -z "$_subject" ] || echo "Subject: $_subject"
 367                 echo "MIME-Version: 1.0"
 368                 echo "Content-Type: text/plain; charset=UTF-8"
 369                 echo "Content-Transfer-Encoding: 8bit"
 370                 [ -z "$_references" ] || echo "References: <$_references>"
 371                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 372                 echo "Auto-Submitted: auto-generated"
 373                 echo ""
 374                 cat
 375         } | _sendmail "$@"
 376 }
 377
 378 # Usage: mail [-s <subject>] <addr> [<addr>...]
 379 mail() {
 380         mailref "" "$@"
 381 }
 382
 383 # bang CMD... will execute the command with well-defined failure mode;
 384 # set bang_action to string of the failed action ('clone', 'update', ...);
 385 # re-define the bang_trap() function to do custom cleanup before bailing out
 386 bang() {
 387         bang_errcode=
 388         bang_catch "$@"
 389         [ "${bang_errcode:-0}" = "0" ] || bang_failed
 390 }
 391
 392 bang_catch() {
 393         bang_active=1
 394         bang_cmd="$*"
 395         bang_errcode=0
 396         if [ "${show_progress:-0}" != "0" ]; then
 397                 exec 3>&1
 398                 read -r bang_errcode <<-EOT || :
 399                 $(
 400                         exec 4>&3 3>&1 1>&4 4>&-
 401                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 402                 )
 403                 EOT
 404                 exec 3>&-
 405                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 406                         # All right. Cool.
 407                         bang_active=
 408                         bang_cmd=
 409                         return;
 410                 fi
 411         else
 412                 if "$@" >>"$bang_log" 2>&1; then
 413                         # All right. Cool.
 414                         bang_active=
 415                         bang_cmd=
 416                         return;
 417                 else
 418                         bang_errcode="$?"
 419                 fi
 420         fi
 421 }
 422
 423 bang_failed() {
 424         bang_active=
 425         unset GIT_DIR
 426         >.banged
 427         cat "$bang_log" >.banglog
 428         echo "" >>.banglog
 429         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 430         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 431         if [ "${show_progress:-0}" != "0" ]; then
 432                 echo ""
 433                 echo "$bang_cmd failed with error code $bang_errcode"
 434         fi
 435         if [ -e .bangagain ]; then
 436                 git config --remove-section girocco.bang 2>/dev/null || :
 437                 rm -f .bangagain
 438         fi
 439         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 440         bangcount=$(( ${bangcount:-0} + 1 ))
 441         git config --int girocco.bang.count $bangcount
 442         if [ $bangcount -eq 1 ]; then
 443                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 444         fi
 445         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 446            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 447            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 448                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 449                 bangaddrs=
 450                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 451                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 452                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 453                 rsubj=
 454                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 455                 [ -z "$bangaddrs" ] ||
 456                 {
 457                         echo "$bang_cmd failed with error code $bang_errcode"
 458                         echo ""
 459                         rsubj=
 460                         if [ $bangcount -gt 1 ]; then
 461                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 462                                 echo ""
 463                         fi
 464                         echo "you will not receive any more notifications until recovery"
 465                         echo "this status message may be disabled on the project admin page"
 466                         echo ""
 467                         echo "Log follows:"
 468                         echo ""
 469                         cat "$bang_log"
 470                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 471                 git config --bool girocco.bang.messagesent true
 472         fi
 473         bangthrottle=
 474         [ $bangcount -lt 15 ] ||
 475                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 476                 bangthrottle=1
 477         bang_trap $bangthrottle
 478         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 479         exit $bang_errcode
 480 }
 481
 482 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 483 # Identical to bang CMD... except the command is eval'd instead of executed.
 484 bang_eval() {
 485         bang eval "$*"
 486 }
 487
 488 # Default bang settings:
 489 bang_setup() {
 490         bang_active=
 491         bang_action="lame_programmer"
 492         bang_trap() { :; }
 493         bang_tmpdir="${TMPDIR:-/tmp}"
 494         bang_tmpdir="${bang_tmpdir%/}"
 495         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 496         is_git_dir . || {
 497                 echo "bang_setup called with current directory not a git directory" >&2
 498                 exit 1
 499         }
 500         trap 'rm -f "$bang_log"' EXIT
 501         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 502         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 503 }
 504
 505 # Remove banged status
 506 bang_reset() {
 507         rm -f .banged .bangagain .banglog
 508         git config --remove-section girocco.bang 2>/dev/null || :
 509 }
 510
 511 # Check to see if banged status
 512 is_banged() {
 513         [ -e .banged ]
 514 }
 515
 516 # Check to see if banged message was sent
 517 was_banged_message_sent() {
 518         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 519 }
 520
 521 # Progress report - if show_progress is set, shows the given message.
 522 progress() {
 523         [ "${show_progress:-0}" = "0" ] || echo "$*"
 524 }
 525
 526 # Project config accessors; must be run in project directory
 527 config_get() {
 528         case "$1" in
 529         *.*)
 530                 git config "$1";;
 531         *)
 532                 git config "gitweb.$1";;
 533         esac
 534 }
 535
 536 config_set() {
 537         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 538 }
 539
 540 config_set_raw() {
 541         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 542 }
 543
 544 config_get_date_seconds() {
 545         _dt="$(config_get "$1")" || :
 546         [ -n "$_dt" ] || return 1
 547         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 548         [ -n "$_ds" ] || return 1
 549         echo "$_ds"
 550 }
 551
 552 # Tool for checking whether given number of seconds has not passed yet
 553 check_interval() {
 554         os="$(config_get_date_seconds "$1")" || return 1
 555         ns="$(date +%s)"
 556         [ $ns -lt $(($os+$2)) ]
 557 }
 558
 559 # Check if we are running with effective root permissions
 560 is_root() {
 561         [ "$(id -u 2>/dev/null)" = "0" ]
 562 }
 563
 564 # Check to see if the single argument is a Git directory
 565 is_git_dir() {
 566         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 567         # And we are slightly more picky (must be refs/.+ not refs/.*)
 568         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 569         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 570         if [ -L "$1/HEAD" ]; then
 571                 _hr="$(readlink "$1/HEAD")"
 572                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 573         fi
 574         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 575         read -r _hr <"$1/HEAD" || return 1
 576         case "$_hr" in
 577                 $octet20*)
 578                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 579                         return 0;;
 580                 ref:refs/?*)
 581                         return 0;;
 582                 ref:*)
 583                         _hr="${_hr##ref:*[ $tab]}"
 584                         case "$_hr" in "refs/"?*) return 0;; esac
 585         esac
 586         return 1
 587 }
 588
 589 # List all Git repositories, with given prefix if specified, one-per-line
 590 # All project names starting with _ are always excluded from the result
 591 get_repo_list() {
 592         if [ -n "$1" ]; then
 593                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 594         else
 595                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 596         fi | while IFS=: read name id; do
 597                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 598         done
 599 }
 600
 601 # set the variable named by the first argument to the project part (i.e. WITH
 602 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 603 # specified by the second argument.
 604 # This function cannot be fooled by symbolic links.
 605 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 606 # The directory specified by the second argument must exist.
 607 v_get_proj_from_dir() {
 608         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 609         [ -d "$2" ] || return 1
 610         case "$2" in
 611         "$cfg_reporoot/"?*)
 612                 # Simple case that does not need any fancy footwork
 613                 _projpart="${2#$cfg_reporoot/}"
 614                 ;;
 615         *)
 616                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 617                 _abspd="$(cd "$2" && pwd -P)"
 618                 case "$_abspd" in
 619                 "$_absrr/"?*)
 620                         # The normal case
 621                         _projpart="${_abspd#$_absrr/}"
 622                         ;;
 623                 *)
 624                         # Must have been reached via a symbolic link, but
 625                         # we have no way to know the source, so use a
 626                         # generic "_external" leader combined with just the
 627                         # trailing directory name
 628                         _abspd="${_abspd%/}"
 629                         _abspd="${_abspd%/.git}"
 630                         _projpart="_external/${_abspd##*/}"
 631                         ;;
 632                 esac
 633         esac
 634         eval "$1="'"$_projpart"'
 635 }
 636
 637 # Return success if the given project name has at least one immediate child fork
 638 # that has a non-zero length alternates file
 639 has_forks_with_alternates() {
 640         _prj="${1%.git}"
 641         [ -n "$_prj" ] || return 1
 642         [ -d "$cfg_reporoot/$_prj" ] || return 1
 643         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 644         if
 645                 get_repo_list "$_prj/[^/:][^/:]*:" |
 646                 while read -r _prjname && [ -n "$_prjname" ]; do
 647                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 648                         exit 1 # will only exit implicit subshell created by '|'
 649                 done
 650         then
 651                 return 1
 652         fi
 653         return 0
 654 }
 655
 656 # returns empty string and error for empty string otherwise one of
 657 #   m  =>  normal Git mirror
 658 #   s  =>  mirror from svn source
 659 #   d  =>  mirror from darcs source
 660 #   b  =>  mirror from bzr source
 661 #   h  =>  mirror from hg source
 662 #   w  =>  mirror from mediawiki source
 663 #   f  =>  mirror from other fast-import source
 664 # note that if the string is non-empty and none of s, d, b or h match the
 665 # return will always be type m regardless of whether it's a valid Git URL
 666 get_url_mirror_type() {
 667         case "$1" in
 668                 "")
 669                         return 1
 670                         ;;
 671                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 672                         echo 's'
 673                         ;;
 674                 darcs://*)
 675                         echo 'd'
 676                         ;;
 677                 bzr://*)
 678                         echo 'b'
 679                         ;;
 680                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 681                         echo 'h'
 682                         ;;
 683                 mediawiki::*)
 684                         echo 'w'
 685                         ;;
 686                 *)
 687                         echo 'm'
 688                         ;;
 689         esac
 690         return 0
 691 }
 692
 693 # returns false for empty string
 694 # returns true if the passed in url is a mirror using git fast-import
 695 is_gfi_mirror_url() {
 696         [ -n "$1" ] || return 1
 697         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 698                 d|b|h|w|f)
 699                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 700                         # and so do generic "f" fast-import mirrors
 701                         return 0
 702                         ;;
 703                 *)
 704                         # Don't think git-svn currently uses git fast-import
 705                         # And Git mirrors certainly do not
 706                         return 1
 707                         ;;
 708         esac
 709         # assume it does not use git fast-import
 710         return 1
 711 }
 712
 713 # returns false for empty string
 714 # returns true if the passed in url is a mirror using git-svn
 715 is_svn_mirror_url() {
 716         [ -n "$1" ] || return 1
 717         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 718 }
 719
 720 # returns mirror url for gitweb.baseurl of git directory
 721 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 722 # will fail if the directory does not have .nofetch and gitweb.baseurl
 723 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 724 # automatically strips any leading "disabled " prefix before returning result
 725 get_mirror_url() {
 726         _gitdir="${1:-.}"
 727         # always return empty for non-mirrors
 728         ! [ -e "$_gitdir/.nofetch" ] || return 0
 729         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 730         _url="${_url##* }"
 731         [ -n "$_url" ] || return 1
 732         printf '%s\n' "$_url"
 733         return 0
 734 }
 735
 736 # returns get_url_mirror_type for gitweb.baseurl of git directory
 737 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 738 # will fail if the directory does not have .nofetch and gitweb.baseurl
 739 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 740 # automatically strips any leading "disabled " prefix before testing
 741 get_mirror_type() {
 742         _url="$(get_mirror_url "$@")" || return 1
 743         [ -n "$_url" ] || return 0
 744         get_url_mirror_type "$_url"
 745 }
 746
 747 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 748 is_gfi_mirror() {
 749         _url="$(get_mirror_url "$@")" || return 1
 750         is_gfi_mirror_url "$_url"
 751 }
 752
 753 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 754 is_svn_mirror() {
 755         _url="$(get_mirror_url "$@")" || return 1
 756         is_svn_mirror_url "$_url"
 757 }
 758
 759 # current directory must already be set to Git repository
 760 # if girocco.headok is already true succeeds without doing anything
 761 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 762 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 763 # then refs/heads/trunk and finally the first top-level head from
 764 # refs/heads/* (i.e. only two slashes in the name) and finally any
 765 # existing refs/heads.  The first one to succeed wins and sets headok=true
 766 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 767 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 768 # when the repository is being set up -- if the HEAD is later deleted (through
 769 # a push or fetch --prune) that's no longer our responsibility to fix
 770 check_and_set_head() {
 771         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 772         if git rev-parse --verify --quiet HEAD >/dev/null; then
 773                 git config --bool girocco.headok true
 774                 return 0
 775         fi
 776         for _hr in refs/heads/master refs/heads/trunk; do
 777                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 778                         _update_head_symref "$_hr"
 779                         return 0
 780                 fi
 781         done
 782         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 783         while read -r _hr; do
 784                 case "${_hr#refs/heads/}" in */*) :;; *)
 785                         _update_head_symref "$_hr"
 786                         exit 1 # exit subshell created by "|"
 787                 esac
 788         done || return 0
 789         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 790         if [ -n "$_hr" ]; then
 791                 _update_head_symref "$_hr"
 792                 return 0
 793         fi
 794         return 1
 795 }
 796 _update_head_symref() {
 797         git symbolic-ref HEAD "$1"
 798         git config --bool girocco.headok true
 799         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 800 }
 801
 802 # current directory must already be set to Git repository
 803 # if the directory needs to have gc run and .needsgc is not already set
 804 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 805 # Girocco shouldn't generate any loose objects but we check for that anyway
 806 check_and_set_needsgc() {
 807         ! [ -e .needsgc ] || return 0
 808         _packs=
 809         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 810         if [ "${_packs:-0}" -ge 20 ]; then
 811                 >.needsgc
 812                 return 0
 813         fi
 814         _logfiles=
 815         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 816         if [ "${_logfiles:-0}" -ge 50 ]; then
 817                 >.needsgc
 818                 return 0
 819         fi
 820         # Truly git gc only checks the number of objects in the objects/17 directory
 821         # We check for -ge 10 which should make the probability of having more than
 822         # 5120 (20*256) loose objects present when there are less than 10 in
 823         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 824         _objfiles=
 825         ! [ -d objects/17 ] ||
 826         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 827         if [ "${_objfiles:-0}" -ge 10 ]; then
 828                 >.needsgc
 829                 return 0
 830         fi
 831 }
 832
 833 # A well-known UTF-8 locale is required for some of the fast-import providers
 834 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 835 # but that is not reliably UTF-8 but rather usually US-ASCII.
 836 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 837 # install time and store that in $var_utf8_locale if one is found.
 838 # If we cannot find one in the `locale -a` output then we just use a well-known
 839 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 840 # it.  We only set this temporarily when running the fast-import providers.
 841 set_utf8_locale() {
 842         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 843         export LC_ALL
 844 }
 845
 846 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 847 git_hg_fetch() (
 848         set_utf8_locale
 849         _python="${PYTHON:-python}"
 850         rm -f hg2git-marks.old hg2git-marks.new
 851         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 852                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 853                         if [ -n "$var_have_git_185" ]; then
 854                                 git cat-file --batch-check=':%(rest) %(objectname)'
 855                         else
 856                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 857                         fi
 858                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 859                 if [ -n "$var_have_git_171" ] &&
 860                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 861                         if [ -z "$var_have_git_185" ] ||
 862                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 863                                 _nm='hg-fast-export'
 864                                 GIT_AUTHOR_NAME="$_nm"
 865                                 GIT_COMMITTER_NAME="$_nm"
 866                                 GIT_AUTHOR_EMAIL="$_nm"
 867                                 GIT_COMMITTER_EMAIL="$_nm"
 868                                 export GIT_AUTHOR_NAME
 869                                 export GIT_COMMITTER_NAME
 870                                 export GIT_AUTHOR_EMAIL
 871                                 export GIT_COMMITTER_EMAIL
 872                                 git notes --ref=refs/notes/hg prune
 873                                 unset GIT_AUTHOR_NAME
 874                                 unset GIT_COMMITTER_NAME
 875                                 unset GIT_AUTHOR_EMAIL
 876                                 unset GIT_COMMITTER_EMAIL
 877                         fi
 878                 fi
 879         else
 880                 >hg2git-marks.old
 881         fi
 882         _err1=
 883         _err2=
 884         exec 3>&1
 885         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 886         $(
 887                 exec 4>&3 3>&1 1>&4 4>&-
 888                 {
 889                         _e1=0
 890                         _af="$(git config hg.authorsfile)" || :
 891                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 892                                 --repo    "$(pwd)/repo.hg" \
 893                                 --marks   "$(pwd)/hg2git-marks.old" \
 894                                 --mapping "$(pwd)/hg2git-mapping" \
 895                                 --heads   "$(pwd)/hg2git-heads" \
 896                                 --status  "$(pwd)/hg2git-state" \
 897                                 -U unknown --force --flatten --hg-hash'
 898                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 899                         eval "$_cmd" 3>&- || _e1=$?
 900                         echo $_e1 >&3
 901                 } |
 902                 {
 903                         _e2=0
 904                         git fast-import \
 905                                 --import-marks="$(pwd)/hg2git-marks.old" \
 906                                 --export-marks="$(pwd)/hg2git-marks.new" \
 907                                 --export-pack-edges="$(pwd)/gfi-packs" \
 908                                 --force 3>&- || _e2=$?
 909                         echo $_e2 >&3
 910                 }
 911         )
 912         EOT
 913         exec 3>&-
 914         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 915         mv -f hg2git-marks.new hg2git-marks
 916         rm -f hg2git-marks.old
 917         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 918         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 919 )