shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  66 _fcp() {
  67         if _fp="$(command -v "$1" 2>/dev/null)"; then
  68                 printf '%s\n' "$_fp"
  69         else
  70                 printf '%s\n' "$1"
  71         fi
  72 }
  73
  74 get_girocco_config_var_list() (
  75         # Same as get_girocco_config_pm_var_list except that
  76         # the following variables (all starting with var_) are added:
  77         # var_group             cfg_owning_group if defined otherwise `id -gn`
  78         # var_git_ver           The version number part from `git version`
  79         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  80         # var_sh_bin            Full path to the posix sh interpreter to use
  81         # var_perl_bin          Full path to the perl interpreter to use
  82         # var_gzip_bin          Full path to the gzip executable to use
  83         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  84         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  85         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  86         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  87         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  88         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  89         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  90         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  91         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  92         # var_window_memory     Value to use for repack --window-memory=
  93         # var_big_file_threshold Value to use for core.bigFileThreshold
  94         # var_redelta_threshold Recompute deltas if no more than this many objs
  95         # var_upload_window     If not "", pack.window to use for upload-pack
  96         # var_log_window_size   Value to use for git-svn --log-window-size=
  97         # var_utf8_locale       Value to use for a UTF-8 locale if available
  98         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  99         # var_du_exclude        Option to exclude PATTERN from du if available
 100         # var_du_follow         Option to follow command line sym links if available
 101         _cfg_vars="$(get_girocco_config_pm_var_list)"
 102         eval "$_cfg_vars"
 103         printf '%s\n' "$_cfg_vars"
 104         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 105         _gver="$("$cfg_git_bin" version 2>/dev/null |
 106                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 107         printf 'var_git_ver=%s\n' "$_gver"
 108         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 109         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 110         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 111         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 112         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 113         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 114         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 115         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 116         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 121         __girocco_conf="$GIROCCO_CONF"
 122         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 123         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 124         printf "var_window_memory=%s\n" \
 125                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 126                 -MGirocco::Util -e 'print calc_windowmemory')"
 127         printf "var_big_file_threshold=%s\n" \
 128                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 129                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 130         printf "var_redelta_threshold=%s\n" \
 131                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 133         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 134            [ "$cfg_upload_pack_window" -le 50 ]; then
 135                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 136         else
 137                 printf "var_upload_window=%s\n" ""
 138         fi
 139         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 140         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 141         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 142                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 143                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 144                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 145         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 146         # On some broken platforms running xargs without -r and empty input runs the command
 147         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 148         # The disk usage report produces better numbers if du has an exclude option
 149         _x0="$(basename "$0")"
 150         _x0="${_x0%?}?*"
 151         for _duopt in --exclude -I; do
 152                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 153                         printf 'var_du_exclude=%s\n' "$_duopt"
 154                         break
 155                 fi
 156         done
 157         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 158                 printf 'var_du_follow=%s\n' "-H"
 159                 break
 160         fi
 161 )
 162
 163 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 164 # definitions from it rather than running Perl.
 165 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 166         # Import all the variables from Girocco::Config to the local environment,
 167         eval "$(get_girocco_config_var_list)"
 168 else
 169         # Import the variables from shlib_vars.sh which avoids needlessly
 170         # running another copy of Perl
 171         . "@basedir@/shlib_vars.sh"
 172 fi
 173
 174 # git_add_config "some.var=value"
 175 # every ' in value must be replaced with the 4-character sequence '\'' before
 176 # calling this function or Git will barf.  Will not be effective unless running
 177 # Git version 1.7.3 or later.
 178 git_add_config() {
 179         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 180         export GIT_CONFIG_PARAMETERS
 181 }
 182
 183 # Make sure we have a reproducible environment by using a controlled HOME dir
 184 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 185 HOME="$cfg_chroot/etc/girocco"
 186 TMPDIR="/tmp"
 187 GIT_CONFIG_NOSYSTEM=1
 188 GIT_ATTR_NOSYSTEM=1
 189 GIT_NO_REPLACE_OBJECTS=1
 190 GIT_TERMINAL_PROMPT=0
 191 GIT_PAGER="cat"
 192 PAGER="cat"
 193 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 194 GIT_SVN_NOTTY=1
 195 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 196 export XDG_CONFIG_HOME
 197 export HOME
 198 export TMPDIR
 199 export GIT_CONFIG_NOSYSTEM
 200 export GIT_ATTR_NOSYSTEM
 201 export GIT_NO_REPLACE_OBJECTS
 202 export GIT_TERMINAL_PROMPT
 203 export GIT_PAGER
 204 export PAGER
 205 export GIT_ASKPASS
 206 export GIT_SVN_NOTTY
 207 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 208 unset GIT_USER_AGENT
 209 unset GIT_HTTP_USER_AGENT
 210 if [ -n "$defined_cfg_git_client_ua" ]; then
 211         GIT_USER_AGENT="$cfg_git_client_ua"
 212         export GIT_USER_AGENT
 213         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 214         export GIT_HTTP_USER_AGENT
 215 fi
 216 unset GIT_CONFIG_PARAMETERS
 217 git_add_config "core.ignoreCase=false"
 218 git_add_config "core.pager=cat"
 219 if [ -n "$cfg_git_no_mmap" ]; then
 220         # Just like compiling with NO_MMAP
 221         git_add_config "core.packedGitWindowSize=1m"
 222 else
 223         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 224         git_add_config "core.packedGitWindowSize=32m"
 225 fi
 226 [ -z "$var_big_file_threshold" ] ||
 227         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 228 git_add_config "gc.auto=0"
 229
 230 # Make sure any sendmail.pl config is always available
 231 unset SENDMAIL_PL_HOST
 232 unset SENDMAIL_PL_PORT
 233 unset SENDMAIL_PL_NCBIN
 234 unset SENDMAIL_PL_NCOPT
 235 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 236 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 237 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 238 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 239
 240 # Set PATH and PYTHON to the values set by Config.pm, if any
 241 unset PYTHON
 242 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 243 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 244
 245 # Extra GIT variables that generally ought to be cleared, but whose clearing
 246 # could potentially interfere with the correct operation of hook scripts so
 247 # they are segregated into a separate function for use as appropriate
 248 clean_git_env() {
 249         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 250         unset GIT_CONFIG
 251         unset GIT_DIR
 252         unset GIT_GRAFT_FILE
 253         unset GIT_INDEX_FILE
 254         unset GIT_OBJECT_DIRECTORY
 255         unset GIT_NAMESPACE
 256 }
 257
 258 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 259 # and nc_openbsd to the desired executables because when using
 260 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 261 # different and unexpected ways:
 262 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 263 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 264 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 265 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 266 # setting only while running func which none of the /bin/sh's do.
 267 #
 268 # Instead we'd like to use an alias that provides the desired behavior without
 269 # any of the bad (a), (b) or (c) effects.
 270 #
 271 # However, unfortunately, some of the crazy /bin/sh implementations do not
 272 # recognize alias expansions when preceded by variable assignments!
 273 #
 274 # So we are left with git() {} and nc_openbsd() {} functions and in the
 275 # case of git() {} we can compensate for (b) and (c) failing to export
 276 # but not (a) and (b) persisting the values so the caller will simply
 277 # have to beware and explicitly unset any variables that should not persist
 278 # beyond the function call itself.
 279
 280 git() (
 281         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 282         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 283         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 284         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 285         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 286         exec "$cfg_git_bin" "$@"
 287 )
 288
 289 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 290 # that allows us to use git update-ref --stdin where supported and the slow shell
 291 # script where not, but only the "delete" operation is currently supported.
 292 git_updateref_stdin() {
 293         if [ -n "$var_have_git_185" ]; then
 294                 git update-ref --stdin
 295         else
 296                 while read -r _op _ref; do
 297                         case "$_op" in
 298                                 delete)
 299                                         git update-ref -d "$_ref"
 300                                         ;;
 301                                 *)
 302                                         echo "bad git_updateref_stdin op: $_op" >&2
 303                                         exit 1
 304                                         ;;
 305                         esac
 306                 done
 307         fi
 308 }
 309
 310 # see comments for git() -- callers must explicitly export all variables
 311 # intended for the commands these functions run before calling them
 312 perl() { command "${var_perl_bin:-perl}" "$@"; }
 313 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 314
 315 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 316
 317 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 318
 319 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 320
 321 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 322
 323 # Some platforms' broken xargs runs the command always at least once even if
 324 # there's no input unless given a special option.  Automatically supply the
 325 # option on those platforms by providing an xargs function.
 326 xargs() { command xargs $var_xargs_r "$@"; }
 327
 328 _addrlist() {
 329         _list=
 330         for _addr in "$@"; do
 331                 [ -z "$_list" ] || _list="$_list, "
 332                 _list="$_list$_addr"
 333         done
 334         echo "$_list"
 335 }
 336
 337 _sendmail() {
 338         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 339         if [ -n "$cfg_sender" ]; then
 340                 "$_mailer" -i -f "$cfg_sender" "$@"
 341         else
 342                 "$_mailer" -i "$@"
 343         fi
 344 }
 345
 346 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 347 # "References:" header.  It may be "" to suppress the "References" header.
 348 # Following arguments are just like mail function
 349 mailref() {
 350         _references=
 351         if [ $# -ge 1 ]; then
 352                 _references="$1"
 353                 shift
 354         fi
 355         _subject=
 356         if [ "$1" = "-s" ]; then
 357                 shift
 358                 _subject="$1"
 359                 shift
 360         fi
 361         {
 362                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 363                 echo "To: $(_addrlist "$@")"
 364                 [ -z "$_subject" ] || echo "Subject: $_subject"
 365                 echo "MIME-Version: 1.0"
 366                 echo "Content-Type: text/plain; charset=UTF-8"
 367                 echo "Content-Transfer-Encoding: 8bit"
 368                 [ -z "$_references" ] || echo "References: <$_references>"
 369                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 370                 echo "Auto-Submitted: auto-generated"
 371                 echo ""
 372                 cat
 373         } | _sendmail "$@"
 374 }
 375
 376 # Usage: mail [-s <subject>] <addr> [<addr>...]
 377 mail() {
 378         mailref "" "$@"
 379 }
 380
 381 # bang CMD... will execute the command with well-defined failure mode;
 382 # set bang_action to string of the failed action ('clone', 'update', ...);
 383 # re-define the bang_trap() function to do custom cleanup before bailing out
 384 bang() {
 385         bang_errcode=
 386         bang_catch "$@"
 387         [ "${bang_errcode:-0}" = "0" ] || bang_failed
 388 }
 389
 390 bang_catch() {
 391         bang_active=1
 392         bang_cmd="$*"
 393         bang_errcode=0
 394         if [ "${show_progress:-0}" != "0" ]; then
 395                 exec 3>&1
 396                 read -r bang_errcode <<-EOT || :
 397                 $(
 398                         exec 4>&3 3>&1 1>&4 4>&-
 399                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 400                 )
 401                 EOT
 402                 exec 3>&-
 403                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 404                         # All right. Cool.
 405                         bang_active=
 406                         bang_cmd=
 407                         return;
 408                 fi
 409         else
 410                 if "$@" >>"$bang_log" 2>&1; then
 411                         # All right. Cool.
 412                         bang_active=
 413                         bang_cmd=
 414                         return;
 415                 else
 416                         bang_errcode="$?"
 417                 fi
 418         fi
 419 }
 420
 421 bang_failed() {
 422         bang_active=
 423         unset GIT_DIR
 424         >.banged
 425         cat "$bang_log" >.banglog
 426         echo "" >>.banglog
 427         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 428         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 429         if [ "${show_progress:-0}" != "0" ]; then
 430                 echo ""
 431                 echo "$bang_cmd failed with error code $bang_errcode"
 432         fi
 433         if [ -e .bangagain ]; then
 434                 git config --remove-section girocco.bang 2>/dev/null || :
 435                 rm -f .bangagain
 436         fi
 437         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 438         bangcount=$(( ${bangcount:-0} + 1 ))
 439         git config --int girocco.bang.count $bangcount
 440         if [ $bangcount -eq 1 ]; then
 441                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 442         fi
 443         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 444            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 445            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 446                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 447                 bangaddrs=
 448                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 449                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 450                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 451                 rsubj=
 452                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 453                 [ -z "$bangaddrs" ] ||
 454                 {
 455                         echo "$bang_cmd failed with error code $bang_errcode"
 456                         echo ""
 457                         rsubj=
 458                         if [ $bangcount -gt 1 ]; then
 459                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 460                                 echo ""
 461                         fi
 462                         echo "you will not receive any more notifications until recovery"
 463                         echo "this status message may be disabled on the project admin page"
 464                         echo ""
 465                         echo "Log follows:"
 466                         echo ""
 467                         cat "$bang_log"
 468                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 469                 git config --bool girocco.bang.messagesent true
 470         fi
 471         bangthrottle=
 472         [ $bangcount -lt 15 ] ||
 473                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 474                 bangthrottle=1
 475         bang_trap $bangthrottle
 476         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 477         exit $bang_errcode
 478 }
 479
 480 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 481 # Identical to bang CMD... except the command is eval'd instead of executed.
 482 bang_eval() {
 483         bang eval "$*"
 484 }
 485
 486 # Default bang settings:
 487 bang_setup() {
 488         bang_active=
 489         bang_action="lame_programmer"
 490         bang_trap() { :; }
 491         bang_tmpdir="${TMPDIR:-/tmp}"
 492         bang_tmpdir="${bang_tmpdir%/}"
 493         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 494         is_git_dir . || {
 495                 echo "bang_setup called with current directory not a git directory" >&2
 496                 exit 1
 497         }
 498         trap 'rm -f "$bang_log"' EXIT
 499         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 500         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 501 }
 502
 503 # Remove banged status
 504 bang_reset() {
 505         rm -f .banged .bangagain .banglog
 506         git config --remove-section girocco.bang 2>/dev/null || :
 507 }
 508
 509 # Check to see if banged status
 510 is_banged() {
 511         [ -e .banged ]
 512 }
 513
 514 # Check to see if banged message was sent
 515 was_banged_message_sent() {
 516         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 517 }
 518
 519 # Progress report - if show_progress is set, shows the given message.
 520 progress() {
 521         [ "${show_progress:-0}" = "0" ] || echo "$*"
 522 }
 523
 524 # Project config accessors; must be run in project directory
 525 config_get() {
 526         case "$1" in
 527         *.*)
 528                 git config "$1";;
 529         *)
 530                 git config "gitweb.$1";;
 531         esac
 532 }
 533
 534 config_set() {
 535         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 536 }
 537
 538 config_set_raw() {
 539         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 540 }
 541
 542 config_get_date_seconds() {
 543         _dt="$(config_get "$1")" || :
 544         [ -n "$_dt" ] || return 1
 545         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 546         [ -n "$_ds" ] || return 1
 547         echo "$_ds"
 548 }
 549
 550 # Tool for checking whether given number of seconds has not passed yet
 551 check_interval() {
 552         os="$(config_get_date_seconds "$1")" || return 1
 553         ns="$(date +%s)"
 554         [ $ns -lt $(($os+$2)) ]
 555 }
 556
 557 # Check if we are running with effective root permissions
 558 is_root() {
 559         [ "$(id -u 2>/dev/null)" = "0" ]
 560 }
 561
 562 # Check to see if the single argument is a Git directory
 563 is_git_dir() {
 564         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 565         # And we are slightly more picky (must be refs/.+ not refs/.*)
 566         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 567         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 568         if [ -L "$1/HEAD" ]; then
 569                 _hr="$(readlink "$1/HEAD")"
 570                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 571         fi
 572         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 573         read -r _hr <"$1/HEAD" || return 1
 574         case "$_hr" in
 575                 $octet20*)
 576                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 577                         return 0;;
 578                 ref:refs/?*)
 579                         return 0;;
 580                 ref:*)
 581                         _hr="${_hr##ref:*[ $tab]}"
 582                         case "$_hr" in "refs/"?*) return 0;; esac
 583         esac
 584         return 1
 585 }
 586
 587 # List all Git repositories, with given prefix if specified, one-per-line
 588 # All project names starting with _ are always excluded from the result
 589 get_repo_list() {
 590         if [ -n "$1" ]; then
 591                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 592         else
 593                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 594         fi | while IFS=: read name id; do
 595                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 596         done
 597 }
 598
 599 # set the variable named by the first argument to the project part (i.e. WITH
 600 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 601 # specified by the second argument.
 602 # This function cannot be fooled by symbolic links.
 603 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 604 # The directory specified by the second argument must exist.
 605 v_get_proj_from_dir() {
 606         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 607         [ -d "$2" ] || return 1
 608         case "$2" in
 609         "$cfg_reporoot/"?*)
 610                 # Simple case that does not need any fancy footwork
 611                 _projpart="${2#$cfg_reporoot/}"
 612                 ;;
 613         *)
 614                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 615                 _abspd="$(cd "$2" && pwd -P)"
 616                 case "$_abspd" in
 617                 "$_absrr/"?*)
 618                         # The normal case
 619                         _projpart="${_abspd#$_absrr/}"
 620                         ;;
 621                 *)
 622                         # Must have been reached via a symbolic link, but
 623                         # we have no way to know the source, so use a
 624                         # generic "_external" leader combined with just the
 625                         # trailing directory name
 626                         _abspd="${_abspd%/}"
 627                         _abspd="${_abspd%/.git}"
 628                         _projpart="_external/${_abspd##*/}"
 629                         ;;
 630                 esac
 631         esac
 632         eval "$1="'"$_projpart"'
 633 }
 634
 635 # Return success if the given project name has at least one immediate child fork
 636 # that has a non-zero length alternates file
 637 has_forks_with_alternates() {
 638         _prj="${1%.git}"
 639         [ -n "$_prj" ] || return 1
 640         [ -d "$cfg_reporoot/$_prj" ] || return 1
 641         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 642         if
 643                 get_repo_list "$_prj/[^/:][^/:]*:" |
 644                 while read -r _prjname && [ -n "$_prjname" ]; do
 645                         ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
 646                         exit 1 # will only exit implicit subshell created by '|'
 647                 done
 648         then
 649                 return 1
 650         fi
 651         return 0
 652 }
 653
 654 # returns empty string and error for empty string otherwise one of
 655 #   m  =>  normal Git mirror
 656 #   s  =>  mirror from svn source
 657 #   d  =>  mirror from darcs source
 658 #   b  =>  mirror from bzr source
 659 #   h  =>  mirror from hg source
 660 #   w  =>  mirror from mediawiki source
 661 #   f  =>  mirror from other fast-import source
 662 # note that if the string is non-empty and none of s, d, b or h match the
 663 # return will always be type m regardless of whether it's a valid Git URL
 664 get_url_mirror_type() {
 665         case "$1" in
 666                 "")
 667                         return 1
 668                         ;;
 669                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 670                         echo 's'
 671                         ;;
 672                 darcs://*)
 673                         echo 'd'
 674                         ;;
 675                 bzr://*)
 676                         echo 'b'
 677                         ;;
 678                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 679                         echo 'h'
 680                         ;;
 681                 mediawiki::*)
 682                         echo 'w'
 683                         ;;
 684                 *)
 685                         echo 'm'
 686                         ;;
 687         esac
 688         return 0
 689 }
 690
 691 # returns false for empty string
 692 # returns true if the passed in url is a mirror using git fast-import
 693 is_gfi_mirror_url() {
 694         [ -n "$1" ] || return 1
 695         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 696                 d|b|h|w|f)
 697                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 698                         # and so do generic "f" fast-import mirrors
 699                         return 0
 700                         ;;
 701                 *)
 702                         # Don't think git-svn currently uses git fast-import
 703                         # And Git mirrors certainly do not
 704                         return 1
 705                         ;;
 706         esac
 707         # assume it does not use git fast-import
 708         return 1
 709 }
 710
 711 # returns false for empty string
 712 # returns true if the passed in url is a mirror using git-svn
 713 is_svn_mirror_url() {
 714         [ -n "$1" ] || return 1
 715         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 716 }
 717
 718 # returns mirror url for gitweb.baseurl of git directory
 719 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 720 # will fail if the directory does not have .nofetch and gitweb.baseurl
 721 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 722 # automatically strips any leading "disabled " prefix before returning result
 723 get_mirror_url() {
 724         _gitdir="${1:-.}"
 725         # always return empty for non-mirrors
 726         ! [ -e "$_gitdir/.nofetch" ] || return 0
 727         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 728         _url="${_url##* }"
 729         [ -n "$_url" ] || return 1
 730         printf '%s\n' "$_url"
 731         return 0
 732 }
 733
 734 # returns get_url_mirror_type for gitweb.baseurl of git directory
 735 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 736 # will fail if the directory does not have .nofetch and gitweb.baseurl
 737 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 738 # automatically strips any leading "disabled " prefix before testing
 739 get_mirror_type() {
 740         _url="$(get_mirror_url "$@")" || return 1
 741         [ -n "$_url" ] || return 0
 742         get_url_mirror_type "$_url"
 743 }
 744
 745 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 746 is_gfi_mirror() {
 747         _url="$(get_mirror_url "$@")" || return 1
 748         is_gfi_mirror_url "$_url"
 749 }
 750
 751 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 752 is_svn_mirror() {
 753         _url="$(get_mirror_url "$@")" || return 1
 754         is_svn_mirror_url "$_url"
 755 }
 756
 757 # current directory must already be set to Git repository
 758 # if girocco.headok is already true succeeds without doing anything
 759 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 760 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 761 # then refs/heads/trunk and finally the first top-level head from
 762 # refs/heads/* (i.e. only two slashes in the name) and finally any
 763 # existing refs/heads.  The first one to succeed wins and sets headok=true
 764 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 765 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 766 # when the repository is being set up -- if the HEAD is later deleted (through
 767 # a push or fetch --prune) that's no longer our responsibility to fix
 768 check_and_set_head() {
 769         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 770         if git rev-parse --verify --quiet HEAD >/dev/null; then
 771                 git config --bool girocco.headok true
 772                 return 0
 773         fi
 774         for _hr in refs/heads/master refs/heads/trunk; do
 775                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 776                         _update_head_symref "$_hr"
 777                         return 0
 778                 fi
 779         done
 780         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 781         while read -r _hr; do
 782                 case "${_hr#refs/heads/}" in */*) :;; *)
 783                         _update_head_symref "$_hr"
 784                         exit 1 # exit subshell created by "|"
 785                 esac
 786         done || return 0
 787         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 788         if [ -n "$_hr" ]; then
 789                 _update_head_symref "$_hr"
 790                 return 0
 791         fi
 792         return 1
 793 }
 794 _update_head_symref() {
 795         git symbolic-ref HEAD "$1"
 796         git config --bool girocco.headok true
 797         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 798 }
 799
 800 # current directory must already be set to Git repository
 801 # if the directory needs to have gc run and .needsgc is not already set
 802 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 803 # Girocco shouldn't generate any loose objects but we check for that anyway
 804 check_and_set_needsgc() {
 805         ! [ -e .needsgc ] || return 0
 806         _packs=
 807         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 808         if [ "${_packs:-0}" -ge 20 ]; then
 809                 >.needsgc
 810                 return 0
 811         fi
 812         _logfiles=
 813         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 814         if [ "${_logfiles:-0}" -ge 50 ]; then
 815                 >.needsgc
 816                 return 0
 817         fi
 818         # Truly git gc only checks the number of objects in the objects/17 directory
 819         # We check for -ge 10 which should make the probability of having more than
 820         # 5120 (20*256) loose objects present when there are less than 10 in
 821         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 822         _objfiles=
 823         ! [ -d objects/17 ] ||
 824         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 825         if [ "${_objfiles:-0}" -ge 10 ]; then
 826                 >.needsgc
 827                 return 0
 828         fi
 829 }
 830
 831 # A well-known UTF-8 locale is required for some of the fast-import providers
 832 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 833 # but that is not reliably UTF-8 but rather usually US-ASCII.
 834 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 835 # install time and store that in $var_utf8_locale if one is found.
 836 # If we cannot find one in the `locale -a` output then we just use a well-known
 837 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 838 # it.  We only set this temporarily when running the fast-import providers.
 839 set_utf8_locale() {
 840         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 841         export LC_ALL
 842 }
 843
 844 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 845 git_hg_fetch() (
 846         set_utf8_locale
 847         _python="${PYTHON:-python}"
 848         rm -f hg2git-marks.old hg2git-marks.new
 849         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 850                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 851                         if [ -n "$var_have_git_185" ]; then
 852                                 git cat-file --batch-check=':%(rest) %(objectname)'
 853                         else
 854                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 855                         fi
 856                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 857                 if [ -n "$var_have_git_171" ] &&
 858                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 859                         if [ -z "$var_have_git_185" ] ||
 860                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 861                                 _nm='hg-fast-export'
 862                                 GIT_AUTHOR_NAME="$_nm"
 863                                 GIT_COMMITTER_NAME="$_nm"
 864                                 GIT_AUTHOR_EMAIL="$_nm"
 865                                 GIT_COMMITTER_EMAIL="$_nm"
 866                                 export GIT_AUTHOR_NAME
 867                                 export GIT_COMMITTER_NAME
 868                                 export GIT_AUTHOR_EMAIL
 869                                 export GIT_COMMITTER_EMAIL
 870                                 git notes --ref=refs/notes/hg prune
 871                                 unset GIT_AUTHOR_NAME
 872                                 unset GIT_COMMITTER_NAME
 873                                 unset GIT_AUTHOR_EMAIL
 874                                 unset GIT_COMMITTER_EMAIL
 875                         fi
 876                 fi
 877         else
 878                 >hg2git-marks.old
 879         fi
 880         _err1=
 881         _err2=
 882         exec 3>&1
 883         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 884         $(
 885                 exec 4>&3 3>&1 1>&4 4>&-
 886                 {
 887                         _e1=0
 888                         _af="$(git config hg.authorsfile)" || :
 889                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 890                                 --repo    "$(pwd)/repo.hg" \
 891                                 --marks   "$(pwd)/hg2git-marks.old" \
 892                                 --mapping "$(pwd)/hg2git-mapping" \
 893                                 --heads   "$(pwd)/hg2git-heads" \
 894                                 --status  "$(pwd)/hg2git-state" \
 895                                 -U unknown --force --flatten --hg-hash'
 896                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 897                         eval "$_cmd" 3>&- || _e1=$?
 898                         echo $_e1 >&3
 899                 } |
 900                 {
 901                         _e2=0
 902                         git fast-import \
 903                                 --import-marks="$(pwd)/hg2git-marks.old" \
 904                                 --export-marks="$(pwd)/hg2git-marks.new" \
 905                                 --export-pack-edges="$(pwd)/gfi-packs" \
 906                                 --force 3>&- || _e2=$?
 907                         echo $_e2 >&3
 908                 }
 909         )
 910         EOT
 911         exec 3>&-
 912         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 913         mv -f hg2git-marks.new hg2git-marks
 914         rm -f hg2git-marks.old
 915         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 916         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 917 )