shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # hash patterns
   7 hexdig='[0-9a-f]'
   8 octet="$hexdig$hexdig"
   9 octet4="$octet$octet$octet$octet"
  10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  11 octet20="$octet4$octet4$octet4$octet4$octet4"
  12 # tab (single \t between single quotes)
  13 tab='   '
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" ] || [ -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         inc_basedir=@basedir@
  54         [ "@basedir@" != '@'basedir'@' ] || inc_basedir="$PWD"
  55         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  56         perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf -le \
  57                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  58                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  59                         $val =~ s/([\\"\$\`])/\\$1/gos;
  60                         $val =~ s/(?:\r\n|\r|\n)$//os;
  61                         print "cfg_$_=\"$val\"";
  62                         print "defined_cfg_$_=",
  63                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  64                 }'
  65 )
  66
  67 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
  68 _fcp() {
  69         if _fp="$(command -v "$1" 2>/dev/null)"; then
  70                 printf '%s\n' "$_fp"
  71         else
  72                 printf '%s\n' "$1"
  73         fi
  74 }
  75
  76 get_girocco_config_var_list() (
  77         # Same as get_girocco_config_pm_var_list except that
  78         # the following variables (all starting with var_) are added:
  79         # var_group             cfg_owning_group if defined otherwise `id -gn`
  80         # var_git_ver           The version number part from `git version`
  81         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  82         # var_sh_bin            Full path to the posix sh interpreter to use
  83         # var_perl_bin          Full path to the perl interpreter to use
  84         # var_gzip_bin          Full path to the gzip executable to use
  85         # var_nc_openbsd_bin    Full path to the netcat (nc) with -U support
  86         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  87         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  88         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  89         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  90         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  91         # var_have_git_210      Set to 1 if git version >= 2.1.0 otherwise ''
  92         # var_have_git_235      Set to 1 if git version >= 2.3.5 otherwise ''
  93         # var_have_git_260      Set to 1 if git version >= 2.6.0 otherwise ''
  94         # var_have_git_2101     Set to 1 if git version >= 2.10.1 otherwise ''
  95         # var_window_memory     Value to use for repack --window-memory=
  96         # var_big_file_threshold Value to use for core.bigFileThreshold
  97         # var_redelta_threshold Recompute deltas if no more than this many objs
  98         # var_upload_window     If not "", pack.window to use for upload-pack
  99         # var_log_window_size   Value to use for git-svn --log-window-size=
 100         # var_utf8_locale       Value to use for a UTF-8 locale if available
 101         # var_xargs_r           A "-r" if xargs needs it to behave correctly
 102         # var_du_exclude        Option to exclude PATTERN from du if available
 103         # var_du_follow         Option to follow command line sym links if available
 104         _cfg_vars="$(get_girocco_config_pm_var_list)"
 105         eval "$_cfg_vars"
 106         printf '%s\n' "$_cfg_vars"
 107         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
 108         _gver="$("$cfg_git_bin" version 2>/dev/null |
 109                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
 110         printf 'var_git_ver=%s\n' "$_gver"
 111         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
 112         printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
 113         printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
 114         printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
 115         printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
 116         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 117         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 118         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 119         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 120         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 121         printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
 122         printf 'var_have_git_235=%s\n' "$([ $(vcmp "$_gver" 2.3.5) -ge 0 ] && echo 1)"
 123         printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
 124         printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
 125         __girocco_conf="$GIROCCO_CONF"
 126         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 127         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 128         inc_basedir=@basedir@
 129         [ "@basedir@" != '@'basedir'@' ] || inc_basedir="$PWD"
 130         printf "var_window_memory=%s\n" \
 131                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 132                 -MGirocco::Util -e 'print calc_windowmemory')"
 133         printf "var_big_file_threshold=%s\n" \
 134                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 135                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 136         printf "var_redelta_threshold=%s\n" \
 137                 "$(perl -I"$inc_basedir" $__girocco_extrainc -M$__girocco_conf \
 138                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 139         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
 140            [ "$cfg_upload_pack_window" -le 50 ]; then
 141                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 142         else
 143                 printf "var_upload_window=%s\n" ""
 144         fi
 145         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 146         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 147         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
 148                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
 149                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
 150                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 151         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 152         # On some broken platforms running xargs without -r and empty input runs the command
 153         printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
 154         # The disk usage report produces better numbers if du has an exclude option
 155         _x0="${0##*/}"
 156         _x0="${_x0%?}?*"
 157         for _duopt in --exclude -I; do
 158                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 159                         printf 'var_du_exclude=%s\n' "$_duopt"
 160                         break
 161                 fi
 162         done
 163         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 164                 printf 'var_du_follow=%s\n' "-H"
 165                 break
 166         fi
 167 )
 168
 169 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 170 # definitions from it rather than running Perl.
 171 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 172         # Import all the variables from Girocco::Config to the local environment,
 173         eval "$(get_girocco_config_var_list)"
 174 else
 175         # Import the variables from shlib_vars.sh which avoids needlessly
 176         # running another copy of Perl
 177         . "@basedir@/shlib_vars.sh"
 178 fi
 179
 180 # git_add_config "some.var=value"
 181 # every ' in value must be replaced with the 4-character sequence '\'' before
 182 # calling this function or Git will barf.  Will not be effective unless running
 183 # Git version 1.7.3 or later.
 184 git_add_config() {
 185         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 186         export GIT_CONFIG_PARAMETERS
 187 }
 188
 189 # Make sure we have a reproducible environment by using a controlled HOME dir
 190 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 191 HOME="$cfg_chroot/etc/girocco"
 192 TMPDIR="/tmp"
 193 GIT_CONFIG_NOSYSTEM=1
 194 GIT_ATTR_NOSYSTEM=1
 195 GIT_NO_REPLACE_OBJECTS=1
 196 GIT_TERMINAL_PROMPT=0
 197 GIT_PAGER="cat"
 198 PAGER="cat"
 199 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 200 GIT_SVN_NOTTY=1
 201 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
 202 GIT_SSH="$cfg_basedir/bin/git-ssh"
 203 SVN_SSH="$cfg_basedir/bin/git-ssh"
 204 export XDG_CONFIG_HOME
 205 export HOME
 206 export TMPDIR
 207 export GIT_CONFIG_NOSYSTEM
 208 export GIT_ATTR_NOSYSTEM
 209 export GIT_NO_REPLACE_OBJECTS
 210 export GIT_TERMINAL_PROMPT
 211 export GIT_PAGER
 212 export PAGER
 213 export GIT_ASKPASS
 214 export GIT_SVN_NOTTY
 215 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
 216 export GIT_SSH
 217 export SVN_SSH
 218 unset GIT_USER_AGENT
 219 unset GIT_HTTP_USER_AGENT
 220 if [ -n "$defined_cfg_git_client_ua" ]; then
 221         GIT_USER_AGENT="$cfg_git_client_ua"
 222         export GIT_USER_AGENT
 223 fi
 224 unset GIT_CONFIG_PARAMETERS
 225 git_add_config "core.ignoreCase=false"
 226 git_add_config "core.pager=cat"
 227 if [ -n "$cfg_git_no_mmap" ]; then
 228         # Just like compiling with NO_MMAP
 229         git_add_config "core.packedGitWindowSize=1m"
 230 else
 231         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 232         git_add_config "core.packedGitWindowSize=32m"
 233 fi
 234 [ -z "$var_big_file_threshold" ] ||
 235         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 236 git_add_config "gc.auto=0"
 237
 238 # Make sure any sendmail.pl config is always available
 239 unset SENDMAIL_PL_HOST
 240 unset SENDMAIL_PL_PORT
 241 unset SENDMAIL_PL_NCBIN
 242 unset SENDMAIL_PL_NCOPT
 243 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 244 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 245 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 246 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 247
 248 # Set PATH and PYTHON to the values set by Config.pm, if any
 249 unset PYTHON
 250 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 251 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 252
 253 # Extra GIT variables that generally ought to be cleared, but whose clearing
 254 # could potentially interfere with the correct operation of hook scripts so
 255 # they are segregated into a separate function for use as appropriate
 256 clean_git_env() {
 257         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 258         unset GIT_CONFIG
 259         unset GIT_DIR
 260         unset GIT_GRAFT_FILE
 261         unset GIT_INDEX_FILE
 262         unset GIT_OBJECT_DIRECTORY
 263         unset GIT_NAMESPACE
 264 }
 265
 266 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 267 # and nc_openbsd to the desired executables because when using
 268 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 269 # different and unexpected ways:
 270 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 271 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 272 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 273 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 274 # setting only while running func which none of the /bin/sh's do.
 275 #
 276 # Instead we'd like to use an alias that provides the desired behavior without
 277 # any of the bad (a), (b) or (c) effects.
 278 #
 279 # However, unfortunately, some of the crazy /bin/sh implementations do not
 280 # recognize alias expansions when preceded by variable assignments!
 281 #
 282 # So we are left with git() {} and nc_openbsd() {} functions and in the
 283 # case of git() {} we can compensate for (b) and (c) failing to export
 284 # but not (a) and (b) persisting the values so the caller will simply
 285 # have to beware and explicitly unset any variables that should not persist
 286 # beyond the function call itself.
 287
 288 git() (
 289         [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
 290         [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
 291         [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
 292         [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
 293         [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
 294         exec "$cfg_git_bin" "$@"
 295 )
 296
 297 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
 298 # that allows us to use git update-ref --stdin where supported and the slow shell
 299 # script where not, but only the "delete" operation is currently supported.
 300 git_updateref_stdin() {
 301         if [ -n "$var_have_git_185" ]; then
 302                 git update-ref --stdin
 303         else
 304                 while read -r _op _ref; do
 305                         case "$_op" in
 306                                 delete)
 307                                         git update-ref -d "$_ref"
 308                                         ;;
 309                                 *)
 310                                         echo "bad git_updateref_stdin op: $_op" >&2
 311                                         exit 1
 312                                         ;;
 313                         esac
 314                 done
 315         fi
 316 }
 317
 318 # see comments for git() -- callers must explicitly export all variables
 319 # intended for the commands these functions run before calling them
 320 perl() { command "${var_perl_bin:-perl}" "$@"; }
 321 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 322
 323 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
 324
 325 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 326
 327 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
 328
 329 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 330
 331 # Some platforms' broken xargs runs the command always at least once even if
 332 # there's no input unless given a special option.  Automatically supply the
 333 # option on those platforms by providing an xargs function.
 334 xargs() { command xargs $var_xargs_r "$@"; }
 335
 336 _addrlist() {
 337         _list=
 338         for _addr in "$@"; do
 339                 [ -z "$_list" ] || _list="$_list, "
 340                 _list="$_list$_addr"
 341         done
 342         echo "$_list"
 343 }
 344
 345 _sendmail() {
 346         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 347         if [ -n "$cfg_sender" ]; then
 348                 "$_mailer" -i -f "$cfg_sender" "$@"
 349         else
 350                 "$_mailer" -i "$@"
 351         fi
 352 }
 353
 354 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
 355 # "References:" header.  It may be "" to suppress the "References" header.
 356 # Following arguments are just like mail function
 357 mailref() {
 358         _references=
 359         if [ $# -ge 1 ]; then
 360                 _references="$1"
 361                 shift
 362         fi
 363         _subject=
 364         if [ "$1" = "-s" ]; then
 365                 shift
 366                 _subject="$1"
 367                 shift
 368         fi
 369         {
 370                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 371                 echo "To: $(_addrlist "$@")"
 372                 [ -z "$_subject" ] || echo "Subject: $_subject"
 373                 echo "MIME-Version: 1.0"
 374                 echo "Content-Type: text/plain; charset=utf-8; format=fixed"
 375                 echo "Content-Transfer-Encoding: 8bit"
 376                 [ -z "$_references" ] || echo "References: <$_references>"
 377                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 378                 echo "Auto-Submitted: auto-generated"
 379                 echo ""
 380                 cat
 381         } | _sendmail "$@"
 382 }
 383
 384 # Usage: mail [-s <subject>] <addr> [<addr>...]
 385 mail() {
 386         mailref "" "$@"
 387 }
 388
 389 # bang CMD... will execute the command with well-defined failure mode;
 390 # set bang_action to string of the failed action ('clone', 'update', ...);
 391 # re-define the bang_trap() function to do custom cleanup before bailing out
 392 bang() {
 393         bang_errcode=
 394         bang_catch "$@"
 395         [ "${bang_errcode:-0}" = "0" ] || bang_failed
 396 }
 397
 398 bang_catch() {
 399         bang_active=1
 400         bang_cmd="$*"
 401         bang_errcode=0
 402         if [ "${show_progress:-0}" != "0" ]; then
 403                 exec 3>&1
 404                 read -r bang_errcode <<-EOT || :
 405                 $(
 406                         exec 4>&3 3>&1 1>&4 4>&-
 407                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 408                 )
 409                 EOT
 410                 exec 3>&-
 411                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 412                         # All right. Cool.
 413                         bang_active=
 414                         bang_cmd=
 415                         return;
 416                 fi
 417         else
 418                 if "$@" >>"$bang_log" 2>&1; then
 419                         # All right. Cool.
 420                         bang_active=
 421                         bang_cmd=
 422                         return;
 423                 else
 424                         bang_errcode="$?"
 425                 fi
 426         fi
 427 }
 428
 429 bang_failed() {
 430         bang_active=
 431         unset GIT_DIR
 432         >.banged
 433         cat "$bang_log" >.banglog
 434         echo "" >>.banglog
 435         echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
 436         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 437         if [ "${show_progress:-0}" != "0" ]; then
 438                 echo ""
 439                 echo "$bang_cmd failed with error code $bang_errcode"
 440         fi
 441         if [ -e .bangagain ]; then
 442                 git config --remove-section girocco.bang 2>/dev/null || :
 443                 rm -f .bangagain
 444         fi
 445         bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
 446         bangcount=$(( ${bangcount:-0} + 1 ))
 447         git config --int girocco.bang.count $bangcount
 448         if [ $bangcount -eq 1 ]; then
 449                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 450         fi
 451         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
 452            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
 453            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 454                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 455                 bangaddrs=
 456                 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
 457                 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
 458                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 459                 rsubj=
 460                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 461                 [ -z "$bangaddrs" ] ||
 462                 {
 463                         echo "$bang_cmd failed with error code $bang_errcode"
 464                         echo ""
 465                         rsubj=
 466                         if [ $bangcount -gt 1 ]; then
 467                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 468                                 echo ""
 469                         fi
 470                         echo "you will not receive any more notifications until recovery"
 471                         echo "this status message may be disabled on the project admin page"
 472                         echo ""
 473                         echo "Log follows:"
 474                         echo ""
 475                         cat "$bang_log"
 476                 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 477                 git config --bool girocco.bang.messagesent true
 478         fi
 479         bangthrottle=
 480         [ $bangcount -lt 15 ] ||
 481                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
 482                 bangthrottle=1
 483         bang_trap $bangthrottle
 484         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 485         exit $bang_errcode
 486 }
 487
 488 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 489 # Identical to bang CMD... except the command is eval'd instead of executed.
 490 bang_eval() {
 491         bang eval "$*"
 492 }
 493
 494 # Default bang settings:
 495 bang_setup() {
 496         bang_active=
 497         bang_action="lame_programmer"
 498         bang_trap() { :; }
 499         bang_tmpdir="${TMPDIR:-/tmp}"
 500         bang_tmpdir="${bang_tmpdir%/}"
 501         bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
 502         is_git_dir . || {
 503                 echo "bang_setup called with current directory not a git directory" >&2
 504                 exit 1
 505         }
 506         trap 'rm -f "$bang_log"' EXIT
 507         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 508         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 509 }
 510
 511 # Remove banged status
 512 bang_reset() {
 513         rm -f .banged .bangagain .banglog
 514         git config --remove-section girocco.bang 2>/dev/null || :
 515 }
 516
 517 # Check to see if banged status
 518 is_banged() {
 519         [ -e .banged ]
 520 }
 521
 522 # Check to see if banged message was sent
 523 was_banged_message_sent() {
 524         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 525 }
 526
 527 # Progress report - if show_progress is set, shows the given message.
 528 progress() {
 529         [ "${show_progress:-0}" = "0" ] || echo "$*"
 530 }
 531
 532 # Project config accessors; must be run in project directory
 533 config_get() {
 534         case "$1" in
 535         *.*)
 536                 git config "$1";;
 537         *)
 538                 git config "gitweb.$1";;
 539         esac
 540 }
 541
 542 config_set() {
 543         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 544 }
 545
 546 config_set_raw() {
 547         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 548 }
 549
 550 config_get_date_seconds() {
 551         _dt="$(config_get "$1")" || :
 552         [ -n "$_dt" ] || return 1
 553         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 554         [ -n "$_ds" ] || return 1
 555         echo "$_ds"
 556 }
 557
 558 # Tool for checking whether given number of seconds has not passed yet
 559 check_interval() {
 560         os="$(config_get_date_seconds "$1")" || return 1
 561         ns="$(date +%s)"
 562         [ $ns -lt $(($os+$2)) ]
 563 }
 564
 565 # Check if we are running with effective root permissions
 566 is_root() {
 567         [ "$(id -u 2>/dev/null)" = "0" ]
 568 }
 569
 570 # Check to see if the single argument (default ".") is a Git directory
 571 is_git_dir() {
 572         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 573         # And we are slightly more picky (must be refs/.+ not refs/.*)
 574         [ $# -ne 0 ] || set -- "."
 575         [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
 576         [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
 577         if [ -L "$1/HEAD" ]; then
 578                 _hr="$(readlink "$1/HEAD")"
 579                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 580         fi
 581         [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
 582         read -r _hr <"$1/HEAD" || return 1
 583         case "$_hr" in
 584                 $octet20*)
 585                         [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
 586                         return 0;;
 587                 ref:refs/?*)
 588                         return 0;;
 589                 ref:*)
 590                         _hr="${_hr##ref:*[ $tab]}"
 591                         case "$_hr" in "refs/"?*) return 0;; esac
 592         esac
 593         return 1
 594 }
 595
 596 # Check to see if the single argument (default ".") is a directory with no refs
 597 is_empty_refs_dir() {
 598         [ $# -ne 0 ] || set -- "."
 599         if [ -s "$1/packed-refs" ]; then
 600                 # could be a packed-refs file with just a '# pack-refs ..." line
 601                 # null hash lines and peel lines do not count either
 602                 _refcnt="$(( $(LC_ALL=C sed <"$1/packed-refs" \
 603                         -e "/^00* /d" \
 604                         -e "/^$octet20$hexdig* refs\/[^ $tab]*\$/!d" | wc -l) ))"
 605                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 606         fi
 607         if [ -d "$1/refs" ]; then
 608                 # quick and dirty check, doesn't try to validate contents
 609                 # or ignore embedded symbolic refs
 610                 _refcnt="$(( $(find -L "$1/refs" -type f -print 2>/dev/null | head -n 1 | LC_ALL=C wc -l) ))"
 611                 [ "${_refcnt:-0}" -eq 0 ] || return 1
 612         fi
 613         # last chance a detached HEAD (we ignore any linked working trees though)
 614         [ -s "$1/HEAD" ] && read -r _hr <"$1/HEAD" && [ -n "$_hr" ] || return 0
 615         [ "${_hr#*[!0-9a-f]}" != "$_hr" ] || [ "${_hr#*[!0]}" = "$_hr" ] || [ "${#_hr}" -lt 40 ] || return 1
 616         return 0
 617 }
 618
 619 # List all Git repositories, with given prefix if specified, one-per-line
 620 # All project names starting with _ are always excluded from the result
 621 get_repo_list() {
 622         if [ -n "$1" ]; then
 623                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 624         else
 625                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 626         fi | while IFS=: read name id; do
 627                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 628         done
 629 }
 630
 631 # set the variable named by the first argument to the project part (i.e. WITH
 632 # the trailing ".git" but WITHOUT the leading $cfg_reporoot) of the directory
 633 # specified by the second argument.
 634 # This function cannot be fooled by symbolic links.
 635 # If the second argument is omitted (or empty) use $(pwd -P) instead.
 636 # The directory specified by the second argument must exist.
 637 v_get_proj_from_dir() {
 638         [ -n "$2" ] || set -- "$1" "$(pwd -P)"
 639         [ -d "$2" ] || return 1
 640         case "$2" in
 641         "$cfg_reporoot/"?*)
 642                 # Simple case that does not need any fancy footwork
 643                 _projpart="${2#$cfg_reporoot/}"
 644                 ;;
 645         *)
 646                 _absrr="$(cd "$cfg_reporoot" && pwd -P)"
 647                 _abspd="$(cd "$2" && pwd -P)"
 648                 case "$_abspd" in
 649                 "$_absrr/"?*)
 650                         # The normal case
 651                         _projpart="${_abspd#$_absrr/}"
 652                         ;;
 653                 *)
 654                         # Must have been reached via a symbolic link, but
 655                         # we have no way to know the source, so use a
 656                         # generic "_external" leader combined with just the
 657                         # trailing directory name
 658                         _abspd="${_abspd%/}"
 659                         _abspd="${_abspd%/.git}"
 660                         _projpart="_external/${_abspd##*/}"
 661                         ;;
 662                 esac
 663         esac
 664         eval "$1="'"$_projpart"'
 665 }
 666
 667 # Returns success if "$1" does not exist or contains only blank lines and comments
 668 # The parsing rules are in Git's sha1-file.c parse_alt_odb_entry function;
 669 # the format for blank lines and comments has been the same since Git v0.99.5
 670 is_empty_alternates_file() {
 671         [ -n "$1" ] || return 0
 672         [ -e "$1" ] && [ -f "$1" ] && [ -s "$1" ] || return 0
 673         [ -r "$1" ] || return 1
 674         LC_ALL=C awk <"$1" '!/^$/ && !/^#/ {exit 1}'
 675 }
 676
 677 # Return success if the given project name has at least one immediate child fork
 678 # that has a non-empty alternates file
 679 has_forks_with_alternates() {
 680         _prj="${1%.git}"
 681         [ -n "$_prj" ] || return 1
 682         [ -d "$cfg_reporoot/$_prj" ] || return 1
 683         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 684         if
 685                 get_repo_list "$_prj/[^/:][^/:]*:" |
 686                 while read -r _prjname && [ -n "$_prjname" ]; do
 687                         is_empty_alternates_file "$cfg_reporoot/$_prjname.git/objects/info/alternates" ||
 688                         exit 1 # will only exit implicit subshell created by '|'
 689                 done
 690         then
 691                 return 1
 692         fi
 693         return 0
 694 }
 695
 696 # returns empty string and error for empty string otherwise one of
 697 #   m  =>  normal Git mirror
 698 #   s  =>  mirror from svn source
 699 #   d  =>  mirror from darcs source
 700 #   b  =>  mirror from bzr source
 701 #   h  =>  mirror from hg source
 702 #   w  =>  mirror from mediawiki source
 703 #   f  =>  mirror from other fast-import source
 704 # note that if the string is non-empty and none of s, d, b or h match the
 705 # return will always be type m regardless of whether it's a valid Git URL
 706 get_url_mirror_type() {
 707         case "$1" in
 708                 "")
 709                         return 1
 710                         ;;
 711                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 712                         echo 's'
 713                         ;;
 714                 darcs://* | darcs+http://* | darcs+https://*)
 715                         echo 'd'
 716                         ;;
 717                 bzr://*)
 718                         echo 'b'
 719                         ;;
 720                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 721                         echo 'h'
 722                         ;;
 723                 mediawiki::*)
 724                         echo 'w'
 725                         ;;
 726                 *)
 727                         echo 'm'
 728                         ;;
 729         esac
 730         return 0
 731 }
 732
 733 # returns false for empty string
 734 # returns true if the passed in url is a mirror using git fast-import
 735 is_gfi_mirror_url() {
 736         [ -n "$1" ] || return 1
 737         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 738                 d|b|h|w|f)
 739                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 740                         # and so do generic "f" fast-import mirrors
 741                         return 0
 742                         ;;
 743                 *)
 744                         # Don't think git-svn currently uses git fast-import
 745                         # And Git mirrors certainly do not
 746                         return 1
 747                         ;;
 748         esac
 749         # assume it does not use git fast-import
 750         return 1
 751 }
 752
 753 # returns false for empty string
 754 # returns true if the passed in url is a mirror using git-svn
 755 is_svn_mirror_url() {
 756         [ -n "$1" ] || return 1
 757         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 758 }
 759
 760 # returns mirror url for gitweb.baseurl of git directory
 761 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 762 # will fail if the directory does not have .nofetch and gitweb.baseurl
 763 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 764 # automatically strips any leading "disabled " prefix before returning result
 765 get_mirror_url() {
 766         _gitdir="${1:-.}"
 767         # always return empty for non-mirrors
 768         ! [ -e "$_gitdir/.nofetch" ] || return 0
 769         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
 770         _url="${_url##* }"
 771         [ -n "$_url" ] || return 1
 772         printf '%s\n' "$_url"
 773         return 0
 774 }
 775
 776 # returns get_url_mirror_type for gitweb.baseurl of git directory
 777 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 778 # will fail if the directory does not have .nofetch and gitweb.baseurl
 779 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 780 # automatically strips any leading "disabled " prefix before testing
 781 get_mirror_type() {
 782         _url="$(get_mirror_url "$@")" || return 1
 783         [ -n "$_url" ] || return 0
 784         get_url_mirror_type "$_url"
 785 }
 786
 787 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 788 is_gfi_mirror() {
 789         _url="$(get_mirror_url "$@")" || return 1
 790         is_gfi_mirror_url "$_url"
 791 }
 792
 793 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 794 is_svn_mirror() {
 795         _url="$(get_mirror_url "$@")" || return 1
 796         is_svn_mirror_url "$_url"
 797 }
 798
 799 # current directory must already be set to Git repository
 800 # if girocco.headok is already true succeeds without doing anything
 801 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 802 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 803 # then refs/heads/trunk and finally the first top-level head from
 804 # refs/heads/* (i.e. only two slashes in the name) and finally any
 805 # existing refs/heads.  The first one to succeed wins and sets headok=true
 806 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 807 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 808 # when the repository is being set up -- if the HEAD is later deleted (through
 809 # a push or fetch --prune) that's no longer our responsibility to fix
 810 check_and_set_head() {
 811         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 812         if git rev-parse --verify --quiet HEAD >/dev/null; then
 813                 git config --bool girocco.headok true
 814                 return 0
 815         fi
 816         for _hr in refs/heads/master refs/heads/trunk; do
 817                 if git rev-parse --verify --quiet "$_hr" >/dev/null; then
 818                         _update_head_symref "$_hr"
 819                         return 0
 820                 fi
 821         done
 822         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 823         while read -r _hr; do
 824                 case "${_hr#refs/heads/}" in */*) :;; *)
 825                         _update_head_symref "$_hr"
 826                         exit 1 # exit subshell created by "|"
 827                 esac
 828         done || return 0
 829         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
 830         if [ -n "$_hr" ]; then
 831                 _update_head_symref "$_hr"
 832                 return 0
 833         fi
 834         return 1
 835 }
 836 _update_head_symref() {
 837         git symbolic-ref HEAD "$1"
 838         git config --bool girocco.headok true
 839         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 840 }
 841
 842 # current directory must already be set to Git repository
 843 # if the directory needs to have gc run and .needsgc is not already set
 844 # then .needsgc will be set triggering a "mini" gc at the next opportunity
 845 # Girocco shouldn't generate any loose objects but we check for that anyway
 846 check_and_set_needsgc() {
 847         # If there's a .needspack file and ANY loose objects with a newer timestamp
 848         # then also set .needsgc otherwise remove it.  The only caller that may set
 849         # .needspack is a mirror therefore we don't have to worry about removing a
 850         # .needspack out from under a simultaneous creator.  We always do this and
 851         # do it first to try and avoid leaving a stale .needspack lying around.
 852         if [ -e .needspack ]; then
 853                 _objfiles=
 854                 _objfiles="$(( $(find -L objects/$octet -maxdepth 1 -newer .needspack -name "$octet19*" -type f -print 2>/dev/null |
 855                          head -n 1 | LC_ALL=C wc -l) +0 ))"
 856                 if [ "${_objfiles:-0}" = "0" ]; then
 857                         rm -f .needspack
 858                 else
 859                         [ -e .needsgc ] || >.needsgc
 860                 fi
 861         fi
 862         ! [ -e .needsgc ] || return 0
 863         _packs=
 864         { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
 865         if [ "${_packs:-0}" -ge 20 ]; then
 866                 >.needsgc
 867                 return 0
 868         fi
 869         _logfiles=
 870         { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
 871         if [ "${_logfiles:-0}" -ge 50 ]; then
 872                 >.needsgc
 873                 return 0
 874         fi
 875         # Truly git gc only checks the number of objects in the objects/17 directory
 876         # We check for -ge 10 which should make the probability of having more than
 877         # 5120 (20*256) loose objects present when there are less than 10 in
 878         # objects/17 vanishingly small (20 is the threshold we use for pack files)
 879         _objfiles=
 880         ! [ -d objects/17 ] ||
 881         { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
 882         if [ "${_objfiles:-0}" -ge 10 ]; then
 883                 >.needsgc
 884                 return 0
 885         fi
 886 }
 887
 888 # A well-known UTF-8 locale is required for some of the fast-import providers
 889 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 890 # but that is not reliably UTF-8 but rather usually US-ASCII.
 891 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 892 # install time and store that in $var_utf8_locale if one is found.
 893 # If we cannot find one in the `locale -a` output then we just use a well-known
 894 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 895 # it.  We only set this temporarily when running the fast-import providers.
 896 set_utf8_locale() {
 897         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 898         export LC_ALL
 899 }
 900
 901 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 902 git_hg_fetch() (
 903         set_utf8_locale
 904         _python="${PYTHON:-python}"
 905         rm -f hg2git-marks.old hg2git-marks.new
 906         if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
 907                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 908                         if [ -n "$var_have_git_185" ]; then
 909                                 git cat-file --batch-check=':%(rest) %(objectname)'
 910                         else
 911                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 912                         fi
 913                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 914                 if [ -n "$var_have_git_171" ] &&
 915                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 916                         if [ -z "$var_have_git_185" ] ||
 917                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 918                                 _nm='hg-fast-export'
 919                                 GIT_AUTHOR_NAME="$_nm"
 920                                 GIT_COMMITTER_NAME="$_nm"
 921                                 GIT_AUTHOR_EMAIL="$_nm"
 922                                 GIT_COMMITTER_EMAIL="$_nm"
 923                                 export GIT_AUTHOR_NAME
 924                                 export GIT_COMMITTER_NAME
 925                                 export GIT_AUTHOR_EMAIL
 926                                 export GIT_COMMITTER_EMAIL
 927                                 git notes --ref=refs/notes/hg prune
 928                                 unset GIT_AUTHOR_NAME
 929                                 unset GIT_COMMITTER_NAME
 930                                 unset GIT_AUTHOR_EMAIL
 931                                 unset GIT_COMMITTER_EMAIL
 932                         fi
 933                 fi
 934         else
 935                 >hg2git-marks.old
 936         fi
 937         _err1=
 938         _err2=
 939         exec 3>&1
 940         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 941         $(
 942                 exec 4>&3 3>&1 1>&4 4>&-
 943                 {
 944                         _e1=0
 945                         _af="$(git config hg.authorsfile)" || :
 946                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 947                                 --repo    "$(pwd)/repo.hg" \
 948                                 --marks   "$(pwd)/hg2git-marks.old" \
 949                                 --mapping "$(pwd)/hg2git-mapping" \
 950                                 --heads   "$(pwd)/hg2git-heads" \
 951                                 --status  "$(pwd)/hg2git-state" \
 952                                 -U unknown --force --flatten --hg-hash'
 953                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 954                         eval "$_cmd" 3>&- || _e1=$?
 955                         echo $_e1 >&3
 956                 } |
 957                 {
 958                         _e2=0
 959                         git fast-import \
 960                                 --import-marks="$(pwd)/hg2git-marks.old" \
 961                                 --export-marks="$(pwd)/hg2git-marks.new" \
 962                                 --export-pack-edges="$(pwd)/gfi-packs" \
 963                                 --force 3>&- || _e2=$?
 964                         echo $_e2 >&3
 965                 }
 966         )
 967         EOT
 968         exec 3>&-
 969         [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
 970         mv -f hg2git-marks.new hg2git-marks
 971         rm -f hg2git-marks.old
 972         git for-each-ref --format='%(refname) %(objectname)' refs/heads |
 973         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 974 )