shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # SHA-1 patterns
   7 octet='[0-9a-f][0-9a-f]'
   8 octet4="$octet$octet$octet$octet"
   9 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  10 octet20="$octet4$octet4$octet4$octet4$octet4"
  11 nullsha="0000000000000000000000000000000000000000"
  12 # tab
  13 tab="$(printf '\t')"
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" -o -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 unset orig_path
  46 get_girocco_config_pm_var_list() (
  47         # Export all the variables from Girocco::Config to suitable var= lines
  48         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  49         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  50         __girocco_conf="$GIROCCO_CONF"
  51         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  52         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  53         [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
  54         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  55                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  56                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  57                         $val =~ s/([\\"\$\`])/\\$1/gos;
  58                         $val =~ s/(?:\r\n|\r|\n)$//os;
  59                         print "cfg_$_=\"$val\"";
  60                         print "defined_cfg_$_=",
  61                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  62                 }'
  63 )
  64
  65 get_girocco_config_var_list() (
  66         # Same as get_girocco_config_pm_var_list except that
  67         # the following variables (all starting with var_) are added:
  68         # var_group             cfg_owning_group if defined otherwise `id -gn`
  69         # var_git_ver           The version number part from `git version`
  70         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  71         # var_sh_bin            Full path to the posix sh interpreter to use
  72         # var_perl_bin          Full path to the perl interpreter to use
  73         # var_gzip_bin          Full path to the gzip executable to use
  74         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  75         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  76         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  77         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  78         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  79         # var_window_memory     Value to use for repack --window-memory=
  80         # var_big_file_threshold Value to use for core.bigFileThreshold
  81         # var_redelta_threshold Recompute deltas if no more than this many objs
  82         # var_upload_window     If not "", pack.window to use for upload-pack
  83         # var_log_window_size   Value to use for git-svn --log-window-size=
  84         # var_utf8_locale       Value to use for a UTF-8 locale if available
  85         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  86         # var_du_exclude        Option to exclude PATTERN from du if available
  87         # var_du_follow         Option to follow command line sym links if available
  88         _cfg_vars="$(get_girocco_config_pm_var_list)"
  89         eval "$_cfg_vars"
  90         printf '%s\n' "$_cfg_vars"
  91         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
  92         _gver="$("$cfg_git_bin" version 2>/dev/null | \
  93                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
  94         printf 'var_git_ver=%s\n' "$_gver"
  95         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
  96         printf 'var_sh_bin="%s"\n' "${cfg_posix_sh_bin:-/bin/sh}"
  97         printf 'var_perl_bin="%s"\n' "${cfg_perl_bin:-$(unset -f perl; command -v perl)}"
  98         printf 'var_gzip_bin="%s"\n' "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}"
  99         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
 100         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
 101         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 102         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 103         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 104         __girocco_conf="$GIROCCO_CONF"
 105         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 106         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 107         printf "var_window_memory=%s\n" \
 108                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 109                 -MGirocco::Util -e 'print calc_windowmemory')"
 110         printf "var_big_file_threshold=%s\n" \
 111                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 112                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 113         printf "var_redelta_threshold=%s\n" \
 114                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 115                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 116         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] && \
 117            [ "$cfg_upload_pack_window" -le 50 ]; then
 118                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 119         else
 120                 printf "var_upload_window=%s\n" ""
 121         fi
 122         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 123         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 124         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' | \
 125                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' | \
 126                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort | \
 127                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 128         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 129         # On some broken platforms running xargs without -r and empty input runs the command
 130         printf 'var_xargs_r=%s\n' "$(: | command xargs echo -r)"
 131         # The disk usage report produces better numbers if du has an exclude option
 132         _x0="$(basename "$0")"
 133         _x0="${_x0%?}?*"
 134         for _duopt in --exclude -I; do
 135                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 136                         printf 'var_du_exclude=%s\n' "$_duopt"
 137                         break
 138                 fi
 139         done
 140         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 141                 printf 'var_du_follow=%s\n' "-H"
 142                 break
 143         fi
 144 )
 145
 146 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 147 # definitions from it rather than running Perl.
 148 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 149         # Import all the variables from Girocco::Config to the local environment,
 150         eval "$(get_girocco_config_var_list)"
 151 else
 152         # Import the variables from shlib_vars.sh which avoids needlessly
 153         # running another copy of Perl
 154         . "@basedir@/shlib_vars.sh"
 155 fi
 156
 157 # git_add_config "some.var=value"
 158 # every ' in value must be replaced with the 4-character sequence '\'' before
 159 # calling this function or Git will barf.  Will not be effective unless running
 160 # Git version 1.7.3 or later.
 161 git_add_config() {
 162         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 163         export GIT_CONFIG_PARAMETERS
 164 }
 165
 166 # Make sure we have a reproducible environment by using a controlled HOME dir
 167 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 168 HOME="$cfg_chroot/etc/girocco"
 169 TMPDIR="/tmp"
 170 GIT_CONFIG_NOSYSTEM=1
 171 GIT_ATTR_NOSYSTEM=1
 172 GIT_NO_REPLACE_OBJECTS=1
 173 GIT_TERMINAL_PROMPT=0
 174 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 175 export XDG_CONFIG_HOME
 176 export HOME
 177 export TMPDIR
 178 export GIT_CONFIG_NOSYSTEM
 179 export GIT_ATTR_NOSYSTEM
 180 export GIT_NO_REPLACE_OBJECTS
 181 export GIT_TERMINAL_PROMPT
 182 export GIT_ASKPASS
 183 unset GIT_USER_AGENT
 184 unset GIT_HTTP_USER_AGENT
 185 if [ -n "$defined_cfg_git_client_ua" ]; then
 186         GIT_USER_AGENT="$cfg_git_client_ua"
 187         export GIT_USER_AGENT
 188         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 189         export GIT_HTTP_USER_AGENT
 190 fi
 191 unset GIT_CONFIG_PARAMETERS
 192 git_add_config "core.ignoreCase=false"
 193 if [ -n "$cfg_git_no_mmap" ]; then
 194         # Just like compiling with NO_MMAP
 195         git_add_config "core.packedGitWindowSize=1m"
 196 else
 197         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 198         git_add_config "core.packedGitWindowSize=32m"
 199 fi
 200 [ -z "$var_big_file_threshold" ] ||
 201         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 202
 203 # Make sure any sendmail.pl config is always available
 204 unset SENDMAIL_PL_HOST
 205 unset SENDMAIL_PL_PORT
 206 unset SENDMAIL_PL_NCBIN
 207 unset SENDMAIL_PL_NCOPT
 208 [ -z "$cfg_sendmail_pl_host"  ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host"   && export SENDMAIL_PL_HOST; }
 209 [ -z "$cfg_sendmail_pl_port"  ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port"   && export SENDMAIL_PL_PORT; }
 210 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
 211 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
 212
 213 # Set PATH and PYTHON to the values set by Config.pm, if any
 214 unset PYTHON
 215 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
 216 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
 217
 218 # Extra GIT variables that generally ought to be cleared, but whose clearing
 219 # could potentially interfere with the correct operation of hook scripts so
 220 # they are segregated into a separate function for use as appropriate
 221 clean_git_env() {
 222         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 223         unset GIT_CONFIG
 224         unset GIT_DIR
 225         unset GIT_GRAFT_FILE
 226         unset GIT_INDEX_FILE
 227         unset GIT_OBJECT_DIRECTORY
 228         unset GIT_NAMESPACE
 229 }
 230
 231 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 232 # and nc_openbsd to the desired executables because when using
 233 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 234 # different and unexpected ways:
 235 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 236 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 237 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 238 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 239 # setting only while running func which none of the /bin/sh's do.
 240 #
 241 # Instead we'd like to use an alias that provides the desired behavior without
 242 # any of the bad (a), (b) or (c) effects.
 243 #
 244 # However, unfortunately, some of the crazy /bin/sh implementations do not
 245 # recognize alias expansions when preceded by variable assignments!
 246 #
 247 # So we are left with git() {} and nc_openbsd() {} functions and in the
 248 # case of git() {} we can compensate for (b) and (c) failing to export
 249 # but not (a) and (b) persisting the values so the caller will simply
 250 # have to beware and explicitly unset any variables that should not persist
 251 # beyond the function call itself.
 252
 253 git() (
 254         [ "${GIT_DIR+set}" = "set" ] && export GIT_DIR
 255         [ "${GIT_SSL_NO_VERIFY+set}" = "set" ] && export GIT_SSL_NO_VERIFY
 256         [ "${GIT_TRACE_PACKET+set}" = "set" ] && export GIT_TRACE_PACKET
 257         [ "${GIT_USER_AGENT+set}" = "set" ] && export GIT_USER_AGENT
 258         [ "${GIT_HTTP_USER_AGENT+set}" = "set" ] && export GIT_HTTP_USER_AGENT
 259         exec "$cfg_git_bin" "$@"
 260 )
 261
 262 # see comments for git() -- callers must explicitly export all variables
 263 # intended for the commands these functions run before calling them
 264 perl() { command "${var_perl_bin:-perl}" "$@"; }
 265 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 266
 267 nc_openbsd() { "$cfg_nc_openbsd_bin" "$@"; }
 268
 269 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 270
 271 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 272
 273 # Some platforms' broken xargs runs the command always at least once even if
 274 # there's no input unless given a special option.  Automatically supply the
 275 # option on those platforms by providing an xargs function.
 276 xargs() { command xargs $var_xargs_r "$@"; }
 277
 278 _addrlist() {
 279         _list=
 280         for _addr in "$@"; do
 281                 [ -z "$_list" ] || _list="$_list, "
 282                 _list="$_list$_addr"
 283         done
 284         echo "$_list"
 285 }
 286
 287 _sendmail() {
 288         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 289         if [ -n "$cfg_sender" ]; then
 290                 "$_mailer" -i -f "$cfg_sender" "$@"
 291         else
 292                 "$_mailer" -i "$@"
 293         fi
 294 }
 295
 296 mail() {
 297         _subject=
 298         if [ "$1" = "-s" ]; then
 299                 shift
 300                 _subject="$1"
 301                 shift
 302         fi
 303         {
 304                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 305                 echo "To: $(_addrlist "$@")"
 306                 [ -z "$_subject" ] || echo "Subject: $_subject"
 307                 echo "MIME-Version: 1.0"
 308                 echo "Content-Type: text/plain; charset=utf-8"
 309                 echo "Content-Transfer-Encoding: 8bit"
 310                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 311                 echo "Auto-Submitted: auto-generated"
 312                 echo ""
 313                 cat
 314         } | _sendmail "$@"
 315 }
 316
 317 # bang CMD... will execute the command with well-defined failure mode;
 318 # set bang_action to string of the failed action ('clone', 'update', ...);
 319 # re-define the bang_trap() function to do custom cleanup before bailing out
 320 bang() {
 321         bang_active=1
 322         bang_cmd="$*"
 323         bang_errcode=0
 324         if [ -n "$show_progress" ]; then
 325                 exec 3>&1
 326                 read -r bang_errcode <<-EOT || :
 327                 $(
 328                         exec 4>&3 3>&1 1>&4 4>&-
 329                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 330                 )
 331                 EOT
 332                 exec 3>&-
 333                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 334                         # All right. Cool.
 335                         bang_active=
 336                         bang_cmd=
 337                         return;
 338                 fi
 339         else
 340                 if "$@" >>"$bang_log" 2>&1; then
 341                         # All right. Cool.
 342                         bang_active=
 343                         bang_cmd=
 344                         return;
 345                 else
 346                         bang_errcode="$?"
 347                 fi
 348         fi
 349         bang_failed
 350 }
 351
 352 bang_failed() {
 353         bang_active=
 354         unset GIT_DIR
 355         touch .banged
 356         cat "$bang_log" > .banglog
 357         echo "" >> .banglog
 358         echo "$bang_cmd failed with error code $bang_errcode" >> .banglog
 359         if [ -n "$show_progress" ]; then
 360                 echo ""
 361                 echo "$bang_cmd failed with error code $bang_errcode"
 362         fi
 363         if [ -e .bangagain ]; then
 364                 git config --remove-section girocco.bang 2>/dev/null || :
 365                 rm -f .bangagain
 366         fi
 367         bangcount="$(git config --int girocco.bang.count 2>/dev/null || :)"
 368         : ${bangcount:=0}
 369         bangcount=$(( $bangcount + 1 ))
 370         git config --int girocco.bang.count $bangcount
 371         if [ $bangcount -eq 1 ]; then
 372                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 373         fi
 374         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] && \
 375            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] && \
 376            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 377                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 378                 bangaddrs=''
 379                 [ "$bangmailok" = "false" -o -z "$mail" ] || bangaddrs="$mail"
 380                 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] ||
 381                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 382                 rsubj=
 383                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 384                 [ -z "$bangaddrs" ] ||
 385                 {
 386                         echo "$bang_cmd failed with error code $bang_errcode"
 387                         echo ""
 388                         rsubj=
 389                         if [ $bangcount -gt 1 ]; then
 390                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 391                                 echo ""
 392                         fi
 393                         echo "you will not receive any more notifications until recovery"
 394                         echo "this status message may be disabled on the project admin page"
 395                         echo ""
 396                         echo "Log follows:"
 397                         echo ""
 398                         cat "$bang_log"
 399                 } | mail -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 400                 git config --bool girocco.bang.messagesent true
 401         fi
 402         bangthrottle=
 403         [ $bangcount -lt 15 ] || \
 404                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) || \
 405                 bangthrottle=1
 406         bang_trap $bangthrottle
 407         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 408         exit $bang_errcode
 409 }
 410
 411 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 412 # Identical to bang CMD... except the command is eval'd instead of executed.
 413 bang_eval() {
 414         bang eval "$*"
 415 }
 416
 417 # Default bang settings:
 418 bang_setup() {
 419         bang_active=
 420         bang_action="lame_programmer"
 421         bang_trap() { :; }
 422         bang_log="$(mktemp -t repomgr-XXXXXX)"
 423         is_git_dir . || {
 424                 echo "bang_setup called with current directory not a git directory" >&2
 425                 exit 1
 426         }
 427         trap 'rm -f "$bang_log"' EXIT
 428         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 429         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 430 }
 431
 432 # Remove banged status
 433 bang_reset() {
 434         rm -f .banged .bangagain .banglog
 435         git config --remove-section girocco.bang 2>/dev/null || :
 436 }
 437
 438 # Check to see if banged status
 439 is_banged() {
 440         [ -e .banged ]
 441 }
 442
 443 # Check to see if banged message was sent
 444 was_banged_message_sent() {
 445         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 446 }
 447
 448 # Progress report - if show_progress is set, shows the given message.
 449 progress() {
 450         [ ! -n "$show_progress" ] || echo "$@"
 451 }
 452
 453 # Project config accessors; must be run in project directory
 454 config_get() {
 455         case "$1" in
 456         *.*)
 457                 git config "$1";;
 458         *)
 459                 git config "gitweb.$1";;
 460         esac
 461 }
 462
 463 config_set() {
 464         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 465 }
 466
 467 config_set_raw() {
 468         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 469 }
 470
 471 config_get_date_seconds() {
 472         _dt="$(config_get "$1" || :)"
 473         [ -n "$_dt" ] || return 1
 474         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 475         [ -n "$_ds" ] || return 1
 476         echo "$_ds"
 477 }
 478
 479 # Tool for checking whether given number of seconds has not passed yet
 480 check_interval() {
 481         os="$(config_get_date_seconds "$1")" || return 1
 482         ns="$(date +%s)"
 483         [ $ns -lt $(($os+$2)) ]
 484 }
 485
 486 # Check if we are running with effective root permissions
 487 is_root() {
 488         [ "$(id -u 2>/dev/null)" = "0" ]
 489 }
 490
 491 # Check to see if the single argument is a Git directory
 492 is_git_dir() {
 493         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 494         # And we are slightly more picky (must be refs/.+ not refs/.*)
 495         [ -d "$1/objects" -a -x "$1/objects" ] || return 1
 496         [ -d "$1/refs" -a -x "$1/refs" ] || return 1
 497         if [ -L "$1/HEAD" ]; then
 498                 _hr="$(readlink "$1/HEAD")"
 499                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 500         fi
 501         [ -f "$1/HEAD" -a -r "$1/HEAD" ] || return 1
 502         read -r _hr <"$1/HEAD" || return 1
 503         case "$_hr" in
 504                 $octet20 | ref:refs/?*)
 505                         return 0;;
 506                 ref:*)
 507                         _hr="${_hr##ref:*[ $tab]}"
 508                         case "$_hr" in "refs/"?*) return 0;; esac
 509         esac
 510         return 1
 511 }
 512
 513 # List all Git repositories, with given prefix if specified, one-per-line
 514 # All project names starting with _ are always excluded from the result
 515 get_repo_list() {
 516         if [ -n "$1" ]; then
 517                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 518         else
 519                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 520         fi | while IFS=: read name id; do
 521                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 522         done
 523 }
 524
 525 # Return success if the given project name has any forks
 526 has_forks() {
 527         _prj="${1%.git}"
 528         [ -n "$_prj" ] || return 1
 529         [ -d "$cfg_reporoot/$_prj" ] || return 1
 530         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 531         test $(get_repo_list "$_prj/[^/][^/]*:" | LC_ALL=C wc -l) -gt 0
 532 }
 533
 534 # returns empty string and error for empty string otherwise one of
 535 #   m  =>  normal Git mirror
 536 #   s  =>  mirror from svn source
 537 #   d  =>  mirror from darcs source
 538 #   b  =>  mirror from bzr source
 539 #   h  =>  mirror from hg source
 540 #   w  =>  mirror from mediawiki source
 541 #   f  =>  mirror from other fast-import source
 542 # note that if the string is non-empty and none of s, d, b or h match the
 543 # return will always be type m regardless of whether it's a valid Git URL
 544 get_url_mirror_type() {
 545         case "$1" in
 546                 "")
 547                         return 1
 548                         ;;
 549                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 550                         echo 's'
 551                         ;;
 552                 darcs://*)
 553                         echo 'd'
 554                         ;;
 555                 bzr://*)
 556                         echo 'b'
 557                         ;;
 558                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 559                         echo 'h'
 560                         ;;
 561                 mediawiki::*)
 562                         echo 'w'
 563                         ;;
 564                 *)
 565                         echo 'm'
 566                         ;;
 567         esac
 568         return 0
 569 }
 570
 571 # returns false for empty string
 572 # returns true if the passed in url is a mirror using git fast-import
 573 is_gfi_mirror_url() {
 574         [ -n "$1" ] || return 1
 575         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 576                 d|b|h|w|f)
 577                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 578                         # and so do generic "f" fast-import mirrors
 579                         return 0
 580                         ;;
 581                 *)
 582                         # Don't think git-svn currently uses git fast-import
 583                         # And Git mirrors certainly do not
 584                         return 1
 585                         ;;
 586         esac
 587         # assume it does not use git fast-import
 588         return 1
 589 }
 590
 591 # returns false for empty string
 592 # returns true if the passed in url is a mirror using git-svn
 593 is_svn_mirror_url() {
 594         [ -n "$1" ] || return 1
 595         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 596 }
 597
 598 # returns mirror url for gitweb.baseurl of git directory
 599 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 600 # will fail if the directory does not have .nofetch and gitweb.baseurl
 601 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 602 # automatically strips any leading "disabled " prefix before returning result
 603 get_mirror_url() {
 604         _gitdir="${1:-.}"
 605         # always return empty for non-mirrors
 606         [ ! -e "$_gitdir/.nofetch" ] || return 0
 607         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null || :)"
 608         _url="${_url##* }"
 609         [ -n "$_url" ] || return 1
 610         printf '%s\n' "$_url"
 611         return 0
 612 }
 613
 614 # returns get_url_mirror_type for gitweb.baseurl of git directory
 615 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 616 # will fail if the directory does not have .nofetch and gitweb.baseurl
 617 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 618 # automatically strips any leading "disabled " prefix before testing
 619 get_mirror_type() {
 620         _url="$(get_mirror_url "$@")" || return 1
 621         get_url_mirror_type "$_url"
 622 }
 623
 624 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 625 is_gfi_mirror() {
 626         _url="$(get_mirror_url "$@")" || return 1
 627         is_gfi_mirror_url "$_url"
 628 }
 629
 630 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 631 is_svn_mirror() {
 632         _url="$(get_mirror_url "$@")" || return 1
 633         is_svn_mirror_url "$_url"
 634 }
 635
 636 # current directory must already be set to Git repository
 637 # if girocco.headok is already true succeeds without doing anything
 638 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 639 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 640 # then refs/heads/trunk and finally the first top-level head from
 641 # refs/heads/* (i.e. only two slashes in the name) and finally any
 642 # existing refs/heads.  The first one to succeed wins and sets headok=true
 643 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 644 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 645 # when the repository is being set up -- if the HEAD is later deleted (through
 646 # a push or fetch --prune) that's no longer our responsibility to fix
 647 check_and_set_head() {
 648         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 649         if git rev-parse --verify --quiet HEAD >/dev/null; then
 650                 git config --bool girocco.headok true
 651                 return 0
 652         fi
 653         for _hr in refs/heads/master refs/heads/trunk; do
 654                 if git rev-parse --verify --quiet "$_hr"; then
 655                         _update_head_symref "$_hr"
 656                         return 0
 657                 fi
 658         done
 659         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 660         while read -r _hr; do
 661                 case "${_hr#refs/heads/}" in */*) :;; *)
 662                         _update_head_symref "$_hr"
 663                         exit 1 # exit subshell created by "|"
 664                 esac
 665         done || return 0
 666         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1 || :)"
 667         if [ -n "$_hr" ]; then
 668                 _update_head_symref "$_hr"
 669                 return 0
 670         fi
 671         return 1
 672 }
 673 _update_head_symref() {
 674         git symbolic-ref HEAD "$1"
 675         git config --bool girocco.headok true
 676         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 677 }
 678
 679 # A well-known UTF-8 locale is required for some of the fast-import providers
 680 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 681 # but that is not reliably UTF-8 but rather usually US-ASCII.
 682 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 683 # install time and store that in $var_utf8_locale if one is found.
 684 # If we cannot find one in the `locale -a` output then we just use a well-known
 685 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 686 # it.  We only set this temporarily when running the fast-import providers.
 687 set_utf8_locale() {
 688         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 689         export LC_ALL
 690 }
 691
 692 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 693 git_hg_fetch() (
 694         set_utf8_locale
 695         _python="${PYTHON:-python}"
 696         rm -f hg2git-marks.old hg2git-marks.new
 697         if [ -f hg2git-marks -a -s hg2git-marks ]; then
 698                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 699                         if [ -n "$var_have_git_185" ]; then
 700                                 git cat-file --batch-check=':%(rest) %(objectname)'
 701                         else
 702                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 703                         fi
 704                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 705                 if [ -n "$var_have_git_171" ] && \
 706                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 707                         if [ -z "$var_have_git_185" ] || \
 708                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 709                                 _nm='hg-fast-export'
 710                                 GIT_AUTHOR_NAME="$_nm"
 711                                 GIT_COMMITTER_NAME="$_nm"
 712                                 GIT_AUTHOR_EMAIL="$_nm"
 713                                 GIT_COMMITTER_EMAIL="$_nm"
 714                                 export GIT_AUTHOR_NAME
 715                                 export GIT_COMMITTER_NAME
 716                                 export GIT_AUTHOR_EMAIL
 717                                 export GIT_COMMITTER_EMAIL
 718                                 git notes --ref=refs/notes/hg prune
 719                                 unset GIT_AUTHOR_NAME
 720                                 unset GIT_COMMITTER_NAME
 721                                 unset GIT_AUTHOR_EMAIL
 722                                 unset GIT_COMMITTER_EMAIL
 723                         fi
 724                 fi
 725         else
 726                 >hg2git-marks.old
 727         fi
 728         _err1=
 729         _err2=
 730         exec 3>&1
 731         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 732         $(
 733                 exec 4>&3 3>&1 1>&4 4>&-
 734                 {
 735                         _e1=0
 736                         _af="$(git config hg.authorsfile || :)"
 737                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 738                                 --repo    "$(pwd)/repo.hg" \
 739                                 --marks   "$(pwd)/hg2git-marks.old" \
 740                                 --mapping "$(pwd)/hg2git-mapping" \
 741                                 --heads   "$(pwd)/hg2git-heads" \
 742                                 --status  "$(pwd)/hg2git-state" \
 743                                 -U unknown --force --flatten --hg-hash'
 744                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 745                         eval "$_cmd" 3>&- || _e1=$?
 746                         echo $_e1 >&3
 747                 } | \
 748                 {
 749                         _e2=0
 750                         git fast-import \
 751                                 --import-marks="$(pwd)/hg2git-marks.old" \
 752                                 --export-marks="$(pwd)/hg2git-marks.new" \
 753                                 --export-pack-edges="$(pwd)/gfi-packs" \
 754                                 --force 3>&- || _e2=$?
 755                         echo $_e2 >&3
 756                 }
 757         )
 758         EOT
 759         exec 3>&-
 760         [ "$_err1" = 0 -a "$_err2" = 0 ] || return 1
 761         mv -f hg2git-marks.new hg2git-marks
 762         rm -f hg2git-marks.old
 763         git for-each-ref --format='%(refname) %(objectname)' refs/heads | \
 764         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 765 )