shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # SHA-1 patterns
   7 octet='[0-9a-f][0-9a-f]'
   8 octet4="$octet$octet$octet$octet"
   9 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  10 octet20="$octet4$octet4$octet4$octet4$octet4"
  11 nullsha="0000000000000000000000000000000000000000"
  12 # tab
  13 tab="$(printf '\t')"
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" -o -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 get_girocco_config_pm_var_list() {
  46         # Export all the variables from Girocco::Config to suitable var= lines
  47         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  48         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  49         __girocco_conf="$GIROCCO_CONF"
  50         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  51         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  52         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  53                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  54                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  55                         $val =~ s/([\\"\$\`])/\\$1/gos;
  56                         $val =~ s/(?:\r\n|\r|\n)$//os;
  57                         print "cfg_$_=\"$val\"";
  58                         print "defined_cfg_$_=",
  59                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  60                 }'
  61 }
  62
  63 get_girocco_config_var_list() (
  64         # Same as get_girocco_config_pm_var_list except that
  65         # the following variables (all starting with var_) are added:
  66         # var_group             cfg_owning_group if defined otherwise `id -gn`
  67         # var_git_ver           The version number part from `git version`
  68         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  69         # var_sh_bin            Full path to the posix sh interpreter to use
  70         # var_perl_bin          Full path to the perl interpreter to use
  71         # var_gzip_bin          Full path to the gzip executable to use
  72         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  73         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  74         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  75         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  76         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  77         # var_window_memory     Value to use for repack --window-memory=
  78         # var_big_file_threshold Value to use for core.bigFileThreshold
  79         # var_redelta_threshold Recompute deltas if no more than this many objs
  80         # var_upload_window     If not "", pack.window to use for upload-pack
  81         # var_log_window_size   Value to use for git-svn --log-window-size=
  82         # var_utf8_locale       Value to use for a UTF-8 locale if available
  83         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  84         # var_du_exclude        Option to exclude PATTERN from du if available
  85         # var_du_follow         Option to follow command line sym links if available
  86         _cfg_vars="$(get_girocco_config_pm_var_list)"
  87         eval "$_cfg_vars"
  88         printf '%s\n' "$_cfg_vars"
  89         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
  90         _gver="$("$cfg_git_bin" version 2>/dev/null | \
  91                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
  92         printf 'var_git_ver=%s\n' "$_gver"
  93         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
  94         printf 'var_sh_bin="%s"\n' "${cfg_posix_sh_bin:-/bin/sh}"
  95         printf 'var_perl_bin="%s"\n' "${cfg_perl_bin:-$(unset -f perl; command -v perl)}"
  96         printf 'var_gzip_bin="%s"\n' "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}"
  97         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
  98         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
  99         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
 100         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
 101         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
 102         __girocco_conf="$GIROCCO_CONF"
 103         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
 104         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 105         printf "var_window_memory=%s\n" \
 106                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 107                 -MGirocco::Util -e 'print calc_windowmemory')"
 108         printf "var_big_file_threshold=%s\n" \
 109                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 110                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 111         printf "var_redelta_threshold=%s\n" \
 112                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 113                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 114         if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] && \
 115            [ "$cfg_upload_pack_window" -le 50 ]; then
 116                 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
 117         else
 118                 printf "var_upload_window=%s\n" ""
 119         fi
 120         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 121         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 122         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' | \
 123                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' | \
 124                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort | \
 125                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 126         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 127         # On some broken platforms running xargs without -r and empty input runs the command
 128         printf 'var_xargs_r=%s\n' "$(: | command xargs echo -r)"
 129         # The disk usage report produces better numbers if du has an exclude option
 130         _x0="$(basename "$0")"
 131         _x0="${_x0%?}?*"
 132         for _duopt in --exclude -I; do
 133                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 134                         printf 'var_du_exclude=%s\n' "$_duopt"
 135                         break
 136                 fi
 137         done
 138         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 139                 printf 'var_du_follow=%s\n' "-H"
 140                 break
 141         fi
 142 )
 143
 144 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 145 # definitions from it rather than running Perl.
 146 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 147         # Import all the variables from Girocco::Config to the local environment,
 148         eval "$(get_girocco_config_var_list)"
 149 else
 150         # Import the variables from shlib_vars.sh which avoids needlessly
 151         # running another copy of Perl
 152         . "@basedir@/shlib_vars.sh"
 153 fi
 154
 155 # git_add_config "some.var=value"
 156 # every ' in value must be replaced with the 4-character sequence '\'' before
 157 # calling this function or Git will barf.  Will not be effective unless running
 158 # Git version 1.7.3 or later.
 159 git_add_config() {
 160         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 161         export GIT_CONFIG_PARAMETERS
 162 }
 163
 164 # Make sure we have a reproducible environment by using a controlled HOME dir
 165 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 166 HOME="$cfg_chroot/etc/girocco"
 167 TMPDIR="/tmp"
 168 GIT_CONFIG_NOSYSTEM=1
 169 GIT_ATTR_NOSYSTEM=1
 170 GIT_NO_REPLACE_OBJECTS=1
 171 GIT_TERMINAL_PROMPT=0
 172 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 173 export XDG_CONFIG_HOME
 174 export HOME
 175 export TMPDIR
 176 export GIT_CONFIG_NOSYSTEM
 177 export GIT_ATTR_NOSYSTEM
 178 export GIT_NO_REPLACE_OBJECTS
 179 export GIT_TERMINAL_PROMPT
 180 export GIT_ASKPASS
 181 unset GIT_USER_AGENT
 182 unset GIT_HTTP_USER_AGENT
 183 if [ -n "$defined_cfg_git_client_ua" ]; then
 184         GIT_USER_AGENT="$cfg_git_client_ua"
 185         export GIT_USER_AGENT
 186         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 187         export GIT_HTTP_USER_AGENT
 188 fi
 189 unset GIT_CONFIG_PARAMETERS
 190 git_add_config "core.ignoreCase=false"
 191 if [ -n "$cfg_git_no_mmap" ]; then
 192         # Just like compiling with NO_MMAP
 193         git_add_config "core.packedGitWindowSize=1m"
 194 else
 195         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 196         git_add_config "core.packedGitWindowSize=32m"
 197 fi
 198 [ -z "$var_big_file_threshold" ] ||
 199         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 200
 201 # Extra GIT variables that generally ought to be cleared, but whose clearing
 202 # could potentially interfere with the correct operation of hook scripts so
 203 # they are segregated into a separate function for use as appropriate
 204 clean_git_env() {
 205         unset GIT_ALTERNATE_OBJECT_DIRECTORIES
 206         unset GIT_CONFIG
 207         unset GIT_DIR
 208         unset GIT_GRAFT_FILE
 209         unset GIT_INDEX_FILE
 210         unset GIT_OBJECT_DIRECTORY
 211         unset GIT_NAMESPACE
 212 }
 213
 214 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 215 # and nc_openbsd to the desired executables because when using
 216 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 217 # different and unexpected ways:
 218 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 219 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 220 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 221 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 222 # setting only while running func which none of the /bin/sh's do.
 223 #
 224 # Instead we'd like to use an alias that provides the desired behavior without
 225 # any of the bad (a), (b) or (c) effects.
 226 #
 227 # However, unfortunately, some of the crazy /bin/sh implementations do not
 228 # recognize alias expansions when preceded by variable assignments!
 229 #
 230 # So we are left with git() {} and nc_openbsd() {} functions and in the
 231 # case of git() {} we can compensate for (b) and (c) failing to export
 232 # but not (a) and (b) persisting the values so the caller will simply
 233 # have to beware and explicitly unset any variables that should not persist
 234 # beyond the function call itself.
 235
 236 git() (
 237         [ "${GIT_DIR+set}" = "set" ] && export GIT_DIR
 238         [ "${GIT_SSL_NO_VERIFY+set}" = "set" ] && export GIT_SSL_NO_VERIFY
 239         [ "${GIT_TRACE_PACKET+set}" = "set" ] && export GIT_TRACE_PACKET
 240         [ "${GIT_USER_AGENT+set}" = "set" ] && export GIT_USER_AGENT
 241         [ "${GIT_HTTP_USER_AGENT+set}" = "set" ] && export GIT_HTTP_USER_AGENT
 242         exec "$cfg_git_bin" "$@"
 243 )
 244
 245 # see comments for git() -- callers must explicitly export all variables
 246 # intended for the commands these functions run before calling them
 247 perl() { command "${var_perl_bin:-perl}" "$@"; }
 248 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
 249
 250 nc_openbsd() { "$cfg_nc_openbsd_bin" "$@"; }
 251
 252 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 253
 254 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 255
 256 # Some platforms' broken xargs runs the command always at least once even if
 257 # there's no input unless given a special option.  Automatically supply the
 258 # option on those platforms by providing an xargs function.
 259 xargs() { command xargs $var_xargs_r "$@"; }
 260
 261 _addrlist() {
 262         _list=
 263         for _addr in "$@"; do
 264                 [ -z "$_list" ] || _list="$_list, "
 265                 _list="$_list$_addr"
 266         done
 267         echo "$_list"
 268 }
 269
 270 _sendmail() {
 271         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 272         if [ -n "$cfg_sender" ]; then
 273                 "$_mailer" -i -f "$cfg_sender" "$@"
 274         else
 275                 "$_mailer" -i "$@"
 276         fi
 277 }
 278
 279 mail() {
 280         _subject=
 281         if [ "$1" = "-s" ]; then
 282                 shift
 283                 _subject="$1"
 284                 shift
 285         fi
 286         {
 287                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 288                 echo "To: $(_addrlist "$@")"
 289                 [ -z "$_subject" ] || echo "Subject: $_subject"
 290                 echo "MIME-Version: 1.0"
 291                 echo "Content-Type: text/plain; charset=utf-8"
 292                 echo "Content-Transfer-Encoding: 8bit"
 293                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 294                 echo "Auto-Submitted: auto-generated"
 295                 echo ""
 296                 cat
 297         } | _sendmail "$@"
 298 }
 299
 300 # bang CMD... will execute the command with well-defined failure mode;
 301 # set bang_action to string of the failed action ('clone', 'update', ...);
 302 # re-define the bang_trap() function to do custom cleanup before bailing out
 303 bang() {
 304         bang_active=1
 305         bang_cmd="$*"
 306         bang_errcode=0
 307         if [ -n "$show_progress" ]; then
 308                 exec 3>&1
 309                 read -r bang_errcode <<-EOT || :
 310                 $(
 311                         exec 4>&3 3>&1 1>&4 4>&-
 312                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 313                 )
 314                 EOT
 315                 exec 3>&-
 316                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 317                         # All right. Cool.
 318                         bang_active=
 319                         bang_cmd=
 320                         return;
 321                 fi
 322         else
 323                 if "$@" >>"$bang_log" 2>&1; then
 324                         # All right. Cool.
 325                         bang_active=
 326                         bang_cmd=
 327                         return;
 328                 else
 329                         bang_errcode="$?"
 330                 fi
 331         fi
 332         bang_failed
 333 }
 334
 335 bang_failed() {
 336         bang_active=
 337         unset GIT_DIR
 338         touch .banged
 339         cat "$bang_log" > .banglog
 340         echo "" >> .banglog
 341         echo "$bang_cmd failed with error code $bang_errcode" >> .banglog
 342         if [ -n "$show_progress" ]; then
 343                 echo ""
 344                 echo "$bang_cmd failed with error code $bang_errcode"
 345         fi
 346         if [ -e .bangagain ]; then
 347                 git config --remove-section girocco.bang 2>/dev/null || :
 348                 rm -f .bangagain
 349         fi
 350         bangcount="$(git config --int girocco.bang.count 2>/dev/null || :)"
 351         : ${bangcount:=0}
 352         bangcount=$(( $bangcount + 1 ))
 353         git config --int girocco.bang.count $bangcount
 354         if [ $bangcount -eq 1 ]; then
 355                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 356         fi
 357         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] && \
 358            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] && \
 359            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 360                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 361                 bangaddrs=''
 362                 [ "$bangmailok" = "false" -o -z "$mail" ] || bangaddrs="$mail"
 363                 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] ||
 364                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 365                 rsubj=
 366                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 367                 [ -z "$bangaddrs" ] ||
 368                 {
 369                         echo "$bang_cmd failed with error code $bang_errcode"
 370                         echo ""
 371                         rsubj=
 372                         if [ $bangcount -gt 1 ]; then
 373                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 374                                 echo ""
 375                         fi
 376                         echo "you will not receive any more notifications until recovery"
 377                         echo "this status message may be disabled on the project admin page"
 378                         echo ""
 379                         echo "Log follows:"
 380                         echo ""
 381                         cat "$bang_log"
 382                 } | mail -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 383                 git config --bool girocco.bang.messagesent true
 384         fi
 385         bangthrottle=
 386         [ $bangcount -lt 15 ] || \
 387                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) || \
 388                 bangthrottle=1
 389         bang_trap $bangthrottle
 390         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 391         exit $bang_errcode
 392 }
 393
 394 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 395 # Identical to bang CMD... except the command is eval'd instead of executed.
 396 bang_eval() {
 397         bang eval "$*"
 398 }
 399
 400 # Default bang settings:
 401 bang_setup() {
 402         bang_active=
 403         bang_action="lame_programmer"
 404         bang_trap() { :; }
 405         bang_log="$(mktemp -t repomgr-XXXXXX)"
 406         is_git_dir . || {
 407                 echo "bang_setup called with current directory not a git directory" >&2
 408                 exit 1
 409         }
 410         trap 'rm -f "$bang_log"' EXIT
 411         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 412         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 413 }
 414
 415 # Remove banged status
 416 bang_reset() {
 417         rm -f .banged .bangagain .banglog
 418         git config --remove-section girocco.bang 2>/dev/null || :
 419 }
 420
 421 # Check to see if banged status
 422 is_banged() {
 423         [ -e .banged ]
 424 }
 425
 426 # Check to see if banged message was sent
 427 was_banged_message_sent() {
 428         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 429 }
 430
 431 # Progress report - if show_progress is set, shows the given message.
 432 progress() {
 433         [ ! -n "$show_progress" ] || echo "$@"
 434 }
 435
 436 # Project config accessors; must be run in project directory
 437 config_get() {
 438         case "$1" in
 439         *.*)
 440                 git config "$1";;
 441         *)
 442                 git config "gitweb.$1";;
 443         esac
 444 }
 445
 446 config_set() {
 447         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 448 }
 449
 450 config_set_raw() {
 451         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 452 }
 453
 454 config_get_date_seconds() {
 455         _dt="$(config_get "$1" || :)"
 456         [ -n "$_dt" ] || return 1
 457         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 458         [ -n "$_ds" ] || return 1
 459         echo "$_ds"
 460 }
 461
 462 # Tool for checking whether given number of seconds has not passed yet
 463 check_interval() {
 464         os="$(config_get_date_seconds "$1")" || return 1
 465         ns="$(date +%s)"
 466         [ $ns -lt $(($os+$2)) ]
 467 }
 468
 469 # Check if we are running with effective root permissions
 470 is_root() {
 471         [ "$(id -u 2>/dev/null)" = "0" ]
 472 }
 473
 474 # Check to see if the single argument is a Git directory
 475 is_git_dir() {
 476         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 477         # And we are slightly more picky (must be refs/.+ not refs/.*)
 478         [ -d "$1/objects" -a -x "$1/objects" ] || return 1
 479         [ -d "$1/refs" -a -x "$1/refs" ] || return 1
 480         if [ -L "$1/HEAD" ]; then
 481                 _hr="$(readlink "$1/HEAD")"
 482                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 483         fi
 484         [ -f "$1/HEAD" -a -r "$1/HEAD" ] || return 1
 485         read -r _hr <"$1/HEAD" || return 1
 486         case "$_hr" in
 487                 $octet20 | ref:refs/?*)
 488                         return 0;;
 489                 ref:*)
 490                         _hr="${_hr##ref:*[ $tab]}"
 491                         case "$_hr" in "refs/"?*) return 0;; esac
 492         esac
 493         return 1
 494 }
 495
 496 # List all Git repositories, with given prefix if specified, one-per-line
 497 # All project names starting with _ are always excluded from the result
 498 get_repo_list() {
 499         if [ -n "$1" ]; then
 500                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 501         else
 502                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 503         fi | while IFS=: read name id; do
 504                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 505         done
 506 }
 507
 508 # Return success if the given project name has any forks
 509 has_forks() {
 510         _prj="${1%.git}"
 511         [ -n "$_prj" ] || return 1
 512         [ -d "$cfg_reporoot/$_prj" ] || return 1
 513         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 514         test $(get_repo_list "$_prj/[^/][^/]*:" | LC_ALL=C wc -l) -gt 0
 515 }
 516
 517 # returns empty string and error for empty string otherwise one of
 518 #   m  =>  normal Git mirror
 519 #   s  =>  mirror from svn source
 520 #   d  =>  mirror from darcs source
 521 #   b  =>  mirror from bzr source
 522 #   h  =>  mirror from hg source
 523 #   w  =>  mirror from mediawiki source
 524 #   f  =>  mirror from other fast-import source
 525 # note that if the string is non-empty and none of s, d, b or h match the
 526 # return will always be type m regardless of whether it's a valid Git URL
 527 get_url_mirror_type() {
 528         case "$1" in
 529                 "")
 530                         return 1
 531                         ;;
 532                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 533                         echo 's'
 534                         ;;
 535                 darcs://*)
 536                         echo 'd'
 537                         ;;
 538                 bzr://*)
 539                         echo 'b'
 540                         ;;
 541                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 542                         echo 'h'
 543                         ;;
 544                 mediawiki::*)
 545                         echo 'w'
 546                         ;;
 547                 *)
 548                         echo 'm'
 549                         ;;
 550         esac
 551         return 0
 552 }
 553
 554 # returns false for empty string
 555 # returns true if the passed in url is a mirror using git fast-import
 556 is_gfi_mirror_url() {
 557         [ -n "$1" ] || return 1
 558         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 559                 d|b|h|w|f)
 560                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 561                         # and so do generic "f" fast-import mirrors
 562                         return 0
 563                         ;;
 564                 *)
 565                         # Don't think git-svn currently uses git fast-import
 566                         # And Git mirrors certainly do not
 567                         return 1
 568                         ;;
 569         esac
 570         # assume it does not use git fast-import
 571         return 1
 572 }
 573
 574 # returns false for empty string
 575 # returns true if the passed in url is a mirror using git-svn
 576 is_svn_mirror_url() {
 577         [ -n "$1" ] || return 1
 578         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 579 }
 580
 581 # returns mirror url for gitweb.baseurl of git directory
 582 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 583 # will fail if the directory does not have .nofetch and gitweb.baseurl
 584 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 585 # automatically strips any leading "disabled " prefix before returning result
 586 get_mirror_url() {
 587         _gitdir="${1:-.}"
 588         # always return empty for non-mirrors
 589         [ ! -e "$_gitdir/.nofetch" ] || return 0
 590         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null || :)"
 591         _url="${_url##* }"
 592         [ -n "$_url" ] || return 1
 593         printf '%s\n' "$_url"
 594         return 0
 595 }
 596
 597 # returns get_url_mirror_type for gitweb.baseurl of git directory
 598 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 599 # will fail if the directory does not have .nofetch and gitweb.baseurl
 600 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 601 # automatically strips any leading "disabled " prefix before testing
 602 get_mirror_type() {
 603         _url="$(get_mirror_url "$@")" || return 1
 604         get_url_mirror_type "$_url"
 605 }
 606
 607 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 608 is_gfi_mirror() {
 609         _url="$(get_mirror_url "$@")" || return 1
 610         is_gfi_mirror_url "$_url"
 611 }
 612
 613 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 614 is_svn_mirror() {
 615         _url="$(get_mirror_url "$@")" || return 1
 616         is_svn_mirror_url "$_url"
 617 }
 618
 619 # current directory must already be set to Git repository
 620 # if girocco.headok is already true succeeds without doing anything
 621 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 622 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 623 # then refs/heads/trunk and finally the first top-level head from
 624 # refs/heads/* (i.e. only two slashes in the name) and finally any
 625 # existing refs/heads.  The first one to succeed wins and sets headok=true
 626 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 627 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 628 # when the repository is being set up -- if the HEAD is later deleted (through
 629 # a push or fetch --prune) that's no longer our responsibility to fix
 630 check_and_set_head() {
 631         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 632         if git rev-parse --verify --quiet HEAD >/dev/null; then
 633                 git config --bool girocco.headok true
 634                 return 0
 635         fi
 636         for _hr in refs/heads/master refs/heads/trunk; do
 637                 if git rev-parse --verify --quiet "$_hr"; then
 638                         _update_head_symref "$_hr"
 639                         return 0
 640                 fi
 641         done
 642         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 643         while read -r _hr; do
 644                 case "${_hr#refs/heads/}" in */*) :;; *)
 645                         _update_head_symref "$_hr"
 646                         exit 1 # exit subshell created by "|"
 647                 esac
 648         done || return 0
 649         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1 || :)"
 650         if [ -n "$_hr" ]; then
 651                 _update_head_symref "$_hr"
 652                 return 0
 653         fi
 654         return 1
 655 }
 656 _update_head_symref() {
 657         git symbolic-ref HEAD "$1"
 658         git config --bool girocco.headok true
 659         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 660 }
 661
 662 # A well-known UTF-8 locale is required for some of the fast-import providers
 663 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 664 # but that is not reliably UTF-8 but rather usually US-ASCII.
 665 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 666 # install time and store that in $var_utf8_locale if one is found.
 667 # If we cannot find one in the `locale -a` output then we just use a well-known
 668 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 669 # it.  We only set this temporarily when running the fast-import providers.
 670 set_utf8_locale() {
 671         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 672         export LC_ALL
 673 }
 674
 675 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 676 git_hg_fetch() (
 677         set_utf8_locale
 678         _python="${PYTHON:-python}"
 679         rm -f hg2git-marks.old hg2git-marks.new
 680         if [ -f hg2git-marks -a -s hg2git-marks ]; then
 681                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 682                         if [ -n "$var_have_git_185" ]; then
 683                                 git cat-file --batch-check=':%(rest) %(objectname)'
 684                         else
 685                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 686                         fi
 687                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 688                 if [ -n "$var_have_git_171" ] && \
 689                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 690                         if [ -z "$var_have_git_185" ] || \
 691                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 692                                 _nm='hg-fast-export'
 693                                 GIT_AUTHOR_NAME="$_nm"
 694                                 GIT_COMMITTER_NAME="$_nm"
 695                                 GIT_AUTHOR_EMAIL="$_nm"
 696                                 GIT_COMMITTER_EMAIL="$_nm"
 697                                 export GIT_AUTHOR_NAME
 698                                 export GIT_COMMITTER_NAME
 699                                 export GIT_AUTHOR_EMAIL
 700                                 export GIT_COMMITTER_EMAIL
 701                                 git notes --ref=refs/notes/hg prune
 702                                 unset GIT_AUTHOR_NAME
 703                                 unset GIT_COMMITTER_NAME
 704                                 unset GIT_AUTHOR_EMAIL
 705                                 unset GIT_COMMITTER_EMAIL
 706                         fi
 707                 fi
 708         else
 709                 >hg2git-marks.old
 710         fi
 711         _err1=
 712         _err2=
 713         exec 3>&1
 714         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 715         $(
 716                 exec 4>&3 3>&1 1>&4 4>&-
 717                 {
 718                         _e1=0
 719                         _af="$(git config hg.authorsfile || :)"
 720                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 721                                 --repo    "$(pwd)/repo.hg" \
 722                                 --marks   "$(pwd)/hg2git-marks.old" \
 723                                 --mapping "$(pwd)/hg2git-mapping" \
 724                                 --heads   "$(pwd)/hg2git-heads" \
 725                                 --status  "$(pwd)/hg2git-state" \
 726                                 -U unknown --force --flatten --hg-hash'
 727                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 728                         eval "$_cmd" 3>&- || _e1=$?
 729                         echo $_e1 >&3
 730                 } | \
 731                 {
 732                         _e2=0
 733                         git fast-import \
 734                                 --import-marks="$(pwd)/hg2git-marks.old" \
 735                                 --export-marks="$(pwd)/hg2git-marks.new" \
 736                                 --export-pack-edges="$(pwd)/gfi-packs" \
 737                                 --force 3>&- || _e2=$?
 738                         echo $_e2 >&3
 739                 }
 740         )
 741         EOT
 742         exec 3>&-
 743         [ "$_err1" = 0 -a "$_err2" = 0 ] || return 1
 744         mv -f hg2git-marks.new hg2git-marks
 745         rm -f hg2git-marks.old
 746         git for-each-ref --format='%(refname) %(objectname)' refs/heads | \
 747         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 748 )