shlib.sh

   1 #!/bin/sh
   2 #
   3 # This is generic shell library for all the scripts used by Girocco;
   4 # most importantly, it introduces all the $cfg_* shell variables.
   5
   6 # SHA-1 patterns
   7 octet='[0-9a-f][0-9a-f]'
   8 octet4="$octet$octet$octet$octet"
   9 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
  10 octet20="$octet4$octet4$octet4$octet4$octet4"
  11 nullsha="0000000000000000000000000000000000000000"
  12 # tab
  13 tab="$(printf '\t')"
  14
  15 # set a sane umask that never excludes any user or group permissions
  16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
  17
  18 vcmp() {
  19         # Compare $1 to $2 each of which must match \d+(\.\d+)*
  20         # An empty string ('') for $1 or $2 is treated like 0
  21         # Outputs:
  22         #  -1 if $1 < $2
  23         #   0 if $1 = $2
  24         #   1 if $1 > $2
  25         # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
  26         while
  27                 _a="${1%%.*}"
  28                 _b="${2%%.*}"
  29                 [ -n "$_a" -o -n "$_b" ]
  30         do
  31                 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
  32                         echo -1
  33                         return
  34                 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
  35                         echo 1
  36                         return
  37                 fi
  38                 _a2="${1#$_a}"
  39                 _b2="${2#$_b}"
  40                 set -- "${_a2#.}" "${_b2#.}"
  41         done
  42         echo 0
  43 }
  44
  45 get_girocco_config_pm_var_list() {
  46         # Export all the variables from Girocco::Config to suitable var= lines
  47         # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
  48         # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
  49         __girocco_conf="$GIROCCO_CONF"
  50         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  51         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
  52         perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
  53                 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
  54                         my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
  55                         $val =~ s/([\\"\$\`])/\\$1/gos;
  56                         $val =~ s/(?:\r\n|\r|\n)$//os;
  57                         print "cfg_$_=\"$val\"";
  58                         print "defined_cfg_$_=",
  59                                 (defined(${$Girocco::Config::{$_}})?"1":"");
  60                 }'
  61 }
  62
  63 get_girocco_config_var_list() (
  64         # Same as get_girocco_config_pm_var_list except that
  65         # the following variables (all starting with var_) are added:
  66         # var_group             cfg_owning_group if defined otherwise `id -gn`
  67         # var_git_ver           The version number part from `git version`
  68         # var_git_exec_path     The result of $cfg_git_bin --exec-dir
  69         # var_perl_bin          Full path to the perl interpreter to use
  70         # var_have_git_171      Set to 1 if git version >= 1.7.1 otherwise ''
  71         # var_have_git_172      Set to 1 if git version >= 1.7.2 otherwise ''
  72         # var_have_git_173      Set to 1 if git version >= 1.7.3 otherwise ''
  73         # var_have_git_1710     Set to 1 if git version >= 1.7.10 otherwise ''
  74         # var_have_git_185      Set to 1 if git version >= 1.8.5 otherwise ''
  75         # var_window_memory     Value to use for repack --window-memory=
  76         # var_big_file_threshold Value to use for core.bigFileThreshold
  77         # var_redelta_threshold Recompute deltas if no more than this many objs
  78         # var_log_window_size   Value to use for git-svn --log-window-size=
  79         # var_utf8_locale       Value to use for a UTF-8 locale if available
  80         # var_xargs_r           A "-r" if xargs needs it to behave correctly
  81         # var_du_exclude        Option to exclude PATTERN from du if available
  82         # var_du_follow         Option to follow command line sym links if available
  83         _cfg_vars="$(get_girocco_config_pm_var_list)"
  84         eval "$_cfg_vars"
  85         printf '%s\n' "$_cfg_vars"
  86         printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
  87         _gver="$("$cfg_git_bin" version 2>/dev/null | \
  88                 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
  89         printf 'var_git_ver=%s\n' "$_gver"
  90         printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
  91         printf 'var_perl_bin="%s"\n' "${cfg_perl_bin:-$(unset -f perl; command -v perl)}"
  92         printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
  93         printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
  94         printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
  95         printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
  96         printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
  97         __girocco_conf="$GIROCCO_CONF"
  98         [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
  99         [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
 100         printf "var_window_memory=%s\n" \
 101                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 102                 -MGirocco::Util -e 'print calc_windowmemory')"
 103         printf "var_big_file_threshold=%s\n" \
 104                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 105                 -MGirocco::Util -e 'print calc_bigfilethreshold')"
 106         printf "var_redelta_threshold=%s\n" \
 107                 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
 108                 -MGirocco::Util -e 'print calc_redeltathreshold')"
 109         printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
 110         # We parse the output of `locale -a` and select a suitable UTF-8 locale.
 111         _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' | \
 112                 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' | \
 113                 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort | \
 114                 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
 115         [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
 116         # On some broken platforms running xargs without -r and empty input runs the command
 117         printf 'var_xargs_r=%s\n' "$(: | command xargs echo -r)"
 118         # The disk usage report produces better numbers if du has an exclude option
 119         _x0="$(basename "$0")"
 120         _x0="${_x0%?}?*"
 121         for _duopt in --exclude -I; do
 122                 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
 123                         printf 'var_du_exclude=%s\n' "$_duopt"
 124                         break
 125                 fi
 126         done
 127         if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
 128                 printf 'var_du_follow=%s\n' "-H"
 129                 break
 130         fi
 131 )
 132
 133 # If basedir has been replaced, and shlib_vars.sh exists, get the config
 134 # definitions from it rather than running Perl.
 135 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
 136         # Import all the variables from Girocco::Config to the local environment,
 137         eval "$(get_girocco_config_var_list)"
 138 else
 139         # Import the variables from shlib_vars.sh which avoids needlessly
 140         # running another copy of Perl
 141         . "@basedir@/shlib_vars.sh"
 142 fi
 143
 144 # git_add_config "some.var=value"
 145 # every ' in value must be replaced with the 4-character sequence '\'' before
 146 # calling this function or Git will barf.  Will not be effective unless running
 147 # Git version 1.7.3 or later.
 148 git_add_config() {
 149         GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
 150         export GIT_CONFIG_PARAMETERS
 151 }
 152
 153 # Make sure we have a reproducible environment by using a controlled HOME dir
 154 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
 155 HOME="$cfg_chroot/etc/girocco"
 156 TMPDIR="/tmp"
 157 GIT_CONFIG_NOSYSTEM=1
 158 GIT_ATTR_NOSYSTEM=1
 159 GIT_NO_REPLACE_OBJECTS=1
 160 GIT_TERMINAL_PROMPT=0
 161 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
 162 export XDG_CONFIG_HOME
 163 export HOME
 164 export TMPDIR
 165 export GIT_CONFIG_NOSYSTEM
 166 export GIT_ATTR_NOSYSTEM
 167 export GIT_NO_REPLACE_OBJECTS
 168 export GIT_TERMINAL_PROMPT
 169 export GIT_ASKPASS
 170 unset GIT_USER_AGENT
 171 unset GIT_HTTP_USER_AGENT
 172 if [ -n "$defined_cfg_git_client_ua" ]; then
 173         GIT_USER_AGENT="$cfg_git_client_ua"
 174         export GIT_USER_AGENT
 175         GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
 176         export GIT_HTTP_USER_AGENT
 177 fi
 178 unset GIT_CONFIG_PARAMETERS
 179 git_add_config "core.ignoreCase=false"
 180 if [ -n "$cfg_git_no_mmap" ]; then
 181         # Just like compiling with NO_MMAP
 182         git_add_config "core.packedGitWindowSize=1m"
 183 else
 184         # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
 185         git_add_config "core.packedGitWindowSize=32m"
 186 fi
 187 [ -z "$var_big_file_threshold" ] ||
 188         git_add_config "core.bigFileThreshold=$var_big_file_threshold"
 189
 190 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
 191 # and nc_openbsd to the desired executables because when using
 192 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
 193 # different and unexpected ways:
 194 #   a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
 195 #   b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
 196 #   c) treat "ENV_VAR=xxx" like a temporary setting only while running func
 197 # None of these are good.  We want a temporary "export ENV_VAR=xxx"
 198 # setting only while running func which none of the /bin/sh's do.
 199 #
 200 # Instead we'd like to use an alias that provides the desired behavior without
 201 # any of the bad (a), (b) or (c) effects.
 202 #
 203 # However, unfortunately, some of the crazy /bin/sh implementations do not
 204 # recognize alias expansions when preceded by variable assignments!
 205 #
 206 # So we are left with git() {} and nc_openbsd() {} functions and in the
 207 # case of git() {} we can compensate for (b) and (c) failing to export
 208 # but not (a) and (b) persisting the values so the caller will simply
 209 # have to beware and explicitly unset any variables that should not persist
 210 # beyond the function call itself.
 211
 212 git() (
 213         [ "${GIT_DIR+set}" = "set" ] && export GIT_DIR
 214         [ "${GIT_SSL_NO_VERIFY+set}" = "set" ] && export GIT_SSL_NO_VERIFY
 215         [ "${GIT_TRACE_PACKET+set}" = "set" ] && export GIT_TRACE_PACKET
 216         [ "${GIT_USER_AGENT+set}" = "set" ] && export GIT_USER_AGENT
 217         [ "${GIT_HTTP_USER_AGENT+set}" = "set" ] && export GIT_HTTP_USER_AGENT
 218         exec "$cfg_git_bin" "$@"
 219 )
 220
 221 # see comments for git() -- callers must explicitly export all variables
 222 # intended for the perl interpreter before using this function
 223 perl() { command "${var_perl_bin:-perl}" "$@"; }
 224
 225 nc_openbsd() { "$cfg_nc_openbsd_bin" "$@"; }
 226
 227 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
 228
 229 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
 230
 231 # Some platforms' broken xargs runs the command always at least once even if
 232 # there's no input unless given a special option.  Automatically supply the
 233 # option on those platforms by providing an xargs function.
 234 xargs() { command xargs $var_xargs_r "$@"; }
 235
 236 _addrlist() {
 237         _list=
 238         for _addr in "$@"; do
 239                 [ -z "$_list" ] || _list="$_list, "
 240                 _list="$_list$_addr"
 241         done
 242         echo "$_list"
 243 }
 244
 245 _sendmail() {
 246         _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
 247         if [ -n "$cfg_sender" ]; then
 248                 "$_mailer" -i -f "$cfg_sender" "$@"
 249         else
 250                 "$_mailer" -i "$@"
 251         fi
 252 }
 253
 254 mail() {
 255         _subject=
 256         if [ "$1" = "-s" ]; then
 257                 shift
 258                 _subject="$1"
 259                 shift
 260         fi
 261         {
 262                 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
 263                 echo "To: $(_addrlist "$@")"
 264                 [ -z "$_subject" ] || echo "Subject: $_subject"
 265                 echo "MIME-Version: 1.0"
 266                 echo "Content-Type: text/plain; charset=utf-8"
 267                 echo "Content-Transfer-Encoding: 8bit"
 268                 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
 269                 echo "Auto-Submitted: auto-generated"
 270                 echo ""
 271                 cat
 272         } | _sendmail "$@"
 273 }
 274
 275 # bang CMD... will execute the command with well-defined failure mode;
 276 # set bang_action to string of the failed action ('clone', 'update', ...);
 277 # re-define the bang_trap() function to do custom cleanup before bailing out
 278 bang() {
 279         bang_active=1
 280         bang_cmd="$*"
 281         bang_errcode=0
 282         if [ -n "$show_progress" ]; then
 283                 exec 3>&1
 284                 read -r bang_errcode <<-EOT || :
 285                 $(
 286                         exec 4>&3 3>&1 1>&4 4>&-
 287                         { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 288                 )
 289                 EOT
 290                 exec 3>&-
 291                 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 292                         # All right. Cool.
 293                         bang_active=
 294                         bang_cmd=
 295                         return;
 296                 fi
 297         else
 298                 if "$@" >>"$bang_log" 2>&1; then
 299                         # All right. Cool.
 300                         bang_active=
 301                         bang_cmd=
 302                         return;
 303                 else
 304                         bang_errcode="$?"
 305                 fi
 306         fi
 307         bang_failed
 308 }
 309
 310 bang_failed() {
 311         bang_active=
 312         unset GIT_DIR
 313         touch .banged
 314         cat "$bang_log" > .banglog
 315         echo "" >> .banglog
 316         echo "$bang_cmd failed with error code $bang_errcode" >> .banglog
 317         if [ -n "$show_progress" ]; then
 318                 echo ""
 319                 echo "$bang_cmd failed with error code $bang_errcode"
 320         fi
 321         if [ -e .bangagain ]; then
 322                 git config --remove-section girocco.bang 2>/dev/null || :
 323                 rm -f .bangagain
 324         fi
 325         bangcount="$(git config --int girocco.bang.count 2>/dev/null || :)"
 326         : ${bangcount:=0}
 327         bangcount=$(( $bangcount + 1 ))
 328         git config --int girocco.bang.count $bangcount
 329         if [ $bangcount -eq 1 ]; then
 330                 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
 331         fi
 332         if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] && \
 333            [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] && \
 334            ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
 335                 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
 336                 bangaddrs=''
 337                 [ "$bangmailok" = "false" -o -z "$mail" ] || bangaddrs="$mail"
 338                 [ -z "$cfg_admincc" -o "$cfg_admincc" = "0" -o -z "$cfg_admin" ] ||
 339                 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
 340                 rsubj=
 341                 [ $bangcount -le 1 ] || rsubj=" repeatedly"
 342                 [ -z "$bangaddrs" ] ||
 343                 {
 344                         echo "$bang_cmd failed with error code $bang_errcode"
 345                         echo ""
 346                         rsubj=
 347                         if [ $bangcount -gt 1 ]; then
 348                                 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
 349                                 echo ""
 350                         fi
 351                         echo "you will not receive any more notifications until recovery"
 352                         echo "this status message may be disabled on the project admin page"
 353                         echo ""
 354                         echo "Log follows:"
 355                         echo ""
 356                         cat "$bang_log"
 357                 } | mail -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 358                 git config --bool girocco.bang.messagesent true
 359         fi
 360         bangthrottle=
 361         [ $bangcount -lt 15 ] || \
 362                 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) || \
 363                 bangthrottle=1
 364         bang_trap $bangthrottle
 365         [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
 366         exit $bang_errcode
 367 }
 368
 369 # bang_eval CMD... will evaluate the command with well-defined failure mode;
 370 # Identical to bang CMD... except the command is eval'd instead of executed.
 371 bang_eval() {
 372         bang eval "$*"
 373 }
 374
 375 # Default bang settings:
 376 bang_setup() {
 377         bang_active=
 378         bang_action="lame_programmer"
 379         bang_trap() { :; }
 380         bang_log="$(mktemp -t repomgr-XXXXXX)"
 381         is_git_dir . || {
 382                 echo "bang_setup called with current directory not a git directory" >&2
 383                 exit 1
 384         }
 385         trap 'rm -f "$bang_log"' EXIT
 386         trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
 387         trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 388 }
 389
 390 # Remove banged status
 391 bang_reset() {
 392         rm -f .banged .bangagain .banglog
 393         git config --remove-section girocco.bang 2>/dev/null || :
 394 }
 395
 396 # Check to see if banged status
 397 is_banged() {
 398         [ -e .banged ]
 399 }
 400
 401 # Check to see if banged message was sent
 402 was_banged_message_sent() {
 403         [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
 404 }
 405
 406 # Progress report - if show_progress is set, shows the given message.
 407 progress() {
 408         [ ! -n "$show_progress" ] || echo "$@"
 409 }
 410
 411 # Project config accessors; must be run in project directory
 412 config_get() {
 413         case "$1" in
 414         *.*)
 415                 git config "$1";;
 416         *)
 417                 git config "gitweb.$1";;
 418         esac
 419 }
 420
 421 config_set() {
 422         git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
 423 }
 424
 425 config_set_raw() {
 426         git config "$1" "$2"        && chgrp $var_group config && chmod g+w config
 427 }
 428
 429 config_get_date_seconds() {
 430         _dt="$(config_get "$1" || :)"
 431         [ -n "$_dt" ] || return 1
 432         _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
 433         [ -n "$_ds" ] || return 1
 434         echo "$_ds"
 435 }
 436
 437 # Tool for checking whether given number of seconds has not passed yet
 438 check_interval() {
 439         os="$(config_get_date_seconds "$1")" || return 1
 440         ns="$(date +%s)"
 441         [ $ns -lt $(($os+$2)) ]
 442 }
 443
 444 # Check if we are running with effective root permissions
 445 is_root() {
 446         [ "$(id -u 2>/dev/null)" = "0" ]
 447 }
 448
 449 # Check to see if the single argument is a Git directory
 450 is_git_dir() {
 451         # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
 452         # And we are slightly more picky (must be refs/.+ not refs/.*)
 453         [ -d "$1/objects" -a -x "$1/objects" ] || return 1
 454         [ -d "$1/refs" -a -x "$1/refs" ] || return 1
 455         if [ -L "$1/HEAD" ]; then
 456                 _hr="$(readlink "$1/HEAD")"
 457                 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
 458         fi
 459         [ -f "$1/HEAD" -a -r "$1/HEAD" ] || return 1
 460         read -r _hr <"$1/HEAD" || return 1
 461         case "$_hr" in
 462                 $octet20 | ref:refs/?*)
 463                         return 0;;
 464                 ref:*)
 465                         _hr="${_hr##ref:*[ $tab]}"
 466                         case "$_hr" in "refs/"?*) return 0;; esac
 467         esac
 468         return 1
 469 }
 470
 471 # List all Git repositories, with given prefix if specified, one-per-line
 472 # All project names starting with _ are always excluded from the result
 473 get_repo_list() {
 474         if [ -n "$1" ]; then
 475                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
 476         else
 477                 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
 478         fi | while IFS=: read name id; do
 479                 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
 480         done
 481 }
 482
 483 # Return success if the given project name has any forks
 484 has_forks() {
 485         _prj="${1%.git}"
 486         [ -n "$_prj" ] || return 1
 487         [ -d "$cfg_reporoot/$_prj" ] || return 1
 488         is_git_dir "$cfg_reporoot/$_prj.git" || return 1
 489         test $(get_repo_list "$_prj/[^/][^/]*:" | LC_ALL=C wc -l) -gt 0
 490 }
 491
 492 # returns empty string and error for empty string otherwise one of
 493 #   m  =>  normal Git mirror
 494 #   s  =>  mirror from svn source
 495 #   d  =>  mirror from darcs source
 496 #   b  =>  mirror from bzr source
 497 #   h  =>  mirror from hg source
 498 #   w  =>  mirror from mediawiki source
 499 #   f  =>  mirror from other fast-import source
 500 # note that if the string is non-empty and none of s, d, b or h match the
 501 # return will always be type m regardless of whether it's a valid Git URL
 502 get_url_mirror_type() {
 503         case "$1" in
 504                 "")
 505                         return 1
 506                         ;;
 507                 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
 508                         echo 's'
 509                         ;;
 510                 darcs://*)
 511                         echo 'd'
 512                         ;;
 513                 bzr://*)
 514                         echo 'b'
 515                         ;;
 516                 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
 517                         echo 'h'
 518                         ;;
 519                 mediawiki::*)
 520                         echo 'w'
 521                         ;;
 522                 *)
 523                         echo 'm'
 524                         ;;
 525         esac
 526         return 0
 527 }
 528
 529 # returns false for empty string
 530 # returns true if the passed in url is a mirror using git fast-import
 531 is_gfi_mirror_url() {
 532         [ -n "$1" ] || return 1
 533         case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
 534                 d|b|h|w|f)
 535                         # darcs, bzr, hg and mediawiki mirrors use git fast-import
 536                         # and so do generic "f" fast-import mirrors
 537                         return 0
 538                         ;;
 539                 *)
 540                         # Don't think git-svn currently uses git fast-import
 541                         # And Git mirrors certainly do not
 542                         return 1
 543                         ;;
 544         esac
 545         # assume it does not use git fast-import
 546         return 1
 547 }
 548
 549 # returns false for empty string
 550 # returns true if the passed in url is a mirror using git-svn
 551 is_svn_mirror_url() {
 552         [ -n "$1" ] || return 1
 553         [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
 554 }
 555
 556 # returns mirror url for gitweb.baseurl of git directory
 557 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 558 # will fail if the directory does not have .nofetch and gitweb.baseurl
 559 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 560 # automatically strips any leading "disabled " prefix before returning result
 561 get_mirror_url() {
 562         _gitdir="${1:-.}"
 563         # always return empty for non-mirrors
 564         [ ! -e "$_gitdir/.nofetch" ] || return 0
 565         _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null || :)"
 566         _url="${_url##* }"
 567         [ -n "$_url" ] || return 1
 568         printf '%s\n' "$_url"
 569         return 0
 570 }
 571
 572 # returns get_url_mirror_type for gitweb.baseurl of git directory
 573 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
 574 # will fail if the directory does not have .nofetch and gitweb.baseurl
 575 # comes back empty -- otherwise .nofetch directories succeed with a "" return
 576 # automatically strips any leading "disabled " prefix before testing
 577 get_mirror_type() {
 578         _url="$(get_mirror_url "$@")" || return 1
 579         get_url_mirror_type "$_url"
 580 }
 581
 582 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
 583 is_gfi_mirror() {
 584         _url="$(get_mirror_url "$@")" || return 1
 585         is_gfi_mirror_url "$_url"
 586 }
 587
 588 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
 589 is_svn_mirror() {
 590         _url="$(get_mirror_url "$@")" || return 1
 591         is_svn_mirror_url "$_url"
 592 }
 593
 594 # current directory must already be set to Git repository
 595 # if girocco.headok is already true succeeds without doing anything
 596 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
 597 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
 598 # then refs/heads/trunk and finally the first top-level head from
 599 # refs/heads/* (i.e. only two slashes in the name) and finally any
 600 # existing refs/heads.  The first one to succeed wins and sets headok=true
 601 # and then a successful exit.  Otherwise headok is left unset with a failure exit
 602 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
 603 # when the repository is being set up -- if the HEAD is later deleted (through
 604 # a push or fetch --prune) that's no longer our responsibility to fix
 605 check_and_set_head() {
 606         [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
 607         if git rev-parse --verify --quiet HEAD >/dev/null; then
 608                 git config --bool girocco.headok true
 609                 return 0
 610         fi
 611         for _hr in refs/heads/master refs/heads/trunk; do
 612                 if git rev-parse --verify --quiet "$_hr"; then
 613                         _update_head_symref "$_hr"
 614                         return 0
 615                 fi
 616         done
 617         git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
 618         while read -r _hr; do
 619                 case "${_hr#refs/heads/}" in */*) :;; *)
 620                         _update_head_symref "$_hr"
 621                         exit 1 # exit subshell created by "|"
 622                 esac
 623         done || return 0
 624         _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1 || :)"
 625         if [ -n "$_hr" ]; then
 626                 _update_head_symref "$_hr"
 627                 return 0
 628         fi
 629         return 1
 630 }
 631 _update_head_symref() {
 632         git symbolic-ref HEAD "$1"
 633         git config --bool girocco.headok true
 634         ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
 635 }
 636
 637 # A well-known UTF-8 locale is required for some of the fast-import providers
 638 # in order to avoid mangling characters.  Ideally we could use "POSIX.UTF-8"
 639 # but that is not reliably UTF-8 but rather usually US-ASCII.
 640 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
 641 # install time and store that in $var_utf8_locale if one is found.
 642 # If we cannot find one in the `locale -a` output then we just use a well-known
 643 # UTF-8 locale and hope for the best.  We set LC_ALL to our choice and export
 644 # it.  We only set this temporarily when running the fast-import providers.
 645 set_utf8_locale() {
 646         LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
 647         export LC_ALL
 648 }
 649
 650 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
 651 git_hg_fetch() (
 652         set_utf8_locale
 653         _python="${PYTHON:-python}"
 654         rm -f hg2git-marks.old hg2git-marks.new
 655         if [ -f hg2git-marks -a -s hg2git-marks ]; then
 656                 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
 657                         if [ -n "$var_have_git_185" ]; then
 658                                 git cat-file --batch-check=':%(rest) %(objectname)'
 659                         else
 660                                 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
 661                         fi
 662                 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
 663                 if [ -n "$var_have_git_171" ] && \
 664                    git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
 665                         if [ -z "$var_have_git_185" ] || \
 666                            ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
 667                                 _nm='hg-fast-export'
 668                                 GIT_AUTHOR_NAME="$_nm"
 669                                 GIT_COMMITTER_NAME="$_nm"
 670                                 GIT_AUTHOR_EMAIL="$_nm"
 671                                 GIT_COMMITTER_EMAIL="$_nm"
 672                                 export GIT_AUTHOR_NAME
 673                                 export GIT_COMMITTER_NAME
 674                                 export GIT_AUTHOR_EMAIL
 675                                 export GIT_COMMITTER_EMAIL
 676                                 git notes --ref=refs/notes/hg prune
 677                                 unset GIT_AUTHOR_NAME
 678                                 unset GIT_COMMITTER_NAME
 679                                 unset GIT_AUTHOR_EMAIL
 680                                 unset GIT_COMMITTER_EMAIL
 681                         fi
 682                 fi
 683         else
 684                 >hg2git-marks.old
 685         fi
 686         _err1=
 687         _err2=
 688         exec 3>&1
 689         { read -r _err1 || :; read -r _err2 || :; } <<-EOT
 690         $(
 691                 exec 4>&3 3>&1 1>&4 4>&-
 692                 {
 693                         _e1=0
 694                         _af="$(git config hg.authorsfile || :)"
 695                         _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
 696                                 --repo    "$(pwd)/repo.hg" \
 697                                 --marks   "$(pwd)/hg2git-marks.old" \
 698                                 --mapping "$(pwd)/hg2git-mapping" \
 699                                 --heads   "$(pwd)/hg2git-heads" \
 700                                 --status  "$(pwd)/hg2git-state" \
 701                                 -U unknown --force --flatten --hg-hash'
 702                         [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
 703                         eval "$_cmd" 3>&- || _e1=$?
 704                         echo $_e1 >&3
 705                 } | \
 706                 {
 707                         _e2=0
 708                         git fast-import \
 709                                 --import-marks="$(pwd)/hg2git-marks.old" \
 710                                 --export-marks="$(pwd)/hg2git-marks.new" \
 711                                 --export-pack-edges="$(pwd)/gfi-packs" \
 712                                 --force 3>&- || _e2=$?
 713                         echo $_e2 >&3
 714                 }
 715         )
 716         EOT
 717         exec 3>&-
 718         [ "$_err1" = 0 -a "$_err2" = 0 ] || return 1
 719         mv -f hg2git-marks.new hg2git-marks
 720         rm -f hg2git-marks.old
 721         git for-each-ref --format='%(refname) %(objectname)' refs/heads | \
 722         LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads
 723 )