update.sh: tame the wild show_progress=1 beast
[girocco/readme.git] / shlib.sh
blob86ee4f70c3ec0aad1f91234e7acca675dc2b021b
1 #!/bin/sh
3 # This is generic shell library for all the scripts used by Girocco;
4 # most importantly, it introduces all the $cfg_* shell variables.
6 # hash patterns
7 hexdig='[0-9a-f]'
8 octet="$hexdig$hexdig"
9 octet4="$octet$octet$octet$octet"
10 octet19="$octet4$octet4$octet4$octet4$octet$octet$octet"
11 octet20="$octet4$octet4$octet4$octet4$octet4"
12 # tab (single \t between single quotes)
13 tab=' '
15 # set a sane umask that never excludes any user or group permissions
16 umask $(printf '0%03o' $(( $(umask) & ~0770 )) )
18 vcmp() {
19 # Compare $1 to $2 each of which must match \d+(\.\d+)*
20 # An empty string ('') for $1 or $2 is treated like 0
21 # Outputs:
22 # -1 if $1 < $2
23 # 0 if $1 = $2
24 # 1 if $1 > $2
25 # Note that `vcmp 1.8 1.8.0.0.0.0` correctly outputs 0.
26 while
27 _a="${1%%.*}"
28 _b="${2%%.*}"
29 [ -n "$_a" ] || [ -n "$_b" ]
31 if [ "${_a:-0}" -lt "${_b:-0}" ]; then
32 echo -1
33 return
34 elif [ "${_a:-0}" -gt "${_b:-0}" ]; then
35 echo 1
36 return
38 _a2="${1#$_a}"
39 _b2="${2#$_b}"
40 set -- "${_a2#.}" "${_b2#.}"
41 done
42 echo 0
45 unset orig_path
46 get_girocco_config_pm_var_list() (
47 # Export all the variables from Girocco::Config to suitable var= lines
48 # prefixing them with 'cfg_'. E.g. $cfg_admin is admin's mail address now
49 # and also setting a 'defined_cfg_' prefix to 1 if they are not undef.
50 __girocco_conf="$GIROCCO_CONF"
51 [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
52 [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
53 [ -z "$orig_path" ] || { PATH="$orig_path" && export PATH; }
54 perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf -le \
55 'foreach (sort {uc($a) cmp uc($b)} keys %Girocco::Config::) {
56 my $val = ${$Girocco::Config::{$_}}; defined($val) or $val="";
57 $val =~ s/([\\"\$\`])/\\$1/gos;
58 $val =~ s/(?:\r\n|\r|\n)$//os;
59 print "cfg_$_=\"$val\"";
60 print "defined_cfg_$_=",
61 (defined(${$Girocco::Config::{$_}})?"1":"");
65 # Returns full command path for "$1" if it's a valid command otherwise returns "$1"
66 _fcp() {
67 if _fp="$(command -v "$1" 2>/dev/null)"; then
68 printf '%s\n' "$_fp"
69 else
70 printf '%s\n' "$1"
74 get_girocco_config_var_list() (
75 # Same as get_girocco_config_pm_var_list except that
76 # the following variables (all starting with var_) are added:
77 # var_group cfg_owning_group if defined otherwise `id -gn`
78 # var_git_ver The version number part from `git version`
79 # var_git_exec_path The result of $cfg_git_bin --exec-dir
80 # var_sh_bin Full path to the posix sh interpreter to use
81 # var_perl_bin Full path to the perl interpreter to use
82 # var_gzip_bin Full path to the gzip executable to use
83 # var_nc_openbsd_bin Full path to the netcat (nc) with -U support
84 # var_have_git_171 Set to 1 if git version >= 1.7.1 otherwise ''
85 # var_have_git_172 Set to 1 if git version >= 1.7.2 otherwise ''
86 # var_have_git_173 Set to 1 if git version >= 1.7.3 otherwise ''
87 # var_have_git_1710 Set to 1 if git version >= 1.7.10 otherwise ''
88 # var_have_git_185 Set to 1 if git version >= 1.8.5 otherwise ''
89 # var_have_git_210 Set to 1 if git version >= 2.1.0 otherwise ''
90 # var_have_git_260 Set to 1 if git version >= 2.6.0 otherwise ''
91 # var_have_git_2101 Set to 1 if git version >= 2.10.1 otherwise ''
92 # var_window_memory Value to use for repack --window-memory=
93 # var_big_file_threshold Value to use for core.bigFileThreshold
94 # var_redelta_threshold Recompute deltas if no more than this many objs
95 # var_upload_window If not "", pack.window to use for upload-pack
96 # var_log_window_size Value to use for git-svn --log-window-size=
97 # var_utf8_locale Value to use for a UTF-8 locale if available
98 # var_xargs_r A "-r" if xargs needs it to behave correctly
99 # var_du_exclude Option to exclude PATTERN from du if available
100 # var_du_follow Option to follow command line sym links if available
101 _cfg_vars="$(get_girocco_config_pm_var_list)"
102 eval "$_cfg_vars"
103 printf '%s\n' "$_cfg_vars"
104 printf 'var_group=%s\n' "${cfg_owning_group:-$(id -gn)}"
105 _gver="$("$cfg_git_bin" version 2>/dev/null |
106 LC_ALL=C sed -ne 's/^[^0-9]*\([0-9][0-9]*\(\.[0-9][0-9]*\)*\).*$/\1/p')"
107 printf 'var_git_ver=%s\n' "$_gver"
108 printf 'var_git_exec_path="%s"\n' "$("$cfg_git_bin" --exec-path 2>/dev/null)"
109 printf 'var_sh_bin="%s"\n' "$(_fcp "${cfg_posix_sh_bin:-/bin/sh}")"
110 printf 'var_perl_bin="%s"\n' "$(_fcp "${cfg_perl_bin:-$(unset -f perl; command -v perl)}")"
111 printf 'var_gzip_bin="%s"\n' "$(_fcp "${cfg_gzip_bin:-$(unset -f gzip; command -v gzip)}")"
112 printf 'var_nc_openbsd_bin="%s"\n' "$(_fcp "${cfg_nc_openbsd_bin:-$(unset -f nc; command -v nc)}")"
113 printf 'var_have_git_171=%s\n' "$([ $(vcmp "$_gver" 1.7.1) -ge 0 ] && echo 1)"
114 printf 'var_have_git_172=%s\n' "$([ $(vcmp "$_gver" 1.7.2) -ge 0 ] && echo 1)"
115 printf 'var_have_git_173=%s\n' "$([ $(vcmp "$_gver" 1.7.3) -ge 0 ] && echo 1)"
116 printf 'var_have_git_1710=%s\n' "$([ $(vcmp "$_gver" 1.7.10) -ge 0 ] && echo 1)"
117 printf 'var_have_git_185=%s\n' "$([ $(vcmp "$_gver" 1.8.5) -ge 0 ] && echo 1)"
118 printf 'var_have_git_210=%s\n' "$([ $(vcmp "$_gver" 2.1.0) -ge 0 ] && echo 1)"
119 printf 'var_have_git_260=%s\n' "$([ $(vcmp "$_gver" 2.6.0) -ge 0 ] && echo 1)"
120 printf 'var_have_git_2101=%s\n' "$([ $(vcmp "$_gver" 2.10.1) -ge 0 ] && echo 1)"
121 __girocco_conf="$GIROCCO_CONF"
122 [ -n "$__girocco_conf" ] || __girocco_conf="Girocco::Config"
123 [ -z "$basedir" ] || __girocco_extrainc="-I$basedir"
124 printf "var_window_memory=%s\n" \
125 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
126 -MGirocco::Util -e 'print calc_windowmemory')"
127 printf "var_big_file_threshold=%s\n" \
128 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
129 -MGirocco::Util -e 'print calc_bigfilethreshold')"
130 printf "var_redelta_threshold=%s\n" \
131 "$(perl -I@basedir@ $__girocco_extrainc -M$__girocco_conf \
132 -MGirocco::Util -e 'print calc_redeltathreshold')"
133 if [ -n "$cfg_upload_pack_window" ] && [ "$cfg_upload_pack_window" -ge 2 ] &&
134 [ "$cfg_upload_pack_window" -le 50 ]; then
135 printf "var_upload_window=%s\n" "$cfg_upload_pack_window"
136 else
137 printf "var_upload_window=%s\n" ""
139 printf 'var_log_window_size=%s\n' "${cfg_svn_log_window_size:-250}"
140 # We parse the output of `locale -a` and select a suitable UTF-8 locale.
141 _guess_locale="$(locale -a | LC_ALL=C grep -viE '^(posix|c)(\..*)?$' |
142 LC_ALL=C grep -iE '\.utf-?8$' | LC_ALL=C sed -e 's/\.[Uu][Tt][Ff]-*8$//' |
143 LC_ALL=C sed -e '/en_US/ s/^/0 /; /en_US/ !s/^/1 /' | LC_ALL=C sort |
144 head -n 1 | LC_ALL=C cut -d ' ' -f 2)"
145 [ -z "$_guess_locale" ] || printf 'var_utf8_locale=%s.UTF-8\n' "$_guess_locale"
146 # On some broken platforms running xargs without -r and empty input runs the command
147 printf 'var_xargs_r=%s\n' "$(</dev/null command xargs printf %s -r)"
148 # The disk usage report produces better numbers if du has an exclude option
149 _x0="$(basename "$0")"
150 _x0="${_x0%?}?*"
151 for _duopt in --exclude -I; do
152 if _test="$(du $_duopt 's?lib.s*' $_duopt "$_x0" "$0" 2>/dev/null)" && [ -z "$_test" ]; then
153 printf 'var_du_exclude=%s\n' "$_duopt"
154 break
156 done
157 if _test="$(du -H "$0" 2>/dev/null)" && [ -n "$_test" ]; then
158 printf 'var_du_follow=%s\n' "-H"
159 break
163 # If basedir has been replaced, and shlib_vars.sh exists, get the config
164 # definitions from it rather than running Perl.
165 if [ "@basedir@" = '@'basedir'@' ] || ! [ -r "@basedir@/shlib_vars.sh" ]; then
166 # Import all the variables from Girocco::Config to the local environment,
167 eval "$(get_girocco_config_var_list)"
168 else
169 # Import the variables from shlib_vars.sh which avoids needlessly
170 # running another copy of Perl
171 . "@basedir@/shlib_vars.sh"
174 # git_add_config "some.var=value"
175 # every ' in value must be replaced with the 4-character sequence '\'' before
176 # calling this function or Git will barf. Will not be effective unless running
177 # Git version 1.7.3 or later.
178 git_add_config() {
179 GIT_CONFIG_PARAMETERS="${GIT_CONFIG_PARAMETERS:+$GIT_CONFIG_PARAMETERS }'$1'"
180 export GIT_CONFIG_PARAMETERS
183 # Make sure we have a reproducible environment by using a controlled HOME dir
184 XDG_CONFIG_HOME="$cfg_chroot/var/empty"
185 HOME="$cfg_chroot/etc/girocco"
186 TMPDIR="/tmp"
187 GIT_CONFIG_NOSYSTEM=1
188 GIT_ATTR_NOSYSTEM=1
189 GIT_NO_REPLACE_OBJECTS=1
190 GIT_TERMINAL_PROMPT=0
191 GIT_PAGER="cat"
192 PAGER="cat"
193 GIT_ASKPASS="$cfg_basedir/bin/git-askpass-password"
194 GIT_SVN_NOTTY=1
195 GIROCCO_SUPPRESS_AUTO_GC_UPDATE=1
196 export XDG_CONFIG_HOME
197 export HOME
198 export TMPDIR
199 export GIT_CONFIG_NOSYSTEM
200 export GIT_ATTR_NOSYSTEM
201 export GIT_NO_REPLACE_OBJECTS
202 export GIT_TERMINAL_PROMPT
203 export GIT_PAGER
204 export PAGER
205 export GIT_ASKPASS
206 export GIT_SVN_NOTTY
207 export GIROCCO_SUPPRESS_AUTO_GC_UPDATE
208 unset GIT_USER_AGENT
209 unset GIT_HTTP_USER_AGENT
210 if [ -n "$defined_cfg_git_client_ua" ]; then
211 GIT_USER_AGENT="$cfg_git_client_ua"
212 export GIT_USER_AGENT
213 GIT_HTTP_USER_AGENT="$cfg_git_client_ua"
214 export GIT_HTTP_USER_AGENT
216 unset GIT_CONFIG_PARAMETERS
217 git_add_config "core.ignoreCase=false"
218 git_add_config "core.pager=cat"
219 if [ -n "$cfg_git_no_mmap" ]; then
220 # Just like compiling with NO_MMAP
221 git_add_config "core.packedGitWindowSize=1m"
222 else
223 # Always use the 32-bit default (32m) even on 64-bit to avoid memory blowout
224 git_add_config "core.packedGitWindowSize=32m"
226 [ -z "$var_big_file_threshold" ] ||
227 git_add_config "core.bigFileThreshold=$var_big_file_threshold"
229 # Make sure any sendmail.pl config is always available
230 unset SENDMAIL_PL_HOST
231 unset SENDMAIL_PL_PORT
232 unset SENDMAIL_PL_NCBIN
233 unset SENDMAIL_PL_NCOPT
234 [ -z "$cfg_sendmail_pl_host" ] || { SENDMAIL_PL_HOST="$cfg_sendmail_pl_host" && export SENDMAIL_PL_HOST; }
235 [ -z "$cfg_sendmail_pl_port" ] || { SENDMAIL_PL_PORT="$cfg_sendmail_pl_port" && export SENDMAIL_PL_PORT; }
236 [ -z "$cfg_sendmail_pl_ncbin" ] || { SENDMAIL_PL_NCBIN="$cfg_sendmail_pl_ncbin" && export SENDMAIL_PL_NCBIN; }
237 [ -z "$cfg_sendmail_pl_ncopt" ] || { SENDMAIL_PL_NCOPT="$cfg_sendmail_pl_ncopt" && export SENDMAIL_PL_NCOPT; }
239 # Set PATH and PYTHON to the values set by Config.pm, if any
240 unset PYTHON
241 [ -z "$cfg_python" ] || { PYTHON="$cfg_python" && export PYTHON; }
242 [ -z "$cfg_path" ] || { orig_path="$PATH" && PATH="$cfg_path" && export PATH; }
244 # Extra GIT variables that generally ought to be cleared, but whose clearing
245 # could potentially interfere with the correct operation of hook scripts so
246 # they are segregated into a separate function for use as appropriate
247 clean_git_env() {
248 unset GIT_ALTERNATE_OBJECT_DIRECTORIES
249 unset GIT_CONFIG
250 unset GIT_DIR
251 unset GIT_GRAFT_FILE
252 unset GIT_INDEX_FILE
253 unset GIT_OBJECT_DIRECTORY
254 unset GIT_NAMESPACE
257 # We cannot use a git() {} or nc_openbsd() {} function to redirect git
258 # and nc_openbsd to the desired executables because when using
259 # "ENV_VAR=xxx func" the various /bin/sh implementations behave in various
260 # different and unexpected ways:
261 # a) treat "ENV_VAR=xxx" like a separate, preceding "export ENV_VAR=xxx"
262 # b) treat "ENV_VAR=xxx" like a separate, prededing "ENV_VAR=xxx"
263 # c) treat "ENV_VAR=xxx" like a temporary setting only while running func
264 # None of these are good. We want a temporary "export ENV_VAR=xxx"
265 # setting only while running func which none of the /bin/sh's do.
267 # Instead we'd like to use an alias that provides the desired behavior without
268 # any of the bad (a), (b) or (c) effects.
270 # However, unfortunately, some of the crazy /bin/sh implementations do not
271 # recognize alias expansions when preceded by variable assignments!
273 # So we are left with git() {} and nc_openbsd() {} functions and in the
274 # case of git() {} we can compensate for (b) and (c) failing to export
275 # but not (a) and (b) persisting the values so the caller will simply
276 # have to beware and explicitly unset any variables that should not persist
277 # beyond the function call itself.
279 git() (
280 [ z"${GIT_DIR+set}" != z"set" ] || export GIT_DIR
281 [ z"${GIT_SSL_NO_VERIFY+set}" != z"set" ] || export GIT_SSL_NO_VERIFY
282 [ z"${GIT_TRACE_PACKET+set}" != z"set" ] || export GIT_TRACE_PACKET
283 [ z"${GIT_USER_AGENT+set}" != z"set" ] || export GIT_USER_AGENT
284 [ z"${GIT_HTTP_USER_AGENT+set}" != z"set" ] || export GIT_HTTP_USER_AGENT
285 exec "$cfg_git_bin" "$@"
288 # Since we do not yet require at least Git 1.8.5 this is a compatibility function
289 # that allows us to use git update-ref --stdin where supported and the slow shell
290 # script where not, but only the "delete" operation is currently supported.
291 git_updateref_stdin() {
292 if [ -n "$var_have_git_185" ]; then
293 git update-ref --stdin
294 else
295 while read -r _op _ref; do
296 case "$_op" in
297 delete)
298 git update-ref -d "$_ref"
301 echo "bad git_updateref_stdin op: $_op" >&2
302 exit 1
304 esac
305 done
309 # see comments for git() -- callers must explicitly export all variables
310 # intended for the commands these functions run before calling them
311 perl() { command "${var_perl_bin:-perl}" "$@"; }
312 gzip() { command "${var_gzip_bin:-gzip}" "$@"; }
314 nc_openbsd() { command "$var_nc_openbsd_bin" "$@"; }
316 list_packs() { command "$cfg_basedir/bin/list_packs" "$@"; }
318 readlink() { command "$cfg_basedir/bin/readlink" "$@"; }
320 strftime() { command "$cfg_basedir/bin/strftime" "$@"; }
322 # Some platforms' broken xargs runs the command always at least once even if
323 # there's no input unless given a special option. Automatically supply the
324 # option on those platforms by providing an xargs function.
325 xargs() { command xargs $var_xargs_r "$@"; }
327 _addrlist() {
328 _list=
329 for _addr in "$@"; do
330 [ -z "$_list" ] || _list="$_list, "
331 _list="$_list$_addr"
332 done
333 echo "$_list"
336 _sendmail() {
337 _mailer="${cfg_sendmail_bin:-/usr/sbin/sendmail}"
338 if [ -n "$cfg_sender" ]; then
339 "$_mailer" -i -f "$cfg_sender" "$@"
340 else
341 "$_mailer" -i "$@"
345 # First argument is an id WITHOUT surrounding '<' and '>' to use in a
346 # "References:" header. It may be "" to suppress the "References" header.
347 # Following arguments are just like mail function
348 mailref() {
349 _references=
350 if [ $# -ge 1 ]; then
351 _references="$1"
352 shift
354 _subject=
355 if [ "$1" = "-s" ]; then
356 shift
357 _subject="$1"
358 shift
361 echo "From: \"$cfg_name\" ($cfg_title) <$cfg_admin>"
362 echo "To: $(_addrlist "$@")"
363 [ -z "$_subject" ] || echo "Subject: $_subject"
364 echo "MIME-Version: 1.0"
365 echo "Content-Type: text/plain; charset=UTF-8"
366 echo "Content-Transfer-Encoding: 8bit"
367 [ -z "$_references" ] || echo "References: <$_references>"
368 [ -n "$cfg_suppress_x_girocco" ] || echo "X-Girocco: $cfg_gitweburl"
369 echo "Auto-Submitted: auto-generated"
370 echo ""
372 } | _sendmail "$@"
375 # Usage: mail [-s <subject>] <addr> [<addr>...]
376 mail() {
377 mailref "" "$@"
380 # bang CMD... will execute the command with well-defined failure mode;
381 # set bang_action to string of the failed action ('clone', 'update', ...);
382 # re-define the bang_trap() function to do custom cleanup before bailing out
383 bang() {
384 bang_active=1
385 bang_cmd="$*"
386 bang_errcode=0
387 if [ "${show_progress:-0}" != "0" ]; then
388 exec 3>&1
389 read -r bang_errcode <<-EOT || :
391 exec 4>&3 3>&1 1>&4 4>&-
392 { "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
395 exec 3>&-
396 if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
397 # All right. Cool.
398 bang_active=
399 bang_cmd=
400 return;
402 else
403 if "$@" >>"$bang_log" 2>&1; then
404 # All right. Cool.
405 bang_active=
406 bang_cmd=
407 return;
408 else
409 bang_errcode="$?"
412 bang_failed
415 bang_failed() {
416 bang_active=
417 unset GIT_DIR
418 >.banged
419 cat "$bang_log" >.banglog
420 echo "" >>.banglog
421 echo "$bang_cmd failed with error code $bang_errcode" >>.banglog
422 if [ "${show_progress:-0}" != "0" ]; then
423 echo ""
424 echo "$bang_cmd failed with error code $bang_errcode"
426 if [ -e .bangagain ]; then
427 git config --remove-section girocco.bang 2>/dev/null || :
428 rm -f .bangagain
430 bangcount="$(git config --int girocco.bang.count 2>/dev/null)" || :
431 bangcount=$(( ${bangcount:-0} + 1 ))
432 git config --int girocco.bang.count $bangcount
433 if [ $bangcount -eq 1 ]; then
434 git config girocco.bang.firstfail "$(TZ=UTC date "+%Y-%m-%d %T UTC")"
436 if [ $bangcount -ge $cfg_min_mirror_failure_message_count ] &&
437 [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" != "true" ] &&
438 ! check_interval "girocco.bang.firstfail" $cfg_min_mirror_failure_message_interval; then
439 bangmailok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
440 bangaddrs=
441 [ "$bangmailok" = "false" ] || [ -z "$mail" ] || bangaddrs="$mail"
442 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
443 if [ -z "$bangaddrs" ]; then bangaddrs="$cfg_admin"; else bangaddrs="$bangaddrs,$cfg_admin"; fi
444 rsubj=
445 [ $bangcount -le 1 ] || rsubj=" repeatedly"
446 [ -z "$bangaddrs" ] ||
448 echo "$bang_cmd failed with error code $bang_errcode"
449 echo ""
450 rsubj=
451 if [ $bangcount -gt 1 ]; then
452 echo "$bangcount consecutive update failures have occurred since $(config_get girocco.bang.firstfail)"
453 echo ""
455 echo "you will not receive any more notifications until recovery"
456 echo "this status message may be disabled on the project admin page"
457 echo ""
458 echo "Log follows:"
459 echo ""
460 cat "$bang_log"
461 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
462 git config --bool girocco.bang.messagesent true
464 bangthrottle=
465 [ $bangcount -lt 15 ] ||
466 check_interval "girocco.bang.firstfail" $(( $cfg_min_mirror_interval * 3 / 2 )) ||
467 bangthrottle=1
468 bang_trap $bangthrottle
469 [ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
470 exit $bang_errcode
473 # bang_eval CMD... will evaluate the command with well-defined failure mode;
474 # Identical to bang CMD... except the command is eval'd instead of executed.
475 bang_eval() {
476 bang eval "$*"
479 # Default bang settings:
480 bang_setup() {
481 bang_active=
482 bang_action="lame_programmer"
483 bang_trap() { :; }
484 bang_tmpdir="${TMPDIR:-/tmp}"
485 bang_tmpdir="${bang_tmpdir%/}"
486 bang_log="$(mktemp "${bang_tmpdir:-/tmp}/repomgr-XXXXXX")"
487 is_git_dir . || {
488 echo "bang_setup called with current directory not a git directory" >&2
489 exit 1
491 trap 'rm -f "$bang_log"' EXIT
492 trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
493 trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
496 # Remove banged status
497 bang_reset() {
498 rm -f .banged .bangagain .banglog
499 git config --remove-section girocco.bang 2>/dev/null || :
502 # Check to see if banged status
503 is_banged() {
504 [ -e .banged ]
507 # Check to see if banged message was sent
508 was_banged_message_sent() {
509 [ "$(git config --bool girocco.bang.messagesent 2>/dev/null || :)" = "true" ]
512 # Progress report - if show_progress is set, shows the given message.
513 progress() {
514 [ "${show_progress:-0}" = "0" ] || echo "$*"
517 # Project config accessors; must be run in project directory
518 config_get() {
519 case "$1" in
520 *.*)
521 git config "$1";;
523 git config "gitweb.$1";;
524 esac
527 config_set() {
528 git config "gitweb.$1" "$2" && chgrp $var_group config && chmod g+w config
531 config_set_raw() {
532 git config "$1" "$2" && chgrp $var_group config && chmod g+w config
535 config_get_date_seconds() {
536 _dt="$(config_get "$1")" || :
537 [ -n "$_dt" ] || return 1
538 _ds="$(perl -I@basedir@ -MGirocco::Util -e "print parse_any_date('$_dt')")"
539 [ -n "$_ds" ] || return 1
540 echo "$_ds"
543 # Tool for checking whether given number of seconds has not passed yet
544 check_interval() {
545 os="$(config_get_date_seconds "$1")" || return 1
546 ns="$(date +%s)"
547 [ $ns -lt $(($os+$2)) ]
550 # Check if we are running with effective root permissions
551 is_root() {
552 [ "$(id -u 2>/dev/null)" = "0" ]
555 # Check to see if the single argument is a Git directory
556 is_git_dir() {
557 # Just like Git's test except we ignore GIT_OBJECT_DIRECTORY
558 # And we are slightly more picky (must be refs/.+ not refs/.*)
559 [ -d "$1/objects" ] && [ -x "$1/objects" ] || return 1
560 [ -d "$1/refs" ] && [ -x "$1/refs" ] || return 1
561 if [ -L "$1/HEAD" ]; then
562 _hr="$(readlink "$1/HEAD")"
563 case "$_hr" in "refs/"?*) :;; *) return 1;; esac
565 [ -f "$1/HEAD" ] && [ -r "$1/HEAD" ] || return 1
566 read -r _hr <"$1/HEAD" || return 1
567 case "$_hr" in
568 $octet20*)
569 [ "${_hr#*[!0-9a-f]}" = "$_hr" ] || return 1
570 return 0;;
571 ref:refs/?*)
572 return 0;;
573 ref:*)
574 _hr="${_hr##ref:*[ $tab]}"
575 case "$_hr" in "refs/"?*) return 0;; esac
576 esac
577 return 1
580 # List all Git repositories, with given prefix if specified, one-per-line
581 # All project names starting with _ are always excluded from the result
582 get_repo_list() {
583 if [ -n "$1" ]; then
584 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group | LC_ALL=C grep "^$1"
585 else
586 LC_ALL=C cut -d : -f 1,3 "$cfg_chroot"/etc/group
587 fi | while IFS=: read name id; do
588 [ $id -lt 65536 ] || case "$name" in _*) :;; ?*) echo "$name"; esac
589 done
592 # Return success if the given project name has at least one immediate child fork
593 # that has a non-zero length alternates file
594 has_forks_with_alternates() {
595 _prj="${1%.git}"
596 [ -n "$_prj" ] || return 1
597 [ -d "$cfg_reporoot/$_prj" ] || return 1
598 is_git_dir "$cfg_reporoot/$_prj.git" || return 1
600 get_repo_list "$_prj/[^/:][^/:]*:" |
601 while read -r _prjname && [ -n "$_prjname" ]; do
602 ! [ -s "$cfg_reporoot/$_prjname.git/objects/info/alternates" ] ||
603 exit 1 # will only exit implicit subshell created by '|'
604 done
605 then
606 return 1
608 return 0
611 # returns empty string and error for empty string otherwise one of
612 # m => normal Git mirror
613 # s => mirror from svn source
614 # d => mirror from darcs source
615 # b => mirror from bzr source
616 # h => mirror from hg source
617 # w => mirror from mediawiki source
618 # f => mirror from other fast-import source
619 # note that if the string is non-empty and none of s, d, b or h match the
620 # return will always be type m regardless of whether it's a valid Git URL
621 get_url_mirror_type() {
622 case "$1" in
624 return 1
626 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
627 echo 's'
629 darcs://*)
630 echo 'd'
632 bzr://*)
633 echo 'b'
635 hg+http://* | hg+https://* | hg+file://* | hg+ssh://* | hg::*)
636 echo 'h'
638 mediawiki::*)
639 echo 'w'
642 echo 'm'
644 esac
645 return 0
648 # returns false for empty string
649 # returns true if the passed in url is a mirror using git fast-import
650 is_gfi_mirror_url() {
651 [ -n "$1" ] || return 1
652 case "$(get_url_mirror_type "$1" 2>/dev/null || :)" in
653 d|b|h|w|f)
654 # darcs, bzr, hg and mediawiki mirrors use git fast-import
655 # and so do generic "f" fast-import mirrors
656 return 0
659 # Don't think git-svn currently uses git fast-import
660 # And Git mirrors certainly do not
661 return 1
663 esac
664 # assume it does not use git fast-import
665 return 1
668 # returns false for empty string
669 # returns true if the passed in url is a mirror using git-svn
670 is_svn_mirror_url() {
671 [ -n "$1" ] || return 1
672 [ "$(get_url_mirror_type "$1" 2>/dev/null || :)" = "s" ]
675 # returns mirror url for gitweb.baseurl of git directory
676 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
677 # will fail if the directory does not have .nofetch and gitweb.baseurl
678 # comes back empty -- otherwise .nofetch directories succeed with a "" return
679 # automatically strips any leading "disabled " prefix before returning result
680 get_mirror_url() {
681 _gitdir="${1:-.}"
682 # always return empty for non-mirrors
683 ! [ -e "$_gitdir/.nofetch" ] || return 0
684 _url="$(GIT_DIR="$_gitdir" config_get baseurl 2>/dev/null)" || :
685 _url="${_url##* }"
686 [ -n "$_url" ] || return 1
687 printf '%s\n' "$_url"
688 return 0
691 # returns get_url_mirror_type for gitweb.baseurl of git directory
692 # (GIT_DIR) passed in as the argument (which defaults to "." if omitted)
693 # will fail if the directory does not have .nofetch and gitweb.baseurl
694 # comes back empty -- otherwise .nofetch directories succeed with a "" return
695 # automatically strips any leading "disabled " prefix before testing
696 get_mirror_type() {
697 _url="$(get_mirror_url "$@")" || return 1
698 [ -n "$_url" ] || return 0
699 get_url_mirror_type "$_url"
702 # returns true if the passed in git dir (defaults to ".") is a mirror using git fast-import
703 is_gfi_mirror() {
704 _url="$(get_mirror_url "$@")" || return 1
705 is_gfi_mirror_url "$_url"
708 # returns true if the passed in git dir (defaults to ".") is a mirror using git-svn
709 is_svn_mirror() {
710 _url="$(get_mirror_url "$@")" || return 1
711 is_svn_mirror_url "$_url"
714 # current directory must already be set to Git repository
715 # if girocco.headok is already true succeeds without doing anything
716 # if rev-parse --verify HEAD succeeds sets headok=true and succeeds
717 # otherwise tries to set HEAD to a symbolic ref to refs/heads/master
718 # then refs/heads/trunk and finally the first top-level head from
719 # refs/heads/* (i.e. only two slashes in the name) and finally any
720 # existing refs/heads. The first one to succeed wins and sets headok=true
721 # and then a successful exit. Otherwise headok is left unset with a failure exit
722 # We use the girocco.headok flag to make sure we only force a valid HEAD symref
723 # when the repository is being set up -- if the HEAD is later deleted (through
724 # a push or fetch --prune) that's no longer our responsibility to fix
725 check_and_set_head() {
726 [ "$(git config --bool girocco.headok 2>/dev/null || :)" != "true" ] || return 0
727 if git rev-parse --verify --quiet HEAD >/dev/null; then
728 git config --bool girocco.headok true
729 return 0
731 for _hr in refs/heads/master refs/heads/trunk; do
732 if git rev-parse --verify --quiet "$_hr"; then
733 _update_head_symref "$_hr"
734 return 0
736 done
737 git for-each-ref --format="%(refname)" refs/heads 2>/dev/null |
738 while read -r _hr; do
739 case "${_hr#refs/heads/}" in */*) :;; *)
740 _update_head_symref "$_hr"
741 exit 1 # exit subshell created by "|"
742 esac
743 done || return 0
744 _hr="$(git for-each-ref --format="%(refname)" refs/heads 2>/dev/null | head -n 1)" || :
745 if [ -n "$_hr" ]; then
746 _update_head_symref "$_hr"
747 return 0
749 return 1
751 _update_head_symref() {
752 git symbolic-ref HEAD "$1"
753 git config --bool girocco.headok true
754 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
757 # current directory must already be set to Git repository
758 # if the directory needs to have gc run and .needsgc is not already set
759 # then .needsgc will be set triggering a "mini" gc at the next opportunity
760 # Girocco shouldn't generate any loose objects but we check for that anyway
761 check_and_set_needsgc() {
762 ! [ -e .needsgc ] || return 0
763 _packs=
764 { _packs="$(list_packs --quiet --count --exclude-no-idx --exclude-keep objects/pack || :)" || :; } 2>/dev/null
765 if [ "${_packs:-0}" -ge 20 ]; then
766 >.needsgc
767 return 0
769 _logfiles=
770 { _logfiles="$(($(find -L reflogs -maxdepth 1 -type f -print | wc -l || :)+0))" || :; } 2>/dev/null
771 if [ "${_logfiles:-0}" -ge 50 ]; then
772 >.needsgc
773 return 0
775 # Truly git gc only checks the number of objects in the objects/17 directory
776 # We check for -ge 10 which should make the probability of having more than
777 # 5120 (20*256) loose objects present when there are less than 10 in
778 # objects/17 vanishingly small (20 is the threshold we use for pack files)
779 _objfiles=
780 ! [ -d objects/17 ] ||
781 { _objfiles="$(($(find -L objects/17 -type f -name "$octet19*" -print | wc -l || :)+0))" || :; } 2>/dev/null
782 if [ "${_objfiles:-0}" -ge 10 ]; then
783 >.needsgc
784 return 0
788 # A well-known UTF-8 locale is required for some of the fast-import providers
789 # in order to avoid mangling characters. Ideally we could use "POSIX.UTF-8"
790 # but that is not reliably UTF-8 but rather usually US-ASCII.
791 # We parse the output of `locale -a` and select a suitable UTF-8 locale at
792 # install time and store that in $var_utf8_locale if one is found.
793 # If we cannot find one in the `locale -a` output then we just use a well-known
794 # UTF-8 locale and hope for the best. We set LC_ALL to our choice and export
795 # it. We only set this temporarily when running the fast-import providers.
796 set_utf8_locale() {
797 LC_ALL="${var_utf8_locale:-en_US.UTF-8}"
798 export LC_ALL
801 # hg-fast-export | git fast-import with error handling in current directory GIT_DIR
802 git_hg_fetch() (
803 set_utf8_locale
804 _python="${PYTHON:-python}"
805 rm -f hg2git-marks.old hg2git-marks.new
806 if [ -f hg2git-marks ] && [ -s hg2git-marks ]; then
807 LC_ALL=C sed 's/^:\([^ ][^ ]*\) \([^ ][^ ]*\)$/\2 \1/' <hg2git-marks | {
808 if [ -n "$var_have_git_185" ]; then
809 git cat-file --batch-check=':%(rest) %(objectname)'
810 else
811 LC_ALL=C sed 's/^\([^ ][^ ]*\) \([^ ][^ ]*\)$/:\2 \1/'
813 } | LC_ALL=C sed '/ missing$/d' >hg2git-marks.old
814 if [ -n "$var_have_git_171" ] &&
815 git rev-parse --quiet --verify refs/notes/hg >/dev/null; then
816 if [ -z "$var_have_git_185" ] ||
817 ! LC_ALL=C cmp -s hg2git-marks hg2git-marks.old; then
818 _nm='hg-fast-export'
819 GIT_AUTHOR_NAME="$_nm"
820 GIT_COMMITTER_NAME="$_nm"
821 GIT_AUTHOR_EMAIL="$_nm"
822 GIT_COMMITTER_EMAIL="$_nm"
823 export GIT_AUTHOR_NAME
824 export GIT_COMMITTER_NAME
825 export GIT_AUTHOR_EMAIL
826 export GIT_COMMITTER_EMAIL
827 git notes --ref=refs/notes/hg prune
828 unset GIT_AUTHOR_NAME
829 unset GIT_COMMITTER_NAME
830 unset GIT_AUTHOR_EMAIL
831 unset GIT_COMMITTER_EMAIL
834 else
835 >hg2git-marks.old
837 _err1=
838 _err2=
839 exec 3>&1
840 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
842 exec 4>&3 3>&1 1>&4 4>&-
844 _e1=0
845 _af="$(git config hg.authorsfile)" || :
846 _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
847 --repo "$(pwd)/repo.hg" \
848 --marks "$(pwd)/hg2git-marks.old" \
849 --mapping "$(pwd)/hg2git-mapping" \
850 --heads "$(pwd)/hg2git-heads" \
851 --status "$(pwd)/hg2git-state" \
852 -U unknown --force --flatten --hg-hash'
853 [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
854 eval "$_cmd" 3>&- || _e1=$?
855 echo $_e1 >&3
858 _e2=0
859 git fast-import \
860 --import-marks="$(pwd)/hg2git-marks.old" \
861 --export-marks="$(pwd)/hg2git-marks.new" \
862 --export-pack-edges="$(pwd)/gfi-packs" \
863 --force 3>&- || _e2=$?
864 echo $_e2 >&3
868 exec 3>&-
869 [ "$_err1" = 0 ] && [ "$_err2" = 0 ] || return 1
870 mv -f hg2git-marks.new hg2git-marks
871 rm -f hg2git-marks.old
872 git for-each-ref --format='%(refname) %(objectname)' refs/heads |
873 LC_ALL=C sed -e 's,^refs/heads/,:,' >hg2git-heads