jailsetup.sh: set default diff.renameLimit=250
[girocco/readme.git] / jobd / update.sh
blob2efed86303e050a9ff9a28d8b37685fd79a4f998
1 #!/bin/sh
3 . @basedir@/shlib.sh
5 set -e
7 if [ $# -ne 1 ]; then
8 echo "Usage: update.sh projname" >&2
9 exit 1
12 # date -R is linux-only, POSIX equivalent is '+%a, %d %b %Y %T %z'
13 datefmt='+%a, %d %b %Y %T %z'
15 git_fetch_q_progress() {
16 PATH="$var_git_exec_path:$cfg_basedir/bin:$PATH" @basedir@/jobd/git-fetch-q-progress.sh "$@"
19 # freshen_loose_objects full-sha ...
20 # if "$n" is a loose object, set its modification time to now
21 # otherwise silently do nothing with no error. To facilitate conversion
22 # of mirror projects to push projects we also add group write permission.
23 freshen_loose_objects() {
24 _list=
25 for _sha; do
26 _fn="${_sha#??}"
27 _shard="${_sha%$_fn}"
28 _list="$_list objects/$_shard/$_fn"
29 done
30 if [ -n "$_list" ]; then
31 chmod ug+w $_list 2>/dev/null || :
32 touch -c $_list 2>/dev/null || :
36 # darcs fast-export | git fast-import with error handling
37 git_darcs_fetch() (
38 set_utf8_locale
39 _err1=
40 _err2=
41 exec 3>&1
42 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
44 exec 4>&3 3>&1 1>&4 4>&-
46 _e1=0
47 "$cfg_basedir"/bin/darcs-fast-export \
48 --export-marks="$(pwd)/dfe-marks" \
49 --import-marks="$(pwd)/dfe-marks" "$1" 3>&- || _e1=$?
50 echo $_e1 >&3
51 } |
53 _e2=0
54 git fast-import \
55 --export-marks="$(pwd)/gfi-marks" \
56 --export-pack-edges="$(pwd)/gfi-packs" \
57 --import-marks="$(pwd)/gfi-marks" \
58 --force 3>&- || _e2=$?
59 echo $_e2 >&3
62 EOT
63 exec 3>&-
64 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
65 return $?
68 # bzr fast-export | git fast-import with error handling
69 git_bzr_fetch() (
70 set_utf8_locale
71 BZR_LOG=/dev/null
72 export BZR_LOG
73 _err1=
74 _err2=
75 exec 3>&1
76 { read -r _err1 || :; read -r _err2 || :; } <<-EOT
78 exec 4>&3 3>&1 1>&4 4>&-
80 _e1=0
81 bzr fast-export --plain \
82 --export-marks="$(pwd)/bfe-marks" \
83 --import-marks="$(pwd)/bfe-marks" "$1" 3>&- || _e1=$?
84 echo $_e1 >&3
85 } |
87 _e2=0
88 git fast-import \
89 --export-marks="$(pwd)/gfi-marks" \
90 --export-pack-edges="$(pwd)/gfi-packs" \
91 --import-marks="$(pwd)/gfi-marks" \
92 --force 3>&- || _e2=$?
93 echo $_e2 >&3
96 EOT
97 exec 3>&-
98 [ "$_err1" = 0 ] && [ "$_err2" = 0 ]
99 return $?
102 [ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 0; }
104 umask 002
105 [ "$cfg_permission_control" != "Hooks" ] || umask 000
106 clean_git_env
108 proj="${1%.git}"
109 cd "$cfg_reporoot/$proj.git"
111 # Activate a mini-gc if needed
112 # We do this here as well as after a successful fetch so that if we're stuck
113 # in a fetch loop where fetches are succeeding in fetching new packs but the
114 # ref update is failing for some reason (perhaps a non-commit under refs/heads)
115 # and a previous invokation therefore had a "bang" exit then we will still
116 # get the .needsgc flag set in a timely fashion to avoid excess pack build up.
117 check_and_set_needsgc
119 trap 'if [ $? != 0 ]; then echo "update failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT
120 trap 'exit 130' INT
121 trap 'exit 143' TERM
123 if [ "${force_update:-0}" = "0" ] && check_interval lastrefresh $cfg_min_mirror_interval; then
124 progress "= [$proj] update skip (last at $(config_get lastrefresh))"
125 exit 0
127 if [ -e .nofetch ]; then
128 progress "x [$proj] update disabled (.nofetch exists)"
129 exit 0
131 progress "+ [$proj] update ($(date))"
133 # Any pre-existing FETCH_HEAD from a previous fetch, failed or not, is garbage
134 rm -f FETCH_HEAD
136 # A previous failed update attempt can leave a huge tmp_pack_XXXXXX file behind.
137 # Since no pushes are allowed to mirrors, we know that any such files that exist
138 # at this point in time are garbage and can be safely deleted, we do not even
139 # need to check how old they are. A tmp_idx_XXXXXX file is also created during
140 # the later stages of the fetch process, so we kill any of those as well.
141 find -L objects/pack -maxdepth 1 -type f -name "tmp_pack_?*" -exec rm -f '{}' + || :
142 find -L objects/pack -maxdepth 1 -type f -name "tmp_idx_?*" -exec rm -f '{}' + || :
144 # Make sure we have a reflogs subdirectory and abort the update if not
145 # This should not count as a normal "bang" failure if unsuccessful
146 [ -d reflogs ] || mkdir -p reflogs >/dev/null 2>&1 || :
147 [ -d reflogs ]
149 keep_bang_log=
150 do_check_after_refs=1
151 bang_setup
152 bang_action="update"
153 bang_trap() {
154 if [ -n "$1" ]; then
155 # Throttle retries
156 # Since gitweb shows the .last_refresh date, it's safe to update
157 # gitweb.lastrefresh to throttle the updates w/o corrupting the
158 # last refresh date display on the gitweb summary page
159 # It's therefore important that we do NOT touch .last_refresh here
160 config_set lastrefresh "$(date "$datefmt")"
164 bang echo "Project: $proj"
165 bang echo " Date: $(TZ=UTC date '+%Y-%m-%d %T UTC')"
166 bang echo ""
167 mail="$(config_get owner)" || :
168 url="$(config_get baseurl)" || :
169 case "$url" in *" "*|*" "*|"")
170 bang_eval 'echo "Bad mirror URL (\"$url\")"; ! :'
171 exit 1
172 esac
173 bang echo "Mirroring from URL \"$url\""
174 bang echo ""
175 statusok="$(git config --bool gitweb.statusupdates 2>/dev/null || echo true)"
176 mailaddrs=
177 [ "$statusok" = "false" ] || [ -z "$mail" ] || mailaddrs="$mail"
178 [ -z "$cfg_admincc" ] || [ "$cfg_admincc" = "0" ] || [ -z "$cfg_admin" ] ||
179 if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs,$cfg_admin"; fi
181 bang_eval "git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
182 bang_eval "LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-before"
184 check_after_refs() {
185 [ -n "$do_check_after_refs" ] || return 0
186 bang_eval "git for-each-ref --format '%(refname) %(objectname)' >.refs-temp"
187 bang_eval "LC_ALL=C sort -b -k1,1 <.refs-temp >.refs-after"
188 refschanged=
189 cmp -s .refs-before .refs-after || refschanged=1
190 do_check_after_refs=
193 ! [ -e .delaygc ] || >.allowgc || :
194 svnpackcreated=
196 # Make sure we don't get any unwanted loose objects
197 # Starting with Git v2.10.0 fast-import can generate loose objects unless we
198 # tweak its configuration to prevent that
199 git_add_config 'fetch.unpackLimit=1'
200 # Note the git config documentation is wrong
201 # transfer.unpackLimit, if set, overrides fetch.unpackLimit
202 git_add_config 'transfer.unpackLimit=1'
203 # But not the Git v2.10.0 and later fastimport.unpackLimit which improperly uses <= instead of <
204 git_add_config 'fastimport.unpackLimit=0'
206 case "$url" in
207 svn://* | svn+http://* | svn+https://* | svn+file://* | svn+ssh://*)
208 [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 0; }
209 # Use an 'anonsvn' username as is commonly used for anonymous svn
210 # Use an 'anonsvn' password as is commonly used for anonymous svn
211 GIT_ASKPASS_PASSWORD=anonsvn
212 export GIT_ASKPASS_PASSWORD
213 # Update the git svn url to match baseurl but be cognizant of any
214 # needed prefix changes. See the comments in taskd/clone.sh about
215 # why we need to put up with a prefix in the first place.
216 case "$url" in svn+ssh://*) svnurl="$url";; *) svnurl="${url#svn+}";; esac
217 svnurl="${svnurl%/}"
218 svnurlold="$(config_get svnurl)" || :
219 if [ "$svnurl" != "$svnurlold" ]; then
220 # We better already have an svn-remote.svn.fetch setting
221 bang test -n "$(git config --get-all svn-remote.svn.fetch)" || :
222 # the only way to truly know what the proper prefix is
223 # is to attempt a fresh git-svn init -s on the new url
224 rm -rf svn-new-url || :
225 # We require svn info to succeed on the URL otherwise it's
226 # simply not a valid URL and without using -s on the init it
227 # will not otherwise be tested until the fetch
228 bang eval 'svn --non-interactive --username anonsvn --password anonsvn info "$svnurl" >/dev/null'
229 bang mkdir svn-new-url
230 GIT_DIR=svn-new-url bang git init --bare --quiet
231 # We initially use -s for the init which will possibly shorten
232 # the URL. However, the shortening can fail if a password is
233 # not required for the longer version but is for the shorter,
234 # so try again without -s if the -s version fails.
235 cmdstr='git svn init --username=anonsvn --prefix "" -s "$svnurl" </dev/null >/dev/null 2>&1 || '
236 cmdstr="$cmdstr"'git svn init --username=anonsvn --prefix "" "$svnurl" </dev/null >/dev/null 2>&1'
237 GIT_DIR=svn-new-url bang eval "$cmdstr"
238 gitsvnurl="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url)" || :
239 gitsvnfetch="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch)" || :
240 gitsvnprefixnew="${gitsvnfetch%%:*}"
241 gitsvnsuffixnew="${gitsvnprefixnew##*/}"
242 gitsvnprefixnew="${gitsvnprefixnew%$gitsvnsuffixnew}"
243 rm -rf svn-new-url || :
244 # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to .
245 GIT_DIR=.
246 if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then
247 # The url has been changed.
248 # We must update the url and replace the prefix on all config items
249 gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | head -1)" || :
250 gitsvnprefixold="${gitsvnfetch%%:*}"
251 gitsvnsuffixold="${gitsvnprefixold##*/}"
252 gitsvnprefixold="${gitsvnprefixold%$gitsvnsuffixold}"
253 git config --remove-section 'svn-remote.svnnew' 2>/dev/null || :
254 git config 'svn-remote.svnnew.url' "$gitsvnurl"
255 git config --get-regexp '^svn-remote\.svn\.' |
256 while read -r sname sval; do
257 case "$sname" in
258 svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags)
259 sname="${sname#svn-remote.svn.}"
260 sval="${sval#$gitsvnprefixold}"
261 bang git config --add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval"
262 esac
263 done
264 test $? -eq 0
265 bang git config -f svn/.metadata svn-remote.svn.reposRoot "$gitsvnurl"
266 bang git config --remove-section svn-remote.svn
267 bang git config --rename-section svn-remote.svnnew svn-remote.svn
269 bang config_set svnurl "$svnurl"
271 # remove any stale *.lock files greater than 1 hour old in case
272 # git-svn was killed on the last update because it took too long
273 find -L svn -type f -name '*.lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || :
274 # remember the starting time so we can easily combine fetched loose objects
275 # we sleep for 1 second after creating .svnpack to make sure all objects are newer
276 if ! [ -e .svnpack ]; then
277 svnpackcreated=1
278 rm -f .svnpack
279 >.svnpack
280 sleep 1
282 GIT_DIR=. bang git svn fetch --log-window-size=$var_log_window_size --username=anonsvn --quiet </dev/null
283 # git svn does not preserve group permissions in the svn subdirectory
284 chmod -R ug+rw,o+r svn
285 # git svn also leaves behind ref turds that end with @nnn
286 # We get rid of them now
287 git for-each-ref --format='%(refname)' |
288 LC_ALL=C sed '/^..*@[1-9][0-9]*$/!d; s/^/delete /' |
289 git_updateref_stdin
290 unset GIT_ASKPASS_PASSWORD
292 darcs://*)
293 [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 0; }
294 httpurl="http://${url#darcs://}"
295 # remove any stale lock files greater than 1 hour old in case
296 # darcs_fast_export was killed on the last update because it took too long
297 find -L *.darcs -maxdepth 2 -type f -name 'lock' -mmin +60 -exec rm -f '{}' + 2>/dev/null || :
298 bang git_darcs_fetch "$httpurl"
300 bzr://*)
301 [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 0; }
302 bzrurl="${url#bzr://}"
303 bang git_bzr_fetch "$bzrurl"
305 hg+http://* | hg+https://* | hg+file://* | hg+ssh://*)
306 [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; }
307 # We just remove hg+ here, so hg+http://... becomes http://...
308 hgurl="${url#hg+}"
309 # Fetch any new updates
310 bang hg -R "$(pwd)/repo.hg" pull
311 # Do the fast-export | fast-import
312 bang git_hg_fetch
315 [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url"
316 pruneopt=--prune
317 [ "$(git config --bool fetch.prune 2>/dev/null || :)" != "false" ] || pruneopt=
318 if ! is_gfi_mirror_url "$url"; then
319 lastwasclean=
320 [ "$(git config --bool girocco.lastupdateclean 2>/dev/null || :)" != "true" ] || lastwasclean=1
321 nextisclean=
322 [ "$(git config --bool girocco.cleanmirror 2>/dev/null || :)" != "true" ] || nextisclean=1
323 if [ "$nextisclean" != "$lastwasclean" ]; then
324 if [ -n "$nextisclean" ]; then
325 git config --replace-all remote.origin.fetch "+refs/heads/*:refs/heads/*"
326 git config --add remote.origin.fetch "+refs/tags/*:refs/tags/*"
327 git config --add remote.origin.fetch "+refs/notes/*:refs/notes/*"
328 git config --add remote.origin.fetch "+refs/top-bases/*:refs/top-bases/*"
329 else
330 git config --replace-all remote.origin.fetch "+refs/*:refs/*"
334 # remember the starting time so we can easily detect new packs for fast-import mirrors
335 # we sleep for 1 second after creating .gfipack to make sure all packs are newer
336 if is_gfi_mirror_url "$url" && [ ! -e .gfipack ]; then
337 rm -f .gfipack
338 >.gfipack
339 sleep 1
341 fetcharg="default"
342 git config remotes.default >/dev/null 2>&1 || fetcharg="--all"
343 fetchcmd="git fetch"
344 [ "$show_progress" != "0" ] || fetchcmd="git fetch -q"
345 if [ -n "$var_have_git_171" ] && [ "${show_progress:-0}" != "0" ]; then
346 # git fetch learned --progress in v1.7.1
347 case "$show_progress" in
348 [2-9]*|1[0-9]*)
349 # full volume progress with all the spammy noise
350 fetchcmd="git fetch --progress"
353 # a kinder, gentler progress that doesn't leave one
354 # covered all over in exploded bits of spam afterwards
355 fetchcmd="git_fetch_q_progress"
357 esac
359 # It's possible for a fetch to actually do something while still returning
360 # a non-zero result (perhaps some of the refs were updated but some were
361 # not -- a malicious Git-impersonation trying to set refs/heads/... refs
362 # to non-commit objects for example).
363 GIT_SSL_NO_VERIFY=1 bang_catch eval "$fetchcmd" $pruneopt --multiple "$fetcharg"
364 # If we did fetch anything, don't treat it as an error, but do keep the log;
365 # otherwise invoke bang_failed as for a normal failure
366 if [ "${bang_errcode:-0}" != "0" ]; then
367 save_bang_errcode="$bang_errcode"
368 check_after_refs
369 if [ -n "$refschanged" ]; then
370 keep_bang_log="git fetch${pruneopt:+ $pruneopt} --multiple $fetcharg"
371 else
372 bang_cmd="git fetch${pruneopt:+ $pruneopt} --multiple $fetcharg"
373 bang_errcode="$save_bang_errcode"
374 bang_failed
377 if ! is_gfi_mirror_url "$url" && [ "$nextisclean" != "$lastwasclean" ]; then
378 if [ -n "$nextisclean" ]; then
379 # We must manually purge the unclean refs now as even prune won't do it
380 git for-each-ref --format='%(refname)' |
381 LC_ALL=C sed \
382 -e '/^refs\/heads\//d' \
383 -e '/^refs\/tags\//d' \
384 -e '/^refs\/notes\//d' \
385 -e '/^refs\/top-bases\//d' \
386 -e 's/^/delete /' |
387 git_updateref_stdin
389 git config --bool girocco.lastupdateclean ${nextisclean:-0}
391 if [ -e .gfipack ] && is_gfi_mirror_url "$url"; then
392 find -L objects/pack -type f -newer .gfipack -name "pack-$octet20*.pack" -print >>gfi-packs
393 rm -f .gfipack
396 esac
398 # The objects subdirectories permissions must be updated now.
399 # In the case of a dumb http clone, the permissions will not be correct
400 # (missing group write) despite the core.sharedrepository=1 setting!
401 # The objects themselves seem to have the correct permissions.
402 # This problem appears to have been fixed in the most recent git versions.
403 perms=g+w
404 [ "$cfg_permission_control" != "Hooks" ] || perms=go+w
405 chmod $perms $(find -L objects -maxdepth 1 -type d) 2>/dev/null || :
407 bang git update-server-info
409 # We maintain the last refresh date in two places deliberately
410 # so that it's available as part of the config data and also
411 # as a standalone file timestamp that can be accessed without git.
412 bang config_set lastrefresh "$(date "$datefmt")"
413 { >.last_refresh; } 2>/dev/null || :
415 # Check to see if any refs changed
416 check_after_refs
418 # There's no way to get git svn to fetch packs, so we always need to run
419 # a mini-gc afterwards if svn actually fetched anything, but that's okay
420 # because it will be very quick
421 if [ -z "$refschanged" ] && [ -n "$svnpackcreated" ]; then
422 # we created the .svnpack but didn't actually fetch anything
423 # so remove it to avoid forcing a mini-gc if not necessary
424 rm -f .svnpack
426 if [ -e .svnpack ] && ! [ -e .needsgc ]; then
427 >.needsgc
430 # Force a mini-gc if $Girocco::Config::delay_gfi_redelta is false and there's
431 # at least one gfi pack present now
432 if [ -z "$cfg_delay_gfi_redelta" ] && ! [ -e .needsgc ] &&
433 [ -f gfi-packs ] && [ -s gfi-packs ] && is_gfi_mirror_url "$url"; then
434 >.needsgc
437 # Activate a mini-gc if needed
438 check_and_set_needsgc
440 # Look at which refs changed and trigger ref-change for these
441 sockpath="$cfg_chroot/etc/taskd.socket"
442 if [ -n "$refschanged" ]; then
443 bang config_set lastreceive "$(date '+%a, %d %b %Y %T %z')"
444 # We always use UTC for the log timestamp so that chroot and non-chroot match up.
445 # We don't have to worry about multiple log files since only one update runs
446 lognamets="$(TZ=UTC date '+%Y%m%d_%H%M%S')"
447 loghhmmss="${lognamets##*_}"
448 logname="reflogs/${lognamets%%_*}"
449 # We freshen the mod time to now on any old or new ref that is a loose object
450 # For old refs we do it so we will be able to keep them around for 1 day
451 # For new refs we do it in case we are about to run gc and the new ref
452 # actually points to an oldish loose object that had been unreachable
453 # We probably do not need to do it for new refs as Git tries to do that,
454 # but since we're already doing it for old refs (which Git does not do),
455 # it's almost no extra work for new refs, just in case.
457 echo "ref-changes %$proj% $proj"
458 LC_ALL=C join .refs-before .refs-after |
459 LC_ALL=C sed -e '/^[^ ][^ ]* \([^ ][^ ]*\) \1$/d' |
460 while read ref old new; do
461 echo "$loghhmmss $old $new $ref" >&3
462 freshen_loose_objects "$old" "$new"
463 echo "$old $new $ref"
464 done
465 LC_ALL=C join -v 1 .refs-before .refs-after |
466 while read ref old; do
467 echo "$loghhmmss $old 0000000000000000000000000000000000000000 $ref" >&3
468 freshen_loose_objects "$old"
469 echo "$old 0000000000000000000000000000000000000000 $ref"
470 done
471 LC_ALL=C join -v 2 .refs-before .refs-after |
472 while read ref new; do
473 echo "$loghhmmss 0000000000000000000000000000000000000000 $new $ref" >&3
474 freshen_loose_objects "$new"
475 echo "0000000000000000000000000000000000000000 $new $ref"
476 done
477 git for-each-ref --format='%(objectname) %(objectname) %(refname)' refs/heads
478 echo "done ref-changes %$proj% $proj"
479 } >.refs-temp 3>>"$logname"
480 if [ -S "$sockpath" ]; then
481 trap ':' PIPE
482 nc_openbsd -w 15 -U "$sockpath" <.refs-temp || :
483 trap - PIPE
485 bang config_set lastchange "$(date '+%a, %d %b %Y %T %z')"
486 bang_eval "git for-each-ref --sort=-committerdate --format='%(committerdate:iso8601)' \
487 --count=1 refs/heads >info/lastactivity"
488 ! [ -d htmlcache ] || { >htmlcache/changed; } 2>/dev/null || :
489 rm -f .delaygc .allowgc
491 [ "${cfg_autogchack:-0}" != "0" ] &&
492 [ "$(git config --get --bool girocco.autogchack 2>/dev/null)" != "false" ]
493 then
494 mv -f .refs-after .refs-last
498 # If the repository does not yet have a valid HEAD symref try to set one
499 # If an empty repository was cloned and then later becomes unempty you just
500 # lose out on the fancy "symref=HEAD:" logic and get this version instead
501 check_and_set_head || :
503 rm -f .refs-before .refs-after .refs-temp FETCH_HEAD
505 if is_banged; then
506 [ -z "$mailaddrs" ] || ! was_banged_message_sent ||
508 echo "$proj update succeeded - failure recovery"
509 echo "this status message may be disabled on the project admin page"
510 } | mailref "update@$cfg_gitweburl/$proj.git" -s "[$cfg_name] $proj update succeeded" "$mailaddrs" || :
511 bang_reset
514 if [ -n "$keep_bang_log" ] && [ -s "$bang_log" ]; then
515 cat "$bang_log" >.banglog
516 echo "" >>.banglog
517 echo "$keep_bang_log failed with error code $save_bang_errcode" >>.banglog
520 progress "- [$proj] update ($(date))"