From ed7efe56dffa7e8bd909b7731f639cefe8c46858 Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Tue, 23 Jul 2013 13:37:29 -0700 Subject: [PATCH] Improve svn mirror support * Properly deal with git-svn's tendency to shorten the base url and then stick a prefix on the svn-remove.svn.* config items * Handle changing the base url when the prefix thing is going on * Work properly when an svn mirror is restarted by overwriting any leftover svn-remote.svn.* items with the new ones rather than ending up with a bunch of duplicates that break things * Get rid of the @nnn ref turds git svn leaves behind --- jobd/update.sh | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- taskd/clone.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 99 insertions(+), 14 deletions(-) diff --git a/jobd/update.sh b/jobd/update.sh index 9dc7ceb..2fe8992 100755 --- a/jobd/update.sh +++ b/jobd/update.sh @@ -54,12 +54,50 @@ show_progress=1 bang git for-each-ref --format '%(refname) %(objectname)' | sort case "$url" in svn://* | svn+http://* | svn+https://*) - # update the git svn url to match baseurl + # Update the git svn url to match baseurl but be cognizant of any + # needed prefix changes. See the comments in taskd/clone.sh about + # why we need to put up with a prefix in the first place. svnurl="${url#svn+}" svnurl="${svnurl%/}" - if [ "$svnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then - bang config_set_raw svn-remote.svn.url "$svnurl" - bang git config -f svn/.metadata svn-remote.svn.reposRoot "$svnurl" + svnurlold="$(config_get svnurl)" + if [ "$svnurl" != "$svnurlold" ]; then + # We better already have an svn-remote.svn.fetch setting + bang test -n "$(git config --get-all svn-remote.svn.fetch || :)" + # the only way to truly know what the proper prefix is + # is to attempt a fresh git-svn init -s on the new url + rm -rf svn-new-url || : + bang mkdir svn-new-url + GIT_DIR=svn-new-url bang git init --bare --quiet + GIT_DIR=svn-new-url bang git svn init -s "$svnurl" > /dev/null 2>&1 + gitsvnurl="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.url || :)" + gitsvnfetch="$(GIT_DIR=svn-new-url git config --get svn-remote.svn.fetch || :)" + gitsvnprefixnew="${gitsvnfetch%trunk:refs/remotes/trunk}" + rm -rf svn-new-url || : + # Using GIT_DIR= with bang leaves it set to svn-new-url, so reset it to . + GIT_DIR=. + if [ "$gitsvnurl" != "$(git config --get svn-remote.svn.url || :)" ]; then + # The url has been changed. + # We must update the url and replace the prefix on all config items + gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | head -1 || :)" + gitsvnprefixold="${gitsvnfetch%%:*}" + gitsvnprefixold="${gitsvnprefixold%/*}" + [ -z "$gitsvnprefixold" ] || gitsvnprefixold="$gitsvnprefixold/" + git config --remove-section 'svn-remote.svnnew' 2>/dev/null || : + git config 'svn-remote.svnnew.url' "$gitsvnurl" + { git config --get-regexp '^svn-remote\.svn\.' || :; } | \ + { while read sname sval; do + case "$sname" in + svn-remote.svn.fetch|svn-remote.svn.branches|svn-remote.svn.tags) + sname="${sname#svn-remote.svn.}" + sval="${sval#$gitsvnprefixold}" + bang git config --add "svn-remote.svnnew.$sname" "${gitsvnprefixnew}$sval" + esac + done; } + bang git config -f svn/.metadata svn-remote.svn.reposRoot "$gitsvnurl" + bang git config --remove-section svn-remote.svn + bang git config --rename-section svn-remote.svnnew svn-remote.svn + fi + bang config_set svnurl "$svnurl" fi # remove any stale *.lock files greater than 1 hour old in case # git-svn was killed on the last update because it took too long @@ -67,6 +105,18 @@ case "$url" in GIT_DIR=. bang git svn fetch --quiet # git svn does not preserve group permissions in the svn subdirectory chmod -R ug+rw,o+r svn + # git svn also leaves behind ref turds that end with @nnn + # We get rid of them now + git show-ref | \ + { while read sha1 ref; do + case "$ref" in + ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\ + ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\ + ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\ + ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) + git update-ref -d "$ref" + esac + done; } ;; darcs://*) httpurl="${url/darcs:\/\//http://}" diff --git a/taskd/clone.sh b/taskd/clone.sh index a9b11f1..8d3b2fb 100755 --- a/taskd/clone.sh +++ b/taskd/clone.sh @@ -26,35 +26,70 @@ mail="$(config_get owner)" echo "Initiating mirroring..." case "$url" in svn://* | svn+http://* | svn+https://*) - # we just remove svn+ here, so svn+http://... becomes http://... + # We just remove svn+ here, so svn+http://... becomes http://... + # We also remove a trailing '/' to match what git-svn will do svnurl="${url#svn+}" + svnurl="${svnurl%/}" + # We must use GIT_DIR=. here or ever so "helpful" git-svn will + # create a .git subdirectory! GIT_DIR=. git svn init -s "$svnurl" - # ask git-svn to store everything in the normal non-remote locations - GIT_DIR=. git config svn-remote.svn.fetch 'trunk:refs/heads/master' - GIT_DIR=. git config svn-remote.svn.branches 'branches/*:refs/heads/*' - GIT_DIR=. git config svn-remote.svn.tags 'tags/*:refs/tags/*' + # We need to remember this url so we can detect changes because + # ever so "helpful" git-svn may shorten it! + config_set svnurl "$svnurl" + # At this point, since we asked for a standard layout (-s) git-svn + # may have been "helpful" and adjusted our $svnurl to a prefix and + # then glued the removed suffix onto the front of any svn-remote.svn.* + # config items. We could avoid this by not using the '-s' option + # but then we might not get all the history. If, for example, we + # are cloning an http://svn.example.com/repos/public repository that + # early in its history moved trunk => public/trunk we would miss that + # earlier history without allowing the funky shorten+prefix behavior. + # So we read back the svn-remote.svn.fetch configuration and compute + # the prefix. This way we are sure to get the correct prefix. + gitsvnurl="$(git config --get svn-remote.svn.url || :)" + gitsvnfetch="$(git config --get-all svn-remote.svn.fetch | tail -1 || :)" + gitsvnprefix="${gitsvnfetch%trunk:refs/remotes/trunk}" + # Ask git-svn to store everything in the normal non-remote + # locations being careful to use the correct prefix + git config --replace-all svn-remote.svn.fetch "${gitsvnprefix}trunk:refs/heads/master" + git config --replace-all svn-remote.svn.branches "${gitsvnprefix}branches/*:refs/heads/*" + git config --replace-all svn-remote.svn.tags "${gitsvnprefix}tags/*:refs/tags/*" # look for additional non-standard directories to fetch # check for standard layout at the same time foundstd= foundfile= - { svn ls "$svnurl" 2>/dev/null || :; } | \ + { svn ls "$gitsvnurl/${gitsvnprefix}" 2>/dev/null || :; } | \ { while read file; do case $file in # skip the already-handled standard ones and any with a space or tab *' '*|*' '*) :;; trunk/|branches/|tags/) foundstd=1;; # only fetch extra directories from the $svnurl root (not any files) - *?/) GIT_DIR=. git config --add svn-remote.svn.fetch "${file%/}:refs/heads/${file%/}";; + *?/) git config --add svn-remote.svn.fetch \ + "${gitsvnprefix}${file%/}:refs/heads/${file%/}";; *?) foundfile=1;; esac; done # if files found and no standard directories present use a simpler layout if [ -z "$foundstd" ] && [ -n "$foundfile" ]; then - GIT_DIR=. git config --unset svn-remote.svn.branches - GIT_DIR=. git config --unset svn-remote.svn.tags - GIT_DIR=. git config --replace-all svn-remote.svn.fetch ':refs/heads/master' + git config --unset svn-remote.svn.branches + git config --unset svn-remote.svn.tags + git config --replace-all svn-remote.svn.fetch ':refs/heads/master' fi; } + # Again, be careful to use GIT_DIR=. here or else new .git subdirectory! GIT_DIR=. git svn fetch --quiet # git svn does not preserve group permissions in the svn subdirectory chmod -R ug+rw,o+r svn + # git svn also leaves behind ref turds that end with @nnn + # We get rid of them now + git show-ref | \ + { while read sha1 ref; do + case "$ref" in + ?*@[1-9]|?*@[1-9][0-9]|?*@[1-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9]|\ + ?*@[1-9][0-9][0-9][0-9][0-9]|?*@[1-9][0-9][0-9][0-9][0-9][0-9]|\ + ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9]|\ + ?*@[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) + git update-ref -d "$ref" + esac + done; } ;; darcs://*) httpurl="${url/darcs:\/\//http://}" -- 2.11.4.GIT