mirroring: add support for hg+https? mirror source URLs
authorKyle J. McKay <mackyle@gmail.com>
Thu, 17 Apr 2014 08:50:15 +0000 (17 01:50 -0700)
committerKyle J. McKay <mackyle@gmail.com>
Thu, 17 Apr 2014 08:50:15 +0000 (17 01:50 -0700)
.gitmodules
Girocco/Config.pm
Girocco/Util.pm
fast-export.git [new submodule]
install.sh
jobd/update.sh
shlib.sh
taskd/clone.sh

index cbc0aac..43e0b05 100644 (file)
@@ -10,3 +10,6 @@
 [submodule "ezcert.git"]
        path = ezcert.git
        url = http://repo.or.cz/r/ezcert.git
+[submodule "fast-export.git"]
+       path = fast-export.git
+       url = http://repo.or.cz/r/fast-export/rorcz.git
index 0c9fa83..bd93958 100644 (file)
@@ -190,6 +190,16 @@ our $mirror_darcs = 1;
 # in PATH (it's a python script so python is required as well).
 our $mirror_bzr = 1;
 
+# If $mirror is true and $mirror_hg is true then mirrors from hg source
+# repositories will be allowed (and be converted to Git).  These URLs have
+# the form hg+http://... or hg+https://...
+# Note that for this to work the "hg" command line command must be available
+# in PATH and so must python (required to run the hg-fast-export.py script).
+# Note that if the PYTHON environment variable is set that will be used instead
+# of just plain "python" to run the hg-fast-export.py script (which needs to
+# be able to import from mercurial).
+our $mirror_hg = 1;
+
 
 ## Paths
 
index adc0f20..9c26a98 100644 (file)
@@ -177,6 +177,9 @@ sub valid_repo_url {
        $Girocco::Config::mirror_bzr &&
                $url =~ /^bzr:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os
                and return 1;
+       $Girocco::Config::mirror_hg &&
+               $url =~ /^hg\+https?:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os
+               and return 1;
        return $url =~  /^(https?|git):\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/;
 }
 my %_badtags;
diff --git a/fast-export.git b/fast-export.git
new file mode 160000 (submodule)
index 0000000..0b3dd8a
--- /dev/null
@@ -0,0 +1 @@
+Subproject commit 0b3dd8a684f8f76a07dc14ef20ab516bf44fb142
index 547abb3..4d71d08 100755 (executable)
@@ -177,6 +177,15 @@ mkdir -p "$cfg_basedir"/bin
 cp bzr-fastimport.git/exporters/darcs/darcs-fast-export "$cfg_basedir"/bin
 
 
+echo "*** Setting up hg-fast-export from fast-export.git..."
+if [ ! -f fast-export.git/hg-fast-export.py -o ! -f fast-export.git/hg2git.py ]; then
+       echo "ERROR: fast-export.git is not checked out! Did you _REALLY_ read INSTALL?" >&2
+       exit 1
+fi
+mkdir -p "$cfg_basedir"/bin
+cp fast-export.git/hg-fast-export.py fast-export.git/hg2git.py "$cfg_basedir"/bin
+
+
 echo "*** Setting up our part of the website..."
 mkdir -p "$cfg_webroot" "$cfg_cgiroot"
 cp cgi/*.cgi gitweb/gitweb_config.perl "$cfg_cgiroot"
index 75814aa..56c7bb5 100755 (executable)
@@ -165,6 +165,15 @@ case "$url" in
                bzrurl="${url#bzr://}"
                bang git_bzr_fetch "$bzrurl"
                ;;
+       hg+http://* | hg+https://*)
+               [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; }
+               # We just remove hg+ here, so hg+http://... becomes http://...
+               hgurl="${url#hg+}"
+               # Fetch any new updates
+               bang hg -R "$(pwd)/repo.hg" pull
+               # Do the fast-export | fast-import
+               bang git_hg_fetch
+               ;;
        *)
                [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url"
                GIT_SSL_NO_VERIFY=1 bang git remote update
index 3dde0cb..373e267 100644 (file)
--- a/shlib.sh
+++ b/shlib.sh
@@ -221,3 +221,53 @@ get_repo_list() {
                [ $id -lt 65536 ] || echo "$name"
        done
 }
+
+
+# hg-fast-export | git fast-import with error handling in current directory GIT_DIR
+git_hg_fetch() {
+       _python="${PYTHON:-python}"
+       _err1=
+       _err2=
+       exec 3>&1
+       { read -r _err1 || :; read -r _err2 || :; } <<-EOT
+       $(
+               exec 4>&3 3>&1 1>&4 4>&-
+               {
+                       _e1=0
+                       [ -f hg2git-marks ] || touch hg2git-marks
+                       _af="$(git config hg.authorsfile || :)"
+                       _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \
+                               --repo    "$(pwd)/repo.hg" \
+                               --marks   "$(pwd)/hg2git-marks" \
+                               --mapping "$(pwd)/hg2git-mapping" \
+                               --heads   "$(pwd)/hg2git-heads" \
+                               --status  "$(pwd)/hg2git-state" \
+                               -U unknown --force --flatten'
+                       [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"'
+                       eval "$_cmd" 3>&- || _e1=$?
+                       echo $_e1 >&3
+               } | \
+               {
+                       _e2=0
+                       rm -f hg2git-marks.new
+                       git fast-import --force --export-marks="$(pwd)/hg2git-marks.new" 3>&- || _e2=$?
+                       echo $_e2 >&3
+               }
+       )
+       EOT
+       exec 3>&-
+       [ "$_err1" = 0 -a "$_err2" = 0 ] || return 1
+       if [ -f hg2git-marks ]; then
+               rm -f hg2git-marks.old
+               mv hg2git-marks hg2git-marks.old
+       else
+               touch hg2git-marks.old
+       fi
+       cat hg2git-marks.old hg2git-marks.new | LC_ALL=C uniq > hg2git-marks
+       rm hg2git-marks.old hg2git-marks.new
+       rm -f hg2git-heads
+       git branch --no-color | \
+       while IFS= read -r _head; do
+               echo ":${_head#??} $(git rev-parse "refs/heads/${_head#??}")"
+       done > hg2git-heads
+}
index f112012..9e2ec94 100755 (executable)
@@ -154,6 +154,19 @@ case "$url" in
                bzrurl="${url#bzr://}"
                git_bzr_fetch "$bzrurl"
                ;;
+       hg+http://* | hg+https://*)
+               [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; }
+               # We just remove hg+ here, so hg+http://... becomes http://...
+               hgurl="${url#hg+}"
+               # Remove any left-over repo.hg dir from a previous failed attempt
+               rm -rf repo.hg
+               # Remove any left-over export files from a previous failed attempt
+               rm -f hg2git-heads hg2git-mapping hg2git-marks* hg2git-state
+               # Perform the initial hg clone
+               hg clone -U "$hgurl" "$(pwd)/repo.hg"
+               # Do the fast-export | fast-import
+               git_hg_fetch
+               ;;
        *)
                git remote rm origin >/dev/null 2>&1 || :
                # starting with Git 1.7.5 --mirror by itself will spew a warning