From 9ab75f7479f1aa3567cdbcb6fee4991dd6437caf Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Thu, 17 Apr 2014 01:50:15 -0700 Subject: [PATCH] mirroring: add support for hg+https? mirror source URLs --- .gitmodules | 3 +++ Girocco/Config.pm | 10 ++++++++++ Girocco/Util.pm | 3 +++ fast-export.git | 1 + install.sh | 9 +++++++++ jobd/update.sh | 9 +++++++++ shlib.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ taskd/clone.sh | 13 +++++++++++++ 8 files changed, 98 insertions(+) create mode 160000 fast-export.git diff --git a/.gitmodules b/.gitmodules index cbc0aac..43e0b05 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "ezcert.git"] path = ezcert.git url = http://repo.or.cz/r/ezcert.git +[submodule "fast-export.git"] + path = fast-export.git + url = http://repo.or.cz/r/fast-export/rorcz.git diff --git a/Girocco/Config.pm b/Girocco/Config.pm index 0c9fa83..bd93958 100644 --- a/Girocco/Config.pm +++ b/Girocco/Config.pm @@ -190,6 +190,16 @@ our $mirror_darcs = 1; # in PATH (it's a python script so python is required as well). our $mirror_bzr = 1; +# If $mirror is true and $mirror_hg is true then mirrors from hg source +# repositories will be allowed (and be converted to Git). These URLs have +# the form hg+http://... or hg+https://... +# Note that for this to work the "hg" command line command must be available +# in PATH and so must python (required to run the hg-fast-export.py script). +# Note that if the PYTHON environment variable is set that will be used instead +# of just plain "python" to run the hg-fast-export.py script (which needs to +# be able to import from mercurial). +our $mirror_hg = 1; + ## Paths diff --git a/Girocco/Util.pm b/Girocco/Util.pm index adc0f20..9c26a98 100644 --- a/Girocco/Util.pm +++ b/Girocco/Util.pm @@ -177,6 +177,9 @@ sub valid_repo_url { $Girocco::Config::mirror_bzr && $url =~ /^bzr:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os and return 1; + $Girocco::Config::mirror_hg && + $url =~ /^hg\+https?:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os + and return 1; return $url =~ /^(https?|git):\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/; } my %_badtags; diff --git a/fast-export.git b/fast-export.git new file mode 160000 index 0000000..0b3dd8a --- /dev/null +++ b/fast-export.git @@ -0,0 +1 @@ +Subproject commit 0b3dd8a684f8f76a07dc14ef20ab516bf44fb142 diff --git a/install.sh b/install.sh index 547abb3..4d71d08 100755 --- a/install.sh +++ b/install.sh @@ -177,6 +177,15 @@ mkdir -p "$cfg_basedir"/bin cp bzr-fastimport.git/exporters/darcs/darcs-fast-export "$cfg_basedir"/bin +echo "*** Setting up hg-fast-export from fast-export.git..." +if [ ! -f fast-export.git/hg-fast-export.py -o ! -f fast-export.git/hg2git.py ]; then + echo "ERROR: fast-export.git is not checked out! Did you _REALLY_ read INSTALL?" >&2 + exit 1 +fi +mkdir -p "$cfg_basedir"/bin +cp fast-export.git/hg-fast-export.py fast-export.git/hg2git.py "$cfg_basedir"/bin + + echo "*** Setting up our part of the website..." mkdir -p "$cfg_webroot" "$cfg_cgiroot" cp cgi/*.cgi gitweb/gitweb_config.perl "$cfg_cgiroot" diff --git a/jobd/update.sh b/jobd/update.sh index 75814aa..56c7bb5 100755 --- a/jobd/update.sh +++ b/jobd/update.sh @@ -165,6 +165,15 @@ case "$url" in bzrurl="${url#bzr://}" bang git_bzr_fetch "$bzrurl" ;; + hg+http://* | hg+https://*) + [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 0; } + # We just remove hg+ here, so hg+http://... becomes http://... + hgurl="${url#hg+}" + # Fetch any new updates + bang hg -R "$(pwd)/repo.hg" pull + # Do the fast-export | fast-import + bang git_hg_fetch + ;; *) [ "$url" = "$(git config --get remote.origin.url || :)" ] || bang config_set_raw remote.origin.url "$url" GIT_SSL_NO_VERIFY=1 bang git remote update diff --git a/shlib.sh b/shlib.sh index 3dde0cb..373e267 100644 --- a/shlib.sh +++ b/shlib.sh @@ -221,3 +221,53 @@ get_repo_list() { [ $id -lt 65536 ] || echo "$name" done } + + +# hg-fast-export | git fast-import with error handling in current directory GIT_DIR +git_hg_fetch() { + _python="${PYTHON:-python}" + _err1= + _err2= + exec 3>&1 + { read -r _err1 || :; read -r _err2 || :; } <<-EOT + $( + exec 4>&3 3>&1 1>&4 4>&- + { + _e1=0 + [ -f hg2git-marks ] || touch hg2git-marks + _af="$(git config hg.authorsfile || :)" + _cmd='GIT_DIR="$(pwd)" "$_python" "$cfg_basedir/bin/hg-fast-export.py" \ + --repo "$(pwd)/repo.hg" \ + --marks "$(pwd)/hg2git-marks" \ + --mapping "$(pwd)/hg2git-mapping" \ + --heads "$(pwd)/hg2git-heads" \ + --status "$(pwd)/hg2git-state" \ + -U unknown --force --flatten' + [ -z "$_af" ] || _cmd="$_cmd"' --authors "$_af"' + eval "$_cmd" 3>&- || _e1=$? + echo $_e1 >&3 + } | \ + { + _e2=0 + rm -f hg2git-marks.new + git fast-import --force --export-marks="$(pwd)/hg2git-marks.new" 3>&- || _e2=$? + echo $_e2 >&3 + } + ) + EOT + exec 3>&- + [ "$_err1" = 0 -a "$_err2" = 0 ] || return 1 + if [ -f hg2git-marks ]; then + rm -f hg2git-marks.old + mv hg2git-marks hg2git-marks.old + else + touch hg2git-marks.old + fi + cat hg2git-marks.old hg2git-marks.new | LC_ALL=C uniq > hg2git-marks + rm hg2git-marks.old hg2git-marks.new + rm -f hg2git-heads + git branch --no-color | \ + while IFS= read -r _head; do + echo ":${_head#??} $(git rev-parse "refs/heads/${_head#??}")" + done > hg2git-heads +} diff --git a/taskd/clone.sh b/taskd/clone.sh index f112012..9e2ec94 100755 --- a/taskd/clone.sh +++ b/taskd/clone.sh @@ -154,6 +154,19 @@ case "$url" in bzrurl="${url#bzr://}" git_bzr_fetch "$bzrurl" ;; + hg+http://* | hg+https://*) + [ -n "$cfg_mirror_hg" ] || { echo "Mirroring hg is disabled" >&2; exit 1; } + # We just remove hg+ here, so hg+http://... becomes http://... + hgurl="${url#hg+}" + # Remove any left-over repo.hg dir from a previous failed attempt + rm -rf repo.hg + # Remove any left-over export files from a previous failed attempt + rm -f hg2git-heads hg2git-mapping hg2git-marks* hg2git-state + # Perform the initial hg clone + hg clone -U "$hgurl" "$(pwd)/repo.hg" + # Do the fast-export | fast-import + git_hg_fetch + ;; *) git remote rm origin >/dev/null 2>&1 || : # starting with Git 1.7.5 --mirror by itself will spew a warning -- 2.11.4.GIT