From 9391b74a74955f2e92a0859e9a12dedec4d4a55e Mon Sep 17 00:00:00 2001 From: "Kyle J. McKay" Date: Wed, 16 Apr 2014 19:03:59 -0700 Subject: [PATCH] mirroring: add individual foreign vcs mirror control When mirroring is enabled ($Girocco::Config::mirror is true) allow each supported foreign vcs to be individually enabled/disabled as an allowed mirror source. Additionally make sure that mirrors are not updated, cloned or optimized if $Girocco::Config::mirror is not true. --- Girocco/Config.pm | 33 ++++++++++++++++++++++++++++++++- Girocco/Util.pm | 15 +++++++++++++-- jobd/gc.sh | 28 +++++++++++++++------------- jobd/update.sh | 4 ++++ taskd/clone.sh | 4 ++++ 5 files changed, 68 insertions(+), 16 deletions(-) diff --git a/Girocco/Config.pm b/Girocco/Config.pm index 595cc6b..0c9fa83 100644 --- a/Girocco/Config.pm +++ b/Girocco/Config.pm @@ -71,7 +71,7 @@ our $giroccobranch = undef; ## Feature knobs -# Enable mirroring mode if true +# Enable mirroring mode if true (see "Foreign VCS mirrors" section below) our $mirror = 1; # Enable push mode if true @@ -160,6 +160,37 @@ our $update_pwd_db = 0; our $sshd_jail_port = 22; +## Foreign VCS mirrors + +# Note that if any of these settings are changed from true to false, then +# any pre-existing mirrors using the now-disabled foreign VCS will stop +# updating, new mirrors using the now-disabled foreign VCS will be disallowed +# and attempts to update ANY project settings for a pre-existing project that +# uses a now-disabled foreign VCS source URL will also be disallowed. + +# If $mirror is true and $mirror_svn is true then mirrors from svn source +# repositories will be allowed (and be converted to Git). These URLs have +# the form svn://... or svn+http://... or svn+https://... +# Note that for this to work the "svn" command line command must be available +# in PATH and the "git svn" commands must work (which generally requires both +# Perl and the subversion perl bindings be installed). +our $mirror_svn = 1; + +# If $mirror is true and $mirror_darcs is true then mirrors from darcs source +# repositories will be allowed (and be converted to Git). These URLs have +# the form darcs://... +# Note that for this to work the "darcs" command line command must be available +# in PATH and so must python (required to run the darcs-fast-export script). +our $mirror_darcs = 1; + +# If $mirror is true and $mirror_bzr is true then mirrors from bzr source +# repositories will be allowed (and be converted to Git). These URLs have +# the form bzr://... +# Note that for this to work the "bzr" command line command must be available +# in PATH (it's a python script so python is required as well). +our $mirror_bzr = 1; + + ## Paths # Path where the main chunk of Girocco files will be installed diff --git a/Girocco/Util.pm b/Girocco/Util.pm index 8fcd498..adc0f20 100644 --- a/Girocco/Util.pm +++ b/Girocco/Util.pm @@ -165,8 +165,19 @@ sub valid_web_url { /^https?:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~:?&=;-]*)?(#[a-zA-Z0-9._-]+)?$/; } sub valid_repo_url { - local $_ = $_[0]; - /^(https?|git|svn(\+http)?|svn(\+https)?|darcs|bzr):\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/; + my $url = shift || ''; + # Currently neither username nor password is allowed in the URL and IPv6 + # literal addresses are not accepted either. + $Girocco::Config::mirror_svn && + $url =~ /^svn(\+https?)?:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os + and return 1; + $Girocco::Config::mirror_darcs && + $url =~ /^darcs:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os + and return 1; + $Girocco::Config::mirror_bzr && + $url =~ /^bzr:\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/os + and return 1; + return $url =~ /^(https?|git):\/\/[a-zA-Z0-9.:-]+(\/[_\%a-zA-Z0-9.\/~-]*)?$/; } my %_badtags; BEGIN { diff --git a/jobd/gc.sh b/jobd/gc.sh index ddedf30..698f3ff 100755 --- a/jobd/gc.sh +++ b/jobd/gc.sh @@ -119,21 +119,23 @@ git update-server-info # darcs:// mirrors have a xxx.log file that will grow endlessly # if this is a mirror and the file exists, shorten it to 10000 lines # also take this opportunity to optimize the darcs repo -if [ ! -e .nofetch ]; then +if [ ! -e .nofetch ] && [ -n "$cfg_mirror" ]; then url="$(config_get baseurl || :)" case "$url" in darcs://*) - url="${url%/}" - basedarcs="$(basename "${url#darcs:/}")" - if [ -f "$basedarcs.log" ]; then - tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$" - mv -f "$basedarcs.log.$$" "$basedarcs.log" - fi - if [ -d "$basedarcs.darcs" ]; then - ( - cd "$basedarcs.darcs" - # Note that this does not optimize _darcs/inventories/ :( - darcs optimize - ) + if [ -n "$cfg_mirror_darcs" ]; then + url="${url%/}" + basedarcs="$(basename "${url#darcs:/}")" + if [ -f "$basedarcs.log" ]; then + tail -n 10000 "$basedarcs.log" > "$basedarcs.log.$$" + mv -f "$basedarcs.log.$$" "$basedarcs.log" + fi + if [ -d "$basedarcs.darcs" ]; then + ( + cd "$basedarcs.darcs" + # Note that this does not optimize _darcs/inventories/ :( + darcs optimize + ) + fi fi esac fi diff --git a/jobd/update.sh b/jobd/update.sh index 2f99fdc..75814aa 100755 --- a/jobd/update.sh +++ b/jobd/update.sh @@ -43,6 +43,7 @@ git_bzr_fetch() { } set -e +[ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 0; } trap 'if [ $? != 0 ]; then echo "update failed dir: $PWD" >&2; fi; rm -f "$bang_log"' EXIT umask 002 @@ -74,6 +75,7 @@ bang_eval "sort -k1b,1 <.refs-temp >.refs-before" case "$url" in svn://* | svn+http://* | svn+https://*) + [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 0; } # Update the git svn url to match baseurl but be cognizant of any # needed prefix changes. See the comments in taskd/clone.sh about # why we need to put up with a prefix in the first place. @@ -151,6 +153,7 @@ case "$url" in done; } ;; darcs://*) + [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 0; } httpurl="http://${url#darcs://}" # remove any stale lock files greater than 1 hour old in case # darcs_fast_export was killed on the last update because it took too long @@ -158,6 +161,7 @@ case "$url" in bang git_darcs_fetch "$httpurl" ;; bzr://*) + [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 0; } bzrurl="${url#bzr://}" bang git_bzr_fetch "$bzrurl" ;; diff --git a/taskd/clone.sh b/taskd/clone.sh index 9f960df..f112012 100755 --- a/taskd/clone.sh +++ b/taskd/clone.sh @@ -48,6 +48,7 @@ proj="${projdir%.git}" cd "$cfg_reporoot/$projdir" trap "echo '@OVER@'; touch .clone_failed" EXIT +[ -n "$cfg_mirror" ] || { echo "Mirroring is disabled" >&2; exit 1; } url="$(config_get baseurl)" if [ "$cfg_project_owners" = "source" ]; then @@ -62,6 +63,7 @@ if [ -z "$mailaddrs" ]; then mailaddrs="$cfg_admin"; else mailaddrs="$mailaddrs, echo "Initiating mirroring..." case "$url" in svn://* | svn+http://* | svn+https://*) + [ -n "$cfg_mirror_svn" ] || { echo "Mirroring svn is disabled" >&2; exit 1; } # We just remove svn+ here, so svn+http://... becomes http://... # We also remove a trailing '/' to match what git-svn will do svnurl="${url#svn+}" @@ -139,12 +141,14 @@ case "$url" in done; } ;; darcs://*) + [ -n "$cfg_mirror_darcs" ] || { echo "Mirroring darcs is disabled" >&2; exit 1; } httpurl="http://${url#darcs://}" # Remove any left-over .darcs dirs from a previous failed attempt rm -rf *.darcs git_darcs_fetch "$httpurl" ;; bzr://*) + [ -n "$cfg_mirror_bzr" ] || { echo "Mirroring bzr is disabled" >&2; exit 1; } # we just remove bzr:// here, a typical bzr url is just # "lp:foo" bzrurl="${url#bzr://}" -- 2.11.4.GIT