From db33815f0fae6e879bf4def3039d6e892474ca47 Mon Sep 17 00:00:00 2001 From: Petr Baudis Date: Fri, 15 Aug 2008 20:26:54 +0200 Subject: [PATCH] Cloning design overhaul - daemon instead of cronjobs The clonecheck cronjob was removed, as was Girocco::Config::mqueuedir. Instead, mirroring/cloned.pl listens to requests at /tmp/girocco.cloned (sent by new Girocco::Project::clone()) and runs jobs/clone.sh on its own. jobs/clone.sh does not mail failures but instead saves them to .clonelog within the repository and regproj.cgi shows this file online. On failure, repository is not removed but just .clone_failed is touched. --- Girocco/Config.pm | 3 -- Girocco/Project.pm | 21 ++++++++---- cgi/regproj.cgi | 22 +++++++++++- install.sh | 9 +---- jobs/clonecheck.sh | 14 -------- mirroring/Girocco | 1 + mirroring/clone.sh | 26 +++++++++------ mirroring/cloned.pl | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 149 insertions(+), 43 deletions(-) delete mode 100755 jobs/clonecheck.sh create mode 120000 mirroring/Girocco create mode 100755 mirroring/cloned.pl diff --git a/Girocco/Config.pm b/Girocco/Config.pm index 7bd7fd3..d2a269d 100644 --- a/Girocco/Config.pm +++ b/Girocco/Config.pm @@ -40,9 +40,6 @@ our $mob = "mob"; # This will get COMPLETELY OVERWRITTEN by each make install! our $basedir = '/home/repo/repomgr'; -# Path to the directory of the mirror queue (undef if not $mirror) -our $mqueuedir = '/home/repo/repodata'; - # The repository collection our $reporoot = "/srv/git"; diff --git a/Girocco/Project.pm b/Girocco/Project.pm index 05672d1..a396160 100644 --- a/Girocco/Project.pm +++ b/Girocco/Project.pm @@ -204,8 +204,7 @@ sub _new { sub ghost { my $class = shift; my ($name, $mirror) = @_; - my $self = $class->_new($name, - $mirror ? "$Girocco::Config::mqueuedir/to-clone" : $Girocco::Config::reporoot); + my $self = $class->_new($name, $Girocco::Config::reporoot); $self->{users} = []; $self->{mirror} = $mirror; $self; @@ -349,6 +348,19 @@ sub conjure { $self->_nofetch(1); } +sub clone { + my $self = shift; + + use IO::Socket; + my $sock = IO::Socket::UNIX->new('/tmp/girocco.cloned') or die "cannot connect to girocco.cloned: $!"; + $sock->print($self->{name}.".git\n"); + # Just ignore reply, we are going to succeed anyway and the I/O + # would apparently get quite hairy. + $sock->flush(); + sleep 2; # *cough* + $sock->close(); +} + sub update { my $self = shift; @@ -409,10 +421,7 @@ sub valid_name { sub does_exist { my ($name) = @_; valid_name($name) or die "tried to query for project with invalid name $name!"; - (available($name) - or ($Girocco::Config::mqueuedir - and (-d $Girocco::Config::mqueuedir."/cloning/$name" - or -d $Girocco::Config::mqueuedir."/to-clone/$name"))); + available($name); } sub available { my ($name) = @_; diff --git a/cgi/regproj.cgi b/cgi/regproj.cgi index 83a03f6..61dfa65 100755 --- a/cgi/regproj.cgi +++ b/cgi/regproj.cgi @@ -67,12 +67,32 @@ if ($cgi->param('mode')) { my $proj = Girocco::Project->ghost($name, $mirror); if ($proj->cgi_fill($gcgi)) { if ($mirror) { + $| = 1; unless ($Girocco::Config::mirror) { $gcgi->err("Mirroring mode is not enabled at this site."); exit; } $proj->premirror; - print "

Initiated mirroring of ".$cgi->param('url').". You will be notified about the result by mail.

\n"; + print "

Initiated mirroring of ".$cgi->param('url')." to $Girocco::Config::name project ". + "$name.git:

\n"; + $proj->clone; + print "
\n";
+
+			open LOG, $proj->{path}.'/.clonelog' or die "clonelog: $!";
+			tailf: for (;;) {
+				my $curpos;
+				for ($curpos = tell(LOG); ; $curpos = tell(LOG)) {
+					chomp;
+					$_ eq '@OVER@' and last tailf;
+					print "$_\n";
+				}
+				sleep 1;
+				seek(LOG, $curpos, 0);  # seek to where we had been
+			}
+			close LOG;
+
+			print "
\n"; + } else { unless ($Girocco::Config::push) { $gcgi->err("Push mode is not enabled at this site."); diff --git a/install.sh b/install.sh index 64451d9..0dbf50b 100755 --- a/install.sh +++ b/install.sh @@ -28,19 +28,12 @@ perl -I. -M$GIROCCO_CONF -i -pe 's/\@basedir\@/"$Girocco::Config::basedir"/g' "$ if [ -n "$cfg_mirror" ]; then - echo "*** Setting up mirror queue..." - mkdir -p "$cfg_mqueuedir/to-clone" "$cfg_mqueuedir/cloning" - chown "$cgi_mirror_user"."$cfg_owning_group" "$cfg_mqueuedir/to-clone" "$cfg_mqueuedir/cloning" || - echo "WARNING: Cannot chown $cgi_mirror_user.$cfg_owning_group the mirroring directories" - chmod 02775 "$cfg_mqueuedir/to-clone" "$cfg_mqueuedir/cloning" || - echo "WARNING: Cannot chmod 02775 the mirroring directories" - if [ ! -x "$($cfg_git_bin --exec-path)/git-mirror" ]; then echo "ERROR: Your $cfg_git_bin does not know about git-mirror; git.git submodule has it but you need to install it manually" fi echo "--- Recommended crontab for $cfg_mirror_user:" - echo "*/1 * * * * /usr/bin/nice -n 18 $cfg_basedir/jobs/clonecheck.sh" echo "*/10 * * * * /usr/bin/nice -n 18 $cfg_basedir/jobs/updatecheck.sh # adjust frequency based on number of repos" + echo "--- Remember to start $cfg_basedir/mirroring/cloned.pl" fi if [ -n "$cfg_push" -a "$cfg_permission_control" = "Group" ]; then echo "--- Recommended crontab for root:" diff --git a/jobs/clonecheck.sh b/jobs/clonecheck.sh deleted file mode 100755 index 01433fd..0000000 --- a/jobs/clonecheck.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -. @basedir@/shlib.sh - -shopt -s nullglob - -cd "$cfg_mqueuedir"/to-clone -get_repo_list_here | - while read dir; do - mkdir -m 0775 -p "$cfg_reporoot/$(dirname "$dir")" - mv "$dir" "$cfg_reporoot/$(dirname "$dir")" - "$cfg_basedir"/mirroring/clone.sh "$dir" - exit # next dir at next cron invocation, to be safe - done diff --git a/mirroring/Girocco b/mirroring/Girocco new file mode 120000 index 0000000..f897157 --- /dev/null +++ b/mirroring/Girocco @@ -0,0 +1 @@ +../Girocco \ No newline at end of file diff --git a/mirroring/clone.sh b/mirroring/clone.sh index 2e632c8..580b78b 100755 --- a/mirroring/clone.sh +++ b/mirroring/clone.sh @@ -1,4 +1,6 @@ #!/bin/bash +# +# Invoked from mirroring/cloned.pl . @basedir@/shlib.sh @@ -7,25 +9,24 @@ set -e projdir="$1" proj="${projdir%.git}" -bang_setup -bang_action="clone" -bang_trap() { - # This removes our stage directory _and_ gets rid of the group record. - perl -I@basedir@ -MGirocco::Project -e 'Girocco::Project->load('"'$proj'"')->delete;' -} - cd "$cfg_reporoot/$projdir" +trap "echo '@OVER@'; touch .clone_failed" EXIT url="$(cat base_url)" mail="$(cat owner)" # Configure -bang git config --bool --add mirror.allowed true -bang git remote add origin "$url" # superfluous? +echo "Configuring repository..." +git config --bool --add mirror.allowed true +git remote add origin "$url" # superfluous? # Initial mirror -bang git mirror "$url" -bang git update-server-info +echo "Initiating mirroring..." +git mirror "$url" +# The rest +echo "Final touches..." +git update-server-info +trap "" EXIT mail -s "[$cfg_name] $proj clone completed" "$mail,$cfg_admin" < 0) { + logmsg "reaped $waitedpid" . ($? ? " with exit $?" : ''); + } + $SIG{CHLD} = \&REAPER; # loathe sysV +} + +$SIG{CHLD} = \&REAPER; # Apollo 440 + +sub spawn { + my $coderef = shift; + + my $pid = fork; + if (not defined $pid) { + logmsg "cannot fork: $!"; + return; + } elsif ($pid) { + logmsg "begat $pid"; + return; # I'm the parent + } + + open STDIN, "<&Client" or die "can't dup client to stdin"; + open STDOUT, ">&Client" or die "can't dup client to stdout"; + exit &$coderef(); +} + + +while (1) { + unless (accept(Client, Server)) { + logmsg "accept failed: $!"; + next; + } + logmsg "connection on $NAME"; + spawn sub { + my $name = <>; + print STDERR "read\n"; + chomp $name; + print STDERR "print\n"; + print "1\n"; + print STDERR "cloning $name\n"; + open STDOUT, ">".$Girocco::Config::reporoot."/".$name."/.clonelog" or die "cannot open clonelog: $!"; + open STDERR, ">&STDOUT"; + open STDIN, "