From ac065b636d2e66b955bf0376f749c37fff3e439b Mon Sep 17 00:00:00 2001 From: "Andreas J. Koenig" Date: Thu, 9 Oct 2008 05:51:45 +0200 Subject: [PATCH] verbose output cosmetics; notes about several todos --- Todo | 28 +++++++++++++++++++++ lib/File/Rsync/Mirror/Recent.pm | 50 ++++++++++++++++++++++++++----------- lib/File/Rsync/Mirror/Recentfile.pm | 23 +++++++++-------- 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/Todo b/Todo index 8c475f4..f52d31c 100644 --- a/Todo +++ b/Todo @@ -1,3 +1,31 @@ +2008-10-09 Andreas J. Koenig + + * The multiple-rrr way of doing things needs a new option to rmirror, + like piecemeal or so. Not urgent because after the first pass through, + things run smoothely. It's only ugly during the first pass. + + * I have the suspicion that the code is broken that decides if the + neighboring RF needs to be seeded. I fear when too much time has gone + between two calls (in our case more than one hour), it would not seed + the neighbor. Of course this will never be noticed, so we need a good + test for it. + + * local/localroot confusion + + * accounts for early birds + + * hardcoded 20 seconds + + * who mirrors the index? + + * which CPAN mirrors offer rsync? + + * visit all XXX, visit all _float places + + * rename the pathdb stuff, it's too confusing. No idea how. + + * rrr-inotify, backpan, rrr-register + 2008-10-08 Andreas J. Koenig * current bugs: the pathdb seems to get no reset, the seeding of the diff --git a/lib/File/Rsync/Mirror/Recent.pm b/lib/File/Rsync/Mirror/Recent.pm index be352d3..84ab5dc 100644 --- a/lib/File/Rsync/Mirror/Recent.pm +++ b/lib/File/Rsync/Mirror/Recent.pm @@ -405,7 +405,7 @@ Testing this ATM with: ignore_link_stat_errors => 1, localroot => "/home/ftp/pub/PAUSE/authors", remote => "pause.perl.org::authors/RECENT.recent", - max_files_per_connection => 5, + max_files_per_connection => 5000, rsync_options => { compress => 1, links => 1, @@ -418,6 +418,36 @@ Testing this ATM with: ); $rrr->rmirror ( "skip-deletes" => 1, loop => 1 ); +And since the above seems to work, I try now without the llop +parameter: + + use File::Rsync::Mirror::Recent; + my @rrr; + for my $t ("authors","modules"){ + my $rrr = File::Rsync::Mirror::Recent->new( + ignore_link_stat_errors => 1, + localroot => "/home/ftp/pub/PAUSE/$t", + remote => "pause.perl.org::$t/RECENT.recent", + max_files_per_connection => 512, + rsync_options => { + compress => 1, + links => 1, + times => 1, + checksum => 0, + }, + verbose => 1, + _runstatusfile => "recent-rmirror-state-$t.yml", + _logfilefordone => "recent-rmirror-donelog-$t.log", + ); + push @rrr, $rrr; + } + while (){ + for my $rrr (@rrr){ + $rrr->rmirror ( "skip-deletes" => 1 ); + } + warn "sleeping 23\n"; sleep 23; + } + =cut @@ -469,18 +499,8 @@ sub rmirror { WORKUNIT: while (time < $ttleave) { if ($rf->uptodate) { my $sleep = $rf->sleep_per_connection; - $sleep = 0.42 unless defined $sleep; # XXX double accessor! - for ($sleep) { - if ($rf->verbose) { - printf STDERR - ( - "Napping (%s/%s) ...\n", - $_, - $rf->interval, - ); - } - Time::HiRes::sleep $_ if $_; - } + $sleep = 0.42 unless defined $sleep; # XXX accessor! + Time::HiRes::sleep $sleep; $rfs->[$i+1]->done->merge($rf->done) if $i < $#$rfs; next RECENTFILE; } else { @@ -523,9 +543,9 @@ sub rmirror { if ($sleep > 0.01) { printf STDERR ( - "Dormitory (%ssecs\@%d)\n", - $sleep, + "Dorm %d (%s secs)\n", time, + $sleep, ); sleep $sleep; } else { diff --git a/lib/File/Rsync/Mirror/Recentfile.pm b/lib/File/Rsync/Mirror/Recentfile.pm index f952213..c794048 100644 --- a/lib/File/Rsync/Mirror/Recentfile.pm +++ b/lib/File/Rsync/Mirror/Recentfile.pm @@ -23,7 +23,7 @@ for my $package ( $HAVE->{$package} = eval qq{ require $package; }; } use Config; -use File::Basename qw(dirname fileparse); +use File::Basename qw(basename dirname fileparse); use File::Copy qw(cp); use File::Path qw(mkpath); use File::Rsync::Mirror::Recentfile::FakeBigFloat qw(:all); @@ -542,14 +542,15 @@ sub get_remote_recentfile_as_tempfile { $trfilename, ); if ($self->verbose) { - my $doing = -e $dst ? "Syncing" : "Getting"; + my $doing = -e $dst ? "Sync" : "Get"; + my $display_dst = join "/", "...", basename(dirname($dst)), basename($dst); printf STDERR ( - "%s (1/1/%s\@%d) temporary %s ... ", + "%-4s %d (1/1/%s) temp %s ... ", $doing, - $self->interval, time, - $dst, + $self->interval, + $display_dst, ); } my $gaveup = 0; @@ -612,13 +613,13 @@ sub get_remotefile { my $dst = File::Spec->catfile($self->localroot, $path); mkpath dirname $dst; if ($self->verbose) { - my $doing = -e $dst ? "Syncing" : "Getting"; + my $doing = -e $dst ? "Sync" : "Get"; printf STDERR ( - "%s (1/1/%s\@%d) %s ... ", + "%-4s %d (1/1/%s) %s ... ", $doing, - $self->interval, time, + $self->interval, $path, ); } @@ -1048,15 +1049,15 @@ sub _mirror_item_new { $options, ) = @_; if ($self->verbose) { - my $doing = -e $dst ? "Syncing" : "Getting"; + my $doing = -e $dst ? "Sync" : "Get"; printf STDERR ( - "%s (%d/%d/%s\@%d) %s ... ", + "%-4s %d (%d/%d/%s) %s ... ", $doing, + time, 1+$i, 1+$last_item, $self->interval, - time, $recent_event->{path}, ); } -- 2.11.4.GIT