bin/rsync-over-recentfile-2.pl

   1 #!/usr/bin/perl
   2
   3 =pod
   4
   5
   6
   7 =cut
   8
   9 use strict;
  10 use warnings;
  11 use File::Basename qw(dirname);
  12 use File::Path qw(mkpath);
  13 use File::Spec;
  14 use Getopt::Long qw(GetOptions);
  15 use List::Util qw(min);
  16 use Time::HiRes qw(sleep);
  17
  18 use lib "/home/k/sources/rersyncrecent/lib/";
  19 require File::Rsync::Mirror::Recentfile;
  20
  21 our %Opt;
  22 GetOptions(\%Opt,
  23            "use_interval=i",
  24            "loops=i",
  25            "verbose!",
  26           ) or die;
  27
  28 my %max_epoch_ever;
  29
  30 my $print_leading_newline = 0;
  31 my $loop = 0;
  32 ITERATION: while () {
  33   last if $Opt{loops} && $loop++ >= $Opt{loops};
  34   my $iteration_start = time;
  35
  36   for my $rmodule (qw(authors modules)) {
  37     my $rf = File::Rsync::Mirror::Recentfile->new
  38         (
  39          canonize => "naive_path_normalize",
  40          filenameroot => "RECENT",
  41          ignore_link_stat_errors => 1,
  42          interval => q(6h),
  43          localroot => "/home/ftp/pub/PAUSE/$rmodule",
  44          remote_dir => $rmodule,
  45          remote_host => "k75",
  46          remote_module => "PAUSE",
  47          rsync_options => {
  48                            # intenionally not using archive=>1 because it contains "r"
  49                            compress => 0,
  50                            'rsync-path' => '/usr/bin/rsync',
  51                            links => 1,
  52                            times => 1,
  53                            'omit-dir-times' => 1,
  54                            checksum => 1,
  55                           },
  56          verbose => $Opt{verbose},
  57         );
  58
  59     my $trecentfile = eval {$rf->get_remote_recentfile_as_tempfile();};
  60     if ($@) {
  61       warn sprintf "Warning: %s", $@; # XXX need a logging mechanism
  62       sleep 5;
  63       next ITERATION;
  64     }
  65     my($recent_data) = $rf->recent_events_from_tempfile();
  66     my @error;
  67     my $i = 0;
  68     my $total = @$recent_data;
  69   UPLOADITEM: for my $recent_event (reverse @$recent_data) {
  70       $i++;
  71       if ($recent_event->{type} eq "new"){
  72         my $must_get;
  73         $max_epoch_ever{$rmodule} ||= 0;
  74         if ($Opt{use_interval} && $recent_event->{epoch}+$Opt{use_interval} < time) {
  75           next UPLOADITEM;
  76         } elsif ($recent_event->{epoch} <= $max_epoch_ever{$rmodule}) {
  77           next UPLOADITEM;
  78         } else {
  79           $must_get++;
  80         }
  81         if ($must_get) {
  82           my $dst = $rf->local_path($recent_event->{path});
  83           my $doing = -e $dst ? "Syncing" : "Getting";
  84           {
  85             printf(
  86                    "%s%s (%d/%d) %s\n",
  87                    $print_leading_newline ? "\n" : "",
  88                    $doing,
  89                    $i,
  90                    $total,
  91                    $recent_event->{path},
  92                   );
  93             $print_leading_newline = 0;
  94           }
  95           eval { $rf->mirror_path($recent_event->{path}) };
  96           if ($@) {
  97             push @error, $@;
  98             sleep 1;
  99             next UPLOADITEM;
 100           }
 101         }
 102       } elsif ($recent_event->{type} eq "delete") {
 103         print "d";
 104       } else {
 105         warn "Warning: invalid upload type '$recent_event->{type}'"; # XXX logging
 106       }
 107       $max_epoch_ever{$rmodule} = $recent_event->{epoch} if $recent_event->{epoch} > $max_epoch_ever{$rmodule};
 108     }
 109     if (@error) {
 110       # XXX this seems a bit too drastic
 111       my $errors = @error;
 112       my @disperrors = splice @error, 0, min(10, scalar @error);
 113       my $disperrors = @disperrors;
 114       warn "Warning: Ran into $errors errors, $disperrors follow:
 115 @disperrors
 116 ";
 117       sleep 12;
 118       $max_epoch_ever{$rmodule} = 0;
 119     }
 120     rename $trecentfile, $rf->rfile;
 121   }
 122
 123   my $minimum_time_per_loop = 60;
 124   { local $| = 1; print "."; $print_leading_newline = 1; }
 125   if (time - $iteration_start < $minimum_time_per_loop) {
 126     sleep $iteration_start + $minimum_time_per_loop - time;
 127   }
 128   { local $| = 1; print "~"; $print_leading_newline = 1; }
 129 }
 130
 131 print "\n";
 132
 133 __END__
 134
 135 # Local Variables:
 136 # mode: cperl
 137 # cperl-indent-level: 2
 138 # End: