12 $Opt{verbose} ||= $ENV{PERL_RERSYNCRECENT_TEST_VERBOSE};
16 # neither LibMagic nor MMagic tell them apart
21 $HAVE->{$package} = eval qq{ require $package; };
26 use File::Basename qw(dirname);
27 use File::Copy qw(cp);
28 use File::Path qw(mkpath rmtree);
29 use File::Rsync::Mirror::Recent;
30 use File::Rsync::Mirror::Recentfile;
31 use List::MoreUtils qw(uniq);
33 use Time::HiRes qw(time sleep);
36 my $root_from = "t/ta";
38 my $statusfile = "t/recent-rmirror-state.yml";
39 rmtree [$root_from, $root_to];
45 $test_counter = $tests;
52 $tests += @serializers;
53 if ($HAVE->{"File::LibMagic"}||$HAVE->{"File::MMagic"}) {
54 $tests += @serializers;
57 printf "#test_counter[%d]\n", $test_counter;
59 my $ttt = "$root_from/ttt";
60 open my $fh, ">", $ttt or die "Could not open: $!";
62 close $fh or die "Could not close: $!";
64 if ($HAVE->{"File::LibMagic"}) {
65 $fm = File::LibMagic->new();
66 } elsif ($HAVE->{"File::MMagic"}) {
67 $fm = File::MMagic->new();
69 for my $s (@serializers) {
70 my $rf = File::Rsync::Mirror::Recentfile->new
72 filenameroot => "RECENT",
74 localroot => $root_from,
75 serializer_suffix => $s,
77 $rf->update($ttt,"new");
79 my $magic = $fm->checktype_filename("$root_from/RECENT-1m$s");
81 ("Got a magic[%s] for s[%s]: [%s]",
87 my $content = do {open my $fh, "$root_from/RECENT-1m$s";local $/;<$fh>};
88 $content = Dumpvalue->new()->stringify($content);
89 my $want_length = 42; # or maybe 3 more
90 substr($content,$want_length) = "..." if length $content > 3+$want_length;
91 ok($content, "Got a substr for s[$s]: [$content]");
95 rmtree [$root_from, $root_to];
98 # very small tree, aggregate it
102 $test_counter = $tests;
103 @intervals = qw( 2s 4s 8s 16s 32s Z );
104 $tests += 2 + 2 * (7 + 10 * @intervals); # test_counter
106 printf "#test_counter[%d]\n", $test_counter;
107 ok(1, "starting smalltree block");
108 is 6, scalar @intervals, "array has 6 elements: @intervals";
109 printf "#test_counter[%d]\n", $test_counter+=2;
111 my $rf0 = File::Rsync::Mirror::Recentfile->new
113 aggregator => [@intervals[1..$#intervals]],
114 interval => $intervals[0],
115 localroot => $root_from,
123 my $timestampfutured = 0;
124 for my $iv (@intervals) {
133 mkpath dirname $file;
134 open my $fh, ">", $file or die "Could not open '$file': $!";
135 print $fh time, ":", $file, "\n";
136 close $fh or die "Could not close '$file': $!";
137 $rf0->update($file,"new");
138 if ($pass==1 && !$timestampfutured) {
139 my $recent_events = $rf0->recent_events;
140 $recent_events->[0]{epoch} += 987654321;
141 $rf0->write_recent($recent_events);
146 my $recent_events = $rf0->recent_events;
147 # faking internals as if the contents were wide-spread in time
148 for my $evi (0..$#$recent_events) {
149 my $ev = $recent_events->[$evi];
150 $ev->{epoch} -= 2**($evi*.25);
152 $rf0->write_recent($recent_events);
154 my $filesize_threshold = 1750; # XXX may be system dependent
156 for my $iv (@intervals) {
157 my $rf = "$root_from/RECENT-$iv.yaml";
158 my $filesize = -s $rf;
159 $size_before{$iv} = $filesize;
160 # now they have $filesize_threshold+ bytes because they were merged for the
161 # first time ever and could not be truncated for this reason.
162 ok( $filesize > $filesize_threshold, "file $iv (before merging) has good size[$filesize]");
163 utime 0, 0, $rf; # so that the next aggregate isn't skipped
165 printf "#test_counter[%d]\n", $test_counter+=6;
166 open my $fh, ">", "$root_from/finissage" or die "Could not open: $!";
168 close $fh or die "Could not close: $!";
169 $rf0->update("$root_from/finissage","new");
170 $rf0 = File::Rsync::Mirror::Recentfile->new_from_file("$root_from/RECENT-2s.yaml");
172 for my $iv (@intervals) {
173 my $filesize = -s "$root_from/RECENT-$iv.yaml";
174 # now they have <$filesize_threshold bytes because the second aggregate could
176 ok($iv eq "Z" || $filesize<$size_before{$iv}, "file $iv (after merging) has good size[$filesize<$size_before{$iv}]");
178 printf "#test_counter[%d]\n", $test_counter+=6;
179 my $dagg1 = $rf0->_debug_aggregate;
180 Time::HiRes::sleep 1.2;
182 my $dagg2 = $rf0->_debug_aggregate;
184 my $recc = File::Rsync::Mirror::Recent->new
186 local => "$root_from/RECENT-2s.yaml",
188 ok $recc->overview, "overview created";
189 # diag $recc->overview;
191 printf "#test_counter[%d]\n", $test_counter+=1;
192 for my $dirti (0,1) {
193 open my $fh2, ">", "$root_from/dirty$dirti" or die "Could not open: $!";
194 print $fh2 "dirty$dirti";
195 close $fh2 or die "Could not close: $!";
196 $rf0->update("$root_from/dirty$dirti","new","999.999");
197 $recent_events = $rf0->recent_events;
198 is $recent_events->[-1]{epoch}, "999.999", "found the dirty timestamp during dirti[$dirti]";
199 printf "#test_counter[%d]\n", $test_counter+=1;
200 $rf0->aggregate(force => 1);
201 my $recc = File::Rsync::Mirror::Recent->new
203 localroot => $root_from,
204 local => "$root_from/RECENT.recent",
207 for my $rf (@{$recc->recentfiles}) {
208 my $isec = $rf->interval_secs;
209 my $re = $rf->recent_events;
210 is $re->[-1]{epoch}, "999.999", "found the dirty timestamp in isec[$isec]";
211 my $dirtymark = $rf->dirtymark;
212 ok $dirtymark, "dirtymark[$dirtymark]";
213 $seen{ $rf->dirtymark }++;
215 printf "#test_counter[%d]\n", $test_counter+=12;
216 is scalar keys %seen, 1, "all recentfiles have the same dirtymark";
217 printf "#test_counter[%d]\n", $test_counter+=1;
219 $rf0->aggregate(force => 1);
220 my $rfs = $recc->recentfiles;
221 for my $i (0..$#$rfs) {
223 my $re = $rf->recent_events;
225 is $re->[-1]{epoch}, "999.999", "found the dirty timestamp on i[$i]";
227 isnt $re->[-1]{epoch}, "999.999", "dirty timestamp gone on i[$i]";
229 my $dirtymark = $rf->dirtymark;
230 ok $dirtymark, "dirtymark[$dirtymark]";
231 $seen{ $rf->dirtymark }++;
233 printf "#test_counter[%d]\n", $test_counter+=12;
234 is scalar keys %seen, 1, "all recentfiles have the same dirtymark";
235 printf "#test_counter[%d]\n", $test_counter+=1;
242 rmtree [$root_from, $root_to];
245 # replay a short history, run aggregate on it, add files, aggregate again
248 $test_counter = $tests;
251 printf "#test_counter[%d]\n", $test_counter;
252 ok(1, "starting short history block");
253 my $rf = File::Rsync::Mirror::Recentfile->new_from_file("t/RECENT-6h.yaml");
254 my $recent_events = $rf->recent_events;
255 my $recent_events_cnt = scalar @$recent_events;
259 "found $recent_events_cnt events",
262 $rf->localroot($root_from);
263 $rf->comment("produced during the test 02-operation.t");
264 $rf->aggregator([qw(10s 30s 1m 1h Z)]);
266 my $start = Time::HiRes::time;
267 for my $e (@$recent_events) {
272 $pass==0 ? $root_from : $root_to,
275 mkpath dirname $file;
276 open my $fh, ">", $file or die "Could not open '$file': $!";
277 print $fh time, ":", $file, "\n";
278 close $fh or die "Could not close '$file': $!";
280 $rf->update($file,$e->{type});
285 my $took = Time::HiRes::time - $start;
286 ok $took > 0, "creating the tree and aggregate took $took seconds";
287 my $dagg1 = $rf->_debug_aggregate;
289 my $file_from = "$root_from/anotherfilefromtesting$i";
290 open my $fh, ">", $file_from or die "Could not open: $!";
291 print $fh time, ":", $file_from;
292 close $fh or die "Could not close: $!";
293 $rf->update($file_from,"new");
296 my $dagg2 = $rf->_debug_aggregate;
298 ok($dagg1->[0]{size} < $dagg2->[0]{size}, "The second 5s file size larger: $dagg1->[0]{size} < $dagg2->[0]{size}");
299 ok($dagg1->[1]{mtime} <= $dagg2->[1]{mtime}, "The second 30s file timestamp larger: $dagg1->[1]{mtime} <= $dagg2->[1]{mtime}");
300 is $dagg1->[2]{size}, $dagg2->[2]{size}, "The 1m file size unchanged";
301 is $dagg1->[3]{mtime}, $dagg2->[3]{mtime}, "The 1h file timestamp unchanged";
302 ok -l "t/ta/RECENT.recent", "found the symlink";
303 my $have_slept = my $have_worked = 0;
304 $start = Time::HiRes::time;
311 ($i<25) ? ($i%12) : $i,
313 open my $fh, ">", $file or die "Could not open '$file': $!";
314 print $fh time, ":", $file, "\n";
315 close $fh or die "Could not close '$file': $!";
316 my $another_rf = File::Rsync::Mirror::Recentfile->new
319 localroot => $root_from,
320 aggregator => [qw(10s 30s 1m 1h Z)],
322 $another_rf->update($file,"new");
323 my $should_have = 97 + (($i<25) ? ($i < 12 ? ($i+1) : 12) : ($i-12));
324 my($news,$filtered_news);
326 $another_rf->aggregate;
329 my $recc = File::Rsync::Mirror::Recent->new
331 local => "$root_from/RECENT-5s.yaml",
333 $news = $recc->news ();
334 $filtered_news = [ uniq map { $_->{path} } @$news ];
336 is scalar @$filtered_news, $should_have, "should_have[$should_have]" or die;
337 $debug->[$i] = $news;
338 my $rf2 = File::Rsync::Mirror::Recentfile->new_from_file("$root_from/RECENT-5s.yaml");
339 my $rece = $rf2->recent_events;
340 my $rececnt = @$rece;
341 my $span = $rece->[0]{epoch} - $rece->[-1]{epoch};
342 $have_worked = Time::HiRes::time - $start - $have_slept;
344 && ($i<50 ? $span <= 5 # we have run aggregate, so it guaranteed(*)
345 : $i < 90 ? 1 # we have not yet spent 5 seconds, so cannot predict
346 : $span > 5 # we have certainly written enough files now, must happen
349 ("i[%s]cnt[%s]span[%s]worked[%6.4f]",
355 $have_slept += Time::HiRes::sleep 0.2;
357 # (*) "<=" instead of "<" because of rounding errors
364 $test_counter = $tests;
367 printf "#test_counter[%d]\n", $test_counter;
368 my $rf = File::Rsync::Mirror::Recentfile->new
370 filenameroot => "RECENT",
372 localroot => $root_to,
373 max_rsync_errors => 0,
374 remote_dir => $root_from,
376 max_files_per_connection => 512,
381 # not available in rsync 3.0.3: 'omit-dir-times' => 1,
389 $success = $rf->mirror;
390 my $re = $rf->recent_events;
391 $somefile_epoch = $re->[24]{epoch};
392 } elsif (1 == $pass) {
393 $success = $rf->mirror(after => $somefile_epoch);
395 ok($success, "mirrored pass[$pass] without dying");
398 my $recc = File::Rsync::Mirror::Recent->new
399 ( # ($root_from, $root_to)
400 local => "$root_from/RECENT-5s.yaml",
403 #diag $recc->overview;
406 my $recc = File::Rsync::Mirror::Recent->new
408 # ignore_link_stat_errors => 1,
409 localroot => $root_to,
410 remote => "$root_from/RECENT-5s.yaml",
412 max_files_per_connection => 512,
423 my $recc = File::Rsync::Mirror::Recent->new
424 ( # ($root_from, $root_to)
425 local => "$root_from/RECENT-5s.yaml",
428 #diag $recc->overview;
431 my $recc = File::Rsync::Mirror::Recent->new
433 # ignore_link_stat_errors => 1,
434 localroot => $root_to,
435 remote => "$root_from/RECENT.recent",
436 verbose => $Opt{verbose},
437 max_files_per_connection => 512,
444 _runstatusfile => $statusfile,
447 my $rf2 = File::Rsync::Mirror::Recentfile->new_from_file("$root_from/RECENT-5s.yaml");
448 my $file = "$root_from/about-re-mirroring.txt";
449 open my $fh, ">", $file or die "Could not open '$file': $!";
451 close $fh or die "Could not close '$file': $!";
452 $rf2->update($file, "new");
456 my $recc = File::Rsync::Mirror::Recent->new
458 # ignore_link_stat_errors => 1,
459 localroot => $root_to,
460 local => "$root_to/RECENT.recent",
463 for my $rf (@{$recc->recentfiles}) {
464 my $dirtymark = $rf->dirtymark or next;
465 $seen{ $dirtymark }++;
467 is scalar keys %seen, 1, "all recentfiles have the same dirtymark or we don't know it";
471 rmtree [$root_from, $root_to, $statusfile] unless $Opt{verbose};
479 # cperl-indent-level: 4