bin/get-historic-reportflow.pl

   1 use strict;
   2 use warnings;
   3
   4 =head1 NAME
   5
   6
   7
   8 =head1 SYNOPSIS
   9
  10
  11
  12 =head1 OPTIONS
  13
  14 =over 8
  15
  16 =cut
  17
  18 my @opt = <<'=back' =~ /B<--(\S+)>/g;
  19
  20 =item B<--help|h!>
  21
  22 This help
  23
  24 =back
  25
  26 =head1 DESCRIPTION
  27
  28 Most probably a one-off script because we hope to get this in munin then.
  29
  30 =cut
  31
  32
  33 use FindBin;
  34 use lib "$FindBin::Bin/../lib";
  35 BEGIN {
  36     push @INC, qw(       );
  37 }
  38
  39 use Dumpvalue;
  40 use Fcntl ":seek";
  41 use File::Basename qw(dirname);
  42 use File::Path qw(mkpath);
  43 use File::Spec;
  44 use File::Temp;
  45 use Getopt::Long;
  46 use Hash::Util qw(lock_keys);
  47 use Time::HiRes qw(sleep time);
  48
  49 our %Opt;
  50 lock_keys %Opt, map { /([^=]+)/ } @opt;
  51 GetOptions(\%Opt,
  52            @opt,
  53           ) or pod2usage(1);
  54
  55 my $mb_log = "$ENV{HOME}/var/metabase-log/metabase.log-proc.log.1";
  56 # 20121018T195811:INFO[13627]: Newlines written: 506, Keeping in mem records: 1000
  57
  58 my $refill_dir = "$ENV{HOME}/var/refill-cpanstatsdb";
  59 # .../2012/10/20121027T2308-24182412-24182626.json.gz => 215 records
  60
  61 my $Pstamp_s = "";
  62 my $Ptell = 202000;
  63 for (my $t = 1328313600; $t < time; $t+=300) {
  64     my($mb_cnt,$refill_cnt) = (0,0);
  65     my @time = gmtime($t);
  66     $time[4]++;
  67     $time[5]+=1900;
  68     my $stamp_s = sprintf "%04d%02d%02dT%02d%02d%02d", @time[5,4,3,2,1,0];
  69     open my $fh, $mb_log or die "Could not open $mb_log\: $!";
  70     seek $fh, $Ptell, SEEK_SET;
  71     while (<$fh>) {
  72         next unless /^(\d{8}T\d{6}):INFO\[\d+\]: Newlines written:\s(\d+),/;
  73         my($tstamp,$tcnt) = ($1,$2);
  74         last if $tstamp gt $stamp_s;
  75         next if $tstamp lt $Pstamp_s;
  76         next unless $tcnt;
  77         $mb_cnt+=$tcnt;
  78     }
  79     $Ptell = tell $fh;
  80     $Ptell -= 512; # certainly too much
  81     opendir my $dh, $refill_dir or die "Could not opendir $refill_dir\: $!";
  82     my %logfiles;
  83     for my $y (sort readdir $dh) {
  84         next unless $y =~ /^(\d{4})$/;
  85         last if $y gt $stamp_s;
  86         next if $y lt substr($stamp_s,0,4);
  87         opendir my $dh2, "$refill_dir/$y" or die "Could not opendir $refill_dir/$y\: $!";
  88         for my $m (sort readdir $dh2) {
  89             next unless $m =~ /^(\d{2})$/;
  90             last if "$y$m" gt $stamp_s;
  91             next if "$y$m" lt substr($stamp_s,0,6);
  92             opendir my $dh3, "$refill_dir/$y/$m" or die "Could not opendir $refill_dir/$y/$m\: $!";
  93             for my $logfile (sort readdir $dh3) {
  94                 next unless $logfile =~ /(\d{8}T\d{4})-(\d+)-(\d+)\.json\.gz$/;
  95                 my($tstamp,$tstart,$tend) = ($1,$2,$3);
  96                 last if $tstamp gt $stamp_s;
  97                 next if $tstamp le substr($Pstamp_s,0,13);
  98                 my $tcnt = $tend - $tstart + 1;
  99                 $refill_cnt+=$tcnt;
 100                 $logfiles{$logfile}=1;
 101             }
 102         }
 103     }
 104     next unless $mb_cnt || $refill_cnt;
 105     printf "%s %d %8d %8d %s\n", $stamp_s, $t, $mb_cnt, $refill_cnt, join("|",sort keys %logfiles);
 106     $Pstamp_s = $stamp_s;
 107 }