bin/analysis-count-everything.pl

   1 #!/usr/bin/perl
   2
   3 # use 5.010;
   4 use strict;
   5 use warnings;
   6
   7 =head1 NAME
   8
   9
  10
  11 =head1 SYNOPSIS
  12
  13
  14
  15 =head1 OPTIONS
  16
  17 =over 8
  18
  19 =cut
  20
  21 my @opt = <<'=back' =~ /B<--(\S+)>/g;
  22
  23 =item B<--help|h!>
  24
  25 This help
  26
  27 =item B<--sleep=f>
  28
  29 Sleep that much between two calls to distrofacts.
  30
  31 =item B<--verbose|v!>
  32
  33 Even more verbose than we already are by default (but this may change)
  34
  35 =back
  36
  37 =head1 DESCRIPTION
  38
  39 Our first iteration is just a letter of intent that we want to become
  40 better on empty pages.
  41
  42 =head1 HISTORY
  43
  44 Up to commit 32b522a we were using
  45
  46   redis->srandmember("analysis:distv:legalset")
  47
  48 but that started to dry out mid-July 2014, so we switched to a
  49 postgres implementation.
  50
  51 =cut
  52
  53
  54 use FindBin;
  55 use lib "$FindBin::Bin/../lib";
  56 BEGIN {
  57     push @INC, qw(       );
  58 }
  59
  60 use Dumpvalue;
  61 use File::Basename qw(dirname);
  62 use File::Path qw(mkpath);
  63 use File::Spec;
  64 use File::Temp;
  65 use Getopt::Long;
  66 use Pod::Usage;
  67 use Hash::Util qw(lock_keys);
  68 use Time::HiRes qw(sleep);
  69
  70 our %Opt;
  71 lock_keys %Opt, map { /([^=|!]+)/ } @opt;
  72 GetOptions(\%Opt,
  73            @opt,
  74           ) or pod2usage(1);
  75 if ($Opt{help}) {
  76     pod2usage(0);
  77 }
  78 $Opt{sleep} //= 30;
  79
  80 my($workdir);
  81 use FindBin;
  82 use lib "$FindBin::Bin/../CPAN-Blame/lib";
  83 use CPAN::Blame::Config::Cnntp;
  84 BEGIN {
  85     $workdir = File::Spec->catdir
  86         ($CPAN::Blame::Config::Cnntp::Config->{solver_vardir},
  87          "workdir");
  88 }
  89 use IPC::ConcurrencyLimit;
  90
  91 my($basename) = File::Basename::basename(__FILE__);
  92 my $limit = IPC::ConcurrencyLimit->new
  93     (
  94      max_procs => 1,
  95      path      => "$workdir/IPC-ConcurrencyLimit-$basename",
  96     );
  97 my $limitid = $limit->get_lock;
  98 if (not $limitid) {
  99     warn "Another process appears to be still running. Exiting.";
 100     exit(0);
 101 }
 102
 103 use Redis;
 104 my $redis = Redis->new(reconnect => 120, every => 1000);
 105 sub mypgdbi () {
 106     require DBI;
 107     my $dbi = DBI->connect ("dbi:Pg:dbname=analysis");
 108 }
 109 my $sth = mypgdbi()->prepare("SELECT EXTRACT(EPOCH FROM cuti_ts) from distlookup where distv=?");
 110 my $sth2 = mypgdbi()->prepare("SELECT cuti_ts is null as N0, count(*) from distlookup group by N0");
 111 my $sth3 = mypgdbi()->prepare("SELECT distv from distlookup where (cuti_ts is null or now() - cuti_ts > ?) and distv not like '%-' limit 1");
 112 my $start_days = 20;
 113 setpriority(0, 0, 5); # renice
 114 while () {
 115     $sth3->execute("$start_days days");
 116     if ($sth3->rows <= 0){
 117         if ($start_days > 1) {
 118             $start_days--;
 119             sleep 1;
 120             next;
 121         } else {
 122             last;
 123         }
 124     }
 125     my($distv) = $sth3->fetchrow_array;
 126     next if $distv =~ /-$/; # MyConfig2- KeywordsSpider- BitArray1- (last one reported to Barbie)
 127     warn "Running 'cnntp-solver.pl --cpanstats_distrofacts $distv'" if $Opt{verbose};
 128     my @system = ($^X,"$FindBin::Bin/cnntp-solver.pl", "--cpanstats_distrofacts", $distv);
 129     0 == system(@system) or die "Problems while running '@system'";
 130     my($legalset_total) = $redis->scard("analysis:distv:legalset");
 131     my($calctimestamp_total) = $redis->zcard("analysis:distv:calctimestamp");
 132     my($uncounted_total, $counted_total);
 133     {
 134         $sth2->execute;
 135         while (my @row = $sth2->fetchrow_array) {
 136             if ($row[0]) {
 137                 $uncounted_total = $row[1];
 138             } else {
 139                 $counted_total = $row[1];
 140             }
 141         }
 142     }
 143     warn "distv[$distv]redis:legalset_total[$legalset_total]redis:calctimestamp_total[$calctimestamp_total]pg:uncounted_total[$uncounted_total]pg:counted_total[$counted_total]\n";
 144     sleep $Opt{sleep};
 145 }
 146
 147 # Local Variables:
 148 # mode: cperl
 149 # cperl-indent-level: 4
 150 # End: