bin/basesmoker-antihang.pl

   1 #!/usr/bin/perl
   2
   3 # use 5.010;
   4 use strict;
   5 use warnings;
   6
   7 =head1 NAME
   8
   9
  10
  11 =head1 SYNOPSIS
  12
  13
  14
  15 =head1 OPTIONS
  16
  17 =over 8
  18
  19 =cut
  20
  21 my @opt = <<'=back' =~ /B<--(\S+)>/g;
  22
  23 =item B<--help|h!>
  24
  25 This help
  26
  27 =item B<--longsleep=i>
  28
  29 After how many seconds we look again whether a new megalog file is
  30 there. Defaults to 600.
  31
  32 =item B<--megalog=s>
  33
  34 Do not try to determine the megalog file, only follow this one.
  35
  36 =item B<--sleep=i>
  37
  38 After how many seconds we look again into the megalog file. Defaults to 15.
  39
  40 A longer value reduces the noise on the console, a shorter ensures
  41 that we identify each problem earlier. For a while 120 was a good
  42 default but then I discovered that PDL-2.006 was hung repeatedly
  43 during testing and then I decided to reduce to 15. Now 150 in order to
  44 have a whole day in one screen window. Update 2022-12-25: since we
  45 were unable to kill Monoceros-0.28 on focal in a reasonable amount of
  46 time, we once again switch back to 15.
  47
  48 =back
  49
  50 =head1 DESCRIPTION
  51
  52
  53
  54 =head1 BUGS
  55
  56 - distrofilenames hardcoded
  57 - no globs
  58
  59 =cut
  60
  61
  62 use FindBin;
  63 use lib "$FindBin::Bin/../lib";
  64 BEGIN {
  65     push @INC, qw(       );
  66 }
  67 our $HAVE_SHUFFLE;
  68 BEGIN {
  69     $HAVE_SHUFFLE = eval { require Algorithm::Numerical::Shuffle; 1; };
  70 }
  71 use File::ReadBackwards;
  72 use File::Which ();
  73 use Getopt::Long;
  74 use Hash::Util qw(lock_keys);
  75 use List::Util qw(reduce);
  76 use Pod::Usage qw(pod2usage);
  77 use Proc::ProcessTable;
  78 use Sys::Hostname ();
  79
  80 File::Which::which('lsof') or die "required external command 'lsof' not installed, refusing to run";
  81
  82 our %Opt;
  83 lock_keys %Opt, map { /([^=|]+)/ } @opt;
  84 GetOptions(\%Opt,
  85            @opt,
  86           ) or pod2usage(1);
  87 if ($Opt{help}) {
  88     pod2usage(0);
  89 }
  90 $Opt{sleep} = 15 unless defined $Opt{sleep};
  91 $Opt{longsleep} = 600 unless defined $Opt{longsleep};
  92
  93 sub counting_sleep ($) {
  94     my $sleep = shift;
  95     my $start_time = time;
  96     my $start_time_str = scalar localtime $start_time;
  97     my $eta = $start_time + $sleep;
  98     local($|)=1;
  99     while () {
 100         my $left = $eta - time;
 101         last if $left < 0;
 102         printf "\r%s: sleeping %d: %d    ", $start_time_str, $sleep, $left;
 103         sleep 1;
 104     }
 105     print "\n";
 106 }
 107
 108 sub last_megalog () {
 109   PICKMEGA: while () {
 110         my $dh;
 111         until (opendir $dh, ".") {
 112             warn "Couldn't opendir .: $!; Sleeping\n";
 113             counting_sleep $Opt{sleep};
 114         }
 115         my @have_mtime = grep { -e $_ } readdir $dh;
 116         local($^T) = time;
 117         my @mfiles = grep { -M $_ < 60/86400 }
 118             grep { /megalog-\d+-\d+-\d+T\d+:\d+:\d+.log$/ } @have_mtime;
 119         unless (@mfiles) {
 120             warn "Couldn't find any recently changed megalog file; Sleeping\n";
 121             counting_sleep $Opt{longsleep};
 122             next PICKMEGA;
 123         }
 124         if ($HAVE_SHUFFLE) {
 125             Algorithm::Numerical::Shuffle::shuffle(\@mfiles);
 126         } else {
 127             @mfiles = sort { $b cmp $a } @mfiles;
 128         }
 129         my $self_host;
 130         for my $mfile (@mfiles) {
 131             open my $fh, "<", $mfile or next;
 132             local $/ = "\n";
 133             while (<$fh>) {
 134                 next unless m{^perl\|-\>\s+\S+installed-perls/(?:perl|host/([^/]+))/};
 135                 my $megalog_host = $1 || "k83";
 136                 $self_host //= Sys::Hostname::hostname;
 137                 if ($self_host eq $megalog_host) {
 138                     return $mfile;
 139                 }
 140             }
 141         }
 142         warn "Couldn't find an interesting megalog file; Sleeping\n";
 143         counting_sleep $Opt{longsleep};
 144     }
 145 }
 146
 147 {
 148 my $Lfile;
 149 my $Lsize;
 150 sub megalog_is_growing ($) {
 151     my($file) = @_;
 152     my $size0;
 153     if ($Lfile && $Lfile eq $file) {
 154         $size0 = $Lsize;
 155     } else {
 156         $size0 = -s $file;
 157         my $sleep = 3; # arbitrary
 158         counting_sleep $sleep;
 159     }
 160     my $size1 = -s $file;
 161     $Lfile = $file;
 162     $Lsize = $size1;
 163     my $growing;
 164     if ($size1 > $size0) {
 165         $growing = 1; # "growing";
 166     } else {
 167         $growing = 0; # "NOT growing";
 168     }
 169     warn sprintf "watching %s%s growing\n", $file, $growing ? "" : " NOT";
 170     return $growing;
 171 }
 172 }
 173
 174 sub kill_stalled_leaves ($) {
 175     my($proc) = @_;
 176     my $ppt = Proc::ProcessTable->new( 'enable_ttys' => 0 );
 177     my $ref = $ppt->table;
 178     my %parent_map = ($proc => 1);
 179     my %family;
 180     while () {
 181         my $keys = keys %parent_map;
 182         for my $p (@$ref) {
 183             next if $p->state eq "defunct"; # zombie
 184             if (exists $parent_map{$p->pid} && ! exists $family{$p->pid}) {
 185                 $family{$p->pid} = $p;
 186             }
 187             next unless $parent_map{$p->ppid};
 188             $parent_map{$p->pid} = $p->ppid;
 189         }
 190         next if keys %parent_map > $keys;
 191         last;
 192     }
 193     my $FORMAT = "%-6s %-6s %-8s %-24s %s\n";
 194     printf($FORMAT, "PID", "PPID", "STAT", "START", "COMMAND");
 195     my @tokill;
 196     my %map2 = %parent_map;
 197     while (%map2) {
 198         my %values = map {($_ => 1)} values %map2; # parents
 199         my @leaves = grep { ! $values{$_} } keys %map2;
 200         unless (@tokill) {
 201             @tokill = @leaves;
 202         }
 203         my $tp = pop @leaves;
 204         delete $map2{$tp};
 205         my $p = delete $family{$tp}
 206             or die "Panic: unexpected non-value in family tp='$tp'";
 207         printf($FORMAT,
 208                $p->pid,
 209                $p->ppid,
 210                $p->state,
 211                scalar(localtime($p->start)),
 212                $p->cmndline);
 213     }
 214     warn "ATTENTION: About to kill @tokill\n";
 215     my $signaled = kill 15, @tokill;
 216     warn "successfully signaled $signaled processes with signal 15\n";
 217     sleep 2;
 218  TOKILL: for my $pid (@tokill) {
 219         if (kill 0, $pid) {
 220             warn "process $pid still alive? killing with SIGKILL\n";
 221             if (kill 9, $pid) {
 222                 warn "sent signal successfully\n";
 223             } else {
 224                 warn "could not signal $pid\: $!\n";
 225             }
 226         } else {
 227             warn "verified that $pid is dead\n";
 228         }
 229     }
 230 }
 231
 232 # these times are observed values how long the slowest
 233 # operation of each distro can take without output
 234 # when it is not hanging. undef stands for not
 235 # measured
 236 my $TIMEOUTS =
 237 {
 238     "ARCv2-1.05"                     => 60, # seen 4
 239     "Acme-FSM-v2.3.5"                => 180, # seen 62
 240     "Acme-Tests-0.03"                => 30, # seen hanging
 241     # "WOLDRICH/Acme-DependOnEverything-0.06.tar.gz"
 242     "Acme-DependOnEverything-0.06"   => 60, # scheint ein Bremser zu sein
 243     "Algorithm-LinearManifoldDataClusterer-1.01" => 60, # seen 2
 244     "Alien-Boost-ProgramOptions-1.7" => 1000, # seen 2, aber auch 418, da passiert irgendein Download
 245     "Alien-FLTK2-0.09296"            => undef,
 246     "Alien-KentSrc-0.4"              => 60, # seen 1, but also 2971
 247     "Alien-Plotly-Kaleido-0.003"     => 120, # seen 1; still runs 2000 seconds because we can hardly kill it
 248     "Alien-QtSmoke-4.3.3"            => 60, # pass:fail=0:0 since 2009
 249     "Alien-SDL-1.444"                => undef,
 250     "Analizo-1.23.0"                 => 600, # seen 190
 251     "AnyEvent-AggressiveIdle-0.04"   => 120, # seen 3
 252     "AnyEvent-Fork-Pool-1.2"         => 30, # seen 2
 253     "AnyEvent-Gearman-0.10"          => 60, # seen 6, seen hanging with 5.22.2
 254     "AnyEvent-IRC-Server-0.03"       => 60, # seen 6
 255     "AnyEvent-JSONRPC-Lite-0.15"     => undef,
 256     "AnyEvent-RetryTimer-0.1"        => 60, # seen 2
 257     "AnyEvent-SIP-0.002"             => 60, # https://github.com/xsawyerx/anyevent-sip/issues/1
 258     "AnyEvent-Tickit-0.01"           => 60, # seen nothing
 259     "Apache-SessionX-2.01"           => undef,
 260     "App-shcompgen-0.321"            => 60, # seen 0.48
 261     "App-Tel-0.201601"               => 60, # seen 3
 262     "Argon-0.18"                     => 60, # seen 6
 263     "Attean-0.032"                   => 480,, # seen 175
 264     "Authen-Krb5-Easy-0.90"          => 30, # http://www.cpantesters.org/cpan/report/8f9d58e6-fbd7-11e6-91d1-8b9078ddaa53 looks like waiting for input, released 2002
 265     "DB_File-SV18x-kit-0.06"         => 60, # did not look
 266     "Badger-0.09"                    => 60, # seen 9, seen hanging as 5.12.5
 267     "Benchmark-Thread-Size-0.09"     => 60, # seen 3, seen > 2200 with 5.35.10
 268     "Bio-Medpost-0.04"               => 60, # seen nothing, abandoned
 269     "Business-OnlinePayment-Exact-0.01" => 60, # seen nothing
 270     "CGI-Application-4.50"           => 30,
 271     "CGI-Application-Server-0.063"   => 30, # seen 3
 272     "CGI-Compile-0.26"               => 180, # seen 1, 2, 26, 46; 2020-08-03
 273     "CGI-Debug-1.0"                  => 30, # abandoned since 2000?
 274     "CPANPLUS-Daemon-0.02"           => 120, # seen 1
 275     "CPAN-Reporter-Smoker-0.29"      => 300, # seen 41, but today >4000
 276     "CPAN-Testers-WWW-Reports-Query-Report-0.05" => 120, # no passes with perl >= 5.34
 277     "CSS-Inliner-3935"               => 10, #3934 whole job took 5 seconds
 278     "CSS-Sass-3.3.6"                 => 60, # seen 7, seen hanging with 5.12.5
 279     "Cache-Memcached-Semaphore-0.3"  => 60, # seems abandoned
 280     "Catalog-0.5"                    => 60, # 0 reports ever, fills disk quickly with "(default /home/httpd/cgi-bin, type '' for empty string) cgidir : /home/httpd/cgi-bin is not an existing directory"
 281     "Catalog-1.02"                   => 60, # 0 reports ever, fills disk quickly with "(default /home/httpd/cgi-bin, type '' for empty string) cgidir : /home/httpd/cgi-bin is not an existing directory"
 282     "Catalyst-View-Reproxy-0.05"     => 120, # seen 13
 283     "Catmandu-XSD-0.05"              => 60, # seen 10
 284     "Chart-GRACE-0.95"               => 60, # declared abandoned in 01.DISABLED
 285     "Class-Std-0.011"                => 10,
 286     "Cluster-Init-0.215"             => 120, # seen 4332! pass:fail:unknown=(0/7/1), abandoned since 2003
 287     "Concurrent-Object-1.07"         => 30, # seen 1
 288     "Continuity-1.6"                 => 30, # seen 2 (without skipping tests)
 289     "Coro-6.41"                      => 10,
 290     "Costume-Doublet-0.001"          => 120, # seen 0
 291     "DBIx-Broker-1.14"               => 60, # seen nothing, only red but one unavailable one
 292     "DBIx-ObjectMapper-0.3013"       => undef,
 293     "DBIx-SQLCrosstab-1.17"          => 300, # watched 120
 294     "DBIx-XMLServer-0.02"            => 60,
 295     "DBD-iPod-0.01"                  => 60, # seen 0
 296     "DDCCI-0.003"                    => 60, # seen 0: interactively asks whether we have a DDC/CI standard compliant monitor attached
 297     "DJabberd-0.85"                  => undef,
 298     "Daemon-Device-1.07"             => 60, # seen 7
 299     "Data-RecordStore-6.06"          => 60, # seen 2
 300     "Data-STUID-0.01"                => 60, # nothing seen
 301     "Data-Serializer-Sereal-1.05"    => 60, # seen 0
 302     "Date-LastModified-0.60"         => 120, # seen 30
 303     "Debug-Client-0.31"              => 120, # seen 38
 304     "Devel-Debug-DBGp-0.22"          => 120, # seen 21
 305     "Devel-GlobalDestruction-XS-0.03"=> 60, # seen 1, hanger on 5.8.2
 306     "Devel-TrackSIG-0.03"            => 10,
 307     "Devel-Trepan-0.73"              => 300,
 308     "Devel-Trepan-v1.0.0"            => 300,
 309     "Dist-Zilla-Plugin-StaticInstall-0.012" => 120, # seen 19
 310     "Dpkg-1.21.10"                   => 60, # seen 14
 311     "ETL-Yertl-0.043"                => 300, # seen 42
 312     "Emacs-EPL-0.7"                  => 30, # seen 1
 313     "Enbugger-2.016"                 => 300, # seen 38-96
 314     "Event-RPC-1.10"                 => 120, # seen 8
 315     "FAQ-OMatic-2.702"               => 60, # nothing seen
 316     "FAQ-OMatic-2.717"               => 60, # seen 1
 317     "FAQ-OMatic-2.719"               => 60, # seen 1
 318     "FFI-Platypus-0.40"              => 300, # seen 160
 319     "Feersum-1.405"                  => 30, # fails always anyway
 320     "File-BSDGlob-0.94"              => 30, # seen 1 https://rt.cpan.org/Ticket/Display.html?id=95586
 321     "File-Flock-2014.01"             => 90, # seen 14
 322     "File-Lock-Multi-1.02"           => 90, # seen 19
 323     "Forecast-IO-0.21"               => 30, # asks questions, https://rt.cpan.org/Public/Bug/Display.html?id=86798
 324     "FreeRADIUS-Database-0.06"       => 90, # seen 8
 325     "Games-AlphaBeta-v0.4.7"         => undef,
 326     "Games-Baseball-Scorecard-0.03"  => 10, # seen 0
 327     "Games-Rezrov-0.15"              => 120, # seen nothing
 328     "Games-Rezrov-0.20"              => 120, # seen nothing
 329     "Gearman-1.12"                   => 180, # seen 17
 330     "Gearman-Driver-0.02008"         => 600, # seen 104
 331     "Gearman-Starter-0.03"           => 60, # seen 2
 332     "Gearman-SlotManager-0.3"        => 10, # 2012, pass:fail=10:270
 333     "GH-0.69"                        => 10, # pass:fail=6:270
 334     "Hardware-1Wire-HA7Net-1.01"     => 60, # only red
 335     "Hash-SharedMem-0.005"           => 300, # seen 130
 336     "HPCI-0.75"                      => 900, # seen 432: difficult to kill, leave is a sleep most of the time
 337     "HTML-EmbeddedPerl-0.91"         => 30, # seen 0, https://rt.cpan.org/Ticket/Display.html?id=88731
 338     "HTML-EP-0.1135"                 => 10, # not current, but is indexed for HTML::EP::Examples::Glimpse
 339     "HTTP-Proxy-0.304"               => 30, # have reported the hang
 340     "HTTP-ProxySelector-Persistent-0.02" => 60, # seen nothing
 341     "HTTP-Server-EV-0.69"            => 60, # seen 3
 342     "HTTP-Server-Simple-0.52"        => 180, # seen 20
 343     "Image-Signature-0.01"           => 60, # seen 13, ~/.cpan/plugins/CPAN::Plugin::TraceDeps/20210806T140441.log
 344     "IO-EventMux-2.02"               => 60, # seen 1, https://rt.cpan.org/Ticket/Display.html?id=120205
 345     "IO-Socket-ByteCounter-v0.0.2"   => 60, # seen 0
 346     # OLEG/IO-Socket-Socks-Wrapper-0.17.tar.gz
 347     "IO-Socket-Socks-Wrapper-0.17"   => 120, # seen 40; t/10_event_loop_mojo.t ...... skipped: Mojolicious 4.85+ required for this test
 348     "IPC-AnyEvent-Gearman-0.8"       => 180, # seen 29
 349     "IPC-Exe-2.002001"               => 5,
 350     "IPC-Pipeline-0.8"               => 30,
 351     "IPC-Shareable-1.05"             => 120, # seen 13
 352     "IPC-Transit-1.171860"           => 300, # seen 107, only fails
 353     "IS-Init-0.93"                   => 300, # seen 174
 354     "JIRA-REST-Class-0.12"           => 60, # seen 3
 355     "JTM-Boilerplate-2.190700"       => 60, # seen 3
 356     "JTM-Boilerplate-2.211420"       => 60, # did not look
 357     "Job-Async-0.004"                => 60, # seen 1
 358     "Jvm-0.9.2"                      => 60, # seen nothing
 359     "Luka-1.07"                      => 60, # pass:fail = 0:74; was hanging for two hours
 360     "LWP-Protocol-Coro-http-v1.0.7"  => 10,
 361     "Lingua-POSAlign-0.01"           => 10, # seen 0
 362     "MListbox-1.11"                  => 120, # nothing seen
 363     "MMM-Text-Search-0.07"           => 300, # seen nothing
 364     "Mail-Abuse-1.026"               => 60, # just added to abandoned
 365     "Mail-MtPolicyd-2.05"            => 60, # seen nothing
 366     "Mail-SendEasy-1.2"              => 60, # seen nothing
 367     "Mail-SpamAssassin-Contrib-Plugin-IPFilter-1.2" => 180, # seen 25
 368     "Mail-Webmail-Yahoo-0.601"       => 60, # seen 1
 369     "Mason-Tidy-2.57"                => 60, # seen 1
 370     "Math-BigInt-Random-OO-0.04"     => 300, # seen 70, see https://rt.cpan.org/Ticket/Display.html?id=132452
 371     "Math-Matlab-0.08"               => 60, # seen 1
 372     "Math-Prime-Util-0.73"           => 60, # seen 7; https://rt.cpan.org/Ticket/Display.html?id=139739
 373     "MediaWiki-Bot-5.006003"         => 60, # seen 17
 374     "Minion-Backend-Storable-7.012"  => 60, # seen 3
 375     "Module-Build-0.4005"            => 60, # eh schon alt, egal
 376     "Module-Build-Convert-0.49"      => 60, # seen 1
 377     "Module-Refresh-0.17"            => 60, # seen 2, seen hanging in v5.12.5
 378     "Mojolicious-Plugin-AssetPack-1.13" => 120, # seen 25
 379     "Mojolicious-Plugin-AssetPack-1.24" => 120,
 380     "Mojolicious-Plugin-Minion-Starter-0.005" => 60, # seen 3
 381     "Mojolicious-Plugin-OAuth2-Server-0.48" => 120, # seen 20
 382     "Mojo-ACME-0.13"                 => 60, # seen 1
 383     "Mojo-Promise-Role-Repeat-0.006" => 120,
 384     "Monoceros-0.29"                 => 180, # seen 90
 385     "MooseX-Workers-0.24"            => 60, # seen 10
 386     "MySQL-TableInfo-1.01"           => 60, # nothing seen
 387     "Net-AMQP-RabbitMQ-0.007001"     => undef,
 388     "Net-AMQP-RabbitMQ-1.400000"     => 110, # seen 53 total, but no stalls>127
 389     "Net-AMQP-RabbitMQ-2.30000"      => 110,
 390     "Net-Async-Blockchain-0.003"     => 120, # seen 8
 391     "Net-Dropbear-0.16"              => 800, # seen 460 and 17
 392     "Net-FTPServer-1.125"            => 60, # much red, no release since 2012
 393     "Net-Goofey-1.4"                 => 60, # seen nothing
 394     "Net-HTTPS-NB-0.15"              => 60, # seen 7
 395     "Net-LDAP-Server-Test-0.14"      => undef,
 396     "Net-Netcat-0.05"                => undef,
 397     "Net-OpenID-JanRain-1.1.1"       => 60, # only red
 398     "Net-OpenSRS-0.06"               => 60, # seen "No tests defined"
 399     "Net-Proxy-Type-0.09"            => 60, # seen 7
 400     "Net-WhoisNG-0.09"               => 60, # seen 1; only fails since 2018
 401     "NetServer-Generic-1.03"         => 3600, # haengt, aber am Schluss gibts ok; heute aber nicht bei k93bionic/v5.31.2/2aaf
 402     "Number-Phone-3.5000"            => 600, # seen 420
 403     "Number-Phone-AU-0.02"           => 60, # seen 1
 404     "ODS-0.04"                       => 60, # seen 9
 405     "Object-Lazy-0.13"               => 30,
 406     "OpenFrame-Segment-Apache-1.20"  => 60, # seen 1
 407     "Otogiri-Plugin-BulkInsert-0.02" => 60, # seen 4
 408     "PBJ-JNI-0.1"                    => 120, # nothing seen
 409     "PDF-Create-1.43"                => 60, # seen 3
 410     "PDL-2.006"                      => undef,
 411     "POE-Component-Client-HTTP-0.949" => 120, # seen 33
 412     "POE-Component-Client-Keepalive-0.272" => 300, # seen 40
 413     # BINGOS/POE-Component-CPAN-YACSmoke-1.38.tar.gz
 414     "POE-Component-CPAN-Reporter-0.08" => 120, # seen 7
 415     "POE-Component-CPAN-YACSmoke-1.38" => 180, # seen 7
 416     "POE-Component-DirWatch-0.300004" => 180, # seen 26
 417     "POE-Component-Server-Inet-0.06" => 120, # seen 34
 418     "POE-Component-Server-SimpleHTTP-PreFork-2.10" => 1200, # seen 305
 419     "POE-Loop-IO_Async-0.004"        => 120, # seen 36
 420     "POE-Quickie-0.18"               => undef,
 421     "POEx-Role-PSGIServer-1.150280"  => 30, # DEPREACATED
 422     "POE-XS-Loop-Poll-1.000"         => 180, # seen 74
 423     "POEx-Tickit-0.04"               => 60,
 424     "PPerl-0.25"                     => 60, # seen nothing
 425     "Parallel-DataPipe-0.12"         => 120, # seen 1
 426     "PerlQt-3.006"                   => 120, # seen 13
 427     "PerlQt-3.008"                   => 120, # seen 13
 428     "Perl6-Pod-Slide-0.10"           => 60, # seen 2
 429     "Plugin-Simple-1.01"             => 120, # seen 5
 430     "Pod-Trial-LinkImg-0.005"        => 30, # seen 1
 431     "Proch-N50-1.5.0"                => 60, # seen 3
 432     "Project-Euler-0.20"             => 10,
 433     "Prophet-0.751"                  => 120, # seen 40
 434     "RAS-HiPerARC-1.03"              => 60, # seen "Terminated" in http://www.cpantesters.org/cpan/report/4a40775e-7d78-11e9-ac9a-059c23201cad
 435     "RAS-PortMaster-1.16"            => 120, # seen nothing
 436     "RT-Client-REST-0.43"            => 20,
 437     "RT-Extension-ShiftPlanning-0.01.tar.gz" => 20, # seen 1
 438     "Redis-1.995"                    => 180, # seen 75
 439     "Redis-Jet-0.09"                 => 60, # seen 2
 440     "Rx-0.53"                        => 30, # asks questions
 441     "SNMP-Util-1.8"                  => 60, # abandoned
 442     "Search-Sitemap-2.13"            => 120, # seen 6
 443     "Senna-0.51"                     => 60, # https://rt.cpan.org/Ticket/Display.html?id=105318
 444     "Server-Starter-0.12"            => undef,
 445     "Server-Starter-0.19"            => undef,
 446     "Server-Starter-0.32"            => undef,
 447     "Server-Starter-0.34"            => 300, # seen 94
 448     "Sport-Analytics-NHL-1.53"       => 300, # seen 152
 449     "StandupGenerator-0.5"           => 60, # only red results and hanging
 450     "Starlet-0.31"                   => 300, # seen 16
 451     "Starman-0.4015"                 => 360, # seen 152
 452     "Sub-Call-Tail-0.05"             => 1,
 453     "Sudo-0.33"                      => 1800, # seen 941
 454     "Sybase-Xfer-0.1"                => 123, # nothing seen
 455     "Sybase-Xfer-0.63"               => 123, # nothing seen
 456     "Syntax-Keyword-Try-0.10"        => 60, # seen 4
 457     "Sys-SigAction-0.23"             => 90, # seen 11
 458     "Tangram-2.04"                   => 60, # nothing seen
 459     "Tapper-Testplan-4.1.2"          => undef,
 460     "Tcl-pTk-1.09"                   => 420, # seen 163
 461     "Test-HTTP-Server-Simple-0.11"   => 60, # seen 0
 462     "Test-Moose-More-0.027"          => 1, # haengt anscheinend immer, braucht keine Chance
 463     "Test-SFTP-1.10"                 => 180, # seen 24
 464     "Test-WWW-Simple-0.39"           => 120, # seen 38
 465     "Text-Glob-DWIW-0.01"            => 120, # seen 17
 466     "Thread-Isolate-0.05"            => 180, # seen 26
 467     "Thread-Queue-Multiplex-0.92"    => 1200, # seen 816
 468     "Thread-Workers-0.06"            => 60, # seen 11
 469     "Tk-804.030"                     => 10,
 470     "Tk-GridEntry-1.0"               => 60,
 471     "Tk-LockDisplay-1.3"             => 30, # nothing seen
 472     "Tk-MListbox-1.11"               => 120, # nothing seen
 473     "Tk-PopEntry-0.06"               => 60, # nothing seen
 474     "Tk-QuickTk-0.92"                => 60, # seen 0
 475     "Tk-TM-0.53"                     => 60,
 476     "Tk-Updown-1.0"                  => 30, # seen hanging when Tk is installed
 477     "TkUtil-Configure-0.03"          => 10, # seen 0
 478     "TripleStore-0.03"               => 0, # seen 60
 479     "Twiggy-0.1025"                  => 120, # seen 6
 480     "Twiggy-Prefork-0.08"            => 120, # seen 8
 481     "UNIVERSAL-isa-1.20171012"       => 60, # seen 1
 482     "Vim-Debug-0.904"                => 120, # seen 14
 483     "WWW-Google-Groups-0.09"         => 60, # always red
 484     "WWW-Search-Googlism-0.02"       => 60, # released 2003
 485     "WWW-VieDeMerde-0.21"            => 60, # seen nothing
 486     "WWW-UsePerl-Journal-0.26"       => 60, # seen 21
 487     "WebService-RESTCountries-0.3"   => 60, # seen 7 https://github.com/kianmeng/webservice-restcountries/issues/1
 488     "WebService-Async-UserAgent-0.006" => 60, # seen 1
 489     "What-1.00"                      => 60, # seen 0
 490     "Win32-MSAgent-0.07"             => 60, # nothing seen
 491     "Winamp-Control-0.2.1"           => 60, # vermute userinputversuch
 492     "X11-Protocol-0.56"              => 60, # seen ok 1..3
 493     "XAO-MySQL-1.0"                  => 10, # sometimes hangs
 494     "XML-ExtOn-0.17"                 => 60, # seen 1
 495     "XML-Grammar-Fortune-0.0501"     => undef,
 496     "Yandex-Disk-0.07"               => 2400, # seen 1775
 497     "ZeroMQ-PubSub-0.10"             => undef,
 498     "code-UnifdefPlus-v0.4.0"        => 60, # nothing seen
 499     "go-db-perl-0.04"                => 60, # no pass since 2009
 500     "forks-0.36"                     => 300, # seen 149
 501     "math-image-109"                 => undef,
 502     "nsapi_perl-0.24"                => 60, # seem nothing
 503     "perldap-1.4"                    => 60, # look like abandoned since 1999
 504 };
 505
 506 while () {
 507     my $file = $Opt{megalog} || last_megalog;
 508     if (megalog_is_growing($file)) {
 509         my $bw = File::ReadBackwards->new( $file ) or
 510             die "can't read '$file' $!" ;
 511         my $log_line;
 512         my $proc;
 513         my $max = 0;
 514         my $min = 9999999999;
 515         while( defined( $log_line = $bw->readline ) ) {
 516             # hanging Starlet:
 517             next if $log_line =~
 518                 /^(\QCan't exec "": No such file or directory at\E
 519                  |\Qnew worker \E\d+\Q seems to have failed to start, exit status\E
 520                  |\Qstarting new worker\E
 521                  )/x;
 522             last unless $log_line =~ /====/;
 523             next unless $log_line =~ /={10}monitoring proc (\d+) perl (\S+) secs ([0-9\.]+)={7}/; # external invariable
 524             $proc = $1;
 525             my $perl = $2;
 526             my $time = $3;
 527             $max = $time if $time > $max;
 528             $min = $time if $time < $min;
 529         }
 530         if ($proc) {
 531             my $stalled = $max - $min;
 532             if ($stalled > 3) { # arbitrary
 533                 my $cwd = `lsof -p $proc | awk '\$4=="cwd"{print \$9}'`;
 534                 chomp $cwd;
 535                 warn "process $proc stalled for $stalled seconds in $cwd\n";
 536                 my $timeout = 3600; # arbitrary
 537                 while (my($distro,$ttimeout) = each %$TIMEOUTS) {
 538                     if ($cwd =~ m!/\Q$distro\E-!) {
 539                         $timeout = $ttimeout || 120;
 540                         last;
 541                     }
 542                 }
 543                 keys %$TIMEOUTS; # do we need to reset the iterator here?
 544                 if ($stalled >= $timeout) { # arbitrary
 545                     warn sprintf "no output for %d seconds (timeout=%d)\n", $stalled, $timeout;
 546                     kill_stalled_leaves ($proc);
 547                 }
 548             } else {
 549                 warn "working OK\n";
 550             }
 551         } else {
 552             warn "is busy\n";
 553         }
 554     }
 555     counting_sleep $Opt{sleep}; # arbitrary
 556 }
 557
 558 # Local Variables:
 559 # mode: cperl
 560 # cperl-indent-level: 4
 561 # End: