bin/m-l-smokerpt-download.pl

   1 #!/usr/bin/perl
   2
   3 # use 5.010;
   4 use strict;
   5 use warnings;
   6
   7 =head1 NAME
   8
   9
  10
  11 =head1 SYNOPSIS
  12
  13
  14
  15 =head1 OPTIONS
  16
  17 =over 8
  18
  19 =cut
  20
  21 my @opt = <<'=back' =~ /B<--(\S+)>/g;
  22
  23 =item B<--help|h!>
  24
  25 This help
  26
  27 =back
  28
  29 =head1 DESCRIPTION
  30
  31 m-l.org/~perl/smoke/perl/... holds one month worth of smokereports.
  32
  33 This script fetches a collection of reports and stores them in a
  34 single YAML file.
  35
  36 =head1 BUGS
  37
  38 This is a one-off script with shortcuts.
  39
  40 =head1 SEE ALSO
  41
  42 m-l-smokerpt-parse.pl for parsing the contents.
  43
  44 =cut
  45
  46
  47 use FindBin;
  48 use lib "$FindBin::Bin/../lib";
  49 BEGIN {
  50     push @INC, qw(       );
  51 }
  52
  53 use Dumpvalue;
  54 use File::Basename qw(dirname);
  55 use File::Path qw(mkpath);
  56 use File::Spec;
  57 use File::Temp;
  58 use Getopt::Long;
  59 use Hash::Util qw(lock_keys);
  60
  61 our %Opt;
  62 lock_keys %Opt, map { /([^=|!]+)/ } @opt;
  63 GetOptions(\%Opt,
  64            @opt,
  65           ) or pod2usage(1);
  66
  67 our $DIR_URL = "http://m-l.org/~perl/smoke/perl/linux/blead_clang/";
  68
  69 use LWP::UserAgent ();
  70 use POSIX ();
  71 use Scalar::Util ();
  72 use Text::Trim qw(trim);
  73 use XML::LibXML ();
  74 use YAML::Syck ();
  75
  76 my $ua = LWP::UserAgent->new;
  77 my $p  = XML::LibXML->new;
  78 my $S;
  79
  80 my $resp = $ua->get($DIR_URL);
  81 if ($resp->is_success) {
  82     my $content = $resp->decoded_content;
  83     my $dom = XML::LibXML->load_html(string => $content);
  84  TR: for my $tr_ele ($dom->findnodes("/html/body/table/tr")){
  85         my($filename_attr) = $tr_ele->findnodes("td[2]/a/\@href") or next;
  86         my $filename = $filename_attr->value;
  87         next TR unless $filename =~ /^rpt.*\.rpt$/;
  88         my $timestamp = $tr_ele->findnodes("td[3]/.");
  89         trim $timestamp;
  90         my $resp = $ua->get("$DIR_URL/$filename");
  91         if ($resp->is_success) {
  92             my $content = $resp->decoded_content;
  93             $S->{$filename}{content} = $content;
  94             $S->{$filename}{timestamp} = $timestamp;
  95         } else {
  96             die
  97         }
  98     }
  99 } else {
 100     die
 101 }
 102
 103 my $ts = POSIX::strftime "%FT%T", localtime;
 104 my $outfile = __FILE__ . "-$ts.out";
 105 YAML::Syck::DumpFile($outfile, $S);
 106
 107 # Local Variables:
 108 # mode: cperl
 109 # cperl-indent-level: 4
 110 # End: