t/perf/aggregate.perl

   1 #!/usr/bin/perl
   2
   3 use lib '../../perl/build/lib';
   4 use strict;
   5 use warnings;
   6 use Getopt::Long;
   7 use Git;
   8 use Cwd qw(realpath);
   9
  10 sub get_times {
  11         my $name = shift;
  12         open my $fh, "<", $name or return undef;
  13         my $line = <$fh>;
  14         return undef if not defined $line;
  15         close $fh or die "cannot close $name: $!";
  16         # times
  17         if ($line =~ /^(?:(\d+):)?(\d+):(\d+(?:\.\d+)?) (\d+(?:\.\d+)?) (\d+(?:\.\d+)?)$/) {
  18                 my $rt = ((defined $1 ? $1 : 0.0)*60+$2)*60+$3;
  19                 return ($rt, $4, $5);
  20         # size
  21         } elsif ($line =~ /^\d+$/) {
  22                 return $&;
  23         } else {
  24                 die "bad input line: $line";
  25         }
  26 }
  27
  28 sub relative_change {
  29         my ($r, $firstr) = @_;
  30         if ($firstr > 0) {
  31                 return sprintf "%+.1f%%", 100.0*($r-$firstr)/$firstr;
  32         } elsif ($r == 0) {
  33                 return "=";
  34         } else {
  35                 return "+inf";
  36         }
  37 }
  38
  39 sub format_times {
  40         my ($r, $u, $s, $firstr) = @_;
  41         # no value means we did not finish the test
  42         if (!defined $r) {
  43                 return "<missing>";
  44         }
  45         # a single value means we have a size, not times
  46         if (!defined $u) {
  47                 return format_size($r, $firstr);
  48         }
  49         # otherwise, we have real/user/system times
  50         my $out = sprintf "%.2f(%.2f+%.2f)", $r, $u, $s;
  51         $out .= ' ' . relative_change($r, $firstr) if defined $firstr;
  52         return $out;
  53 }
  54
  55 sub usage {
  56         print <<EOT;
  57 ./aggregate.perl [options] [--] [<dir_or_rev>...] [--] [<test_script>...] >
  58
  59   Options:
  60     --codespeed          * Format output for Codespeed
  61     --reponame    <str>  * Send given reponame to codespeed
  62     --sort-by     <str>  * Sort output (only "regression" criteria is supported)
  63     --subsection  <str>  * Use results from given subsection
  64
  65 EOT
  66         exit(1);
  67 }
  68
  69 sub human_size {
  70         my $n = shift;
  71         my @units = ('', qw(K M G));
  72         while ($n > 900 && @units > 1) {
  73                 $n /= 1000;
  74                 shift @units;
  75         }
  76         return $n unless length $units[0];
  77         return sprintf '%.1f%s', $n, $units[0];
  78 }
  79
  80 sub format_size {
  81         my ($size, $first) = @_;
  82         # match the width of a time: 0.00(0.00+0.00)
  83         my $out = sprintf '%15s', human_size($size);
  84         $out .= ' ' . relative_change($size, $first) if defined $first;
  85         return $out;
  86 }
  87
  88 my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
  89     $codespeed, $sortby, $subsection, $reponame);
  90
  91 Getopt::Long::Configure qw/ require_order /;
  92
  93 my $rc = GetOptions("codespeed"     => \$codespeed,
  94                     "reponame=s"    => \$reponame,
  95                     "sort-by=s"     => \$sortby,
  96                     "subsection=s"  => \$subsection);
  97 usage() unless $rc;
  98
  99 while (scalar @ARGV) {
 100         my $arg = $ARGV[0];
 101         my $dir;
 102         my $prefix = '';
 103         last if -f $arg or $arg eq "--";
 104         if (! -d $arg) {
 105                 my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
 106                 $dir = "build/".$rev;
 107         } elsif ($arg eq '.') {
 108                 $dir = '.';
 109         } else {
 110                 $dir = realpath($arg);
 111                 $dirnames{$dir} = $dir;
 112                 $prefix .= 'bindir';
 113         }
 114         push @dirs, $dir;
 115         $dirnames{$dir} ||= $arg;
 116         $prefix .= $dir;
 117         $prefix =~ tr/^a-zA-Z0-9/_/c;
 118         $prefixes{$dir} = $prefix . '.';
 119         shift @ARGV;
 120 }
 121
 122 if (not @dirs) {
 123         @dirs = ('.');
 124 }
 125 $dirnames{'.'} = $dirabbrevs{'.'} = "this tree";
 126 $prefixes{'.'} = '';
 127
 128 shift @ARGV if scalar @ARGV and $ARGV[0] eq "--";
 129
 130 @tests = @ARGV;
 131 if (not @tests) {
 132         @tests = glob "p????-*.sh";
 133 }
 134
 135 my $resultsdir = "test-results";
 136
 137 if (! $subsection and
 138     exists $ENV{GIT_PERF_SUBSECTION} and
 139     $ENV{GIT_PERF_SUBSECTION} ne "") {
 140         $subsection = $ENV{GIT_PERF_SUBSECTION};
 141 }
 142
 143 if ($subsection) {
 144         $resultsdir .= "/" . $subsection;
 145 }
 146
 147 my @subtests;
 148 my %shorttests;
 149 for my $t (@tests) {
 150         $t =~ s{(?:.*/)?(p(\d+)-[^/]+)\.sh$}{$1} or die "bad test name: $t";
 151         my $n = $2;
 152         my $fname = "$resultsdir/$t.subtests";
 153         open my $fp, "<", $fname or die "cannot open $fname: $!";
 154         for (<$fp>) {
 155                 chomp;
 156                 /^(\d+)$/ or die "malformed subtest line: $_";
 157                 push @subtests, "$t.$1";
 158                 $shorttests{"$t.$1"} = "$n.$1";
 159         }
 160         close $fp or die "cannot close $fname: $!";
 161 }
 162
 163 sub read_descr {
 164         my $name = shift;
 165         open my $fh, "<", $name or return "<error reading description>";
 166         binmode $fh, ":utf8" or die "PANIC on binmode: $!";
 167         my $line = <$fh>;
 168         close $fh or die "cannot close $name";
 169         chomp $line;
 170         return $line;
 171 }
 172
 173 sub have_duplicate {
 174         my %seen;
 175         for (@_) {
 176                 return 1 if exists $seen{$_};
 177                 $seen{$_} = 1;
 178         }
 179         return 0;
 180 }
 181 sub have_slash {
 182         for (@_) {
 183                 return 1 if m{/};
 184         }
 185         return 0;
 186 }
 187
 188 sub display_dir {
 189         my ($d) = @_;
 190         return exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d};
 191 }
 192
 193 sub print_default_results {
 194         my %descrs;
 195         my $descrlen = 4; # "Test"
 196         for my $t (@subtests) {
 197                 $descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
 198                 $descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
 199         }
 200
 201         my %newdirabbrevs = %dirabbrevs;
 202         while (!have_duplicate(values %newdirabbrevs)) {
 203                 %dirabbrevs = %newdirabbrevs;
 204                 last if !have_slash(values %dirabbrevs);
 205                 %newdirabbrevs = %dirabbrevs;
 206                 for (values %newdirabbrevs) {
 207                         s{^[^/]*/}{};
 208                 }
 209         }
 210
 211         my %times;
 212         my @colwidth = ((0)x@dirs);
 213         for my $i (0..$#dirs) {
 214                 my $w = length display_dir($dirs[$i]);
 215                 $colwidth[$i] = $w if $w > $colwidth[$i];
 216         }
 217         for my $t (@subtests) {
 218                 my $firstr;
 219                 for my $i (0..$#dirs) {
 220                         my $d = $dirs[$i];
 221                         my $base = "$resultsdir/$prefixes{$d}$t";
 222                         $times{$prefixes{$d}.$t} = [get_times("$base.result")];
 223                         my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
 224                         my $w = length format_times($r,$u,$s,$firstr);
 225                         $colwidth[$i] = $w if $w > $colwidth[$i];
 226                         $firstr = $r unless defined $firstr;
 227                 }
 228         }
 229         my $totalwidth = 3*@dirs+$descrlen;
 230         $totalwidth += $_ for (@colwidth);
 231
 232         printf "%-${descrlen}s", "Test";
 233         for my $i (0..$#dirs) {
 234                 printf "   %-$colwidth[$i]s", display_dir($dirs[$i]);
 235         }
 236         print "\n";
 237         print "-"x$totalwidth, "\n";
 238         for my $t (@subtests) {
 239                 printf "%-${descrlen}s", $descrs{$t};
 240                 my $firstr;
 241                 for my $i (0..$#dirs) {
 242                         my $d = $dirs[$i];
 243                         my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
 244                         printf "   %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
 245                         $firstr = $r unless defined $firstr;
 246                 }
 247                 print "\n";
 248         }
 249 }
 250
 251 sub print_sorted_results {
 252         my ($sortby) = @_;
 253
 254         if ($sortby ne "regression") {
 255                 print "Only 'regression' is supported as '--sort-by' argument\n";
 256                 usage();
 257         }
 258
 259         my @evolutions;
 260         for my $t (@subtests) {
 261                 my ($prevr, $prevu, $prevs, $prevrev);
 262                 for my $i (0..$#dirs) {
 263                         my $d = $dirs[$i];
 264                         my ($r, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.result");
 265                         if ($i > 0 and defined $r and defined $prevr and $prevr > 0) {
 266                                 my $percent = 100.0 * ($r - $prevr) / $prevr;
 267                                 push @evolutions, { "percent"  => $percent,
 268                                                     "test"     => $t,
 269                                                     "prevrev"  => $prevrev,
 270                                                     "rev"      => $d,
 271                                                     "prevr"    => $prevr,
 272                                                     "r"        => $r,
 273                                                     "prevu"    => $prevu,
 274                                                     "u"        => $u,
 275                                                     "prevs"    => $prevs,
 276                                                     "s"        => $s};
 277                         }
 278                         ($prevr, $prevu, $prevs, $prevrev) = ($r, $u, $s, $d);
 279                 }
 280         }
 281
 282         my @sorted_evolutions = sort { $b->{percent} <=> $a->{percent} } @evolutions;
 283
 284         for my $e (@sorted_evolutions) {
 285                 printf "%+.1f%%", $e->{percent};
 286                 print " " . $e->{test};
 287                 print " " . format_times($e->{prevr}, $e->{prevu}, $e->{prevs});
 288                 print " " . format_times($e->{r}, $e->{u}, $e->{s});
 289                 print " " . display_dir($e->{prevrev});
 290                 print " " . display_dir($e->{rev});
 291                 print "\n";
 292         }
 293 }
 294
 295 sub print_codespeed_results {
 296         my ($subsection) = @_;
 297
 298         my $project = "Git";
 299
 300         my $executable = `uname -s -m`;
 301         chomp $executable;
 302
 303         if ($subsection) {
 304                 $executable .= ", " . $subsection;
 305         }
 306
 307         my $environment;
 308         if ($reponame) {
 309                 $environment = $reponame;
 310         } elsif (exists $ENV{GIT_PERF_REPO_NAME} and $ENV{GIT_PERF_REPO_NAME} ne "") {
 311                 $environment = $ENV{GIT_PERF_REPO_NAME};
 312         } else {
 313                 $environment = `uname -r`;
 314                 chomp $environment;
 315         }
 316
 317         my @data;
 318
 319         for my $t (@subtests) {
 320                 for my $d (@dirs) {
 321                         my $commitid = $prefixes{$d};
 322                         $commitid =~ s/^build_//;
 323                         $commitid =~ s/\.$//;
 324                         my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.result");
 325
 326                         my %vals = (
 327                                 "commitid" => $commitid,
 328                                 "project" => $project,
 329                                 "branch" => $dirnames{$d},
 330                                 "executable" => $executable,
 331                                 "benchmark" => $shorttests{$t} . " " . read_descr("$resultsdir/$t.descr"),
 332                                 "environment" => $environment,
 333                                 "result_value" => $result_value,
 334                             );
 335                         push @data, \%vals;
 336                 }
 337         }
 338
 339         require JSON;
 340         print JSON::to_json(\@data, {utf8 => 1, pretty => 1, canonical => 1}), "\n";
 341 }
 342
 343 binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
 344
 345 if ($codespeed) {
 346         print_codespeed_results($subsection);
 347 } elsif (defined $sortby) {
 348         print_sorted_results($sortby);
 349 } else {
 350         print_default_results();
 351 }