tests/test-perf

   1 #!/usr/bin/env perl
   2
   3 #
   4 # test-perf
   5 #
   6 # Author:
   7 #   Gabriel Burt <gburt@novell.com>
   8 #
   9 # Copyright (C) 2009 Novell, Inc.
  10 #
  11 # Permission is hereby granted, free of charge, to any person obtaining
  12 # a copy of this software and associated documentation files (the
  13 # "Software"), to deal in the Software without restriction, including
  14 # without limitation the rights to use, copy, modify, merge, publish,
  15 # distribute, sublicense, and/or sell copies of the Software, and to
  16 # permit persons to whom the Software is furnished to do so, subject to
  17 # the following conditions:
  18 #
  19 # The above copyright notice and this permission notice shall be
  20 # included in all copies or substantial portions of the Software.
  21 #
  22 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  26 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  27 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  28 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29 #
  30
  31 use strict;
  32
  33 chomp(my $pwd = `pwd`);
  34 chdir "../bin" || die "couldn't cd ../bin";
  35
  36 my $RUNS_PER_TEST = 3;
  37 my $ENV_OPTIONS = "TZ=America/Chicago LC_ALL=it_IT LANG=it_IT";
  38 my $original_rev = current_rev();
  39
  40 sub current_rev {
  41     my $current_rev = `git rev-parse \@\{0\};`;
  42     chomp($current_rev);
  43     return $current_rev;
  44 }
  45
  46 sub usage {
  47     print "test-perf [--name=NAME] [SHORT-NAME=GIT-REV | GIT-REV | GIT-REV-EXPR]...\n\n";
  48     print "  NAME specifies the directory that the NUnit xml results will be placed in\n";
  49     print "  SHORT-NAME lets you alias a git revision\n\n";
  50     print "  GIT-REV and GIT-REV-EXPR must be parsable by git rev-parse\n\n";
  51     print "  Examples:\n\n";
  52     print "     test-perf --name=a-vs-b a=HEAD~3 b=34e934\n";
  53     print "     test-perf HEAD~3 34e934 # same as above, just without nice aliases\n";
  54     print "     test-perf\n";
  55     exit;
  56 }
  57
  58 my $run_name = "results";
  59 my %revisions;
  60 my @ordered_revisions;
  61 my $rev_count = 0;
  62 foreach my $in (@ARGV) {
  63     if ($in =~ m/--help/) {
  64         usage ();
  65     } elsif ($in =~ m/^--name=(\S+)$/) {
  66         $run_name = $1;
  67     } elsif ($in =~ m/^(\S+)=(\S+)$/) {
  68         my $rev = `git rev-parse $2`;
  69         chomp $rev;
  70         $revisions{$1} = $rev;
  71         push @ordered_revisions, $1;
  72         $rev_count++;
  73     } else {
  74         my @expanded = `git rev-parse $in`;
  75         foreach my $rev (@expanded) {
  76             chomp $rev;
  77             my $key = length(@expanded) == 1 ? $in : $rev;
  78             $revisions{$key} = $rev;
  79             push @ordered_revisions, $key;
  80             $rev_count++;
  81         }
  82     }
  83 }
  84
  85 if ($rev_count == 0) {
  86     usage ();
  87 }
  88
  89 print "Will test:\n";
  90 foreach my $key (@ordered_revisions) {
  91     my $val = $revisions{$key};
  92     if ($key eq $val) {
  93         print "  $val\n";
  94     } else {
  95         print "  $val ($key)\n";
  96     }
  97 }
  98 print "\n";
  99
 100 my $results_dir = "$pwd/$run_name";
 101 print "Warning: results dir $results_dir already exists\n\n" if -d $results_dir;
 102 `mkdir -p $results_dir`;
 103
 104 # Backup the database so we can restore it after each test
 105 `cp ~/.config/banshee-1/banshee.db ~/.config/banshee-1/banshee.db.bak`;
 106
 107 foreach my $rev_name (@ordered_revisions) {
 108     print "* $rev_name\n";
 109
 110     my $rev = $revisions{$rev_name};
 111     my $rev_dir = "$results_dir/$rev_name";
 112
 113     if (-d $rev_dir) {
 114         print "  - Removing existing results dir ($rev_dir)\n";
 115         unlink $rev_dir;
 116     }
 117     mkdir $rev_dir;
 118
 119     # TODO output these as well ?
 120     # load
 121     # hwinfo --cpu
 122     # hwinfo --memory
 123     # $(BUILD_VENDOR_ID)
 124     # $(BUILD_HOST_OS)
 125     # $(BUILD_HOST_CPU)
 126     # $(BUILD_TIME)
 127     # put a chronologically-ordered identifier for this commit into its result dir (run-info file)
 128     my $alias = $rev_name eq $rev ? "" : " $rev_name";
 129     my $info = `git rev-list -n 1 --pretty=format:"%ai %h %at" $rev | grep -v commit`;
 130     $info =~ s/^(\S+)\s+\S+\s+\S+\s+(\S+)\s+(\S+)$/$1 $2$alias\n$3/;
 131     chomp($info);
 132     `echo "$info" > $rev_dir/run-info`;
 133
 134     chdir "..";
 135
 136     my $cur = current_rev ();
 137     if ($rev ne current_rev()) {
 138         print "  - Checking out\n";
 139         `git checkout $rev >/dev/null`;
 140         print "  - Building $rev_name\n";
 141     } else {
 142         print "  - Building $rev_name\n";
 143     }
 144
 145     my $make_ret = `make`;
 146     chdir "bin";
 147     `cp ../tests/Performance/Performance.dll .`;
 148
 149     print "  - Testing $rev_name\n";
 150     #foreach my $asm (glob("*.dll"))
 151     foreach my $asm (glob("Performance.dll")) {
 152         for (my $i = 0; $i < $RUNS_PER_TEST; $i++) {
 153             print "    - Run $i\n";
 154             `$ENV_OPTIONS nunit-console2 -nologo -noshadow -xml=$rev_dir/$i-$asm.xml $asm 2>/dev/null`;
 155             `cp ~/.config/banshee-1/banshee.db.bak ~/.config/banshee-1/banshee.db`;
 156         }
 157     }
 158
 159     print "\n\n";
 160 }
 161
 162 if (current_rev () ne $original_rev) {
 163     `git checkout $original_rev`;
 164 }
 165
 166 print "Output is in $run_name/\n";