errors/do-tests.pl

   1 #!/usr/bin/perl -w
   2
   3 use strict;
   4
   5 unless ($#ARGV == 2) {
   6     print STDERR "Usage: $0 profile compiler glob-pattern\n";
   7     exit 1;
   8 }
   9
  10 #
  11 # Expected value constants
  12 #
  13 my $EXPECTING_WRONG_ERROR = 1;
  14 my $EXPECTING_NO_ERROR    = 2;
  15 my %expecting_map = ();
  16 my %ignore_map = ();
  17
  18 my $profile = $ARGV [0];
  19 my $compile = $ARGV [1];
  20 my $files   = $ARGV [2];
  21
  22 if (open (EXPECT_WRONG, "<$profile-expect-wrong-error")) {
  23         $expecting_map{$_} = $EXPECTING_WRONG_ERROR
  24         foreach map {
  25                 chomp,                     # remove trailing \n
  26                 s/\#.*//g,                 # remove # style comments
  27                 s/\s//g;                    # remove whitespace
  28                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  29         } <EXPECT_WRONG>;
  30
  31         close EXPECT_WRONG;
  32 }
  33
  34 if (open (EXPECT_NO, "<$profile-expect-no-error")) {
  35         $expecting_map{$_} = $EXPECTING_NO_ERROR
  36         foreach map {
  37                 chomp,                     # remove trailing \n
  38                 s/\#.*//g,                 # remove # style comments
  39                 s/\s//g;                    # remove whitespace
  40                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  41         } <EXPECT_NO>;
  42
  43         close EXPECT_NO;
  44 }
  45
  46 if (open (IGNORE, "<$profile-ignore-tests")) {
  47         $ignore_map{$_} = 1
  48         foreach map {
  49                 chomp,                     # remove trailing \n
  50                 s/\#.*//g,                 # remove # style comments
  51                 s/\s//g;                    # remove whitespace
  52                 $_ eq "" ? () : $_;        # now copy over non empty stuff
  53         } <IGNORE>;
  54
  55         close IGNORE;
  56 }
  57
  58 my $RESULT_UNEXPECTED_CORRECT_ERROR     = 1;
  59 my $RESULT_CORRECT_ERROR                = 2;
  60 my $RESULT_UNEXPECTED_INCORRECT_ERROR   = 3;
  61 my $RESULT_EXPECTED_INCORRECT_ERROR     = 4;
  62 my $RESULT_UNEXPECTED_NO_ERROR          = 5;
  63 my $RESULT_EXPECTED_NO_ERROR            = 6;
  64 my $RESULT_UNEXPECTED_CRASH             = 7;
  65
  66 my @statuses = (
  67         "UNEXPECTED TEST HARNESS INTERNAL ERROR",
  68         "OK (still listed as a failure)",
  69         "OK",
  70         "REGRESSION: An incorrect error was reported",
  71         "KNOWN ISSUE: Incorrect error reported",
  72         "REGRESSION: No error reported",
  73         "KNOWN ISSUE: No error reported",
  74         "UNEXPECTED CRASH"
  75 );
  76
  77 my @status_items = (
  78         [],  # should be empty
  79         [],
  80         [],
  81         [],
  82         [],
  83         [],
  84         [],
  85         [],
  86 );
  87
  88 my %results_map = ();
  89 my $total = 0;
  90 my $textmsg = "";
  91
  92 open (PROFILELOG, ">$profile.log") or die "Cannot open log file $profile.log";
  93
  94 foreach (glob ($files)) {
  95         my $tname = $_;
  96         my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/);
  97         my $options = `sed -n 's,^// Compiler options:,,p' $_`;
  98         chomp $options;
  99
 100         if (exists $ignore_map {$_}) {
 101             print "IGNORED: $_\n";
 102             print PROFILELOG "IGNORED: $_\n";
 103             next;
 104         }
 105
 106         $total++;
 107         my $testlogfile="$profile-$_.log";
 108         system "$compile --expect-error $error_number $options -out:$profile-$_.junk $_ > $testlogfile 2>&1";
 109
 110         exit 1 if $? & 127;
 111
 112         my $exit_value = $? >> 8;
 113
 114         my $status;
 115
 116         if ($exit_value == 0) {
 117                 system "rm -f $testlogfile";
 118                 $status = (exists $expecting_map {$_})
 119                     ? $RESULT_UNEXPECTED_CORRECT_ERROR : $RESULT_CORRECT_ERROR;
 120         } elsif ($exit_value == 1) {
 121                 $status = (exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR)
 122                     ? $RESULT_EXPECTED_INCORRECT_ERROR : $RESULT_UNEXPECTED_INCORRECT_ERROR;
 123         } elsif ($exit_value == 2) {
 124                 $status = (exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR)
 125                     ? $RESULT_EXPECTED_NO_ERROR : $RESULT_UNEXPECTED_NO_ERROR;
 126         } else {
 127                 $status = (exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR)
 128                     ? $RESULT_EXPECTED_INCORRECT_ERROR : $RESULT_UNEXPECTED_CRASH;
 129         }
 130
 131         $textmsg = "FAIL";
 132
 133         if ($status == $RESULT_EXPECTED_NO_ERROR ||
 134             $status == $RESULT_UNEXPECTED_CORRECT_ERROR ||
 135             $status == $RESULT_EXPECTED_INCORRECT_ERROR ||
 136             $status == $RESULT_CORRECT_ERROR){
 137             $textmsg = "PASS";
 138         }
 139         push @{$status_items [$status]}, $_;
 140         print PROFILELOG "$textmsg: $tname $statuses[$status]\n"; print "$textmsg: $tname $statuses[$status]\n";
 141         $results_map{$_} = $status;
 142 }
 143 print "\n";
 144 my $correct = scalar @{$status_items [$RESULT_CORRECT_ERROR]};
 145 my $pct = sprintf("%.2f",($correct / $total) * 100);
 146 print PROFILELOG $correct, " correctly detected errors ($pct %) \n\n";
 147 print $correct, " correctly detected errors ($pct %) \n\n";
 148
 149 if (scalar @{$status_items [$RESULT_UNEXPECTED_CRASH]} > 0) {
 150     print PROFILELOG scalar @{$status_items [$RESULT_UNEXPECTED_CRASH]}, " compiler crashes\n";
 151     print scalar @{$status_items [$RESULT_UNEXPECTED_CRASH]}, " compiler crashes\n";
 152     print PROFILELOG "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_CRASH]};
 153     print "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_CRASH]};
 154 }
 155
 156 if (scalar @{$status_items [$RESULT_UNEXPECTED_CORRECT_ERROR]} > 0) {
 157     print PROFILELOG scalar @{$status_items [$RESULT_UNEXPECTED_CORRECT_ERROR]}, " fixed error report(s), remove it from expect-wrong-error or expect-no-error !\n";
 158     print scalar @{$status_items [$RESULT_UNEXPECTED_CORRECT_ERROR]}, " fixed error report(s), remove it from expect-wrong-error or expect-no-error !\n";
 159     print PROFILELOG "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_CORRECT_ERROR]};
 160     print "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_CORRECT_ERROR]};
 161 }
 162
 163 if (scalar @{$status_items [$RESULT_UNEXPECTED_INCORRECT_ERROR]} > 0) {
 164     print PROFILELOG scalar @{$status_items [$RESULT_UNEXPECTED_INCORRECT_ERROR]}, " new incorrect error report(s) !\n";
 165     print scalar @{$status_items [$RESULT_UNEXPECTED_INCORRECT_ERROR]}, " new incorrect error report(s) !\n";
 166     print PROFILELOG "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_INCORRECT_ERROR]};
 167     print "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_INCORRECT_ERROR]};
 168 }
 169
 170 if (scalar @{$status_items [$RESULT_UNEXPECTED_NO_ERROR]} > 0) {
 171     print PROFILELOG scalar @{$status_items [$RESULT_UNEXPECTED_NO_ERROR]}, " new missing error report(s) !\n";
 172     print scalar @{$status_items [$RESULT_UNEXPECTED_NO_ERROR]}, " new missing error report(s) !\n";
 173     print PROFILELOG "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_NO_ERROR]};
 174     print "$_\n" foreach @{$status_items [$RESULT_UNEXPECTED_NO_ERROR]};
 175 }
 176
 177 exit ((
 178         scalar @{$status_items [$RESULT_UNEXPECTED_INCORRECT_ERROR]} +
 179         scalar @{$status_items [$RESULT_UNEXPECTED_NO_ERROR       ]} +
 180         scalar @{$status_items [$RESULT_UNEXPECTED_CRASH          ]}
 181 ) == 0 ? 0 : 1);