4 print STDERR
"Usage: $0 profile compiler glob-pattern\n";
9 # Expected value constants
11 my $EXPECTING_WRONG_ERROR = 1;
12 my $EXPECTING_NO_ERROR = 2;
13 my %expecting_map = ();
16 my $profile = $ARGV [0];
17 my $compile = $ARGV [1];
18 my $files = $ARGV [2];
20 if (open (EXPECT_WRONG
, "<$profile-expect-wrong-error")) {
21 $expecting_map{$_} = $EXPECTING_WRONG_ERROR
23 chomp, # remove trailing \n
24 s/\#.*//g, # remove # style comments
25 s/\s//g; # remove whitespace
26 $_ eq "" ?
() : $_; # now copy over non empty stuff
32 if (open (EXPECT_NO
, "<$profile-expect-no-error")) {
33 $expecting_map{$_} = $EXPECTING_NO_ERROR
35 chomp, # remove trailing \n
36 s/\#.*//g, # remove # style comments
37 s/\s//g; # remove whitespace
38 $_ eq "" ?
() : $_; # now copy over non empty stuff
44 if (open (IGNORE
, "<$profile-ignore-tests")) {
47 chomp, # remove trailing \n
48 s/\#.*//g, # remove # style comments
49 s/\s//g; # remove whitespace
50 $_ eq "" ?
() : $_; # now copy over non empty stuff
56 my $RESULT_UNEXPECTED_CORRECT_ERROR = 1;
57 my $RESULT_CORRECT_ERROR = 2;
58 my $RESULT_UNEXPECTED_INCORRECT_ERROR = 3;
59 my $RESULT_EXPECTED_INCORRECT_ERROR = 4;
60 my $RESULT_UNEXPECTED_NO_ERROR = 5;
61 my $RESULT_EXPECTED_NO_ERROR = 6;
62 my $RESULT_UNEXPECTED_CRASH = 7;
67 "UNEXPECTED INCORRECT ERROR",
69 "UNEXPECTED NO ERROR",
87 foreach (glob ($files)) {
89 my ($error_number) = (/[a-z]*(\d+)(-\d+)?\.cs/);
90 my $options = `sed -n 's,^// Compiler options:,,p' $_`;
94 if (exists $ignore_map {$_}) {
100 my $testlogfile="$profile-$_.log";
101 system "$compile --expect-error $error_number $options -out:$profile-$_.junk $_ > $testlogfile 2>&1";
105 my $exit_value = $?
>> 8;
109 if ($exit_value > 2) {
110 if (exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR) {
111 $status = $RESULT_EXPECTED_INCORRECT_ERROR;
113 $status = $RESULT_UNEXPECTED_CRASH;
117 if ($exit_value == 0) {
118 system "rm -f $testlogfile";
119 $status = $RESULT_UNEXPECTED_CORRECT_ERROR if exists $expecting_map {$_};
120 $status = $RESULT_CORRECT_ERROR unless exists $expecting_map {$_};
123 if ($exit_value == 1) {
124 $status = $RESULT_UNEXPECTED_INCORRECT_ERROR unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
125 $status = $RESULT_EXPECTED_INCORRECT_ERROR if exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_WRONG_ERROR;
128 if ($exit_value == 2) {
129 $status = $RESULT_UNEXPECTED_NO_ERROR unless exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
130 $status = $RESULT_EXPECTED_NO_ERROR if exists $expecting_map {$_} and $expecting_map {$_} == $EXPECTING_NO_ERROR;
134 push @
{@status_items [($status - 1)]}, $_;
135 print "@statuses[($status - 1)]\n";
136 $results_map{$_} = $status;
139 my $correct = scalar @
{@status_items [($RESULT_CORRECT_ERROR - 1)]};
140 print $correct, " correctly detected errors (", sprintf("%.2f",($correct / $total) * 100), " %) \n\n";
142 if (scalar @
{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]} > 0) {
143 print scalar @
{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]}, " compiler crashes\n";
144 print, print "\n" foreach @
{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]};
147 if (scalar @
{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]} > 0) {
148 print scalar @
{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]}, " fixed error report(s), remove it from expect-wrong-error or expect-no-error !\n";
149 print, print "\n" foreach @
{@status_items [($RESULT_UNEXPECTED_CORRECT_ERROR - 1)]};
152 if (scalar @
{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} > 0) {
153 print scalar @
{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]}, " new incorrect error report(s) !\n";
154 print, print "\n" foreach @
{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]};
157 if (scalar @
{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]} > 0) {
158 print scalar @
{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]}, " new missing error report(s) !\n";
159 print, print "\n" foreach @
{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]};
163 scalar @
{@status_items [($RESULT_UNEXPECTED_INCORRECT_ERROR - 1)]} +
164 scalar @
{@status_items [($RESULT_UNEXPECTED_NO_ERROR - 1)]} +
165 scalar @
{@status_items [($RESULT_UNEXPECTED_CRASH - 1)]}