rbtree: add rb_search_exact()
[nasm.git] / test / performtest.pl
blobf7865b39fa31b0acca7b5748aaa5f895972dac73
1 #!/usr/bin/perl
2 #Perform tests on nasm
4 use strict;
5 use warnings;
7 use Getopt::Long qw(GetOptions);
8 use Pod::Usage qw(pod2usage);
10 use File::Basename qw(fileparse);
11 use File::Compare qw(compare compare_text);
12 use File::Copy qw(move);
13 use File::Path qw(mkpath rmtree);
15 #sub debugprint { print (pop() . "\n"); }
16 sub debugprint { }
18 my $globalresult = 0;
20 #Process one testfile
21 sub perform {
22 my ($clean, $diff, $golden, $nasm, $quiet, $testpath) = @_;
23 my ($stdoutfile, $stderrfile) = ("stdout", "stderr");
25 my ($testname, $ignoredpath, $ignoredsuffix) = fileparse($testpath, ".asm");
26 debugprint $testname;
28 my $outputdir = $golden ? "golden" : "testresults";
30 mkdir "$outputdir" unless -d "$outputdir";
32 if ($clean) {
33 rmtree "$outputdir/$testname";
34 return;
37 if(-d "$outputdir/$testname") {
38 rmtree "$outputdir/$testname";
41 open(TESTFILE, '<', $testpath) or (warn "Can't open $testpath\n", return);
42 TEST:
43 while(<TESTFILE>) {
44 #See if there is a test case
45 last unless /Testname=(.*);\s*Arguments=(.*);\s*Files=(.*)/;
46 my ($subname, $arguments, $files) = ($1, $2, $3);
47 debugprint("$subname | $arguments | $files");
49 #Call nasm with this test case
50 system("$nasm $arguments $testpath > $stdoutfile 2> $stderrfile");
51 debugprint("$nasm $arguments $testpath > $stdoutfile 2> $stderrfile ----> $?");
53 #Move the output to the test dir
54 mkpath("$outputdir/$testname/$subname");
55 foreach(split / /,$files) {
56 if (-f $_) {
57 move($_, "$outputdir/$testname/$subname/$_") or die $!
60 unlink ("$stdoutfile", "$stderrfile"); #Just to be sure
62 if($golden) {
63 print "Test $testname/$subname created.\n" unless $quiet;
64 } else {
65 #Compare them with the golden files
66 my $result = 0;
67 my @failedfiles = ();
68 foreach(split / /, $files) {
69 if(-f "$outputdir/$testname/$subname/$_") {
70 my $temp;
71 if($_ eq $stdoutfile or $_ eq $stderrfile) {
72 #Compare stdout and stderr in text mode so line ending changes won't matter
73 $temp = compare_text("$outputdir/$testname/$subname/$_", "golden/$testname/$subname/$_",
74 sub { my ($a, $b) = @_;
75 $a =~ s/\r//g;
76 $b =~ s/\r//g;
77 $a ne $b; } );
78 } else {
79 $temp = compare("$outputdir/$testname/$subname/$_", "golden/$testname/$subname/$_");
82 if($temp == 1) {
83 #different
84 $result = 1;
85 $globalresult = 1;
86 push @failedfiles, $_;
87 } elsif($temp == -1) {
88 #error
89 print "Can't compare at $testname/$subname file $_\n";
90 next TEST;
92 } elsif (-f "golden/$testname/$subname/$_") {
93 #File exists in golden but not in output
94 $result = 1;
95 $globalresult = 1;
96 push @failedfiles, $_;
100 if($result == 0) {
101 print "Test $testname/$subname succeeded.\n" unless $quiet;
102 } elsif ($result == 1) {
103 print "Test $testname/$subname failed on @failedfiles.\n";
104 if($diff) {
105 for(@failedfiles) {
106 if($_ eq $stdoutfile or $_ eq $stderrfile) {
107 system "diff -u golden/$testname/$subname/$_ $outputdir/$testname/$subname/$_";
108 print "\n";
112 } else {
113 die "Impossible result";
117 close(TESTFILE);
120 my $nasm;
121 my $clean = 0;
122 my $diff = 0;
123 my $golden = 0;
124 my $help = 0;
125 my $verbose = 0;
127 GetOptions('clean' => \$clean,
128 'diff'=> \$diff,
129 'golden' => \$golden,
130 'help' => \$help,
131 'verbose' => \$verbose,
132 'nasm=s' => \$nasm
133 ) or pod2usage();
135 pod2usage() if $help;
136 die "Please specify either --nasm or --clean. Use --help for help.\n"
137 unless $nasm or $clean;
138 die "Please specify the test files, e.g. *.asm\n" unless @ARGV;
140 unless (!defined $nasm or -x $nasm) {
141 warn "Warning: $nasm may not be executable. Expect problems.\n\n";
142 sleep 5;
145 perform($clean, $diff, $golden, $nasm, ! $verbose, $_) foreach @ARGV;
146 exit $globalresult;
148 __END__
150 =head1 NAME
152 performtest.pl - NASM regression tester based on golden files
154 =head1 SYNOPSIS
156 performtest.pl [options] [testfile.asm ...]
158 Runs NASM on the specified test files and compare the results
159 with "golden" output files.
161 Options:
162 --clean Clean up test results (or golden files with --golden)
163 --diff Execute diff when stdout or stderr don't match
164 --golden Create golden files
165 --help Get this help
166 --nasm=file Specify the file name for the NASM executable, e.g. ../nasm
167 --verbose Get more output
169 If --clean is not specified, --nasm is required.
171 testfile.asm ...:
172 One or more files that NASM should be tested with,
173 often *.asm in the test directory.
174 It should contain one or more option lines at the start,
175 in the following format:
177 ;Testname=<testname>; Arguments=<arguments to nasm>; Files=<output files>
179 If no such lines are found at the start, the file is skipped.
180 testname should ideally describe the arguments, eg. unoptimized for -O0.
181 arguments can be an optimization level (-O), an output format (-f),
182 an output file specifier (-o) etc.
183 The output files should be a space seperated list of files that will
184 be checked for regressions. This should often be the output file
185 and the special files stdout and stderr.
187 Any mismatch could be a regression,
188 but it doesn't have to be. COFF files have a timestamp which
189 makes this method useless. ELF files have a comment section
190 with the current version of NASM, so they will change each version number.
192 =cut