tests/run_make_tests.pl

   1 #!/usr/bin/env perl
   2 # -*-perl-*-
   3
   4 # Test driver for the Make test suite
   5
   6 # Usage:  run_make_tests  [testname]
   7 #                         [-debug]
   8 #                         [-help]
   9 #                         [-verbose]
  10 #                         [-keep]
  11 #                         [-make <make prog>]
  12 #                        (and others)
  13
  14 # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
  15 # 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software
  16 # Foundation, Inc.
  17 # This file is part of GNU Make.
  18 #
  19 # GNU Make is free software; you can redistribute it and/or modify it under
  20 # the terms of the GNU General Public License as published by the Free Software
  21 # Foundation; either version 3 of the License, or (at your option) any later
  22 # version.
  23 #
  24 # GNU Make is distributed in the hope that it will be useful, but WITHOUT ANY
  25 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  26 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
  27 # details.
  28 #
  29 # You should have received a copy of the GNU General Public License along with
  30 # this program.  If not, see <http://www.gnu.org/licenses/>.
  31
  32 %FEATURES = ();
  33
  34 $valgrind = 0;              # invoke make with valgrind
  35 $valgrind_args = '';
  36 $memcheck_args = '--num-callers=15 --tool=memcheck --leak-check=full';
  37 $massif_args = '--num-callers=15 --tool=massif --alloc-fn=xmalloc --alloc-fn=xcalloc --alloc-fn=xrealloc --alloc-fn=xstrdup --alloc-fn=xstrndup';
  38 $pure_log = undef;
  39
  40 $command_string = '';
  41
  42 $all_tests = 0;
  43
  44 require "test_driver.pl";
  45
  46 # Some target systems might not have the POSIX module...
  47 $has_POSIX = eval { require "POSIX.pm" };
  48
  49 #$SIG{INT} = sub { print STDERR "Caught a signal!\n"; die @_; };
  50
  51 sub valid_option
  52 {
  53    local($option) = @_;
  54
  55    if ($option =~ /^-make([-_]?path)?$/i) {
  56        $make_path = shift @argv;
  57        if (!-f $make_path) {
  58            print "$option $make_path: Not found.\n";
  59            exit 0;
  60        }
  61        return 1;
  62    }
  63
  64    if ($option =~ /^-all([-_]?tests)?$/i) {
  65        $all_tests = 1;
  66        return 1;
  67    }
  68
  69    if ($option =~ /^-(valgrind|memcheck)$/i) {
  70        $valgrind = 1;
  71        $valgrind_args = $memcheck_args;
  72        return 1;
  73    }
  74
  75    if ($option =~ /^-massif$/i) {
  76        $valgrind = 1;
  77        $valgrind_args = $massif_args;
  78        return 1;
  79    }
  80
  81 # This doesn't work--it _should_!  Someone badly needs to fix this.
  82 #
  83 #   elsif ($option =~ /^-work([-_]?dir)?$/)
  84 #   {
  85 #      $workdir = shift @argv;
  86 #      return 1;
  87 #   }
  88
  89    return 0;
  90 }
  91
  92
  93 # This is an "all-in-one" function.  Arguments are as follows:
  94 #
  95 #  [0] (string):  The makefile to be tested.  undef means use the last one.
  96 #  [1] (string):  Arguments to pass to make.
  97 #  [2] (string):  Answer we should get back.
  98 #  [3] (integer): Exit code we expect.  A missing code means 0 (success)
  99
 100 $old_makefile = undef;
 101
 102 sub run_make_test
 103 {
 104   local ($makestring, $options, $answer, $err_code, $timeout) = @_;
 105
 106   # If the user specified a makefile string, create a new makefile to contain
 107   # it.  If the first value is not defined, use the last one (if there is
 108   # one).
 109
 110   if (! defined $makestring) {
 111     defined $old_makefile
 112       || die "run_make_test(undef) invoked before run_make_test('...')\n";
 113     $makefile = $old_makefile;
 114   } else {
 115     if (! defined($makefile)) {
 116       $makefile = &get_tmpfile();
 117     }
 118
 119     # Make sure it ends in a newline.
 120     $makestring && $makestring !~ /\n$/s and $makestring .= "\n";
 121
 122     # Replace @MAKEFILE@ with the makefile name and @MAKE@ with the path to
 123     # make
 124     $makestring =~ s/#MAKEFILE#/$makefile/g;
 125     $makestring =~ s/#MAKEPATH#/$mkpath/g;
 126     $makestring =~ s/#MAKE#/$make_name/g;
 127     $makestring =~ s/#PERL#/$perl_name/g;
 128     $makestring =~ s/#PWD#/$pwd/g;
 129
 130     # Populate the makefile!
 131     open(MAKEFILE, "> $makefile") || die "Failed to open $makefile: $!\n";
 132     print MAKEFILE $makestring;
 133     close(MAKEFILE) || die "Failed to write $makefile: $!\n";
 134   }
 135
 136   # Do the same processing on $answer as we did on $makestring.
 137
 138   $answer && $answer !~ /\n$/s and $answer .= "\n";
 139   $answer =~ s/#MAKEFILE#/$makefile/g;
 140   $answer =~ s/#MAKEPATH#/$mkpath/g;
 141   $answer =~ s/#MAKE#/$make_name/g;
 142   $answer =~ s/#PERL#/$perl_name/g;
 143   $answer =~ s/#PWD#/$pwd/g;
 144
 145   run_make_with_options($makefile, $options, &get_logfile(0),
 146                         $err_code, $timeout);
 147   &compare_output($answer, &get_logfile(1));
 148
 149   $old_makefile = $makefile;
 150   $makefile = undef;
 151 }
 152
 153 # The old-fashioned way...
 154 sub run_make_with_options {
 155   local ($filename,$options,$logname,$expected_code,$timeout) = @_;
 156   local($code);
 157   local($command) = $make_path;
 158
 159   $expected_code = 0 unless defined($expected_code);
 160
 161   # Reset to reflect this one test.
 162   $test_passed = 1;
 163
 164   if ($filename) {
 165     $command .= " -f $filename";
 166   }
 167
 168   if ($options) {
 169     $command .= " $options";
 170   }
 171
 172   $command_string = "$command\n";
 173
 174   if ($valgrind) {
 175     print VALGRIND "\n\nExecuting: $command\n";
 176   }
 177
 178
 179   {
 180       my $old_timeout = $test_timeout;
 181       $timeout and $test_timeout = $timeout;
 182
 183       # If valgrind is enabled, turn off the timeout check
 184       $valgrind and $test_timeout = 0;
 185
 186       $code = &run_command_with_output($logname,$command);
 187
 188       $test_timeout = $old_timeout;
 189   }
 190
 191   # Check to see if we have Purify errors.  If so, keep the logfile.
 192   # For this to work you need to build with the Purify flag -exit-status=yes
 193
 194   if ($pure_log && -f $pure_log) {
 195     if ($code & 0x7000) {
 196       $code &= ~0x7000;
 197
 198       # If we have a purify log, save it
 199       $tn = $pure_testname . ($num_of_logfiles ? ".$num_of_logfiles" : "");
 200       print("Renaming purify log file to $tn\n") if $debug;
 201       rename($pure_log, "$tn")
 202         || die "Can't rename $log to $tn: $!\n";
 203       ++$purify_errors;
 204     } else {
 205       unlink($pure_log);
 206     }
 207   }
 208
 209   if ($code != $expected_code) {
 210     print "Error running $make_path (expected $expected_code; got $code): $command\n";
 211     $test_passed = 0;
 212     $runf = &get_runfile;
 213     &create_file (&get_runfile, $command_string);
 214     # If it's a SIGINT, stop here
 215     if ($code & 127) {
 216       print STDERR "\nCaught signal ".($code & 127)."!\n";
 217       ($code & 127) == 2 and exit($code);
 218     }
 219     return 0;
 220   }
 221
 222   if ($profile & $vos) {
 223     system "add_profile $make_path";
 224   }
 225
 226   return 1;
 227 }
 228
 229 sub print_usage
 230 {
 231    &print_standard_usage ("run_make_tests",
 232                           "[-make_path make_pathname] [-memcheck] [-massif]",);
 233 }
 234
 235 sub print_help
 236 {
 237    &print_standard_help (
 238         "-make_path",
 239         "\tYou may specify the pathname of the copy of make to run.",
 240         "-valgrind",
 241         "-memcheck",
 242         "\tRun the test suite under valgrind's memcheck tool.",
 243         "\tChange the default valgrind args with the VALGRIND_ARGS env var.",
 244         "-massif",
 245         "\tRun the test suite under valgrind's massif toool.",
 246         "\tChange the default valgrind args with the VALGRIND_ARGS env var."
 247        );
 248 }
 249
 250 sub get_this_pwd {
 251   $delete_command = 'rm -f';
 252   if ($has_POSIX) {
 253     $__pwd = POSIX::getcwd();
 254   } elsif ($vos) {
 255     $delete_command = "delete_file -no_ask";
 256     $__pwd = `++(current_dir)`;
 257   } else {
 258     # No idea... just try using pwd as a last resort.
 259     chop ($__pwd = `pwd`);
 260   }
 261
 262   return $__pwd;
 263 }
 264
 265 sub set_defaults
 266 {
 267    # $profile = 1;
 268    $testee = "GNU make";
 269    $make_path = "make";
 270    $tmpfilesuffix = "mk";
 271    $pwd = &get_this_pwd;
 272 }
 273
 274 sub set_more_defaults
 275 {
 276    local($string);
 277    local($index);
 278
 279    # find the type of the port.  We do this up front to have a single
 280    # point of change if it needs to be tweaked.
 281    #
 282    # This is probably not specific enough.
 283    #
 284    if ($osname =~ /Windows/i || $osname =~ /MINGW32/i || $osname =~ /CYGWIN_NT/i) {
 285      $port_type = 'W32';
 286    }
 287    # Bleah, the osname is so variable on DOS.  This kind of bites.
 288    # Well, as far as I can tell if we check for some text at the
 289    # beginning of the line with either no spaces or a single space, then
 290    # a D, then either "OS", "os", or "ev" and a space.  That should
 291    # match and be pretty specific.
 292    elsif ($osname =~ /^([^ ]*|[^ ]* [^ ]*)D(OS|os|ev) /) {
 293      $port_type = 'DOS';
 294    }
 295    # Check for OS/2
 296    elsif ($osname =~ m%OS/2%) {
 297      $port_type = 'OS/2';
 298    }
 299    # Everything else, right now, is UNIX.  Note that we should integrate
 300    # the VOS support into this as well and get rid of $vos; we'll do
 301    # that next time.
 302    else {
 303      $port_type = 'UNIX';
 304    }
 305
 306    # On DOS/Windows system the filesystem apparently can't track
 307    # timestamps with second granularity (!!).  Change the sleep time
 308    # needed to force a file to be considered "old".
 309    $wtime = $port_type eq 'UNIX' ? 1 : $port_type eq 'OS/2' ? 2 : 4;
 310
 311    print "Port type: $port_type\n" if $debug;
 312    print "Make path: $make_path\n" if $debug;
 313
 314    # Find the full pathname of Make.  For DOS systems this is more
 315    # complicated, so we ask make itself.
 316    my $mk = `sh -c 'echo "all:;\@echo \\\$(MAKE)" | $make_path -f-'`;
 317    chop $mk;
 318    $mk or die "FATAL ERROR: Cannot determine the value of \$(MAKE):\n
 319 'echo \"all:;\@echo \\\$(MAKE)\" | $make_path -f-' failed!\n";
 320    $make_path = $mk;
 321    print "Make\t= `$make_path'\n" if $debug;
 322
 323    $string = `$make_path -v -f /dev/null 2> /dev/null`;
 324
 325    $string =~ /^(GNU Make [^,\n]*)/;
 326    $testee_version = "$1\n";
 327
 328    $string = `sh -c "$make_path -f /dev/null 2>&1"`;
 329    if ($string =~ /(.*): \*\*\* No targets\.  Stop\./) {
 330      $make_name = $1;
 331    }
 332    else {
 333      if ($make_path =~ /$pathsep([^\n$pathsep]*)$/) {
 334        $make_name = $1;
 335      }
 336      else {
 337        $make_name = $make_path;
 338      }
 339    }
 340
 341    # prepend pwd if this is a relative path (ie, does not
 342    # start with a slash, but contains one).  Thanks for the
 343    # clue, Roland.
 344
 345    if (index ($make_path, ":") != 1 && index ($make_path, "/") > 0)
 346    {
 347       $mkpath = "$pwd$pathsep$make_path";
 348    }
 349    else
 350    {
 351       $mkpath = $make_path;
 352    }
 353
 354    # Get Purify log info--if any.
 355
 356    if (exists $ENV{PURIFYOPTIONS}
 357        && $ENV{PURIFYOPTIONS} =~ /.*-logfile=([^ ]+)/) {
 358      $pure_log = $1 || '';
 359      $pure_log =~ s/%v/$make_name/;
 360      $purify_errors = 0;
 361    }
 362
 363    $string = `sh -c "$make_path -j 2 -f /dev/null 2>&1"`;
 364    if ($string =~ /not supported/) {
 365      $parallel_jobs = 0;
 366    }
 367    else {
 368      $parallel_jobs = 1;
 369    }
 370
 371    %FEATURES = map { $_ => 1 } split /\s+/, `sh -c "echo '\\\$(info \\\$(.FEATURES))' | $make_path -f- 2>/dev/null"`;
 372
 373    # Set up for valgrind, if requested.
 374
 375    if ($valgrind) {
 376      my $args = $valgrind_args;
 377      open(VALGRIND, "> valgrind.out")
 378        || die "Cannot open valgrind.out: $!\n";
 379      #  -q --leak-check=yes
 380      exists $ENV{VALGRIND_ARGS} and $args = $ENV{VALGRIND_ARGS};
 381      $make_path = "valgrind --log-fd=".fileno(VALGRIND)." $args $make_path";
 382      # F_SETFD is 2
 383      fcntl(VALGRIND, 2, 0) or die "fcntl(setfd) failed: $!\n";
 384      system("echo Starting on `date` 1>&".fileno(VALGRIND));
 385      print "Enabled valgrind support.\n";
 386    }
 387 }
 388
 389 sub setup_for_test
 390 {
 391   $makefile = &get_tmpfile;
 392   if (-f $makefile) {
 393     unlink $makefile;
 394   }
 395
 396   # Get rid of any Purify logs.
 397   if ($pure_log) {
 398     ($pure_testname = $testname) =~ tr,/,_,;
 399     $pure_testname = "$pure_log.$pure_testname";
 400     system("rm -f $pure_testname*");
 401     print("Purify testfiles are: $pure_testname*\n") if $debug;
 402   }
 403 }
 404
 405 exit !&toplevel;