gen-testsuite-part

   1 #! /usr/bin/env perl
   2 # Automatically compute some dependencies for the hand-written tests
   3 # of the Automake testsuite.  Also, automatically generate some more
   4 # tests from them (for particular cases/setups only).
   5
   6 # Copyright (C) 2011-2013 Free Software Foundation, Inc.
   7
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2, or (at your option)
  11 # any later version.
  12
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 #--------------------------------------------------------------------------
  22
  23 use warnings FATAL => "all";
  24 use strict;
  25 use File::Basename ();
  26 use constant TRUE => 1;
  27 use constant FALSE => 0;
  28
  29 my $me = File::Basename::basename $0;
  30
  31 # For use in VPATH builds.
  32 my $srcdir = ".";
  33
  34 # The testsuite subdirectory, relative to the top-lever source directory.
  35 my $testdir = "t";
  36
  37 # Where testsuite-related helper scripts, data files and shell libraries
  38 # are placed.  Relative to the top-lever source directory.
  39 my $testauxdir = "$testdir/ax";
  40
  41 #--------------------------------------------------------------------------
  42
  43 sub unindent ($)
  44 {
  45   my $text = shift;
  46   $text =~ /^(\s*)/;
  47   my $indentation = $1;
  48   $text =~ s/^$indentation//gm;
  49   return $text;
  50 }
  51
  52 sub atomic_write ($$;$)
  53 {
  54   my ($outfile, $func) = (shift, shift);
  55   my $perms = @_ > 0 ? shift : 0777;
  56   my $tmpfile = "$outfile-t";
  57   foreach my $f ($outfile, $tmpfile)
  58     {
  59       unlink $f or die "$me: cannot unlink '$f': $!\n"
  60         if -e $f;
  61     }
  62   open (my $fh, ">$tmpfile")
  63     or die "$me: can't write to '$tmpfile': $!\n";
  64   $func->($fh);
  65   close $fh
  66     or die "$me: closing '$tmpfile': $!\n";
  67   chmod ($perms & ~umask, $tmpfile)
  68     or die "$me: cannot change perms for '$tmpfile': $!\n";
  69   rename ($tmpfile, $outfile)
  70     or die "$me: renaming '$tmpfile' -> '$outfile: $!\n'";
  71 }
  72
  73 sub line_match ($$)
  74 {
  75   my ($re, $file) = (shift, shift);
  76   # Try both builddir and srcdir, with builddir first, to play nice
  77   # with VPATH builds.
  78   open (FH, "<$file") or open (FH, "<$srcdir/$file")
  79     or die "$me: cannot open file '$file': $!\n";
  80   my $ret = 0;
  81   while (defined (my $line = <FH>))
  82     {
  83       if ($line =~ $re)
  84         {
  85           $ret = 1;
  86           last;
  87         }
  88     }
  89   close FH or die "$me: cannot close file '$file': $!\n";
  90   return $ret;
  91 }
  92
  93 sub write_wrapper_script ($$$)
  94 {
  95   my ($file_handle, $wrapped_test, $shell_setup_code, $creator_name) = @_;
  96   print $file_handle unindent <<EOF;
  97     #! /bin/sh
  98     # This file has been automatically generated.  DO NOT EDIT BY HAND!
  99     . test-lib.sh
 100     $shell_setup_code
 101     # In the spirit of VPATH, we prefer a test in the build tree
 102     # over one in the source tree.
 103     for dir in . "\$am_top_srcdir"; do
 104       if test -f "\$dir/$wrapped_test"; then
 105         echo "\$0: will source \$dir/$wrapped_test"
 106         . "\$dir/$wrapped_test"; exit \$?
 107       fi
 108     done
 109     echo "\$0: cannot find wrapped test '$wrapped_test'" >&2
 110     exit 99
 111 EOF
 112 }
 113
 114 sub get_list_of_tests ()
 115 {
 116   my $make = defined $ENV{MAKE} ? $ENV{MAKE} : "make";
 117   # Unset MAKEFLAGS, for when we are called from make itself.
 118   my $cmd = "MAKEFLAGS= && unset MAKEFLAGS && cd '$srcdir' && "
 119             . "$make -s -f $testdir/list-of-tests.mk print-list-of-tests";
 120   my @tests_list = split /\s+/, `$cmd`;
 121   die "$me: cannot get list of tests\n" unless $? == 0 && @tests_list;
 122   my $ok = 1;
 123   foreach my $test (@tests_list)
 124     {
 125       # Respect VPATH builds.
 126       next if -f $test || -f "$srcdir/$test";
 127       warn "$me: test '$test' not found\n";
 128       $ok = 0;
 129     }
 130   die "$me: some test scripts not found\n" if !$ok;
 131   return @tests_list;
 132 }
 133
 134 sub parse_options (@)
 135 {
 136   use Getopt::Long qw/GetOptions/;
 137   local @ARGV = @_;
 138   GetOptions ('srcdir=s' => \$srcdir) or die "$me: usage error\n";
 139   die "$me: too many arguments\n" if @ARGV > 0;
 140   die "$me: srcdir '$srcdir': not a directory\n" unless -d $srcdir;
 141 }
 142
 143 #--------------------------------------------------------------------------
 144
 145 my %deps_extractor =
 146   (
 147     libtool_macros =>
 148       {
 149         line_matcher => qr/^\s*required=.*\blibtool/,
 150         nodist_prereqs => "$testdir/libtool-macros.log",
 151       },
 152     gettext_macros =>
 153       {
 154         line_matcher => qr/^\s*required=.*\bgettext/,
 155         nodist_prereqs => "$testdir/gettext-macros.log",
 156       },
 157     pkgconfig_macros =>
 158       {
 159         line_matcher => qr/^\s*required=.*\bpkg-config/,
 160         nodist_prereqs => "$testdir/pkg-config-macros.log",
 161       },
 162     use_trivial_test_driver =>
 163       {
 164         line_matcher => qr/\btrivial-test-driver\b/,
 165         dist_prereqs => "$testauxdir/trivial-test-driver",
 166       },
 167     check_testsuite_summary =>
 168       {
 169         line_matcher => qr/\btestsuite-summary-checks\.sh\b/,
 170         dist_prereqs => "$testauxdir/testsuite-summary-checks.sh",
 171       },
 172     extract_testsuite_summary =>
 173       {
 174         line_matcher => qr/\bextract-testsuite-summary\.pl\b/,
 175         dist_prereqs => "$testauxdir/extract-testsuite-summary.pl",
 176       },
 177     check_tap_testsuite_summary =>
 178       {
 179         line_matcher => qr/\btap-summary-aux\.sh\b/,
 180         dist_prereqs => "$testauxdir/tap-summary-aux.sh",
 181       },
 182     on_tap_with_common_setup =>
 183       {
 184         line_matcher => qr/\btap-setup\.sh\b/,
 185         dist_prereqs => "$testauxdir/tap-setup.sh",
 186         nodist_prereqs => "$testdir/tap-common-setup.log",
 187       },
 188     depcomp =>
 189       {
 190         line_matcher => qr/\bdepcomp\.sh\b/,
 191         dist_prereqs => "$testauxdir/depcomp.sh",
 192       },
 193   );
 194
 195 #--------------------------------------------------------------------------
 196
 197 my %test_generators =
 198   (
 199     #
 200     # Any test script in the Automake testsuite that checks features of
 201     # the Automake-provided parallel testsuite harness might want to
 202     # define a sibling test that does similar checks, but for the old
 203     # serial testsuite harness instead.
 204     #
 205     # Individual tests can request the creation of such a sibling by
 206     # making the string "try-with-serial-tests" appear any line of the
 207     # test itself.
 208     #
 209     serial_testsuite_harness =>
 210       {
 211         line_matcher     => qr/\btry-with-serial-tests\b/,
 212         shell_setup_code => 'am_serial_tests=yes',
 213       },
 214     #
 215     # For each test script in the Automake testsuite that tests features
 216     # of one or more automake-provided shell script from the 'lib/'
 217     # subdirectory by running those scripts directly (i.e., not thought
 218     # make calls and automake-generated makefiles), define a sibling test
 219     # that does likewise, but running the said script with the configure
 220     # time $SHELL instead of the default system shell /bin/sh.
 221     #
 222     # A test is considered a candidate for sibling-generation if it calls
 223     # the 'get_shell_script' function anywhere.
 224     #
 225     # Individual tests can prevent the creation of such a sibling by
 226     # explicitly setting the '$am_test_prefer_config_shell' variable
 227     # to either "yes" or "no".
 228     # The rationale for this is that if the variable is set to "yes",
 229     # the test already uses $SHELL, so that a sibling would be just a
 230     # duplicate; while if the variable is set to "no", the test doesn't
 231     # support, or is not meant to use, $SHELL to run the script under
 232     # testing, and forcing it to do so in the sibling would likely
 233     # cause a spurious failure.
 234     #
 235     prefer_config_shell =>
 236       {
 237         line_matcher =>
 238           qr/(^|\s)get_shell_script\s/,
 239         line_rejecter =>
 240           qr/\bam_test_prefer_config_shell=/,
 241         shell_setup_code =>
 242           'am_test_prefer_config_shell=yes',
 243       },
 244     #
 245     # Tests on tap support should be run with both the perl and awk
 246     # implementations of the TAP driver (they run with the awk one
 247     # by default).
 248     #
 249     perl_tap_driver =>
 250       {
 251         line_matcher =>
 252           qr<(?:\bfetch_tap_driver\b|[\s/]tap-setup\.sh\b)>,
 253         line_rejecter =>
 254           qr/\bam_tap_implementation=/,
 255         shell_setup_code =>
 256           'am_tap_implementation=perl',
 257       },
 258   );
 259
 260 #--------------------------------------------------------------------------
 261
 262 parse_options @ARGV;
 263
 264 my @all_tests = get_list_of_tests;
 265 my @generated_tests = (); # Will be updated later.
 266
 267 print "## -*- Makefile -*-\n";
 268 print "## Generated by $me.  DO NOT EDIT BY HAND!\n\n";
 269
 270 print <<EOF;
 271
 272 ## --------------------------------------------- ##
 273 ##  Autogenerated tests and their dependencies.  ##
 274 ## --------------------------------------------- ##
 275
 276 EOF
 277
 278 # A test script '$test' can possibly match more than one condition, so
 279 # for each tests we need to keep a list of generated wrapper tests.
 280 # Since what defines these wrapper scripts is the set of initializations
 281 # that are issued before sourcing the original, wrapped tests, these
 282 # initializations is what we put in our list entries.
 283 # The list will be saved in the hash entry '$wrapper_setups{$test}'.
 284 my %wrapper_setups = ();
 285 foreach my $test (@all_tests)
 286   {
 287     my @setups = ('');
 288     foreach my $x (values %test_generators)
 289     {
 290       next
 291         if not line_match $x->{line_matcher}, $test;
 292       next
 293         if $x->{line_rejecter} and line_match $x->{line_rejecter}, $test;
 294       @setups = map { ($_, "$_\n$x->{shell_setup_code}") } @setups;
 295     }
 296     @setups = grep { $_ ne '' } @setups;
 297     $wrapper_setups{$test} = \@setups if @setups;
 298   }
 299 # And now create all the wrapper tests.
 300 while (my ($wrapped_test, $setup_list) = each %wrapper_setups)
 301   {
 302     (my $base = $wrapped_test) =~ s/\.([^.]*)$//;
 303     my $suf = $1 or die "$me: test '$wrapped_test' lacks a suffix\n";
 304     my $count = 0;
 305     foreach my $setup (@$setup_list)
 306       {
 307         $count++;
 308         my $wbase = "$base-w" . ($count > 1 ? $count : '');
 309         my $wrapper_test =  "$wbase.$suf";
 310         # Register wrapper test as "autogenerated".
 311         push @generated_tests, $wrapper_test;
 312         # Create wrapper test.
 313         atomic_write $wrapper_test,
 314                      sub { write_wrapper_script $_[0], $wrapped_test,
 315                                                 $setup },
 316                      0555;
 317         # The generated test works by sourcing the original test, so that
 318         # it has to be re-run every time that changes ...
 319         print "$wbase.log: $wrapped_test\n";
 320         # ... but also every time the prerequisites of the wrapped test
 321         # changes.  The simpler (although suboptimal) way to do so is to
 322         # ensure that the wrapped tests runs before the wrapper one (in
 323         # case it needs to be re-run *at all*).
 324         # FIXME: we could maybe refactor the script to find a more
 325         # granular way to express such implicit dependencies.
 326         print "$wbase.log: $base.log\n";
 327       }
 328   }
 329
 330 print <<EOF;
 331
 332 ## ---------------------------------------------------- ##
 333 ##  Ad-hoc autogenerated tests and their dependencies.  ##
 334 ## ---------------------------------------------------- ##
 335
 336 EOF
 337
 338 print "## Tests on automatic dependency tracking (see 'depcomp.sh').\n";
 339
 340 # Key: depmode, value: list of required programs.
 341 my %depmodes =
 342   (
 343     auto         => ["cc"],
 344     disabled     => ["cc"],
 345     makedepend   => ["cc", "makedepend"],
 346     dashmstdout  => ["gcc"],
 347     cpp          => ["gcc"],
 348 # This was for older (pre-3.x) GCC versions (newer versions
 349 # have depmode "gcc3").  But other compilers use this depmode
 350 # as well (for example, the IMB xlc/xlC compilers, and the HP
 351 # C compiler, see 'lib/depcomp' for more info), so it's not
 352 # obsolete, and it's worth giving it some coverage.
 353     gcc          => ["gcc"],
 354 # This is for older (pre-7) msvc versions.  Newer versions
 355 # have depmodes "msvc7" and "msvc7msys".
 356     msvisualcpp  => ["cl", "cygpath"],
 357     msvcmsys     => ["cl", "mingw"],
 358   );
 359
 360 foreach my $lt (TRUE, FALSE)
 361   {
 362     foreach my $m (keys %depmodes)
 363       {
 364         my $planned = ($lt && $m eq "auto") ? 84 : 28;
 365         my @required =
 366           (
 367             @{$depmodes{$m}},
 368             $lt ? ("libtoolize",) : (),
 369           );
 370         my @vars_init =
 371           (
 372             "am_create_testdir=empty",
 373             "depmode=$m",
 374             "depcomp_with_libtool=" . ($lt ? "yes" : "no"),
 375           );
 376         my $test = "$testdir/depcomp" . ($lt ? "-lt-" : "-") . "$m.tap";
 377         # Register wrapper test as "autogenerated" ...
 378         push @generated_tests, $test;
 379         # ... and create it.
 380         atomic_write ($test, sub
 381           {
 382             my $file_handle = shift;
 383             print $file_handle unindent <<EOF;
 384               #! /bin/sh
 385               # Automatically generated test.  DO NOT EDIT BY HAND!
 386               @vars_init
 387               required="@required"
 388               . test-init.sh
 389               plan_ $planned
 390               . depcomp.sh
 391               exit \$?
 392 EOF
 393           },
 394           0555);
 395       }
 396    }
 397
 398 # Update generated makefile fragment to account for all the generated tests.
 399 print "generated_TESTS =\n";
 400 map { print "generated_TESTS += $_\n" } @generated_tests;
 401
 402 # The test scripts are scanned for automatic dependency generation *after*
 403 # the generated tests have been created, so they too can be scanned.  To
 404 # do so correctly, we need to update the list in '@all_tests' to make it
 405 # comprise also the freshly-generated tests.
 406
 407 push @all_tests, @generated_tests;
 408
 409 print <<EOF;
 410
 411 ## ----------------------------- ##
 412 ##  Autogenerated dependencies.  ##
 413 ## ----------------------------- ##
 414
 415 EOF
 416
 417 while (my ($k, $x) = each %deps_extractor)
 418   {
 419     my $dist_prereqs = $x->{dist_prereqs} || "";
 420     my $nodist_prereqs = $x->{nodist_prereqs} || "";
 421     my @tests = grep { line_match $x->{line_matcher}, $_ } @all_tests;
 422     map { s/\.[^.]*$//; s/$/\.log/; } (my @logs = @tests);
 423     print "## Added by deps-extracting key '$k'.\n";
 424     ## The list of all tests which have a dependency detected by the
 425     ## current key.
 426     print join(" \\\n  ", "${k}_TESTS =", @tests) . "\n";
 427     print "EXTRA_DIST += $dist_prereqs\n";
 428     map { print "$_: $dist_prereqs $nodist_prereqs\n" } @logs;
 429     print "\n";
 430   }
 431
 432 __END__