gen-testsuite-part

   1 #! /usr/bin/env perl
   2 # Automatically compute some dependencies for the hand-written tests
   3 # of the Automake testsuite.  Also, automatically generate some more
   4 # tests from them (for particular cases/setups only).
   5
   6 # Copyright (C) 2011-2012 Free Software Foundation, Inc.
   7
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2, or (at your option)
  11 # any later version.
  12
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 #--------------------------------------------------------------------------
  22
  23 use warnings FATAL => "all";
  24 use strict;
  25 use File::Basename ();
  26 use constant TRUE => 1;
  27 use constant FALSE => 0;
  28
  29 my $me = File::Basename::basename $0;
  30
  31 # For use in VPATH builds.
  32 my $srcdir = ".";
  33
  34 #--------------------------------------------------------------------------
  35
  36 sub unindent ($)
  37 {
  38   my $text = shift;
  39   $text =~ /^(\s*)/;
  40   my $indentation = $1;
  41   $text =~ s/^$indentation//gm;
  42   return $text;
  43 }
  44
  45 sub atomic_write ($$;$)
  46 {
  47   my ($outfile, $func) = (shift, shift);
  48   my $perms = @_ > 0 ? shift : 0777;
  49   my $tmpfile = "$outfile-t";
  50   foreach my $f ($outfile, $tmpfile)
  51     {
  52       unlink $f or die "$me: cannot unlink '$f': $!\n"
  53         if -e $f;
  54     }
  55   open (my $fh, ">$tmpfile")
  56     or die "$me: can't write to '$tmpfile': $!\n";
  57   $func->($fh);
  58   close $fh
  59     or die "$me: closing '$tmpfile': $!\n";
  60   chmod ($perms & ~umask, $tmpfile)
  61     or die "$me: cannot change perms for '$tmpfile': $!\n";
  62   rename ($tmpfile, $outfile)
  63     or die "$me: renaming '$tmpfile' -> '$outfile: $!\n'";
  64 }
  65
  66 sub line_match ($$)
  67 {
  68   my ($re, $file) = (shift, shift);
  69   # Try both builddir and srcdir, with builddir first, to play nice
  70   # with VPATH builds.
  71   open (FH, "<$file") or open (FH, "<$srcdir/$file")
  72     or die "$me: cannot open file '$file': $!\n";
  73   my $ret = 0;
  74   while (defined (my $line = <FH>))
  75     {
  76       if ($line =~ $re)
  77         {
  78           $ret = 1;
  79           last;
  80         }
  81     }
  82   close FH or die "$me: cannot close file '$file': $!\n";
  83   return $ret;
  84 }
  85
  86 sub write_wrapper_script ($$$)
  87 {
  88   my ($file_handle, $wrapped_test, $shell_setup_code, $creator_name) = @_;
  89   # FIXME: we use some creative quoting in the generated scripts,
  90   # FIXME: to please maintainer-check.
  91   print $file_handle unindent <<EOF;
  92     #! /bin/sh
  93     # This file has been automatically generated.  DO NOT EDIT BY HAND!
  94     . ./defs-static || exit '99';
  95     $shell_setup_code
  96     # In the spirit of VPATH, we prefer a test in the build tree
  97     # over one in the source tree.
  98     for dir in . "\$am_top_srcdir"; do
  99       if test -f "\$dir/$wrapped_test"; then
 100         echo "\$0: will source \$dir/$wrapped_test"
 101         . "\$dir/$wrapped_test"; exit "\$?"
 102       fi
 103     done
 104     echo "\$0: cannot find wrapped test '$wrapped_test'" >&2
 105     exit '99'
 106 EOF
 107 }
 108
 109 sub get_list_of_tests ()
 110 {
 111   my $make = defined $ENV{MAKE} ? $ENV{MAKE} : "make";
 112   # Unset MAKEFLAGS, for when we are called from make itself.
 113   my $cmd = "MAKEFLAGS= && unset MAKEFLAGS && cd '$srcdir' && "
 114             . "$make -s -f t/list-of-tests.mk print-list-of-tests";
 115   my @tests_list = split /\s+/, `$cmd`;
 116   die "$me: cannot get list of tests\n" unless $? == 0 && @tests_list;
 117   my $ok = 1;
 118   foreach my $test (@tests_list)
 119     {
 120       # Respect VPATH builds.
 121       next if -f $test || -f "$srcdir/$test";
 122       warn "$me: test '$test' not found\n";
 123       $ok = 0;
 124     }
 125   die "$me: some test scripts not found\n" if !$ok;
 126   return @tests_list;
 127 }
 128
 129 sub parse_options (@)
 130 {
 131   use Getopt::Long qw/GetOptions/;
 132   local @ARGV = @_;
 133   GetOptions ('srcdir=s' => \$srcdir) or die "$me: usage error\n";
 134   die "$me: too many arguments\n" if @ARGV > 0;
 135   die "$me: srcdir '$srcdir': not a directory\n" unless -d $srcdir;
 136 }
 137
 138 #--------------------------------------------------------------------------
 139
 140 # Where testsuite-related helper scripts, data files and shell libraries
 141 # are placed.  Relative to the 't/' subdirectory.
 142 my $auxdir = "ax";
 143
 144 my %deps_extractor =
 145   (
 146     libtool_macros =>
 147       {
 148         line_matcher => qr/^\s*required=.*\blibtool/,
 149         nodist_prereqs => "t/libtool-macros.log",
 150       },
 151     gettext_macros =>
 152       {
 153         line_matcher => qr/^\s*required=.*\bgettext/,
 154         nodist_prereqs => "t/gettext-macros.log",
 155       },
 156     use_trivial_test_driver =>
 157       {
 158         line_matcher => qr/\btrivial-test-driver\b/,
 159         dist_prereqs => "t/$auxdir/trivial-test-driver",
 160       },
 161     check_testsuite_summary =>
 162       {
 163         line_matcher => qr/\btestsuite-summary-checks\.sh\b/,
 164         dist_prereqs => "t/$auxdir/testsuite-summary-checks.sh",
 165       },
 166     extract_testsuite_summary =>
 167       {
 168         line_matcher => qr/\bextract-testsuite-summary\.pl\b/,
 169         dist_prereqs => "t/$auxdir/extract-testsuite-summary.pl",
 170       },
 171     check_tap_testsuite_summary =>
 172       {
 173         line_matcher => qr/\btap-summary-aux\.sh\b/,
 174         dist_prereqs => "t/$auxdir/tap-summary-aux.sh",
 175       },
 176     on_tap_with_common_setup =>
 177       {
 178         line_matcher => qr/\btap-setup\.sh\b/,
 179         dist_prereqs => "t/$auxdir/tap-setup.sh",
 180         nodist_prereqs => "t/tap-common-setup.log",
 181       },
 182     depcomp =>
 183       {
 184         line_matcher => qr/\bdepcomp\.sh\b/,
 185         dist_prereqs => "t/$auxdir/depcomp.sh",
 186       },
 187   );
 188
 189 #--------------------------------------------------------------------------
 190
 191 my %test_generators =
 192   (
 193     #
 194     # For each test script in the Automake testsuite that itself tests
 195     # features of the TESTS automake interface, define a sibling test
 196     # that does likewise, but with the option 'parallel-tests' enabled.
 197     #
 198     # A test is considered a candidate for sibling-generation if any
 199     # Makefile.am generated by it define the TESTS variable.
 200     #
 201     # Individual tests can prevent the creation of such a sibling by
 202     # explicitly setting the '$am_parallel_tests' variable to either "yes"
 203     # or "no".  The rationale for this is that if the variable is set to
 204     # "yes", the test already uses the 'parallel-tests' option, so that
 205     # a sibling would be just a duplicate; while if the variable is set
 206     # to "no", the test doesn't support, or is not meant to run with, the
 207     # 'parallel-tests' option, and forcing it to do so in the sibling
 208     # would likely cause a spurious failure.
 209     #
 210     parallel_testsuite_harness =>
 211       {
 212         line_matcher =>
 213           qr/(?:^|\s)TESTS\s*=/,
 214         line_rejecter =>
 215           qr/(?:^[^#]*\bparallel-tests\b)|\bam_parallel_tests=/,
 216         shell_setup_code =>
 217           'am_parallel_tests=yes'
 218       },
 219     #
 220     # For each test script in the Automake testsuite that tests features
 221     # of one or more automake-provided shell script from the 'lib/'
 222     # subdirectory by running those scripts directly (i.e., not thought
 223     # make calls and automake-generated makefiles), define a sibling test
 224     # that does likewise, but running the said script with the configure
 225     # time $SHELL instead of the default system shell /bin/sh.
 226     #
 227     # A test is considered a candidate for sibling-generation if it calls
 228     # the 'get_shell_script' function anywhere.
 229     #
 230     # Individual tests can prevent the creation of such a sibling by
 231     # explicitly setting the '$am_test_prefer_config_shell' variable
 232     # to either "yes" or "no".
 233     # The rationale for this is that if the variable is set to "yes",
 234     # the test already uses $SHELL, so that a sibling would be just a
 235     # duplicate; while if the variable is set to "no", the test doesn't
 236     # support, or is not meant to use, $SHELL to run the script under
 237     # testing, and forcing it to do so in the sibling would likely
 238     # cause a spurious failure.
 239     #
 240     prefer_config_shell =>
 241       {
 242         line_matcher =>
 243           qr/(^|\s)get_shell_script\s/,
 244         line_rejecter =>
 245           qr/\bam_test_prefer_config_shell=/,
 246         shell_setup_code =>
 247           'am_test_prefer_config_shell=yes',
 248       },
 249     #
 250     # Tests on tap support should be run with both the perl and awk
 251     # implementations of the TAP driver (they run with the awk one
 252     # by default).
 253     #
 254     perl_tap_driver =>
 255       {
 256         line_matcher =>
 257           qr<(?:\bfetch_tap_driver\b|[\s/]tap-setup\.sh\b)>,
 258         line_rejecter =>
 259           qr/\bam_tap_implementation=/,
 260         shell_setup_code =>
 261           'am_tap_implementation=perl',
 262       },
 263   );
 264
 265 #--------------------------------------------------------------------------
 266
 267 parse_options @ARGV;
 268
 269 my @all_tests = get_list_of_tests;
 270 my @generated_tests = (); # Will be updated later.
 271
 272 print "## -*- Makefile -*-\n";
 273 print "## Generated by $me.  DO NOT EDIT BY HAND!\n\n";
 274
 275 print <<EOF;
 276
 277 ## --------------------------------------------- ##
 278 ##  Autogenerated tests and their dependencies.  ##
 279 ## --------------------------------------------- ##
 280
 281 EOF
 282
 283 # FIXME: the following is not really right, since cannot compose wrapping
 284 # of tests matching more than one condition.  Still, there should be no
 285 # such test at the moment, so the limitation is (temporarily) acceptable.
 286 while (my ($k, $g) = each %test_generators)
 287   {
 288     my @wrapped_tests = grep {
 289       line_match ($g->{line_matcher}, $_)
 290         && !line_match ($g->{line_rejecter}, $_)
 291     } @all_tests;
 292     foreach my $wrapped_test (@wrapped_tests)
 293       {
 294         (my $base = $wrapped_test) =~ s/\.([^.]*)$//;
 295         my $suf = $1 or die "$me: test '$wrapped_test' lacks a suffix\n";
 296         my $wrapper_test =  "$base-w.$suf";
 297         # Register wrapper test as "autogenerated".
 298         push @generated_tests, $wrapper_test;
 299         # Create wrapper test.
 300         atomic_write $wrapper_test,
 301                      sub { write_wrapper_script $_[0], $wrapped_test,
 302                            $g->{shell_setup_code} },
 303                      0555;
 304         # The generated test works by sourcing the original test, so that
 305         # it has to be re-run every time that changes ...
 306         print "$base-w.log: $wrapped_test\n";
 307         # ... but also every time the prerequisites of the wrapped test
 308         # changes.  The simpler (although suboptimal) way to do so is to
 309         # ensure that the wrapped tests runs before the wrappee one (in
 310         # case it needs to be re-run *at all*.
 311         # FIXME: we could maybe refactor the script to find a more
 312         # granular way to express such implicit dependencies.
 313         print "$base-w.log: $base.log\n";
 314       }
 315   }
 316
 317 print <<EOF;
 318
 319 ## ---------------------------------------------------- ##
 320 ##  Ad-hoc autogenerated tests and their dependencies.  ##
 321 ## ---------------------------------------------------- ##
 322
 323 EOF
 324
 325 print "## Tests on automatic dependency tracking (see 'depcomp.sh').\n";
 326
 327 # Key: depmode, value: list of required programs.
 328 my %depmodes =
 329   (
 330     auto         => ["cc"],
 331     disabled     => ["cc"],
 332     makedepend   => ["cc", "makedepend"],
 333     dashmstdout  => ["gcc"],
 334     cpp          => ["gcc"],
 335 # This is for older (pre-3.x) GCC versions.  Newer versions
 336 # have depmode "gcc3".
 337     gcc          => ["gcc"],
 338 # This is for older (pre-7) msvc versions.  Newer versions
 339 # have depmodes "msvc7" and "msvc7msys".
 340     msvisualcpp  => ["cl", "cygpath"],
 341     msvcmsys     => ["cl", "mingw"],
 342   );
 343
 344 foreach my $lt (TRUE, FALSE)
 345   {
 346     foreach my $m (keys %depmodes)
 347       {
 348         my $planned = ($lt && $m eq "auto") ? 84 : 28;
 349         my @required =
 350           (
 351             @{$depmodes{$m}},
 352             $lt ? ("libtoolize",) : (),
 353           );
 354         my @vars_init =
 355           (
 356             "am_create_testdir=empty",
 357             "depmode=$m",
 358             "depcomp_with_libtool=" . ($lt ? "yes" : "no"),
 359           );
 360         my $test = "t/depcomp" . ($lt ? "-lt-" : "-") . $m . ".tap";
 361         # Register wrapper test as "autogenerated" ...
 362         push @generated_tests, $test;
 363         # ... and create it.
 364         atomic_write ($test, sub
 365           {
 366             my $file_handle = shift;
 367             print $file_handle unindent <<EOF;
 368               #! /bin/sh
 369               # Automatically generated test.  DO NOT EDIT BY HAND!
 370               @vars_init
 371               required="@required"
 372               . ./defs || Exit 1
 373               plan_ $planned
 374               . "\$am_testauxdir/depcomp.sh"; exit "\$?"
 375 EOF
 376           },
 377           0555);
 378       }
 379    }
 380
 381 # Update generated makefile fragment to account for all the generated tests.
 382 print "generated_TESTS =\n";
 383 map { print "generated_TESTS += $_\n" } @generated_tests;
 384
 385 # The test scripts are scanned for automatic dependency generation *after*
 386 # the generated tests have been created, so they too can be scanned.  To
 387 # do so correctly, we need to update the list in '@all_tests' to make it
 388 # comprise also the freshly-generated tests.
 389
 390 push @all_tests, @generated_tests;
 391
 392 print <<EOF;
 393
 394 ## ----------------------------- ##
 395 ##  Autogenerated dependencies.  ##
 396 ## ----------------------------- ##
 397
 398 EOF
 399
 400 while (my ($k, $x) = each %deps_extractor)
 401   {
 402     my $dist_prereqs = $x->{dist_prereqs} || "";
 403     my $nodist_prereqs = $x->{nodist_prereqs} || "";
 404     my @tests = grep { line_match $x->{line_matcher}, $_ } @all_tests;
 405     map { s/\.[^.]*$//; s/$/\.log/; } (my @logs = @tests);
 406     print "## Added by deps-extracting key '$k'.\n";
 407     ## The list of all tests which have a dependency detected by the
 408     ## current key.
 409     print join(" \\\n  ", "${k}_TESTS =", @tests) . "\n";
 410     print "EXTRA_DIST += $dist_prereqs\n";
 411     map { print "$_: $dist_prereqs $nodist_prereqs\n" } @logs;
 412     print "\n";
 413   }
 414
 415 __END__