4 # Modification history:
5 # Written 91-12-02 through 92-01-01 by Stephen McGee.
6 # Modified 92-02-11 through 92-02-22 by Chris Arthur to further generalize.
7 # End of modification history
9 # Test driver routines used by a number of test suites, including
10 # those for SCS, make, roll_dir, and scan_deps (?).
12 # this routine controls the whole mess; each test suite sets up a few
13 # variables and then calls &toplevel, which does all the real work.
15 # $Id: test_driver.pl,v 1.13 2004/09/21 05:39:04 psmith Exp $
18 # The number of test categories we've run
20 # The number of test categroies that have passed
21 $categories_passed = 0;
22 # The total number of individual tests that have been run
24 # The total number of individual tests that have passed
25 $total_tests_passed = 0;
26 # The number of tests in this category that have been run
28 # The number of tests in this category that have passed
32 # Yeesh. This whole test environment is such a hack!
37 # Get a clean environment
41 # Pull in benign variables from the user's environment
43 foreach (# UNIX-specific things
44 'TZ', 'LANG', 'TMPDIR', 'HOME', 'USER', 'LOGNAME', 'PATH',
47 # Windows NT-specific stuff
49 # DJGPP-specific stuff
50 'DJDIR', 'DJGPP', 'SHELL', 'COMSPEC', 'HOSTNAME', 'LFN',
51 'FNCASE', '387', 'EMU387', 'GROUP'
53 $makeENV{$_} = $ENV{$_} if $ENV{$_};
56 # Replace the environment with the new one
60 # We used to say "%ENV = ();" but this doesn't work in Perl 5.000
61 # through Perl 5.004. It was fixed in Perl 5.004_01, but we don't
62 # want to require that here, so just delete each one individually.
64 foreach $v (keys %ENV) {
70 $| = 1; # unbuffered output
72 $debug = 0; # debug flag
73 $profile = 0; # profiling flag
74 $verbose = 0; # verbose mode flag
75 $detail = 0; # detailed verbosity
76 $keep = 0; # keep temp files around
77 $workdir = "work"; # The directory where the test will start running
78 $scriptdir = "scripts"; # The directory where we find the test scripts
79 $tmpfilesuffix = "t"; # the suffix used on tmpfiles
80 $default_output_stack_level = 0; # used by attach_default_output, etc.
81 $default_input_stack_level = 0; # used by attach_default_input, etc.
82 $cwd = "."; # don't we wish we knew
83 $cwdslash = ""; # $cwd . $pathsep, but "" rather than "./"
85 &get_osname
; # sets $osname, $vos, $pathsep, and $short_filenames
87 &set_defaults
; # suite-defined
89 &parse_command_line
(@ARGV);
91 print "OS name = `$osname'\n" if $debug;
93 $workpath = "$cwdslash$workdir";
94 $scriptpath = "$cwdslash$scriptdir";
96 &set_more_defaults
; # suite-defined
102 print "Clearing $workpath...\n";
103 &remove_directory_tree
("$workpath/")
104 || &error
("Couldn't wipe out $workpath\n");
108 mkdir ($workpath, 0777) || &error
("Couldn't mkdir $workpath: $!\n");
113 &error
("Failed to find $scriptpath containing perl test scripts.\n");
118 print "Making work dirs...\n";
119 foreach $test (@TESTS)
121 if ($test =~ /^([^\/]+)\
//)
124 push (@rmdirs, $dir);
126 || mkdir ("$workpath/$dir", 0777)
127 || &error
("Couldn't mkdir $workpath/$dir: $!\n");
133 print "Finding tests...\n";
134 opendir (SCRIPTDIR
, $scriptpath)
135 || &error
("Couldn't opendir $scriptpath: $!\n");
136 @dirs = grep (!/^(\.\.?|CVS|RCS)$/, readdir (SCRIPTDIR
) );
137 closedir (SCRIPTDIR
);
140 next if ($dir =~ /^\.\.?$/ || $dir eq 'CVS' || $dir eq 'RCS'
141 || ! -d
"$scriptpath/$dir");
142 push (@rmdirs, $dir);
143 mkdir ("$workpath/$dir", 0777)
144 || &error
("Couldn't mkdir $workpath/$dir: $!\n");
145 opendir (SCRIPTDIR
, "$scriptpath/$dir")
146 || &error
("Couldn't opendir $scriptpath/$dir: $!\n");
147 @files = grep (!/^(\.\.?|CVS|RCS)$/, readdir (SCRIPTDIR
) );
148 closedir (SCRIPTDIR
);
149 foreach $test (@files)
151 next if $test =~ /~$/ || -d
$test;
152 push (@TESTS, "$dir/$test");
159 &error
("\nNo tests in $scriptpath, and none were specified.\n");
166 foreach $dir (@rmdirs)
168 rmdir ("$workpath/$dir");
173 $categories_failed = $categories_run - $categories_passed;
174 $total_tests_failed = $total_tests_run - $total_tests_passed;
176 if ($total_tests_failed)
178 print "\n$total_tests_failed Test";
179 print "s" unless $total_tests_failed == 1;
180 print " in $categories_failed Categor";
181 print ($categories_failed == 1 ?
"y" : "ies");
182 print " Failed (See .$diffext files in $workdir dir for details) :-(\n\n";
187 print "\n$total_tests_passed Test";
188 print "s" unless $total_tests_passed == 1;
189 print " in $categories_passed Categor";
190 print ($categories_passed == 1 ?
"y" : "ies");
191 print " Complete ... No Failures :-)\n\n";
198 # Set up an initial value. In perl5 we can do it the easy way.
200 $osname = defined($^O
) ?
$^O
: '';
202 # See if the filesystem supports long file names with multiple
204 $short_filenames = 0;
205 (open (TOUCHFD
, "> fancy.file.name") && close (TOUCHFD
))
206 || ($short_filenames = 1);
207 unlink ("fancy.file.name") || ($short_filenames = 1);
209 if (! $short_filenames) {
210 # Thanks go to meyering@cs.utexas.edu (Jim Meyering) for suggesting a
211 # better way of doing this. (We used to test for existence of a /mnt
212 # dir, but that apparently fails on an SGI Indigo (whatever that is).)
213 # Because perl on VOS translates /'s to >'s, we need to test for
214 # VOSness rather than testing for Unixness (ie, try > instead of /).
216 mkdir (".ostest", 0777) || &error
("Couldn't create .ostest: $!\n", 1);
217 open (TOUCHFD
, "> .ostest>ick") && close (TOUCHFD
);
218 chdir (".ostest") || &error
("Couldn't chdir to .ostest: $!\n", 1);
221 if (! $short_filenames && -f
"ick")
229 # the following is regrettably knarly, but it seems to be the only way
230 # to not get ugly error messages if uname can't be found.
231 # Hmmm, BSD/OS 2.0's uname -a is excessively verbose. Let's try it
232 # with switches first.
233 eval "chop (\$osname = `sh -c 'uname -nmsr 2>&1'`)";
234 if ($osname =~ /not found/i)
236 $osname = "(something unixy with no uname)";
238 elsif ($@
ne "" || $?
)
240 eval "chop (\$osname = `sh -c 'uname -a 2>&1'`)";
243 $osname = "(something unixy)";
250 if (! $short_filenames) {
251 chdir ("..") || &error
("Couldn't chdir to ..: $!\n", 1);
252 unlink (".ostest>ick");
253 rmdir (".ostest") || &error
("Couldn't rmdir .ostest: $!\n", 1);
257 sub parse_command_line
261 # use @ARGV if no args were passed in
268 # look at each option; if we don't recognize it, maybe the suite-specific
269 # command line parsing code will...
273 $option = shift @argv;
274 if ($option =~ /^-debug$/i)
276 print "\nDEBUG ON\n";
279 elsif ($option =~ /^-usage$/i)
284 elsif ($option =~ /^-(h|help)$/i)
289 elsif ($option =~ /^-profile$/i)
293 elsif ($option =~ /^-verbose$/i)
297 elsif ($option =~ /^-detail$/i)
302 elsif ($option =~ /^-keep$/i)
306 elsif (&valid_option
($option))
308 # The suite-defined subroutine takes care of the option
310 elsif ($option =~ /^-/)
312 print "Invalid option: $option\n";
316 else # must be the name of a test
318 $option =~ s/\.pl$//;
319 push(@TESTS,$option);
326 local($num) = shift @_;
343 local($width, $string) = @_;
346 if (length ($string))
348 $pad = " " x
( ($width - length ($string) + 1) / 2);
359 $info = "Running tests for $testee on $osname\n"; # $testee is suite-defined
360 $len = &max
(length ($line), length ($testee_version),
361 length ($banner_info), 73) + 5;
362 $line = ("-" x
$len) . "\n";
368 &print_centered
($len, $line);
369 &print_centered
($len, $info);
370 &print_centered
($len, $testee_version); # suite-defined
371 &print_centered
($len, $banner_info); # suite-defined
372 &print_centered
($len, $line);
380 foreach $testname (sort @TESTS)
383 $suite_passed = 1; # reset by test on failure
384 $num_of_logfiles = 0;
385 $num_of_tmpfiles = 0;
388 $old_makefile = undef;
389 $testname =~ s/^$scriptpath$pathsep//;
390 $perl_testname = "$scriptpath$pathsep$testname";
391 $testname =~ s/(\.pl|\.perl)$//;
392 $testpath = "$workpath$pathsep$testname";
393 # Leave enough space in the extensions to append a number, even
394 # though it needs to fit into 8+3 limits.
395 if ($short_filenames) {
407 $log_filename = "$testpath.$logext";
408 $diff_filename = "$testpath.$diffext";
409 $base_filename = "$testpath.$baseext";
410 $tmp_filename = "$testpath.$tmpfilesuffix";
412 &setup_for_test
; # suite-defined
414 $output = "........................................................ ";
416 substr($output,0,length($testname)) = "$testname ";
420 # Run the actual test!
423 $code = do $perl_testname;
425 $total_tests_run += $tests_run;
426 $total_tests_passed += $tests_passed;
434 warn "\n*** Test died ($testname): $@\n";
438 warn "\n*** Couldn't run $perl_testname\n";
441 elsif ($code == -1) {
444 elsif ($code != 1 && $code != -1) {
446 warn "\n*** Test returned $code\n";
450 ++$categories_passed;
451 $status = "ok ($tests_passed passed)";
452 for ($i = $num_of_tmpfiles; $i; $i--)
454 &delete ($tmp_filename . &num_suffix
($i) );
457 for ($i = $num_of_logfiles ?
$num_of_logfiles : 1; $i; $i--)
459 &delete ($log_filename . &num_suffix
($i) );
460 &delete ($base_filename . &num_suffix
($i) );
464 $status = "FAILED ($tests_passed/$tests_run passed)";
471 # If the verbose option has been specified, then a short description
472 # of each test is printed before displaying the results of each test
473 # describing WHAT is being tested.
479 print "\nWHAT IS BEING TESTED\n";
480 print "--------------------";
482 print "\n\n$description\n\n";
485 # If the detail option has been specified, then the details of HOW
486 # the test is testing what it says it is testing in the verbose output
487 # will be displayed here before the results of the test are displayed.
491 print "\nHOW IT IS TESTED\n";
492 print "----------------";
493 print "\n\n$details\n\n";
500 # If the keep flag is not set, this subroutine deletes all filenames that
509 return (unlink @files);
515 sub print_standard_usage
517 local($plname,@moreusage) = @_;
520 print "Usage: perl $plname [testname] [-verbose] [-detail] [-keep]\n";
521 print " [-profile] [-usage] [-help] "
523 foreach $line (@moreusage)
529 sub print_standard_help
531 local(@morehelp) = @_;
536 $line = "Test Driver For $testee";
538 $line = "=" x
length ($line);
544 . "${t}You may, if you wish, run only ONE test if you know the name\n"
545 . "${t}of that test and specify this name anywhere on the command\n"
546 . "${t}line. Otherwise ALL existing tests in the scripts directory\n"
547 . "${t}will be run.\n"
549 . "${t}If this option is given, a description of every test is\n"
550 . "${t}displayed before the test is run. (Not all tests may have\n"
551 . "${t}descriptions at this time)\n"
553 . "${t}If this option is given, a detailed description of every\n"
554 . "${t}test is displayed before the test is run. (Not all tests\n"
555 . "${t}have descriptions at this time)\n"
557 . "${t}If this option is given, then the profile file\n"
558 . "${t}is added to other profiles every time $testee is run.\n"
559 . "${t}This option only works on VOS at this time.\n"
561 . "${t}You may give this option if you DO NOT want ANY\n"
562 . "${t}of the files generated by the tests to be deleted. \n"
563 . "${t}Without this option, all files generated by the test will\n"
564 . "${t}be deleted IF THE TEST PASSES.\n"
566 . "${t}Use this option if you would like to see all of the system\n"
567 . "${t}calls issued and their return status while running the tests\n"
568 . "${t}This can be helpful if you're having a problem adding a test\n"
569 . "${t}to the suite, or if the test fails!\n";
571 foreach $line (@morehelp)
574 if (substr ($tline, 0, 1) eq "\t")
576 substr ($tline, 0, 1) = $t;
582 #######################################################################
583 ########### Generic Test Driver Subroutines ###########
584 #######################################################################
593 $depth = defined ($_[0]) ?
$_[0] : 1;
594 ($package, $filename, $linenum) = caller ($depth + 1);
595 return "$filename: $linenum";
600 local($message) = $_[0];
601 local($caller) = &get_caller
(1);
605 $caller = &get_caller
($_[1] + 1) . " -> $caller";
608 die "$caller: $message";
613 local($answer,$logfile) = @_;
616 print "Comparing Output ........ " if $debug;
618 $slurp = &read_file_into_string
($logfile);
620 # For make, get rid of any time skew error before comparing--too bad this
621 # has to go into the "generic" driver code :-/
622 $slurp =~ s/^.*modification time .*in the future.*\n//gm;
623 $slurp =~ s/^.*Clock skew detected.*\n//gm;
627 if ($slurp eq $answer && $test_passed)
629 print "ok\n" if $debug;
634 if ($slurp ne $answer) {
635 print "DIFFERENT OUTPUT\n" if $debug;
637 &create_file
(&get_basefile
, $answer);
639 print "\nCreating Difference File ...\n" if $debug;
641 # Create the difference file
642 local($command) = "diff -c " . &get_basefile
. " " . $logfile;
643 &run_command_with_output
(&get_difffile
,$command);
651 sub read_file_into_string
653 local($filename) = @_;
654 local($oldslash) = $/;
658 open (RFISFILE
, $filename) || return "";
659 local ($slurp) = <RFISFILE
>;
667 sub attach_default_output
669 local ($filename) = @_;
674 $code = system "++attach_default_output_hack $filename";
675 $code == -2 || &error
("adoh death\n", 1);
679 open ("SAVEDOS" . $default_output_stack_level . "out", ">&STDOUT")
680 || &error
("ado: $! duping STDOUT\n", 1);
681 open ("SAVEDOS" . $default_output_stack_level . "err", ">&STDERR")
682 || &error
("ado: $! duping STDERR\n", 1);
684 open (STDOUT
, "> " . $filename)
685 || &error
("ado: $filename: $!\n", 1);
686 open (STDERR
, ">&STDOUT")
687 || &error
("ado: $filename: $!\n", 1);
689 $default_output_stack_level++;
692 # close the current stdout/stderr, and restore the previous ones from
695 sub detach_default_output
701 $code = system "++detach_default_output_hack";
702 $code == -2 || &error
("ddoh death\n", 1);
706 if (--$default_output_stack_level < 0)
708 &error
("default output stack has flown under!\n", 1);
714 open (STDOUT
, ">&SAVEDOS" . $default_output_stack_level . "out")
715 || &error
("ddo: $! duping STDOUT\n", 1);
716 open (STDERR
, ">&SAVEDOS" . $default_output_stack_level . "err")
717 || &error
("ddo: $! duping STDERR\n", 1);
719 close ("SAVEDOS" . $default_output_stack_level . "out")
720 || &error
("ddo: $! closing SCSDOSout\n", 1);
721 close ("SAVEDOS" . $default_output_stack_level . "err")
722 || &error
("ddo: $! closing SAVEDOSerr\n", 1);
725 # run one command (passed as a list of arg 0 - n), returning 0 on success
726 # and nonzero on failure.
734 print "\nrun_command: @_\n";
736 print "run_command: \"@_\" returned $code.\n";
743 # run one command (passed as a list of arg 0 - n, with arg 0 being the
744 # second arg to this routine), returning 0 on success and non-zero on failure.
745 # The first arg to this routine is a filename to connect to the stdout
746 # & stderr of the child process.
748 sub run_command_with_output
750 local ($filename) = shift;
753 &attach_default_output
($filename);
755 &detach_default_output
;
758 print "run_command_with_output: \"@_\" returned $code.\n";
764 # performs the equivalent of an "rm -rf" on the first argument. Like
765 # rm, if the path ends in /, leaves the (now empty) directory; otherwise
768 sub remove_directory_tree
770 local ($targetdir) = @_;
771 local ($nuketop) = 1;
774 $ch = substr ($targetdir, length ($targetdir) - 1);
775 if ($ch eq "/" || $ch eq $pathsep)
777 $targetdir = substr ($targetdir, 0, length ($targetdir) - 1);
786 &remove_directory_tree_inner
("RDT00", $targetdir) || return 0;
789 rmdir $targetdir || return 0;
795 sub remove_directory_tree_inner
797 local ($dirhandle, $targetdir) = @_;
799 local ($subdirhandle);
801 opendir ($dirhandle, $targetdir) || return 0;
802 $subdirhandle = $dirhandle;
804 while ($object = readdir ($dirhandle))
806 if ($object =~ /^(\.\.?|CVS|RCS)$/)
811 $object = "$targetdir$pathsep$object";
814 if (-d _
&& &remove_directory_tree_inner
($subdirhandle, $object))
816 rmdir $object || return 0;
820 unlink $object || return 0;
823 closedir ($dirhandle);
827 # We used to use this behavior for this function:
831 # local (@filenames) = @_;
832 # local ($now) = time;
835 # foreach $file (@filenames)
837 # utime ($now, $now, $file)
838 # || (open (TOUCHFD, ">> $file") && close (TOUCHFD))
839 # || &error ("Couldn't touch $file: $!\n", 1);
844 # But this behaves badly on networked filesystems where the time is
845 # skewed, because it sets the time of the file based on the _local_
846 # host. Normally when you modify a file, it's the _remote_ host that
847 # determines the modtime, based on _its_ clock. So, instead, now we open
848 # the file and write something into it to force the remote host to set
849 # the modtime correctly according to its clock.
857 (open(T
, ">> $file") && print(T
"\n") && close(T
))
858 || &error
("Couldn't touch $file: $!\n", 1);
862 # Touch with a time offset. To DTRT, call touch() then use stat() to get the
863 # access/mod time for each file and apply the offset.
867 local ($off) = shift;
872 local (@s) = stat($_[0]);
874 utime($s[8]+$off, $s[9]+$off, @_);
877 # open a file, write some stuff to it, and close it.
881 local ($filename, @lines) = @_;
883 open (CF
, "> $filename") || &error
("Couldn't open $filename: $!\n", 1);
884 foreach $line (@lines)
891 # create a directory tree described by an associative array, wherein each
892 # key is a relative pathname (using slashes) and its associated value is
894 # DIR indicates a directory
895 # FILE:contents indicates a file, which should contain contents +\n
896 # LINK:target indicates a symlink, pointing to $basedir/target
897 # The first argument is the dir under which the structure will be created
898 # (the dir will be made and/or cleaned if necessary); the second argument
899 # is the associative array.
903 local ($basedir, %dirtree) = @_;
906 &remove_directory_tree
("$basedir");
907 mkdir ($basedir, 0777) || &error
("Couldn't mkdir $basedir: $!\n", 1);
909 foreach $path (sort keys (%dirtree))
911 if ($dirtree {$path} =~ /^DIR$/)
913 mkdir ("$basedir/$path", 0777)
914 || &error
("Couldn't mkdir $basedir/$path: $!\n", 1);
916 elsif ($dirtree {$path} =~ /^FILE:(.*)$/)
918 &create_file
("$basedir/$path", $1 . "\n");
920 elsif ($dirtree {$path} =~ /^LINK:(.*)$/)
922 symlink ("$basedir/$1", "$basedir/$path")
923 || &error
("Couldn't symlink $basedir/$path -> $basedir/$1: $!\n", 1);
927 &error
("Bogus dirtree type: \"$dirtree{$path}\"\n", 1);
930 if ($just_setup_tree)
932 die "Tree is setup...\n";
936 # compare a directory tree with an associative array in the format used
937 # by create_dir_tree, above.
938 # The first argument is the dir under which the structure should be found;
939 # the second argument is the associative array.
943 local ($basedir, %dirtree) = @_;
954 opendir (DIR
, $basedir) || &error
("Couldn't open $basedir: $!\n", 1);
955 @allfiles = grep (!/^(\.\.?|CVS|RCS)$/, readdir (DIR
) );
959 print "dirtree: (%dirtree)\n$basedir: (@allfiles)\n";
962 foreach $path (sort keys (%dirtree))
966 print "Checking $path ($dirtree{$path}).\n";
970 foreach $i (0 .. $#allfiles)
972 if ($allfiles[$i] eq $path)
974 splice (@allfiles, $i, 1); # delete it
977 print " Zapped $path; files now (@allfiles).\n";
979 lstat ("$basedir/$path");
987 print "compare_dir_tree: $path does not exist.\n";
992 if ($dirtree {$path} =~ /^DIR$/)
994 if (-d _
&& opendir (DIR
, "$basedir/$path") )
996 @files = readdir (DIR
);
998 @files = grep (!/^(\.\.?|CVS|RCS)$/ && ($_ = "$path/$_"), @files);
999 push (@allfiles, @files);
1002 print " Read in $path; new files (@files).\n";
1007 print "compare_dir_tree: $path is not a dir.\n";
1011 elsif ($dirtree {$path} =~ /^FILE:(.*)$/)
1015 print "compare_dir_tree: $path is not a file.\n";
1022 $contents = &read_file_into_string
("$basedir/$path");
1023 if ($contents ne "$1\n")
1025 print "compare_dir_tree: $path contains wrong stuff."
1026 . " Is:\n$contentsShould be:\n$1\n";
1031 elsif ($dirtree {$path} =~ /^LINK:(.*)$/)
1036 print "compare_dir_tree: $path is not a link.\n";
1041 $contents = readlink ("$basedir/$path");
1042 $contents =~ tr/>/\//;
1043 $fulltarget = "$basedir/$target";
1044 $fulltarget =~ tr/>/\//;
1045 if (!($contents =~ /$fulltarget$/))
1049 $target = $fulltarget;
1051 print "compare_dir_tree: $path should be link to $target, "
1052 . "not $contents.\n";
1058 &error
("Bogus dirtree type: \"$dirtree{$path}\"\n", 1);
1064 print "leftovers: (@allfiles).\n";
1067 foreach $file (@allfiles)
1069 print "compare_dir_tree: $file should not exist.\n";
1076 # this subroutine generates the numeric suffix used to keep tmp filenames,
1077 # log filenames, etc., unique. If the number passed in is 1, then a null
1078 # string is returned; otherwise, we return ".n", where n + 1 is the number
1086 return "$extext$num";
1092 # This subroutine returns a log filename with a number appended to
1093 # the end corresponding to how many logfiles have been created in the
1094 # current running test. An optional parameter may be passed (0 or 1).
1095 # If a 1 is passed, then it does NOT increment the logfile counter
1096 # and returns the name of the latest logfile. If either no parameter
1097 # is passed at all or a 0 is passed, then the logfile counter is
1098 # incremented and the new name is returned.
1102 local($no_increment) = @_;
1104 $num_of_logfiles += !$no_increment;
1106 return ($log_filename . &num_suffix
($num_of_logfiles));
1109 # This subroutine returns a base (answer) filename with a number
1110 # appended to the end corresponding to how many logfiles (and thus
1111 # base files) have been created in the current running test.
1112 # NO PARAMETERS ARE PASSED TO THIS SUBROUTINE.
1116 return ($base_filename . &num_suffix
($num_of_logfiles));
1119 # This subroutine returns a difference filename with a number appended
1120 # to the end corresponding to how many logfiles (and thus diff files)
1121 # have been created in the current running test.
1125 return ($diff_filename . &num_suffix
($num_of_logfiles));
1128 # just like logfile, only a generic tmp filename for use by the test.
1129 # they are automatically cleaned up unless -keep was used, or the test fails.
1130 # Pass an argument of 1 to return the same filename as the previous call.
1134 local($no_increment) = @_;
1136 $num_of_tmpfiles += !$no_increment;
1138 return ($tmp_filename . &num_suffix
($num_of_tmpfiles));