4 # Script to do automated testing and data collection for various test
5 # files, so that we don't have to do this by hand on every test file.
6 # It attempts to collect some diagnostic info about size and speed that
7 # should be useful in the future as the library gets tuned for size
8 # and speed. In addition, it tests static and shared linkage, iff each
12 # mkcheck [01] (path to build) (path to src) (path to install)
14 ### XXX There are a lot of tests in here for OS-specific stuff. If we
15 ### move to a 'configure.target' method of determining those extra
16 ### flags and whatnot, we can take out all those things and source
17 ### that file from here. (Write that file with this in mind...)
19 ### XXX Note that breaking out of this with ^C will not work. Dunno why.
24 # WHICH determines if you are
25 # (0) testing the build binary and headers, or
26 # (1) testing the installed binary and headers, or
28 if [ "$WHICH"x
= 0x
] && [ $# -eq 3 ]; then
29 echo "running mkcheck"
30 echo "$0: testing the build directory"
31 query
="--built-library"
32 elif [ "$WHICH"x
= 1x
] && [ $# -eq 4 ]; then
33 echo "running mkcheck"
34 echo "$0: testing the install directory $4"
35 query
="--installed-library"
37 echo 'Usage: mkcheck 0 (path to build) (path to src)'
38 echo ' mkcheck 1 (path to build) (path to src) (path to install)'
42 # Now that we've successfully translated the numerical option into
43 # a symbolic one, we can safely ignore it.
46 # This has been true all along. Found out about it the hard way...
48 1*) echo 'You need bash 2.x to run mkcheck. Exiting.'; exit 1 ;;
52 # Compute the flags necessary to run the testsuite.
55 set `../tests_flags ${query} $*` ||
exit 1
56 BUILD_DIR
=$1; SRC_DIR
=$2; PREFIX_DIR
=$3; CXX
=$4; CXXFLAGS
=$5; INCLUDES
=$6; LIBS
=$7;
59 # Build libtoolized surrogates to compile and run testcases. Also,
60 # adjust CXX so that the newly built compiler can find headers.
61 if [ x
$WHICH = x0
] ; then
62 CXX
="$CXX -B`dirname $CXX`/"
64 LIBTOOL
="$BUILD_DIR/libtool"
65 LTEXE
="$LIBTOOL --mode=execute"
66 LTCXX
="$LIBTOOL --tag=CXX --mode=link $CXX $CXXFLAGS $INCLUDES $LIBS"
68 # specific libtool flag(s) to force the use of shared libraries, if any
71 # specific libtool flag(s) to force the use of static libraries, if any
73 #ST_FLAG="-all-static"
75 # Set up the testing directory, which should be in a directory called
76 # "testsuite" in the root level of the build directory.
78 # help libtool keep quiet
79 if [ ! -d ${TEST_DIR}/.libs
]; then
83 # the name of the file that will collect and hold all this useful data:
84 RESULTS_FILE
="$TEST_DIR/$(date +%Y%m%d)-mkcheck.txt"
86 # the name of the log file that will append compiler diagnostics:
87 LOG_FILE
="$TEST_DIR/$(date +%Y%m%d)-mkchecklog.txt"
89 # the names of the specific test files to be run
90 TESTS_FILE
="$TEST_DIR/$(date +%Y%m%d)-mkcheckfiles.txt"
92 # the heap size and virtual mem limit for testsuite binaries
93 # See http://gcc.gnu.org/ml/libstdc++/2000-10/msg00029.html
97 # 2: clean, make files, append general test info
100 # Remove old executables.
101 rm -rf "$TEST_DIR"/*exe
103 # Remove old core files (which now get left in cwd, not $TEST_DIR).
106 if [ -f $RESULTS_FILE ]; then
109 if [ -f $LOG_FILE ]; then
113 # Make a list of the files we're going to run, or use an old one if it exists.
114 if [ ! -f "$TESTS_FILE" ]; then
115 echo "making file $TESTS_FILE"
116 for LONG_NAME
in $SRC_DIR/testsuite
/*/*.cc
118 DIR_NAME
=$
(dirname $LONG_NAME)
119 SHORT_NAME
="`basename $DIR_NAME`/`basename $LONG_NAME`"
120 echo "$SHORT_NAME" >> $TESTS_FILE
124 # Nasty solution to replace GNU date(1)'s %s time_t output function.
125 TIMER_COMMAND
=$TEST_DIR/printnow.exe
126 if [ ! -x "$TIMER_COMMAND" ]; then
127 echo "making utility $TIMER_COMMAND"
128 gcc
-o "$TIMER_COMMAND" "$SRC_DIR/testsuite/printnow.c"
129 strip
"$TIMER_COMMAND"
132 # Copy over the data files for filebufs
133 cp $SRC_DIR/testsuite
/27_io
/*.txt
$TEST_DIR
134 cp $SRC_DIR/testsuite
/27_io
/*.tst
$TEST_DIR
135 chmod u
+w
$TEST_DIR/*.txt
136 chmod u
+w
$TEST_DIR/*.tst
138 # Emit useful info about compiler and platform
139 echo "host: $(uname -mrsv)" >> $RESULTS_FILE
140 echo "compiler: $($CXX -v 2>&1)" >> $RESULTS_FILE
141 echo "compiler flags: $CXXFLAGS" >> $RESULTS_FILE
142 echo "date: $(date +%Y%m%d)" >> $RESULTS_FILE
143 echo "" >> $RESULTS_FILE
145 explanation
='+: pass, -b: build failure, -r: run failure, x: disabled'
146 printf "%s\n %s\n" 'p == pass/fail execution test' "$explanation" \
148 echo "ctime == time to compile and link" >> $RESULTS_FILE
149 echo "etime == time for executable to run" >> $RESULTS_FILE
150 echo "text == size of the executable text section" >> $RESULTS_FILE
151 echo "data == size of the executable data section" >> $RESULTS_FILE
152 echo "total == size of the executable" >> $RESULTS_FILE
153 echo "" >> $RESULTS_FILE
155 echo "p" |
awk '{printf("%s ", $1)}' >> $RESULTS_FILE
156 echo "ctime" "etime" |
awk '{printf("%s\t%s\t", $1, $2)}' >> $RESULTS_FILE
157 echo "text" "data" |
awk '{printf("%s\t%s\t", $1, $2)}' >> $RESULTS_FILE
158 echo "total" "name" |
awk '{printf("%s\t%s\t", $1, $2)}' >> $RESULTS_FILE
159 echo "" >> $RESULTS_FILE
161 # Counters. These could be members of an array, but they'd all have to
162 # become individuals anyhow if we ever change this script to super-portable sh.
170 # 2.5: support functions
173 # Figure out how to extract size information from binaries. We take
174 # the text of the value we want as an argument, and leave the size in
175 # the appropriate variable.
177 # We discover what kind of size(1) we are using *once* and build a shell
178 # function named 'size_command' to wrap it. (The "function" keyword is
179 # redundant here, but helps me read it, so there.) Previously we were
180 # re-discovering the size(1) arguments three times for each test; sloooow.
182 # It is VERY IMPORTANT not to compare these numbers across platforms.
183 # Different size(1)'s extract section information differently. For
184 # example, using the native Sun size(1) and GNU size(1) built for Suns
185 # on the exact same binary will give very different numbers, due to all
186 # the variance in command-line options and arbitrary names of ELF sections.
188 # and suddenly we go to 2-space indentations...
191 if size
--version 2> /dev
/null |
grep -c GNU
> /dev
/null
;
192 then # Then we're using a GNU size(1) built for this platform.
193 # We lose .rodata and .data1 and who knows what else... kludge.
194 function size_command
()
197 TEXT
) TEXT
=$
(size
-A $EXENAME |
grep ^.text |
awk '{print $2}') ;;
198 DATA
) DATA
=$
(size
-A $EXENAME |
grep -w ^.data |
awk '{print $2}') ;;
199 SIZE
) SIZE
=$
(size
-A $EXENAME |
grep otal |
awk '{print $2}') ;;
203 # Not using GNU size; check for platform. These numbers seem to match
204 # up to text/data/total, although their meanings seem to be different.
205 # THIS TABLE IS SORTED. KEEP IT THAT WAY.
208 function size_command
()
211 TEXT
) TEXT
=$
(size
-X32_64 $EXENAME |
awk '{print $2}') ;;
212 DATA
) DATA
=$
(size
-X32_64 $EXENAME |
awk '{print $4}') ;;
213 SIZE
) SIZE
=$
(size
-X32_64 $EXENAME |
awk '{print $12}') ;;
218 function size_command
()
221 TEXT
) TEXT
=$
(size
-4 $EXENAME |
awk '{print $1}') ;;
222 DATA
) DATA
=$
(size
-4 $EXENAME |
awk '{print $3}') ;;
223 SIZE
) SIZE
=$
(size
-4 $EXENAME |
awk '{print $7}') ;;
228 function size_command
()
231 TEXT
) TEXT
=$
(size
$EXENAME |
awk '{print $1}') ;;
232 DATA
) DATA
=$
(size
$EXENAME |
awk '{print $3}') ;;
233 SIZE
) SIZE
=$
(size
$EXENAME |
awk '{print $7}') ;;
238 echo ' * Warning! Skipping section sizes!' 1>&2
239 function size_command
()
252 # Test for file output
255 # This checks for emitted output files, which is useful when
256 # testing file-related output. The rules for this working are as
257 # follows: the emitted file must have the ".txt" extension, and be
258 # based on the actual *.cc file's name. For example, 27/filbuf.cc
259 # currently outputs files named 27/filebuf-2.txt and 27/filebuf-3.txt.
260 # Also, the first emitted file must be in the form $NAME-1.txt.
261 # The control file must follow the same constraints, but have a
262 # ".tst" extension. Thus, you have 27/filebuf-2.tst, etc.
264 # NAME contains the source name, like 27/filebuf.cc
265 # From that NAME, we want to generate some possible names, using
266 # ls on MATCH, a pattern description generated with sed.
268 # this is the name of the resulting diff file, if any
269 DIFF_FILE
="`echo $TEST_NAME | sed 's/cc$/diff/'`"
270 # construct wildcard names, ie for $NAME=filebuf.cc, makes "filebuf*.tst"
271 DATA_FILES
="`echo $TEST_NAME | sed 's/\.cc/\*\.tst/g'`"
272 # make sure there is at least one, then go
273 ST_E
="`echo $TEST_NAME | sed 's/\.cc/\-1\.tst/g'`"
274 if [ -f $ST_E ]; then
275 # list of actual files that match the wildcard above, ie
277 ST_MATCH_LIST
="`ls $DATA_FILES`"
278 for i
in $ST_MATCH_LIST; do
279 # ST_OUT_FILE is generated in the build directory.
280 PRE_NAME2
="$TEST_DIR/`basename $i`"
281 ST_OUT_FILE
="`echo $PRE_NAME2 | sed 's/tst$/txt/'`"
282 diff $ST_OUT_FILE $i > $DIFF_FILE
283 if [ -s $DIFF_FILE ]; then
291 # the file does no output, and didn't abnormally
292 # terminate, so assume passed.
299 # 3: compile, link, execute, time
301 # Abstract out the common code for compiling, linking, executing and printing.
304 # NB: S_FLAG has to be last argument because it may be null, and
305 # error checking hasn't been invented yet.
310 SRC_NAME
="$SRC_DIR/testsuite/$1"
311 TEST_NAME
="$TEST_DIR/`basename $NAME`"
313 # This would be deliciously easy if GNU date's %s were always around.
314 # There are three ways to do this: 1) use the builtin 'time' like we
315 # do later; then getting compiler errors into LOG_FILE is a nightmare.
316 # 2) Grab the output of a formatted date(1) and do the math; harder
317 # and harder as we try compiling at, say, top of the hour; we would
318 # eventually have to calculate time_t anyhow. Or 3) just grab two
319 # time_t's (no more overhead than grabbing two date(1)'s).
320 compiler_invocation
="$LTCXX $S_FLAG $SRC_NAME -o $EXENAME"
321 COMP_TIME_START
=$
($TIMER_COMMAND)
322 $compiler_invocation >> compile.out
2>&1
323 COMP_TIME_END
=$
($TIMER_COMMAND)
325 if [ $COMP_TIME_START -lt $COMP_TIME_END ]; then
326 C_TIME
=$
[ $COMP_TIME_END - $COMP_TIME_START ]
331 if [ -f $EXENAME ]; then
337 # Actually run the executable and time it. Note that output
338 # printed by the executable will be lost and cannot be redirected,
339 # because we need to capture the output of 'time'. Bummer.
340 TIMEFORMAT
='timemark %R'
341 E_TIME_TEXT
="$(exec 2>&1; \
342 ulimit -d $MAX_MEM_USAGE; ulimit -v $MAX_MEM_USAGE; \
343 time $LTEXE $EXENAME)"
344 E_ABNORMAL_TERMINATION
=$?
345 E_TIME
="$(echo $E_TIME_TEXT | awk '{print $2}')"
346 # joining those two commands does not work due to quoting problems:
347 #E_TIME="$(exec 2>&1; time $EXENAME | awk '{print $2}')"
348 # this will work as a fallback on certain systems...?
349 #E_TIME=$(exec 2>&1; time $EXENAME | cut -d ' ' -f 2)
351 if [ "$E_ABNORMAL_TERMINATION" -ne 0 ]; then
354 # sometimes you want to save all core files for review:
355 #mv ./core $EXENAME.core
356 # sometimes the OS allows you to name core files yourself:
357 #mv ./*core $EXENAME.core
358 #mv ./core* $EXENAME.core
363 # sometimes you want to save all failing exe files for review:
364 if [ "$RESULT" = "+" ]; then
368 # the file did not compile/link.
369 printf "\n" >> $LOG_FILE
370 `cat compile.out >> $LOG_FILE`
378 # update the counters
379 if test "$RESULT" = "+" ; then
380 if test x
"$S_FLAG" = x
"$ST_FLAG"; then
381 static_pass
=`expr $static_pass + 1`
383 shared_pass
=`expr $shared_pass + 1`
386 if test x
"$S_FLAG" = x
"$ST_FLAG"; then
387 static_fail
=`expr $static_fail + 1`
389 shared_fail
=`expr $shared_fail + 1`
393 printf "%s\t" "$RESULT"
394 printf "%-2s %d\t%.3f\t%s\t%s\t%s\t%s %s\n" \
395 "$RESULT" $C_TIME $E_TIME $TEXT $DATA $SIZE $NAME "$S_FLAG" \
401 echo "Detailed test results in .${RESULTS_FILE/$BUILD_DIR}"
403 echo "------------------------------------------------------------------------"
404 printf "static\tshared\ttest\n"
405 echo "------------------------------------------------------------------------"
407 TEST_TIME_START
=$
($TIMER_COMMAND)
408 for NAME
in `cat $TESTS_FILE`
410 PRE_NAME
="$TEST_DIR/`basename $NAME`"
411 ST_NAME
="`echo $PRE_NAME | sed 's/cc$/st-exe/'`"
412 SH_NAME
="`echo $PRE_NAME | sed 's/cc$/sh-exe/'`"
414 if test @enable_static@
= yes; then
415 test_file
$NAME $ST_NAME "$ST_FLAG"
418 printf "static skipped\n" >> $RESULTS_FILE
420 if test @enable_shared@
= yes; then
421 test_file
$NAME $SH_NAME "$SH_FLAG"
424 printf "shared skipped\n" >> $RESULTS_FILE
426 printf "%s\n" "$NAME"
428 echo "" >> $RESULTS_FILE
430 TEST_TIME_END
=$
($TIMER_COMMAND)
436 # grep can count faster than we can...
437 total_failures
=`expr ${shared_fail} + ${static_fail}`
438 total_successes
=`expr ${shared_pass} + ${static_pass}`
439 resultstext
="pass/fail results: ${shared_pass}/${shared_fail} shared + ${static_pass}/${static_fail} static = ${total_successes}/${total_failures} total"
440 if [ $total_failures -eq 0 ]; then
441 resultstext
="${resultstext}, WIN WIN"
444 $resultstext" $RESULTS_FILE > ${RESULTS_FILE}.tmp
445 mv ${RESULTS_FILE}.tmp
$RESULTS_FILE
447 if [ $TEST_TIME_START -lt $TEST_TIME_END ]; then
448 TEST_TIME
=$
[ $TEST_TIME_END - $TEST_TIME_START ]
449 echo "testrun == $TEST_TIME seconds"
450 echo "testrun == $TEST_TIME seconds" >> $RESULTS_FILE