5 # Copyright (C) 2004, 2005, 2006 Free Software Foundation, Inc.
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
21 # [STDOUT := ignore], STDERR, [POST-TEST-CODE])
22 # ---------------------------------------------------------------
23 # Create a new test named TITLE that runs a minimal Autotest test suite,
24 # SUITE-CODE. Call AT_XFAIL_IF with XFAIL-CONDITION. STATUS and STDERR pass
25 # directly to the AT_CHECK that call the minimal test suite. STDOUT is not
26 # used, but it is reserved for future use. Run POST-TEST-CODE
27 # at the top level after the micro-suite has been run.
28 m4_define([AT_CHECK_AT],
31 AT_KEYWORDS([autotest])
32 AT_CAPTURE_FILE([micro-suite.log])
35 AT_DATA([package.m4],[[
36 m4_define([AT_PACKAGE_NAME], [GNU Nonsense])
37 m4_define([AT_PACKAGE_TARNAME], [nonsense])
38 m4_define([AT_PACKAGE_VERSION], [1.0])
39 m4_define([AT_PACKAGE_STRING], [GNU Nonsense 1.0])
40 m4_define([AT_PACKAGE_BUGREPORT], [bug-autoconf@gnu.org])
43 AT_DATA([mysuite.at], [$2])
45 # Do not use `testsuite' as the name of the small test suite, or the
46 # log file it generates will overwrite the log that the Autoconf test
47 # suite produces for this test case.
48 AT_CHECK_AUTOM4TE([--language=autotest -o micro-suite mysuite.at])
49 AT_CHECK([$CONFIG_SHELL ./micro-suite], m4_default([$4], 0), [ignore], [$6])
50 AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x], m4_default([$4], 0), [ignore], [$6])
55 # AT_CHECK_AT_TEST(TITLE, SUITE-SNIPPET, ...)
56 # -----------------------------------------------------------------------
57 # Wrapper for AT_CHECK_AT that surrounds SUITE-SNIPPET with a boilerplate
58 # AT_INIT, AT_SETUP, and AT_CLEANUP and passes other arguments verbatim.
59 m4_define([AT_CHECK_AT_TEST],
62 AT_INIT([artificial test suite])
63 AT_SETUP([my only test])
66 ]], m4_shiftn(2, $@))])
68 # Here documents for these tests contain forbidden macros.
69 m4_pattern_allow([^AT_])
73 m4_define([AT_NO_CMDSUBST],
74 [if (eval 'foo=$(echo bar) && test "$foo" = bar') >/dev/null 2>&1; then false; else :; fi])
77 ## ------------------ ##
78 ## Empty test suite. ##
79 ## ------------------ ##
81 # This is not a sensible thing to do, but the user should not get an unhelpful
83 AT_CHECK_AT([Empty test suite],
84 [[AT_INIT([empty test suite])
87 # Next level of emptiness.
88 AT_CHECK_AT_TEST([Empty test], [])
90 # And finally, an empty check should not cause a syntax error.
91 AT_CHECK_AT_TEST([Empty check], [AT_CHECK])
93 ## ----------------------------------------------------- ##
94 ## Newlines and command substitutions in test commands. ##
95 ## ----------------------------------------------------- ##
97 AT_CHECK_AT_TEST([Truth],
98 [AT_CHECK([:], 0, [], [])])
100 AT_CHECK_AT_TEST([Fallacy],
101 [AT_CHECK([false], ignore, [], [])])
103 AT_CHECK_AT_TEST([Literal multiline command],
104 [AT_CHECK([echo Auto'
109 AT_CHECK_AT_TEST([Multiline parameter expansion],
112 AT_CHECK([echo "$FOO"], 0, [one
116 AT_CHECK_AT_TEST([Backquote command substition],
117 [AT_CHECK([echo `echo hi`], 0, [hi
121 AT_CHECK_AT_TEST([Multiline backquote command substition],
122 [AT_DATA([myfile],[foo
125 AT_CHECK([echo "`cat myfile`"], 0, [foo
129 AT_CHECK_AT_TEST([Parenthetical command substition],
130 [AT_CHECK([echo $(echo hi)], 0, [hi
134 AT_CHECK_AT_TEST([Multiline parenthetical command substition],
135 [AT_DATA([myfile],[foo
138 AT_CHECK([echo "$(cat myfile)"], 0, [foo
144 ## ------------------------- ##
145 ## ${...} in test commands. ##
146 ## ------------------------- ##
148 # If this invalid parameter expansion capsizes the test suite, the entire
149 # AT_SETUP ... AT_CLEANUP subshell will exit, and the commands it runs will
150 # appear to have succeeded. Therefore, we verify a failing test case.
152 AT_CHECK_AT_TEST([Invalid brace-enclosed parameter expansion],
153 [AT_CHECK([echo '${=invalid}'], 0, [wrong])], [false], 1, ignore, ignore)
156 ## ---------------------------- ##
157 ## M4 macros in test commands. ##
158 ## ---------------------------- ##
160 # The last paragaph in the comment above _AT_DECIDE_TRACEABLE illustrates why
161 # this test fails (except with Korn shell-style quoting $'foo\nbar').
162 AT_CHECK_AT_TEST([Multiline command from M4 expansion],
163 [m4_define([GNU], ['foo
165 AT_CHECK([echo GNU], 0, [foo
167 ], [])], [case `( set -x; echo 'foo
168 bar') 2>&1` in *\$\'foo\\nbar\'*) false;; *) :;; esac])
170 AT_CHECK_AT_TEST([Double-M4-quoted command],
171 [m4_define([GNU], ['foo
173 AT_CHECK([[echo GNU]], 0, [[GNU
177 ## -------------------------------------- ##
178 ## Backslash-<newline> in test commands. ##
179 ## -------------------------------------- ##
181 AT_CHECK_AT_TEST([BS-newline in command],
182 [AT_CHECK([echo Auto"\
186 AT_CHECK_AT_TEST([^BS-newline in command],
191 AT_CHECK_AT_TEST([BSx641-newline in command],
192 [AT_CHECK([printf '%s\n' Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
193 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\conf
196 AT_CHECK_AT_TEST([BS-BS-newline in command],
197 [AT_CHECK([printf '%s\n' Auto"\\
202 # A `^BS-BS-newline in command' test will run a command named `\'. No, thanks.
204 AT_CHECK_AT_TEST([BSx640-newline in command],
205 [AT_CHECK([printf '%s\n' Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
206 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
210 # This command has both escaped and unescaped newlines.
211 AT_CHECK_AT_TEST([Newline-CODE-BS-newline in command],
212 [AT_CHECK([printf '%s\n' Auto'
218 AT_CHECK_AT_TEST([Single-quote-BS-newline in command],
219 [AT_CHECK([printf '%s\n' Auto'\
224 AT_CHECK_AT_TEST([Single-quote-newline-BS-newline in command],
225 [AT_CHECK([printf '%s\n' Auto'
233 ## ------------------------------- ##
234 ## Funny characters in test names. ##
235 ## ------------------------------- ##
237 # AT_CHECK_AT_TITLE(TITLE, TITLE-TO-TEST, EXPANDED-TITLE-TO-TEST
239 # ---------------------------------------------------------------
240 # Create a new test named TITLE that runs an Autotest test suite
241 # comprised of a trivial test named TITLE-TO-TEST, which expands
242 # to EXPANDED-TITLE-TO-TEST. XFAIL-CONDITION passes verbatim to
244 m4_define([AT_CHECK_AT_TITLE],
247 m4@&t@_define([macro_name], [[macro_expanded]])
248 m4@&t@_define([macro_expanded], [[macro_overexpanded]])
249 m4@&t@_define([macro_backquote], [`])
250 m4@&t@_define([macro_single_quote], ['])
251 m4@&t@_define([macro_double_quote], ["])
252 m4@&t@_define([macro_backslash], [\\])
253 AT_INIT([artificial test suite])
257 ]], [$4], [], [], [],
258 [AT_CHECK([[$CONFIG_SHELL ./micro-suite |
259 sed -n 's/[^:]*: \(.*[^ ]\)[ ]*ok.*/\1/p']],,
262 AT_CHECK([[$CONFIG_SHELL ./micro-suite -l |
263 sed -n 's/.*[0-9]: [^ ][^ ]*[ ][ ]*\(.*[^ ]\)[ ]*/\1/p']],,
266 AT_CHECK([[sed -n 's/[^.]*\. \(.*\) ([^)]*): ok.*/\1/p' micro-suite.log]],,
271 m4_define([AT_CHECK_AT_TITLE_CHAR],
272 [AT_CHECK_AT_TITLE([$1 in a test title], [A $2 in my name],
273 [A ]m4_ifval([$3], [[$3]], [[$2]])[ in my name], $4)])
275 AT_CHECK_AT_TITLE_CHAR([Backquote], [`])
276 AT_CHECK_AT_TITLE_CHAR([Single-quote], ['])
277 AT_CHECK_AT_TITLE_CHAR([Double-quote], ["])
278 AT_CHECK_AT_TITLE_CHAR([Backslash], [\\])
279 AT_CHECK_AT_TITLE_CHAR([Brackets], [[[]]], [[]])
280 AT_CHECK_AT_TITLE_CHAR([Pound], [[#]], [#])
281 AT_CHECK_AT_TITLE_CHAR([Comma], [,])
283 AT_CHECK_AT_TITLE_CHAR([Quoted Macro], [[macro_name]],
285 AT_CHECK_AT_TITLE_CHAR([Macro], [macro_name],
287 AT_CHECK_AT_TITLE_CHAR([Macro with backquote], [macro_backquote],
289 AT_CHECK_AT_TITLE_CHAR([Macro with single-quote], [macro_single_quote], ['])
290 AT_CHECK_AT_TITLE_CHAR([Macro with double-quote], [macro_double_quote],
292 # This test neither fails nor passes reliably.
293 # AT_CHECK_AT_TITLE_CHAR([Macro with backslash], [macro_backslash],
297 ## ----------------- ##
298 ## Debugging a test. ##
299 ## ----------------- ##
301 AT_CHECK_AT_TEST([Debugging a successful test],
302 [AT_CHECK([:])], [], [], [], [ignore],
303 [# Without options, when all tests pass, no test directory should exist.
304 AT_CHECK([test -d micro-suite.dir/1 && exit 42
305 ./micro-suite -d 1], [], [ignore], [ignore])
306 # Running with -d should leave a reproducible test group.
307 # Also, running the test script from the test group locks the
308 # directory from removal on some platforms; the script should still be
309 # able to run even if rmdir fails.
310 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
311 # Running a debugging script implies -d.
312 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
315 AT_CHECK_AT_TEST([Debugging script and environment],
316 [AT_CHECK([test "$MY_VAR" = pass || exit 42])],
317 [], [1], [], [ignore], [
318 # Changing environment outside of debugging script is not preserved.
319 AT_CHECK([(cd micro-suite.dir/1 && MY_VAR=pass ./run)],
320 [0], [ignore], [ignore])
321 AT_CHECK([(cd micro-suite.dir/1 && ./run)],
322 [1], [ignore], [ignore])
323 # Changing environment as argument to debugging script is preserved.
324 AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR=pass)],
325 [0], [ignore], [ignore])
326 AT_CHECK([(cd micro-suite.dir/1; ./run)],
327 [0], [ignore], [ignore])
330 # The run script must still be valid when shell metacharacters are passed
331 # in via an environment option.
332 AT_CHECK_AT_TEST([Debugging a failed test],
333 [AT_CHECK([test "$MY_VAR" = "one space" || exit 42])],
334 [], [1], [], [ignore], [
335 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='two spaces')],
336 [1], [ignore], [ignore])
337 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='one space')],
338 [0], [ignore], [ignore])
345 AT_SETUP([Keywords and ranges])
346 AT_KEYWORDS([autotest])
349 [[m4_define([AT_PACKAGE_STRING],[k])
350 m4_define([AT_PACKAGE_BUGREPORT],[devnull])
369 AT_CHECK_AUTOM4TE([--language=autotest -o k k.at])
371 # AT_CHECK_EGREP(PATTERN, STATUS, COUNT)
372 m4_define([AT_CHECK_EGREP],
373 [AT_CHECK([$EGREP -c '$1' stdout], $2, [$3
377 # AT_CHECK_KEYS(TESTSUITE-OPTIONS, PATTERN1, COUNT1, PATTERN2, COUNT2)
378 m4_define([AT_CHECK_KEYS],
379 [AT_CHECK([./k $1], 0, [stdout])
380 AT_CHECK_EGREP([$2], 0, [$3])
381 AT_CHECK_EGREP([$4], 1, [$5])
384 AT_CHECK_KEYS([-k key1], [first|both], [2], [none|second], [0])
385 AT_CHECK_KEYS([-k key2], [second|both], [2], [none|first], [0])
386 AT_CHECK_KEYS([-k key1,key2], [both], [1], [none|first|second], [0])
387 AT_CHECK_KEYS([-k key1 -k key2], [first|second|both], [3], [none], [0])
388 AT_CHECK_KEYS([-k '!key1'], [none|second], [2], [first|both], [0])
389 AT_CHECK_KEYS([-k '!key2'], [none|first], [2], [second|both], [0])
390 AT_CHECK_KEYS([-k '!key1,key2'], [second], [1], [none|first|both], [0])
391 AT_CHECK_KEYS([-k 'key1,!key2'], [first], [1], [none|second|both], [0])
392 AT_CHECK_KEYS([-k '!key1,!key2'], [none], [1], [first|second|both], [0])
393 AT_CHECK_KEYS([-k '!key1' -k KEY2], [none|second|both], [3], [first], [0])
394 AT_CHECK_KEYS([-k key1 -k '!key2'], [none|first|both], [3], [second], [0])
395 AT_CHECK_KEYS([-k '!KEY1' -k '!key2'], [none|first|second], [3], [both], [0])
397 AT_CHECK_KEYS([-k none], [none], [1], [first|second|both], [0])
398 AT_CHECK_KEYS([-k key1,both], [both], [1], [none|first|second], [0])
399 AT_CHECK_KEYS([-k key1 -k both], [first|both], [2], [none|second], [0])
400 AT_CHECK_KEYS([-k none,first], [successful], [1], [none|first|second|both], [0])
401 AT_CHECK_KEYS([-k none,first,second,both], [successful], [1], [none|first|second|both], [0])
402 AT_CHECK_KEYS([-k !none,first], [first], [1], [none|second|both], [0])
404 AT_CHECK_KEYS([-k '.*eco.*'], [second], [1], [none|first|both], [0])
405 AT_CHECK_KEYS([-k 'ECO'], [successful], [1], [none|first|second|both], [0])
406 AT_CHECK_KEYS([-k '.*eco'], [successful], [1], [none|first|second|both], [0])
407 AT_CHECK_KEYS([-k 'eco.*'], [successful], [1], [none|first|second|both], [0])
408 AT_CHECK_KEYS([-k 'fir.*'], [first], [1], [none|second|both], [0])
410 AT_CHECK_KEYS([1-2], [none|first], [2], [second|both], [0])
411 AT_CHECK_KEYS([1-3 2-1], [none|first|second], [3], [both], [0])
412 AT_CHECK_KEYS([-3], [none|first|second], [3], [both], [0])
413 AT_CHECK_KEYS([4-], [both], [1], [none|first|second], [0])
414 AT_CHECK_KEYS([-k second 4-], [second|both], [2], [none|first], [0])
418 ## ------------------- ##
419 ## srcdir propagation. ##
420 ## ------------------- ##
422 AT_SETUP([srcdir propagation])
423 AT_KEYWORDS([autotest])
424 AT_CAPTURE_FILE([suite.log])
426 mkdir pkg vpath-outside vpath-abs
427 mkdir pkg/t pkg/vpath-inside
431 AT_DATA([pkg/configure.ac], [[AC_INIT
432 AC_CONFIG_AUX_DIR($top_srcdir/build-aux)
433 AC_CONFIG_TESTDIR([t])
440 AT_DATA([pkg/t/suite.at],
441 [[m4_define([AT_PACKAGE_NAME], [GNU Nonsense])
442 m4_define([AT_PACKAGE_TARNAME], [nonsense])
443 m4_define([AT_PACKAGE_VERSION], [1.0])
444 m4_define([AT_PACKAGE_STRING], [GNU Nonsense 1.0])
445 m4_define([AT_PACKAGE_BUGREPORT], [bug-autoconf@gnu.org])
446 AT_INIT([suite to check srcdir])
447 AT_SETUP([my only test])
448 AT_CHECK([test -f "$top_srcdir"/a && test -f "$srcdir"/b])
452 AT_CHECK_AUTOM4TE([--language=autotest -o suite suite.at])
455 # Build directory totally outside source directory.
457 AT_CHECK([top_srcdir=$abs_top_srcdir ../pkg/configure], [0], [ignore])
459 AT_CHECK([../../pkg/t/suite], [0], [ignore])
462 # Build directory totally outside source directory (absolute).
465 AT_CHECK([top_srcdir=$abs_top_srcdir "$my_srcdir"/configure], [0], [ignore])
467 AT_CHECK(["$my_srcdir"/t/suite], [0], [ignore])
470 # Build directory as subdirectory of source directory.
472 AT_CHECK([top_srcdir=$abs_top_srcdir ../configure], [0], [ignore])
474 AT_CHECK([../../t/suite], [0], [ignore])
477 # Build directory as parent of source directory.
478 AT_CHECK([top_srcdir=$abs_top_srcdir pkg/configure], [0], [ignore])
480 AT_CHECK([../pkg/t/suite], [0], [ignore])
483 # Build directory as source directory.
487 AT_CHECK([./suite], [0], [ignore])