* doc/autoconf.texi (Limitations of Usual Tools): Warn about other
[autoconf.git] / tests / autotest.at
blobb632f064049aecb3991e126ba6d0e43376789455
1 #                                                       -*- Autotest -*-
3 AT_BANNER([Autotest.])
5 # Copyright (C) 2004, 2005, 2006 Free Software Foundation, Inc.
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
10 # any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 # 02110-1301, USA.
22 # AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
23 #             [STDOUT := ignore], STDERR, [POST-TEST-CODE])
24 # ---------------------------------------------------------------
25 # Create a new test named TITLE that runs a minimal Autotest test suite,
26 # SUITE-CODE.  Call AT_XFAIL_IF with XFAIL-CONDITION.  STATUS and STDERR pass
27 # directly to the AT_CHECK that call the minimal test suite.  STDOUT is not
28 # used, but it is reserved for future use.  Run POST-TEST-CODE
29 # at the top level after the micro-suite has been run.
30 m4_define([AT_CHECK_AT],
32 AT_SETUP([$1])
33 AT_KEYWORDS([autotest])
34 AT_CAPTURE_FILE([micro-suite.log])
35 AT_XFAIL_IF([$3])
37 AT_DATA([package.m4],[[
38 m4_define([AT_PACKAGE_NAME],      [GNU Nonsense])
39 m4_define([AT_PACKAGE_TARNAME],   [nonsense])
40 m4_define([AT_PACKAGE_VERSION],   [1.0])
41 m4_define([AT_PACKAGE_STRING],    [GNU Nonsense 1.0])
42 m4_define([AT_PACKAGE_BUGREPORT], [bug-autoconf@gnu.org])
43 ]])
45 AT_DATA([mysuite.at], [$2])
47 # Do not use `testsuite' as the name of the small test suite, or the
48 # log file it generates will overwrite the log that the Autoconf test
49 # suite produces for this test case.
50 AT_CHECK_AUTOM4TE([--language=autotest -o micro-suite mysuite.at])
51 AT_CHECK([$CONFIG_SHELL ./micro-suite],       m4_default([$4], 0), [ignore], [$6])
52 AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x], m4_default([$4], 0), [ignore], [$6])
54 AT_CLEANUP
55 ])# AT_CHECK_AT
57 # AT_CHECK_AT_TEST(TITLE, SUITE-SNIPPET, ...)
58 # -----------------------------------------------------------------------
59 # Wrapper for AT_CHECK_AT that surrounds SUITE-SNIPPET with a boilerplate
60 # AT_INIT, AT_SETUP, and AT_CLEANUP and passes other arguments verbatim.
61 m4_define([AT_CHECK_AT_TEST],
62 [AT_CHECK_AT([$1],
64 AT_INIT([artificial test suite])
65 AT_SETUP([my only test])
67 AT_CLEANUP
68 ]], m4_shiftn(2, $@))])
70 # Here documents for these tests contain forbidden macros.
71 m4_pattern_allow([^AT_])
73 # AT_NO_CMDSUBST
74 # --------------
75 m4_define([AT_NO_CMDSUBST],
76 [if (eval 'foo=$(echo bar) && test "$foo" = bar') >/dev/null 2>&1; then false; else :; fi])
79 ## ------------------ ##
80 ## Empty test suite.  ##
81 ## ------------------ ##
83 # This is not a sensible thing to do, but the user should not get an unhelpful
84 # error message.
85 AT_CHECK_AT([Empty test suite],
86 [[AT_INIT([empty test suite])
87 ]])
89 # Next level of emptiness.
90 AT_CHECK_AT_TEST([Empty test], [])
92 # And finally, an empty check should not cause a syntax error.
93 AT_CHECK_AT_TEST([Empty check], [AT_CHECK])
95 ## ----------------------------------------------------- ##
96 ## Newlines and command substitutions in test commands.  ##
97 ## ----------------------------------------------------- ##
99 AT_CHECK_AT_TEST([Truth],
100   [AT_CHECK([:], 0, [], [])])
102 AT_CHECK_AT_TEST([Fallacy],
103   [AT_CHECK([false], ignore, [], [])])
105 AT_CHECK_AT_TEST([Literal multiline command],
106   [AT_CHECK([echo Auto'
107 'conf], 0, [Auto
108 conf
109 ], [])])
111 AT_CHECK_AT_TEST([Multiline parameter expansion],
112   [FOO='one
113 two'
114    AT_CHECK([echo "$FOO"], 0, [one
116 ], [])])
118 AT_CHECK_AT_TEST([Backquote command substition],
119   [AT_CHECK([echo `echo hi`], 0, [hi
120 ], [])])
123 AT_CHECK_AT_TEST([Multiline backquote command substition],
124   [AT_DATA([myfile],[foo
127    AT_CHECK([echo "`cat myfile`"], 0, [foo
129 ], [])])
131 AT_CHECK_AT_TEST([Parenthetical command substition],
132   [AT_CHECK([echo $(echo hi)], 0, [hi
133 ], [])],
134   [AT_NO_CMDSUBST])
136 AT_CHECK_AT_TEST([Multiline parenthetical command substition],
137   [AT_DATA([myfile],[foo
140    AT_CHECK([echo "$(cat myfile)"], 0, [foo
142 ], [])],
143   [AT_NO_CMDSUBST])
146 ## ------------------------- ##
147 ## ${...} in test commands.  ##
148 ## ------------------------- ##
150 # If this invalid parameter expansion capsizes the test suite, the entire
151 # AT_SETUP ... AT_CLEANUP subshell will exit, and the commands it runs will
152 # appear to have succeeded.  Therefore, we verify a failing test case.
154 AT_CHECK_AT_TEST([Invalid brace-enclosed parameter expansion],
155   [AT_CHECK([echo '${=invalid}'], 0, [wrong])], [false], 1, ignore, ignore)
158 ## ---------------------------- ##
159 ## M4 macros in test commands.  ##
160 ## ---------------------------- ##
162 # The last paragaph in the comment above _AT_DECIDE_TRACEABLE illustrates why
163 # this test fails (except with Korn shell-style quoting $'foo\nbar').
164 AT_CHECK_AT_TEST([Multiline command from M4 expansion],
165   [m4_define([GNU], ['foo
166 bar'])
167    AT_CHECK([echo GNU], 0, [foo
169 ], [])], [case `( set -x; echo 'foo
170 bar') 2>&1` in *\$\'foo\\nbar\'*) false;; *) :;; esac])
172 AT_CHECK_AT_TEST([Double-M4-quoted command],
173   [m4_define([GNU], ['foo
174 bar'])
175    AT_CHECK([[echo GNU]], 0, [[GNU
176 ]], [])])
179 ## -------------------------------------- ##
180 ## Backslash-<newline> in test commands.  ##
181 ## -------------------------------------- ##
183 AT_CHECK_AT_TEST([BS-newline in command],
184   [AT_CHECK([echo Auto"\
185 "conf], 0, [Autoconf
186 ], [])])
188 AT_CHECK_AT_TEST([^BS-newline in command],
189   [AT_CHECK([\
190 echo GNU], 0, [GNU
191 ], [])])
193 AT_CHECK_AT_TEST([BSx641-newline in command],
194   [AT_CHECK([printf '%s\n' Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
195 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\conf
196 ], [])])
198 AT_CHECK_AT_TEST([BS-BS-newline in command],
199   [AT_CHECK([printf '%s\n' Auto"\\
200 "conf], 0, [Auto\
201 conf
202 ], [])])
204 # A `^BS-BS-newline in command' test will run a command named `\'.  No, thanks.
206 AT_CHECK_AT_TEST([BSx640-newline in command],
207   [AT_CHECK([printf '%s\n' Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
208 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
209 conf
210 ], [])])
212 # This command has both escaped and unescaped newlines.
213 AT_CHECK_AT_TEST([Newline-CODE-BS-newline in command],
214   [AT_CHECK([printf '%s\n' Auto'
215 'co\
216 nf], 0, [Auto
217 conf
218 ], [])])
220 AT_CHECK_AT_TEST([Single-quote-BS-newline in command],
221   [AT_CHECK([printf '%s\n' Auto'\
222 'conf], 0, [Auto\
223 conf
224 ], [])])
226 AT_CHECK_AT_TEST([Single-quote-newline-BS-newline in command],
227   [AT_CHECK([printf '%s\n' Auto'
229 'conf], 0, [Auto
231 conf
232 ], [])])
235 ## ------------------------------- ##
236 ## Funny characters in test names. ##
237 ## ------------------------------- ##
239 # AT_CHECK_AT_TITLE(TITLE, TITLE-TO-TEST, EXPANDED-TITLE-TO-TEST
240 #                   [XFAIL-CONDITION])
241 # ---------------------------------------------------------------
242 # Create a new test named TITLE that runs an Autotest test suite
243 # comprised of a trivial test named TITLE-TO-TEST, which expands
244 # to EXPANDED-TITLE-TO-TEST.  XFAIL-CONDITION passes verbatim to
245 # AT_CHECK_AT.
246 m4_define([AT_CHECK_AT_TITLE],
247 [AT_CHECK_AT([$1],
249 m4@&t@_define([macro_name],         [[macro_expanded]])
250 m4@&t@_define([macro_expanded], [[macro_overexpanded]])
251 m4@&t@_define([macro_backquote],    [`])
252 m4@&t@_define([macro_single_quote], ['])
253 m4@&t@_define([macro_double_quote], ["])
254 m4@&t@_define([macro_backslash],   [\\])
255 AT_INIT([artificial test suite])
256 AT_SETUP([$2])
257 AT_CHECK([:])
258 AT_CLEANUP
259 ]], [$4], [], [], [],
260 [AT_CHECK([[$CONFIG_SHELL ./micro-suite |
261             sed -n 's/[^:]*: \(.*[^      ]\)[    ]*ok.*/\1/p']],,
262 [[$3
264 AT_CHECK([[$CONFIG_SHELL ./micro-suite -l |
265            sed -n 's/.*[0-9]: [^         ][^     ]*[     ][      ]*\(.*[^        ]\)[    ]*/\1/p']],,
266 [[$3
268 AT_CHECK([[sed -n 's/[^.]*\. \(.*\) ([^)]*): ok.*/\1/p' micro-suite.log]],,
269 [[$3
271 ])])
273 m4_define([AT_CHECK_AT_TITLE_CHAR],
274 [AT_CHECK_AT_TITLE([$1 in a test title], [A $2 in my name],
275                    [A ]m4_ifval([$3], [[$3]], [[$2]])[ in my name], $4)])
277 AT_CHECK_AT_TITLE_CHAR([Backquote],    [`])
278 AT_CHECK_AT_TITLE_CHAR([Single-quote], ['])
279 AT_CHECK_AT_TITLE_CHAR([Double-quote], ["])
280 AT_CHECK_AT_TITLE_CHAR([Backslash],   [\\])
281 AT_CHECK_AT_TITLE_CHAR([Brackets],  [[[]]], [[]])
282 AT_CHECK_AT_TITLE_CHAR([Pound],      [[#]], [#])
283 AT_CHECK_AT_TITLE_CHAR([Comma],        [,])
285 AT_CHECK_AT_TITLE_CHAR([Quoted Macro],                  [[macro_name]],
286                                                           [macro_name])
287 AT_CHECK_AT_TITLE_CHAR([Macro],                           [macro_name],
288                                                       [macro_expanded])
289 AT_CHECK_AT_TITLE_CHAR([Macro with backquote],       [macro_backquote],
290                                                                    [`], [:])
291 AT_CHECK_AT_TITLE_CHAR([Macro with single-quote], [macro_single_quote], ['])
292 AT_CHECK_AT_TITLE_CHAR([Macro with double-quote], [macro_double_quote],
293                                                                    ["], [:])
294 # This test neither fails nor passes reliably.
295 # AT_CHECK_AT_TITLE_CHAR([Macro with backslash],       [macro_backslash],
296 #                                                                  [\\], [:])
299 ## ----------------- ##
300 ## Debugging a test. ##
301 ## ----------------- ##
303 AT_CHECK_AT_TEST([Debugging a successful test],
304   [AT_CHECK([:])], [], [], [], [ignore],
305 [# Without options, when all tests pass, no test directory should exist.
306 AT_CHECK([test -d micro-suite.dir/1 && exit 42
307           ./micro-suite -d 1], [], [ignore], [ignore])
308 # Running with -d should leave a reproducible test group.
309 # Also, running the test script from the test group locks the
310 # directory from removal on some platforms; the script should still be
311 # able to run even if rmdir fails.
312 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
313 # Running a debugging script implies -d.
314 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
317 AT_CHECK_AT_TEST([Debugging script and environment],
318   [AT_CHECK([test "$MY_VAR" = pass || exit 42])],
319   [], [1], [], [ignore], [
320 # Changing environment outside of debugging script is not preserved.
321 AT_CHECK([(cd micro-suite.dir/1 && MY_VAR=pass ./run)],
322          [0], [ignore], [ignore])
323 AT_CHECK([(cd micro-suite.dir/1 && ./run)],
324          [1], [ignore], [ignore])
325 # Changing environment as argument to debugging script is preserved.
326 AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR=pass)],
327          [0], [ignore], [ignore])
328 AT_CHECK([(cd micro-suite.dir/1; ./run)],
329          [0], [ignore], [ignore])
332 # The run script must still be valid when shell metacharacters are passed
333 # in via an environment option.
334 AT_CHECK_AT_TEST([Debugging a failed test],
335   [AT_CHECK([test "$MY_VAR" = "one space" || exit 42])],
336   [], [1], [], [ignore], [
337 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='two  spaces')],
338          [1], [ignore], [ignore])
339 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='one space')],
340          [0], [ignore], [ignore])
344 ## --------- ##
345 ## Keywords. ##
346 ## --------- ##
347 AT_SETUP([Keywords and ranges])
348 AT_KEYWORDS([autotest])
350 AT_DATA([k.at],
351 [[m4_define([AT_PACKAGE_STRING],[k])
352 m4_define([AT_PACKAGE_BUGREPORT],[devnull])
353 AT_INIT
354 AT_SETUP(none)
355 AT_CHECK(:)
356 AT_CLEANUP
357 AT_SETUP(first)
358 AT_KEYWORDS(key1)
359 AT_CHECK(:)
360 AT_CLEANUP
361 AT_SETUP(second)
362 AT_KEYWORDS(key2)
363 AT_CHECK(:)
364 AT_CLEANUP
365 AT_SETUP(both)
366 AT_KEYWORDS(key1)
367 AT_KEYWORDS(key2)
368 AT_CHECK(:)
369 AT_CLEANUP
371 AT_CHECK_AUTOM4TE([--language=autotest -o k k.at])
373 # AT_CHECK_EGREP(PATTERN, STATUS, COUNT)
374 m4_define([AT_CHECK_EGREP],
375 [AT_CHECK([$EGREP -c '$1' stdout], $2, [$3
376 ], ignore)
379 # AT_CHECK_KEYS(TESTSUITE-OPTIONS, PATTERN1, COUNT1, PATTERN2, COUNT2)
380 m4_define([AT_CHECK_KEYS],
381 [AT_CHECK([./k $1], 0, [stdout])
382 AT_CHECK_EGREP([$2], 0, [$3])
383 AT_CHECK_EGREP([$4], 1, [$5])
386 AT_CHECK_KEYS([-k key1], [first|both], [2], [none|second], [0])
387 AT_CHECK_KEYS([-k key2], [second|both], [2], [none|first], [0])
388 AT_CHECK_KEYS([-k key1,key2], [both], [1], [none|first|second], [0])
389 AT_CHECK_KEYS([-k key1 -k key2], [first|second|both], [3], [none], [0])
390 AT_CHECK_KEYS([-k '!key1'], [none|second], [2], [first|both], [0])
391 AT_CHECK_KEYS([-k '!key2'], [none|first], [2], [second|both], [0])
392 AT_CHECK_KEYS([-k '!key1,key2'], [second], [1], [none|first|both], [0])
393 AT_CHECK_KEYS([-k 'key1,!key2'], [first], [1], [none|second|both], [0])
394 AT_CHECK_KEYS([-k '!key1,!key2'], [none], [1], [first|second|both], [0])
395 AT_CHECK_KEYS([-k '!key1' -k KEY2], [none|second|both], [3], [first], [0])
396 AT_CHECK_KEYS([-k key1 -k '!key2'], [none|first|both], [3], [second], [0])
397 AT_CHECK_KEYS([-k '!KEY1' -k '!key2'], [none|first|second], [3], [both], [0])
399 AT_CHECK_KEYS([-k none], [none], [1], [first|second|both], [0])
400 AT_CHECK_KEYS([-k key1,both], [both], [1], [none|first|second], [0])
401 AT_CHECK_KEYS([-k key1 -k both], [first|both], [2], [none|second], [0])
402 AT_CHECK_KEYS([-k none,first], [successful], [1], [none|first|second|both], [0])
403 AT_CHECK_KEYS([-k none,first,second,both], [successful], [1], [none|first|second|both], [0])
404 AT_CHECK_KEYS([-k !none,first], [first], [1], [none|second|both], [0])
406 AT_CHECK_KEYS([-k '.*eco.*'], [second], [1], [none|first|both], [0])
407 AT_CHECK_KEYS([-k 'ECO'], [successful], [1], [none|first|second|both], [0])
408 AT_CHECK_KEYS([-k '.*eco'], [successful], [1], [none|first|second|both], [0])
409 AT_CHECK_KEYS([-k 'eco.*'], [successful], [1], [none|first|second|both], [0])
410 AT_CHECK_KEYS([-k 'fir.*'], [first], [1], [none|second|both], [0])
412 AT_CHECK_KEYS([1-2], [none|first], [2], [second|both], [0])
413 AT_CHECK_KEYS([1-3 2-1], [none|first|second], [3], [both], [0])
414 AT_CHECK_KEYS([-3], [none|first|second], [3], [both], [0])
415 AT_CHECK_KEYS([4-], [both], [1], [none|first|second], [0])
416 AT_CHECK_KEYS([-k second 4-], [second|both], [2], [none|first], [0])
417 AT_CLEANUP