tests/autotest.at

   1 #                                                       -*- Autotest -*-
   2
   3 AT_BANNER([Autotest.])
   4
   5 # Copyright (C) 2004, 2005, 2006 Free Software Foundation, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20 # 02110-1301, USA.
  21
  22 # AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
  23 #             [STDOUT := ignore], STDERR, [POST-TEST-CODE])
  24 # ---------------------------------------------------------------
  25 # Create a new test named TITLE that runs a minimal Autotest test suite,
  26 # SUITE-CODE.  Call AT_XFAIL_IF with XFAIL-CONDITION.  STATUS and STDERR pass
  27 # directly to the AT_CHECK that call the minimal test suite.  STDOUT is not
  28 # used, but it is reserved for future use.  Run POST-TEST-CODE
  29 # at the top level after the micro-suite has been run.
  30 m4_define([AT_CHECK_AT],
  31 [
  32 AT_SETUP([$1])
  33 AT_KEYWORDS([autotest])
  34 AT_CAPTURE_FILE([micro-suite.log])
  35 AT_XFAIL_IF([$3])
  36
  37 AT_DATA([package.m4],[[
  38 m4_define([AT_PACKAGE_NAME],      [GNU Nonsense])
  39 m4_define([AT_PACKAGE_TARNAME],   [nonsense])
  40 m4_define([AT_PACKAGE_VERSION],   [1.0])
  41 m4_define([AT_PACKAGE_STRING],    [GNU Nonsense 1.0])
  42 m4_define([AT_PACKAGE_BUGREPORT], [bug-autoconf@gnu.org])
  43 ]])
  44
  45 AT_DATA([mysuite.at], [$2])
  46
  47 # Do not use `testsuite' as the name of the small test suite, or the
  48 # log file it generates will overwrite the log that the Autoconf test
  49 # suite produces for this test case.
  50 AT_CHECK_AUTOM4TE([--language=autotest -o micro-suite mysuite.at])
  51 AT_CHECK([$CONFIG_SHELL ./micro-suite],       m4_default([$4], 0), [ignore], [$6])
  52 AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x], m4_default([$4], 0), [ignore], [$6])
  53 $7
  54 AT_CLEANUP
  55 ])# AT_CHECK_AT
  56
  57 # AT_CHECK_AT_TEST(TITLE, SUITE-SNIPPET, ...)
  58 # -----------------------------------------------------------------------
  59 # Wrapper for AT_CHECK_AT that surrounds SUITE-SNIPPET with a boilerplate
  60 # AT_INIT, AT_SETUP, and AT_CLEANUP and passes other arguments verbatim.
  61 m4_define([AT_CHECK_AT_TEST],
  62 [AT_CHECK_AT([$1],
  63 [[
  64 AT_INIT([artificial test suite])
  65 AT_SETUP([my only test])
  66 $2
  67 AT_CLEANUP
  68 ]], m4_shiftn(2, $@))])
  69
  70 # Here documents for these tests contain forbidden macros.
  71 m4_pattern_allow([^AT_])
  72
  73 # AT_NO_CMDSUBST
  74 # --------------
  75 m4_define([AT_NO_CMDSUBST],
  76 [if (eval 'foo=$(echo bar) && test "$foo" = bar') >/dev/null 2>&1; then false; else :; fi])
  77
  78
  79 ## ------------------ ##
  80 ## Empty test suite.  ##
  81 ## ------------------ ##
  82
  83 # This is not a sensible thing to do, but the user should not get an unhelpful
  84 # error message.
  85 AT_CHECK_AT([Empty test suite],
  86 [[AT_INIT([empty test suite])
  87 ]])
  88
  89 # Next level of emptiness.
  90 AT_CHECK_AT_TEST([Empty test], [])
  91
  92 # And finally, an empty check should not cause a syntax error.
  93 AT_CHECK_AT_TEST([Empty check], [AT_CHECK])
  94
  95 ## ----------------------------------------------------- ##
  96 ## Newlines and command substitutions in test commands.  ##
  97 ## ----------------------------------------------------- ##
  98
  99 AT_CHECK_AT_TEST([Truth],
 100   [AT_CHECK([:], 0, [], [])])
 101
 102 AT_CHECK_AT_TEST([Fallacy],
 103   [AT_CHECK([false], ignore, [], [])])
 104
 105 AT_CHECK_AT_TEST([Literal multiline command],
 106   [AT_CHECK([echo Auto'
 107 'conf], 0, [Auto
 108 conf
 109 ], [])])
 110
 111 AT_CHECK_AT_TEST([Multiline parameter expansion],
 112   [FOO='one
 113 two'
 114    AT_CHECK([echo "$FOO"], 0, [one
 115 two
 116 ], [])])
 117
 118 AT_CHECK_AT_TEST([Backquote command substition],
 119   [AT_CHECK([echo `echo hi`], 0, [hi
 120 ], [])])
 121
 122
 123 AT_CHECK_AT_TEST([Multiline backquote command substition],
 124   [AT_DATA([myfile],[foo
 125 bar
 126 ])
 127    AT_CHECK([echo "`cat myfile`"], 0, [foo
 128 bar
 129 ], [])])
 130
 131 AT_CHECK_AT_TEST([Parenthetical command substition],
 132   [AT_CHECK([echo $(echo hi)], 0, [hi
 133 ], [])],
 134   [AT_NO_CMDSUBST])
 135
 136 AT_CHECK_AT_TEST([Multiline parenthetical command substition],
 137   [AT_DATA([myfile],[foo
 138 bar
 139 ])
 140    AT_CHECK([echo "$(cat myfile)"], 0, [foo
 141 bar
 142 ], [])],
 143   [AT_NO_CMDSUBST])
 144
 145
 146 ## ------------------------- ##
 147 ## ${...} in test commands.  ##
 148 ## ------------------------- ##
 149
 150 # If this invalid parameter expansion capsizes the test suite, the entire
 151 # AT_SETUP ... AT_CLEANUP subshell will exit, and the commands it runs will
 152 # appear to have succeeded.  Therefore, we verify a failing test case.
 153
 154 AT_CHECK_AT_TEST([Invalid brace-enclosed parameter expansion],
 155   [AT_CHECK([echo '${=invalid}'], 0, [wrong])], [false], 1, ignore, ignore)
 156
 157
 158 ## ---------------------------- ##
 159 ## M4 macros in test commands.  ##
 160 ## ---------------------------- ##
 161
 162 # The last paragaph in the comment above _AT_DECIDE_TRACEABLE illustrates why
 163 # this test fails.
 164 AT_CHECK_AT_TEST([Multiline command from M4 expansion],
 165   [m4_define([GNU], ['foo
 166 bar'])
 167    AT_CHECK([echo GNU], 0, [foo
 168 bar
 169 ], [])], [:])
 170
 171 AT_CHECK_AT_TEST([Double-M4-quoted command],
 172   [m4_define([GNU], ['foo
 173 bar'])
 174    AT_CHECK([[echo GNU]], 0, [[GNU
 175 ]], [])])
 176
 177
 178 ## -------------------------------------- ##
 179 ## Backslash-<newline> in test commands.  ##
 180 ## -------------------------------------- ##
 181
 182 AT_CHECK_AT_TEST([BS-newline in command],
 183   [AT_CHECK([echo Auto"\
 184 "conf], 0, [Autoconf
 185 ], [])])
 186
 187 AT_CHECK_AT_TEST([^BS-newline in command],
 188   [AT_CHECK([\
 189 echo GNU], 0, [GNU
 190 ], [])])
 191
 192 AT_CHECK_AT_TEST([BSx641-newline in command],
 193   [AT_CHECK([echo Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 194 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\conf
 195 ], [])])
 196
 197 AT_CHECK_AT_TEST([BS-BS-newline in command],
 198   [AT_CHECK([echo Auto"\\
 199 "conf], 0, [Auto\
 200 conf
 201 ], [])])
 202
 203 # A `^BS-BS-newline in command' test will run a command named `\'.  No, thanks.
 204
 205 AT_CHECK_AT_TEST([BSx640-newline in command],
 206   [AT_CHECK([echo Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 207 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 208 conf
 209 ], [])])
 210
 211 # This command has both escaped and unescaped newlines.
 212 AT_CHECK_AT_TEST([Newline-CODE-BS-newline in command],
 213   [AT_CHECK([echo Auto'
 214 'co\
 215 nf], 0, [Auto
 216 conf
 217 ], [])])
 218
 219 AT_CHECK_AT_TEST([Single-quote-BS-newline in command],
 220   [AT_CHECK([echo Auto'\
 221 'conf], 0, [Auto\
 222 conf
 223 ], [])])
 224
 225 AT_CHECK_AT_TEST([Single-quote-newline-BS-newline in command],
 226   [AT_CHECK([echo Auto'
 227 \
 228 'conf], 0, [Auto
 229 \
 230 conf
 231 ], [])])
 232
 233
 234 ## ------------------------------- ##
 235 ## Funny characters in test names. ##
 236 ## ------------------------------- ##
 237
 238 # AT_CHECK_AT_TITLE(TITLE, TITLE-TO-TEST, [XFAIL-CONDITION])
 239 # ----------------------------------------------------------
 240 # Create a new test named TITLE that runs an Autotest test suite
 241 # comprised of a trivial test named TITLE-TO-TEST.  XFAIL-CONDITION
 242 # passes verbatim to AT_CHECK_AT.
 243 m4_define([AT_CHECK_AT_TITLE],
 244 [AT_CHECK_AT([$1],
 245 [[
 246 AT_INIT([artificial test suite])
 247 AT_SETUP([$2])
 248 AT_CHECK([:])
 249 AT_CLEANUP
 250 ]], [$3])])
 251
 252 m4_define([AT_CHECK_AT_TITLE_CHAR],
 253 [AT_CHECK_AT_TITLE([$1 in a test title], [A $2 in my name], $3)])
 254
 255 AT_CHECK_AT_TITLE_CHAR([Backquote],    [`], [:])
 256 AT_CHECK_AT_TITLE_CHAR([Single-quote], ['], [:])
 257 AT_CHECK_AT_TITLE_CHAR([Double-quote], ["], [:])
 258 AT_CHECK_AT_TITLE_CHAR([Backslash],    [\])
 259
 260
 261 ## ----------------- ##
 262 ## Debugging a test. ##
 263 ## ----------------- ##
 264
 265 AT_CHECK_AT_TEST([Debugging a successful test],
 266   [AT_CHECK([:])], [], [], [], [ignore],
 267 [# Without options, when all tests pass, no test directory should exist.
 268 AT_CHECK([test -d micro-suite.dir/1 && exit 42
 269           ./micro-suite -d 1], [], [ignore], [ignore])
 270 # Running with -d should leave a reproducible test group.
 271 # Also, running the test script from the test group locks the
 272 # directory from removal on some platforms; the script should still be
 273 # able to run even if rmdir fails.
 274 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
 275 # Running a debugging script implies -d.
 276 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
 277 ])
 278
 279 AT_CHECK_AT_TEST([Debugging script and environment],
 280   [AT_CHECK([test "$MY_VAR" = pass || exit 42])],
 281   [], [1], [], [ignore], [
 282 # Changing environment outside of debugging script is not preserved.
 283 AT_CHECK([(cd micro-suite.dir/1 && MY_VAR=pass ./run)],
 284          [0], [ignore], [ignore])
 285 AT_CHECK([(cd micro-suite.dir/1 && ./run)],
 286          [1], [ignore], [ignore])
 287 # Changing environment as argument to debugging script is preserved.
 288 AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR=pass)],
 289          [0], [ignore], [ignore])
 290 AT_CHECK([(cd micro-suite.dir/1; ./run)],
 291          [0], [ignore], [ignore])
 292 ])
 293
 294 # The run script must still be valid when shell metacharacters are passed
 295 # in via an environment option.
 296 AT_CHECK_AT_TEST([Debugging a failed test],
 297   [AT_CHECK([test "$MY_VAR" = "one space" || exit 42])],
 298   [], [1], [], [ignore], [
 299 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='two  spaces')],
 300          [1], [ignore], [ignore])
 301 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='one space')],
 302          [0], [ignore], [ignore])
 303 ])
 304
 305
 306 ## --------- ##
 307 ## Keywords. ##
 308 ## --------- ##
 309 AT_SETUP([Keywords and ranges])
 310 AT_KEYWORDS([autotest])
 311
 312 AT_DATA([k.at],
 313 [[m4_define([AT_PACKAGE_STRING],[k])
 314 m4_define([AT_PACKAGE_BUGREPORT],[devnull])
 315 AT_INIT
 316 AT_SETUP(none)
 317 AT_CHECK(:)
 318 AT_CLEANUP
 319 AT_SETUP(first)
 320 AT_KEYWORDS(key1)
 321 AT_CHECK(:)
 322 AT_CLEANUP
 323 AT_SETUP(second)
 324 AT_KEYWORDS(key2)
 325 AT_CHECK(:)
 326 AT_CLEANUP
 327 AT_SETUP(both)
 328 AT_KEYWORDS(key1)
 329 AT_KEYWORDS(key2)
 330 AT_CHECK(:)
 331 AT_CLEANUP
 332 ]])
 333 AT_CHECK_AUTOM4TE([--language=autotest -o k k.at])
 334
 335 # AT_CHECK_EGREP(PATTERN, STATUS, COUNT)
 336 m4_define([AT_CHECK_EGREP],
 337 [AT_CHECK([$EGREP -c '$1' stdout], $2, [$3
 338 ], ignore)
 339 ])
 340
 341 # AT_CHECK_KEYS(TESTSUITE-OPTIONS, PATTERN1, COUNT1, PATTERN2, COUNT2)
 342 m4_define([AT_CHECK_KEYS],
 343 [AT_CHECK([./k $1], 0, [stdout])
 344 AT_CHECK_EGREP([$2], 0, [$3])
 345 AT_CHECK_EGREP([$4], 1, [$5])
 346 ])
 347
 348 AT_CHECK_KEYS([-k key1], [first|both], [2], [none|second], [0])
 349 AT_CHECK_KEYS([-k key2], [second|both], [2], [none|first], [0])
 350 AT_CHECK_KEYS([-k key1,key2], [both], [1], [none|first|second], [0])
 351 AT_CHECK_KEYS([-k key1 -k key2], [first|second|both], [3], [none], [0])
 352 AT_CHECK_KEYS([-k '!key1'], [none|second], [2], [first|both], [0])
 353 AT_CHECK_KEYS([-k '!key2'], [none|first], [2], [second|both], [0])
 354 AT_CHECK_KEYS([-k '!key1,key2'], [second], [1], [none|first|both], [0])
 355 AT_CHECK_KEYS([-k 'key1,!key2'], [first], [1], [none|second|both], [0])
 356 AT_CHECK_KEYS([-k '!key1,!key2'], [none], [1], [first|second|both], [0])
 357 AT_CHECK_KEYS([-k '!key1' -k KEY2], [none|second|both], [3], [first], [0])
 358 AT_CHECK_KEYS([-k key1 -k '!key2'], [none|first|both], [3], [second], [0])
 359 AT_CHECK_KEYS([-k '!KEY1' -k '!key2'], [none|first|second], [3], [both], [0])
 360
 361 AT_CHECK_KEYS([-k none], [none], [1], [first|second|both], [0])
 362 AT_CHECK_KEYS([-k key1,both], [both], [1], [none|first|second], [0])
 363 AT_CHECK_KEYS([-k key1 -k both], [first|both], [2], [none|second], [0])
 364 AT_CHECK_KEYS([-k none,first], [successful], [1], [none|first|second|both], [0])
 365 AT_CHECK_KEYS([-k none,first,second,both], [successful], [1], [none|first|second|both], [0])
 366 AT_CHECK_KEYS([-k !none,first], [first], [1], [none|second|both], [0])
 367
 368 AT_CHECK_KEYS([-k '.*eco.*'], [second], [1], [none|first|both], [0])
 369 AT_CHECK_KEYS([-k 'ECO'], [successful], [1], [none|first|second|both], [0])
 370 AT_CHECK_KEYS([-k '.*eco'], [successful], [1], [none|first|second|both], [0])
 371 AT_CHECK_KEYS([-k 'eco.*'], [successful], [1], [none|first|second|both], [0])
 372 AT_CHECK_KEYS([-k 'fir.*'], [first], [1], [none|second|both], [0])
 373
 374 AT_CHECK_KEYS([1-2], [none|first], [2], [second|both], [0])
 375 AT_CHECK_KEYS([1-3 2-1], [none|first|second], [3], [both], [0])
 376 AT_CHECK_KEYS([-3], [none|first|second], [3], [both], [0])
 377 AT_CHECK_KEYS([4-], [both], [1], [none|first|second], [0])
 378 AT_CHECK_KEYS([-k second 4-], [second|both], [2], [none|first], [0])
 379 AT_CLEANUP