tests/autotest.at

   1 #                                                       -*- Autotest -*-
   2
   3 AT_BANNER([Autotest.])
   4
   5 # Copyright (C) 2004, 2005, 2006 Free Software Foundation, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2, or (at your option)
  10 # any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20 # 02110-1301, USA.
  21
  22 # AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0],
  23 #             [STDOUT := ignore], STDERR, [POST-TEST-CODE])
  24 # ---------------------------------------------------------------
  25 # Create a new test named TITLE that runs a minimal Autotest test suite,
  26 # SUITE-CODE.  Call AT_XFAIL_IF with XFAIL-CONDITION.  STATUS and STDERR pass
  27 # directly to the AT_CHECK that call the minimal test suite.  STDOUT is not
  28 # used, but it is reserved for future use.  Run POST-TEST-CODE
  29 # at the top level after the micro-suite has been run.
  30 m4_define([AT_CHECK_AT],
  31 [
  32 AT_SETUP([$1])
  33 AT_KEYWORDS([autotest])
  34 AT_CAPTURE_FILE([micro-suite.log])
  35 AT_XFAIL_IF([$3])
  36
  37 AT_DATA([package.m4],[[
  38 m4_define([AT_PACKAGE_NAME],      [GNU Nonsense])
  39 m4_define([AT_PACKAGE_TARNAME],   [nonsense])
  40 m4_define([AT_PACKAGE_VERSION],   [1.0])
  41 m4_define([AT_PACKAGE_STRING],    [GNU Nonsense 1.0])
  42 m4_define([AT_PACKAGE_BUGREPORT], [bug-autoconf@gnu.org])
  43 ]])
  44
  45 AT_DATA([mysuite.at], [$2])
  46
  47 # Do not use `testsuite' as the name of the small test suite, or the
  48 # log file it generates will overwrite the log that the Autoconf test
  49 # suite produces for this test case.
  50 AT_CHECK_AUTOM4TE([--language=autotest -o micro-suite mysuite.at])
  51 AT_CHECK([$CONFIG_SHELL ./micro-suite],       m4_default([$4], 0), [ignore], [$6])
  52 AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x], m4_default([$4], 0), [ignore], [$6])
  53 $7
  54 AT_CLEANUP
  55 ])# AT_CHECK_AT
  56
  57 # AT_CHECK_AT_TEST(TITLE, SUITE-SNIPPET, ...)
  58 # -----------------------------------------------------------------------
  59 # Wrapper for AT_CHECK_AT that surrounds SUITE-SNIPPET with a boilerplate
  60 # AT_INIT, AT_SETUP, and AT_CLEANUP and passes other arguments verbatim.
  61 m4_define([AT_CHECK_AT_TEST],
  62 [AT_CHECK_AT([$1],
  63 [[
  64 AT_INIT([artificial test suite])
  65 AT_SETUP([my only test])
  66 $2
  67 AT_CLEANUP
  68 ]], m4_shiftn(2, $@))])
  69
  70 # Here documents for these tests contain forbidden macros.
  71 m4_pattern_allow([^AT_])
  72
  73 # AT_NO_CMDSUBST
  74 # --------------
  75 m4_define([AT_NO_CMDSUBST],
  76 [if (eval 'foo=$(echo bar) && test "$foo" = bar') >/dev/null 2>&1; then false; else :; fi])
  77
  78
  79 ## ------------------ ##
  80 ## Empty test suite.  ##
  81 ## ------------------ ##
  82
  83 # This is not a sensible thing to do, but the user should not get an unhelpful
  84 # error message.
  85 AT_CHECK_AT([Empty test suite],
  86 [[AT_INIT([empty test suite])
  87 ]])
  88
  89 # Next level of emptiness.
  90 AT_CHECK_AT_TEST([Empty test], [])
  91
  92 # And finally, an empty check should not cause a syntax error.
  93 AT_CHECK_AT_TEST([Empty check], [AT_CHECK])
  94
  95 ## ----------------------------------------------------- ##
  96 ## Newlines and command substitutions in test commands.  ##
  97 ## ----------------------------------------------------- ##
  98
  99 AT_CHECK_AT_TEST([Truth],
 100   [AT_CHECK([:], 0, [], [])])
 101
 102 AT_CHECK_AT_TEST([Fallacy],
 103   [AT_CHECK([false], ignore, [], [])])
 104
 105 AT_CHECK_AT_TEST([Literal multiline command],
 106   [AT_CHECK([echo Auto'
 107 'conf], 0, [Auto
 108 conf
 109 ], [])])
 110
 111 AT_CHECK_AT_TEST([Multiline parameter expansion],
 112   [FOO='one
 113 two'
 114    AT_CHECK([echo "$FOO"], 0, [one
 115 two
 116 ], [])])
 117
 118 AT_CHECK_AT_TEST([Backquote command substition],
 119   [AT_CHECK([echo `echo hi`], 0, [hi
 120 ], [])])
 121
 122
 123 AT_CHECK_AT_TEST([Multiline backquote command substition],
 124   [AT_DATA([myfile],[foo
 125 bar
 126 ])
 127    AT_CHECK([echo "`cat myfile`"], 0, [foo
 128 bar
 129 ], [])])
 130
 131 AT_CHECK_AT_TEST([Parenthetical command substition],
 132   [AT_CHECK([echo $(echo hi)], 0, [hi
 133 ], [])],
 134   [AT_NO_CMDSUBST])
 135
 136 AT_CHECK_AT_TEST([Multiline parenthetical command substition],
 137   [AT_DATA([myfile],[foo
 138 bar
 139 ])
 140    AT_CHECK([echo "$(cat myfile)"], 0, [foo
 141 bar
 142 ], [])],
 143   [AT_NO_CMDSUBST])
 144
 145
 146 ## ------------------------- ##
 147 ## ${...} in test commands.  ##
 148 ## ------------------------- ##
 149
 150 # If this invalid parameter expansion capsizes the test suite, the entire
 151 # AT_SETUP ... AT_CLEANUP subshell will exit, and the commands it runs will
 152 # appear to have succeeded.  Therefore, we verify a failing test case.
 153
 154 AT_CHECK_AT_TEST([Invalid brace-enclosed parameter expansion],
 155   [AT_CHECK([echo '${=invalid}'], 0, [wrong])], [false], 1, ignore, ignore)
 156
 157
 158 ## ---------------------------- ##
 159 ## M4 macros in test commands.  ##
 160 ## ---------------------------- ##
 161
 162 # The last paragaph in the comment above _AT_DECIDE_TRACEABLE illustrates why
 163 # this test fails (except with Korn shell-style quoting $'foo\nbar').
 164 AT_CHECK_AT_TEST([Multiline command from M4 expansion],
 165   [m4_define([GNU], ['foo
 166 bar'])
 167    AT_CHECK([echo GNU], 0, [foo
 168 bar
 169 ], [])], [case `( set -x; echo 'foo
 170 bar') 2>&1` in *\$\'foo\\nbar\'*) false;; *) :;; esac])
 171
 172 AT_CHECK_AT_TEST([Double-M4-quoted command],
 173   [m4_define([GNU], ['foo
 174 bar'])
 175    AT_CHECK([[echo GNU]], 0, [[GNU
 176 ]], [])])
 177
 178
 179 ## -------------------------------------- ##
 180 ## Backslash-<newline> in test commands.  ##
 181 ## -------------------------------------- ##
 182
 183 AT_CHECK_AT_TEST([BS-newline in command],
 184   [AT_CHECK([echo Auto"\
 185 "conf], 0, [Autoconf
 186 ], [])])
 187
 188 AT_CHECK_AT_TEST([^BS-newline in command],
 189   [AT_CHECK([\
 190 echo GNU], 0, [GNU
 191 ], [])])
 192
 193 AT_CHECK_AT_TEST([BSx641-newline in command],
 194   [AT_CHECK([echo Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 195 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\conf
 196 ], [])])
 197
 198 AT_CHECK_AT_TEST([BS-BS-newline in command],
 199   [AT_CHECK([echo Auto"\\
 200 "conf], 0, [Auto\
 201 conf
 202 ], [])])
 203
 204 # A `^BS-BS-newline in command' test will run a command named `\'.  No, thanks.
 205
 206 AT_CHECK_AT_TEST([BSx640-newline in command],
 207   [AT_CHECK([echo Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 208 "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
 209 conf
 210 ], [])])
 211
 212 # This command has both escaped and unescaped newlines.
 213 AT_CHECK_AT_TEST([Newline-CODE-BS-newline in command],
 214   [AT_CHECK([echo Auto'
 215 'co\
 216 nf], 0, [Auto
 217 conf
 218 ], [])])
 219
 220 AT_CHECK_AT_TEST([Single-quote-BS-newline in command],
 221   [AT_CHECK([echo Auto'\
 222 'conf], 0, [Auto\
 223 conf
 224 ], [])])
 225
 226 AT_CHECK_AT_TEST([Single-quote-newline-BS-newline in command],
 227   [AT_CHECK([echo Auto'
 228 \
 229 'conf], 0, [Auto
 230 \
 231 conf
 232 ], [])])
 233
 234
 235 ## ------------------------------- ##
 236 ## Funny characters in test names. ##
 237 ## ------------------------------- ##
 238
 239 # AT_CHECK_AT_TITLE(TITLE, TITLE-TO-TEST, [XFAIL-CONDITION])
 240 # ----------------------------------------------------------
 241 # Create a new test named TITLE that runs an Autotest test suite
 242 # comprised of a trivial test named TITLE-TO-TEST.  XFAIL-CONDITION
 243 # passes verbatim to AT_CHECK_AT.
 244 m4_define([AT_CHECK_AT_TITLE],
 245 [AT_CHECK_AT([$1],
 246 [[
 247 AT_INIT([artificial test suite])
 248 AT_SETUP([$2])
 249 AT_CHECK([:])
 250 AT_CLEANUP
 251 ]], [$3])])
 252
 253 m4_define([AT_CHECK_AT_TITLE_CHAR],
 254 [AT_CHECK_AT_TITLE([$1 in a test title], [A $2 in my name], $3)])
 255
 256 AT_CHECK_AT_TITLE_CHAR([Backquote],    [`], [:])
 257 AT_CHECK_AT_TITLE_CHAR([Single-quote], ['], [:])
 258 AT_CHECK_AT_TITLE_CHAR([Double-quote], ["], [:])
 259 AT_CHECK_AT_TITLE_CHAR([Backslash],    [\])
 260
 261
 262 ## ----------------- ##
 263 ## Debugging a test. ##
 264 ## ----------------- ##
 265
 266 AT_CHECK_AT_TEST([Debugging a successful test],
 267   [AT_CHECK([:])], [], [], [], [ignore],
 268 [# Without options, when all tests pass, no test directory should exist.
 269 AT_CHECK([test -d micro-suite.dir/1 && exit 42
 270           ./micro-suite -d 1], [], [ignore], [ignore])
 271 # Running with -d should leave a reproducible test group.
 272 # Also, running the test script from the test group locks the
 273 # directory from removal on some platforms; the script should still be
 274 # able to run even if rmdir fails.
 275 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
 276 # Running a debugging script implies -d.
 277 AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore])
 278 ])
 279
 280 AT_CHECK_AT_TEST([Debugging script and environment],
 281   [AT_CHECK([test "$MY_VAR" = pass || exit 42])],
 282   [], [1], [], [ignore], [
 283 # Changing environment outside of debugging script is not preserved.
 284 AT_CHECK([(cd micro-suite.dir/1 && MY_VAR=pass ./run)],
 285          [0], [ignore], [ignore])
 286 AT_CHECK([(cd micro-suite.dir/1 && ./run)],
 287          [1], [ignore], [ignore])
 288 # Changing environment as argument to debugging script is preserved.
 289 AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR=pass)],
 290          [0], [ignore], [ignore])
 291 AT_CHECK([(cd micro-suite.dir/1; ./run)],
 292          [0], [ignore], [ignore])
 293 ])
 294
 295 # The run script must still be valid when shell metacharacters are passed
 296 # in via an environment option.
 297 AT_CHECK_AT_TEST([Debugging a failed test],
 298   [AT_CHECK([test "$MY_VAR" = "one space" || exit 42])],
 299   [], [1], [], [ignore], [
 300 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='two  spaces')],
 301          [1], [ignore], [ignore])
 302 AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='one space')],
 303          [0], [ignore], [ignore])
 304 ])
 305
 306
 307 ## --------- ##
 308 ## Keywords. ##
 309 ## --------- ##
 310 AT_SETUP([Keywords and ranges])
 311 AT_KEYWORDS([autotest])
 312
 313 AT_DATA([k.at],
 314 [[m4_define([AT_PACKAGE_STRING],[k])
 315 m4_define([AT_PACKAGE_BUGREPORT],[devnull])
 316 AT_INIT
 317 AT_SETUP(none)
 318 AT_CHECK(:)
 319 AT_CLEANUP
 320 AT_SETUP(first)
 321 AT_KEYWORDS(key1)
 322 AT_CHECK(:)
 323 AT_CLEANUP
 324 AT_SETUP(second)
 325 AT_KEYWORDS(key2)
 326 AT_CHECK(:)
 327 AT_CLEANUP
 328 AT_SETUP(both)
 329 AT_KEYWORDS(key1)
 330 AT_KEYWORDS(key2)
 331 AT_CHECK(:)
 332 AT_CLEANUP
 333 ]])
 334 AT_CHECK_AUTOM4TE([--language=autotest -o k k.at])
 335
 336 # AT_CHECK_EGREP(PATTERN, STATUS, COUNT)
 337 m4_define([AT_CHECK_EGREP],
 338 [AT_CHECK([$EGREP -c '$1' stdout], $2, [$3
 339 ], ignore)
 340 ])
 341
 342 # AT_CHECK_KEYS(TESTSUITE-OPTIONS, PATTERN1, COUNT1, PATTERN2, COUNT2)
 343 m4_define([AT_CHECK_KEYS],
 344 [AT_CHECK([./k $1], 0, [stdout])
 345 AT_CHECK_EGREP([$2], 0, [$3])
 346 AT_CHECK_EGREP([$4], 1, [$5])
 347 ])
 348
 349 AT_CHECK_KEYS([-k key1], [first|both], [2], [none|second], [0])
 350 AT_CHECK_KEYS([-k key2], [second|both], [2], [none|first], [0])
 351 AT_CHECK_KEYS([-k key1,key2], [both], [1], [none|first|second], [0])
 352 AT_CHECK_KEYS([-k key1 -k key2], [first|second|both], [3], [none], [0])
 353 AT_CHECK_KEYS([-k '!key1'], [none|second], [2], [first|both], [0])
 354 AT_CHECK_KEYS([-k '!key2'], [none|first], [2], [second|both], [0])
 355 AT_CHECK_KEYS([-k '!key1,key2'], [second], [1], [none|first|both], [0])
 356 AT_CHECK_KEYS([-k 'key1,!key2'], [first], [1], [none|second|both], [0])
 357 AT_CHECK_KEYS([-k '!key1,!key2'], [none], [1], [first|second|both], [0])
 358 AT_CHECK_KEYS([-k '!key1' -k KEY2], [none|second|both], [3], [first], [0])
 359 AT_CHECK_KEYS([-k key1 -k '!key2'], [none|first|both], [3], [second], [0])
 360 AT_CHECK_KEYS([-k '!KEY1' -k '!key2'], [none|first|second], [3], [both], [0])
 361
 362 AT_CHECK_KEYS([-k none], [none], [1], [first|second|both], [0])
 363 AT_CHECK_KEYS([-k key1,both], [both], [1], [none|first|second], [0])
 364 AT_CHECK_KEYS([-k key1 -k both], [first|both], [2], [none|second], [0])
 365 AT_CHECK_KEYS([-k none,first], [successful], [1], [none|first|second|both], [0])
 366 AT_CHECK_KEYS([-k none,first,second,both], [successful], [1], [none|first|second|both], [0])
 367 AT_CHECK_KEYS([-k !none,first], [first], [1], [none|second|both], [0])
 368
 369 AT_CHECK_KEYS([-k '.*eco.*'], [second], [1], [none|first|both], [0])
 370 AT_CHECK_KEYS([-k 'ECO'], [successful], [1], [none|first|second|both], [0])
 371 AT_CHECK_KEYS([-k '.*eco'], [successful], [1], [none|first|second|both], [0])
 372 AT_CHECK_KEYS([-k 'eco.*'], [successful], [1], [none|first|second|both], [0])
 373 AT_CHECK_KEYS([-k 'fir.*'], [first], [1], [none|second|both], [0])
 374
 375 AT_CHECK_KEYS([1-2], [none|first], [2], [second|both], [0])
 376 AT_CHECK_KEYS([1-3 2-1], [none|first|second], [3], [both], [0])
 377 AT_CHECK_KEYS([-3], [none|first|second], [3], [both], [0])
 378 AT_CHECK_KEYS([4-], [both], [1], [none|first|second], [0])
 379 AT_CHECK_KEYS([-k second 4-], [second|both], [2], [none|first], [0])
 380 AT_CLEANUP