Test/ztst.zsh

   1 #!/bin/zsh -f
   2 # The line above is just for convenience.  Normally tests will be run using
   3 # a specified version of zsh.  With dynamic loading, any required libraries
   4 # must already have been installed in that case.
   5 #
   6 # Takes one argument: the name of the test file.  Currently only one such
   7 # file will be processed each time ztst.zsh is run.  This is slower, but
   8 # much safer in terms of preserving the correct status.
   9 # To avoid namespace pollution, all functions and parameters used
  10 # only by the script begin with ZTST_.
  11 #
  12 # Options (without arguments) may precede the test file argument; these
  13 # are interpreted as shell options to set.  -x is probably the most useful.
  14
  15 # Produce verbose messages if non-zero.
  16 # If 1, produce reports of tests executed; if 2, also report on progress.
  17 # Defined in such a way that any value from the environment is used.
  18 : ${ZTST_verbose:=0}
  19
  20 # We require all options to be reset, not just emulation options.
  21 # Unfortunately, due to the crud which may be in /etc/zshenv this might
  22 # still not be good enough.  Maybe we should trick it somehow.
  23 emulate -R zsh
  24
  25 # Ensure the locale does not screw up sorting.  Don't supply a locale
  26 # unless there's one set, to minimise problems.
  27 [[ -n $LC_ALL ]] && LC_ALL=C
  28 [[ -n $LC_COLLATE ]] && LC_COLLATE=C
  29 [[ -n $LC_NUMERIC ]] && LC_NUMERIC=C
  30 [[ -n $LC_MESSAGES ]] && LC_MESSAGES=C
  31 [[ -n $LANG ]] && LANG=C
  32
  33 # Don't propagate variables that are set by default in the shell.
  34 typeset +x WORDCHARS
  35
  36 # Set the module load path to correspond to this build of zsh.
  37 # This Modules directory should have been created by "make check".
  38 [[ -d Modules/zsh ]] && module_path=( $PWD/Modules )
  39 # Allow this to be passed down.
  40 export MODULE_PATH
  41
  42 # We need to be able to save and restore the options used in the test.
  43 # We use the $options variable of the parameter module for this.
  44 zmodload -i zsh/parameter
  45
  46 # Note that both the following are regular arrays, since we only use them
  47 # in whole array assignments to/from $options.
  48 # Options set in test code (i.e. by default all standard options)
  49 ZTST_testopts=(${(kv)options})
  50
  51 setopt extendedglob nonomatch
  52 while [[ $1 = [-+]* ]]; do
  53   set $1
  54   shift
  55 done
  56 # Options set in main script
  57 ZTST_mainopts=(${(kv)options})
  58
  59 # We run in the current directory, so remember it.
  60 ZTST_testdir=$PWD
  61 ZTST_testname=$1
  62
  63 integer ZTST_testfailed
  64
  65 # This is POSIX nonsense.  Because of the vague feeling someone, somewhere
  66 # may one day need to examine the arguments of "tail" using a standard
  67 # option parser, every Unix user in the world is expected to switch
  68 # to using "tail -n NUM" instead of "tail -NUM".  Older versions of
  69 # tail don't support this.
  70 tail() {
  71   emulate -L zsh
  72
  73   if [[ -z $TAIL_SUPPORTS_MINUS_N ]]; then
  74     local test
  75     test=$(echo "foo\nbar" | command tail -n 1 2>/dev/null)
  76     if [[ $test = bar ]]; then
  77       TAIL_SUPPORTS_MINUS_N=1
  78     else
  79       TAIL_SUPPORTS_MINUS_N=0
  80     fi
  81   fi
  82
  83   integer argi=${argv[(i)-<->]}
  84
  85   if [[ $argi -le $# && $TAIL_SUPPORTS_MINUS_N = 1 ]]; then
  86     argv[$argi]=(-n ${argv[$argi][2,-1]})
  87   fi
  88
  89   command tail "$argv[@]"
  90 }
  91
  92 # The source directory is not necessarily the current directory,
  93 # but if $0 doesn't contain a `/' assume it is.
  94 if [[ $0 = */* ]]; then
  95   ZTST_srcdir=${0%/*}
  96 else
  97   ZTST_srcdir=$PWD
  98 fi
  99 [[ $ZTST_srcdir = /* ]] || ZTST_srcdir="$ZTST_testdir/$ZTST_srcdir"
 100
 101 # Set the function autoload paths to correspond to this build of zsh.
 102 fpath=( $ZTST_srcdir/../Functions/*~*/CVS(/)
 103         $ZTST_srcdir/../Completion
 104         $ZTST_srcdir/../Completion/*/*~*/CVS(/) )
 105
 106 : ${TMPPREFIX:=/tmp/zsh}
 107 # Temporary files for redirection inside tests.
 108 ZTST_in=${TMPPREFIX}.ztst.in.$$
 109 # hold the expected output
 110 ZTST_out=${TMPPREFIX}.ztst.out.$$
 111 ZTST_err=${TMPPREFIX}.ztst.err.$$
 112 # hold the actual output from the test
 113 ZTST_tout=${TMPPREFIX}.ztst.tout.$$
 114 ZTST_terr=${TMPPREFIX}.ztst.terr.$$
 115
 116 ZTST_cleanup() {
 117   cd $ZTST_testdir
 118   rm -rf $ZTST_testdir/dummy.tmp $ZTST_testdir/*.tmp(N) \
 119     ${TMPPREFIX}.ztst*$$(N)
 120 }
 121
 122 # This cleanup always gets performed, even if we abort.  Later,
 123 # we should try and arrange that any test-specific cleanup
 124 # always gets called as well.
 125 ##trap 'print cleaning up...
 126 ##ZTST_cleanup' INT QUIT TERM
 127 # Make sure it's clean now.
 128 rm -rf dummy.tmp *.tmp
 129
 130 # Report failure.  Note that all output regarding the tests goes to stdout.
 131 # That saves an unpleasant mixture of stdout and stderr to sort out.
 132 ZTST_testfailed() {
 133   print -r "Test $ZTST_testname failed: $1"
 134   if [[ -n $ZTST_message ]]; then
 135     print -r "Was testing: $ZTST_message"
 136   fi
 137   print -r "$ZTST_testname: test failed."
 138   if [[ -n $ZTST_failmsg ]]; then
 139     print -r "The following may (or may not) help identifying the cause:
 140 $ZTST_failmsg"
 141   fi
 142   ZTST_testfailed=1
 143   return 1
 144 }
 145
 146 # Print messages if $ZTST_verbose is non-empty
 147 ZTST_verbose() {
 148   local lev=$1
 149   shift
 150   if [[ -n $ZTST_verbose && $ZTST_verbose -ge $lev ]]; then
 151     print -r -u $ZTST_fd -- $*
 152   fi
 153 }
 154 ZTST_hashmark() {
 155   if [[ ZTST_verbose -le 0 && -t $ZTST_fd ]]; then
 156     print -n -u$ZTST_fd -- ${(pl:SECONDS::\#::\#\r:)}
 157   fi
 158   (( SECONDS > COLUMNS+1 && (SECONDS -= COLUMNS) ))
 159 }
 160
 161 if [[ ! -r $ZTST_testname ]]; then
 162   ZTST_testfailed "can't read test file."
 163   exit 1
 164 fi
 165
 166 exec {ZTST_fd}>&1
 167 exec {ZTST_input}<$ZTST_testname
 168
 169 # The current line read from the test file.
 170 ZTST_curline=''
 171 # The current section being run
 172 ZTST_cursect=''
 173
 174 # Get a new input line.  Don't mangle spaces; set IFS locally to empty.
 175 # We shall skip comments at this level.
 176 ZTST_getline() {
 177   local IFS=
 178   while true; do
 179     read -u $ZTST_input -r ZTST_curline || return 1
 180     [[ $ZTST_curline == \#* ]] || return 0
 181   done
 182 }
 183
 184 # Get the name of the section.  It may already have been read into
 185 # $curline, or we may have to skip some initial comments to find it.
 186 # If argument present, it's OK to skip the reset of the current section,
 187 # so no error if we find garbage.
 188 ZTST_getsect() {
 189   local match mbegin mend
 190
 191   while [[ $ZTST_curline != '%'(#b)([[:alnum:]]##)* ]]; do
 192     ZTST_getline || return 1
 193     [[ $ZTST_curline = [[:blank:]]# ]] && continue
 194     if [[ $# -eq 0 && $ZTST_curline != '%'[[:alnum:]]##* ]]; then
 195       ZTST_testfailed "bad line found before or after section:
 196 $ZTST_curline"
 197       exit 1
 198     fi
 199   done
 200   # have the next line ready waiting
 201   ZTST_getline
 202   ZTST_cursect=${match[1]}
 203   ZTST_verbose 2 "ZTST_getsect: read section name: $ZTST_cursect"
 204   return 0
 205 }
 206
 207 # Read in an indented code chunk for execution
 208 ZTST_getchunk() {
 209   # Code chunks are always separated by blank lines or the
 210   # end of a section, so if we already have a piece of code,
 211   # we keep it.  Currently that shouldn't actually happen.
 212   ZTST_code=''
 213   # First find the chunk.
 214   while [[ $ZTST_curline = [[:blank:]]# ]]; do
 215     ZTST_getline || break
 216   done
 217   while [[ $ZTST_curline = [[:blank:]]##[^[:blank:]]* ]]; do
 218     ZTST_code="${ZTST_code:+${ZTST_code}
 219 }${ZTST_curline}"
 220     ZTST_getline || break
 221   done
 222   ZTST_verbose 2 "ZTST_getchunk: read code chunk:
 223 $ZTST_code"
 224   [[ -n $ZTST_code ]]
 225 }
 226
 227 # Read in a piece for redirection.
 228 ZTST_getredir() {
 229   local char=${ZTST_curline[1]} fn
 230   ZTST_redir=${ZTST_curline[2,-1]}
 231   while ZTST_getline; do
 232     [[ $ZTST_curline[1] = $char ]] || break
 233     ZTST_redir="${ZTST_redir}
 234 ${ZTST_curline[2,-1]}"
 235   done
 236   ZTST_verbose 2 "ZTST_getredir: read redir for '$char':
 237 $ZTST_redir"
 238
 239   case $char in
 240     ('<') fn=$ZTST_in
 241     ;;
 242     ('>') fn=$ZTST_out
 243     ;;
 244     ('?') fn=$ZTST_err
 245     ;;
 246     (*)  ZTST_testfailed "bad redir operator: $char"
 247     return 1
 248     ;;
 249   esac
 250   if [[ $ZTST_flags = *q* && $char = '<' ]]; then
 251     # delay substituting output until variables are set
 252     print -r -- "${(e)ZTST_redir}" >>$fn
 253   else
 254     print -r -- "$ZTST_redir" >>$fn
 255   fi
 256
 257   return 0
 258 }
 259
 260 # Execute an indented chunk.  Redirections will already have
 261 # been set up, but we need to handle the options.
 262 ZTST_execchunk() {
 263   options=($ZTST_testopts)
 264   eval "$ZTST_code"
 265   ZTST_status=$?
 266   # careful... ksh_arrays may be in effect.
 267   ZTST_testopts=(${(kv)options[*]})
 268   options=(${ZTST_mainopts[*]})
 269   ZTST_verbose 2 "ZTST_execchunk: status $ZTST_status"
 270   return $ZTST_status
 271 }
 272
 273 # Functions for preparation and cleaning.
 274 # When cleaning up (non-zero string argument), we ignore status.
 275 ZTST_prepclean() {
 276   # Execute indented code chunks.
 277   while ZTST_getchunk; do
 278     ZTST_execchunk >/dev/null || [[ -n $1 ]] || {
 279       [[ -n "$ZTST_unimplemented" ]] ||
 280       ZTST_testfailed "non-zero status from preparation code:
 281 $ZTST_code" && return 0
 282     }
 283   done
 284 }
 285
 286 # diff wrapper
 287 ZTST_diff() {
 288   local diff_out diff_ret
 289
 290   diff_out=$(diff "$@")
 291   diff_ret="$?"
 292   if [[ "$diff_ret" != "0" ]]; then
 293     print -r "$diff_out"
 294   fi
 295
 296   return "$diff_ret"
 297 }
 298
 299 ZTST_test() {
 300   local last match mbegin mend found substlines
 301
 302   while true; do
 303     rm -f $ZTST_in $ZTST_out $ZTST_err
 304     touch $ZTST_in $ZTST_out $ZTST_err
 305     ZTST_message=''
 306     ZTST_failmsg=''
 307     found=0
 308
 309     ZTST_verbose 2 "ZTST_test: looking for new test"
 310
 311     while true; do
 312       ZTST_verbose 2 "ZTST_test: examining line:
 313 $ZTST_curline"
 314       case $ZTST_curline in
 315         (%*) if [[ $found = 0 ]]; then
 316               break 2
 317             else
 318               last=1
 319               break
 320             fi
 321             ;;
 322         ([[:space:]]#)
 323             if [[ $found = 0 ]]; then
 324               ZTST_getline || break 2
 325               continue
 326             else
 327               break
 328             fi
 329             ;;
 330         ([[:space:]]##[^[:space:]]*) ZTST_getchunk
 331           if [[ $ZTST_curline == (#b)([-0-9]##)([[:alpha:]]#)(:*)# ]]; then
 332             ZTST_xstatus=$match[1]
 333             ZTST_flags=$match[2]
 334             ZTST_message=${match[3]:+${match[3][2,-1]}}
 335           else
 336             ZTST_testfailed "expecting test status at:
 337 $ZTST_curline"
 338             return 1
 339           fi
 340           ZTST_getline
 341           found=1
 342           ;;
 343         ('<'*) ZTST_getredir || return 1
 344           found=1
 345           ;;
 346         ('>'*) ZTST_getredir || return 1
 347           found=1
 348           ;;
 349         ('?'*) ZTST_getredir || return 1
 350           found=1
 351           ;;
 352         ('F:'*) ZTST_failmsg="${ZTST_failmsg:+${ZTST_failmsg}
 353 }  ${ZTST_curline[3,-1]}"
 354           ZTST_getline
 355           found=1
 356           ;;
 357         (*) ZTST_testfailed "bad line in test block:
 358 $ZTST_curline"
 359           return 1
 360           ;;
 361       esac
 362     done
 363
 364     # If we found some code to execute...
 365     if [[ -n $ZTST_code ]]; then
 366       ZTST_hashmark
 367       ZTST_verbose 1 "Running test: $ZTST_message"
 368       ZTST_verbose 2 "ZTST_test: expecting status: $ZTST_xstatus"
 369       ZTST_verbose 2 "Input: $ZTST_in, output: $ZTST_out, error: $ZTST_terr"
 370
 371       ZTST_execchunk <$ZTST_in >$ZTST_tout 2>$ZTST_terr
 372
 373       # First check we got the right status, if specified.
 374       if [[ $ZTST_xstatus != - && $ZTST_xstatus != $ZTST_status ]]; then
 375         ZTST_testfailed "bad status $ZTST_status, expected $ZTST_xstatus from:
 376 $ZTST_code${$(<$ZTST_terr):+
 377 Error output:
 378 $(<$ZTST_terr)}"
 379         return 1
 380       fi
 381
 382       ZTST_verbose 2 "ZTST_test: test produced standard output:
 383 $(<$ZTST_tout)
 384 ZTST_test: and standard error:
 385 $(<$ZTST_terr)"
 386
 387       # Now check output and error.
 388       if [[ $ZTST_flags = *q* && -s $ZTST_out ]]; then
 389         substlines="$(<$ZTST_out)"
 390         rm -rf $ZTST_out
 391         print -r -- "${(e)substlines}" >$ZTST_out
 392       fi
 393       if [[ $ZTST_flags != *d* ]] && ! ZTST_diff -c $ZTST_out $ZTST_tout; then
 394         ZTST_testfailed "output differs from expected as shown above for:
 395 $ZTST_code${$(<$ZTST_terr):+
 396 Error output:
 397 $(<$ZTST_terr)}"
 398         return 1
 399       fi
 400       if [[ $ZTST_flags = *q* && -s $ZTST_err ]]; then
 401         substlines="$(<$ZTST_err)"
 402         rm -rf $ZTST_err
 403         print -r -- "${(e)substlines}" >$ZTST_err
 404       fi
 405       if [[ $ZTST_flags != *D* ]] && ! ZTST_diff -c $ZTST_err $ZTST_terr; then
 406         ZTST_testfailed "error output differs from expected as shown above for:
 407 $ZTST_code"
 408         return 1
 409       fi
 410     fi
 411     ZTST_verbose 1 "Test successful."
 412     [[ -n $last ]] && break
 413   done
 414
 415   ZTST_verbose 2 "ZTST_test: all tests successful"
 416
 417   # reset message to keep ZTST_testfailed output correct
 418   ZTST_message=''
 419 }
 420
 421
 422 # Remember which sections we've done.
 423 typeset -A ZTST_sects
 424 ZTST_sects=(prep 0 test 0 clean 0)
 425
 426 print "$ZTST_testname: starting."
 427
 428 # Now go through all the different sections until the end.
 429 # prep section may set ZTST_unimplemented, in this case the actual
 430 # tests will be skipped
 431 ZTST_skipok=
 432 ZTST_unimplemented=
 433 while [[ -z "$ZTST_unimplemented" ]] && ZTST_getsect $ZTST_skipok; do
 434   case $ZTST_cursect in
 435     (prep) if (( ${ZTST_sects[prep]} + ${ZTST_sects[test]} + \
 436                 ${ZTST_sects[clean]} )); then
 437             ZTST_testfailed "\`prep' section must come first"
 438             exit 1
 439           fi
 440           ZTST_prepclean
 441           ZTST_sects[prep]=1
 442           ;;
 443     (test)
 444           if (( ${ZTST_sects[test]} + ${ZTST_sects[clean]} )); then
 445             ZTST_testfailed "bad placement of \`test' section"
 446             exit 1
 447           fi
 448           # careful here: we can't execute ZTST_test before || or &&
 449           # because that affects the behaviour of traps in the tests.
 450           ZTST_test
 451           (( $? )) && ZTST_skipok=1
 452           ZTST_sects[test]=1
 453           ;;
 454     (clean)
 455            if (( ${ZTST_sects[test]} == 0 || ${ZTST_sects[clean]} )); then
 456              ZTST_testfailed "bad use of \`clean' section"
 457            else
 458              ZTST_prepclean 1
 459              ZTST_sects[clean]=1
 460            fi
 461            ZTST_skipok=
 462            ;;
 463     *) ZTST_testfailed "bad section name: $ZTST_cursect"
 464        ;;
 465   esac
 466 done
 467
 468 if [[ -n "$ZTST_unimplemented" ]]; then
 469   print "$ZTST_testname: skipped ($ZTST_unimplemented)"
 470   ZTST_testfailed=2
 471 elif (( ! $ZTST_testfailed )); then
 472   print "$ZTST_testname: all tests successful."
 473 fi
 474 ZTST_cleanup
 475 exit $(( ZTST_testfailed ))