test/tools/libtest.sh

   1 #!/bin/sh
   2 #
   3 # Setup test environment.
   4 #
   5 # Copyright (c) 2014 Jonas Fonseca <jonas.fonseca@gmail.com>
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation; either version 2 of
  10 # the License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 set -eu
  18 if [ -n "${BASH_VERSION:-}" ]; then
  19         set -o pipefail
  20         IFS=$'\n\t'
  21 fi
  22
  23 test="$(basename "$0")"
  24 source_dir="$(cd "$(dirname "$0")" && pwd)"
  25 base_dir="$(echo "$source_dir" | sed -n 's#\(.*/test\)\([/].*\)*#\1#p')"
  26 prefix_dir="$(echo "$source_dir" | sed -n 's#\(.*/test/\)\([/].*\)*#\2#p')"
  27 output_dir="$base_dir/tmp/$prefix_dir/$test"
  28 tmp_dir="$base_dir/tmp"
  29 output_dir="$tmp_dir/$prefix_dir/$test"
  30 work_dir="work dir"
  31
  32 # The locale must specify UTF-8 for Ncurses to output correctly. Since C.UTF-8
  33 # does not exist on Mac OS X, we end up with en_US as the only sane choice.
  34 export LANG=en_US.UTF-8
  35 export LC_ALL=en_US.UTF-8
  36
  37 export PAGER=cat
  38 export TZ=UTC
  39 export TERM=dumb
  40 export HOME="$output_dir"
  41 unset CDPATH
  42 unset VISUAL
  43
  44 # Git env
  45 export GIT_CONFIG_NOSYSTEM
  46 unset GIT_CONFIG GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE
  47 unset GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_AUTHOR_DATE
  48 unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_COMMITTER_DATE
  49 unset GIT_EDITOR GIT_SEQUENCE_EDITOR GIT_PAGER GIT_EXTERNAL_DIFF
  50 unset GIT_NOTES_REF GIT_NOTES_DISPLAY_REF
  51
  52 # Tig env
  53 export TIG_TRACE=
  54 export TIGRC_SYSTEM=
  55 unset TIGRC_USER
  56
  57 # Ncurses env
  58 export ESCDELAY=200
  59 export LINES=30
  60 export COLUMNS=80
  61
  62 # Internal test env
  63 # A comma-separated list of options. See docs in test/README.
  64 export TEST_OPTS="${TEST_OPTS:-}"
  65 # Used by tig_script to set the test "scope" used by test_tig.
  66 export TEST_NAME=
  67
  68 [ -e "$output_dir" ] && rm -rf "$output_dir"
  69 mkdir -p "$output_dir/$work_dir"
  70
  71 if [ ! -d "$tmp_dir/.git" ]; then
  72         # Create a dummy repository to avoid reading .git/config
  73         # settings from the tig repository.
  74         git init -q "$tmp_dir"
  75 fi
  76
  77 # For any utilities used in Tig scripts
  78 BIN_DIR="$HOME/bin"
  79 mkdir -p "$BIN_DIR"
  80 export PATH="$BIN_DIR:$PATH"
  81
  82 executable() {
  83         path="$BIN_DIR/$1"; shift
  84
  85         if [ "$#" = 0 ]; then
  86                 case "$path" in
  87                         stdin|expected*) cat ;;
  88                         *) sed 's/^[ ]//' ;;
  89                 esac > "$path"
  90         else
  91                 printf '%s' "$@" > "$path"
  92         fi
  93         chmod +x "$path"
  94 }
  95
  96 # Setup fake editor
  97 export EDITOR="vim"
  98 executable 'vim' <<EOF
  99 #!/bin/sh
 100
 101 file="\$1"
 102 lineno="\$(expr "\$1" : '+\([0-9]*\)')"
 103 if [ -n "\$lineno" ]; then
 104         file="\$2"
 105 fi
 106
 107 echo "\$@" >> "$HOME/editor.log"
 108 sed -n -e "\${lineno}p" "\$file" >> "$HOME/editor.log"
 109 EOF
 110
 111 cd "$output_dir"
 112
 113 #
 114 # Utilities.
 115 #
 116
 117 die()
 118 {
 119         echo >&2 "$*"
 120         exit 1
 121 }
 122
 123 file() {
 124         path="$1"; shift
 125
 126         mkdir -p "$(dirname "$path")"
 127         if [ "$#" = 0 ]; then
 128                 case "$path" in
 129                         stdin|expected*) cat ;;
 130                         *) sed 's/^[ ]//' ;;
 131                 esac > "$path"
 132         else
 133                 printf '%s' "$@" > "$path"
 134         fi
 135 }
 136
 137 tig_script() {
 138         name="$1"; shift
 139         prefix="${name:+$name.}"
 140
 141         export TIG_SCRIPT="$HOME/${prefix}steps"
 142         export TEST_NAME="$name"
 143
 144         # Ensure that the steps finish by quitting
 145         printf '%s\n:quit\n' "$@" \
 146                 | sed -e 's/^[  ]*//' -e '/^$/d' \
 147                 | sed "s|:save-display\s\+\(\S*\)|:save-display $HOME/\1|" \
 148                 > "$TIG_SCRIPT"
 149 }
 150
 151 steps() {
 152         tig_script "" "$@"
 153 }
 154
 155 stdin() {
 156         file "stdin" "$@"
 157 }
 158
 159 tigrc() {
 160         file "$HOME/.tigrc" "$@"
 161 }
 162
 163 gitconfig() {
 164         file "$HOME/.gitconfig" "$@"
 165 }
 166
 167 in_work_dir()
 168 {
 169         (cd "$work_dir" && $@)
 170 }
 171
 172 auto_detect_debugger() {
 173         for dbg in gdb lldb; do
 174                 dbg="$(command -v "$dbg" 2>/dev/null || true)"
 175                 if [ -n "$dbg" ]; then
 176                         echo "$dbg"
 177                         return
 178                 fi
 179         done
 180
 181         die "Failed to detect a supported debugger"
 182 }
 183
 184 #
 185 # Parse TEST_OPTS
 186 #
 187
 188 expected_status_code=0
 189 diff_color_arg=
 190 [ -t 1 ] && diff_color_arg=--color
 191
 192 indent='            '
 193 verbose=
 194 debugger=
 195 trace=
 196 valgrind=
 197
 198 set -- $TIG_TEST_OPTS $TEST_OPTS
 199
 200 while [ $# -gt 0 ]; do
 201         arg="$1"; shift
 202         case "$arg" in
 203                 verbose) verbose=yes ;;
 204                 no-indent) indent= ;;
 205                 debugger=*) debugger=$(expr "$arg" : 'debugger=\(.*\)') ;;
 206                 debugger) debugger="$(auto_detect_debugger)" ;;
 207                 trace) trace=yes ;;
 208                 valgrind) valgrind="$HOME/valgrind.log" ;;
 209         esac
 210 done
 211
 212 #
 213 # Test runners and assertion checking.
 214 #
 215
 216 assert_equals()
 217 {
 218         file="$1"; shift
 219
 220         file "expected/$file" "$@"
 221
 222         if [ -e "$file" ]; then
 223                 git diff -w --no-index $diff_color_arg "expected/$file" "$file" > "$file.diff" || true
 224                 if [ -s "$file.diff" ]; then
 225                         echo "[FAIL] $file != expected/$file" >> .test-result
 226                         cat "$file.diff" >> .test-result
 227                 else
 228                         echo "  [OK] $file assertion" >> .test-result
 229                 fi
 230                 rm -f "$file.diff"
 231         else
 232                 echo "[FAIL] $file not found" >> .test-result
 233         fi
 234 }
 235
 236 assert_not_exists()
 237 {
 238         file="$1"; shift
 239
 240         if [ -e "$file" ]; then
 241                 echo "[FAIL] $file should not exist" >> .test-result
 242         else
 243                 echo "  [OK] $file does not exist" >> .test-result
 244         fi
 245 }
 246
 247 vars_file="vars"
 248 expected_var_file="$HOME/expected/$vars_file"
 249
 250 executable 'assert-var' <<EOF
 251 #!/bin/sh
 252
 253 mkdir -p "$(dirname "$expected_var_file")"
 254 echo "\$1" >> "$expected_var_file"
 255 echo "\$3" >> "$HOME/$vars_file"
 256 EOF
 257
 258 assert_vars()
 259 {
 260         if [ -e "$expected_var_file" ]; then
 261                 assert_equals "$vars_file" "$(cat "$expected_var_file")"
 262         else
 263                 echo "[FAIL] $expected_var_file not found" >> .test-result
 264         fi
 265 }
 266
 267 show_test_results()
 268 {
 269         if [ -n "$trace" -a -n "$TIG_TRACE" -a -e "$TIG_TRACE" ]; then
 270                 sed "s/^/$indent[trace] /" < "$TIG_TRACE"
 271         fi
 272         if [ -n "$valgrind" -a -e "$valgrind" ]; then
 273                 sed "s/^/$indent[valgrind] /" < "$valgrind"
 274         fi
 275         if [ ! -d "$HOME" ]; then
 276                 echo "Skipped"
 277         elif [ ! -e .test-result ]; then
 278                 [ -e stderr ] &&
 279                         sed "s/^/[stderr] /" < stderr
 280                 [ -e stderr.orig ] &&
 281                         sed "s/^/[stderr] /" < stderr.orig
 282                 echo "No test results found"
 283         elif grep FAIL -q < .test-result; then
 284                 failed="$(grep FAIL < .test-result | wc -l)"
 285                 count="$(sed -n '/\(FAIL\|OK\)/p' < .test-result | wc -l)"
 286
 287                 printf "Failed %d out of %d test(s)%s\n" $failed $count
 288
 289                 # Show output from stderr if no output is expected
 290                 if [ -e stderr ]; then
 291                         [ -e expected/stderr ] ||
 292                                 sed "s/^/[stderr] /" < stderr
 293                 fi
 294
 295                 [ -e .test-result ] &&
 296                         cat .test-result
 297         elif [ "$verbose" ]; then
 298                 count="$(sed -n '/\(OK\)/p' < .test-result | wc -l)"
 299                 printf "Passed %d assertions\n" $count
 300         fi | sed "s/^/$indent| /"
 301 }
 302
 303 trap 'show_test_results' EXIT
 304
 305 test_exec_work_dir()
 306 {
 307         echo "=== $@ ===" >> "$HOME/test-exec.log"
 308         set +e
 309         in_work_dir "$@" 1>>"$HOME/test-exec.log" 2>>"$HOME/test-exec.log"
 310         set -e
 311 }
 312
 313 test_setup()
 314 {
 315         run_setup="$(type test_setup_work_dir 2>/dev/null | grep 'function' || true)"
 316
 317         if [ -n "$run_setup" ]; then
 318                 if test ! -e "$HOME/test-exec.log" || ! grep -q test_setup_work_dir "$HOME/test-exec.log"; then
 319                         test_exec_work_dir test_setup_work_dir
 320                 fi
 321         fi
 322 }
 323
 324 valgrind_exec()
 325 {
 326         kernel="$(uname -s 2>/dev/null || echo unknown)"
 327         kernel_supp="test/tools/valgrind-$kernel.supp"
 328
 329         valgrind_ops=
 330         if [ -e "$kernel_supp" ]; then
 331                 valgrind_ops="$valgrind_ops --suppresions='$kernel_supp'"
 332         fi
 333
 334         valgrind -q --gen-suppressions=all --track-origins=yes --error-exitcode=1 \
 335                 --log-file="$valgrind.orig" $valgrind_ops \
 336                 "$@"
 337
 338         case "$kernel" in
 339         Darwin) grep -v "mach_msg unhandled MACH_SEND_TRAILER option" < "$valgrind.orig" > "$valgrind" ;;
 340         *)      mv "$valgrind.orig" "$valgrind" ;;
 341         esac
 342
 343         rm -f "$valgrind.orig"
 344 }
 345
 346 test_tig()
 347 {
 348         name="$TEST_NAME"
 349         prefix="${name:+$name.}"
 350
 351         test_setup
 352         export TIG_NO_DISPLAY=
 353         if [ -n "$trace" ]; then
 354                 export TIG_TRACE="$HOME/${prefix}tig-trace"
 355         fi
 356         touch "${prefix}stdin" "${prefix}stderr"
 357         if [ -n "$debugger" ]; then
 358                 echo "*** Running tests in '$(pwd)/$work_dir'"
 359                 if [ -s "$work_dir/${prefix}stdin" ]; then
 360                         echo "*** - This test requires data to be injected via stdin."
 361                         echo "***   The expected input file is: '../${prefix}stdin'"
 362                 fi
 363                 if [ "$#" -gt 0 ]; then
 364                         echo "*** - This test expects the following arguments: $@"
 365                 fi
 366                 (cd "$work_dir" && $debugger tig "$@")
 367         else
 368                 set +e
 369                 runner=
 370                 # FIXME: Tell Valgrind to forward status code
 371                 if [ "$expected_status_code" = 0 -a -n "$valgrind" ]; then
 372                         runner=valgrind_exec
 373                 fi
 374                 if [ -s "${prefix}stdin" ]; then
 375                         (cd "$work_dir" && $runner tig "$@") < "${prefix}stdin" > "${prefix}stdout" 2> "${prefix}stderr.orig"
 376                 else
 377                         (cd "$work_dir" && $runner tig "$@") > "${prefix}stdout" 2> "${prefix}stderr.orig"
 378                 fi
 379                 status_code="$?"
 380                 if [ "$status_code" != "$expected_status_code" ]; then
 381                         echo "[FAIL] unexpected status code: $status_code (should be $expected_status_code)" >> .test-result
 382                 fi
 383                 set -e
 384         fi
 385         # Normalize paths in stderr output
 386         if [ -e "${prefix}stderr.orig" ]; then
 387                 sed "s#$output_dir#HOME#" < "${prefix}stderr.orig" > "${prefix}stderr"
 388                 rm -f "${prefix}stderr.orig"
 389         fi
 390         if [ -n "$trace" ]; then
 391                 export TIG_TRACE="$HOME/.tig-trace"
 392                 if [ -n "$name" ]; then
 393                         sed "s#^#[$name] #" < "$HOME/${prefix}tig-trace" >> "$HOME/.tig-trace"
 394                 else
 395                         mv "$HOME/${prefix}tig-trace" "$HOME/.tig-trace"
 396                 fi
 397         fi
 398         if [ -n "$prefix" ]; then
 399                 sed "s#^#[$name] #" < "${prefix}stderr" >> "stderr"
 400         fi
 401 }
 402
 403 test_graph()
 404 {
 405         test-graph $@ > stdout 2> stderr.orig
 406 }
 407
 408 test_case()
 409 {
 410         name="$1"; shift
 411
 412         echo "$name" >> test-cases
 413         cat > "$name.expected"
 414
 415         touch "$name-before.sh" "$name-after.sh" "$name.script" "$name-args"
 416
 417         while [ "$#" -gt 0 ]; do
 418                 arg="$1"; shift
 419                 value="$(expr "$arg" : '--[^=]*=\(.*\)')"
 420
 421                 case "$arg" in
 422                 --before=*) echo "$value" > "$name-before.sh" ;;
 423                 --after=*)  echo "$value" > "$name-after.sh" ;;
 424                 --script=*) echo "$value" > "$name.script" ;;
 425                 --args=*)   echo "$value" > "$name-args" ;;
 426                 --cwd=*)    echo "$value" > "$name-cwd" ;;
 427                 *)          die "Unknown test_case argument: $arg"
 428                 esac
 429         done
 430 }
 431
 432 run_test_cases()
 433 {
 434         if [ ! -e test-cases ]; then
 435                 return
 436         fi
 437         test_setup
 438         for name in $(cat test-cases); do
 439                 tig_script "$name" "
 440                         $(if [ -e "$name.script" ]; then cat "$name.script"; fi)
 441                         :save-display $name.screen
 442                 "
 443                 if [ -e "$name-before.sh" ]; then
 444                         test_exec_work_dir sh "$HOME/$name-before.sh"
 445                 fi
 446                 old_work_dir="$work_dir"
 447                 if [ -e "$name-cwd" ]; then
 448                         work_dir="$work_dir/$(cat "$name-cwd")"
 449                 fi
 450                 test_tig $(if [ -e "$name-args" ]; then cat "$name-args"; fi)
 451                 work_dir="$old_work_dir"
 452                 if [ -e "$name-after.sh" ]; then
 453                         test_exec_work_dir sh "$HOME/$name-after.sh"
 454                 fi
 455
 456                 assert_equals "$name.screen" < "$name.expected"
 457                 assert_equals "$name.stderr" ''
 458         done
 459 }