test/tools/libtest.sh

   1 #!/bin/sh
   2 #
   3 # Setup test environment.
   4 #
   5 # Copyright (c) 2014 Jonas Fonseca <jonas.fonseca@gmail.com>
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation; either version 2 of
  10 # the License, or (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16
  17 set -eu
  18 if [ -n "${BASH_VERSION:-}" ]; then
  19         set -o pipefail
  20         IFS=$'\n\t'
  21 fi
  22
  23 test="$(basename "$0")"
  24 source_dir="$(cd $(dirname "$0") >/dev/null && pwd)"
  25 base_dir="$(echo "$source_dir" | sed -n 's#\(.*/test\)\([/].*\)*#\1#p')"
  26 prefix_dir="$(echo "$source_dir" | sed -n 's#\(.*/test/\)\([/].*\)*#\2#p')"
  27 output_dir="$base_dir/tmp/$prefix_dir/$test"
  28 tmp_dir="$base_dir/tmp"
  29 output_dir="$tmp_dir/$prefix_dir/$test"
  30 work_dir="work dir"
  31
  32 # The locale must specify UTF-8 for Ncurses to output correctly. Since C.UTF-8
  33 # does not exist on Mac OS X, we end up with en_US as the only sane choice.
  34 export LANG=en_US.UTF-8
  35 export LC_ALL=en_US.UTF-8
  36
  37 export PAGER=cat
  38 export TZ=UTC
  39 export TERM=dumb
  40 export HOME="$output_dir"
  41 unset CDPATH
  42 unset VISUAL
  43
  44 # Git env
  45 export GIT_CONFIG_NOSYSTEM
  46 unset GIT_CONFIG GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE
  47 unset GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_AUTHOR_DATE
  48 unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_COMMITTER_DATE
  49 unset GIT_EDITOR GIT_SEQUENCE_EDITOR GIT_PAGER GIT_EXTERNAL_DIFF
  50 unset GIT_NOTES_REF GIT_NOTES_DISPLAY_REF
  51
  52 # Tig env
  53 export TIG_TRACE=
  54 export TIGRC_SYSTEM=
  55 unset TIGRC_USER
  56
  57 # Ncurses env
  58 export ESCDELAY=200
  59 export LINES=30
  60 export COLUMNS=80
  61
  62 # Internal test env
  63 # A comma-separated list of options. See docs in test/README.
  64 export TEST_OPTS="${TEST_OPTS:-}"
  65 # Used by tig_script to set the test "scope" used by test_tig.
  66 export TEST_NAME=
  67
  68 [ -e "$output_dir" ] && rm -rf "$output_dir"
  69 mkdir -p "$output_dir/$work_dir"
  70
  71 if [ ! -d "$tmp_dir/.git" ]; then
  72         # Create a dummy repository to avoid reading .git/config
  73         # settings from the tig repository.
  74         git init -q "$tmp_dir"
  75 fi
  76
  77 # For any utilities used in Tig scripts
  78 BIN_DIR="$HOME/bin"
  79 mkdir -p "$BIN_DIR"
  80 export PATH="$BIN_DIR:$PATH"
  81
  82 executable() {
  83         path="$BIN_DIR/$1"; shift
  84
  85         if [ "$#" = 0 ]; then
  86                 case "$path" in
  87                         stdin|expected*) cat ;;
  88                         *) sed 's/^[ ]//' ;;
  89                 esac > "$path"
  90         else
  91                 printf '%s' "$@" > "$path"
  92         fi
  93         chmod +x "$path"
  94 }
  95
  96 # Setup fake editor
  97 export EDITOR="vim"
  98 executable 'vim' <<EOF
  99 #!/bin/sh
 100
 101 file="\$1"
 102 lineno="\$(expr "\$1" : '+\([0-9]*\)')"
 103 if [ -n "\$lineno" ]; then
 104         file="\$2"
 105 fi
 106
 107 echo "\$@" >> "$HOME/editor.log"
 108 sed -n -e "\${lineno}p" "\$file" >> "$HOME/editor.log" 2>&1
 109 EOF
 110
 111 cd "$output_dir"
 112
 113 #
 114 # Utilities.
 115 #
 116
 117 die()
 118 {
 119         echo >&2 "$*"
 120         exit 1
 121 }
 122
 123 file() {
 124         path="$1"; shift
 125
 126         mkdir -p "$(dirname "$path")"
 127         if [ "$#" = 0 ]; then
 128                 case "$path" in
 129                         stdin|expected*) cat ;;
 130                         *) sed 's/^[ ]//' ;;
 131                 esac > "$path"
 132         else
 133                 printf '%s' "$@" > "$path"
 134         fi
 135 }
 136
 137 tig_script() {
 138         name="$1"; shift
 139         prefix="${name:+$name.}"
 140
 141         export TIG_SCRIPT="$HOME/${prefix}steps"
 142         export TEST_NAME="$name"
 143
 144         # Ensure that the steps finish by quitting
 145         printf '%s\n:quit\n' "$@" \
 146                 | sed -e 's/^[  ]*//' -e '/^$/d' \
 147                 | sed "s|:save-display\s\+\(\S*\)|:save-display $HOME/\1|" \
 148                 > "$TIG_SCRIPT"
 149 }
 150
 151 steps() {
 152         tig_script "" "$@"
 153 }
 154
 155 stdin() {
 156         file "stdin" "$@"
 157 }
 158
 159 tigrc() {
 160         file "$HOME/.tigrc" "$@"
 161 }
 162
 163 gitconfig() {
 164         file "$HOME/.gitconfig" "$@"
 165 }
 166
 167 in_work_dir()
 168 {
 169         (cd "$work_dir" && $@)
 170 }
 171
 172 auto_detect_debugger() {
 173         for dbg in gdb lldb; do
 174                 dbg="$(command -v "$dbg" 2>/dev/null || true)"
 175                 if [ -n "$dbg" ]; then
 176                         echo "$dbg"
 177                         return
 178                 fi
 179         done
 180
 181         die "Failed to detect a supported debugger"
 182 }
 183
 184 #
 185 # Parse TEST_OPTS
 186 #
 187
 188 expected_status_code=0
 189 diff_color_arg=
 190 [ -t 1 ] && diff_color_arg=--color
 191
 192 indent='            '
 193 verbose=
 194 debugger=
 195 trace=
 196 valgrind=
 197
 198 set -- $TIG_TEST_OPTS $TEST_OPTS
 199
 200 while [ $# -gt 0 ]; do
 201         arg="$1"; shift
 202         case "$arg" in
 203                 verbose) verbose=yes ;;
 204                 no-indent) indent= ;;
 205                 debugger=*) debugger=$(expr "$arg" : 'debugger=\(.*\)') ;;
 206                 debugger) debugger="$(auto_detect_debugger)" ;;
 207                 trace) trace=yes ;;
 208                 valgrind) valgrind="$HOME/valgrind.log" ;;
 209         esac
 210 done
 211
 212 #
 213 # Test runners and assertion checking.
 214 #
 215
 216 assert_equals()
 217 {
 218         file="$1"; shift
 219
 220         if [ ! -s "expected/$file" ]; then
 221                 file "expected/$file"
 222         fi
 223
 224         if [ -e "$file" ]; then
 225                 git diff -w --no-index $diff_color_arg "expected/$file" "$file" > "$file.diff" || true
 226                 if [ -s "$file.diff" ]; then
 227                         echo "[FAIL] $file != expected/$file" >> .test-result
 228                         while [ $# -gt 0 ]; do
 229                                 msg="$1"; shift
 230                                 echo "[NOTE] $msg" >> .test-result
 231                         done
 232                         cat "$file.diff" >> .test-result
 233                 else
 234                         echo "  [OK] $file assertion" >> .test-result
 235                 fi
 236                 rm -f "$file.diff"
 237         else
 238                 echo "[FAIL] $file not found" >> .test-result
 239         fi
 240 }
 241
 242 assert_not_exists()
 243 {
 244         file="$1"; shift
 245
 246         if [ -e "$file" ]; then
 247                 echo "[FAIL] $file should not exist" >> .test-result
 248         else
 249                 echo "  [OK] $file does not exist" >> .test-result
 250         fi
 251 }
 252
 253 vars_file="vars"
 254 expected_var_file="$HOME/expected/$vars_file"
 255
 256 executable 'assert-var' <<EOF
 257 #!/bin/sh
 258
 259 mkdir -p "$(dirname "$expected_var_file")"
 260 while [ \$# -gt 0 ]; do
 261         arg="\$1"; shift
 262
 263         case "\$arg" in
 264                 ==) break ;;
 265                 *)  echo "\$arg" >> "$HOME/$vars_file" ;;
 266         esac
 267 done
 268 echo "\$@" >> "$expected_var_file"
 269 EOF
 270
 271 assert_vars()
 272 {
 273         if [ -e "$expected_var_file" ]; then
 274                 assert_equals "$vars_file" < "$expected_var_file"
 275         else
 276                 echo "[FAIL] $expected_var_file not found" >> .test-result
 277         fi
 278 }
 279
 280 show_test_results()
 281 {
 282         if [ -e .test-skipped ]; then
 283                 sed "s/^/$indent[skipped] /" < .test-skipped
 284                 return
 285         fi
 286         if [ -n "$trace" -a -n "$TIG_TRACE" -a -e "$TIG_TRACE" ]; then
 287                 sed "s/^/$indent[trace] /" < "$TIG_TRACE"
 288         fi
 289         if [ -n "$valgrind" -a -e "$valgrind" ]; then
 290                 sed "s/^/$indent[valgrind] /" < "$valgrind"
 291         fi
 292         if [ ! -d "$HOME" ] || [ ! -e .test-result ]; then
 293                 [ -e stderr ] &&
 294                         sed "s/^/[stderr] /" < stderr
 295                 [ -e stderr.orig ] &&
 296                         sed "s/^/[stderr] /" < stderr.orig
 297                 echo "No test results found"
 298         elif grep FAIL -q < .test-result; then
 299                 failed="$(grep FAIL < .test-result | wc -l)"
 300                 count="$(sed -n '/\(FAIL\|OK\)/p' < .test-result | wc -l)"
 301
 302                 printf "Failed %d out of %d test(s)%s\n" $failed $count
 303
 304                 # Show output from stderr if no output is expected
 305                 if [ -e stderr ]; then
 306                         [ -e expected/stderr ] ||
 307                                 sed "s/^/[stderr] /" < stderr
 308                 fi
 309
 310                 # Replace CR used by Git progress messages
 311                 tr '\r' '\n' < .test-result
 312         elif [ "$verbose" ]; then
 313                 count="$(sed -n '/\(OK\)/p' < .test-result | wc -l)"
 314                 printf "Passed %d assertions\n" $count
 315         fi | sed "s/^/$indent| /"
 316 }
 317
 318 trap 'show_test_results' EXIT
 319
 320 test_skip()
 321 {
 322         echo "$@" >> .test-skipped
 323 }
 324
 325 require_git_version()
 326 {
 327         git_version="$(git version | sed 's/git version \([0-9\.]*\).*/\1/')"
 328         actual_major="$(expr "$git_version" : '\([0-9]*\).*')"
 329         actual_minor="$(expr "$git_version" : '[0-9]*\.\([0-9]*\).*')"
 330
 331         required_version="$1"; shift
 332         required_major="$(expr "$required_version" : '\([0-9]*\).*')"
 333         required_minor="$(expr "$required_version" : '[0-9]*\.\([0-9]*\).*')"
 334
 335         if [ "$required_major" -gt "$actual_major" ] ||
 336            [ "$required_minor" -gt "$actual_minor" ]; then
 337                 test_skip "$@"
 338         fi
 339 }
 340
 341 test_require()
 342 {
 343         while [ $# -gt 0 ]; do
 344                 feature="$1"; shift
 345
 346                 case "$feature" in
 347                 git-worktree)
 348                         require_git_version 2.5 \
 349                                 "The test requires git-worktree, available in git version 2.5 or newer"
 350                         ;;
 351                 address-sanitizer)
 352                         if [ "${TIG_ADDRESS_SANITIZER_ENABLED:-no}" != yes ]; then
 353                                 test_skip "The test requires clang and is only run via \`make test-address-sanitizer\`"
 354                         fi
 355                         ;;
 356                 *)
 357                         test_skip "Unknown feature requirement: $feature"
 358                 esac
 359         done
 360 }
 361
 362 test_exec_work_dir()
 363 {
 364         echo "=== $@ ===" >> "$HOME/test-exec.log"
 365         test_exec_log="$HOME/test-exec.log.tmp"
 366         rm -f "$test_exec_log"
 367
 368         set +e
 369         in_work_dir "$@" 1>>"$test_exec_log" 2>>"$test_exec_log"
 370         test_exec_exit_code=$?
 371         set -e
 372
 373         cat "$test_exec_log" >> "$HOME/test-exec.log"
 374         if [ "$test_exec_exit_code" != 0 ]; then
 375                 cmd="$@"
 376                 echo "[FAIL] unexpected exit code while executing '$cmd': $test_exec_exit_code" >> .test-result
 377                 cat "$test_exec_log" >> .test-result
 378                 # Exit gracefully to allow additional tests to run
 379                 exit 0
 380         fi
 381 }
 382
 383 test_setup()
 384 {
 385         if [ -e .test-skipped ]; then
 386                 exit 0
 387         fi
 388
 389         run_setup="$(type test_setup_work_dir 2>/dev/null | grep 'function' || true)"
 390
 391         if [ -n "$run_setup" ]; then
 392                 if test ! -e "$HOME/test-exec.log" || ! grep -q test_setup_work_dir "$HOME/test-exec.log"; then
 393                         test_exec_work_dir test_setup_work_dir
 394                 fi
 395         fi
 396 }
 397
 398 valgrind_exec()
 399 {
 400         kernel="$(uname -s 2>/dev/null || echo unknown)"
 401         kernel_supp="test/tools/valgrind-$kernel.supp"
 402
 403         valgrind_ops=
 404         if [ -e "$kernel_supp" ]; then
 405                 valgrind_ops="$valgrind_ops --suppresions='$kernel_supp'"
 406         fi
 407
 408         valgrind -q --gen-suppressions=all --track-origins=yes --error-exitcode=1 \
 409                 --log-file="$valgrind.orig" $valgrind_ops \
 410                 "$@"
 411
 412         case "$kernel" in
 413         Darwin) grep -v "mach_msg unhandled MACH_SEND_TRAILER option" < "$valgrind.orig" > "$valgrind" ;;
 414         *)      mv "$valgrind.orig" "$valgrind" ;;
 415         esac
 416
 417         rm -f "$valgrind.orig"
 418 }
 419
 420 test_tig()
 421 {
 422         name="$TEST_NAME"
 423         prefix="${name:+$name.}"
 424
 425         test_setup
 426         export TIG_NO_DISPLAY=
 427         if [ -n "$trace" ]; then
 428                 export TIG_TRACE="$HOME/${prefix}tig-trace"
 429         fi
 430         touch "${prefix}stdin" "${prefix}stderr"
 431         if [ -n "$debugger" ]; then
 432                 echo "*** Running tests in '$(pwd)/$work_dir'"
 433                 if [ -s "$work_dir/${prefix}stdin" ]; then
 434                         echo "*** - This test requires data to be injected via stdin."
 435                         echo "***   The expected input file is: '../${prefix}stdin'"
 436                 fi
 437                 if [ "$#" -gt 0 ]; then
 438                         echo "*** - This test expects the following arguments: $@"
 439                 fi
 440                 (cd "$work_dir" && $debugger tig "$@")
 441         else
 442                 set +e
 443                 runner=
 444                 # FIXME: Tell Valgrind to forward status code
 445                 if [ "$expected_status_code" = 0 -a -n "$valgrind" ]; then
 446                         runner=valgrind_exec
 447                 fi
 448                 if [ -s "${prefix}stdin" ]; then
 449                         (cd "$work_dir" && $runner tig "$@") < "${prefix}stdin" > "${prefix}stdout" 2> "${prefix}stderr.orig"
 450                 else
 451                         (cd "$work_dir" && $runner tig "$@") > "${prefix}stdout" 2> "${prefix}stderr.orig"
 452                 fi
 453                 status_code="$?"
 454                 if [ "$status_code" != "$expected_status_code" ]; then
 455                         echo "[FAIL] unexpected status code: $status_code (should be $expected_status_code)" >> .test-result
 456                 fi
 457                 set -e
 458         fi
 459         # Normalize paths in stderr output
 460         if [ -e "${prefix}stderr.orig" ]; then
 461                 sed "s#$output_dir#HOME#" < "${prefix}stderr.orig" > "${prefix}stderr"
 462                 rm -f "${prefix}stderr.orig"
 463         fi
 464         if [ -n "$trace" ]; then
 465                 export TIG_TRACE="$HOME/.tig-trace"
 466                 if [ -n "$name" ]; then
 467                         sed "s#^#[$name] #" < "$HOME/${prefix}tig-trace" >> "$HOME/.tig-trace"
 468                 else
 469                         mv "$HOME/${prefix}tig-trace" "$HOME/.tig-trace"
 470                 fi
 471         fi
 472         if [ -n "$prefix" ]; then
 473                 sed "s#^#[$name] #" < "${prefix}stderr" >> "stderr"
 474         fi
 475 }
 476
 477 test_graph()
 478 {
 479         test-graph $@ > stdout 2> stderr.orig
 480 }
 481
 482 test_case()
 483 {
 484         name="$1"; shift
 485
 486         echo "$name" >> test-cases
 487         cat > "$name.expected"
 488
 489         touch "$name-before" "$name-after" "$name.script" "$name-args"
 490
 491         while [ "$#" -gt 0 ]; do
 492                 arg="$1"; shift
 493                 key="$(expr "X$arg" : 'X--\([^=]*\).*')"
 494                 value="$(expr "X$arg" : 'X--[^=]*=\(.*\)')"
 495
 496                 case "$key" in
 497                 before|after|script|args|cwd)
 498                         echo "$value" > "$name-$key" ;;
 499                 *)      die "Unknown test_case argument: $arg"
 500                 esac
 501         done
 502 }
 503
 504 run_test_cases()
 505 {
 506         if [ ! -e test-cases ]; then
 507                 return
 508         fi
 509         test_setup
 510         for name in $(cat test-cases); do
 511                 tig_script "$name" "
 512                         $(if [ -e "$name-script" ]; then cat "$name-script"; fi)
 513                         :save-display $name.screen
 514                 "
 515                 if [ -e "$name-before" ]; then
 516                         test_exec_work_dir "$SHELL" "$HOME/$name-before"
 517                 fi
 518                 old_work_dir="$work_dir"
 519                 if [ -e "$name-cwd" ]; then
 520                         work_dir="$work_dir/$(cat "$name-cwd")"
 521                 fi
 522                 ORIG_IFS="$IFS"
 523                 IFS=$' '
 524                 test_tig $(if [ -e "$name-args" ]; then cat "$name-args"; fi)
 525                 IFS="$ORIG_IFS"
 526                 work_dir="$old_work_dir"
 527                 if [ -e "$name-after" ]; then
 528                         test_exec_work_dir "$SHELL" "$HOME/$name-after"
 529                 fi
 530
 531                 assert_equals "$name.screen" < "$name.expected"
 532                 assert_equals "$name.stderr" < /dev/null
 533         done
 534 }