util/romcc/test.sh

   1 #!/bin/sh
   2 #
   3 # This file is part of the coreboot project.
   4 #
   5 # Copyright 2016 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; version 2 of the License.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16
  17 # These tests are currently known to be broken and should not result in a
  18 # build failure:
  19 XFAIL_TESTS="
  20         simple_test4.c
  21         simple_test6.c
  22         simple_test25.c
  23         simple_test26.c
  24         simple_test46.c
  25         simple_test47.c
  26         simple_test54.c
  27         simple_test72.c
  28         simple_test73.c
  29         linux_test2.c
  30         linux_test5.c
  31         linux_test10.c
  32         linux_test11.c
  33         linux_test12.c
  34 "
  35
  36 # ------------------------------------------------------------------- #
  37
  38 BASEDIR="$(dirname "$0")"
  39 BUILDDIR="$BASEDIR/build"
  40 LOGDIR="$BUILDDIR/logs"
  41 mkdir -p "$BUILDDIR"
  42 mkdir -p "$LOGDIR"
  43
  44 usage () {
  45         echo "Usage: test.sh [--nocolor] CLASS"
  46         echo ""
  47         echo "CLASS selects a group of tests to run. It must be one of the following:"
  48         echo "  all     - all tests"
  49         echo "  simple  - simple tests"
  50         echo "  linux   - linux programs whose output is checked against a reference"
  51         echo ""
  52         echo "--nocolor disables colors."
  53         exit 1
  54 }
  55
  56 COLORS=1
  57 if [ "$1" = "--nocolor" ]; then
  58         shift
  59         COLORS=0
  60 fi
  61
  62
  63 if [ -t 1 -a "$COLORS" -eq 1 ]; then
  64         red()   { printf "\033[1;31m%s\033[0m" "$*"; }
  65         green() { printf "\033[1;32m%s\033[0m" "$*"; }
  66         blue()  { printf "\033[1;34m%s\033[0m" "$*"; }
  67 else
  68         red()   { printf "%s" "$*"; }
  69         green() { printf "%s" "$*"; }
  70         blue()  { printf "%s" "$*"; }
  71 fi
  72
  73 init_stats() {
  74         NUM_TOTAL=0     # Number of tests that were run
  75         NUM_FAIL=0      # Number of tests that failed unexpectedly
  76         NUM_BROKEN=0    # Number of tests that failed expectedly
  77         NUM_PASS=0      # Number of tests that passed expectedly
  78         NUM_FIXED=0     # Number of tests that passed unexpectedly
  79 }
  80
  81 get_romcc() {
  82         ROMCC="$BUILDDIR/romcc"
  83         if [ ! -f "$ROMCC" ]; then
  84                 echo "romcc not found! Please run \"make\"."
  85                 exit 1
  86         fi
  87 }
  88
  89 init_testing() {
  90         init_stats
  91         get_romcc
  92 }
  93
  94 show_stats() {
  95         printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED
  96         printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN
  97         printf "total:  %s\n" $NUM_TOTAL
  98 }
  99
 100 is_xfail() {
 101         local t
 102         for t in $XFAIL_TESTS; do
 103                 if [ "$t" = "$1" ]; then
 104                         return 0
 105                 fi
 106         done
 107         return 1
 108 }
 109
 110 pass() {
 111         NUM_TOTAL=$((NUM_TOTAL + 1))
 112         NUM_PASS=$((NUM_PASS + 1))
 113
 114         green "passed"
 115         if is_xfail "$(basename "$1")"; then
 116                 blue " (fixed)"
 117                 NUM_FIXED=$((NUM_FIXED + 1))
 118         fi
 119         echo
 120 }
 121
 122 fail() {
 123         NUM_TOTAL=$((NUM_TOTAL + 1))
 124         NUM_FAIL=$((NUM_FAIL + 1))
 125
 126         red "failed"
 127         if is_xfail "$(basename "$1")"; then
 128                 blue " (known broken)"
 129                 NUM_BROKEN=$((NUM_BROKEN + 1))
 130         fi
 131         echo
 132 }
 133
 134 run_simple_test() {
 135         # TODO: "timeout" is not POSIX compliant. Use something that is.
 136         timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S"
 137 }
 138
 139 run_simple_tests() {
 140         echo "Running simple tests..."
 141
 142         local t
 143         for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do
 144                 printf "%s" "$(basename "$t")"
 145
 146                 local result=pass
 147                 local logfile="$LOGDIR/$(basename "$t").log"
 148                 rm "$logfile" >/dev/null 2>&1
 149                 for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \
 150                            "-O -mmmx" "-O -msse" "-O  -mmmx -msse" \
 151                            "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do
 152                         if run_simple_test "$opt" "$t" \
 153                                         >> "$logfile" 2>&1; then
 154                                 printf .
 155                         else
 156                                 result=fail
 157                                 break
 158                         fi
 159                 done
 160                 printf " "
 161                 $result "$t"
 162         done
 163
 164         echo
 165 }
 166
 167 run_linux_test() {
 168         local base="$(basename "$1")"
 169
 170         # TODO: "timeout" is not POSIX compliant. Use something that is.
 171
 172         timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1
 173         as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1
 174         ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \
 175                 "$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1
 176         timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1
 177
 178         diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out"
 179 }
 180
 181 run_linux_tests() {
 182         echo "Running linux tests..."
 183
 184         local t
 185         for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do
 186                 printf "%s... " "$(basename "$t")"
 187
 188                 local logfile="$LOGDIR/$(basename "$t").log"
 189                 if run_linux_test "$t" > "$logfile" 2>&1; then
 190                         pass "$t"
 191                 else
 192                         fail "$t"
 193                 fi
 194         done
 195
 196         echo
 197 }
 198
 199
 200 if [ $# -ne 1 ]; then
 201         usage
 202 fi
 203
 204 CLASS="$1"
 205
 206 case "$CLASS" in
 207         all)
 208                 init_testing
 209                 run_simple_tests
 210                 run_linux_tests
 211                 show_stats
 212                 ;;
 213         simple)
 214                 init_testing
 215                 run_simple_tests
 216                 show_stats
 217                 ;;
 218         linux)
 219                 init_testing
 220                 run_linux_tests
 221                 show_stats
 222                 ;;
 223         *)
 224                 echo "Invalid test class $CLASS"
 225                 echo
 226                 usage
 227                 ;;
 228 esac
 229
 230 if [ $NUM_FAIL -ne $NUM_BROKEN ]; then
 231         exit 1
 232 fi