run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename ${0})
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 gawk
  14 git
  15 i18nspector
  16 libav-tools
  17 libcap2-bin
  18 libsikuli-script-java
  19 libvirt-clients
  20 libvirt-daemon-system
  21 libvirt-dev
  22 libvirt0
  23 libxslt1-dev
  24 openjdk-7-jre
  25 ovmf
  26 python-jabberbot
  27 python-potr
  28 qemu-kvm
  29 qemu-system-x86
  30 radvd
  31 ruby-guestfs
  32 ruby-json
  33 ruby-libvirt
  34 ruby-net-irc
  35 ruby-packetfu
  36 ruby-rb-inotify
  37 ruby-rjb
  38 ruby-rspec
  39 ruby-test-unit
  40 seabios
  41 tcpdump
  42 unclutter
  43 virt-viewer
  44 x11-apps
  45 xvfb
  46 "
  47
  48 usage() {
  49     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  50 Sets up an appropriate environment and invokes cucumber. Note that this script
  51 must be run from the Tails source directory root.
  52
  53 Options for '@product' features:
  54   --capture          Captures failed scenarios into videos stored in the
  55                      temporary directory (see --tmpdir below) using x264
  56                      encoding. Requires x264.
  57   --capture-all      Keep videos for all scenarios, including those that
  58                      succeed (implies --capture).
  59   --pause-on-fail    On failure, pause test suite until pressing Enter. This is
  60                      useful for investigating the state of the VM guest to see
  61                      exactly why a test failed.
  62   --keep-snapshots   Don't ever delete the background snapshots. This can a big
  63                      time saver when debugging new features.
  64   --retry-find       Print a warning whenever Sikuli fails to find an image
  65                      and allow *one* retry after pressing ENTER. This is useful
  66                      for updating outdated images.
  67   --tmpdir           Directory where various temporary files are written
  68                      during a test, e.g. VM snapshots and memory dumps,
  69                      failure screenshots, pcap files and disk images
  70                      (default is TMPDIR in the environment, and if unset,
  71                      /tmp/TailsToaster).
  72   --view             Shows the test session in a windows. Requires x11vnc
  73                      and xtightvncviewer.
  74   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
  75   --iso IMAGE        Test '@product' features using IMAGE.
  76   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
  77                      an older version of Tails, which this options sets to
  78                      IMAGE. If none is given, it defaults to the same IMAGE
  79                      given by --iso, which will be good enough for most testing
  80                      purposes.
  81
  82 Note that '@source' features has no relevant options.
  83
  84 CUCUMBER_ARGS can be used to specify which features to be run, but also any
  85 cucumber option, although then you must pass \`--\` first to let this wrapper
  86 script know that we're done with *its* options. For debugging purposes, a
  87 'debug' formatter has been added so pretty debugging can be enabled with
  88 \`--format debug\`. You could even combine the default (pretty) formatter with
  89 pretty debugging printed to a file with \`--format pretty --format debug
  90 --out debug.log\`.
  91 "
  92 }
  93
  94 error() {
  95     echo "${NAME}: error: ${*}" >&2
  96     usage
  97     exit 1
  98 }
  99
 100 package_installed() {
 101     local ret
 102     set +o pipefail
 103     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 104         ret=0
 105     else
 106         ret=1
 107     fi
 108     set -o pipefail
 109     return ${ret}
 110 }
 111
 112 check_dependencies() {
 113     while [ -n "${1:-}" ]; do
 114         if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
 115             error "'${1}' is missing, please install it and run again."
 116         fi
 117         shift
 118     done
 119 }
 120
 121 display_in_use() {
 122     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 123 }
 124
 125 next_free_display() {
 126     display_nr=0
 127     while display_in_use ":${display_nr}"; do
 128         display_nr=$((display_nr+1))
 129     done
 130     echo ":${display_nr}"
 131 }
 132
 133 test_suite_cleanup() {
 134     (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true
 135 }
 136
 137 start_xvfb() {
 138     Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 &
 139     XVFB_PID=$!
 140     # Wait for Xvfb to run on TARGET_DISPLAY
 141     until display_in_use $TARGET_DISPLAY; do
 142         sleep 1
 143     done
 144     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 145     # Hide the mouse cursor so it won't mess up Sikuli's screen scanning
 146     unclutter -display $TARGET_DISPLAY -root -idle 0 >/dev/null 2>&1 &
 147 }
 148
 149 start_vnc_server() {
 150     check_dependencies x11vnc
 151     VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \
 152                               -bg -nopw 2>&1 | \
 153                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 154     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 155 }
 156
 157 start_vnc_viewer() {
 158     check_dependencies xtightvncviewer
 159     xtightvncviewer -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 &
 160 }
 161
 162 capture_session() {
 163     check_dependencies libvpx1
 164     echo "Capturing guest display into ${CAPTURE_FILE}"
 165     avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
 166         -vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 &
 167 }
 168
 169 # main script
 170
 171 # Unset all environment variables used by this script to pass options
 172 # to cucumber, except TMPDIR since we explicitly want to support
 173 # setting it that way.
 174 CAPTURE=
 175 CAPTURE_ALL=
 176 LOG_FILE=
 177 VNC_VIEWER=
 178 VNC_SERVER=
 179 PAUSE_ON_FAIL=
 180 KEEP_SNAPSHOTS=
 181 SIKULI_RETRY_FINDFAILED=
 182 TAILS_ISO=
 183 OLD_TAILS_ISO=
 184
 185 LONGOPTS="view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,pause-on-fail"
 186 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 187 eval set -- "$OPTS"
 188 while [ $# -gt 0 ]; do
 189     case $1 in
 190         --view)
 191             VNC_VIEWER=yes
 192             VNC_SERVER=yes
 193             ;;
 194         --vnc-server-only)
 195             VNC_VIEWER=
 196             VNC_SERVER=yes
 197             ;;
 198         --capture)
 199             check_dependencies x264
 200             export CAPTURE="yes"
 201             ;;
 202         --capture-all)
 203             check_dependencies x264
 204             export CAPTURE="yes"
 205             export CAPTURE_ALL="yes"
 206             ;;
 207         --pause-on-fail)
 208             export PAUSE_ON_FAIL="yes"
 209             ;;
 210         --keep-snapshots)
 211             export KEEP_SNAPSHOTS="yes"
 212             ;;
 213         --retry-find)
 214             export SIKULI_RETRY_FINDFAILED="yes"
 215             ;;
 216         --tmpdir)
 217             shift
 218             export TMPDIR="$(readlink -f $1)"
 219             ;;
 220         --iso)
 221             shift
 222             export TAILS_ISO="$(readlink -f $1)"
 223             ;;
 224         --old-iso)
 225             shift
 226             export OLD_TAILS_ISO="$(readlink -f $1)"
 227             ;;
 228         --help)
 229             usage
 230             exit 0
 231             ;;
 232         --)
 233             shift
 234             break
 235             ;;
 236     esac
 237     shift
 238 done
 239
 240 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 241
 242 check_dependencies ${GENERAL_DEPENDENCIES}
 243
 244 TARGET_DISPLAY=$(next_free_display)
 245
 246 start_xvfb
 247
 248 if [ -n "${VNC_SERVER}" ]; then
 249     start_vnc_server
 250 fi
 251 if [ -n "${VNC_VIEWER}" ]; then
 252     start_vnc_viewer
 253 fi
 254
 255 export JAVA_HOME="/usr/lib/jvm/java-7-openjdk-amd64"
 256 export SIKULI_HOME="/usr/share/java"
 257 export DISPLAY=${TARGET_DISPLAY}
 258
 259 cucumber ${@}