run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename ${0})
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 gawk
  14 git
  15 i18nspector
  16 libav-tools
  17 libcap2-bin
  18 libsikulixapi-java
  19 libvirt-clients
  20 libvirt-daemon-system
  21 libvirt-dev
  22 libvirt0
  23 obfs4proxy
  24 openssh-server
  25 ovmf
  26 pry
  27 python-jabberbot
  28 python-potr
  29 qemu-kvm
  30 qemu-system-x86
  31 redir
  32 ruby-guestfs
  33 ruby-json
  34 ruby-libvirt
  35 ruby-net-irc
  36 ruby-packetfu
  37 ruby-rb-inotify
  38 ruby-rjb
  39 ruby-rspec
  40 ruby-test-unit
  41 seabios
  42 tcpdump
  43 tor
  44 unclutter
  45 virt-viewer
  46 xvfb
  47 "
  48
  49 usage() {
  50     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  51 Sets up an appropriate environment and invokes cucumber. Note that this script
  52 must be run from the Tails source directory root.
  53
  54 Options for '@product' features:
  55   --artifacts-base-uri URI
  56                      Pretend that the artifact is located at URI when printing
  57                      its location during a scenario failure. This is useful if
  58                      you intend to serve the artifacts via the web, for
  59                      instance.
  60   --capture          Captures failed scenarios into videos stored in the
  61                      temporary directory (see --tmpdir below) using x264
  62                      encoding. Requires x264.
  63   --capture-all      Keep videos for all scenarios, including those that
  64                      succeed (implies --capture).
  65   --interactive-debugging
  66                      On failure, pause test suite until pressing Enter. Also
  67                      offer the option to open an interactive Ruby shell (pry)
  68                      in the Cucumber world's context.
  69   --keep-snapshots   Don't ever delete any snapshots (including ones marked as
  70                      temporary). This can be a big time saver when debugging new
  71                      features.
  72   --retry-find       Print a warning whenever Sikuli fails to find an image
  73                      and allow *one* retry after pressing ENTER. This is useful
  74                      for updating outdated images.
  75   --fuzzy-image-matching
  76                      When Sikuli fails to find an image, let it retry with more
  77                      fuzziness (or \"lower similarity factor\" in Sikuli terms).
  78   --tmpdir           Directory where various temporary files are written
  79                      during a test, e.g. VM snapshots and memory dumps,
  80                      failure screenshots, pcap files and disk images
  81                      (default is TMPDIR in the environment, and if unset,
  82                      /tmp/TailsToaster).
  83   --view             Shows the test session in a windows. Requires x11vnc
  84                      and tigervnc-viewer.
  85   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
  86   --iso IMAGE        Test '@product' features using IMAGE.
  87   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
  88                      an older version of Tails, which this options sets to
  89                      IMAGE. If none is given, it defaults to the same IMAGE
  90                      given by --iso, which will be good enough for most testing
  91                      purposes.
  92
  93 Note that '@source' features has no relevant options.
  94
  95 CUCUMBER_ARGS can be used to specify which features to be run, but also any
  96 cucumber option, although then you must pass \`--\` first to let this wrapper
  97 script know that we're done with *its* options. For debugging purposes, a
  98 'debug' formatter has been added so pretty debugging can be enabled with
  99 \`--format debug\`. You could even combine the default (pretty) formatter with
 100 pretty debugging printed to a file with \`--format pretty --format debug
 101 --out debug.log\`.
 102 "
 103 }
 104
 105 error() {
 106     echo "${NAME}: error: ${*}" >&2
 107     usage
 108     exit 1
 109 }
 110
 111 package_installed() {
 112     local ret
 113     set +o pipefail
 114     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 115         ret=0
 116     else
 117         ret=1
 118     fi
 119     set -o pipefail
 120     return ${ret}
 121 }
 122
 123 check_dependencies() {
 124     while [ -n "${1:-}" ]; do
 125         if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
 126             error "'${1}' is missing, please install it and run again."
 127         fi
 128         shift
 129     done
 130 }
 131
 132 display_in_use() {
 133     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 134 }
 135
 136 next_free_display() {
 137     display_nr=0
 138     while display_in_use ":${display_nr}"; do
 139         display_nr=$((display_nr+1))
 140     done
 141     echo ":${display_nr}"
 142 }
 143
 144 test_suite_cleanup() {
 145     (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true
 146 }
 147
 148 start_xvfb() {
 149     Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 &
 150     XVFB_PID=$!
 151     # Wait for Xvfb to run on TARGET_DISPLAY
 152     until display_in_use $TARGET_DISPLAY; do
 153         sleep 1
 154     done
 155     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 156     # Hide the mouse cursor so it won't mess up Sikuli's screen scanning
 157     unclutter -display $TARGET_DISPLAY -root -idle 0.1 >/dev/null 2>&1 &
 158 }
 159
 160 start_vnc_server() {
 161     check_dependencies x11vnc
 162     VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \
 163                               -bg -nopw -forever 2>&1 | \
 164                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 165     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 166 }
 167
 168 start_vnc_viewer() {
 169     check_dependencies tigervnc-viewer
 170     xtigervncviewer -nojpeg -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 &
 171 }
 172
 173 capture_session() {
 174     check_dependencies libvpx1
 175     echo "Capturing guest display into ${CAPTURE_FILE}"
 176     avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
 177         -vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 &
 178 }
 179
 180 # main script
 181
 182 # Unset all environment variables used by this script to pass options
 183 # to cucumber, except TMPDIR since we explicitly want to support
 184 # setting it that way.
 185 ARTIFACTS_BASE_URI=
 186 CAPTURE=
 187 CAPTURE_ALL=
 188 LOG_FILE=
 189 VNC_VIEWER=
 190 VNC_SERVER=
 191 INTERACTIVE_DEBUGGING=
 192 KEEP_SNAPSHOTS=
 193 SIKULI_RETRY_FINDFAILED=
 194 SIKULI_FUZZY_IMAGE_MATCHING=
 195 TAILS_ISO=
 196 OLD_TAILS_ISO=
 197
 198 LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,fuzzy-image-matching,iso:,old-iso:,interactive-debugging"
 199 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 200 eval set -- "$OPTS"
 201 while [ $# -gt 0 ]; do
 202     case $1 in
 203         --artifacts-base-uri)
 204             shift
 205             export ARTIFACTS_BASE_URI="${1}"
 206             ;;
 207         --view)
 208             VNC_VIEWER=yes
 209             VNC_SERVER=yes
 210             ;;
 211         --vnc-server-only)
 212             VNC_VIEWER=
 213             VNC_SERVER=yes
 214             ;;
 215         --capture)
 216             check_dependencies x264
 217             export CAPTURE="yes"
 218             ;;
 219         --capture-all)
 220             check_dependencies x264
 221             export CAPTURE="yes"
 222             export CAPTURE_ALL="yes"
 223             ;;
 224         --interactive-debugging)
 225             export INTERACTIVE_DEBUGGING="yes"
 226             ;;
 227         --keep-snapshots)
 228             export KEEP_SNAPSHOTS="yes"
 229             ;;
 230         --retry-find)
 231             export SIKULI_RETRY_FINDFAILED="yes"
 232             ;;
 233         --fuzzy-image-matching)
 234             export SIKULI_FUZZY_IMAGE_MATCHING="yes"
 235             ;;
 236         --tmpdir)
 237             shift
 238             export TMPDIR="$(readlink -f $1)"
 239             ;;
 240         --iso)
 241             shift
 242             export TAILS_ISO="$(readlink -f $1)"
 243             ;;
 244         --old-iso)
 245             shift
 246             export OLD_TAILS_ISO="$(readlink -f $1)"
 247             ;;
 248         --help)
 249             usage
 250             exit 0
 251             ;;
 252         --)
 253             shift
 254             break
 255             ;;
 256     esac
 257     shift
 258 done
 259
 260 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 261
 262 check_dependencies ${GENERAL_DEPENDENCIES}
 263
 264 TARGET_DISPLAY=$(next_free_display)
 265
 266 start_xvfb
 267
 268 if [ -n "${VNC_SERVER:-}" ]; then
 269     start_vnc_server
 270 fi
 271 if [ -n "${VNC_VIEWER:-}" ]; then
 272     start_vnc_viewer
 273 fi
 274
 275 export SIKULI_HOME="/usr/share/java"
 276 export DISPLAY=${TARGET_DISPLAY}
 277
 278 cucumber ${@}