run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename ${0})
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 gawk
  14 git
  15 i18nspector
  16 libav-tools
  17 libcap2-bin
  18 libsikulixapi-java
  19 libvirt-clients
  20 libvirt-daemon-system
  21 libvirt-dev
  22 libvirt0
  23 obfs4proxy
  24 openssh-server
  25 ovmf
  26 pry
  27 python-jabberbot
  28 python-potr
  29 qemu-kvm
  30 qemu-system-x86
  31 redir
  32 ruby-guestfs
  33 ruby-json
  34 ruby-libvirt
  35 ruby-net-irc
  36 ruby-packetfu
  37 ruby-rb-inotify
  38 ruby-rjb
  39 ruby-rspec
  40 ruby-test-unit
  41 seabios
  42 tcpdump
  43 tor
  44 unclutter
  45 virt-viewer
  46 xvfb
  47 "
  48
  49 usage() {
  50     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  51 Sets up an appropriate environment and invokes cucumber. Note that this script
  52 must be run from the Tails source directory root.
  53
  54 Options for '@product' features:
  55   --artifacts-base-uri URI
  56                      Pretend that the artifact is located at URI when printing
  57                      its location during a scenario failure. This is useful if
  58                      you intend to serve the artifacts via the web, for
  59                      instance.
  60   --capture          Captures failed scenarios into videos stored in the
  61                      temporary directory (see --tmpdir below) using x264
  62                      encoding. Requires x264.
  63   --capture-all      Keep videos for all scenarios, including those that
  64                      succeed (implies --capture).
  65   --interactive-debugging
  66                      On failure, pause test suite until pressing Enter. Also
  67                      offer the option to open an interactive Ruby shell (pry)
  68                      in the Cucumber world's context.
  69   --keep-snapshots   Don't ever delete any snapshots (including ones marked as
  70                      temporary). This can be a big time saver when debugging new
  71                      features.
  72   --retry-find       Print a warning whenever Sikuli fails to find an image
  73                      and allow *one* retry after pressing ENTER. This is useful
  74                      for updating outdated images.
  75   --fuzzy-image-matching
  76                      When Sikuli fails to find an image, let it retry with more
  77                      fuzziness (or \"lower similarity factor\" in Sikuli terms).
  78   --tmpdir           Directory where various temporary files are written
  79                      during a test, e.g. VM snapshots and memory dumps,
  80                      failure screenshots, pcap files and disk images
  81                      (default is TMPDIR in the environment, and if unset,
  82                      /tmp/TailsToaster).
  83   --view             Shows the test session in a windows. Requires x11vnc
  84                      and tigervnc-viewer.
  85   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
  86   --iso IMAGE        Test '@product' features using IMAGE.
  87   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
  88                      an older version of Tails, which this options sets to
  89                      IMAGE. If none is given, it defaults to the same IMAGE
  90                      given by --iso, which will be good enough for most testing
  91                      purposes.
  92
  93 Note that '@source' features has no relevant options.
  94
  95 CUCUMBER_ARGS can be used to specify which features to be run, but also any
  96 cucumber option, although then you must pass \`--\` first to let this wrapper
  97 script know that we're done with *its* options. For debugging purposes, a
  98 'debug' formatter has been added so pretty debugging can be enabled with
  99 \`--format debug\`. You could even combine the default (pretty) formatter with
 100 pretty debugging printed to a file with \`--format pretty --format debug
 101 --out debug.log\`.
 102 "
 103 }
 104
 105 error() {
 106     echo "${NAME}: error: ${*}" >&2
 107     usage
 108     exit 1
 109 }
 110
 111 package_installed() {
 112     local ret
 113     set +o pipefail
 114     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 115         ret=0
 116     else
 117         ret=1
 118     fi
 119     set -o pipefail
 120     return ${ret}
 121 }
 122
 123 check_dependencies() {
 124     while [ -n "${1:-}" ]; do
 125         if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
 126             error "'${1}' is missing, please install it and run again."
 127         fi
 128         shift
 129     done
 130 }
 131
 132 display_in_use() {
 133     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 134 }
 135
 136 next_free_display() {
 137     display_nr=0
 138     while display_in_use ":${display_nr}"; do
 139         display_nr=$((display_nr+1))
 140     done
 141     echo ":${display_nr}"
 142 }
 143
 144 test_suite_cleanup() {
 145     if [ -n "${XVFB_PID:-}" ]; then
 146         (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true
 147     fi
 148     return $?
 149 }
 150
 151 start_xvfb() {
 152     Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 &
 153     XVFB_PID=$!
 154     # Wait for Xvfb to run on TARGET_DISPLAY
 155     until display_in_use $TARGET_DISPLAY; do
 156         sleep 1
 157     done
 158     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 159     # Hide the mouse cursor so it won't mess up Sikuli's screen scanning
 160     unclutter -display $TARGET_DISPLAY -root -idle 0.1 >/dev/null 2>&1 &
 161 }
 162
 163 start_vnc_server() {
 164     check_dependencies x11vnc
 165     VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \
 166                               -bg -nopw -forever 2>&1 | \
 167                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 168     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 169 }
 170
 171 start_vnc_viewer() {
 172     check_dependencies tigervnc-viewer
 173     xtigervncviewer -nojpeg -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 &
 174 }
 175
 176 capture_session() {
 177     check_dependencies libvpx1
 178     echo "Capturing guest display into ${CAPTURE_FILE}"
 179     avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
 180         -vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 &
 181 }
 182
 183 # main script
 184
 185 # Unset all environment variables used by this script to pass options
 186 # to cucumber, except TMPDIR since we explicitly want to support
 187 # setting it that way.
 188 ARTIFACTS_BASE_URI=
 189 CAPTURE=
 190 CAPTURE_ALL=
 191 LOG_FILE=
 192 VNC_VIEWER=
 193 VNC_SERVER=
 194 INTERACTIVE_DEBUGGING=
 195 KEEP_SNAPSHOTS=
 196 SIKULI_RETRY_FINDFAILED=
 197 SIKULI_FUZZY_IMAGE_MATCHING=
 198 TAILS_ISO=
 199 OLD_TAILS_ISO=
 200
 201 LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,fuzzy-image-matching,iso:,old-iso:,interactive-debugging"
 202 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 203 eval set -- "$OPTS"
 204 while [ $# -gt 0 ]; do
 205     case $1 in
 206         --artifacts-base-uri)
 207             shift
 208             export ARTIFACTS_BASE_URI="${1}"
 209             ;;
 210         --view)
 211             VNC_VIEWER=yes
 212             VNC_SERVER=yes
 213             ;;
 214         --vnc-server-only)
 215             VNC_VIEWER=
 216             VNC_SERVER=yes
 217             ;;
 218         --capture)
 219             check_dependencies x264
 220             export CAPTURE="yes"
 221             ;;
 222         --capture-all)
 223             check_dependencies x264
 224             export CAPTURE="yes"
 225             export CAPTURE_ALL="yes"
 226             ;;
 227         --interactive-debugging)
 228             export INTERACTIVE_DEBUGGING="yes"
 229             ;;
 230         --keep-snapshots)
 231             export KEEP_SNAPSHOTS="yes"
 232             ;;
 233         --retry-find)
 234             export SIKULI_RETRY_FINDFAILED="yes"
 235             ;;
 236         --fuzzy-image-matching)
 237             export SIKULI_FUZZY_IMAGE_MATCHING="yes"
 238             ;;
 239         --tmpdir)
 240             shift
 241             export TMPDIR="$(readlink -f $1)"
 242             ;;
 243         --iso)
 244             shift
 245             export TAILS_ISO="$(readlink -f $1)"
 246             ;;
 247         --old-iso)
 248             shift
 249             export OLD_TAILS_ISO="$(readlink -f $1)"
 250             ;;
 251         --help)
 252             usage
 253             exit 0
 254             ;;
 255         --)
 256             shift
 257             break
 258             ;;
 259     esac
 260     shift
 261 done
 262
 263 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 264
 265 check_dependencies ${GENERAL_DEPENDENCIES}
 266
 267 TARGET_DISPLAY=$(next_free_display)
 268
 269 start_xvfb
 270
 271 if [ -n "${VNC_SERVER:-}" ]; then
 272     start_vnc_server
 273 fi
 274 if [ -n "${VNC_VIEWER:-}" ]; then
 275     start_vnc_viewer
 276 fi
 277
 278 export SIKULI_HOME="/usr/share/java"
 279 export DISPLAY=${TARGET_DISPLAY}
 280
 281 cucumber ${@}