run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename "${0}")
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 ffmpeg
  14 gawk
  15 git
  16 imagemagick
  17 libcap2-bin
  18 libvirt-clients
  19 libvirt-daemon-system
  20 libvirt-dev
  21 libvirt0
  22 obfs4proxy
  23 openssh-server
  24 ovmf
  25 pry
  26 python3-opencv
  27 python3-pil
  28 python3-slixmpp
  29 qemu-system-x86
  30 qrencode
  31 ruby-guestfs
  32 ruby-json
  33 ruby-libvirt
  34 ruby-net-dns
  35 ruby-packetfu
  36 ruby-rb-inotify
  37 ruby-rspec
  38 ruby-test-unit
  39 seabios
  40 tcpdump
  41 tcplay
  42 tor
  43 unclutter
  44 virt-viewer
  45 xdotool
  46 xvfb
  47 "
  48
  49 # Recent tor client require a Chutney network with tor >= 0.4.4.6 (#18190)
  50 MINIMUM_TOR_VERSION=0.4.4.6
  51
  52 usage() {
  53     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  54 Sets up an appropriate environment and invokes cucumber. Note that this script
  55 must be run from the Tails source directory root.
  56
  57 Options for '@product' features:
  58   --allow-non-root   Normally the test suite must be run as root, but if you
  59                      really know what you are doing this option allows any
  60                      user to run it.
  61   --artifacts-base-uri URI
  62                      Pretend that the artifact is located at URI when printing
  63                      its location during a scenario failure. This is useful if
  64                      you intend to serve the artifacts via the web, for
  65                      instance.
  66   --capture          Captures failed scenarios into videos stored in the
  67                      temporary directory (see --tmpdir below) using x264
  68                      encoding. Requires x264.
  69   --capture-all      Keep videos for all scenarios, including those that
  70                      succeed (implies --capture).
  71   --interactive-debugging
  72                      On failure, pause test suite until pressing Enter. Also
  73                      offer the option to open an interactive Ruby shell (pry)
  74                      in the Cucumber world's context.
  75   --image-bumping-mode
  76                      When any image matching fails, enter an interactive mode
  77                      that allows to update the image. If run from a graphical
  78                      environment, any found candidate image will be displayed
  79                      in a pop-up.
  80   --keep-chutney     Don't ever clean Chutney data directory.
  81                      This can be a big time saver when debugging steps
  82                      when --keep-snapshots is not an option.
  83   --keep-snapshots   Don't ever delete any snapshots (including ones marked as
  84                      temporary). This can be a big time saver when debugging new
  85                      features. Implies --keep-chutney.
  86   --disable-chutney  EXPERIMENTAL: All tests will be run using the real Tor
  87                      network, not a simulated one.
  88                      Expect this to break many test cases.
  89   --late-patch FILE  Copies files into the VM before running the test suite.
  90                      This often can avoid a rebuild.
  91                      FILE is a text file where each line contains tab-separated
  92                      source and destination.
  93                      lines without a tab are ignored.
  94                      lines with a leading \"#\" are ignored.
  95   --early-patch      Boots the system with the \"early_patch=umount\" cmdline option.
  96                      See wiki/src/contribute/build/early-patch.mdwn for details
  97                      This is useful when you need something to be patched early
  98                      in the boot process; or if you need to run arbitrary
  99                      commands, not just copying files
 100   --all-tests        Don't skip tests which have a @fragile or @skip_by_default tag.
 101   --tmpdir           Directory where various temporary files are written
 102                      during a test, e.g. VM snapshots and memory dumps,
 103                      failure screenshots, pcap files and disk images
 104                      (default is TMPDIR in the environment, and if unset,
 105                      /tmp/TailsToaster).
 106   --view             Shows the test session in a windows. Requires x11vnc
 107                      and tigervnc-viewer.
 108   --view-interact    The test session can be \"touched\". For debugging purposes
 109                      only.
 110   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
 111   --iso IMAGE        Test '@product' features using IMAGE.
 112   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
 113                      an older version of Tails, which this options sets to
 114                      IMAGE. If none is given, it defaults to the same IMAGE
 115                      given by --iso, which will be good enough for most testing
 116                      purposes.
 117
 118 Note that '@source' features has no relevant options.
 119
 120 CUCUMBER_ARGS can be used to specify which features to be run, but also any
 121 cucumber option, although then you must pass \`--\` first to let this wrapper
 122 script know that we're done with *its* options. For debugging purposes, a
 123 'debug' formatter has been added so pretty debugging can be enabled with
 124 \`--format debug\`. You could even combine the default (pretty) formatter with
 125 pretty debugging printed to a file with \`--format pretty --format debug
 126 --out debug.log\`.
 127 "
 128 }
 129
 130 error() {
 131     echo "${NAME}: error: ${*}" >&2
 132     exit 1
 133 }
 134
 135 package_installed() {
 136     local ret
 137     set +o pipefail
 138     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 139         ret=0
 140     else
 141         ret=1
 142     fi
 143     set -o pipefail
 144     return ${ret}
 145 }
 146
 147 check_dependencies() {
 148     while [ -n "${1:-}" ]; do
 149         if ! command -v "${1}" >/dev/null && ! package_installed "${1}" ; then
 150             error "'${1}' is missing, please install it and run again."
 151         fi
 152         shift
 153     done
 154 }
 155
 156 check_tor_version() {
 157     local tor_version
 158     tor_version=$(tor --version | grep -Po '^Tor version \K.*' | sed --regexp-extended 's,[.]$,,')
 159     echo "tor version: $tor_version"
 160     if dpkg --compare-versions "$tor_version" lt "$MINIMUM_TOR_VERSION"; then
 161         error "Please upgrade to tor ${MINIMUM_TOR_VERSION} or newer."
 162     fi
 163 }
 164
 165 display_in_use() {
 166     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 167 }
 168
 169 next_free_display() {
 170     display_nr=0
 171     while display_in_use ":${display_nr}"; do
 172         display_nr=$((display_nr+1))
 173     done
 174     echo ":${display_nr}"
 175 }
 176
 177 test_suite_cleanup() {
 178     if [ -n "${XVFB_PID:-}" ]; then
 179         (kill -0 "${XVFB_PID}" 2>/dev/null && kill "${XVFB_PID}") || /bin/true
 180     fi
 181     return $?
 182 }
 183
 184 start_xvfb() {
 185     Xvfb "$TARGET_DISPLAY" -screen 0 1024x768x24+32 -noreset >/dev/null 2>&1 &
 186     XVFB_PID=$!
 187     # Wait for Xvfb to run on TARGET_DISPLAY
 188     until display_in_use "$TARGET_DISPLAY"; do
 189         sleep 1
 190     done
 191     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 192     # Hide the mouse cursor so it won't be in the way when we are
 193     # trying to match images.
 194     unclutter -display "$TARGET_DISPLAY" -root -idle 0.1 >/dev/null 2>&1 &
 195 }
 196
 197 start_vnc_server() {
 198     check_dependencies x11vnc
 199     VNC_SERVER_PORT="$(x11vnc -listen localhost -display "${TARGET_DISPLAY}" \
 200                               -bg -nopw -forever 2>&1 | \
 201                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 202     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 203 }
 204
 205 start_vnc_viewer() {
 206     opts=
 207     if [[ "${VNC_VIEWER_VIEWONLY}" = yes ]]; then
 208         opts=-viewonly
 209     fi
 210     check_dependencies tigervnc-viewer
 211     xtigervncviewer \
 212         -nojpeg $opts \
 213         -RemoteResize=0 \
 214         -AcceptClipboard=0 \
 215         -SendClipboard=0 \
 216         -SetPrimary=0 \
 217         -SendPrimary=0 \
 218         "localhost:${VNC_SERVER_PORT}" 1>/dev/null 2>&1 &
 219 }
 220
 221 # main script
 222
 223 # Set default values of environment variables used by this script to
 224 # pass options to cucumber (unless they are already set).
 225 ALLOW_NON_ROOT=${ALLOW_NON_ROOT-}
 226 ARTIFACTS_BASE_URI=${ARTIFACTS_BASE_URI-}
 227 CAPTURE=${CAPTURE-}
 228 CAPTURE_ALL=${CAPTURE_ALL-}
 229 VNC_VIEWER=${VNC_VIEWER-}
 230 VNC_VIEWER_VIEWONLY=${VNC_VIEWER_VIEWONLY-yes}
 231 VNC_SERVER=${VNC_SERVER-}
 232 INTERACTIVE_DEBUGGING=${INTERACTIVE_DEBUGGING-}
 233 IMAGE_BUMPING_MODE=${IMAGE_BUMPING_MODE-}
 234 KEEP_CHUTNEY=${KEEP_CHUTNEY-}
 235 KEEP_SNAPSHOTS=${KEEP_SNAPSHOTS-}
 236 TAILS_ISO=${TAILS_ISO-}
 237 OLD_TAILS_ISO=${OLD_TAILS_ISO-}
 238 EARLY_PATCH=${EARLY_PATCH-}
 239 EXTRA_BOOT_OPTIONS=${EXTRA_BOOT_OPTIONS-}
 240 ALL_TESTS=${ALL_TESTS-}
 241
 242 LONGOPTS="allow-non-root,artifacts-base-uri:,view,view-interact,vnc-server-only,capture,capture-all,help,tmpdir:,keep-chutney,keep-snapshots,disable-chutney,late-patch:,early-patch,extra-boot-options:,all-tests,iso:,old-iso:,interactive-debugging,image-bumping-mode"
 243 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 244 eval set -- "$OPTS"
 245 while [ $# -gt 0 ]; do
 246     case $1 in
 247         --allow-non-root)
 248             if ! /sbin/getcap /usr/bin/tcpdump | grep -q 'cap_net_raw=eip'; then
 249                 error "/usr/bin/tcpdump lacks cap_net_raw=eip"
 250             fi
 251             ALLOW_NON_ROOT="yes"
 252             ;;
 253         --artifacts-base-uri)
 254             shift
 255             export ARTIFACTS_BASE_URI="${1}"
 256             ;;
 257         --view)
 258             VNC_VIEWER=yes
 259             VNC_VIEWER_VIEWONLY=yes
 260             VNC_SERVER=yes
 261             ;;
 262         --view-interact)
 263             VNC_VIEWER=yes
 264             VNC_VIEWER_VIEWONLY=
 265             VNC_SERVER=yes
 266             ;;
 267         --vnc-server-only)
 268             VNC_VIEWER=
 269             VNC_SERVER=yes
 270             ;;
 271         --capture)
 272             check_dependencies x264
 273             export CAPTURE="yes"
 274             ;;
 275         --capture-all)
 276             check_dependencies x264
 277             export CAPTURE="yes"
 278             export CAPTURE_ALL="yes"
 279             ;;
 280         --interactive-debugging)
 281             export INTERACTIVE_DEBUGGING="yes"
 282             ;;
 283         --image-bumping-mode)
 284             export IMAGE_BUMPING_MODE="yes"
 285             ;;
 286         --keep-chutney)
 287             export KEEP_CHUTNEY="yes"
 288             ;;
 289         --keep-snapshots)
 290             export KEEP_CHUTNEY="yes"
 291             export KEEP_SNAPSHOTS="yes"
 292             ;;
 293         --disable-chutney)
 294             export DISABLE_CHUTNEY="yes"
 295             ;;
 296         --late-patch)
 297             shift
 298             LATE_PATCH="$(realpath -s "$1")"
 299             if ! [[ -f "$LATE_PATCH" ]]; then
 300                 echo "--late-patch requires a valid FILE as argument"
 301                 exit 2
 302             fi
 303             export LATE_PATCH
 304             ;;
 305         --early-patch)
 306             export EARLY_PATCH="yes"
 307             ;;
 308         --extra-boot-options)
 309             shift
 310             export EXTRA_BOOT_OPTIONS="$1"
 311             ;;
 312         --all-tests)
 313             ALL_TESTS="yes"
 314             ;;
 315         --tmpdir)
 316             shift
 317             TMPDIR="$(readlink -f "$1")"
 318             export TMPDIR
 319             ;;
 320         --iso)
 321             shift
 322             TAILS_ISO="$(realpath -s "$1")"
 323             export TAILS_ISO
 324             ;;
 325         --old-iso)
 326             shift
 327             OLD_TAILS_ISO="$(realpath -s "$1")"
 328             export OLD_TAILS_ISO
 329             ;;
 330         --help)
 331             usage
 332             exit 0
 333             ;;
 334         --)
 335             shift
 336             break
 337             ;;
 338     esac
 339     shift
 340 done
 341
 342 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 343
 344 if [ "${EUID}" -ne 0 ] && [ -z "${ALLOW_NON_ROOT}" ]; then
 345     error "you are not running as root; if you really know what you are" \
 346           "doing, see the --allow-non-root option"
 347 fi
 348
 349 # shellcheck disable=SC2086
 350 check_dependencies ${GENERAL_DEPENDENCIES}
 351 check_tor_version
 352
 353 TARGET_DISPLAY=$(next_free_display)
 354
 355 start_xvfb
 356
 357 if [ -n "${VNC_SERVER:-}" ]; then
 358     start_vnc_server
 359 fi
 360 if [ -n "${VNC_VIEWER:-}" ]; then
 361     start_vnc_viewer
 362 fi
 363
 364 TAGS_ARGS=""
 365 if [ -n "${JENKINS_URL:-}" ]; then
 366     . auto/scripts/utils.sh
 367
 368     if echo "${GIT_BRANCH}" | grep -q -E '[+-]real-Tor$'; then
 369         export DISABLE_CHUTNEY="yes"
 370         TAGS_ARGS="--tag @supports_real_tor"
 371     # The current Git state may not reflect the state at the time the
 372     # upstream job was started (e.g. since then we git fetch + git
 373     # reset --hard) so we trust the Git state described in Jenkins'
 374     # environment variables instead.
 375     elif echo "${GIT_BRANCH}" | grep -q -E '[+-]force-all-tests$' \
 376        || git_on_a_tag \
 377        || [ "${GIT_BRANCH#origin/}" = feature/bullseye ] \
 378        || [ "${GIT_BRANCH#origin/}" = testing ] \
 379        || [ "${GIT_BRANCH#origin/}" = devel ] \
 380        || [ -n "${ALL_TESTS:-}" ]; then
 381         TAGS_ARGS=""
 382     else
 383         TAGS_ARGS="${TAGS_ARGS} --tag ~@fragile --tag ~@skip_by_default"
 384     fi
 385     if [ "${UPSTREAMJOB_GIT_COMMIT}" != "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}" ] && \
 386            git_only_doc_changes_since "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}"; then
 387         TAGS_ARGS="${TAGS_ARGS} --tag @doc"
 388     fi
 389 fi
 390
 391 export USER_DISPLAY="${DISPLAY:-}"
 392 export DISPLAY=${TARGET_DISPLAY}
 393
 394 # shellcheck disable=SC2086
 395 cucumber --expand ${TAGS_ARGS} "${@}"