run_test_suite

   1 #!/bin/bash
   2
   3 set -e
   4 set -u
   5 set -o pipefail
   6
   7 NAME=$(basename "${0}")
   8
   9 GENERAL_DEPENDENCIES="
  10 cucumber
  11 devscripts
  12 dnsmasq-base
  13 ffmpeg
  14 gawk
  15 git
  16 imagemagick
  17 libcap2-bin
  18 libvirt-clients
  19 libvirt-daemon-system
  20 libvirt-dev
  21 libvirt0
  22 obfs4proxy
  23 openssh-server
  24 ovmf
  25 pry
  26 python3-opencv
  27 python3-pil
  28 python3-slixmpp
  29 qemu-system-x86
  30 qrencode
  31 ruby-debug-inspector
  32 ruby-guestfs
  33 ruby-json
  34 ruby-libvirt
  35 ruby-net-dns
  36 ruby-packetfu
  37 ruby-rb-inotify
  38 ruby-rspec
  39 ruby-test-unit
  40 seabios
  41 tcpdump
  42 tcplay
  43 tor
  44 unclutter
  45 virt-viewer
  46 xdotool
  47 xvfb
  48 "
  49
  50 # Recent tor client require a Chutney network with tor >= 0.4.4.6 (#18190)
  51 MINIMUM_TOR_VERSION=0.4.4.6
  52
  53 usage() {
  54     echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
  55 Sets up an appropriate environment and invokes cucumber. Note that this script
  56 must be run from the Tails source directory root.
  57
  58 Options for '@product' features:
  59   --allow-non-root   Normally the test suite must be run as root, but if you
  60                      really know what you are doing this option allows any
  61                      user to run it.
  62   --artifacts-base-uri URI
  63                      Pretend that the artifact is located at URI when printing
  64                      its location during a scenario failure. This is useful if
  65                      you intend to serve the artifacts via the web, for
  66                      instance.
  67   --capture          Captures failed scenarios into videos stored in the
  68                      temporary directory (see --tmpdir below) using x264
  69                      encoding. Requires x264.
  70   --capture-all      Keep videos for all scenarios, including those that
  71                      succeed (implies --capture).
  72   --interactive-debugging
  73                      On failure, pause test suite until pressing Enter. Also
  74                      offer the option to open an interactive Ruby shell (pry)
  75                      in the Cucumber world's context.
  76   --image-bumping-mode
  77                      When any image matching fails, enter an interactive mode
  78                      that allows to update the image. If run from a graphical
  79                      environment, any found candidate image will be displayed
  80                      in a pop-up.
  81   --keep-chutney     Don't ever clean Chutney data directory.
  82                      This can be a big time saver when debugging steps
  83                      when --keep-snapshots is not an option.
  84   --keep-snapshots   Don't ever delete any snapshots (including ones marked as
  85                      temporary). This can be a big time saver when debugging new
  86                      features. Implies --keep-chutney.
  87   --disable-chutney  EXPERIMENTAL: All tests will be run using the real Tor
  88                      network, not a simulated one.
  89                      Expect this to break many test cases.
  90   --late-patch
  91   --late-patch=FILE
  92                      Copies files into the VM before running the test suite,
  93                      which often can avoid a rebuild.
  94                      FILE is a text file where each line contains tab-separated
  95                      source and destination. If only a source is given, it is
  96                      assumed to be a file inside config/chroot_local-includes,
  97                      and the corresponding destination is inferred. Lines with
  98                      a leading \"#\" are ignored.
  99                      FILE is optional, and if not given we copy in all files in
 100                      config/chroot_local-includes that have changed since the
 101                      commit the system under testing was built from, including
 102                      untracked files.
 103   --early-patch      Boots the system with the \"early_patch=umount\" cmdline option.
 104                      See wiki/src/contribute/build/early-patch.mdwn for details
 105                      This is useful when you need something to be patched early
 106                      in the boot process; or if you need to run arbitrary
 107                      commands, not just copying files
 108   --all-tests        Don't skip tests which have a @fragile or @skip_by_default tag.
 109   --tmpdir           Directory where various temporary files are written
 110                      during a test, e.g. VM snapshots and memory dumps,
 111                      failure screenshots, pcap files and disk images
 112                      (default is TMPDIR in the environment, and if unset,
 113                      /tmp/TailsToaster).
 114   --view             Shows the test session in a windows. Requires x11vnc
 115                      and tigervnc-viewer.
 116   --view-interact    The test session can be \"touched\". For debugging purposes
 117                      only.
 118   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
 119   --iso IMAGE        Test '@product' features using IMAGE.
 120   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
 121                      an older version of Tails, which this options sets to
 122                      IMAGE. If none is given, it defaults to the same IMAGE
 123                      given by --iso, which will be good enough for most testing
 124                      purposes.
 125
 126 Note that '@source' features has no relevant options.
 127
 128 CUCUMBER_ARGS can be used to specify which features to be run, but also any
 129 cucumber option, although then you must pass \`--\` first to let this wrapper
 130 script know that we're done with *its* options. For debugging purposes, a
 131 'debug' formatter has been added so pretty debugging can be enabled with
 132 \`--format debug\`. You could even combine the default (pretty) formatter with
 133 pretty debugging printed to a file with \`--format pretty --format debug
 134 --out debug.log\`.
 135 "
 136 }
 137
 138 error() {
 139     echo "${NAME}: error: ${*}" >&2
 140     exit 1
 141 }
 142
 143 package_installed() {
 144     local ret
 145     set +o pipefail
 146     if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
 147         ret=0
 148     else
 149         ret=1
 150     fi
 151     set -o pipefail
 152     return ${ret}
 153 }
 154
 155 check_dependencies() {
 156     while [ -n "${1:-}" ]; do
 157         if ! command -v "${1}" >/dev/null && ! package_installed "${1}" ; then
 158             error "'${1}' is missing, please install it and run again."
 159         fi
 160         shift
 161     done
 162 }
 163
 164 check_tor_version() {
 165     local tor_version
 166     tor_version=$(tor --version | grep -Po '^Tor version \K.*' | sed --regexp-extended 's,[.]$,,')
 167     echo "tor version: $tor_version"
 168     if dpkg --compare-versions "$tor_version" lt "$MINIMUM_TOR_VERSION"; then
 169         error "Please upgrade to tor ${MINIMUM_TOR_VERSION} or newer."
 170     fi
 171 }
 172
 173 check_virt_viewer_version() {
 174     local version
 175     version="$(dpkg-query --show --showformat='${VERSION}' virt-viewer)"
 176     if dpkg --compare-versions "${version}" ge 10.0; then
 177         if ! echo "${version}" | grep -q 'tails$'; then
 178             error 'Please install virt-viewer from the isotester-bookworm APT suite' \
 179                   '(instructions: https://tails.net/contribute/release_process/test/setup/' \
 180                   'details: https://gitlab.tails.boum.org/tails/tails/-/issues/19064)'
 181         fi
 182     fi
 183 }
 184
 185 display_in_use() {
 186     [ -e "/tmp/.X${1#:}-lock" ] || [ -e "/tmp/.X11-unix/X${1#:}" ]
 187 }
 188
 189 next_free_display() {
 190     display_nr=0
 191     while display_in_use ":${display_nr}"; do
 192         display_nr=$((display_nr+1))
 193     done
 194     echo ":${display_nr}"
 195 }
 196
 197 test_suite_cleanup() {
 198     if [ -n "${XVFB_PID:-}" ]; then
 199         (kill -0 "${XVFB_PID}" 2>/dev/null && kill "${XVFB_PID}") || /bin/true
 200     fi
 201     return $?
 202 }
 203
 204 start_xvfb() {
 205     Xvfb "$TARGET_DISPLAY" -screen 0 1024x768x24+32 -noreset >/dev/null 2>&1 &
 206     XVFB_PID=$!
 207     # Wait for Xvfb to run on TARGET_DISPLAY
 208     until display_in_use "$TARGET_DISPLAY"; do
 209         sleep 1
 210     done
 211     echo "Virtual X framebuffer started on display ${TARGET_DISPLAY}"
 212     # Hide the mouse cursor so it won't be in the way when we are
 213     # trying to match images.
 214     unclutter -display "$TARGET_DISPLAY" -root -idle 0.1 >/dev/null 2>&1 &
 215 }
 216
 217 start_vnc_server() {
 218     check_dependencies x11vnc
 219     # shellcheck disable=SC2086
 220     VNC_SERVER_PORT="$(env -u WAYLAND_DISPLAY x11vnc \
 221                               -listen localhost -display "${TARGET_DISPLAY}" \
 222                               -bg -nopw -forever ${TAILS_X11VNC_OPTS:-} 2>&1 | \
 223                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
 224     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 225 }
 226
 227 start_vnc_viewer() {
 228     opts=
 229     if [[ "${VNC_VIEWER_VIEWONLY}" = yes ]]; then
 230         opts=-viewonly
 231     fi
 232     check_dependencies tigervnc-viewer
 233     xtigervncviewer \
 234         -nojpeg $opts \
 235         -RemoteResize=0 \
 236         -AcceptClipboard=0 \
 237         -SendClipboard=0 \
 238         -SetPrimary=0 \
 239         -SendPrimary=0 \
 240         "localhost:${VNC_SERVER_PORT}" 1>/dev/null 2>&1 &
 241 }
 242
 243 # main script
 244
 245 # Set default values of environment variables used by this script to
 246 # pass options to cucumber (unless they are already set).
 247 ALLOW_NON_ROOT=${ALLOW_NON_ROOT-}
 248 ARTIFACTS_BASE_URI=${ARTIFACTS_BASE_URI-}
 249 CAPTURE=${CAPTURE-}
 250 CAPTURE_ALL=${CAPTURE_ALL-}
 251 VNC_VIEWER=${VNC_VIEWER-}
 252 VNC_VIEWER_VIEWONLY=${VNC_VIEWER_VIEWONLY-yes}
 253 VNC_SERVER=${VNC_SERVER-}
 254 INTERACTIVE_DEBUGGING=${INTERACTIVE_DEBUGGING-}
 255 IMAGE_BUMPING_MODE=${IMAGE_BUMPING_MODE-}
 256 KEEP_CHUTNEY=${KEEP_CHUTNEY-}
 257 KEEP_SNAPSHOTS=${KEEP_SNAPSHOTS-}
 258 TAILS_ISO=${TAILS_ISO-}
 259 OLD_TAILS_ISO=${OLD_TAILS_ISO-}
 260 EARLY_PATCH=${EARLY_PATCH-}
 261 EXTRA_BOOT_OPTIONS=${EXTRA_BOOT_OPTIONS-}
 262 ALL_TESTS=${ALL_TESTS-}
 263
 264 LONGOPTS="allow-non-root,artifacts-base-uri:,view,view-interact,vnc-server-only,capture,capture-all,help,tmpdir:,keep-chutney,keep-snapshots,disable-chutney,late-patch::,early-patch,extra-boot-options:,all-tests,iso:,old-iso:,interactive-debugging,image-bumping-mode"
 265 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 266 eval set -- "$OPTS"
 267 while [ $# -gt 0 ]; do
 268     case $1 in
 269         --allow-non-root)
 270             if ! /sbin/getcap /usr/bin/tcpdump | grep -q 'cap_net_raw=eip'; then
 271                 error "/usr/bin/tcpdump lacks cap_net_raw=eip"
 272             fi
 273             ALLOW_NON_ROOT="yes"
 274             ;;
 275         --artifacts-base-uri)
 276             shift
 277             export ARTIFACTS_BASE_URI="${1}"
 278             ;;
 279         --view)
 280             VNC_VIEWER=yes
 281             VNC_VIEWER_VIEWONLY=yes
 282             VNC_SERVER=yes
 283             ;;
 284         --view-interact)
 285             VNC_VIEWER=yes
 286             VNC_VIEWER_VIEWONLY=
 287             VNC_SERVER=yes
 288             ;;
 289         --vnc-server-only)
 290             VNC_VIEWER=
 291             VNC_SERVER=yes
 292             ;;
 293         --capture)
 294             check_dependencies x264
 295             export CAPTURE="yes"
 296             ;;
 297         --capture-all)
 298             check_dependencies x264
 299             export CAPTURE="yes"
 300             export CAPTURE_ALL="yes"
 301             ;;
 302         --interactive-debugging)
 303             export INTERACTIVE_DEBUGGING="yes"
 304             ;;
 305         --image-bumping-mode)
 306             export IMAGE_BUMPING_MODE="yes"
 307             ;;
 308         --keep-chutney)
 309             export KEEP_CHUTNEY="yes"
 310             ;;
 311         --keep-snapshots)
 312             export KEEP_CHUTNEY="yes"
 313             export KEEP_SNAPSHOTS="yes"
 314             ;;
 315         --disable-chutney)
 316             export DISABLE_CHUTNEY="yes"
 317             ;;
 318         --late-patch)
 319             shift
 320             if [ -n "${1:-}" ]; then
 321                 LATE_PATCH="$(realpath -s "$1")"
 322                 if [ ! -e "$LATE_PATCH" ]; then
 323                     error "--late-patch: FILE doesn't exist"
 324                 fi
 325             else
 326                 LATE_PATCH=''
 327             fi
 328             export LATE_PATCH
 329             ;;
 330         --early-patch)
 331             export EARLY_PATCH="yes"
 332             ;;
 333         --extra-boot-options)
 334             shift
 335             export EXTRA_BOOT_OPTIONS="$1"
 336             ;;
 337         --all-tests)
 338             ALL_TESTS="yes"
 339             ;;
 340         --tmpdir)
 341             shift
 342             TMPDIR="$(readlink -f "$1")"
 343             export TMPDIR
 344             ;;
 345         --iso)
 346             shift
 347             TAILS_ISO="$(realpath -s "$1")"
 348             export TAILS_ISO
 349             ;;
 350         --old-iso)
 351             shift
 352             OLD_TAILS_ISO="$(realpath -s "$1")"
 353             export OLD_TAILS_ISO
 354             ;;
 355         --help)
 356             usage
 357             exit 0
 358             ;;
 359         --)
 360             shift
 361             break
 362             ;;
 363     esac
 364     shift
 365 done
 366
 367 trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
 368
 369 if [ "${EUID}" -ne 0 ] && [ -z "${ALLOW_NON_ROOT}" ]; then
 370     error "you are not running as root; if you really know what you are" \
 371           "doing, see the --allow-non-root option"
 372 fi
 373
 374 # shellcheck disable=SC2086
 375 check_dependencies ${GENERAL_DEPENDENCIES}
 376 check_tor_version
 377 check_virt_viewer_version
 378
 379 TARGET_DISPLAY=$(next_free_display)
 380
 381 start_xvfb
 382
 383 if [ -n "${VNC_SERVER:-}" ]; then
 384     start_vnc_server
 385 fi
 386 if [ -n "${VNC_VIEWER:-}" ]; then
 387     start_vnc_viewer
 388 fi
 389
 390 TAGS_ARGS=""
 391 if [ -n "${JENKINS_URL:-}" ]; then
 392     . auto/scripts/utils.sh
 393
 394     if echo "${GIT_BRANCH}" | grep -q -E '[+-]real-Tor$'; then
 395         export DISABLE_CHUTNEY="yes"
 396         TAGS_ARGS="--tag @supports_real_tor"
 397     # The current Git state may not reflect the state at the time the
 398     # upstream job was started (e.g. since then we git fetch + git
 399     # reset --hard) so we trust the Git state described in Jenkins'
 400     # environment variables instead.
 401     elif echo "${GIT_BRANCH}" | grep -q -E '[+-]force-all-tests$' \
 402        || git_on_a_tag \
 403        || [ "${GIT_BRANCH#origin/}" = feature/bookworm ] \
 404        || [ "${GIT_BRANCH#origin/}" = testing ] \
 405        || [ "${GIT_BRANCH#origin/}" = devel ] \
 406        || [ -n "${ALL_TESTS:-}" ]; then
 407         TAGS_ARGS=""
 408     else
 409         TAGS_ARGS="${TAGS_ARGS} --tag ~@fragile --tag ~@skip_by_default"
 410     fi
 411     if [ "${UPSTREAMJOB_GIT_COMMIT}" != "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}" ] && \
 412            git_only_doc_changes_since "${UPSTREAMJOB_GIT_BASE_BRANCH_HEAD}"; then
 413         TAGS_ARGS="${TAGS_ARGS} --tag @doc"
 414     fi
 415 fi
 416
 417 export USER_DISPLAY="${DISPLAY:-}"
 418 export DISPLAY=${TARGET_DISPLAY}
 419
 420 # shellcheck disable=SC2086
 421 cucumber --expand ${TAGS_ARGS} "${@}"