scripts/device-crash-test

   1 #!/usr/bin/env python
   2 #
   3 #  Copyright (c) 2017 Red Hat Inc
   4 #
   5 # Author:
   6 #  Eduardo Habkost <ehabkost@redhat.com>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License along
  19 # with this program; if not, write to the Free Software Foundation, Inc.,
  20 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21
  22 """
  23 Run QEMU with all combinations of -machine and -device types,
  24 check for crashes and unexpected errors.
  25 """
  26 from __future__ import print_function
  27
  28 import sys
  29 import os
  30 import glob
  31 import logging
  32 import traceback
  33 import re
  34 import random
  35 import argparse
  36 from itertools import chain
  37
  38 from qemu import QEMUMachine
  39
  40 logger = logging.getLogger('device-crash-test')
  41 dbg = logger.debug
  42
  43
  44 # Purposes of the following whitelist:
  45 # * Avoiding verbose log messages when we find known non-fatal
  46 #   (exitcode=1) errors
  47 # * Avoiding fatal errors when we find known crashes
  48 # * Skipping machines/devices that are known not to work out of
  49 #   the box, when running in --quick mode
  50 #
  51 # Keeping the whitelist updated is desirable, but not required,
  52 # because unexpected cases where QEMU exits with exitcode=1 will
  53 # just trigger a INFO message.
  54
  55 # Valid whitelist entry keys:
  56 # * accel: regexp, full match only
  57 # * machine: regexp, full match only
  58 # * device: regexp, full match only
  59 # * log: regexp, partial match allowed
  60 # * exitcode: if not present, defaults to 1. If None, matches any exitcode
  61 # * warn: if True, matching failures will be logged as warnings
  62 # * expected: if True, QEMU is expected to always fail every time
  63 #   when testing the corresponding test case
  64 # * loglevel: log level of log output when there's a match.
  65 ERROR_WHITELIST = [
  66     # Machines that won't work out of the box:
  67     #             MACHINE                         | ERROR MESSAGE
  68     {'machine':'niagara', 'expected':True},       # Unable to load a firmware for -M niagara
  69     {'machine':'boston', 'expected':True},        # Please provide either a -kernel or -bios argument
  70     {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null)
  71
  72     # devices that don't work out of the box because they require extra options to "-device DEV":
  73     #            DEVICE                                    | ERROR MESSAGE
  74     {'device':'.*-(i386|x86_64)-cpu', 'expected':True},    # CPU socket-id is not set
  75     {'device':'icp', 'expected':True},                     # icp_realize: required link 'xics' not found: Property '.xics' not found
  76     {'device':'ics', 'expected':True},                     # ics_base_realize: required link 'xics' not found: Property '.xics' not found
  77     # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True
  78     {'device':'ide-cd'},                                 # No drive specified
  79     {'device':'ide-drive', 'expected':True},               # No drive specified
  80     {'device':'ide-hd', 'expected':True},                  # No drive specified
  81     {'device':'ipmi-bmc-extern', 'expected':True},         # IPMI external bmc requires chardev attribute
  82     {'device':'isa-debugcon', 'expected':True},            # Can't create serial device, empty char device
  83     {'device':'isa-ipmi-bt', 'expected':True},             # IPMI device requires a bmc attribute to be set
  84     {'device':'isa-ipmi-kcs', 'expected':True},            # IPMI device requires a bmc attribute to be set
  85     {'device':'isa-parallel', 'expected':True},            # Can't create serial device, empty char device
  86     {'device':'ivshmem', 'expected':True},                 # You must specify either 'shm' or 'chardev'
  87     {'device':'ivshmem-doorbell', 'expected':True},        # You must specify a 'chardev'
  88     {'device':'ivshmem-plain', 'expected':True},           # You must specify a 'memdev'
  89     {'device':'loader', 'expected':True},                  # please include valid arguments
  90     {'device':'nand', 'expected':True},                    # Unsupported NAND block size 0x1
  91     {'device':'nvdimm', 'expected':True},                  # 'memdev' property is not set
  92     {'device':'nvme', 'expected':True},                    # Device initialization failed
  93     {'device':'pc-dimm', 'expected':True},                 # 'memdev' property is not set
  94     {'device':'pci-bridge', 'expected':True},              # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
  95     {'device':'pci-bridge-seat', 'expected':True},         # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
  96     {'device':'pxb', 'expected':True},                     # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
  97     {'device':'scsi-block', 'expected':True},              # drive property not set
  98     {'device':'scsi-disk', 'expected':True},               # drive property not set
  99     {'device':'scsi-generic', 'expected':True},            # drive property not set
 100     {'device':'scsi-hd', 'expected':True},                 # drive property not set
 101     {'device':'spapr-pci-host-bridge', 'expected':True},   # BUID not specified for PHB
 102     {'device':'spapr-rng', 'expected':True},               # spapr-rng needs an RNG backend!
 103     {'device':'spapr-vty', 'expected':True},               # chardev property not set
 104     {'device':'tpm-tis', 'expected':True},                 # tpm_tis: backend driver with id (null) could not be found
 105     {'device':'unimplemented-device', 'expected':True},    # property 'size' not specified or zero
 106     {'device':'usb-braille', 'expected':True},             # Property chardev is required
 107     {'device':'usb-mtp', 'expected':True},                 # rootdir property must be configured
 108     {'device':'usb-redir', 'expected':True},               # Parameter 'chardev' is missing
 109     {'device':'usb-serial', 'expected':True},              # Property chardev is required
 110     {'device':'usb-storage', 'expected':True},             # drive property not set
 111     {'device':'vfio-amd-xgbe', 'expected':True},           # -device vfio-amd-xgbe: vfio error: wrong host device name
 112     {'device':'vfio-calxeda-xgmac', 'expected':True},      # -device vfio-calxeda-xgmac: vfio error: wrong host device name
 113     {'device':'vfio-pci', 'expected':True},                # No provided host device
 114     {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0
 115     {'device':'vhost-scsi.*', 'expected':True},            # vhost-scsi: missing wwpn
 116     {'device':'vhost-vsock-device', 'expected':True},      # guest-cid property must be greater than 2
 117     {'device':'vhost-vsock-pci', 'expected':True},         # guest-cid property must be greater than 2
 118     {'device':'virtio-9p-ccw', 'expected':True},           # 9pfs device couldn't find fsdev with the id = NULL
 119     {'device':'virtio-9p-device', 'expected':True},        # 9pfs device couldn't find fsdev with the id = NULL
 120     {'device':'virtio-9p-pci', 'expected':True},           # 9pfs device couldn't find fsdev with the id = NULL
 121     {'device':'virtio-blk-ccw', 'expected':True},          # drive property not set
 122     {'device':'virtio-blk-device', 'expected':True},       # drive property not set
 123     {'device':'virtio-blk-device', 'expected':True},       # drive property not set
 124     {'device':'virtio-blk-pci', 'expected':True},          # drive property not set
 125     {'device':'virtio-crypto-ccw', 'expected':True},       # 'cryptodev' parameter expects a valid object
 126     {'device':'virtio-crypto-device', 'expected':True},    # 'cryptodev' parameter expects a valid object
 127     {'device':'virtio-crypto-pci', 'expected':True},       # 'cryptodev' parameter expects a valid object
 128     {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required
 129     {'device':'virtio-input-host-pci', 'expected':True},   # evdev property is required
 130     {'device':'xen-pvdevice', 'expected':True},            # Device ID invalid, it must always be supplied
 131     {'device':'vhost-vsock-ccw', 'expected':True},         # guest-cid property must be greater than 2
 132     {'device':'zpci', 'expected':True},                    # target must be defined
 133     {'device':'pnv-(occ|icp|lpc)', 'expected':True},       # required link 'xics' not found: Property '.xics' not found
 134     {'device':'powernv-cpu-.*', 'expected':True},          # pnv_core_realize: required link 'xics' not found: Property '.xics' not found
 135
 136     # ioapic devices are already created by pc and will fail:
 137     {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed
 138     {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True},     # Only 1 ioapics allowed
 139
 140     # "spapr-cpu-core needs a pseries machine"
 141     {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True},
 142
 143     # KVM-specific devices shouldn't be tried without accel=kvm:
 144     {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True},
 145
 146     # xen-specific machines and devices:
 147     {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True},
 148     {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True},
 149
 150     # this fails on some machine-types, but not all, so they don't have expected=True:
 151     {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide
 152
 153     # Silence INFO messages for errors that are common on multiple
 154     # devices/machines:
 155     {'log':r"No '[\w-]+' bus found for device '[\w-]+'"},
 156     {'log':r"images* must be given with the 'pflash' parameter"},
 157     {'log':r"(Guest|ROM|Flash|Kernel) image must be specified"},
 158     {'log':r"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"},
 159     {'log':r"Couldn't find rom image '[\w-]+\.bin'"},
 160     {'log':r"speed mismatch trying to attach usb device"},
 161     {'log':r"Can't create a second ISA bus"},
 162     {'log':r"duplicate fw_cfg file name"},
 163     # sysbus-related error messages: most machines reject most dynamic sysbus devices:
 164     {'log':r"Option '-device [\w.,-]+' cannot be handled by this machine"},
 165     {'log':r"Device [\w.,-]+ is not supported by this machine yet"},
 166     {'log':r"Device [\w.,-]+ can not be dynamically instantiated"},
 167     {'log':r"Platform Bus: Can not fit MMIO region of size "},
 168     # other more specific errors we will ignore:
 169     {'device':'.*-spapr-cpu-core', 'log':r"CPU core type should be"},
 170     {'log':r"MSI(-X)? is not supported by interrupt controller"},
 171     {'log':r"pxb-pcie? devices cannot reside on a PCIe? bus"},
 172     {'log':r"Ignoring smp_cpus value"},
 173     {'log':r"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"},
 174     {'log':r"This CPU requires a smaller page size than the system is using"},
 175     {'log':r"MSI-X support is mandatory in the S390 architecture"},
 176     {'log':r"rom check and register reset failed"},
 177     {'log':r"Unable to initialize GIC, CPUState for CPU#0 not valid"},
 178     {'log':r"Multiple VT220 operator consoles are not supported"},
 179     {'log':r"core 0 already populated"},
 180     {'log':r"could not find stage1 bootloader"},
 181
 182     # other exitcode=1 failures not listed above will just generate INFO messages:
 183     {'exitcode':1, 'loglevel':logging.INFO},
 184
 185     # KNOWN CRASHES:
 186     # Known crashes will generate error messages, but won't be fatal.
 187     # Those entries must be removed once we fix the crashes.
 188     {'exitcode':-6, 'log':r"Device 'serial0' is in use", 'loglevel':logging.ERROR},
 189     {'exitcode':-6, 'log':r"qemu_net_client_setup: Assertion `!peer->peer' failed", 'loglevel':logging.ERROR},
 190     {'exitcode':-6, 'log':r'RAMBlock "[\w.-]+" already registered', 'loglevel':logging.ERROR},
 191     {'exitcode':-6, 'log':r"find_ram_offset: Assertion `size != 0' failed.", 'loglevel':logging.ERROR},
 192     {'exitcode':-6, 'log':r"add_cpreg_to_hashtable: code should not be reached", 'loglevel':logging.ERROR},
 193     {'exitcode':-6, 'log':r"qemu_alloc_display: Assertion `surface->image != NULL' failed", 'loglevel':logging.ERROR},
 194     {'exitcode':-6, 'log':r"Unexpected error in error_set_from_qdev_prop_error", 'loglevel':logging.ERROR},
 195     {'exitcode':-6, 'log':r"Object .* is not an instance of type spapr-machine", 'loglevel':logging.ERROR},
 196     {'exitcode':-6, 'log':r"Object .* is not an instance of type generic-pc-machine", 'loglevel':logging.ERROR},
 197     {'exitcode':-6, 'log':r"Object .* is not an instance of type e500-ccsr", 'loglevel':logging.ERROR},
 198     {'exitcode':-6, 'log':r"vmstate_register_with_alias_id: Assertion `!se->compat \|\| se->instance_id == 0' failed", 'loglevel':logging.ERROR},
 199
 200     # everything else (including SIGABRT and SIGSEGV) will be a fatal error:
 201     {'exitcode':None, 'fatal':True, 'loglevel':logging.FATAL},
 202 ]
 203
 204
 205 def whitelistTestCaseMatch(wl, t):
 206     """Check if a test case specification can match a whitelist entry
 207
 208     This only checks if a whitelist entry is a candidate match
 209     for a given test case, it won't check if the test case
 210     results/output match the entry.  See whitelistResultMatch().
 211     """
 212     return (('machine' not in wl or
 213              'machine' not in t or
 214              re.match(wl['machine'] + '$', t['machine'])) and
 215             ('accel' not in wl or
 216              'accel' not in t or
 217              re.match(wl['accel'] + '$', t['accel'])) and
 218             ('device' not in wl or
 219              'device' not in t or
 220              re.match(wl['device'] + '$', t['device'])))
 221
 222
 223 def whitelistCandidates(t):
 224     """Generate the list of candidates that can match a test case"""
 225     for i, wl in enumerate(ERROR_WHITELIST):
 226         if whitelistTestCaseMatch(wl, t):
 227             yield (i, wl)
 228
 229
 230 def findExpectedResult(t):
 231     """Check if there's an expected=True whitelist entry for a test case
 232
 233     Returns (i, wl) tuple, where i is the index in
 234     ERROR_WHITELIST and wl is the whitelist entry itself.
 235     """
 236     for i, wl in whitelistCandidates(t):
 237         if wl.get('expected'):
 238             return (i, wl)
 239
 240
 241 def whitelistResultMatch(wl, r):
 242     """Check if test case results/output match a whitelist entry
 243
 244     It is valid to call this function only if
 245     whitelistTestCaseMatch() is True for the entry (e.g. on
 246     entries returned by whitelistCandidates())
 247     """
 248     assert whitelistTestCaseMatch(wl, r['testcase'])
 249     return ((wl.get('exitcode', 1) is None or
 250              r['exitcode'] == wl.get('exitcode', 1)) and
 251             ('log' not in wl or
 252              re.search(wl['log'], r['log'], re.MULTILINE)))
 253
 254
 255 def checkResultWhitelist(r):
 256     """Look up whitelist entry for a given test case result
 257
 258     Returns (i, wl) tuple, where i is the index in
 259     ERROR_WHITELIST and wl is the whitelist entry itself.
 260     """
 261     for i, wl in whitelistCandidates(r['testcase']):
 262         if whitelistResultMatch(wl, r):
 263             return i, wl
 264
 265     raise Exception("this should never happen")
 266
 267
 268 def qemuOptsEscape(s):
 269     """Escape option value QemuOpts"""
 270     return s.replace(",", ",,")
 271
 272
 273 def formatTestCase(t):
 274     """Format test case info as "key=value key=value" for prettier logging output"""
 275     return ' '.join('%s=%s' % (k, v) for k, v in t.items())
 276
 277
 278 def qomListTypeNames(vm, **kwargs):
 279     """Run qom-list-types QMP command, return type names"""
 280     types = vm.command('qom-list-types', **kwargs)
 281     return [t['name'] for t in types]
 282
 283
 284 def infoQDM(vm):
 285     """Parse 'info qdm' output"""
 286     args = {'command-line': 'info qdm'}
 287     devhelp = vm.command('human-monitor-command', **args)
 288     for l in devhelp.split('\n'):
 289         l = l.strip()
 290         if l == '' or l.endswith(':'):
 291             continue
 292         d = {'name': re.search(r'name "([^"]+)"', l).group(1),
 293              'no-user': (re.search(', no-user', l) is not None)}
 294         yield d
 295
 296
 297 class QemuBinaryInfo(object):
 298     def __init__(self, binary, devtype):
 299         if devtype is None:
 300             devtype = 'device'
 301
 302         self.binary = binary
 303         self._machine_info = {}
 304
 305         dbg("devtype: %r", devtype)
 306         args = ['-S', '-machine', 'none,accel=kvm:tcg']
 307         dbg("querying info for QEMU binary: %s", binary)
 308         vm = QEMUMachine(binary=binary, args=args)
 309         vm.launch()
 310         try:
 311             self.alldevs = set(qomListTypeNames(vm, implements=devtype, abstract=False))
 312             # there's no way to query DeviceClass::user_creatable using QMP,
 313             # so use 'info qdm':
 314             self.no_user_devs = set([d['name'] for d in infoQDM(vm, ) if d['no-user']])
 315             self.machines = list(m['name'] for m in vm.command('query-machines'))
 316             self.user_devs = self.alldevs.difference(self.no_user_devs)
 317             self.kvm_available = vm.command('query-kvm')['enabled']
 318         finally:
 319             vm.shutdown()
 320
 321     def machineInfo(self, machine):
 322         """Query for information on a specific machine-type
 323
 324         Results are cached internally, in case the same machine-
 325         type is queried multiple times.
 326         """
 327         if machine in self._machine_info:
 328             return self._machine_info[machine]
 329
 330         mi = {}
 331         args = ['-S', '-machine', '%s' % (machine)]
 332         dbg("querying machine info for binary=%s machine=%s", self.binary, machine)
 333         vm = QEMUMachine(binary=self.binary, args=args)
 334         try:
 335             vm.launch()
 336             mi['runnable'] = True
 337         except KeyboardInterrupt:
 338             raise
 339         except:
 340             dbg("exception trying to run binary=%s machine=%s", self.binary, machine, exc_info=sys.exc_info())
 341             dbg("log: %r", vm.get_log())
 342             mi['runnable'] = False
 343
 344         vm.shutdown()
 345         self._machine_info[machine] = mi
 346         return mi
 347
 348
 349 BINARY_INFO = {}
 350
 351
 352 def getBinaryInfo(args, binary):
 353     if binary not in BINARY_INFO:
 354         BINARY_INFO[binary] = QemuBinaryInfo(binary, args.devtype)
 355     return BINARY_INFO[binary]
 356
 357
 358 def checkOneCase(args, testcase):
 359     """Check one specific case
 360
 361     Returns a dictionary containing failure information on error,
 362     or None on success
 363     """
 364     binary = testcase['binary']
 365     accel = testcase['accel']
 366     machine = testcase['machine']
 367     device = testcase['device']
 368
 369     dbg("will test: %r", testcase)
 370
 371     args = ['-S', '-machine', '%s,accel=%s' % (machine, accel),
 372             '-device', qemuOptsEscape(device)]
 373     cmdline = ' '.join([binary] + args)
 374     dbg("will launch QEMU: %s", cmdline)
 375     vm = QEMUMachine(binary=binary, args=args)
 376
 377     exc_traceback = None
 378     try:
 379         vm.launch()
 380     except KeyboardInterrupt:
 381         raise
 382     except:
 383         exc_traceback = traceback.format_exc()
 384         dbg("Exception while running test case")
 385     finally:
 386         vm.shutdown()
 387         ec = vm.exitcode()
 388         log = vm.get_log()
 389
 390     if exc_traceback is not None or ec != 0:
 391         return {'exc_traceback':exc_traceback,
 392                 'exitcode':ec,
 393                 'log':log,
 394                 'testcase':testcase,
 395                 'cmdline':cmdline}
 396
 397
 398 def binariesToTest(args, testcase):
 399     if args.qemu:
 400         r = args.qemu
 401     else:
 402         r = glob.glob('./*-softmmu/qemu-system-*')
 403     return r
 404
 405
 406 def accelsToTest(args, testcase):
 407     if getBinaryInfo(args, testcase['binary']).kvm_available:
 408         yield 'kvm'
 409     yield 'tcg'
 410
 411
 412 def machinesToTest(args, testcase):
 413     return getBinaryInfo(args, testcase['binary']).machines
 414
 415
 416 def devicesToTest(args, testcase):
 417     return getBinaryInfo(args, testcase['binary']).user_devs
 418
 419
 420 TESTCASE_VARIABLES = [
 421     ('binary', binariesToTest),
 422     ('accel', accelsToTest),
 423     ('machine', machinesToTest),
 424     ('device', devicesToTest),
 425 ]
 426
 427
 428 def genCases1(args, testcases, var, fn):
 429     """Generate new testcases for one variable
 430
 431     If an existing item already has a variable set, don't
 432     generate new items and just return it directly. This
 433     allows the "-t" command-line option to be used to choose
 434     a specific test case.
 435     """
 436     for testcase in testcases:
 437         if var in testcase:
 438             yield testcase.copy()
 439         else:
 440             for i in fn(args, testcase):
 441                 t = testcase.copy()
 442                 t[var] = i
 443                 yield t
 444
 445
 446 def genCases(args, testcase):
 447     """Generate test cases for all variables
 448     """
 449     cases = [testcase.copy()]
 450     for var, fn in TESTCASE_VARIABLES:
 451         dbg("var: %r, fn: %r", var, fn)
 452         cases = genCases1(args, cases, var, fn)
 453     return cases
 454
 455
 456 def casesToTest(args, testcase):
 457     cases = genCases(args, testcase)
 458     if args.random:
 459         cases = list(cases)
 460         cases = random.sample(cases, min(args.random, len(cases)))
 461     if args.debug:
 462         cases = list(cases)
 463         dbg("%d test cases to test", len(cases))
 464     if args.shuffle:
 465         cases = list(cases)
 466         random.shuffle(cases)
 467     return cases
 468
 469
 470 def logFailure(f, level):
 471     t = f['testcase']
 472     logger.log(level, "failed: %s", formatTestCase(t))
 473     logger.log(level, "cmdline: %s", f['cmdline'])
 474     for l in f['log'].strip().split('\n'):
 475         logger.log(level, "log: %s", l)
 476     logger.log(level, "exit code: %r", f['exitcode'])
 477     if f['exc_traceback']:
 478         logger.log(level, "exception:")
 479         for l in f['exc_traceback'].split('\n'):
 480             logger.log(level, "  %s", l.rstrip('\n'))
 481
 482
 483 def main():
 484     parser = argparse.ArgumentParser(description="QEMU -device crash test")
 485     parser.add_argument('-t', metavar='KEY=VALUE', nargs='*',
 486                         help="Limit test cases to KEY=VALUE",
 487                         action='append', dest='testcases', default=[])
 488     parser.add_argument('-d', '--debug', action='store_true',
 489                         help='debug output')
 490     parser.add_argument('-v', '--verbose', action='store_true', default=True,
 491                         help='verbose output')
 492     parser.add_argument('-q', '--quiet', dest='verbose', action='store_false',
 493                         help='non-verbose output')
 494     parser.add_argument('-r', '--random', type=int, metavar='COUNT',
 495                         help='run a random sample of COUNT test cases',
 496                         default=0)
 497     parser.add_argument('--shuffle', action='store_true',
 498                         help='Run test cases in random order')
 499     parser.add_argument('--dry-run', action='store_true',
 500                         help="Don't run any tests, just generate list")
 501     parser.add_argument('-D', '--devtype', metavar='TYPE',
 502                         help="Test only device types that implement TYPE")
 503     parser.add_argument('-Q', '--quick', action='store_true', default=True,
 504                         help="Quick mode: skip test cases that are expected to fail")
 505     parser.add_argument('-F', '--full', action='store_false', dest='quick',
 506                         help="Full mode: test cases that are expected to fail")
 507     parser.add_argument('--strict', action='store_true', dest='strict',
 508                         help="Treat all warnings as fatal")
 509     parser.add_argument('qemu', nargs='*', metavar='QEMU',
 510                         help='QEMU binary to run')
 511     args = parser.parse_args()
 512
 513     if args.debug:
 514         lvl = logging.DEBUG
 515     elif args.verbose:
 516         lvl = logging.INFO
 517     else:
 518         lvl = logging.WARN
 519     logging.basicConfig(stream=sys.stdout, level=lvl, format='%(levelname)s: %(message)s')
 520
 521     fatal_failures = []
 522     wl_stats = {}
 523     skipped = 0
 524     total = 0
 525
 526     tc = {}
 527     dbg("testcases: %r", args.testcases)
 528     if args.testcases:
 529         for t in chain(*args.testcases):
 530             for kv in t.split():
 531                 k, v = kv.split('=', 1)
 532                 tc[k] = v
 533
 534     if len(binariesToTest(args, tc)) == 0:
 535         print("No QEMU binary found", file=sys.stderr)
 536         parser.print_usage(sys.stderr)
 537         return 1
 538
 539     for t in casesToTest(args, tc):
 540         logger.info("running test case: %s", formatTestCase(t))
 541         total += 1
 542
 543         expected_match = findExpectedResult(t)
 544         if (args.quick and
 545                 (expected_match or
 546                  not getBinaryInfo(args, t['binary']).machineInfo(t['machine'])['runnable'])):
 547             dbg("skipped: %s", formatTestCase(t))
 548             skipped += 1
 549             continue
 550
 551         if args.dry_run:
 552             continue
 553
 554         try:
 555             f = checkOneCase(args, t)
 556         except KeyboardInterrupt:
 557             break
 558
 559         if f:
 560             i, wl = checkResultWhitelist(f)
 561             dbg("testcase: %r, whitelist match: %r", t, wl)
 562             wl_stats.setdefault(i, []).append(f)
 563             level = wl.get('loglevel', logging.DEBUG)
 564             logFailure(f, level)
 565             if wl.get('fatal') or (args.strict and level >= logging.WARN):
 566                 fatal_failures.append(f)
 567         else:
 568             dbg("success: %s", formatTestCase(t))
 569             if expected_match:
 570                 logger.warn("Didn't fail as expected: %s", formatTestCase(t))
 571
 572     logger.info("Total: %d test cases", total)
 573     if skipped:
 574         logger.info("Skipped %d test cases", skipped)
 575
 576     if args.debug:
 577         stats = sorted([(len(wl_stats.get(i, [])), wl) for i, wl in enumerate(ERROR_WHITELIST)])
 578         for count, wl in stats:
 579             dbg("whitelist entry stats: %d: %r", count, wl)
 580
 581     if fatal_failures:
 582         for f in fatal_failures:
 583             t = f['testcase']
 584             logger.error("Fatal failure: %s", formatTestCase(t))
 585         logger.error("Fatal failures on some machine/device combinations")
 586         return 1
 587
 588 if __name__ == '__main__':
 589     sys.exit(main())