3 # Copyright (c) 2017 Red Hat Inc
6 # Eduardo Habkost <ehabkost@redhat.com>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License along
19 # with this program; if not, write to the Free Software Foundation, Inc.,
20 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 Run QEMU with all combinations of -machine and -device types,
24 check for crashes and unexpected errors.
26 from __future__
import print_function
36 from itertools
import chain
38 from qemu
import QEMUMachine
40 logger
= logging
.getLogger('device-crash-test')
44 # Purposes of the following whitelist:
45 # * Avoiding verbose log messages when we find known non-fatal
47 # * Avoiding fatal errors when we find known crashes
48 # * Skipping machines/devices that are known not to work out of
49 # the box, when running in --quick mode
51 # Keeping the whitelist updated is desirable, but not required,
52 # because unexpected cases where QEMU exits with exitcode=1 will
53 # just trigger a INFO message.
55 # Valid whitelist entry keys:
56 # * accel: regexp, full match only
57 # * machine: regexp, full match only
58 # * device: regexp, full match only
59 # * log: regexp, partial match allowed
60 # * exitcode: if not present, defaults to 1. If None, matches any exitcode
61 # * warn: if True, matching failures will be logged as warnings
62 # * expected: if True, QEMU is expected to always fail every time
63 # when testing the corresponding test case
64 # * loglevel: log level of log output when there's a match.
66 # Machines that won't work out of the box:
67 # MACHINE | ERROR MESSAGE
68 {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara
69 {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument
70 {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null)
72 # devices that don't work out of the box because they require extra options to "-device DEV":
73 # DEVICE | ERROR MESSAGE
74 {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set
75 {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found
76 {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found
77 # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True
78 {'device':'ide-cd'}, # No drive specified
79 {'device':'ide-drive', 'expected':True}, # No drive specified
80 {'device':'ide-hd', 'expected':True}, # No drive specified
81 {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute
82 {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device
83 {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set
84 {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set
85 {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device
86 {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev'
87 {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev'
88 {'device':'loader', 'expected':True}, # please include valid arguments
89 {'device':'nand', 'expected':True}, # Unsupported NAND block size 0x1
90 {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set
91 {'device':'nvme', 'expected':True}, # Device initialization failed
92 {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set
93 {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
94 {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
95 {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
96 {'device':'scsi-block', 'expected':True}, # drive property not set
97 {'device':'scsi-disk', 'expected':True}, # drive property not set
98 {'device':'scsi-generic', 'expected':True}, # drive property not set
99 {'device':'scsi-hd', 'expected':True}, # drive property not set
100 {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB
101 {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend!
102 {'device':'spapr-vty', 'expected':True}, # chardev property not set
103 {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found
104 {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero
105 {'device':'usb-braille', 'expected':True}, # Property chardev is required
106 {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured
107 {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing
108 {'device':'usb-serial', 'expected':True}, # Property chardev is required
109 {'device':'usb-storage', 'expected':True}, # drive property not set
110 {'device':'vfio-amd-xgbe', 'expected':True}, # -device vfio-amd-xgbe: vfio error: wrong host device name
111 {'device':'vfio-calxeda-xgmac', 'expected':True}, # -device vfio-calxeda-xgmac: vfio error: wrong host device name
112 {'device':'vfio-pci', 'expected':True}, # No provided host device
113 {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0
114 {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn
115 {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2
116 {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2
117 {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
118 {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
119 {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
120 {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set
121 {'device':'virtio-blk-device', 'expected':True}, # drive property not set
122 {'device':'virtio-blk-device', 'expected':True}, # drive property not set
123 {'device':'virtio-blk-pci', 'expected':True}, # drive property not set
124 {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object
125 {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object
126 {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object
127 {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required
128 {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required
129 {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied
130 {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2
131 {'device':'zpci', 'expected':True}, # target must be defined
132 {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found
133 {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found
135 # ioapic devices are already created by pc and will fail:
136 {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed
137 {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed
139 # "spapr-cpu-core needs a pseries machine"
140 {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True},
142 # KVM-specific devices shouldn't be tried without accel=kvm:
143 {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True},
145 # xen-specific machines and devices:
146 {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True},
147 {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True},
149 # this fails on some machine-types, but not all, so they don't have expected=True:
150 {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide
152 # Silence INFO messages for errors that are common on multiple
154 {'log':r
"No '[\w-]+' bus found for device '[\w-]+'"},
155 {'log':r
"images* must be given with the 'pflash' parameter"},
156 {'log':r
"(Guest|ROM|Flash|Kernel) image must be specified"},
157 {'log':r
"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"},
158 {'log':r
"Couldn't find rom image '[\w-]+\.bin'"},
159 {'log':r
"speed mismatch trying to attach usb device"},
160 {'log':r
"Can't create a second ISA bus"},
161 {'log':r
"duplicate fw_cfg file name"},
162 # sysbus-related error messages: most machines reject most dynamic sysbus devices:
163 {'log':r
"Option '-device [\w.,-]+' cannot be handled by this machine"},
164 {'log':r
"Device [\w.,-]+ is not supported by this machine yet"},
165 {'log':r
"Device [\w.,-]+ can not be dynamically instantiated"},
166 {'log':r
"Platform Bus: Can not fit MMIO region of size "},
167 # other more specific errors we will ignore:
168 {'device':'.*-spapr-cpu-core', 'log':r
"CPU core type should be"},
169 {'log':r
"MSI(-X)? is not supported by interrupt controller"},
170 {'log':r
"pxb-pcie? devices cannot reside on a PCIe? bus"},
171 {'log':r
"Ignoring smp_cpus value"},
172 {'log':r
"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"},
173 {'log':r
"This CPU requires a smaller page size than the system is using"},
174 {'log':r
"MSI-X support is mandatory in the S390 architecture"},
175 {'log':r
"rom check and register reset failed"},
176 {'log':r
"Unable to initialize GIC, CPUState for CPU#0 not valid"},
177 {'log':r
"Multiple VT220 operator consoles are not supported"},
178 {'log':r
"core 0 already populated"},
179 {'log':r
"could not find stage1 bootloader"},
181 # other exitcode=1 failures not listed above will just generate INFO messages:
182 {'exitcode':1, 'loglevel':logging
.INFO
},
185 # Known crashes will generate error messages, but won't be fatal.
186 # Those entries must be removed once we fix the crashes.
187 {'exitcode':-6, 'log':r
"Device 'serial0' is in use", 'loglevel':logging
.ERROR
},
188 {'exitcode':-6, 'log':r
"qemu_net_client_setup: Assertion `!peer->peer' failed", 'loglevel':logging
.ERROR
},
189 {'exitcode':-6, 'log':r
'RAMBlock "[\w.-]+" already registered', 'loglevel':logging
.ERROR
},
190 {'exitcode':-6, 'log':r
"find_ram_offset: Assertion `size != 0' failed.", 'loglevel':logging
.ERROR
},
191 {'exitcode':-6, 'log':r
"add_cpreg_to_hashtable: code should not be reached", 'loglevel':logging
.ERROR
},
192 {'exitcode':-6, 'log':r
"qemu_alloc_display: Assertion `surface->image != NULL' failed", 'loglevel':logging
.ERROR
},
193 {'exitcode':-6, 'log':r
"Unexpected error in error_set_from_qdev_prop_error", 'loglevel':logging
.ERROR
},
194 {'exitcode':-6, 'log':r
"Object .* is not an instance of type spapr-machine", 'loglevel':logging
.ERROR
},
195 {'exitcode':-6, 'log':r
"Object .* is not an instance of type generic-pc-machine", 'loglevel':logging
.ERROR
},
196 {'exitcode':-6, 'log':r
"Object .* is not an instance of type e500-ccsr", 'loglevel':logging
.ERROR
},
197 {'exitcode':-6, 'log':r
"vmstate_register_with_alias_id: Assertion `!se->compat \|\| se->instance_id == 0' failed", 'loglevel':logging
.ERROR
},
199 # everything else (including SIGABRT and SIGSEGV) will be a fatal error:
200 {'exitcode':None, 'fatal':True, 'loglevel':logging
.FATAL
},
204 def whitelistTestCaseMatch(wl
, t
):
205 """Check if a test case specification can match a whitelist entry
207 This only checks if a whitelist entry is a candidate match
208 for a given test case, it won't check if the test case
209 results/output match the entry. See whitelistResultMatch().
211 return (('machine' not in wl
or
212 'machine' not in t
or
213 re
.match(wl
['machine'] + '$', t
['machine'])) and
214 ('accel' not in wl
or
216 re
.match(wl
['accel'] + '$', t
['accel'])) and
217 ('device' not in wl
or
219 re
.match(wl
['device'] + '$', t
['device'])))
222 def whitelistCandidates(t
):
223 """Generate the list of candidates that can match a test case"""
224 for i
, wl
in enumerate(ERROR_WHITELIST
):
225 if whitelistTestCaseMatch(wl
, t
):
229 def findExpectedResult(t
):
230 """Check if there's an expected=True whitelist entry for a test case
232 Returns (i, wl) tuple, where i is the index in
233 ERROR_WHITELIST and wl is the whitelist entry itself.
235 for i
, wl
in whitelistCandidates(t
):
236 if wl
.get('expected'):
240 def whitelistResultMatch(wl
, r
):
241 """Check if test case results/output match a whitelist entry
243 It is valid to call this function only if
244 whitelistTestCaseMatch() is True for the entry (e.g. on
245 entries returned by whitelistCandidates())
247 assert whitelistTestCaseMatch(wl
, r
['testcase'])
248 return ((wl
.get('exitcode', 1) is None or
249 r
['exitcode'] == wl
.get('exitcode', 1)) and
251 re
.search(wl
['log'], r
['log'], re
.MULTILINE
)))
254 def checkResultWhitelist(r
):
255 """Look up whitelist entry for a given test case result
257 Returns (i, wl) tuple, where i is the index in
258 ERROR_WHITELIST and wl is the whitelist entry itself.
260 for i
, wl
in whitelistCandidates(r
['testcase']):
261 if whitelistResultMatch(wl
, r
):
264 raise Exception("this should never happen")
267 def qemuOptsEscape(s
):
268 """Escape option value QemuOpts"""
269 return s
.replace(",", ",,")
272 def formatTestCase(t
):
273 """Format test case info as "key=value key=value" for prettier logging output"""
274 return ' '.join('%s=%s' % (k
, v
) for k
, v
in t
.items())
277 def qomListTypeNames(vm
, **kwargs
):
278 """Run qom-list-types QMP command, return type names"""
279 types
= vm
.command('qom-list-types', **kwargs
)
280 return [t
['name'] for t
in types
]
284 """Parse 'info qdm' output"""
285 args
= {'command-line': 'info qdm'}
286 devhelp
= vm
.command('human-monitor-command', **args
)
287 for l
in devhelp
.split('\n'):
289 if l
== '' or l
.endswith(':'):
291 d
= {'name': re
.search(r
'name "([^"]+)"', l
).group(1),
292 'no-user': (re
.search(', no-user', l
) is not None)}
296 class QemuBinaryInfo(object):
297 def __init__(self
, binary
, devtype
):
302 self
._machine
_info
= {}
304 dbg("devtype: %r", devtype
)
305 args
= ['-S', '-machine', 'none,accel=kvm:tcg']
306 dbg("querying info for QEMU binary: %s", binary
)
307 vm
= QEMUMachine(binary
=binary
, args
=args
)
310 self
.alldevs
= set(qomListTypeNames(vm
, implements
=devtype
, abstract
=False))
311 # there's no way to query DeviceClass::user_creatable using QMP,
313 self
.no_user_devs
= set([d
['name'] for d
in infoQDM(vm
, ) if d
['no-user']])
314 self
.machines
= list(m
['name'] for m
in vm
.command('query-machines'))
315 self
.user_devs
= self
.alldevs
.difference(self
.no_user_devs
)
316 self
.kvm_available
= vm
.command('query-kvm')['enabled']
320 def machineInfo(self
, machine
):
321 """Query for information on a specific machine-type
323 Results are cached internally, in case the same machine-
324 type is queried multiple times.
326 if machine
in self
._machine
_info
:
327 return self
._machine
_info
[machine
]
330 args
= ['-S', '-machine', '%s' % (machine
)]
331 dbg("querying machine info for binary=%s machine=%s", self
.binary
, machine
)
332 vm
= QEMUMachine(binary
=self
.binary
, args
=args
)
335 mi
['runnable'] = True
336 except KeyboardInterrupt:
339 dbg("exception trying to run binary=%s machine=%s", self
.binary
, machine
, exc_info
=sys
.exc_info())
340 dbg("log: %r", vm
.get_log())
341 mi
['runnable'] = False
344 self
._machine
_info
[machine
] = mi
351 def getBinaryInfo(args
, binary
):
352 if binary
not in BINARY_INFO
:
353 BINARY_INFO
[binary
] = QemuBinaryInfo(binary
, args
.devtype
)
354 return BINARY_INFO
[binary
]
357 def checkOneCase(args
, testcase
):
358 """Check one specific case
360 Returns a dictionary containing failure information on error,
363 binary
= testcase
['binary']
364 accel
= testcase
['accel']
365 machine
= testcase
['machine']
366 device
= testcase
['device']
368 dbg("will test: %r", testcase
)
370 args
= ['-S', '-machine', '%s,accel=%s' % (machine
, accel
),
371 '-device', qemuOptsEscape(device
)]
372 cmdline
= ' '.join([binary
] + args
)
373 dbg("will launch QEMU: %s", cmdline
)
374 vm
= QEMUMachine(binary
=binary
, args
=args
)
379 except KeyboardInterrupt:
382 exc_traceback
= traceback
.format_exc()
383 dbg("Exception while running test case")
389 if exc_traceback
is not None or ec
!= 0:
390 return {'exc_traceback':exc_traceback
,
397 def binariesToTest(args
, testcase
):
401 r
= glob
.glob('./*-softmmu/qemu-system-*')
405 def accelsToTest(args
, testcase
):
406 if getBinaryInfo(args
, testcase
['binary']).kvm_available
:
411 def machinesToTest(args
, testcase
):
412 return getBinaryInfo(args
, testcase
['binary']).machines
415 def devicesToTest(args
, testcase
):
416 return getBinaryInfo(args
, testcase
['binary']).user_devs
419 TESTCASE_VARIABLES
= [
420 ('binary', binariesToTest
),
421 ('accel', accelsToTest
),
422 ('machine', machinesToTest
),
423 ('device', devicesToTest
),
427 def genCases1(args
, testcases
, var
, fn
):
428 """Generate new testcases for one variable
430 If an existing item already has a variable set, don't
431 generate new items and just return it directly. This
432 allows the "-t" command-line option to be used to choose
433 a specific test case.
435 for testcase
in testcases
:
437 yield testcase
.copy()
439 for i
in fn(args
, testcase
):
445 def genCases(args
, testcase
):
446 """Generate test cases for all variables
448 cases
= [testcase
.copy()]
449 for var
, fn
in TESTCASE_VARIABLES
:
450 dbg("var: %r, fn: %r", var
, fn
)
451 cases
= genCases1(args
, cases
, var
, fn
)
455 def casesToTest(args
, testcase
):
456 cases
= genCases(args
, testcase
)
459 cases
= random
.sample(cases
, min(args
.random
, len(cases
)))
462 dbg("%d test cases to test", len(cases
))
465 random
.shuffle(cases
)
469 def logFailure(f
, level
):
471 logger
.log(level
, "failed: %s", formatTestCase(t
))
472 logger
.log(level
, "cmdline: %s", f
['cmdline'])
473 for l
in f
['log'].strip().split('\n'):
474 logger
.log(level
, "log: %s", l
)
475 logger
.log(level
, "exit code: %r", f
['exitcode'])
476 if f
['exc_traceback']:
477 logger
.log(level
, "exception:")
478 for l
in f
['exc_traceback'].split('\n'):
479 logger
.log(level
, " %s", l
.rstrip('\n'))
483 parser
= argparse
.ArgumentParser(description
="QEMU -device crash test")
484 parser
.add_argument('-t', metavar
='KEY=VALUE', nargs
='*',
485 help="Limit test cases to KEY=VALUE",
486 action
='append', dest
='testcases', default
=[])
487 parser
.add_argument('-d', '--debug', action
='store_true',
489 parser
.add_argument('-v', '--verbose', action
='store_true', default
=True,
490 help='verbose output')
491 parser
.add_argument('-q', '--quiet', dest
='verbose', action
='store_false',
492 help='non-verbose output')
493 parser
.add_argument('-r', '--random', type=int, metavar
='COUNT',
494 help='run a random sample of COUNT test cases',
496 parser
.add_argument('--shuffle', action
='store_true',
497 help='Run test cases in random order')
498 parser
.add_argument('--dry-run', action
='store_true',
499 help="Don't run any tests, just generate list")
500 parser
.add_argument('-D', '--devtype', metavar
='TYPE',
501 help="Test only device types that implement TYPE")
502 parser
.add_argument('-Q', '--quick', action
='store_true', default
=True,
503 help="Quick mode: skip test cases that are expected to fail")
504 parser
.add_argument('-F', '--full', action
='store_false', dest
='quick',
505 help="Full mode: test cases that are expected to fail")
506 parser
.add_argument('--strict', action
='store_true', dest
='strict',
507 help="Treat all warnings as fatal")
508 parser
.add_argument('qemu', nargs
='*', metavar
='QEMU',
509 help='QEMU binary to run')
510 args
= parser
.parse_args()
518 logging
.basicConfig(stream
=sys
.stdout
, level
=lvl
, format
='%(levelname)s: %(message)s')
526 dbg("testcases: %r", args
.testcases
)
528 for t
in chain(*args
.testcases
):
530 k
, v
= kv
.split('=', 1)
533 if len(binariesToTest(args
, tc
)) == 0:
534 print("No QEMU binary found", file=sys
.stderr
)
535 parser
.print_usage(sys
.stderr
)
538 for t
in casesToTest(args
, tc
):
539 logger
.info("running test case: %s", formatTestCase(t
))
542 expected_match
= findExpectedResult(t
)
545 not getBinaryInfo(args
, t
['binary']).machineInfo(t
['machine'])['runnable'])):
546 dbg("skipped: %s", formatTestCase(t
))
554 f
= checkOneCase(args
, t
)
555 except KeyboardInterrupt:
559 i
, wl
= checkResultWhitelist(f
)
560 dbg("testcase: %r, whitelist match: %r", t
, wl
)
561 wl_stats
.setdefault(i
, []).append(f
)
562 level
= wl
.get('loglevel', logging
.DEBUG
)
564 if wl
.get('fatal') or (args
.strict
and level
>= logging
.WARN
):
565 fatal_failures
.append(f
)
567 dbg("success: %s", formatTestCase(t
))
569 logger
.warn("Didn't fail as expected: %s", formatTestCase(t
))
571 logger
.info("Total: %d test cases", total
)
573 logger
.info("Skipped %d test cases", skipped
)
576 stats
= sorted([(len(wl_stats
.get(i
, [])), wl
) for i
, wl
in enumerate(ERROR_WHITELIST
)])
577 for count
, wl
in stats
:
578 dbg("whitelist entry stats: %d: %r", count
, wl
)
581 for f
in fatal_failures
:
583 logger
.error("Fatal failure: %s", formatTestCase(t
))
584 logger
.error("Fatal failures on some machine/device combinations")
587 if __name__
== '__main__':