python/qemu/machine.py

   1 """
   2 QEMU machine module:
   3
   4 The machine module primarily provides the QEMUMachine class,
   5 which provides facilities for managing the lifetime of a QEMU VM.
   6 """
   7
   8 # Copyright (C) 2015-2016 Red Hat Inc.
   9 # Copyright (C) 2012 IBM Corp.
  10 #
  11 # Authors:
  12 #  Fam Zheng <famz@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.  See
  15 # the COPYING file in the top-level directory.
  16 #
  17 # Based on qmp.py.
  18 #
  19
  20 import errno
  21 import logging
  22 import os
  23 import subprocess
  24 import shutil
  25 import signal
  26 import tempfile
  27 from typing import Optional, Type
  28 from types import TracebackType
  29 from . import console_socket
  30
  31 from . import qmp
  32
  33 LOG = logging.getLogger(__name__)
  34
  35
  36 class QEMUMachineError(Exception):
  37     """
  38     Exception called when an error in QEMUMachine happens.
  39     """
  40
  41
  42 class QEMUMachineAddDeviceError(QEMUMachineError):
  43     """
  44     Exception raised when a request to add a device can not be fulfilled
  45
  46     The failures are caused by limitations, lack of information or conflicting
  47     requests on the QEMUMachine methods.  This exception does not represent
  48     failures reported by the QEMU binary itself.
  49     """
  50
  51
  52 class AbnormalShutdown(QEMUMachineError):
  53     """
  54     Exception raised when a graceful shutdown was requested, but not performed.
  55     """
  56
  57
  58 class QEMUMachine:
  59     """
  60     A QEMU VM
  61
  62     Use this object as a context manager to ensure
  63     the QEMU process terminates::
  64
  65         with VM(binary) as vm:
  66             ...
  67         # vm is guaranteed to be shut down here
  68     """
  69
  70     def __init__(self, binary, args=None, wrapper=None, name=None,
  71                  test_dir="/var/tmp", monitor_address=None,
  72                  socket_scm_helper=None, sock_dir=None,
  73                  drain_console=False, console_log=None):
  74         '''
  75         Initialize a QEMUMachine
  76
  77         @param binary: path to the qemu binary
  78         @param args: list of extra arguments
  79         @param wrapper: list of arguments used as prefix to qemu binary
  80         @param name: prefix for socket and log file names (default: qemu-PID)
  81         @param test_dir: where to create socket and log file
  82         @param monitor_address: address for QMP monitor
  83         @param socket_scm_helper: helper program, required for send_fd_scm()
  84         @param sock_dir: where to create socket (overrides test_dir for sock)
  85         @param console_log: (optional) path to console log file
  86         @param drain_console: (optional) True to drain console socket to buffer
  87         @note: Qemu process is not started until launch() is used.
  88         '''
  89         if args is None:
  90             args = []
  91         if wrapper is None:
  92             wrapper = []
  93         if name is None:
  94             name = "qemu-%d" % os.getpid()
  95         if sock_dir is None:
  96             sock_dir = test_dir
  97         self._name = name
  98         self._monitor_address = monitor_address
  99         self._vm_monitor = None
 100         self._qemu_log_path = None
 101         self._qemu_log_file = None
 102         self._popen = None
 103         self._binary = binary
 104         self._args = list(args)     # Force copy args in case we modify them
 105         self._wrapper = wrapper
 106         self._events = []
 107         self._iolog = None
 108         self._socket_scm_helper = socket_scm_helper
 109         self._qmp_set = True   # Enable QMP monitor by default.
 110         self._qmp = None
 111         self._qemu_full_args = None
 112         self._test_dir = test_dir
 113         self._temp_dir = None
 114         self._sock_dir = sock_dir
 115         self._launched = False
 116         self._machine = None
 117         self._console_index = 0
 118         self._console_set = False
 119         self._console_device_type = None
 120         self._console_address = None
 121         self._console_socket = None
 122         self._remove_files = []
 123         self._user_killed = False
 124         self._console_log_path = console_log
 125         if self._console_log_path:
 126             # In order to log the console, buffering needs to be enabled.
 127             self._drain_console = True
 128         else:
 129             self._drain_console = drain_console
 130
 131     def __enter__(self):
 132         return self
 133
 134     def __exit__(self,
 135                  exc_type: Optional[Type[BaseException]],
 136                  exc_val: Optional[BaseException],
 137                  exc_tb: Optional[TracebackType]) -> None:
 138         self.shutdown()
 139
 140     def add_monitor_null(self):
 141         """
 142         This can be used to add an unused monitor instance.
 143         """
 144         self._args.append('-monitor')
 145         self._args.append('null')
 146
 147     def add_fd(self, fd, fdset, opaque, opts=''):
 148         """
 149         Pass a file descriptor to the VM
 150         """
 151         options = ['fd=%d' % fd,
 152                    'set=%d' % fdset,
 153                    'opaque=%s' % opaque]
 154         if opts:
 155             options.append(opts)
 156
 157         # This did not exist before 3.4, but since then it is
 158         # mandatory for our purpose
 159         if hasattr(os, 'set_inheritable'):
 160             os.set_inheritable(fd, True)
 161
 162         self._args.append('-add-fd')
 163         self._args.append(','.join(options))
 164         return self
 165
 166     def send_fd_scm(self, fd=None, file_path=None):
 167         """
 168         Send an fd or file_path to socket_scm_helper.
 169
 170         Exactly one of fd and file_path must be given.
 171         If it is file_path, the helper will open that file and pass its own fd.
 172         """
 173         # In iotest.py, the qmp should always use unix socket.
 174         assert self._qmp.is_scm_available()
 175         if self._socket_scm_helper is None:
 176             raise QEMUMachineError("No path to socket_scm_helper set")
 177         if not os.path.exists(self._socket_scm_helper):
 178             raise QEMUMachineError("%s does not exist" %
 179                                    self._socket_scm_helper)
 180
 181         # This did not exist before 3.4, but since then it is
 182         # mandatory for our purpose
 183         if hasattr(os, 'set_inheritable'):
 184             os.set_inheritable(self._qmp.get_sock_fd(), True)
 185             if fd is not None:
 186                 os.set_inheritable(fd, True)
 187
 188         fd_param = ["%s" % self._socket_scm_helper,
 189                     "%d" % self._qmp.get_sock_fd()]
 190
 191         if file_path is not None:
 192             assert fd is None
 193             fd_param.append(file_path)
 194         else:
 195             assert fd is not None
 196             fd_param.append(str(fd))
 197
 198         devnull = open(os.path.devnull, 'rb')
 199         proc = subprocess.Popen(
 200             fd_param, stdin=devnull, stdout=subprocess.PIPE,
 201             stderr=subprocess.STDOUT, close_fds=False
 202         )
 203         output = proc.communicate()[0]
 204         if output:
 205             LOG.debug(output)
 206
 207         return proc.returncode
 208
 209     @staticmethod
 210     def _remove_if_exists(path):
 211         """
 212         Remove file object at path if it exists
 213         """
 214         try:
 215             os.remove(path)
 216         except OSError as exception:
 217             if exception.errno == errno.ENOENT:
 218                 return
 219             raise
 220
 221     def is_running(self):
 222         """Returns true if the VM is running."""
 223         return self._popen is not None and self._popen.poll() is None
 224
 225     def exitcode(self):
 226         """Returns the exit code if possible, or None."""
 227         if self._popen is None:
 228             return None
 229         return self._popen.poll()
 230
 231     def get_pid(self):
 232         """Returns the PID of the running process, or None."""
 233         if not self.is_running():
 234             return None
 235         return self._popen.pid
 236
 237     def _load_io_log(self):
 238         if self._qemu_log_path is not None:
 239             with open(self._qemu_log_path, "r") as iolog:
 240                 self._iolog = iolog.read()
 241
 242     def _base_args(self):
 243         args = ['-display', 'none', '-vga', 'none']
 244         if self._qmp_set:
 245             if isinstance(self._monitor_address, tuple):
 246                 moncdev = "socket,id=mon,host=%s,port=%s" % (
 247                     self._monitor_address[0],
 248                     self._monitor_address[1])
 249             else:
 250                 moncdev = 'socket,id=mon,path=%s' % self._vm_monitor
 251             args.extend(['-chardev', moncdev, '-mon',
 252                          'chardev=mon,mode=control'])
 253         if self._machine is not None:
 254             args.extend(['-machine', self._machine])
 255         for _ in range(self._console_index):
 256             args.extend(['-serial', 'null'])
 257         if self._console_set:
 258             self._console_address = os.path.join(self._sock_dir,
 259                                                  self._name + "-console.sock")
 260             self._remove_files.append(self._console_address)
 261             chardev = ('socket,id=console,path=%s,server,nowait' %
 262                        self._console_address)
 263             args.extend(['-chardev', chardev])
 264             if self._console_device_type is None:
 265                 args.extend(['-serial', 'chardev:console'])
 266             else:
 267                 device = '%s,chardev=console' % self._console_device_type
 268                 args.extend(['-device', device])
 269         return args
 270
 271     def _pre_launch(self):
 272         self._temp_dir = tempfile.mkdtemp(dir=self._test_dir)
 273         self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log")
 274         self._qemu_log_file = open(self._qemu_log_path, 'wb')
 275
 276         if self._qmp_set:
 277             if self._monitor_address is not None:
 278                 self._vm_monitor = self._monitor_address
 279             else:
 280                 self._vm_monitor = os.path.join(self._sock_dir,
 281                                                 self._name + "-monitor.sock")
 282                 self._remove_files.append(self._vm_monitor)
 283             self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor, server=True,
 284                                                 nickname=self._name)
 285
 286     def _post_launch(self):
 287         if self._qmp:
 288             self._qmp.accept()
 289
 290     def _post_shutdown(self):
 291         """
 292         Called to cleanup the VM instance after the process has exited.
 293         May also be called after a failed launch.
 294         """
 295         # Comprehensive reset for the failed launch case:
 296         self._early_cleanup()
 297
 298         if self._qmp:
 299             self._qmp.close()
 300             self._qmp = None
 301
 302         self._load_io_log()
 303
 304         if self._qemu_log_file is not None:
 305             self._qemu_log_file.close()
 306             self._qemu_log_file = None
 307
 308         self._qemu_log_path = None
 309
 310         if self._temp_dir is not None:
 311             shutil.rmtree(self._temp_dir)
 312             self._temp_dir = None
 313
 314         while len(self._remove_files) > 0:
 315             self._remove_if_exists(self._remove_files.pop())
 316
 317         exitcode = self.exitcode()
 318         if (exitcode is not None and exitcode < 0
 319                 and not (self._user_killed and exitcode == -signal.SIGKILL)):
 320             msg = 'qemu received signal %i; command: "%s"'
 321             if self._qemu_full_args:
 322                 command = ' '.join(self._qemu_full_args)
 323             else:
 324                 command = ''
 325             LOG.warning(msg, -int(exitcode), command)
 326
 327         self._user_killed = False
 328         self._launched = False
 329
 330     def launch(self):
 331         """
 332         Launch the VM and make sure we cleanup and expose the
 333         command line/output in case of exception
 334         """
 335
 336         if self._launched:
 337             raise QEMUMachineError('VM already launched')
 338
 339         self._iolog = None
 340         self._qemu_full_args = None
 341         try:
 342             self._launch()
 343             self._launched = True
 344         except:
 345             self._post_shutdown()
 346
 347             LOG.debug('Error launching VM')
 348             if self._qemu_full_args:
 349                 LOG.debug('Command: %r', ' '.join(self._qemu_full_args))
 350             if self._iolog:
 351                 LOG.debug('Output: %r', self._iolog)
 352             raise
 353
 354     def _launch(self):
 355         """
 356         Launch the VM and establish a QMP connection
 357         """
 358         devnull = open(os.path.devnull, 'rb')
 359         self._pre_launch()
 360         self._qemu_full_args = (self._wrapper + [self._binary] +
 361                                 self._base_args() + self._args)
 362         LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args))
 363         self._popen = subprocess.Popen(self._qemu_full_args,
 364                                        stdin=devnull,
 365                                        stdout=self._qemu_log_file,
 366                                        stderr=subprocess.STDOUT,
 367                                        shell=False,
 368                                        close_fds=False)
 369         self._post_launch()
 370
 371     def _early_cleanup(self) -> None:
 372         """
 373         Perform any cleanup that needs to happen before the VM exits.
 374
 375         May be invoked by both soft and hard shutdown in failover scenarios.
 376         Called additionally by _post_shutdown for comprehensive cleanup.
 377         """
 378         # If we keep the console socket open, we may deadlock waiting
 379         # for QEMU to exit, while QEMU is waiting for the socket to
 380         # become writeable.
 381         if self._console_socket is not None:
 382             self._console_socket.close()
 383             self._console_socket = None
 384
 385     def _hard_shutdown(self) -> None:
 386         """
 387         Perform early cleanup, kill the VM, and wait for it to terminate.
 388
 389         :raise subprocess.Timeout: When timeout is exceeds 60 seconds
 390             waiting for the QEMU process to terminate.
 391         """
 392         self._early_cleanup()
 393         self._popen.kill()
 394         self._popen.wait(timeout=60)
 395
 396     def _soft_shutdown(self, timeout: Optional[int],
 397                        has_quit: bool = False) -> None:
 398         """
 399         Perform early cleanup, attempt to gracefully shut down the VM, and wait
 400         for it to terminate.
 401
 402         :param timeout: Timeout in seconds for graceful shutdown.
 403                         A value of None is an infinite wait.
 404         :param has_quit: When True, don't attempt to issue 'quit' QMP command
 405
 406         :raise ConnectionReset: On QMP communication errors
 407         :raise subprocess.TimeoutExpired: When timeout is exceeded waiting for
 408             the QEMU process to terminate.
 409         """
 410         self._early_cleanup()
 411
 412         if self._qmp is not None:
 413             if not has_quit:
 414                 # Might raise ConnectionReset
 415                 self._qmp.cmd('quit')
 416
 417         # May raise subprocess.TimeoutExpired
 418         self._popen.wait(timeout=timeout)
 419
 420     def _do_shutdown(self, timeout: Optional[int],
 421                      has_quit: bool = False) -> None:
 422         """
 423         Attempt to shutdown the VM gracefully; fallback to a hard shutdown.
 424
 425         :param timeout: Timeout in seconds for graceful shutdown.
 426                         A value of None is an infinite wait.
 427         :param has_quit: When True, don't attempt to issue 'quit' QMP command
 428
 429         :raise AbnormalShutdown: When the VM could not be shut down gracefully.
 430             The inner exception will likely be ConnectionReset or
 431             subprocess.TimeoutExpired. In rare cases, non-graceful termination
 432             may result in its own exceptions, likely subprocess.TimeoutExpired.
 433         """
 434         try:
 435             self._soft_shutdown(timeout, has_quit)
 436         except Exception as exc:
 437             self._hard_shutdown()
 438             raise AbnormalShutdown("Could not perform graceful shutdown") \
 439                 from exc
 440
 441     def shutdown(self, has_quit: bool = False,
 442                  hard: bool = False,
 443                  timeout: Optional[int] = 30) -> None:
 444         """
 445         Terminate the VM (gracefully if possible) and perform cleanup.
 446         Cleanup will always be performed.
 447
 448         If the VM has not yet been launched, or shutdown(), wait(), or kill()
 449         have already been called, this method does nothing.
 450
 451         :param has_quit: When true, do not attempt to issue 'quit' QMP command.
 452         :param hard: When true, do not attempt graceful shutdown, and
 453                      suppress the SIGKILL warning log message.
 454         :param timeout: Optional timeout in seconds for graceful shutdown.
 455                         Default 30 seconds, A `None` value is an infinite wait.
 456         """
 457         if not self._launched:
 458             return
 459
 460         try:
 461             if hard:
 462                 self._user_killed = True
 463                 self._hard_shutdown()
 464             else:
 465                 self._do_shutdown(timeout, has_quit)
 466         finally:
 467             self._post_shutdown()
 468
 469     def kill(self):
 470         """
 471         Terminate the VM forcefully, wait for it to exit, and perform cleanup.
 472         """
 473         self.shutdown(hard=True)
 474
 475     def wait(self, timeout: Optional[int] = 30) -> None:
 476         """
 477         Wait for the VM to power off and perform post-shutdown cleanup.
 478
 479         :param timeout: Optional timeout in seconds. Default 30 seconds.
 480                         A value of `None` is an infinite wait.
 481         """
 482         self.shutdown(has_quit=True, timeout=timeout)
 483
 484     def set_qmp_monitor(self, enabled=True):
 485         """
 486         Set the QMP monitor.
 487
 488         @param enabled: if False, qmp monitor options will be removed from
 489                         the base arguments of the resulting QEMU command
 490                         line. Default is True.
 491         @note: call this function before launch().
 492         """
 493         if enabled:
 494             self._qmp_set = True
 495         else:
 496             self._qmp_set = False
 497             self._qmp = None
 498
 499     def qmp(self, cmd, conv_keys=True, **args):
 500         """
 501         Invoke a QMP command and return the response dict
 502         """
 503         qmp_args = dict()
 504         for key, value in args.items():
 505             if conv_keys:
 506                 qmp_args[key.replace('_', '-')] = value
 507             else:
 508                 qmp_args[key] = value
 509
 510         return self._qmp.cmd(cmd, args=qmp_args)
 511
 512     def command(self, cmd, conv_keys=True, **args):
 513         """
 514         Invoke a QMP command.
 515         On success return the response dict.
 516         On failure raise an exception.
 517         """
 518         reply = self.qmp(cmd, conv_keys, **args)
 519         if reply is None:
 520             raise qmp.QMPError("Monitor is closed")
 521         if "error" in reply:
 522             raise qmp.QMPResponseError(reply)
 523         return reply["return"]
 524
 525     def get_qmp_event(self, wait=False):
 526         """
 527         Poll for one queued QMP events and return it
 528         """
 529         if self._events:
 530             return self._events.pop(0)
 531         return self._qmp.pull_event(wait=wait)
 532
 533     def get_qmp_events(self, wait=False):
 534         """
 535         Poll for queued QMP events and return a list of dicts
 536         """
 537         events = self._qmp.get_events(wait=wait)
 538         events.extend(self._events)
 539         del self._events[:]
 540         self._qmp.clear_events()
 541         return events
 542
 543     @staticmethod
 544     def event_match(event, match=None):
 545         """
 546         Check if an event matches optional match criteria.
 547
 548         The match criteria takes the form of a matching subdict. The event is
 549         checked to be a superset of the subdict, recursively, with matching
 550         values whenever the subdict values are not None.
 551
 552         This has a limitation that you cannot explicitly check for None values.
 553
 554         Examples, with the subdict queries on the left:
 555          - None matches any object.
 556          - {"foo": None} matches {"foo": {"bar": 1}}
 557          - {"foo": None} matches {"foo": 5}
 558          - {"foo": {"abc": None}} does not match {"foo": {"bar": 1}}
 559          - {"foo": {"rab": 2}} matches {"foo": {"bar": 1, "rab": 2}}
 560         """
 561         if match is None:
 562             return True
 563
 564         try:
 565             for key in match:
 566                 if key in event:
 567                     if not QEMUMachine.event_match(event[key], match[key]):
 568                         return False
 569                 else:
 570                     return False
 571             return True
 572         except TypeError:
 573             # either match or event wasn't iterable (not a dict)
 574             return match == event
 575
 576     def event_wait(self, name, timeout=60.0, match=None):
 577         """
 578         event_wait waits for and returns a named event from QMP with a timeout.
 579
 580         name: The event to wait for.
 581         timeout: QEMUMonitorProtocol.pull_event timeout parameter.
 582         match: Optional match criteria. See event_match for details.
 583         """
 584         return self.events_wait([(name, match)], timeout)
 585
 586     def events_wait(self, events, timeout=60.0):
 587         """
 588         events_wait waits for and returns a named event
 589         from QMP with a timeout.
 590
 591         events: a sequence of (name, match_criteria) tuples.
 592                 The match criteria are optional and may be None.
 593                 See event_match for details.
 594         timeout: QEMUMonitorProtocol.pull_event timeout parameter.
 595         """
 596         def _match(event):
 597             for name, match in events:
 598                 if event['event'] == name and self.event_match(event, match):
 599                     return True
 600             return False
 601
 602         # Search cached events
 603         for event in self._events:
 604             if _match(event):
 605                 self._events.remove(event)
 606                 return event
 607
 608         # Poll for new events
 609         while True:
 610             event = self._qmp.pull_event(wait=timeout)
 611             if _match(event):
 612                 return event
 613             self._events.append(event)
 614
 615         return None
 616
 617     def get_log(self):
 618         """
 619         After self.shutdown or failed qemu execution, this returns the output
 620         of the qemu process.
 621         """
 622         return self._iolog
 623
 624     def add_args(self, *args):
 625         """
 626         Adds to the list of extra arguments to be given to the QEMU binary
 627         """
 628         self._args.extend(args)
 629
 630     def set_machine(self, machine_type):
 631         """
 632         Sets the machine type
 633
 634         If set, the machine type will be added to the base arguments
 635         of the resulting QEMU command line.
 636         """
 637         self._machine = machine_type
 638
 639     def set_console(self, device_type=None, console_index=0):
 640         """
 641         Sets the device type for a console device
 642
 643         If set, the console device and a backing character device will
 644         be added to the base arguments of the resulting QEMU command
 645         line.
 646
 647         This is a convenience method that will either use the provided
 648         device type, or default to a "-serial chardev:console" command
 649         line argument.
 650
 651         The actual setting of command line arguments will be be done at
 652         machine launch time, as it depends on the temporary directory
 653         to be created.
 654
 655         @param device_type: the device type, such as "isa-serial".  If
 656                             None is given (the default value) a "-serial
 657                             chardev:console" command line argument will
 658                             be used instead, resorting to the machine's
 659                             default device type.
 660         @param console_index: the index of the console device to use.
 661                               If not zero, the command line will create
 662                               'index - 1' consoles and connect them to
 663                               the 'null' backing character device.
 664         """
 665         self._console_set = True
 666         self._console_device_type = device_type
 667         self._console_index = console_index
 668
 669     @property
 670     def console_socket(self):
 671         """
 672         Returns a socket connected to the console
 673         """
 674         if self._console_socket is None:
 675             self._console_socket = console_socket.ConsoleSocket(
 676                 self._console_address,
 677                 file=self._console_log_path,
 678                 drain=self._drain_console)
 679         return self._console_socket