tests/vm/basevm.py

   1 #
   2 # VM testing base class
   3 #
   4 # Copyright 2017-2019 Red Hat Inc.
   5 #
   6 # Authors:
   7 #  Fam Zheng <famz@redhat.com>
   8 #  Gerd Hoffmann <kraxel@redhat.com>
   9 #
  10 # This code is licensed under the GPL version 2 or later.  See
  11 # the COPYING file in the top-level directory.
  12 #
  13
  14 import os
  15 import re
  16 import sys
  17 import socket
  18 import logging
  19 import time
  20 import datetime
  21 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
  22 from qemu.accel import kvm_available
  23 from qemu.machine import QEMUMachine
  24 import subprocess
  25 import hashlib
  26 import argparse
  27 import atexit
  28 import tempfile
  29 import shutil
  30 import multiprocessing
  31 import traceback
  32 import shlex
  33
  34 SSH_KEY_FILE = os.path.join(os.path.dirname(__file__),
  35                "..", "keys", "id_rsa")
  36 SSH_PUB_KEY_FILE = os.path.join(os.path.dirname(__file__),
  37                    "..", "keys", "id_rsa.pub")
  38
  39 # This is the standard configuration.
  40 # Any or all of these can be overridden by
  41 # passing in a config argument to the VM constructor.
  42 DEFAULT_CONFIG = {
  43     'cpu'             : "max",
  44     'machine'         : 'pc',
  45     'guest_user'      : "qemu",
  46     'guest_pass'      : "qemupass",
  47     'root_pass'       : "qemupass",
  48     'ssh_key_file'    : SSH_KEY_FILE,
  49     'ssh_pub_key_file': SSH_PUB_KEY_FILE,
  50     'memory'          : "4G",
  51     'extra_args'      : [],
  52     'qemu_args'       : "",
  53     'dns'             : "",
  54     'ssh_port'        : 0,
  55     'install_cmds'    : "",
  56     'boot_dev_type'   : "block",
  57     'ssh_timeout'     : 1,
  58 }
  59 BOOT_DEVICE = {
  60     'block' :  "-drive file={},if=none,id=drive0,cache=writeback "\
  61                "-device virtio-blk,drive=drive0,bootindex=0",
  62     'scsi'  :  "-device virtio-scsi-device,id=scsi "\
  63                "-drive file={},format=raw,if=none,id=hd0 "\
  64                "-device scsi-hd,drive=hd0,bootindex=0",
  65 }
  66 class BaseVM(object):
  67
  68     envvars = [
  69         "https_proxy",
  70         "http_proxy",
  71         "ftp_proxy",
  72         "no_proxy",
  73     ]
  74
  75     # The script to run in the guest that builds QEMU
  76     BUILD_SCRIPT = ""
  77     # The guest name, to be overridden by subclasses
  78     name = "#base"
  79     # The guest architecture, to be overridden by subclasses
  80     arch = "#arch"
  81     # command to halt the guest, can be overridden by subclasses
  82     poweroff = "poweroff"
  83     # Time to wait for shutdown to finish.
  84     shutdown_timeout_default = 30
  85     # enable IPv6 networking
  86     ipv6 = True
  87     # This is the timeout on the wait for console bytes.
  88     socket_timeout = 120
  89     # Scale up some timeouts under TCG.
  90     # 4 is arbitrary, but greater than 2,
  91     # since we found we need to wait more than twice as long.
  92     tcg_timeout_multiplier = 4
  93     def __init__(self, args, config=None):
  94         self._guest = None
  95         self._genisoimage = args.genisoimage
  96         self._build_path = args.build_path
  97         self._efi_aarch64 = args.efi_aarch64
  98         # Allow input config to override defaults.
  99         self._config = DEFAULT_CONFIG.copy()
 100         if config != None:
 101             self._config.update(config)
 102         self.validate_ssh_keys()
 103         self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-",
 104                                                          suffix=".tmp",
 105                                                          dir="."))
 106         atexit.register(shutil.rmtree, self._tmpdir)
 107         # Copy the key files to a temporary directory.
 108         # Also chmod the key file to agree with ssh requirements.
 109         self._config['ssh_key'] = \
 110             open(self._config['ssh_key_file']).read().rstrip()
 111         self._config['ssh_pub_key'] = \
 112             open(self._config['ssh_pub_key_file']).read().rstrip()
 113         self._ssh_tmp_key_file = os.path.join(self._tmpdir, "id_rsa")
 114         open(self._ssh_tmp_key_file, "w").write(self._config['ssh_key'])
 115         subprocess.check_call(["chmod", "600", self._ssh_tmp_key_file])
 116
 117         self._ssh_tmp_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
 118         open(self._ssh_tmp_pub_key_file,
 119              "w").write(self._config['ssh_pub_key'])
 120
 121         self.debug = args.debug
 122         self._console_log_path = None
 123         if args.log_console:
 124                 self._console_log_path = \
 125                          os.path.join(os.path.expanduser("~/.cache/qemu-vm"),
 126                                       "{}.install.log".format(self.name))
 127         self._stderr = sys.stderr
 128         self._devnull = open(os.devnull, "w")
 129         if self.debug:
 130             self._stdout = sys.stdout
 131         else:
 132             self._stdout = self._devnull
 133         netdev = "user,id=vnet,hostfwd=:127.0.0.1:{}-:22"
 134         self._args = [ \
 135             "-nodefaults", "-m", self._config['memory'],
 136             "-cpu", self._config['cpu'],
 137             "-netdev",
 138             netdev.format(self._config['ssh_port']) +
 139             (",ipv6=no" if not self.ipv6 else "") +
 140             (",dns=" + self._config['dns'] if self._config['dns'] else ""),
 141             "-device", "virtio-net-pci,netdev=vnet",
 142             "-vnc", "127.0.0.1:0,to=20"]
 143         if args.jobs and args.jobs > 1:
 144             self._args += ["-smp", "%d" % args.jobs]
 145         if kvm_available(self.arch):
 146             self._shutdown_timeout = self.shutdown_timeout_default
 147             self._args += ["-enable-kvm"]
 148         else:
 149             logging.info("KVM not available, not using -enable-kvm")
 150             self._shutdown_timeout = \
 151                 self.shutdown_timeout_default * self.tcg_timeout_multiplier
 152         self._data_args = []
 153
 154         if self._config['qemu_args'] != None:
 155             qemu_args = self._config['qemu_args']
 156             qemu_args = qemu_args.replace('\n',' ').replace('\r','')
 157             # shlex groups quoted arguments together
 158             # we need this to keep the quoted args together for when
 159             # the QEMU command is issued later.
 160             args = shlex.split(qemu_args)
 161             self._config['extra_args'] = []
 162             for arg in args:
 163                 if arg:
 164                     # Preserve quotes around arguments.
 165                     # shlex above takes them out, so add them in.
 166                     if " " in arg:
 167                         arg = '"{}"'.format(arg)
 168                     self._config['extra_args'].append(arg)
 169
 170     def validate_ssh_keys(self):
 171         """Check to see if the ssh key files exist."""
 172         if 'ssh_key_file' not in self._config or\
 173            not os.path.exists(self._config['ssh_key_file']):
 174             raise Exception("ssh key file not found.")
 175         if 'ssh_pub_key_file' not in self._config or\
 176            not os.path.exists(self._config['ssh_pub_key_file']):
 177                raise Exception("ssh pub key file not found.")
 178
 179     def wait_boot(self, wait_string=None):
 180         """Wait for the standard string we expect
 181            on completion of a normal boot.
 182            The user can also choose to override with an
 183            alternate string to wait for."""
 184         if wait_string is None:
 185             if self.login_prompt is None:
 186                 raise Exception("self.login_prompt not defined")
 187             wait_string = self.login_prompt
 188         # Intentionally bump up the default timeout under TCG,
 189         # since the console wait below takes longer.
 190         timeout = self.socket_timeout
 191         if not kvm_available(self.arch):
 192             timeout *= 8
 193         self.console_init(timeout=timeout)
 194         self.console_wait(wait_string)
 195
 196     def _download_with_cache(self, url, sha256sum=None, sha512sum=None):
 197         def check_sha256sum(fname):
 198             if not sha256sum:
 199                 return True
 200             checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
 201             return sha256sum == checksum.decode("utf-8")
 202
 203         def check_sha512sum(fname):
 204             if not sha512sum:
 205                 return True
 206             checksum = subprocess.check_output(["sha512sum", fname]).split()[0]
 207             return sha512sum == checksum.decode("utf-8")
 208
 209         cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
 210         if not os.path.exists(cache_dir):
 211             os.makedirs(cache_dir)
 212         fname = os.path.join(cache_dir,
 213                              hashlib.sha1(url.encode("utf-8")).hexdigest())
 214         if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname):
 215             return fname
 216         logging.debug("Downloading %s to %s...", url, fname)
 217         subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"],
 218                               stdout=self._stdout, stderr=self._stderr)
 219         os.rename(fname + ".download", fname)
 220         return fname
 221
 222     def _ssh_do(self, user, cmd, check):
 223         ssh_cmd = ["ssh",
 224                    "-t",
 225                    "-o", "StrictHostKeyChecking=no",
 226                    "-o", "UserKnownHostsFile=" + os.devnull,
 227                    "-o",
 228                    "ConnectTimeout={}".format(self._config["ssh_timeout"]),
 229                    "-p", self.ssh_port, "-i", self._ssh_tmp_key_file]
 230         # If not in debug mode, set ssh to quiet mode to
 231         # avoid printing the results of commands.
 232         if not self.debug:
 233             ssh_cmd.append("-q")
 234         for var in self.envvars:
 235             ssh_cmd += ['-o', "SendEnv=%s" % var ]
 236         assert not isinstance(cmd, str)
 237         ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd)
 238         logging.debug("ssh_cmd: %s", " ".join(ssh_cmd))
 239         r = subprocess.call(ssh_cmd)
 240         if check and r != 0:
 241             raise Exception("SSH command failed: %s" % cmd)
 242         return r
 243
 244     def ssh(self, *cmd):
 245         return self._ssh_do(self._config["guest_user"], cmd, False)
 246
 247     def ssh_root(self, *cmd):
 248         return self._ssh_do("root", cmd, False)
 249
 250     def ssh_check(self, *cmd):
 251         self._ssh_do(self._config["guest_user"], cmd, True)
 252
 253     def ssh_root_check(self, *cmd):
 254         self._ssh_do("root", cmd, True)
 255
 256     def build_image(self, img):
 257         raise NotImplementedError
 258
 259     def exec_qemu_img(self, *args):
 260         cmd = [os.environ.get("QEMU_IMG", "qemu-img")]
 261         cmd.extend(list(args))
 262         subprocess.check_call(cmd)
 263
 264     def add_source_dir(self, src_dir):
 265         name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5]
 266         tarfile = os.path.join(self._tmpdir, name + ".tar")
 267         logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir)
 268         subprocess.check_call(["./scripts/archive-source.sh", tarfile],
 269                               cwd=src_dir, stdin=self._devnull,
 270                               stdout=self._stdout, stderr=self._stderr)
 271         self._data_args += ["-drive",
 272                             "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
 273                                     (tarfile, name),
 274                             "-device",
 275                             "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)]
 276
 277     def boot(self, img, extra_args=[]):
 278         boot_dev = BOOT_DEVICE[self._config['boot_dev_type']]
 279         boot_params = boot_dev.format(img)
 280         args = self._args + boot_params.split(' ')
 281         args += self._data_args + extra_args + self._config['extra_args']
 282         logging.debug("QEMU args: %s", " ".join(args))
 283         qemu_path = get_qemu_path(self.arch, self._build_path)
 284
 285         # Since console_log_path is only set when the user provides the
 286         # log_console option, we will set drain_console=True so the
 287         # console is always drained.
 288         guest = QEMUMachine(binary=qemu_path, args=args,
 289                             console_log=self._console_log_path,
 290                             drain_console=True)
 291         guest.set_machine(self._config['machine'])
 292         guest.set_console()
 293         try:
 294             guest.launch()
 295         except:
 296             logging.error("Failed to launch QEMU, command line:")
 297             logging.error(" ".join([qemu_path] + args))
 298             logging.error("Log:")
 299             logging.error(guest.get_log())
 300             logging.error("QEMU version >= 2.10 is required")
 301             raise
 302         atexit.register(self.shutdown)
 303         self._guest = guest
 304         # Init console so we can start consuming the chars.
 305         self.console_init()
 306         usernet_info = guest.qmp("human-monitor-command",
 307                                  command_line="info usernet")
 308         self.ssh_port = None
 309         for l in usernet_info["return"].splitlines():
 310             fields = l.split()
 311             if "TCP[HOST_FORWARD]" in fields and "22" in fields:
 312                 self.ssh_port = l.split()[3]
 313         if not self.ssh_port:
 314             raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
 315                             usernet_info)
 316
 317     def console_init(self, timeout = None):
 318         if timeout == None:
 319             timeout = self.socket_timeout
 320         vm = self._guest
 321         vm.console_socket.settimeout(timeout)
 322         self.console_raw_path = os.path.join(vm._temp_dir,
 323                                              vm._name + "-console.raw")
 324         self.console_raw_file = open(self.console_raw_path, 'wb')
 325
 326     def console_log(self, text):
 327         for line in re.split("[\r\n]", text):
 328             # filter out terminal escape sequences
 329             line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line)
 330             line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line)
 331             # replace unprintable chars
 332             line = re.sub("\x1b", "<esc>", line)
 333             line = re.sub("[\x00-\x1f]", ".", line)
 334             line = re.sub("[\x80-\xff]", ".", line)
 335             if line == "":
 336                 continue
 337             # log console line
 338             sys.stderr.write("con recv: %s\n" % line)
 339
 340     def console_wait(self, expect, expectalt = None):
 341         vm = self._guest
 342         output = ""
 343         while True:
 344             try:
 345                 chars = vm.console_socket.recv(1)
 346                 if self.console_raw_file:
 347                     self.console_raw_file.write(chars)
 348                     self.console_raw_file.flush()
 349             except socket.timeout:
 350                 sys.stderr.write("console: *** read timeout ***\n")
 351                 sys.stderr.write("console: waiting for: '%s'\n" % expect)
 352                 if not expectalt is None:
 353                     sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt)
 354                 sys.stderr.write("console: line buffer:\n")
 355                 sys.stderr.write("\n")
 356                 self.console_log(output.rstrip())
 357                 sys.stderr.write("\n")
 358                 raise
 359             output += chars.decode("latin1")
 360             if expect in output:
 361                 break
 362             if not expectalt is None and expectalt in output:
 363                 break
 364             if "\r" in output or "\n" in output:
 365                 lines = re.split("[\r\n]", output)
 366                 output = lines.pop()
 367                 if self.debug:
 368                     self.console_log("\n".join(lines))
 369         if self.debug:
 370             self.console_log(output)
 371         if not expectalt is None and expectalt in output:
 372             return False
 373         return True
 374
 375     def console_consume(self):
 376         vm = self._guest
 377         output = ""
 378         vm.console_socket.setblocking(0)
 379         while True:
 380             try:
 381                 chars = vm.console_socket.recv(1)
 382             except:
 383                 break
 384             output += chars.decode("latin1")
 385             if "\r" in output or "\n" in output:
 386                 lines = re.split("[\r\n]", output)
 387                 output = lines.pop()
 388                 if self.debug:
 389                     self.console_log("\n".join(lines))
 390         if self.debug:
 391             self.console_log(output)
 392         vm.console_socket.setblocking(1)
 393
 394     def console_send(self, command):
 395         vm = self._guest
 396         if self.debug:
 397             logline = re.sub("\n", "<enter>", command)
 398             logline = re.sub("[\x00-\x1f]", ".", logline)
 399             sys.stderr.write("con send: %s\n" % logline)
 400         for char in list(command):
 401             vm.console_socket.send(char.encode("utf-8"))
 402             time.sleep(0.01)
 403
 404     def console_wait_send(self, wait, command):
 405         self.console_wait(wait)
 406         self.console_send(command)
 407
 408     def console_ssh_init(self, prompt, user, pw):
 409         sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" \
 410                      % self._config['ssh_pub_key'].rstrip()
 411         self.console_wait_send("login:",    "%s\n" % user)
 412         self.console_wait_send("Password:", "%s\n" % pw)
 413         self.console_wait_send(prompt,      "mkdir .ssh\n")
 414         self.console_wait_send(prompt,      sshkey_cmd)
 415         self.console_wait_send(prompt,      "chmod 755 .ssh\n")
 416         self.console_wait_send(prompt,      "chmod 644 .ssh/authorized_keys\n")
 417
 418     def console_sshd_config(self, prompt):
 419         self.console_wait(prompt)
 420         self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n")
 421         for var in self.envvars:
 422             self.console_wait(prompt)
 423             self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var)
 424
 425     def print_step(self, text):
 426         sys.stderr.write("### %s ...\n" % text)
 427
 428     def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"):
 429         # Allow more time for VM to boot under TCG.
 430         if not kvm_available(self.arch):
 431             seconds *= self.tcg_timeout_multiplier
 432         starttime = datetime.datetime.now()
 433         endtime = starttime + datetime.timedelta(seconds=seconds)
 434         cmd_success = False
 435         while datetime.datetime.now() < endtime:
 436             if wait_root and self.ssh_root(cmd) == 0:
 437                 cmd_success = True
 438                 break
 439             elif self.ssh(cmd) == 0:
 440                 cmd_success = True
 441                 break
 442             seconds = (endtime - datetime.datetime.now()).total_seconds()
 443             logging.debug("%ds before timeout", seconds)
 444             time.sleep(1)
 445         if not cmd_success:
 446             raise Exception("Timeout while waiting for guest ssh")
 447
 448     def shutdown(self):
 449         self._guest.shutdown(timeout=self._shutdown_timeout)
 450
 451     def wait(self):
 452         self._guest.wait(timeout=self._shutdown_timeout)
 453
 454     def graceful_shutdown(self):
 455         self.ssh_root(self.poweroff)
 456         self._guest.wait(timeout=self._shutdown_timeout)
 457
 458     def qmp(self, *args, **kwargs):
 459         return self._guest.qmp(*args, **kwargs)
 460
 461     def gen_cloud_init_iso(self):
 462         cidir = self._tmpdir
 463         mdata = open(os.path.join(cidir, "meta-data"), "w")
 464         name = self.name.replace(".","-")
 465         mdata.writelines(["instance-id: {}-vm-0\n".format(name),
 466                           "local-hostname: {}-guest\n".format(name)])
 467         mdata.close()
 468         udata = open(os.path.join(cidir, "user-data"), "w")
 469         print("guest user:pw {}:{}".format(self._config['guest_user'],
 470                                            self._config['guest_pass']))
 471         udata.writelines(["#cloud-config\n",
 472                           "chpasswd:\n",
 473                           "  list: |\n",
 474                           "    root:%s\n" % self._config['root_pass'],
 475                           "    %s:%s\n" % (self._config['guest_user'],
 476                                            self._config['guest_pass']),
 477                           "  expire: False\n",
 478                           "users:\n",
 479                           "  - name: %s\n" % self._config['guest_user'],
 480                           "    sudo: ALL=(ALL) NOPASSWD:ALL\n",
 481                           "    ssh-authorized-keys:\n",
 482                           "    - %s\n" % self._config['ssh_pub_key'],
 483                           "  - name: root\n",
 484                           "    ssh-authorized-keys:\n",
 485                           "    - %s\n" % self._config['ssh_pub_key'],
 486                           "locale: en_US.UTF-8\n"])
 487         proxy = os.environ.get("http_proxy")
 488         if not proxy is None:
 489             udata.writelines(["apt:\n",
 490                               "  proxy: %s" % proxy])
 491         udata.close()
 492         subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso",
 493                                "-volid", "cidata", "-joliet", "-rock",
 494                                "user-data", "meta-data"],
 495                               cwd=cidir,
 496                               stdin=self._devnull, stdout=self._stdout,
 497                               stderr=self._stdout)
 498         return os.path.join(cidir, "cloud-init.iso")
 499
 500 def get_qemu_path(arch, build_path=None):
 501     """Fetch the path to the qemu binary."""
 502     # If QEMU environment variable set, it takes precedence
 503     if "QEMU" in os.environ:
 504         qemu_path = os.environ["QEMU"]
 505     elif build_path:
 506         qemu_path = os.path.join(build_path, arch + "-softmmu")
 507         qemu_path = os.path.join(qemu_path, "qemu-system-" + arch)
 508     else:
 509         # Default is to use system path for qemu.
 510         qemu_path = "qemu-system-" + arch
 511     return qemu_path
 512
 513 def get_qemu_version(qemu_path):
 514     """Get the version number from the current QEMU,
 515        and return the major number."""
 516     output = subprocess.check_output([qemu_path, '--version'])
 517     version_line = output.decode("utf-8")
 518     version_num = re.split(' |\(', version_line)[3].split('.')[0]
 519     return int(version_num)
 520
 521 def parse_config(config, args):
 522     """ Parse yaml config and populate our config structure.
 523         The yaml config allows the user to override the
 524         defaults for VM parameters.  In many cases these
 525         defaults can be overridden without rebuilding the VM."""
 526     if args.config:
 527         config_file = args.config
 528     elif 'QEMU_CONFIG' in os.environ:
 529         config_file = os.environ['QEMU_CONFIG']
 530     else:
 531         return config
 532     if not os.path.exists(config_file):
 533         raise Exception("config file {} does not exist".format(config_file))
 534     # We gracefully handle importing the yaml module
 535     # since it might not be installed.
 536     # If we are here it means the user supplied a .yml file,
 537     # so if the yaml module is not installed we will exit with error.
 538     try:
 539         import yaml
 540     except ImportError:
 541         print("The python3-yaml package is needed "\
 542               "to support config.yaml files")
 543         # Instead of raising an exception we exit to avoid
 544         # a raft of messy (expected) errors to stdout.
 545         exit(1)
 546     with open(config_file) as f:
 547         yaml_dict = yaml.safe_load(f)
 548
 549     if 'qemu-conf' in yaml_dict:
 550         config.update(yaml_dict['qemu-conf'])
 551     else:
 552         raise Exception("config file {} is not valid"\
 553                         " missing qemu-conf".format(config_file))
 554     return config
 555
 556 def parse_args(vmcls):
 557
 558     def get_default_jobs():
 559         if multiprocessing.cpu_count() > 1:
 560             if kvm_available(vmcls.arch):
 561                 return multiprocessing.cpu_count() // 2
 562             elif os.uname().machine == "x86_64" and \
 563                  vmcls.arch in ["aarch64", "x86_64", "i386"]:
 564                 # MTTCG is available on these arches and we can allow
 565                 # more cores. but only up to a reasonable limit. User
 566                 # can always override these limits with --jobs.
 567                 return min(multiprocessing.cpu_count() // 2, 8)
 568         else:
 569             return 1
 570
 571     parser = argparse.ArgumentParser(
 572         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 573         description="Utility for provisioning VMs and running builds",
 574         epilog="""Remaining arguments are passed to the command.
 575         Exit codes: 0 = success, 1 = command line error,
 576         2 = environment initialization failed,
 577         3 = test command failed""")
 578     parser.add_argument("--debug", "-D", action="store_true",
 579                         help="enable debug output")
 580     parser.add_argument("--image", "-i", default="%s.img" % vmcls.name,
 581                         help="image file name")
 582     parser.add_argument("--force", "-f", action="store_true",
 583                         help="force build image even if image exists")
 584     parser.add_argument("--jobs", type=int, default=get_default_jobs(),
 585                         help="number of virtual CPUs")
 586     parser.add_argument("--verbose", "-V", action="store_true",
 587                         help="Pass V=1 to builds within the guest")
 588     parser.add_argument("--build-image", "-b", action="store_true",
 589                         help="build image")
 590     parser.add_argument("--build-qemu",
 591                         help="build QEMU from source in guest")
 592     parser.add_argument("--build-target",
 593                         help="QEMU build target", default="check")
 594     parser.add_argument("--build-path", default=None,
 595                         help="Path of build directory, "\
 596                         "for using build tree QEMU binary. ")
 597     parser.add_argument("--interactive", "-I", action="store_true",
 598                         help="Interactively run command")
 599     parser.add_argument("--snapshot", "-s", action="store_true",
 600                         help="run tests with a snapshot")
 601     parser.add_argument("--genisoimage", default="genisoimage",
 602                         help="iso imaging tool")
 603     parser.add_argument("--config", "-c", default=None,
 604                         help="Provide config yaml for configuration. "\
 605                         "See config_example.yaml for example.")
 606     parser.add_argument("--efi-aarch64",
 607                         default="/usr/share/qemu-efi-aarch64/QEMU_EFI.fd",
 608                         help="Path to efi image for aarch64 VMs.")
 609     parser.add_argument("--log-console", action="store_true",
 610                         help="Log console to file.")
 611     parser.add_argument("commands", nargs="*", help="""Remaining
 612         commands after -- are passed to command inside the VM""")
 613
 614     return parser.parse_args()
 615
 616 def main(vmcls, config=None):
 617     try:
 618         if config == None:
 619             config = DEFAULT_CONFIG
 620         args = parse_args(vmcls)
 621         if not args.commands and not args.build_qemu and not args.build_image:
 622             print("Nothing to do?")
 623             return 1
 624         config = parse_config(config, args)
 625         logging.basicConfig(level=(logging.DEBUG if args.debug
 626                                    else logging.WARN))
 627         vm = vmcls(args, config=config)
 628         if args.build_image:
 629             if os.path.exists(args.image) and not args.force:
 630                 sys.stderr.writelines(["Image file exists: %s\n" % args.image,
 631                                       "Use --force option to overwrite\n"])
 632                 return 1
 633             return vm.build_image(args.image)
 634         if args.build_qemu:
 635             vm.add_source_dir(args.build_qemu)
 636             cmd = [vm.BUILD_SCRIPT.format(
 637                    configure_opts = " ".join(args.commands),
 638                    jobs=int(args.jobs),
 639                    target=args.build_target,
 640                    verbose = "V=1" if args.verbose else "")]
 641         else:
 642             cmd = args.commands
 643         img = args.image
 644         if args.snapshot:
 645             img += ",snapshot=on"
 646         vm.boot(img)
 647         vm.wait_ssh()
 648     except Exception as e:
 649         if isinstance(e, SystemExit) and e.code == 0:
 650             return 0
 651         sys.stderr.write("Failed to prepare guest environment\n")
 652         traceback.print_exc()
 653         return 2
 654
 655     exitcode = 0
 656     if vm.ssh(*cmd) != 0:
 657         exitcode = 3
 658     if args.interactive:
 659         vm.ssh()
 660
 661     if not args.snapshot:
 662         vm.graceful_shutdown()
 663
 664     return exitcode