tests/vm/basevm.py

   1 #
   2 # VM testing base class
   3 #
   4 # Copyright 2017-2019 Red Hat Inc.
   5 #
   6 # Authors:
   7 #  Fam Zheng <famz@redhat.com>
   8 #  Gerd Hoffmann <kraxel@redhat.com>
   9 #
  10 # This code is licensed under the GPL version 2 or later.  See
  11 # the COPYING file in the top-level directory.
  12 #
  13
  14 import os
  15 import re
  16 import sys
  17 import socket
  18 import logging
  19 import time
  20 import datetime
  21 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
  22 from qemu.machine import QEMUMachine
  23 from qemu.utils import get_info_usernet_hostfwd_port, kvm_available
  24 import subprocess
  25 import hashlib
  26 import argparse
  27 import atexit
  28 import tempfile
  29 import shutil
  30 import multiprocessing
  31 import traceback
  32 import shlex
  33
  34 SSH_KEY_FILE = os.path.join(os.path.dirname(__file__),
  35                "..", "keys", "id_rsa")
  36 SSH_PUB_KEY_FILE = os.path.join(os.path.dirname(__file__),
  37                    "..", "keys", "id_rsa.pub")
  38
  39 # This is the standard configuration.
  40 # Any or all of these can be overridden by
  41 # passing in a config argument to the VM constructor.
  42 DEFAULT_CONFIG = {
  43     'cpu'             : "max",
  44     'machine'         : 'pc',
  45     'guest_user'      : "qemu",
  46     'guest_pass'      : "qemupass",
  47     'root_user'       : "root",
  48     'root_pass'       : "qemupass",
  49     'ssh_key_file'    : SSH_KEY_FILE,
  50     'ssh_pub_key_file': SSH_PUB_KEY_FILE,
  51     'memory'          : "4G",
  52     'extra_args'      : [],
  53     'qemu_args'       : "",
  54     'dns'             : "",
  55     'ssh_port'        : 0,
  56     'install_cmds'    : "",
  57     'boot_dev_type'   : "block",
  58     'ssh_timeout'     : 1,
  59 }
  60 BOOT_DEVICE = {
  61     'block' :  "-drive file={},if=none,id=drive0,cache=writeback "\
  62                "-device virtio-blk,drive=drive0,bootindex=0",
  63     'scsi'  :  "-device virtio-scsi-device,id=scsi "\
  64                "-drive file={},format=raw,if=none,id=hd0 "\
  65                "-device scsi-hd,drive=hd0,bootindex=0",
  66 }
  67 class BaseVM(object):
  68
  69     envvars = [
  70         "https_proxy",
  71         "http_proxy",
  72         "ftp_proxy",
  73         "no_proxy",
  74     ]
  75
  76     # The script to run in the guest that builds QEMU
  77     BUILD_SCRIPT = ""
  78     # The guest name, to be overridden by subclasses
  79     name = "#base"
  80     # The guest architecture, to be overridden by subclasses
  81     arch = "#arch"
  82     # command to halt the guest, can be overridden by subclasses
  83     poweroff = "poweroff"
  84     # Time to wait for shutdown to finish.
  85     shutdown_timeout_default = 30
  86     # enable IPv6 networking
  87     ipv6 = True
  88     # This is the timeout on the wait for console bytes.
  89     socket_timeout = 120
  90     # Scale up some timeouts under TCG.
  91     # 4 is arbitrary, but greater than 2,
  92     # since we found we need to wait more than twice as long.
  93     tcg_timeout_multiplier = 4
  94     def __init__(self, args, config=None):
  95         self._guest = None
  96         self._genisoimage = args.genisoimage
  97         self._build_path = args.build_path
  98         self._efi_aarch64 = args.efi_aarch64
  99         self._source_path = args.source_path
 100         # Allow input config to override defaults.
 101         self._config = DEFAULT_CONFIG.copy()
 102         if config != None:
 103             self._config.update(config)
 104         self.validate_ssh_keys()
 105         self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-",
 106                                                          suffix=".tmp",
 107                                                          dir="."))
 108         atexit.register(shutil.rmtree, self._tmpdir)
 109         # Copy the key files to a temporary directory.
 110         # Also chmod the key file to agree with ssh requirements.
 111         self._config['ssh_key'] = \
 112             open(self._config['ssh_key_file']).read().rstrip()
 113         self._config['ssh_pub_key'] = \
 114             open(self._config['ssh_pub_key_file']).read().rstrip()
 115         self._ssh_tmp_key_file = os.path.join(self._tmpdir, "id_rsa")
 116         open(self._ssh_tmp_key_file, "w").write(self._config['ssh_key'])
 117         subprocess.check_call(["chmod", "600", self._ssh_tmp_key_file])
 118
 119         self._ssh_tmp_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
 120         open(self._ssh_tmp_pub_key_file,
 121              "w").write(self._config['ssh_pub_key'])
 122
 123         self.debug = args.debug
 124         self._console_log_path = None
 125         if args.log_console:
 126                 self._console_log_path = \
 127                          os.path.join(os.path.expanduser("~/.cache/qemu-vm"),
 128                                       "{}.install.log".format(self.name))
 129         self._stderr = sys.stderr
 130         self._devnull = open(os.devnull, "w")
 131         if self.debug:
 132             self._stdout = sys.stdout
 133         else:
 134             self._stdout = self._devnull
 135         netdev = "user,id=vnet,hostfwd=:127.0.0.1:{}-:22"
 136         self._args = [ \
 137             "-nodefaults", "-m", self._config['memory'],
 138             "-cpu", self._config['cpu'],
 139             "-netdev",
 140             netdev.format(self._config['ssh_port']) +
 141             (",ipv6=no" if not self.ipv6 else "") +
 142             (",dns=" + self._config['dns'] if self._config['dns'] else ""),
 143             "-device", "virtio-net-pci,netdev=vnet",
 144             "-vnc", "127.0.0.1:0,to=20"]
 145         if args.jobs and args.jobs > 1:
 146             self._args += ["-smp", "%d" % args.jobs]
 147         if kvm_available(self.arch):
 148             self._shutdown_timeout = self.shutdown_timeout_default
 149             self._args += ["-enable-kvm"]
 150         else:
 151             logging.info("KVM not available, not using -enable-kvm")
 152             self._shutdown_timeout = \
 153                 self.shutdown_timeout_default * self.tcg_timeout_multiplier
 154         self._data_args = []
 155
 156         if self._config['qemu_args'] != None:
 157             qemu_args = self._config['qemu_args']
 158             qemu_args = qemu_args.replace('\n',' ').replace('\r','')
 159             # shlex groups quoted arguments together
 160             # we need this to keep the quoted args together for when
 161             # the QEMU command is issued later.
 162             args = shlex.split(qemu_args)
 163             self._config['extra_args'] = []
 164             for arg in args:
 165                 if arg:
 166                     # Preserve quotes around arguments.
 167                     # shlex above takes them out, so add them in.
 168                     if " " in arg:
 169                         arg = '"{}"'.format(arg)
 170                     self._config['extra_args'].append(arg)
 171
 172     def validate_ssh_keys(self):
 173         """Check to see if the ssh key files exist."""
 174         if 'ssh_key_file' not in self._config or\
 175            not os.path.exists(self._config['ssh_key_file']):
 176             raise Exception("ssh key file not found.")
 177         if 'ssh_pub_key_file' not in self._config or\
 178            not os.path.exists(self._config['ssh_pub_key_file']):
 179                raise Exception("ssh pub key file not found.")
 180
 181     def wait_boot(self, wait_string=None):
 182         """Wait for the standard string we expect
 183            on completion of a normal boot.
 184            The user can also choose to override with an
 185            alternate string to wait for."""
 186         if wait_string is None:
 187             if self.login_prompt is None:
 188                 raise Exception("self.login_prompt not defined")
 189             wait_string = self.login_prompt
 190         # Intentionally bump up the default timeout under TCG,
 191         # since the console wait below takes longer.
 192         timeout = self.socket_timeout
 193         if not kvm_available(self.arch):
 194             timeout *= 8
 195         self.console_init(timeout=timeout)
 196         self.console_wait(wait_string)
 197
 198     def _download_with_cache(self, url, sha256sum=None, sha512sum=None):
 199         def check_sha256sum(fname):
 200             if not sha256sum:
 201                 return True
 202             checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
 203             return sha256sum == checksum.decode("utf-8")
 204
 205         def check_sha512sum(fname):
 206             if not sha512sum:
 207                 return True
 208             checksum = subprocess.check_output(["sha512sum", fname]).split()[0]
 209             return sha512sum == checksum.decode("utf-8")
 210
 211         cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
 212         if not os.path.exists(cache_dir):
 213             os.makedirs(cache_dir)
 214         fname = os.path.join(cache_dir,
 215                              hashlib.sha1(url.encode("utf-8")).hexdigest())
 216         if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname):
 217             return fname
 218         logging.debug("Downloading %s to %s...", url, fname)
 219         subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"],
 220                               stdout=self._stdout, stderr=self._stderr)
 221         os.rename(fname + ".download", fname)
 222         return fname
 223
 224     def _ssh_do(self, user, cmd, check):
 225         ssh_cmd = ["ssh",
 226                    "-t",
 227                    "-o", "StrictHostKeyChecking=no",
 228                    "-o", "UserKnownHostsFile=" + os.devnull,
 229                    "-o",
 230                    "ConnectTimeout={}".format(self._config["ssh_timeout"]),
 231                    "-p", str(self.ssh_port), "-i", self._ssh_tmp_key_file]
 232         # If not in debug mode, set ssh to quiet mode to
 233         # avoid printing the results of commands.
 234         if not self.debug:
 235             ssh_cmd.append("-q")
 236         for var in self.envvars:
 237             ssh_cmd += ['-o', "SendEnv=%s" % var ]
 238         assert not isinstance(cmd, str)
 239         ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd)
 240         logging.debug("ssh_cmd: %s", " ".join(ssh_cmd))
 241         r = subprocess.call(ssh_cmd)
 242         if check and r != 0:
 243             raise Exception("SSH command failed: %s" % cmd)
 244         return r
 245
 246     def ssh(self, *cmd):
 247         return self._ssh_do(self._config["guest_user"], cmd, False)
 248
 249     def ssh_root(self, *cmd):
 250         return self._ssh_do(self._config["root_user"], cmd, False)
 251
 252     def ssh_check(self, *cmd):
 253         self._ssh_do(self._config["guest_user"], cmd, True)
 254
 255     def ssh_root_check(self, *cmd):
 256         self._ssh_do(self._config["root_user"], cmd, True)
 257
 258     def build_image(self, img):
 259         raise NotImplementedError
 260
 261     def exec_qemu_img(self, *args):
 262         cmd = [os.environ.get("QEMU_IMG", "qemu-img")]
 263         cmd.extend(list(args))
 264         subprocess.check_call(cmd)
 265
 266     def add_source_dir(self, src_dir):
 267         name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5]
 268         tarfile = os.path.join(self._tmpdir, name + ".tar")
 269         logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir)
 270         subprocess.check_call(["./scripts/archive-source.sh", tarfile],
 271                               cwd=src_dir, stdin=self._devnull,
 272                               stdout=self._stdout, stderr=self._stderr)
 273         self._data_args += ["-drive",
 274                             "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
 275                                     (tarfile, name),
 276                             "-device",
 277                             "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)]
 278
 279     def boot(self, img, extra_args=[]):
 280         boot_dev = BOOT_DEVICE[self._config['boot_dev_type']]
 281         boot_params = boot_dev.format(img)
 282         args = self._args + boot_params.split(' ')
 283         args += self._data_args + extra_args + self._config['extra_args']
 284         logging.debug("QEMU args: %s", " ".join(args))
 285         qemu_path = get_qemu_path(self.arch, self._build_path)
 286
 287         # Since console_log_path is only set when the user provides the
 288         # log_console option, we will set drain_console=True so the
 289         # console is always drained.
 290         guest = QEMUMachine(binary=qemu_path, args=args,
 291                             console_log=self._console_log_path,
 292                             drain_console=True)
 293         guest.set_machine(self._config['machine'])
 294         guest.set_console()
 295         try:
 296             guest.launch()
 297         except:
 298             logging.error("Failed to launch QEMU, command line:")
 299             logging.error(" ".join([qemu_path] + args))
 300             logging.error("Log:")
 301             logging.error(guest.get_log())
 302             logging.error("QEMU version >= 2.10 is required")
 303             raise
 304         atexit.register(self.shutdown)
 305         self._guest = guest
 306         # Init console so we can start consuming the chars.
 307         self.console_init()
 308         usernet_info = guest.qmp("human-monitor-command",
 309                                  command_line="info usernet").get("return")
 310         self.ssh_port = get_info_usernet_hostfwd_port(usernet_info)
 311         if not self.ssh_port:
 312             raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
 313                             usernet_info)
 314
 315     def console_init(self, timeout = None):
 316         if timeout == None:
 317             timeout = self.socket_timeout
 318         vm = self._guest
 319         vm.console_socket.settimeout(timeout)
 320         self.console_raw_path = os.path.join(vm._temp_dir,
 321                                              vm._name + "-console.raw")
 322         self.console_raw_file = open(self.console_raw_path, 'wb')
 323
 324     def console_log(self, text):
 325         for line in re.split("[\r\n]", text):
 326             # filter out terminal escape sequences
 327             line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line)
 328             line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line)
 329             # replace unprintable chars
 330             line = re.sub("\x1b", "<esc>", line)
 331             line = re.sub("[\x00-\x1f]", ".", line)
 332             line = re.sub("[\x80-\xff]", ".", line)
 333             if line == "":
 334                 continue
 335             # log console line
 336             sys.stderr.write("con recv: %s\n" % line)
 337
 338     def console_wait(self, expect, expectalt = None):
 339         vm = self._guest
 340         output = ""
 341         while True:
 342             try:
 343                 chars = vm.console_socket.recv(1)
 344                 if self.console_raw_file:
 345                     self.console_raw_file.write(chars)
 346                     self.console_raw_file.flush()
 347             except socket.timeout:
 348                 sys.stderr.write("console: *** read timeout ***\n")
 349                 sys.stderr.write("console: waiting for: '%s'\n" % expect)
 350                 if not expectalt is None:
 351                     sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt)
 352                 sys.stderr.write("console: line buffer:\n")
 353                 sys.stderr.write("\n")
 354                 self.console_log(output.rstrip())
 355                 sys.stderr.write("\n")
 356                 raise
 357             output += chars.decode("latin1")
 358             if expect in output:
 359                 break
 360             if not expectalt is None and expectalt in output:
 361                 break
 362             if "\r" in output or "\n" in output:
 363                 lines = re.split("[\r\n]", output)
 364                 output = lines.pop()
 365                 if self.debug:
 366                     self.console_log("\n".join(lines))
 367         if self.debug:
 368             self.console_log(output)
 369         if not expectalt is None and expectalt in output:
 370             return False
 371         return True
 372
 373     def console_consume(self):
 374         vm = self._guest
 375         output = ""
 376         vm.console_socket.setblocking(0)
 377         while True:
 378             try:
 379                 chars = vm.console_socket.recv(1)
 380             except:
 381                 break
 382             output += chars.decode("latin1")
 383             if "\r" in output or "\n" in output:
 384                 lines = re.split("[\r\n]", output)
 385                 output = lines.pop()
 386                 if self.debug:
 387                     self.console_log("\n".join(lines))
 388         if self.debug:
 389             self.console_log(output)
 390         vm.console_socket.setblocking(1)
 391
 392     def console_send(self, command):
 393         vm = self._guest
 394         if self.debug:
 395             logline = re.sub("\n", "<enter>", command)
 396             logline = re.sub("[\x00-\x1f]", ".", logline)
 397             sys.stderr.write("con send: %s\n" % logline)
 398         for char in list(command):
 399             vm.console_socket.send(char.encode("utf-8"))
 400             time.sleep(0.01)
 401
 402     def console_wait_send(self, wait, command):
 403         self.console_wait(wait)
 404         self.console_send(command)
 405
 406     def console_ssh_init(self, prompt, user, pw):
 407         sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" \
 408                      % self._config['ssh_pub_key'].rstrip()
 409         self.console_wait_send("login:",    "%s\n" % user)
 410         self.console_wait_send("Password:", "%s\n" % pw)
 411         self.console_wait_send(prompt,      "mkdir .ssh\n")
 412         self.console_wait_send(prompt,      sshkey_cmd)
 413         self.console_wait_send(prompt,      "chmod 755 .ssh\n")
 414         self.console_wait_send(prompt,      "chmod 644 .ssh/authorized_keys\n")
 415
 416     def console_sshd_config(self, prompt):
 417         self.console_wait(prompt)
 418         self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n")
 419         for var in self.envvars:
 420             self.console_wait(prompt)
 421             self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var)
 422
 423     def print_step(self, text):
 424         sys.stderr.write("### %s ...\n" % text)
 425
 426     def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"):
 427         # Allow more time for VM to boot under TCG.
 428         if not kvm_available(self.arch):
 429             seconds *= self.tcg_timeout_multiplier
 430         starttime = datetime.datetime.now()
 431         endtime = starttime + datetime.timedelta(seconds=seconds)
 432         cmd_success = False
 433         while datetime.datetime.now() < endtime:
 434             if wait_root and self.ssh_root(cmd) == 0:
 435                 cmd_success = True
 436                 break
 437             elif self.ssh(cmd) == 0:
 438                 cmd_success = True
 439                 break
 440             seconds = (endtime - datetime.datetime.now()).total_seconds()
 441             logging.debug("%ds before timeout", seconds)
 442             time.sleep(1)
 443         if not cmd_success:
 444             raise Exception("Timeout while waiting for guest ssh")
 445
 446     def shutdown(self):
 447         self._guest.shutdown(timeout=self._shutdown_timeout)
 448
 449     def wait(self):
 450         self._guest.wait(timeout=self._shutdown_timeout)
 451
 452     def graceful_shutdown(self):
 453         self.ssh_root(self.poweroff)
 454         self._guest.wait(timeout=self._shutdown_timeout)
 455
 456     def qmp(self, *args, **kwargs):
 457         return self._guest.qmp(*args, **kwargs)
 458
 459     def gen_cloud_init_iso(self):
 460         cidir = self._tmpdir
 461         mdata = open(os.path.join(cidir, "meta-data"), "w")
 462         name = self.name.replace(".","-")
 463         mdata.writelines(["instance-id: {}-vm-0\n".format(name),
 464                           "local-hostname: {}-guest\n".format(name)])
 465         mdata.close()
 466         udata = open(os.path.join(cidir, "user-data"), "w")
 467         print("guest user:pw {}:{}".format(self._config['guest_user'],
 468                                            self._config['guest_pass']))
 469         udata.writelines(["#cloud-config\n",
 470                           "chpasswd:\n",
 471                           "  list: |\n",
 472                           "    root:%s\n" % self._config['root_pass'],
 473                           "    %s:%s\n" % (self._config['guest_user'],
 474                                            self._config['guest_pass']),
 475                           "  expire: False\n",
 476                           "users:\n",
 477                           "  - name: %s\n" % self._config['guest_user'],
 478                           "    sudo: ALL=(ALL) NOPASSWD:ALL\n",
 479                           "    ssh-authorized-keys:\n",
 480                           "    - %s\n" % self._config['ssh_pub_key'],
 481                           "  - name: root\n",
 482                           "    ssh-authorized-keys:\n",
 483                           "    - %s\n" % self._config['ssh_pub_key'],
 484                           "locale: en_US.UTF-8\n"])
 485         proxy = os.environ.get("http_proxy")
 486         if not proxy is None:
 487             udata.writelines(["apt:\n",
 488                               "  proxy: %s" % proxy])
 489         udata.close()
 490         subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso",
 491                                "-volid", "cidata", "-joliet", "-rock",
 492                                "user-data", "meta-data"],
 493                               cwd=cidir,
 494                               stdin=self._devnull, stdout=self._stdout,
 495                               stderr=self._stdout)
 496         return os.path.join(cidir, "cloud-init.iso")
 497
 498 def get_qemu_path(arch, build_path=None):
 499     """Fetch the path to the qemu binary."""
 500     # If QEMU environment variable set, it takes precedence
 501     if "QEMU" in os.environ:
 502         qemu_path = os.environ["QEMU"]
 503     elif build_path:
 504         qemu_path = os.path.join(build_path, arch + "-softmmu")
 505         qemu_path = os.path.join(qemu_path, "qemu-system-" + arch)
 506     else:
 507         # Default is to use system path for qemu.
 508         qemu_path = "qemu-system-" + arch
 509     return qemu_path
 510
 511 def get_qemu_version(qemu_path):
 512     """Get the version number from the current QEMU,
 513        and return the major number."""
 514     output = subprocess.check_output([qemu_path, '--version'])
 515     version_line = output.decode("utf-8")
 516     version_num = re.split(' |\(', version_line)[3].split('.')[0]
 517     return int(version_num)
 518
 519 def parse_config(config, args):
 520     """ Parse yaml config and populate our config structure.
 521         The yaml config allows the user to override the
 522         defaults for VM parameters.  In many cases these
 523         defaults can be overridden without rebuilding the VM."""
 524     if args.config:
 525         config_file = args.config
 526     elif 'QEMU_CONFIG' in os.environ:
 527         config_file = os.environ['QEMU_CONFIG']
 528     else:
 529         return config
 530     if not os.path.exists(config_file):
 531         raise Exception("config file {} does not exist".format(config_file))
 532     # We gracefully handle importing the yaml module
 533     # since it might not be installed.
 534     # If we are here it means the user supplied a .yml file,
 535     # so if the yaml module is not installed we will exit with error.
 536     try:
 537         import yaml
 538     except ImportError:
 539         print("The python3-yaml package is needed "\
 540               "to support config.yaml files")
 541         # Instead of raising an exception we exit to avoid
 542         # a raft of messy (expected) errors to stdout.
 543         exit(1)
 544     with open(config_file) as f:
 545         yaml_dict = yaml.safe_load(f)
 546
 547     if 'qemu-conf' in yaml_dict:
 548         config.update(yaml_dict['qemu-conf'])
 549     else:
 550         raise Exception("config file {} is not valid"\
 551                         " missing qemu-conf".format(config_file))
 552     return config
 553
 554 def parse_args(vmcls):
 555
 556     def get_default_jobs():
 557         if multiprocessing.cpu_count() > 1:
 558             if kvm_available(vmcls.arch):
 559                 return multiprocessing.cpu_count() // 2
 560             elif os.uname().machine == "x86_64" and \
 561                  vmcls.arch in ["aarch64", "x86_64", "i386"]:
 562                 # MTTCG is available on these arches and we can allow
 563                 # more cores. but only up to a reasonable limit. User
 564                 # can always override these limits with --jobs.
 565                 return min(multiprocessing.cpu_count() // 2, 8)
 566         else:
 567             return 1
 568
 569     parser = argparse.ArgumentParser(
 570         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 571         description="Utility for provisioning VMs and running builds",
 572         epilog="""Remaining arguments are passed to the command.
 573         Exit codes: 0 = success, 1 = command line error,
 574         2 = environment initialization failed,
 575         3 = test command failed""")
 576     parser.add_argument("--debug", "-D", action="store_true",
 577                         help="enable debug output")
 578     parser.add_argument("--image", "-i", default="%s.img" % vmcls.name,
 579                         help="image file name")
 580     parser.add_argument("--force", "-f", action="store_true",
 581                         help="force build image even if image exists")
 582     parser.add_argument("--jobs", type=int, default=get_default_jobs(),
 583                         help="number of virtual CPUs")
 584     parser.add_argument("--verbose", "-V", action="store_true",
 585                         help="Pass V=1 to builds within the guest")
 586     parser.add_argument("--build-image", "-b", action="store_true",
 587                         help="build image")
 588     parser.add_argument("--build-qemu",
 589                         help="build QEMU from source in guest")
 590     parser.add_argument("--build-target",
 591                         help="QEMU build target", default="check")
 592     parser.add_argument("--build-path", default=None,
 593                         help="Path of build directory, "\
 594                         "for using build tree QEMU binary. ")
 595     parser.add_argument("--source-path", default=None,
 596                         help="Path of source directory, "\
 597                         "for finding additional files. ")
 598     parser.add_argument("--interactive", "-I", action="store_true",
 599                         help="Interactively run command")
 600     parser.add_argument("--snapshot", "-s", action="store_true",
 601                         help="run tests with a snapshot")
 602     parser.add_argument("--genisoimage", default="genisoimage",
 603                         help="iso imaging tool")
 604     parser.add_argument("--config", "-c", default=None,
 605                         help="Provide config yaml for configuration. "\
 606                         "See config_example.yaml for example.")
 607     parser.add_argument("--efi-aarch64",
 608                         default="/usr/share/qemu-efi-aarch64/QEMU_EFI.fd",
 609                         help="Path to efi image for aarch64 VMs.")
 610     parser.add_argument("--log-console", action="store_true",
 611                         help="Log console to file.")
 612     parser.add_argument("commands", nargs="*", help="""Remaining
 613         commands after -- are passed to command inside the VM""")
 614
 615     return parser.parse_args()
 616
 617 def main(vmcls, config=None):
 618     try:
 619         if config == None:
 620             config = DEFAULT_CONFIG
 621         args = parse_args(vmcls)
 622         if not args.commands and not args.build_qemu and not args.build_image:
 623             print("Nothing to do?")
 624             return 1
 625         config = parse_config(config, args)
 626         logging.basicConfig(level=(logging.DEBUG if args.debug
 627                                    else logging.WARN))
 628         vm = vmcls(args, config=config)
 629         if args.build_image:
 630             if os.path.exists(args.image) and not args.force:
 631                 sys.stderr.writelines(["Image file exists: %s\n" % args.image,
 632                                       "Use --force option to overwrite\n"])
 633                 return 1
 634             return vm.build_image(args.image)
 635         if args.build_qemu:
 636             vm.add_source_dir(args.build_qemu)
 637             cmd = [vm.BUILD_SCRIPT.format(
 638                    configure_opts = " ".join(args.commands),
 639                    jobs=int(args.jobs),
 640                    target=args.build_target,
 641                    verbose = "V=1" if args.verbose else "")]
 642         else:
 643             cmd = args.commands
 644         img = args.image
 645         if args.snapshot:
 646             img += ",snapshot=on"
 647         vm.boot(img)
 648         vm.wait_ssh()
 649     except Exception as e:
 650         if isinstance(e, SystemExit) and e.code == 0:
 651             return 0
 652         sys.stderr.write("Failed to prepare guest environment\n")
 653         traceback.print_exc()
 654         return 2
 655
 656     exitcode = 0
 657     if vm.ssh(*cmd) != 0:
 658         exitcode = 3
 659     if args.interactive:
 660         vm.ssh()
 661
 662     if not args.snapshot:
 663         vm.graceful_shutdown()
 664
 665     return exitcode