2 # VM testing base class
4 # Copyright 2017-2019 Red Hat Inc.
7 # Fam Zheng <famz@redhat.com>
8 # Gerd Hoffmann <kraxel@redhat.com>
10 # This code is licensed under the GPL version 2 or later. See
11 # the COPYING file in the top-level directory.
21 sys
.path
.append(os
.path
.join(os
.path
.dirname(__file__
), '..', '..', 'python'))
22 from qemu
.accel
import kvm_available
23 from qemu
.machine
import QEMUMachine
30 import multiprocessing
34 SSH_KEY_FILE
= os
.path
.join(os
.path
.dirname(__file__
),
35 "..", "keys", "id_rsa")
36 SSH_PUB_KEY_FILE
= os
.path
.join(os
.path
.dirname(__file__
),
37 "..", "keys", "id_rsa.pub")
39 # This is the standard configuration.
40 # Any or all of these can be overridden by
41 # passing in a config argument to the VM constructor.
45 'guest_user' : "qemu",
46 'guest_pass' : "qemupass",
47 'root_pass' : "qemupass",
48 'ssh_key_file' : SSH_KEY_FILE
,
49 'ssh_pub_key_file': SSH_PUB_KEY_FILE
,
56 'boot_dev_type' : "block",
60 'block' : "-drive file={},if=none,id=drive0,cache=writeback "\
61 "-device virtio-blk,drive=drive0,bootindex=0",
62 'scsi' : "-device virtio-scsi-device,id=scsi "\
63 "-drive file={},format=raw,if=none,id=hd0 "\
64 "-device scsi-hd,drive=hd0,bootindex=0",
75 # The script to run in the guest that builds QEMU
77 # The guest name, to be overridden by subclasses
79 # The guest architecture, to be overridden by subclasses
81 # command to halt the guest, can be overridden by subclasses
83 # Time to wait for shutdown to finish.
84 shutdown_timeout_default
= 30
85 # enable IPv6 networking
87 # This is the timeout on the wait for console bytes.
89 # Scale up some timeouts under TCG.
90 # 4 is arbitrary, but greater than 2,
91 # since we found we need to wait more than twice as long.
92 tcg_timeout_multiplier
= 4
93 def __init__(self
, args
, config
=None):
95 self
._genisoimage
= args
.genisoimage
96 self
._build
_path
= args
.build_path
97 self
._efi
_aarch
64 = args
.efi_aarch64
98 # Allow input config to override defaults.
99 self
._config
= DEFAULT_CONFIG
.copy()
101 self
._config
.update(config
)
102 self
.validate_ssh_keys()
103 self
._tmpdir
= os
.path
.realpath(tempfile
.mkdtemp(prefix
="vm-test-",
106 atexit
.register(shutil
.rmtree
, self
._tmpdir
)
107 # Copy the key files to a temporary directory.
108 # Also chmod the key file to agree with ssh requirements.
109 self
._config
['ssh_key'] = \
110 open(self
._config
['ssh_key_file']).read().rstrip()
111 self
._config
['ssh_pub_key'] = \
112 open(self
._config
['ssh_pub_key_file']).read().rstrip()
113 self
._ssh
_tmp
_key
_file
= os
.path
.join(self
._tmpdir
, "id_rsa")
114 open(self
._ssh
_tmp
_key
_file
, "w").write(self
._config
['ssh_key'])
115 subprocess
.check_call(["chmod", "600", self
._ssh
_tmp
_key
_file
])
117 self
._ssh
_tmp
_pub
_key
_file
= os
.path
.join(self
._tmpdir
, "id_rsa.pub")
118 open(self
._ssh
_tmp
_pub
_key
_file
,
119 "w").write(self
._config
['ssh_pub_key'])
121 self
.debug
= args
.debug
122 self
._console
_log
_path
= None
124 self
._console
_log
_path
= \
125 os
.path
.join(os
.path
.expanduser("~/.cache/qemu-vm"),
126 "{}.install.log".format(self
.name
))
127 self
._stderr
= sys
.stderr
128 self
._devnull
= open(os
.devnull
, "w")
130 self
._stdout
= sys
.stdout
132 self
._stdout
= self
._devnull
133 netdev
= "user,id=vnet,hostfwd=:127.0.0.1:{}-:22"
135 "-nodefaults", "-m", self
._config
['memory'],
136 "-cpu", self
._config
['cpu'],
138 netdev
.format(self
._config
['ssh_port']) +
139 (",ipv6=no" if not self
.ipv6
else "") +
140 (",dns=" + self
._config
['dns'] if self
._config
['dns'] else ""),
141 "-device", "virtio-net-pci,netdev=vnet",
142 "-vnc", "127.0.0.1:0,to=20"]
143 if args
.jobs
and args
.jobs
> 1:
144 self
._args
+= ["-smp", "%d" % args
.jobs
]
145 if kvm_available(self
.arch
):
146 self
._shutdown
_timeout
= self
.shutdown_timeout_default
147 self
._args
+= ["-enable-kvm"]
149 logging
.info("KVM not available, not using -enable-kvm")
150 self
._shutdown
_timeout
= \
151 self
.shutdown_timeout_default
* self
.tcg_timeout_multiplier
154 if self
._config
['qemu_args'] != None:
155 qemu_args
= self
._config
['qemu_args']
156 qemu_args
= qemu_args
.replace('\n',' ').replace('\r','')
157 # shlex groups quoted arguments together
158 # we need this to keep the quoted args together for when
159 # the QEMU command is issued later.
160 args
= shlex
.split(qemu_args
)
161 self
._config
['extra_args'] = []
164 # Preserve quotes around arguments.
165 # shlex above takes them out, so add them in.
167 arg
= '"{}"'.format(arg
)
168 self
._config
['extra_args'].append(arg
)
170 def validate_ssh_keys(self
):
171 """Check to see if the ssh key files exist."""
172 if 'ssh_key_file' not in self
._config
or\
173 not os
.path
.exists(self
._config
['ssh_key_file']):
174 raise Exception("ssh key file not found.")
175 if 'ssh_pub_key_file' not in self
._config
or\
176 not os
.path
.exists(self
._config
['ssh_pub_key_file']):
177 raise Exception("ssh pub key file not found.")
179 def wait_boot(self
, wait_string
=None):
180 """Wait for the standard string we expect
181 on completion of a normal boot.
182 The user can also choose to override with an
183 alternate string to wait for."""
184 if wait_string
is None:
185 if self
.login_prompt
is None:
186 raise Exception("self.login_prompt not defined")
187 wait_string
= self
.login_prompt
188 # Intentionally bump up the default timeout under TCG,
189 # since the console wait below takes longer.
190 timeout
= self
.socket_timeout
191 if not kvm_available(self
.arch
):
193 self
.console_init(timeout
=timeout
)
194 self
.console_wait(wait_string
)
196 def _download_with_cache(self
, url
, sha256sum
=None, sha512sum
=None):
197 def check_sha256sum(fname
):
200 checksum
= subprocess
.check_output(["sha256sum", fname
]).split()[0]
201 return sha256sum
== checksum
.decode("utf-8")
203 def check_sha512sum(fname
):
206 checksum
= subprocess
.check_output(["sha512sum", fname
]).split()[0]
207 return sha512sum
== checksum
.decode("utf-8")
209 cache_dir
= os
.path
.expanduser("~/.cache/qemu-vm/download")
210 if not os
.path
.exists(cache_dir
):
211 os
.makedirs(cache_dir
)
212 fname
= os
.path
.join(cache_dir
,
213 hashlib
.sha1(url
.encode("utf-8")).hexdigest())
214 if os
.path
.exists(fname
) and check_sha256sum(fname
) and check_sha512sum(fname
):
216 logging
.debug("Downloading %s to %s...", url
, fname
)
217 subprocess
.check_call(["wget", "-c", url
, "-O", fname
+ ".download"],
218 stdout
=self
._stdout
, stderr
=self
._stderr
)
219 os
.rename(fname
+ ".download", fname
)
222 def _ssh_do(self
, user
, cmd
, check
):
225 "-o", "StrictHostKeyChecking=no",
226 "-o", "UserKnownHostsFile=" + os
.devnull
,
228 "ConnectTimeout={}".format(self
._config
["ssh_timeout"]),
229 "-p", self
.ssh_port
, "-i", self
._ssh
_tmp
_key
_file
]
230 # If not in debug mode, set ssh to quiet mode to
231 # avoid printing the results of commands.
234 for var
in self
.envvars
:
235 ssh_cmd
+= ['-o', "SendEnv=%s" % var
]
236 assert not isinstance(cmd
, str)
237 ssh_cmd
+= ["%s@127.0.0.1" % user
] + list(cmd
)
238 logging
.debug("ssh_cmd: %s", " ".join(ssh_cmd
))
239 r
= subprocess
.call(ssh_cmd
)
241 raise Exception("SSH command failed: %s" % cmd
)
245 return self
._ssh
_do
(self
._config
["guest_user"], cmd
, False)
247 def ssh_root(self
, *cmd
):
248 return self
._ssh
_do
("root", cmd
, False)
250 def ssh_check(self
, *cmd
):
251 self
._ssh
_do
(self
._config
["guest_user"], cmd
, True)
253 def ssh_root_check(self
, *cmd
):
254 self
._ssh
_do
("root", cmd
, True)
256 def build_image(self
, img
):
257 raise NotImplementedError
259 def exec_qemu_img(self
, *args
):
260 cmd
= [os
.environ
.get("QEMU_IMG", "qemu-img")]
261 cmd
.extend(list(args
))
262 subprocess
.check_call(cmd
)
264 def add_source_dir(self
, src_dir
):
265 name
= "data-" + hashlib
.sha1(src_dir
.encode("utf-8")).hexdigest()[:5]
266 tarfile
= os
.path
.join(self
._tmpdir
, name
+ ".tar")
267 logging
.debug("Creating archive %s for src_dir dir: %s", tarfile
, src_dir
)
268 subprocess
.check_call(["./scripts/archive-source.sh", tarfile
],
269 cwd
=src_dir
, stdin
=self
._devnull
,
270 stdout
=self
._stdout
, stderr
=self
._stderr
)
271 self
._data
_args
+= ["-drive",
272 "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
275 "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name
, name
)]
277 def boot(self
, img
, extra_args
=[]):
278 boot_dev
= BOOT_DEVICE
[self
._config
['boot_dev_type']]
279 boot_params
= boot_dev
.format(img
)
280 args
= self
._args
+ boot_params
.split(' ')
281 args
+= self
._data
_args
+ extra_args
+ self
._config
['extra_args']
282 logging
.debug("QEMU args: %s", " ".join(args
))
283 qemu_path
= get_qemu_path(self
.arch
, self
._build
_path
)
285 # Since console_log_path is only set when the user provides the
286 # log_console option, we will set drain_console=True so the
287 # console is always drained.
288 guest
= QEMUMachine(binary
=qemu_path
, args
=args
,
289 console_log
=self
._console
_log
_path
,
291 guest
.set_machine(self
._config
['machine'])
296 logging
.error("Failed to launch QEMU, command line:")
297 logging
.error(" ".join([qemu_path
] + args
))
298 logging
.error("Log:")
299 logging
.error(guest
.get_log())
300 logging
.error("QEMU version >= 2.10 is required")
302 atexit
.register(self
.shutdown
)
304 # Init console so we can start consuming the chars.
306 usernet_info
= guest
.qmp("human-monitor-command",
307 command_line
="info usernet")
309 for l
in usernet_info
["return"].splitlines():
311 if "TCP[HOST_FORWARD]" in fields
and "22" in fields
:
312 self
.ssh_port
= l
.split()[3]
313 if not self
.ssh_port
:
314 raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
317 def console_init(self
, timeout
= None):
319 timeout
= self
.socket_timeout
321 vm
.console_socket
.settimeout(timeout
)
322 self
.console_raw_path
= os
.path
.join(vm
._temp
_dir
,
323 vm
._name
+ "-console.raw")
324 self
.console_raw_file
= open(self
.console_raw_path
, 'wb')
326 def console_log(self
, text
):
327 for line
in re
.split("[\r\n]", text
):
328 # filter out terminal escape sequences
329 line
= re
.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line
)
330 line
= re
.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line
)
331 # replace unprintable chars
332 line
= re
.sub("\x1b", "<esc>", line
)
333 line
= re
.sub("[\x00-\x1f]", ".", line
)
334 line
= re
.sub("[\x80-\xff]", ".", line
)
338 sys
.stderr
.write("con recv: %s\n" % line
)
340 def console_wait(self
, expect
, expectalt
= None):
345 chars
= vm
.console_socket
.recv(1)
346 if self
.console_raw_file
:
347 self
.console_raw_file
.write(chars
)
348 self
.console_raw_file
.flush()
349 except socket
.timeout
:
350 sys
.stderr
.write("console: *** read timeout ***\n")
351 sys
.stderr
.write("console: waiting for: '%s'\n" % expect
)
352 if not expectalt
is None:
353 sys
.stderr
.write("console: waiting for: '%s' (alt)\n" % expectalt
)
354 sys
.stderr
.write("console: line buffer:\n")
355 sys
.stderr
.write("\n")
356 self
.console_log(output
.rstrip())
357 sys
.stderr
.write("\n")
359 output
+= chars
.decode("latin1")
362 if not expectalt
is None and expectalt
in output
:
364 if "\r" in output
or "\n" in output
:
365 lines
= re
.split("[\r\n]", output
)
368 self
.console_log("\n".join(lines
))
370 self
.console_log(output
)
371 if not expectalt
is None and expectalt
in output
:
375 def console_consume(self
):
378 vm
.console_socket
.setblocking(0)
381 chars
= vm
.console_socket
.recv(1)
384 output
+= chars
.decode("latin1")
385 if "\r" in output
or "\n" in output
:
386 lines
= re
.split("[\r\n]", output
)
389 self
.console_log("\n".join(lines
))
391 self
.console_log(output
)
392 vm
.console_socket
.setblocking(1)
394 def console_send(self
, command
):
397 logline
= re
.sub("\n", "<enter>", command
)
398 logline
= re
.sub("[\x00-\x1f]", ".", logline
)
399 sys
.stderr
.write("con send: %s\n" % logline
)
400 for char
in list(command
):
401 vm
.console_socket
.send(char
.encode("utf-8"))
404 def console_wait_send(self
, wait
, command
):
405 self
.console_wait(wait
)
406 self
.console_send(command
)
408 def console_ssh_init(self
, prompt
, user
, pw
):
409 sshkey_cmd
= "echo '%s' > .ssh/authorized_keys\n" \
410 % self
._config
['ssh_pub_key'].rstrip()
411 self
.console_wait_send("login:", "%s\n" % user
)
412 self
.console_wait_send("Password:", "%s\n" % pw
)
413 self
.console_wait_send(prompt
, "mkdir .ssh\n")
414 self
.console_wait_send(prompt
, sshkey_cmd
)
415 self
.console_wait_send(prompt
, "chmod 755 .ssh\n")
416 self
.console_wait_send(prompt
, "chmod 644 .ssh/authorized_keys\n")
418 def console_sshd_config(self
, prompt
):
419 self
.console_wait(prompt
)
420 self
.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n")
421 for var
in self
.envvars
:
422 self
.console_wait(prompt
)
423 self
.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var
)
425 def print_step(self
, text
):
426 sys
.stderr
.write("### %s ...\n" % text
)
428 def wait_ssh(self
, wait_root
=False, seconds
=300, cmd
="exit 0"):
429 # Allow more time for VM to boot under TCG.
430 if not kvm_available(self
.arch
):
431 seconds
*= self
.tcg_timeout_multiplier
432 starttime
= datetime
.datetime
.now()
433 endtime
= starttime
+ datetime
.timedelta(seconds
=seconds
)
435 while datetime
.datetime
.now() < endtime
:
436 if wait_root
and self
.ssh_root(cmd
) == 0:
439 elif self
.ssh(cmd
) == 0:
442 seconds
= (endtime
- datetime
.datetime
.now()).total_seconds()
443 logging
.debug("%ds before timeout", seconds
)
446 raise Exception("Timeout while waiting for guest ssh")
449 self
._guest
.shutdown(timeout
=self
._shutdown
_timeout
)
452 self
._guest
.wait(timeout
=self
._shutdown
_timeout
)
454 def graceful_shutdown(self
):
455 self
.ssh_root(self
.poweroff
)
456 self
._guest
.wait(timeout
=self
._shutdown
_timeout
)
458 def qmp(self
, *args
, **kwargs
):
459 return self
._guest
.qmp(*args
, **kwargs
)
461 def gen_cloud_init_iso(self
):
463 mdata
= open(os
.path
.join(cidir
, "meta-data"), "w")
464 name
= self
.name
.replace(".","-")
465 mdata
.writelines(["instance-id: {}-vm-0\n".format(name
),
466 "local-hostname: {}-guest\n".format(name
)])
468 udata
= open(os
.path
.join(cidir
, "user-data"), "w")
469 print("guest user:pw {}:{}".format(self
._config
['guest_user'],
470 self
._config
['guest_pass']))
471 udata
.writelines(["#cloud-config\n",
474 " root:%s\n" % self
._config
['root_pass'],
475 " %s:%s\n" % (self
._config
['guest_user'],
476 self
._config
['guest_pass']),
479 " - name: %s\n" % self
._config
['guest_user'],
480 " sudo: ALL=(ALL) NOPASSWD:ALL\n",
481 " ssh-authorized-keys:\n",
482 " - %s\n" % self
._config
['ssh_pub_key'],
484 " ssh-authorized-keys:\n",
485 " - %s\n" % self
._config
['ssh_pub_key'],
486 "locale: en_US.UTF-8\n"])
487 proxy
= os
.environ
.get("http_proxy")
488 if not proxy
is None:
489 udata
.writelines(["apt:\n",
490 " proxy: %s" % proxy
])
492 subprocess
.check_call([self
._genisoimage
, "-output", "cloud-init.iso",
493 "-volid", "cidata", "-joliet", "-rock",
494 "user-data", "meta-data"],
496 stdin
=self
._devnull
, stdout
=self
._stdout
,
498 return os
.path
.join(cidir
, "cloud-init.iso")
500 def get_qemu_path(arch
, build_path
=None):
501 """Fetch the path to the qemu binary."""
502 # If QEMU environment variable set, it takes precedence
503 if "QEMU" in os
.environ
:
504 qemu_path
= os
.environ
["QEMU"]
506 qemu_path
= os
.path
.join(build_path
, arch
+ "-softmmu")
507 qemu_path
= os
.path
.join(qemu_path
, "qemu-system-" + arch
)
509 # Default is to use system path for qemu.
510 qemu_path
= "qemu-system-" + arch
513 def get_qemu_version(qemu_path
):
514 """Get the version number from the current QEMU,
515 and return the major number."""
516 output
= subprocess
.check_output([qemu_path
, '--version'])
517 version_line
= output
.decode("utf-8")
518 version_num
= re
.split(' |\(', version_line
)[3].split('.')[0]
519 return int(version_num
)
521 def parse_config(config
, args
):
522 """ Parse yaml config and populate our config structure.
523 The yaml config allows the user to override the
524 defaults for VM parameters. In many cases these
525 defaults can be overridden without rebuilding the VM."""
527 config_file
= args
.config
528 elif 'QEMU_CONFIG' in os
.environ
:
529 config_file
= os
.environ
['QEMU_CONFIG']
532 if not os
.path
.exists(config_file
):
533 raise Exception("config file {} does not exist".format(config_file
))
534 # We gracefully handle importing the yaml module
535 # since it might not be installed.
536 # If we are here it means the user supplied a .yml file,
537 # so if the yaml module is not installed we will exit with error.
541 print("The python3-yaml package is needed "\
542 "to support config.yaml files")
543 # Instead of raising an exception we exit to avoid
544 # a raft of messy (expected) errors to stdout.
546 with
open(config_file
) as f
:
547 yaml_dict
= yaml
.safe_load(f
)
549 if 'qemu-conf' in yaml_dict
:
550 config
.update(yaml_dict
['qemu-conf'])
552 raise Exception("config file {} is not valid"\
553 " missing qemu-conf".format(config_file
))
556 def parse_args(vmcls
):
558 def get_default_jobs():
559 if multiprocessing
.cpu_count() > 1:
560 if kvm_available(vmcls
.arch
):
561 return multiprocessing
.cpu_count() // 2
562 elif os
.uname().machine
== "x86_64" and \
563 vmcls
.arch
in ["aarch64", "x86_64", "i386"]:
564 # MTTCG is available on these arches and we can allow
565 # more cores. but only up to a reasonable limit. User
566 # can always override these limits with --jobs.
567 return min(multiprocessing
.cpu_count() // 2, 8)
571 parser
= argparse
.ArgumentParser(
572 formatter_class
=argparse
.ArgumentDefaultsHelpFormatter
,
573 description
="Utility for provisioning VMs and running builds",
574 epilog
="""Remaining arguments are passed to the command.
575 Exit codes: 0 = success, 1 = command line error,
576 2 = environment initialization failed,
577 3 = test command failed""")
578 parser
.add_argument("--debug", "-D", action
="store_true",
579 help="enable debug output")
580 parser
.add_argument("--image", "-i", default
="%s.img" % vmcls
.name
,
581 help="image file name")
582 parser
.add_argument("--force", "-f", action
="store_true",
583 help="force build image even if image exists")
584 parser
.add_argument("--jobs", type=int, default
=get_default_jobs(),
585 help="number of virtual CPUs")
586 parser
.add_argument("--verbose", "-V", action
="store_true",
587 help="Pass V=1 to builds within the guest")
588 parser
.add_argument("--build-image", "-b", action
="store_true",
590 parser
.add_argument("--build-qemu",
591 help="build QEMU from source in guest")
592 parser
.add_argument("--build-target",
593 help="QEMU build target", default
="check")
594 parser
.add_argument("--build-path", default
=None,
595 help="Path of build directory, "\
596 "for using build tree QEMU binary. ")
597 parser
.add_argument("--interactive", "-I", action
="store_true",
598 help="Interactively run command")
599 parser
.add_argument("--snapshot", "-s", action
="store_true",
600 help="run tests with a snapshot")
601 parser
.add_argument("--genisoimage", default
="genisoimage",
602 help="iso imaging tool")
603 parser
.add_argument("--config", "-c", default
=None,
604 help="Provide config yaml for configuration. "\
605 "See config_example.yaml for example.")
606 parser
.add_argument("--efi-aarch64",
607 default
="/usr/share/qemu-efi-aarch64/QEMU_EFI.fd",
608 help="Path to efi image for aarch64 VMs.")
609 parser
.add_argument("--log-console", action
="store_true",
610 help="Log console to file.")
611 parser
.add_argument("commands", nargs
="*", help="""Remaining
612 commands after -- are passed to command inside the VM""")
614 return parser
.parse_args()
616 def main(vmcls
, config
=None):
619 config
= DEFAULT_CONFIG
620 args
= parse_args(vmcls
)
621 if not args
.commands
and not args
.build_qemu
and not args
.build_image
:
622 print("Nothing to do?")
624 config
= parse_config(config
, args
)
625 logging
.basicConfig(level
=(logging
.DEBUG
if args
.debug
627 vm
= vmcls(args
, config
=config
)
629 if os
.path
.exists(args
.image
) and not args
.force
:
630 sys
.stderr
.writelines(["Image file exists: %s\n" % args
.image
,
631 "Use --force option to overwrite\n"])
633 return vm
.build_image(args
.image
)
635 vm
.add_source_dir(args
.build_qemu
)
636 cmd
= [vm
.BUILD_SCRIPT
.format(
637 configure_opts
= " ".join(args
.commands
),
639 target
=args
.build_target
,
640 verbose
= "V=1" if args
.verbose
else "")]
645 img
+= ",snapshot=on"
648 except Exception as e
:
649 if isinstance(e
, SystemExit) and e
.code
== 0:
651 sys
.stderr
.write("Failed to prepare guest environment\n")
652 traceback
.print_exc()
656 if vm
.ssh(*cmd
) != 0:
661 if not args
.snapshot
:
662 vm
.graceful_shutdown()