tests/docker: fix copying of executable in "update"
[qemu.git] / tests / docker / docker.py
blob0435a55d1009f198be4a1cc0a639eaea21fe5bd3
1 #!/usr/bin/env python3
3 # Docker controlling module
5 # Copyright (c) 2016 Red Hat Inc.
7 # Authors:
8 # Fam Zheng <famz@redhat.com>
10 # This work is licensed under the terms of the GNU GPL, version 2
11 # or (at your option) any later version. See the COPYING file in
12 # the top-level directory.
14 import os
15 import sys
16 import subprocess
17 import json
18 import hashlib
19 import atexit
20 import uuid
21 import argparse
22 import enum
23 import tempfile
24 import re
25 import signal
26 from tarfile import TarFile, TarInfo
27 from io import StringIO, BytesIO
28 from shutil import copy, rmtree
29 from pwd import getpwuid
30 from datetime import datetime, timedelta
33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
36 DEVNULL = open(os.devnull, 'wb')
38 class EngineEnum(enum.IntEnum):
39 AUTO = 1
40 DOCKER = 2
41 PODMAN = 3
43 def __str__(self):
44 return self.name.lower()
46 def __repr__(self):
47 return str(self)
49 @staticmethod
50 def argparse(s):
51 try:
52 return EngineEnum[s.upper()]
53 except KeyError:
54 return s
57 USE_ENGINE = EngineEnum.AUTO
59 def _bytes_checksum(bytes):
60 """Calculate a digest string unique to the text content"""
61 return hashlib.sha1(bytes).hexdigest()
63 def _text_checksum(text):
64 """Calculate a digest string unique to the text content"""
65 return _bytes_checksum(text.encode('utf-8'))
67 def _read_dockerfile(path):
68 return open(path, 'rt', encoding='utf-8').read()
70 def _file_checksum(filename):
71 return _bytes_checksum(open(filename, 'rb').read())
74 def _guess_engine_command():
75 """ Guess a working engine command or raise exception if not found"""
76 commands = []
78 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
79 commands += [["podman"]]
80 if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
81 commands += [["docker"], ["sudo", "-n", "docker"]]
82 for cmd in commands:
83 try:
84 # docker version will return the client details in stdout
85 # but still report a status of 1 if it can't contact the daemon
86 if subprocess.call(cmd + ["version"],
87 stdout=DEVNULL, stderr=DEVNULL) == 0:
88 return cmd
89 except OSError:
90 pass
91 commands_txt = "\n".join([" " + " ".join(x) for x in commands])
92 raise Exception("Cannot find working engine command. Tried:\n%s" %
93 commands_txt)
96 def _copy_with_mkdir(src, root_dir, sub_path='.', name=None):
97 """Copy src into root_dir, creating sub_path as needed."""
98 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
99 try:
100 os.makedirs(dest_dir)
101 except OSError:
102 # we can safely ignore already created directories
103 pass
105 dest_file = "%s/%s" % (dest_dir, name if name else os.path.basename(src))
107 try:
108 copy(src, dest_file)
109 except FileNotFoundError:
110 print("Couldn't copy %s to %s" % (src, dest_file))
111 pass
114 def _get_so_libs(executable):
115 """Return a list of libraries associated with an executable.
117 The paths may be symbolic links which would need to be resolved to
118 ensure the right data is copied."""
120 libs = []
121 ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
122 try:
123 ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
124 for line in ldd_output.split("\n"):
125 search = ldd_re.search(line)
126 if search:
127 try:
128 libs.append(search.group(1))
129 except IndexError:
130 pass
131 except subprocess.CalledProcessError:
132 print("%s had no associated libraries (static build?)" % (executable))
134 return libs
137 def _copy_binary_with_libs(src, bin_dest, dest_dir):
138 """Maybe copy a binary and all its dependent libraries.
140 If bin_dest isn't set we only copy the support libraries because
141 we don't need qemu in the docker path to run (due to persistent
142 mapping). Indeed users may get confused if we aren't running what
143 is in the image.
145 This does rely on the host file-system being fairly multi-arch
146 aware so the file don't clash with the guests layout.
149 if bin_dest:
150 _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
151 else:
152 print("only copying support libraries for %s" % (src))
154 libs = _get_so_libs(src)
155 if libs:
156 for l in libs:
157 so_path = os.path.dirname(l)
158 name = os.path.basename(l)
159 real_l = os.path.realpath(l)
160 _copy_with_mkdir(real_l, dest_dir, so_path, name)
163 def _check_binfmt_misc(executable):
164 """Check binfmt_misc has entry for executable in the right place.
166 The details of setting up binfmt_misc are outside the scope of
167 this script but we should at least fail early with a useful
168 message if it won't work.
170 Returns the configured binfmt path and a valid flag. For
171 persistent configurations we will still want to copy and dependent
172 libraries.
175 binary = os.path.basename(executable)
176 binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
178 if not os.path.exists(binfmt_entry):
179 print ("No binfmt_misc entry for %s" % (binary))
180 return None, False
182 with open(binfmt_entry) as x: entry = x.read()
184 if re.search("flags:.*F.*\n", entry):
185 print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
186 (binary))
187 return None, True
189 m = re.search("interpreter (\S+)\n", entry)
190 interp = m.group(1)
191 if interp and interp != executable:
192 print("binfmt_misc for %s does not point to %s, using %s" %
193 (binary, executable, interp))
195 return interp, True
198 def _read_qemu_dockerfile(img_name):
199 # special case for Debian linux-user images
200 if img_name.startswith("debian") and img_name.endswith("user"):
201 img_name = "debian-bootstrap"
203 df = os.path.join(os.path.dirname(__file__), "dockerfiles",
204 img_name + ".docker")
205 return _read_dockerfile(df)
208 def _dockerfile_preprocess(df):
209 out = ""
210 for l in df.splitlines():
211 if len(l.strip()) == 0 or l.startswith("#"):
212 continue
213 from_pref = "FROM qemu/"
214 if l.startswith(from_pref):
215 # TODO: Alternatively we could replace this line with "FROM $ID"
216 # where $ID is the image's hex id obtained with
217 # $ docker images $IMAGE --format="{{.Id}}"
218 # but unfortunately that's not supported by RHEL 7.
219 inlining = _read_qemu_dockerfile(l[len(from_pref):])
220 out += _dockerfile_preprocess(inlining)
221 continue
222 out += l + "\n"
223 return out
226 class Docker(object):
227 """ Running Docker commands """
228 def __init__(self):
229 self._command = _guess_engine_command()
231 if "docker" in self._command and "TRAVIS" not in os.environ:
232 os.environ["DOCKER_BUILDKIT"] = "1"
233 self._buildkit = True
234 else:
235 self._buildkit = False
237 self._instance = None
238 atexit.register(self._kill_instances)
239 signal.signal(signal.SIGTERM, self._kill_instances)
240 signal.signal(signal.SIGHUP, self._kill_instances)
242 def _do(self, cmd, quiet=True, **kwargs):
243 if quiet:
244 kwargs["stdout"] = DEVNULL
245 return subprocess.call(self._command + cmd, **kwargs)
247 def _do_check(self, cmd, quiet=True, **kwargs):
248 if quiet:
249 kwargs["stdout"] = DEVNULL
250 return subprocess.check_call(self._command + cmd, **kwargs)
252 def _do_kill_instances(self, only_known, only_active=True):
253 cmd = ["ps", "-q"]
254 if not only_active:
255 cmd.append("-a")
257 filter = "--filter=label=com.qemu.instance.uuid"
258 if only_known:
259 if self._instance:
260 filter += "=%s" % (self._instance)
261 else:
262 # no point trying to kill, we finished
263 return
265 print("filter=%s" % (filter))
266 cmd.append(filter)
267 for i in self._output(cmd).split():
268 self._do(["rm", "-f", i])
270 def clean(self):
271 self._do_kill_instances(False, False)
272 return 0
274 def _kill_instances(self, *args, **kwargs):
275 return self._do_kill_instances(True)
277 def _output(self, cmd, **kwargs):
278 try:
279 return subprocess.check_output(self._command + cmd,
280 stderr=subprocess.STDOUT,
281 encoding='utf-8',
282 **kwargs)
283 except TypeError:
284 # 'encoding' argument was added in 3.6+
285 return subprocess.check_output(self._command + cmd,
286 stderr=subprocess.STDOUT,
287 **kwargs).decode('utf-8')
290 def inspect_tag(self, tag):
291 try:
292 return self._output(["inspect", tag])
293 except subprocess.CalledProcessError:
294 return None
296 def get_image_creation_time(self, info):
297 return json.loads(info)[0]["Created"]
299 def get_image_dockerfile_checksum(self, tag):
300 resp = self.inspect_tag(tag)
301 labels = json.loads(resp)[0]["Config"].get("Labels", {})
302 return labels.get("com.qemu.dockerfile-checksum", "")
304 def build_image(self, tag, docker_dir, dockerfile,
305 quiet=True, user=False, argv=None, registry=None,
306 extra_files_cksum=[]):
307 if argv is None:
308 argv = []
310 # pre-calculate the docker checksum before any
311 # substitutions we make for caching
312 checksum = _text_checksum(_dockerfile_preprocess(dockerfile))
314 if registry is not None:
315 sources = re.findall("FROM qemu\/(.*)", dockerfile)
316 # Fetch any cache layers we can, may fail
317 for s in sources:
318 pull_args = ["pull", "%s/qemu/%s" % (registry, s)]
319 if self._do(pull_args, quiet=quiet) != 0:
320 registry = None
321 break
322 # Make substitutions
323 if registry is not None:
324 dockerfile = dockerfile.replace("FROM qemu/",
325 "FROM %s/qemu/" %
326 (registry))
328 tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
329 encoding='utf-8',
330 dir=docker_dir, suffix=".docker")
331 tmp_df.write(dockerfile)
333 if user:
334 uid = os.getuid()
335 uname = getpwuid(uid).pw_name
336 tmp_df.write("\n")
337 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
338 (uname, uid, uname))
340 tmp_df.write("\n")
341 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s\n" % (checksum))
342 for f, c in extra_files_cksum:
343 tmp_df.write("LABEL com.qemu.%s-checksum=%s\n" % (f, c))
345 tmp_df.flush()
347 build_args = ["build", "-t", tag, "-f", tmp_df.name]
348 if self._buildkit:
349 build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
351 if registry is not None:
352 pull_args = ["pull", "%s/%s" % (registry, tag)]
353 self._do(pull_args, quiet=quiet)
354 cache = "%s/%s" % (registry, tag)
355 build_args += ["--cache-from", cache]
356 build_args += argv
357 build_args += [docker_dir]
359 self._do_check(build_args,
360 quiet=quiet)
362 def update_image(self, tag, tarball, quiet=True):
363 "Update a tagged image using "
365 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
367 def image_matches_dockerfile(self, tag, dockerfile):
368 try:
369 checksum = self.get_image_dockerfile_checksum(tag)
370 except Exception:
371 return False
372 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
374 def run(self, cmd, keep, quiet, as_user=False):
375 label = uuid.uuid4().hex
376 if not keep:
377 self._instance = label
379 if as_user:
380 uid = os.getuid()
381 cmd = [ "-u", str(uid) ] + cmd
382 # podman requires a bit more fiddling
383 if self._command[0] == "podman":
384 cmd.insert(0, '--userns=keep-id')
386 ret = self._do_check(["run", "--rm", "--label",
387 "com.qemu.instance.uuid=" + label] + cmd,
388 quiet=quiet)
389 if not keep:
390 self._instance = None
391 return ret
393 def command(self, cmd, argv, quiet):
394 return self._do([cmd] + argv, quiet=quiet)
397 class SubCommand(object):
398 """A SubCommand template base class"""
399 name = None # Subcommand name
401 def shared_args(self, parser):
402 parser.add_argument("--quiet", action="store_true",
403 help="Run quietly unless an error occurred")
405 def args(self, parser):
406 """Setup argument parser"""
407 pass
409 def run(self, args, argv):
410 """Run command.
411 args: parsed argument by argument parser.
412 argv: remaining arguments from sys.argv.
414 pass
417 class RunCommand(SubCommand):
418 """Invoke docker run and take care of cleaning up"""
419 name = "run"
421 def args(self, parser):
422 parser.add_argument("--keep", action="store_true",
423 help="Don't remove image when command completes")
424 parser.add_argument("--run-as-current-user", action="store_true",
425 help="Run container using the current user's uid")
427 def run(self, args, argv):
428 return Docker().run(argv, args.keep, quiet=args.quiet,
429 as_user=args.run_as_current_user)
432 class BuildCommand(SubCommand):
433 """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
434 name = "build"
436 def args(self, parser):
437 parser.add_argument("--include-executable", "-e",
438 help="""Specify a binary that will be copied to the
439 container together with all its dependent
440 libraries""")
441 parser.add_argument("--skip-binfmt",
442 action="store_true",
443 help="""Skip binfmt entry check (used for testing)""")
444 parser.add_argument("--extra-files", nargs='*',
445 help="""Specify files that will be copied in the
446 Docker image, fulfilling the ADD directive from the
447 Dockerfile""")
448 parser.add_argument("--add-current-user", "-u", dest="user",
449 action="store_true",
450 help="Add the current user to image's passwd")
451 parser.add_argument("--registry", "-r",
452 help="cache from docker registry")
453 parser.add_argument("-t", dest="tag",
454 help="Image Tag")
455 parser.add_argument("-f", dest="dockerfile",
456 help="Dockerfile name")
458 def run(self, args, argv):
459 dockerfile = _read_dockerfile(args.dockerfile)
460 tag = args.tag
462 dkr = Docker()
463 if "--no-cache" not in argv and \
464 dkr.image_matches_dockerfile(tag, dockerfile):
465 if not args.quiet:
466 print("Image is up to date.")
467 else:
468 # Create a docker context directory for the build
469 docker_dir = tempfile.mkdtemp(prefix="docker_build")
471 # Validate binfmt_misc will work
472 if args.skip_binfmt:
473 qpath = args.include_executable
474 elif args.include_executable:
475 qpath, enabled = _check_binfmt_misc(args.include_executable)
476 if not enabled:
477 return 1
479 # Is there a .pre file to run in the build context?
480 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
481 if os.path.exists(docker_pre):
482 stdout = DEVNULL if args.quiet else None
483 rc = subprocess.call(os.path.realpath(docker_pre),
484 cwd=docker_dir, stdout=stdout)
485 if rc == 3:
486 print("Skip")
487 return 0
488 elif rc != 0:
489 print("%s exited with code %d" % (docker_pre, rc))
490 return 1
492 # Copy any extra files into the Docker context. These can be
493 # included by the use of the ADD directive in the Dockerfile.
494 cksum = []
495 if args.include_executable:
496 # FIXME: there is no checksum of this executable and the linked
497 # libraries, once the image built any change of this executable
498 # or any library won't trigger another build.
499 _copy_binary_with_libs(args.include_executable,
500 qpath, docker_dir)
502 for filename in args.extra_files or []:
503 _copy_with_mkdir(filename, docker_dir)
504 cksum += [(filename, _file_checksum(filename))]
506 argv += ["--build-arg=" + k.lower() + "=" + v
507 for k, v in os.environ.items()
508 if k.lower() in FILTERED_ENV_NAMES]
509 dkr.build_image(tag, docker_dir, dockerfile,
510 quiet=args.quiet, user=args.user,
511 argv=argv, registry=args.registry,
512 extra_files_cksum=cksum)
514 rmtree(docker_dir)
516 return 0
519 class UpdateCommand(SubCommand):
520 """ Update a docker image with new executables. Args: <tag> <executable>"""
521 name = "update"
523 def args(self, parser):
524 parser.add_argument("tag",
525 help="Image Tag")
526 parser.add_argument("executable",
527 help="Executable to copy")
529 def run(self, args, argv):
530 # Create a temporary tarball with our whole build context and
531 # dockerfile for the update
532 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
533 tmp_tar = TarFile(fileobj=tmp, mode='w')
535 # Add the executable to the tarball, using the current
536 # configured binfmt_misc path. If we don't get a path then we
537 # only need the support libraries copied
538 ff, enabled = _check_binfmt_misc(args.executable)
540 if not enabled:
541 print("binfmt_misc not enabled, update disabled")
542 return 1
544 if ff:
545 tmp_tar.add(args.executable, arcname=ff)
547 # Add any associated libraries
548 libs = _get_so_libs(args.executable)
549 if libs:
550 for l in libs:
551 so_path = os.path.dirname(l)
552 name = os.path.basename(l)
553 real_l = os.path.realpath(l)
554 try:
555 tmp_tar.add(real_l, arcname="%s/%s" % (so_path, name))
556 except FileNotFoundError:
557 print("Couldn't add %s/%s to archive" % (so_path, name))
558 pass
560 # Create a Docker buildfile
561 df = StringIO()
562 df.write(u"FROM %s\n" % args.tag)
563 df.write(u"ADD . /\n")
565 df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
567 df_tar = TarInfo(name="Dockerfile")
568 df_tar.size = df_bytes.getbuffer().nbytes
569 tmp_tar.addfile(df_tar, fileobj=df_bytes)
571 tmp_tar.close()
573 # reset the file pointers
574 tmp.flush()
575 tmp.seek(0)
577 # Run the build with our tarball context
578 dkr = Docker()
579 dkr.update_image(args.tag, tmp, quiet=args.quiet)
581 return 0
584 class CleanCommand(SubCommand):
585 """Clean up docker instances"""
586 name = "clean"
588 def run(self, args, argv):
589 Docker().clean()
590 return 0
593 class ImagesCommand(SubCommand):
594 """Run "docker images" command"""
595 name = "images"
597 def run(self, args, argv):
598 return Docker().command("images", argv, args.quiet)
601 class ProbeCommand(SubCommand):
602 """Probe if we can run docker automatically"""
603 name = "probe"
605 def run(self, args, argv):
606 try:
607 docker = Docker()
608 if docker._command[0] == "docker":
609 print("docker")
610 elif docker._command[0] == "sudo":
611 print("sudo docker")
612 elif docker._command[0] == "podman":
613 print("podman")
614 except Exception:
615 print("no")
617 return
620 class CcCommand(SubCommand):
621 """Compile sources with cc in images"""
622 name = "cc"
624 def args(self, parser):
625 parser.add_argument("--image", "-i", required=True,
626 help="The docker image in which to run cc")
627 parser.add_argument("--cc", default="cc",
628 help="The compiler executable to call")
629 parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
630 help="""Extra paths to (ro) mount into container for
631 reading sources""")
633 def run(self, args, argv):
634 if argv and argv[0] == "--":
635 argv = argv[1:]
636 cwd = os.getcwd()
637 cmd = ["-w", cwd,
638 "-v", "%s:%s:rw" % (cwd, cwd)]
639 if args.paths:
640 for p in args.paths:
641 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
642 cmd += [args.image, args.cc]
643 cmd += argv
644 return Docker().run(cmd, False, quiet=args.quiet,
645 as_user=True)
648 class CheckCommand(SubCommand):
649 """Check if we need to re-build a docker image out of a dockerfile.
650 Arguments: <tag> <dockerfile>"""
651 name = "check"
653 def args(self, parser):
654 parser.add_argument("tag",
655 help="Image Tag")
656 parser.add_argument("dockerfile", default=None,
657 help="Dockerfile name", nargs='?')
658 parser.add_argument("--checktype", choices=["checksum", "age"],
659 default="checksum", help="check type")
660 parser.add_argument("--olderthan", default=60, type=int,
661 help="number of minutes")
663 def run(self, args, argv):
664 tag = args.tag
666 try:
667 dkr = Docker()
668 except subprocess.CalledProcessError:
669 print("Docker not set up")
670 return 1
672 info = dkr.inspect_tag(tag)
673 if info is None:
674 print("Image does not exist")
675 return 1
677 if args.checktype == "checksum":
678 if not args.dockerfile:
679 print("Need a dockerfile for tag:%s" % (tag))
680 return 1
682 dockerfile = _read_dockerfile(args.dockerfile)
684 if dkr.image_matches_dockerfile(tag, dockerfile):
685 if not args.quiet:
686 print("Image is up to date")
687 return 0
688 else:
689 print("Image needs updating")
690 return 1
691 elif args.checktype == "age":
692 timestr = dkr.get_image_creation_time(info).split(".")[0]
693 created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
694 past = datetime.now() - timedelta(minutes=args.olderthan)
695 if created < past:
696 print ("Image created @ %s more than %d minutes old" %
697 (timestr, args.olderthan))
698 return 1
699 else:
700 if not args.quiet:
701 print ("Image less than %d minutes old" % (args.olderthan))
702 return 0
705 def main():
706 global USE_ENGINE
708 parser = argparse.ArgumentParser(description="A Docker helper",
709 usage="%s <subcommand> ..." %
710 os.path.basename(sys.argv[0]))
711 parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
712 help="specify which container engine to use")
713 subparsers = parser.add_subparsers(title="subcommands", help=None)
714 for cls in SubCommand.__subclasses__():
715 cmd = cls()
716 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
717 cmd.shared_args(subp)
718 cmd.args(subp)
719 subp.set_defaults(cmdobj=cmd)
720 args, argv = parser.parse_known_args()
721 if args.engine:
722 USE_ENGINE = args.engine
723 return args.cmdobj.run(args, argv)
726 if __name__ == "__main__":
727 sys.exit(main())