include/fpu/softfloat: Fix compilation with Clang on s390x
[qemu/ar7.git] / tests / docker / docker.py
blob02d8a838472f4c0da73a8060cf76d690dc8c0455
1 #!/usr/bin/env python2
3 # Docker controlling module
5 # Copyright (c) 2016 Red Hat Inc.
7 # Authors:
8 # Fam Zheng <famz@redhat.com>
10 # This work is licensed under the terms of the GNU GPL, version 2
11 # or (at your option) any later version. See the COPYING file in
12 # the top-level directory.
14 from __future__ import print_function
15 import os
16 import sys
17 import subprocess
18 import json
19 import hashlib
20 import atexit
21 import uuid
22 import argparse
23 import tempfile
24 import re
25 import signal
26 from tarfile import TarFile, TarInfo
27 try:
28 from StringIO import StringIO
29 except ImportError:
30 from io import StringIO
31 from shutil import copy, rmtree
32 from pwd import getpwuid
33 from datetime import datetime,timedelta
36 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
39 DEVNULL = open(os.devnull, 'wb')
42 def _text_checksum(text):
43 """Calculate a digest string unique to the text content"""
44 return hashlib.sha1(text).hexdigest()
46 def _file_checksum(filename):
47 return _text_checksum(open(filename, 'rb').read())
49 def _guess_docker_command():
50 """ Guess a working docker command or raise exception if not found"""
51 commands = [["docker"], ["sudo", "-n", "docker"]]
52 for cmd in commands:
53 try:
54 # docker version will return the client details in stdout
55 # but still report a status of 1 if it can't contact the daemon
56 if subprocess.call(cmd + ["version"],
57 stdout=DEVNULL, stderr=DEVNULL) == 0:
58 return cmd
59 except OSError:
60 pass
61 commands_txt = "\n".join([" " + " ".join(x) for x in commands])
62 raise Exception("Cannot find working docker command. Tried:\n%s" % \
63 commands_txt)
65 def _copy_with_mkdir(src, root_dir, sub_path='.'):
66 """Copy src into root_dir, creating sub_path as needed."""
67 dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
68 try:
69 os.makedirs(dest_dir)
70 except OSError:
71 # we can safely ignore already created directories
72 pass
74 dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
75 copy(src, dest_file)
78 def _get_so_libs(executable):
79 """Return a list of libraries associated with an executable.
81 The paths may be symbolic links which would need to be resolved to
82 ensure theright data is copied."""
84 libs = []
85 ldd_re = re.compile(r"(/.*/)(\S*)")
86 try:
87 ldd_output = subprocess.check_output(["ldd", executable])
88 for line in ldd_output.split("\n"):
89 search = ldd_re.search(line)
90 if search and len(search.groups()) == 2:
91 so_path = search.groups()[0]
92 so_lib = search.groups()[1]
93 libs.append("%s/%s" % (so_path, so_lib))
94 except subprocess.CalledProcessError:
95 print("%s had no associated libraries (static build?)" % (executable))
97 return libs
99 def _copy_binary_with_libs(src, dest_dir):
100 """Copy a binary executable and all its dependent libraries.
102 This does rely on the host file-system being fairly multi-arch
103 aware so the file don't clash with the guests layout."""
105 _copy_with_mkdir(src, dest_dir, "/usr/bin")
107 libs = _get_so_libs(src)
108 if libs:
109 for l in libs:
110 so_path = os.path.dirname(l)
111 _copy_with_mkdir(l , dest_dir, so_path)
114 def _check_binfmt_misc(executable):
115 """Check binfmt_misc has entry for executable in the right place.
117 The details of setting up binfmt_misc are outside the scope of
118 this script but we should at least fail early with a useful
119 message if it won't work."""
121 binary = os.path.basename(executable)
122 binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
124 if not os.path.exists(binfmt_entry):
125 print ("No binfmt_misc entry for %s" % (binary))
126 return False
128 with open(binfmt_entry) as x: entry = x.read()
130 qpath = "/usr/bin/%s" % (binary)
131 if not re.search("interpreter %s\n" % (qpath), entry):
132 print ("binfmt_misc for %s does not point to %s" % (binary, qpath))
133 return False
135 return True
138 def _read_qemu_dockerfile(img_name):
139 # special case for Debian linux-user images
140 if img_name.startswith("debian") and img_name.endswith("user"):
141 img_name = "debian-bootstrap"
143 df = os.path.join(os.path.dirname(__file__), "dockerfiles",
144 img_name + ".docker")
145 return open(df, "r").read()
147 def _dockerfile_preprocess(df):
148 out = ""
149 for l in df.splitlines():
150 if len(l.strip()) == 0 or l.startswith("#"):
151 continue
152 from_pref = "FROM qemu:"
153 if l.startswith(from_pref):
154 # TODO: Alternatively we could replace this line with "FROM $ID"
155 # where $ID is the image's hex id obtained with
156 # $ docker images $IMAGE --format="{{.Id}}"
157 # but unfortunately that's not supported by RHEL 7.
158 inlining = _read_qemu_dockerfile(l[len(from_pref):])
159 out += _dockerfile_preprocess(inlining)
160 continue
161 out += l + "\n"
162 return out
164 class Docker(object):
165 """ Running Docker commands """
166 def __init__(self):
167 self._command = _guess_docker_command()
168 self._instances = []
169 atexit.register(self._kill_instances)
170 signal.signal(signal.SIGTERM, self._kill_instances)
171 signal.signal(signal.SIGHUP, self._kill_instances)
173 def _do(self, cmd, quiet=True, **kwargs):
174 if quiet:
175 kwargs["stdout"] = DEVNULL
176 return subprocess.call(self._command + cmd, **kwargs)
178 def _do_check(self, cmd, quiet=True, **kwargs):
179 if quiet:
180 kwargs["stdout"] = DEVNULL
181 return subprocess.check_call(self._command + cmd, **kwargs)
183 def _do_kill_instances(self, only_known, only_active=True):
184 cmd = ["ps", "-q"]
185 if not only_active:
186 cmd.append("-a")
187 for i in self._output(cmd).split():
188 resp = self._output(["inspect", i])
189 labels = json.loads(resp)[0]["Config"]["Labels"]
190 active = json.loads(resp)[0]["State"]["Running"]
191 if not labels:
192 continue
193 instance_uuid = labels.get("com.qemu.instance.uuid", None)
194 if not instance_uuid:
195 continue
196 if only_known and instance_uuid not in self._instances:
197 continue
198 print("Terminating", i)
199 if active:
200 self._do(["kill", i])
201 self._do(["rm", i])
203 def clean(self):
204 self._do_kill_instances(False, False)
205 return 0
207 def _kill_instances(self, *args, **kwargs):
208 return self._do_kill_instances(True)
210 def _output(self, cmd, **kwargs):
211 return subprocess.check_output(self._command + cmd,
212 stderr=subprocess.STDOUT,
213 **kwargs)
215 def inspect_tag(self, tag):
216 try:
217 return self._output(["inspect", tag])
218 except subprocess.CalledProcessError:
219 return None
221 def get_image_creation_time(self, info):
222 return json.loads(info)[0]["Created"]
224 def get_image_dockerfile_checksum(self, tag):
225 resp = self.inspect_tag(tag)
226 labels = json.loads(resp)[0]["Config"].get("Labels", {})
227 return labels.get("com.qemu.dockerfile-checksum", "")
229 def build_image(self, tag, docker_dir, dockerfile,
230 quiet=True, user=False, argv=None, extra_files_cksum=[]):
231 if argv == None:
232 argv = []
234 tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
235 tmp_df.write(dockerfile)
237 if user:
238 uid = os.getuid()
239 uname = getpwuid(uid).pw_name
240 tmp_df.write("\n")
241 tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
242 (uname, uid, uname))
244 tmp_df.write("\n")
245 tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
246 _text_checksum(_dockerfile_preprocess(dockerfile)))
247 for f, c in extra_files_cksum:
248 tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
250 tmp_df.flush()
252 self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
253 [docker_dir],
254 quiet=quiet)
256 def update_image(self, tag, tarball, quiet=True):
257 "Update a tagged image using "
259 self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
261 def image_matches_dockerfile(self, tag, dockerfile):
262 try:
263 checksum = self.get_image_dockerfile_checksum(tag)
264 except Exception:
265 return False
266 return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
268 def run(self, cmd, keep, quiet):
269 label = uuid.uuid1().hex
270 if not keep:
271 self._instances.append(label)
272 ret = self._do_check(["run", "--label",
273 "com.qemu.instance.uuid=" + label] + cmd,
274 quiet=quiet)
275 if not keep:
276 self._instances.remove(label)
277 return ret
279 def command(self, cmd, argv, quiet):
280 return self._do([cmd] + argv, quiet=quiet)
282 class SubCommand(object):
283 """A SubCommand template base class"""
284 name = None # Subcommand name
285 def shared_args(self, parser):
286 parser.add_argument("--quiet", action="store_true",
287 help="Run quietly unless an error occurred")
289 def args(self, parser):
290 """Setup argument parser"""
291 pass
292 def run(self, args, argv):
293 """Run command.
294 args: parsed argument by argument parser.
295 argv: remaining arguments from sys.argv.
297 pass
299 class RunCommand(SubCommand):
300 """Invoke docker run and take care of cleaning up"""
301 name = "run"
302 def args(self, parser):
303 parser.add_argument("--keep", action="store_true",
304 help="Don't remove image when command completes")
305 def run(self, args, argv):
306 return Docker().run(argv, args.keep, quiet=args.quiet)
308 class BuildCommand(SubCommand):
309 """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
310 name = "build"
311 def args(self, parser):
312 parser.add_argument("--include-executable", "-e",
313 help="""Specify a binary that will be copied to the
314 container together with all its dependent
315 libraries""")
316 parser.add_argument("--extra-files", "-f", nargs='*',
317 help="""Specify files that will be copied in the
318 Docker image, fulfilling the ADD directive from the
319 Dockerfile""")
320 parser.add_argument("--add-current-user", "-u", dest="user",
321 action="store_true",
322 help="Add the current user to image's passwd")
323 parser.add_argument("tag",
324 help="Image Tag")
325 parser.add_argument("dockerfile",
326 help="Dockerfile name")
328 def run(self, args, argv):
329 dockerfile = open(args.dockerfile, "rb").read()
330 tag = args.tag
332 dkr = Docker()
333 if "--no-cache" not in argv and \
334 dkr.image_matches_dockerfile(tag, dockerfile):
335 if not args.quiet:
336 print("Image is up to date.")
337 else:
338 # Create a docker context directory for the build
339 docker_dir = tempfile.mkdtemp(prefix="docker_build")
341 # Validate binfmt_misc will work
342 if args.include_executable:
343 if not _check_binfmt_misc(args.include_executable):
344 return 1
346 # Is there a .pre file to run in the build context?
347 docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
348 if os.path.exists(docker_pre):
349 stdout = DEVNULL if args.quiet else None
350 rc = subprocess.call(os.path.realpath(docker_pre),
351 cwd=docker_dir, stdout=stdout)
352 if rc == 3:
353 print("Skip")
354 return 0
355 elif rc != 0:
356 print("%s exited with code %d" % (docker_pre, rc))
357 return 1
359 # Copy any extra files into the Docker context. These can be
360 # included by the use of the ADD directive in the Dockerfile.
361 cksum = []
362 if args.include_executable:
363 # FIXME: there is no checksum of this executable and the linked
364 # libraries, once the image built any change of this executable
365 # or any library won't trigger another build.
366 _copy_binary_with_libs(args.include_executable, docker_dir)
367 for filename in args.extra_files or []:
368 _copy_with_mkdir(filename, docker_dir)
369 cksum += [(filename, _file_checksum(filename))]
371 argv += ["--build-arg=" + k.lower() + "=" + v
372 for k, v in os.environ.iteritems()
373 if k.lower() in FILTERED_ENV_NAMES]
374 dkr.build_image(tag, docker_dir, dockerfile,
375 quiet=args.quiet, user=args.user, argv=argv,
376 extra_files_cksum=cksum)
378 rmtree(docker_dir)
380 return 0
382 class UpdateCommand(SubCommand):
383 """ Update a docker image with new executables. Arguments: <tag> <executable>"""
384 name = "update"
385 def args(self, parser):
386 parser.add_argument("tag",
387 help="Image Tag")
388 parser.add_argument("executable",
389 help="Executable to copy")
391 def run(self, args, argv):
392 # Create a temporary tarball with our whole build context and
393 # dockerfile for the update
394 tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
395 tmp_tar = TarFile(fileobj=tmp, mode='w')
397 # Add the executable to the tarball
398 bn = os.path.basename(args.executable)
399 ff = "/usr/bin/%s" % bn
400 tmp_tar.add(args.executable, arcname=ff)
402 # Add any associated libraries
403 libs = _get_so_libs(args.executable)
404 if libs:
405 for l in libs:
406 tmp_tar.add(os.path.realpath(l), arcname=l)
408 # Create a Docker buildfile
409 df = StringIO()
410 df.write("FROM %s\n" % args.tag)
411 df.write("ADD . /\n")
412 df.seek(0)
414 df_tar = TarInfo(name="Dockerfile")
415 df_tar.size = len(df.buf)
416 tmp_tar.addfile(df_tar, fileobj=df)
418 tmp_tar.close()
420 # reset the file pointers
421 tmp.flush()
422 tmp.seek(0)
424 # Run the build with our tarball context
425 dkr = Docker()
426 dkr.update_image(args.tag, tmp, quiet=args.quiet)
428 return 0
430 class CleanCommand(SubCommand):
431 """Clean up docker instances"""
432 name = "clean"
433 def run(self, args, argv):
434 Docker().clean()
435 return 0
437 class ImagesCommand(SubCommand):
438 """Run "docker images" command"""
439 name = "images"
440 def run(self, args, argv):
441 return Docker().command("images", argv, args.quiet)
444 class ProbeCommand(SubCommand):
445 """Probe if we can run docker automatically"""
446 name = "probe"
448 def run(self, args, argv):
449 try:
450 docker = Docker()
451 if docker._command[0] == "docker":
452 print("yes")
453 elif docker._command[0] == "sudo":
454 print("sudo")
455 except Exception:
456 print("no")
458 return
461 class CcCommand(SubCommand):
462 """Compile sources with cc in images"""
463 name = "cc"
465 def args(self, parser):
466 parser.add_argument("--image", "-i", required=True,
467 help="The docker image in which to run cc")
468 parser.add_argument("--cc", default="cc",
469 help="The compiler executable to call")
470 parser.add_argument("--user",
471 help="The user-id to run under")
472 parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
473 help="""Extra paths to (ro) mount into container for
474 reading sources""")
476 def run(self, args, argv):
477 if argv and argv[0] == "--":
478 argv = argv[1:]
479 cwd = os.getcwd()
480 cmd = ["--rm", "-w", cwd,
481 "-v", "%s:%s:rw" % (cwd, cwd)]
482 if args.paths:
483 for p in args.paths:
484 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
485 if args.user:
486 cmd += ["-u", args.user]
487 cmd += [args.image, args.cc]
488 cmd += argv
489 return Docker().command("run", cmd, args.quiet)
492 class CheckCommand(SubCommand):
493 """Check if we need to re-build a docker image out of a dockerfile.
494 Arguments: <tag> <dockerfile>"""
495 name = "check"
497 def args(self, parser):
498 parser.add_argument("tag",
499 help="Image Tag")
500 parser.add_argument("dockerfile", default=None,
501 help="Dockerfile name", nargs='?')
502 parser.add_argument("--checktype", choices=["checksum", "age"],
503 default="checksum", help="check type")
504 parser.add_argument("--olderthan", default=60, type=int,
505 help="number of minutes")
507 def run(self, args, argv):
508 tag = args.tag
510 try:
511 dkr = Docker()
512 except:
513 print("Docker not set up")
514 return 1
516 info = dkr.inspect_tag(tag)
517 if info is None:
518 print("Image does not exist")
519 return 1
521 if args.checktype == "checksum":
522 if not args.dockerfile:
523 print("Need a dockerfile for tag:%s" % (tag))
524 return 1
526 dockerfile = open(args.dockerfile, "rb").read()
528 if dkr.image_matches_dockerfile(tag, dockerfile):
529 if not args.quiet:
530 print("Image is up to date")
531 return 0
532 else:
533 print("Image needs updating")
534 return 1
535 elif args.checktype == "age":
536 timestr = dkr.get_image_creation_time(info).split(".")[0]
537 created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
538 past = datetime.now() - timedelta(minutes=args.olderthan)
539 if created < past:
540 print ("Image created @ %s more than %d minutes old" %
541 (timestr, args.olderthan))
542 return 1
543 else:
544 if not args.quiet:
545 print ("Image less than %d minutes old" % (args.olderthan))
546 return 0
549 def main():
550 parser = argparse.ArgumentParser(description="A Docker helper",
551 usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
552 subparsers = parser.add_subparsers(title="subcommands", help=None)
553 for cls in SubCommand.__subclasses__():
554 cmd = cls()
555 subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
556 cmd.shared_args(subp)
557 cmd.args(subp)
558 subp.set_defaults(cmdobj=cmd)
559 args, argv = parser.parse_known_args()
560 return args.cmdobj.run(args, argv)
562 if __name__ == "__main__":
563 sys.exit(main())