tests/docker/docker.py

   1 #!/usr/bin/env python2
   2 #
   3 # Docker controlling module
   4 #
   5 # Copyright (c) 2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Fam Zheng <famz@redhat.com>
   9 #
  10 # This work is licensed under the terms of the GNU GPL, version 2
  11 # or (at your option) any later version. See the COPYING file in
  12 # the top-level directory.
  13
  14 import os
  15 import sys
  16 sys.path.append(os.path.join(os.path.dirname(__file__),
  17                              '..', '..', 'scripts'))
  18 import argparse
  19 import subprocess
  20 import json
  21 import hashlib
  22 import atexit
  23 import uuid
  24 import tempfile
  25 import re
  26 import signal
  27 from tarfile import TarFile, TarInfo
  28 from StringIO import StringIO
  29 from shutil import copy, rmtree
  30 from pwd import getpwuid
  31
  32
  33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
  34
  35
  36 DEVNULL = open(os.devnull, 'wb')
  37
  38
  39 def _text_checksum(text):
  40     """Calculate a digest string unique to the text content"""
  41     return hashlib.sha1(text).hexdigest()
  42
  43 def _file_checksum(filename):
  44     return _text_checksum(open(filename, 'rb').read())
  45
  46 def _guess_docker_command():
  47     """ Guess a working docker command or raise exception if not found"""
  48     commands = [["docker"], ["sudo", "-n", "docker"]]
  49     for cmd in commands:
  50         try:
  51             if subprocess.call(cmd + ["images"],
  52                                stdout=DEVNULL, stderr=DEVNULL) == 0:
  53                 return cmd
  54         except OSError:
  55             pass
  56     commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
  57     raise Exception("Cannot find working docker command. Tried:\n%s" % \
  58                     commands_txt)
  59
  60 def _copy_with_mkdir(src, root_dir, sub_path='.'):
  61     """Copy src into root_dir, creating sub_path as needed."""
  62     dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
  63     try:
  64         os.makedirs(dest_dir)
  65     except OSError:
  66         # we can safely ignore already created directories
  67         pass
  68
  69     dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
  70     copy(src, dest_file)
  71
  72
  73 def _get_so_libs(executable):
  74     """Return a list of libraries associated with an executable.
  75
  76     The paths may be symbolic links which would need to be resolved to
  77     ensure theright data is copied."""
  78
  79     libs = []
  80     ldd_re = re.compile(r"(/.*/)(\S*)")
  81     try:
  82         ldd_output = subprocess.check_output(["ldd", executable])
  83         for line in ldd_output.split("\n"):
  84             search = ldd_re.search(line)
  85             if search and len(search.groups()) == 2:
  86                 so_path = search.groups()[0]
  87                 so_lib = search.groups()[1]
  88                 libs.append("%s/%s" % (so_path, so_lib))
  89     except subprocess.CalledProcessError:
  90         print "%s had no associated libraries (static build?)" % (executable)
  91
  92     return libs
  93
  94 def _copy_binary_with_libs(src, dest_dir):
  95     """Copy a binary executable and all its dependant libraries.
  96
  97     This does rely on the host file-system being fairly multi-arch
  98     aware so the file don't clash with the guests layout."""
  99
 100     _copy_with_mkdir(src, dest_dir, "/usr/bin")
 101
 102     libs = _get_so_libs(src)
 103     if libs:
 104         for l in libs:
 105             so_path = os.path.dirname(l)
 106             _copy_with_mkdir(l , dest_dir, so_path)
 107
 108 class Docker(object):
 109     """ Running Docker commands """
 110     def __init__(self):
 111         self._command = _guess_docker_command()
 112         self._instances = []
 113         atexit.register(self._kill_instances)
 114         signal.signal(signal.SIGTERM, self._kill_instances)
 115         signal.signal(signal.SIGHUP, self._kill_instances)
 116
 117     def _do(self, cmd, quiet=True, **kwargs):
 118         if quiet:
 119             kwargs["stdout"] = DEVNULL
 120         return subprocess.call(self._command + cmd, **kwargs)
 121
 122     def _do_check(self, cmd, quiet=True, **kwargs):
 123         if quiet:
 124             kwargs["stdout"] = DEVNULL
 125         return subprocess.check_call(self._command + cmd, **kwargs)
 126
 127     def _do_kill_instances(self, only_known, only_active=True):
 128         cmd = ["ps", "-q"]
 129         if not only_active:
 130             cmd.append("-a")
 131         for i in self._output(cmd).split():
 132             resp = self._output(["inspect", i])
 133             labels = json.loads(resp)[0]["Config"]["Labels"]
 134             active = json.loads(resp)[0]["State"]["Running"]
 135             if not labels:
 136                 continue
 137             instance_uuid = labels.get("com.qemu.instance.uuid", None)
 138             if not instance_uuid:
 139                 continue
 140             if only_known and instance_uuid not in self._instances:
 141                 continue
 142             print "Terminating", i
 143             if active:
 144                 self._do(["kill", i])
 145             self._do(["rm", i])
 146
 147     def clean(self):
 148         self._do_kill_instances(False, False)
 149         return 0
 150
 151     def _kill_instances(self, *args, **kwargs):
 152         return self._do_kill_instances(True)
 153
 154     def _output(self, cmd, **kwargs):
 155         return subprocess.check_output(self._command + cmd,
 156                                        stderr=subprocess.STDOUT,
 157                                        **kwargs)
 158
 159     def get_image_dockerfile_checksum(self, tag):
 160         resp = self._output(["inspect", tag])
 161         labels = json.loads(resp)[0]["Config"].get("Labels", {})
 162         return labels.get("com.qemu.dockerfile-checksum", "")
 163
 164     def build_image(self, tag, docker_dir, dockerfile,
 165                     quiet=True, user=False, argv=None, extra_files_cksum=[]):
 166         if argv == None:
 167             argv = []
 168
 169         tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
 170         tmp_df.write(dockerfile)
 171
 172         if user:
 173             uid = os.getuid()
 174             uname = getpwuid(uid).pw_name
 175             tmp_df.write("\n")
 176             tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
 177                          (uname, uid, uname))
 178
 179         tmp_df.write("\n")
 180         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
 181                      _text_checksum("\n".join([dockerfile] +
 182                                     extra_files_cksum)))
 183         tmp_df.flush()
 184
 185         self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + \
 186                        [docker_dir],
 187                        quiet=quiet)
 188
 189     def update_image(self, tag, tarball, quiet=True):
 190         "Update a tagged image using "
 191
 192         self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
 193
 194     def image_matches_dockerfile(self, tag, dockerfile):
 195         try:
 196             checksum = self.get_image_dockerfile_checksum(tag)
 197         except Exception:
 198             return False
 199         return checksum == _text_checksum(dockerfile)
 200
 201     def run(self, cmd, keep, quiet):
 202         label = uuid.uuid1().hex
 203         if not keep:
 204             self._instances.append(label)
 205         ret = self._do_check(["run", "--label",
 206                              "com.qemu.instance.uuid=" + label] + cmd,
 207                              quiet=quiet)
 208         if not keep:
 209             self._instances.remove(label)
 210         return ret
 211
 212     def command(self, cmd, argv, quiet):
 213         return self._do([cmd] + argv, quiet=quiet)
 214
 215 class SubCommand(object):
 216     """A SubCommand template base class"""
 217     name = None # Subcommand name
 218     def shared_args(self, parser):
 219         parser.add_argument("--quiet", action="store_true",
 220                             help="Run quietly unless an error occured")
 221
 222     def args(self, parser):
 223         """Setup argument parser"""
 224         pass
 225     def run(self, args, argv):
 226         """Run command.
 227         args: parsed argument by argument parser.
 228         argv: remaining arguments from sys.argv.
 229         """
 230         pass
 231
 232 class RunCommand(SubCommand):
 233     """Invoke docker run and take care of cleaning up"""
 234     name = "run"
 235     def args(self, parser):
 236         parser.add_argument("--keep", action="store_true",
 237                             help="Don't remove image when command completes")
 238     def run(self, args, argv):
 239         return Docker().run(argv, args.keep, quiet=args.quiet)
 240
 241 class BuildCommand(SubCommand):
 242     """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
 243     name = "build"
 244     def args(self, parser):
 245         parser.add_argument("--include-executable", "-e",
 246                             help="""Specify a binary that will be copied to the
 247                             container together with all its dependent
 248                             libraries""")
 249         parser.add_argument("--extra-files", "-f", nargs='*',
 250                             help="""Specify files that will be copied in the
 251                             Docker image, fulfilling the ADD directive from the
 252                             Dockerfile""")
 253         parser.add_argument("--add-current-user", "-u", dest="user",
 254                             action="store_true",
 255                             help="Add the current user to image's passwd")
 256         parser.add_argument("tag",
 257                             help="Image Tag")
 258         parser.add_argument("dockerfile",
 259                             help="Dockerfile name")
 260
 261     def run(self, args, argv):
 262         dockerfile = open(args.dockerfile, "rb").read()
 263         tag = args.tag
 264
 265         dkr = Docker()
 266         if "--no-cache" not in argv and \
 267            dkr.image_matches_dockerfile(tag, dockerfile):
 268             if not args.quiet:
 269                 print "Image is up to date."
 270         else:
 271             # Create a docker context directory for the build
 272             docker_dir = tempfile.mkdtemp(prefix="docker_build")
 273
 274             # Is there a .pre file to run in the build context?
 275             docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
 276             if os.path.exists(docker_pre):
 277                 stdout = DEVNULL if args.quiet else None
 278                 rc = subprocess.call(os.path.realpath(docker_pre),
 279                                      cwd=docker_dir, stdout=stdout)
 280                 if rc == 3:
 281                     print "Skip"
 282                     return 0
 283                 elif rc != 0:
 284                     print "%s exited with code %d" % (docker_pre, rc)
 285                     return 1
 286
 287             # Copy any extra files into the Docker context. These can be
 288             # included by the use of the ADD directive in the Dockerfile.
 289             cksum = []
 290             if args.include_executable:
 291                 # FIXME: there is no checksum of this executable and the linked
 292                 # libraries, once the image built any change of this executable
 293                 # or any library won't trigger another build.
 294                 _copy_binary_with_libs(args.include_executable, docker_dir)
 295             for filename in args.extra_files or []:
 296                 _copy_with_mkdir(filename, docker_dir)
 297                 cksum += [_file_checksum(filename)]
 298
 299             argv += ["--build-arg=" + k.lower() + "=" + v
 300                         for k, v in os.environ.iteritems()
 301                         if k.lower() in FILTERED_ENV_NAMES]
 302             dkr.build_image(tag, docker_dir, dockerfile,
 303                             quiet=args.quiet, user=args.user, argv=argv,
 304                             extra_files_cksum=cksum)
 305
 306             rmtree(docker_dir)
 307
 308         return 0
 309
 310 class UpdateCommand(SubCommand):
 311     """ Update a docker image with new executables. Arguments: <tag> <executable>"""
 312     name = "update"
 313     def args(self, parser):
 314         parser.add_argument("tag",
 315                             help="Image Tag")
 316         parser.add_argument("executable",
 317                             help="Executable to copy")
 318
 319     def run(self, args, argv):
 320         # Create a temporary tarball with our whole build context and
 321         # dockerfile for the update
 322         tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
 323         tmp_tar = TarFile(fileobj=tmp, mode='w')
 324
 325         # Add the executable to the tarball
 326         bn = os.path.basename(args.executable)
 327         ff = "/usr/bin/%s" % bn
 328         tmp_tar.add(args.executable, arcname=ff)
 329
 330         # Add any associated libraries
 331         libs = _get_so_libs(args.executable)
 332         if libs:
 333             for l in libs:
 334                 tmp_tar.add(os.path.realpath(l), arcname=l)
 335
 336         # Create a Docker buildfile
 337         df = StringIO()
 338         df.write("FROM %s\n" % args.tag)
 339         df.write("ADD . /\n")
 340         df.seek(0)
 341
 342         df_tar = TarInfo(name="Dockerfile")
 343         df_tar.size = len(df.buf)
 344         tmp_tar.addfile(df_tar, fileobj=df)
 345
 346         tmp_tar.close()
 347
 348         # reset the file pointers
 349         tmp.flush()
 350         tmp.seek(0)
 351
 352         # Run the build with our tarball context
 353         dkr = Docker()
 354         dkr.update_image(args.tag, tmp, quiet=args.quiet)
 355
 356         return 0
 357
 358 class CleanCommand(SubCommand):
 359     """Clean up docker instances"""
 360     name = "clean"
 361     def run(self, args, argv):
 362         Docker().clean()
 363         return 0
 364
 365 class ImagesCommand(SubCommand):
 366     """Run "docker images" command"""
 367     name = "images"
 368     def run(self, args, argv):
 369         return Docker().command("images", argv, args.quiet)
 370
 371 def main():
 372     parser = argparse.ArgumentParser(description="A Docker helper",
 373             usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
 374     subparsers = parser.add_subparsers(title="subcommands", help=None)
 375     for cls in SubCommand.__subclasses__():
 376         cmd = cls()
 377         subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
 378         cmd.shared_args(subp)
 379         cmd.args(subp)
 380         subp.set_defaults(cmdobj=cmd)
 381     args, argv = parser.parse_known_args()
 382     return args.cmdobj.run(args, argv)
 383
 384 if __name__ == "__main__":
 385     sys.exit(main())