tests/docker/docker.py

   1 #!/usr/bin/env python2
   2 #
   3 # Docker controlling module
   4 #
   5 # Copyright (c) 2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Fam Zheng <famz@redhat.com>
   9 #
  10 # This work is licensed under the terms of the GNU GPL, version 2
  11 # or (at your option) any later version. See the COPYING file in
  12 # the top-level directory.
  13
  14 import os
  15 import sys
  16 import subprocess
  17 import json
  18 import hashlib
  19 import atexit
  20 import uuid
  21 import argparse
  22 import tempfile
  23 import re
  24 from shutil import copy, rmtree
  25
  26 def _text_checksum(text):
  27     """Calculate a digest string unique to the text content"""
  28     return hashlib.sha1(text).hexdigest()
  29
  30 def _guess_docker_command():
  31     """ Guess a working docker command or raise exception if not found"""
  32     commands = [["docker"], ["sudo", "-n", "docker"]]
  33     for cmd in commands:
  34         if subprocess.call(cmd + ["images"],
  35                            stdout=subprocess.PIPE,
  36                            stderr=subprocess.PIPE) == 0:
  37             return cmd
  38     commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
  39     raise Exception("Cannot find working docker command. Tried:\n%s" % \
  40                     commands_txt)
  41
  42 def _copy_with_mkdir(src, root_dir, sub_path):
  43     """Copy src into root_dir, creating sub_path as needed."""
  44     dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
  45     try:
  46         os.makedirs(dest_dir)
  47     except OSError:
  48         # we can safely ignore already created directories
  49         pass
  50
  51     dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
  52     copy(src, dest_file)
  53
  54
  55 def _get_so_libs(executable):
  56     """Return a list of libraries associated with an executable.
  57
  58     The paths may be symbolic links which would need to be resolved to
  59     ensure theright data is copied."""
  60
  61     libs = []
  62     ldd_re = re.compile(r"(/.*/)(\S*)")
  63     try:
  64         ldd_output = subprocess.check_output(["ldd", executable])
  65         for line in ldd_output.split("\n"):
  66             search = ldd_re.search(line)
  67             if search and len(search.groups()) == 2:
  68                 so_path = search.groups()[0]
  69                 so_lib = search.groups()[1]
  70                 libs.append("%s/%s" % (so_path, so_lib))
  71     except subprocess.CalledProcessError:
  72         print "%s had no associated libraries (static build?)" % (executable)
  73
  74     return libs
  75
  76 def _copy_binary_with_libs(src, dest_dir):
  77     """Copy a binary executable and all its dependant libraries.
  78
  79     This does rely on the host file-system being fairly multi-arch
  80     aware so the file don't clash with the guests layout."""
  81
  82     _copy_with_mkdir(src, dest_dir, "/usr/bin")
  83
  84     libs = _get_so_libs(src)
  85     if libs:
  86         for l in libs:
  87             so_path = os.path.dirname(l)
  88             _copy_with_mkdir(l , dest_dir, so_path)
  89
  90 class Docker(object):
  91     """ Running Docker commands """
  92     def __init__(self):
  93         self._command = _guess_docker_command()
  94         self._instances = []
  95         atexit.register(self._kill_instances)
  96
  97     def _do(self, cmd, quiet=True, **kwargs):
  98         if quiet:
  99             kwargs["stdout"] = subprocess.PIPE
 100         return subprocess.call(self._command + cmd, **kwargs)
 101
 102     def _do_kill_instances(self, only_known, only_active=True):
 103         cmd = ["ps", "-q"]
 104         if not only_active:
 105             cmd.append("-a")
 106         for i in self._output(cmd).split():
 107             resp = self._output(["inspect", i])
 108             labels = json.loads(resp)[0]["Config"]["Labels"]
 109             active = json.loads(resp)[0]["State"]["Running"]
 110             if not labels:
 111                 continue
 112             instance_uuid = labels.get("com.qemu.instance.uuid", None)
 113             if not instance_uuid:
 114                 continue
 115             if only_known and instance_uuid not in self._instances:
 116                 continue
 117             print "Terminating", i
 118             if active:
 119                 self._do(["kill", i])
 120             self._do(["rm", i])
 121
 122     def clean(self):
 123         self._do_kill_instances(False, False)
 124         return 0
 125
 126     def _kill_instances(self):
 127         return self._do_kill_instances(True)
 128
 129     def _output(self, cmd, **kwargs):
 130         return subprocess.check_output(self._command + cmd,
 131                                        stderr=subprocess.STDOUT,
 132                                        **kwargs)
 133
 134     def get_image_dockerfile_checksum(self, tag):
 135         resp = self._output(["inspect", tag])
 136         labels = json.loads(resp)[0]["Config"].get("Labels", {})
 137         return labels.get("com.qemu.dockerfile-checksum", "")
 138
 139     def build_image(self, tag, docker_dir, dockerfile, quiet=True, argv=None):
 140         if argv == None:
 141             argv = []
 142
 143         tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
 144         tmp_df.write(dockerfile)
 145
 146         tmp_df.write("\n")
 147         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
 148                      _text_checksum(dockerfile))
 149         tmp_df.flush()
 150
 151         self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
 152                  [docker_dir],
 153                  quiet=quiet)
 154
 155     def image_matches_dockerfile(self, tag, dockerfile):
 156         try:
 157             checksum = self.get_image_dockerfile_checksum(tag)
 158         except Exception:
 159             return False
 160         return checksum == _text_checksum(dockerfile)
 161
 162     def run(self, cmd, keep, quiet):
 163         label = uuid.uuid1().hex
 164         if not keep:
 165             self._instances.append(label)
 166         ret = self._do(["run", "--label",
 167                         "com.qemu.instance.uuid=" + label] + cmd,
 168                        quiet=quiet)
 169         if not keep:
 170             self._instances.remove(label)
 171         return ret
 172
 173 class SubCommand(object):
 174     """A SubCommand template base class"""
 175     name = None # Subcommand name
 176     def shared_args(self, parser):
 177         parser.add_argument("--quiet", action="store_true",
 178                             help="Run quietly unless an error occured")
 179
 180     def args(self, parser):
 181         """Setup argument parser"""
 182         pass
 183     def run(self, args, argv):
 184         """Run command.
 185         args: parsed argument by argument parser.
 186         argv: remaining arguments from sys.argv.
 187         """
 188         pass
 189
 190 class RunCommand(SubCommand):
 191     """Invoke docker run and take care of cleaning up"""
 192     name = "run"
 193     def args(self, parser):
 194         parser.add_argument("--keep", action="store_true",
 195                             help="Don't remove image when command completes")
 196     def run(self, args, argv):
 197         return Docker().run(argv, args.keep, quiet=args.quiet)
 198
 199 class BuildCommand(SubCommand):
 200     """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
 201     name = "build"
 202     def args(self, parser):
 203         parser.add_argument("--include-executable", "-e",
 204                             help="""Specify a binary that will be copied to the
 205                             container together with all its dependent
 206                             libraries""")
 207         parser.add_argument("tag",
 208                             help="Image Tag")
 209         parser.add_argument("dockerfile",
 210                             help="Dockerfile name")
 211
 212     def run(self, args, argv):
 213         dockerfile = open(args.dockerfile, "rb").read()
 214         tag = args.tag
 215
 216         dkr = Docker()
 217         if dkr.image_matches_dockerfile(tag, dockerfile):
 218             if not args.quiet:
 219                 print "Image is up to date."
 220         else:
 221             # Create a docker context directory for the build
 222             docker_dir = tempfile.mkdtemp(prefix="docker_build")
 223
 224             # Is there a .pre file to run in the build context?
 225             docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
 226             if os.path.exists(docker_pre):
 227                 rc = subprocess.call(os.path.realpath(docker_pre),
 228                                      cwd=docker_dir)
 229                 if rc == 3:
 230                     print "Skip"
 231                     return 0
 232                 elif rc != 0:
 233                     print "%s exited with code %d" % (docker_pre, rc)
 234                     return 1
 235
 236             # Do we include a extra binary?
 237             if args.include_executable:
 238                 _copy_binary_with_libs(args.include_executable,
 239                                        docker_dir)
 240
 241             dkr.build_image(tag, docker_dir, dockerfile,
 242                             quiet=args.quiet, argv=argv)
 243
 244             rmtree(docker_dir)
 245
 246         return 0
 247
 248 class CleanCommand(SubCommand):
 249     """Clean up docker instances"""
 250     name = "clean"
 251     def run(self, args, argv):
 252         Docker().clean()
 253         return 0
 254
 255 def main():
 256     parser = argparse.ArgumentParser(description="A Docker helper",
 257             usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
 258     subparsers = parser.add_subparsers(title="subcommands", help=None)
 259     for cls in SubCommand.__subclasses__():
 260         cmd = cls()
 261         subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
 262         cmd.shared_args(subp)
 263         cmd.args(subp)
 264         subp.set_defaults(cmdobj=cmd)
 265     args, argv = parser.parse_known_args()
 266     return args.cmdobj.run(args, argv)
 267
 268 if __name__ == "__main__":
 269     sys.exit(main())