tests/docker/docker.py

   1 #!/usr/bin/env python3
   2 #
   3 # Docker controlling module
   4 #
   5 # Copyright (c) 2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Fam Zheng <famz@redhat.com>
   9 #
  10 # This work is licensed under the terms of the GNU GPL, version 2
  11 # or (at your option) any later version. See the COPYING file in
  12 # the top-level directory.
  13
  14 import os
  15 import sys
  16 import subprocess
  17 import json
  18 import hashlib
  19 import atexit
  20 import uuid
  21 import argparse
  22 import enum
  23 import tempfile
  24 import re
  25 import signal
  26 from tarfile import TarFile, TarInfo
  27 from io import StringIO, BytesIO
  28 from shutil import copy, rmtree
  29 from pwd import getpwuid
  30 from datetime import datetime, timedelta
  31
  32
  33 FILTERED_ENV_NAMES = ['ftp_proxy', 'http_proxy', 'https_proxy']
  34
  35
  36 DEVNULL = open(os.devnull, 'wb')
  37
  38 class EngineEnum(enum.IntEnum):
  39     AUTO = 1
  40     DOCKER = 2
  41     PODMAN = 3
  42
  43     def __str__(self):
  44         return self.name.lower()
  45
  46     def __repr__(self):
  47         return str(self)
  48
  49     @staticmethod
  50     def argparse(s):
  51         try:
  52             return EngineEnum[s.upper()]
  53         except KeyError:
  54             return s
  55
  56
  57 USE_ENGINE = EngineEnum.AUTO
  58
  59 def _bytes_checksum(bytes):
  60     """Calculate a digest string unique to the text content"""
  61     return hashlib.sha1(bytes).hexdigest()
  62
  63 def _text_checksum(text):
  64     """Calculate a digest string unique to the text content"""
  65     return _bytes_checksum(text.encode('utf-8'))
  66
  67 def _read_dockerfile(path):
  68     return open(path, 'rt', encoding='utf-8').read()
  69
  70 def _file_checksum(filename):
  71     return _bytes_checksum(open(filename, 'rb').read())
  72
  73
  74 def _guess_engine_command():
  75     """ Guess a working engine command or raise exception if not found"""
  76     commands = []
  77
  78     if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.PODMAN]:
  79         commands += [["podman"]]
  80     if USE_ENGINE in [EngineEnum.AUTO, EngineEnum.DOCKER]:
  81         commands += [["docker"], ["sudo", "-n", "docker"]]
  82     for cmd in commands:
  83         try:
  84             # docker version will return the client details in stdout
  85             # but still report a status of 1 if it can't contact the daemon
  86             if subprocess.call(cmd + ["version"],
  87                                stdout=DEVNULL, stderr=DEVNULL) == 0:
  88                 return cmd
  89         except OSError:
  90             pass
  91     commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
  92     raise Exception("Cannot find working engine command. Tried:\n%s" %
  93                     commands_txt)
  94
  95
  96 def _copy_with_mkdir(src, root_dir, sub_path='.', name=None):
  97     """Copy src into root_dir, creating sub_path as needed."""
  98     dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
  99     try:
 100         os.makedirs(dest_dir)
 101     except OSError:
 102         # we can safely ignore already created directories
 103         pass
 104
 105     dest_file = "%s/%s" % (dest_dir, name if name else os.path.basename(src))
 106
 107     try:
 108         copy(src, dest_file)
 109     except FileNotFoundError:
 110         print("Couldn't copy %s to %s" % (src, dest_file))
 111         pass
 112
 113
 114 def _get_so_libs(executable):
 115     """Return a list of libraries associated with an executable.
 116
 117     The paths may be symbolic links which would need to be resolved to
 118     ensure the right data is copied."""
 119
 120     libs = []
 121     ldd_re = re.compile(r"(?:\S+ => )?(\S*) \(:?0x[0-9a-f]+\)")
 122     try:
 123         ldd_output = subprocess.check_output(["ldd", executable]).decode('utf-8')
 124         for line in ldd_output.split("\n"):
 125             search = ldd_re.search(line)
 126             if search:
 127                 try:
 128                     libs.append(search.group(1))
 129                 except IndexError:
 130                     pass
 131     except subprocess.CalledProcessError:
 132         print("%s had no associated libraries (static build?)" % (executable))
 133
 134     return libs
 135
 136
 137 def _copy_binary_with_libs(src, bin_dest, dest_dir):
 138     """Maybe copy a binary and all its dependent libraries.
 139
 140     If bin_dest isn't set we only copy the support libraries because
 141     we don't need qemu in the docker path to run (due to persistent
 142     mapping). Indeed users may get confused if we aren't running what
 143     is in the image.
 144
 145     This does rely on the host file-system being fairly multi-arch
 146     aware so the file don't clash with the guests layout.
 147     """
 148
 149     if bin_dest:
 150         _copy_with_mkdir(src, dest_dir, os.path.dirname(bin_dest))
 151     else:
 152         print("only copying support libraries for %s" % (src))
 153
 154     libs = _get_so_libs(src)
 155     if libs:
 156         for l in libs:
 157             so_path = os.path.dirname(l)
 158             name = os.path.basename(l)
 159             real_l = os.path.realpath(l)
 160             _copy_with_mkdir(real_l, dest_dir, so_path, name)
 161
 162
 163 def _check_binfmt_misc(executable):
 164     """Check binfmt_misc has entry for executable in the right place.
 165
 166     The details of setting up binfmt_misc are outside the scope of
 167     this script but we should at least fail early with a useful
 168     message if it won't work.
 169
 170     Returns the configured binfmt path and a valid flag. For
 171     persistent configurations we will still want to copy and dependent
 172     libraries.
 173     """
 174
 175     binary = os.path.basename(executable)
 176     binfmt_entry = "/proc/sys/fs/binfmt_misc/%s" % (binary)
 177
 178     if not os.path.exists(binfmt_entry):
 179         print ("No binfmt_misc entry for %s" % (binary))
 180         return None, False
 181
 182     with open(binfmt_entry) as x: entry = x.read()
 183
 184     if re.search("flags:.*F.*\n", entry):
 185         print("binfmt_misc for %s uses persistent(F) mapping to host binary" %
 186               (binary))
 187         return None, True
 188
 189     m = re.search("interpreter (\S+)\n", entry)
 190     interp = m.group(1)
 191     if interp and interp != executable:
 192         print("binfmt_misc for %s does not point to %s, using %s" %
 193               (binary, executable, interp))
 194
 195     return interp, True
 196
 197
 198 def _read_qemu_dockerfile(img_name):
 199     # special case for Debian linux-user images
 200     if img_name.startswith("debian") and img_name.endswith("user"):
 201         img_name = "debian-bootstrap"
 202
 203     df = os.path.join(os.path.dirname(__file__), "dockerfiles",
 204                       img_name + ".docker")
 205     return _read_dockerfile(df)
 206
 207
 208 def _dockerfile_preprocess(df):
 209     out = ""
 210     for l in df.splitlines():
 211         if len(l.strip()) == 0 or l.startswith("#"):
 212             continue
 213         from_pref = "FROM qemu/"
 214         if l.startswith(from_pref):
 215             # TODO: Alternatively we could replace this line with "FROM $ID"
 216             # where $ID is the image's hex id obtained with
 217             #    $ docker images $IMAGE --format="{{.Id}}"
 218             # but unfortunately that's not supported by RHEL 7.
 219             inlining = _read_qemu_dockerfile(l[len(from_pref):])
 220             out += _dockerfile_preprocess(inlining)
 221             continue
 222         out += l + "\n"
 223     return out
 224
 225
 226 class Docker(object):
 227     """ Running Docker commands """
 228     def __init__(self):
 229         self._command = _guess_engine_command()
 230
 231         if "docker" in self._command and "TRAVIS" not in os.environ:
 232             os.environ["DOCKER_BUILDKIT"] = "1"
 233             self._buildkit = True
 234         else:
 235             self._buildkit = False
 236
 237         self._instance = None
 238         atexit.register(self._kill_instances)
 239         signal.signal(signal.SIGTERM, self._kill_instances)
 240         signal.signal(signal.SIGHUP, self._kill_instances)
 241
 242     def _do(self, cmd, quiet=True, **kwargs):
 243         if quiet:
 244             kwargs["stdout"] = DEVNULL
 245         return subprocess.call(self._command + cmd, **kwargs)
 246
 247     def _do_check(self, cmd, quiet=True, **kwargs):
 248         if quiet:
 249             kwargs["stdout"] = DEVNULL
 250         return subprocess.check_call(self._command + cmd, **kwargs)
 251
 252     def _do_kill_instances(self, only_known, only_active=True):
 253         cmd = ["ps", "-q"]
 254         if not only_active:
 255             cmd.append("-a")
 256
 257         filter = "--filter=label=com.qemu.instance.uuid"
 258         if only_known:
 259             if self._instance:
 260                 filter += "=%s" % (self._instance)
 261             else:
 262                 # no point trying to kill, we finished
 263                 return
 264
 265         print("filter=%s" % (filter))
 266         cmd.append(filter)
 267         for i in self._output(cmd).split():
 268             self._do(["rm", "-f", i])
 269
 270     def clean(self):
 271         self._do_kill_instances(False, False)
 272         return 0
 273
 274     def _kill_instances(self, *args, **kwargs):
 275         return self._do_kill_instances(True)
 276
 277     def _output(self, cmd, **kwargs):
 278         try:
 279             return subprocess.check_output(self._command + cmd,
 280                                            stderr=subprocess.STDOUT,
 281                                            encoding='utf-8',
 282                                            **kwargs)
 283         except TypeError:
 284             # 'encoding' argument was added in 3.6+
 285             return subprocess.check_output(self._command + cmd,
 286                                            stderr=subprocess.STDOUT,
 287                                            **kwargs).decode('utf-8')
 288
 289
 290     def inspect_tag(self, tag):
 291         try:
 292             return self._output(["inspect", tag])
 293         except subprocess.CalledProcessError:
 294             return None
 295
 296     def get_image_creation_time(self, info):
 297         return json.loads(info)[0]["Created"]
 298
 299     def get_image_dockerfile_checksum(self, tag):
 300         resp = self.inspect_tag(tag)
 301         labels = json.loads(resp)[0]["Config"].get("Labels", {})
 302         return labels.get("com.qemu.dockerfile-checksum", "")
 303
 304     def build_image(self, tag, docker_dir, dockerfile,
 305                     quiet=True, user=False, argv=None, registry=None,
 306                     extra_files_cksum=[]):
 307         if argv is None:
 308             argv = []
 309
 310         # pre-calculate the docker checksum before any
 311         # substitutions we make for caching
 312         checksum = _text_checksum(_dockerfile_preprocess(dockerfile))
 313
 314         if registry is not None:
 315             sources = re.findall("FROM qemu\/(.*)", dockerfile)
 316             # Fetch any cache layers we can, may fail
 317             for s in sources:
 318                 pull_args = ["pull", "%s/qemu/%s" % (registry, s)]
 319                 if self._do(pull_args, quiet=quiet) != 0:
 320                     registry = None
 321                     break
 322             # Make substitutions
 323             if registry is not None:
 324                 dockerfile = dockerfile.replace("FROM qemu/",
 325                                                 "FROM %s/qemu/" %
 326                                                 (registry))
 327
 328         tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
 329                                              encoding='utf-8',
 330                                              dir=docker_dir, suffix=".docker")
 331         tmp_df.write(dockerfile)
 332
 333         if user:
 334             uid = os.getuid()
 335             uname = getpwuid(uid).pw_name
 336             tmp_df.write("\n")
 337             tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" %
 338                          (uname, uid, uname))
 339
 340         tmp_df.write("\n")
 341         tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s\n" % (checksum))
 342         for f, c in extra_files_cksum:
 343             tmp_df.write("LABEL com.qemu.%s-checksum=%s\n" % (f, c))
 344
 345         tmp_df.flush()
 346
 347         build_args = ["build", "-t", tag, "-f", tmp_df.name]
 348         if self._buildkit:
 349             build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
 350
 351         if registry is not None:
 352             pull_args = ["pull", "%s/%s" % (registry, tag)]
 353             self._do(pull_args, quiet=quiet)
 354             cache = "%s/%s" % (registry, tag)
 355             build_args += ["--cache-from", cache]
 356         build_args += argv
 357         build_args += [docker_dir]
 358
 359         self._do_check(build_args,
 360                        quiet=quiet)
 361
 362     def update_image(self, tag, tarball, quiet=True):
 363         "Update a tagged image using "
 364
 365         self._do_check(["build", "-t", tag, "-"], quiet=quiet, stdin=tarball)
 366
 367     def image_matches_dockerfile(self, tag, dockerfile):
 368         try:
 369             checksum = self.get_image_dockerfile_checksum(tag)
 370         except Exception:
 371             return False
 372         return checksum == _text_checksum(_dockerfile_preprocess(dockerfile))
 373
 374     def run(self, cmd, keep, quiet, as_user=False):
 375         label = uuid.uuid4().hex
 376         if not keep:
 377             self._instance = label
 378
 379         if as_user:
 380             uid = os.getuid()
 381             cmd = [ "-u", str(uid) ] + cmd
 382             # podman requires a bit more fiddling
 383             if self._command[0] == "podman":
 384                 cmd.insert(0, '--userns=keep-id')
 385
 386         ret = self._do_check(["run", "--rm", "--label",
 387                              "com.qemu.instance.uuid=" + label] + cmd,
 388                              quiet=quiet)
 389         if not keep:
 390             self._instance = None
 391         return ret
 392
 393     def command(self, cmd, argv, quiet):
 394         return self._do([cmd] + argv, quiet=quiet)
 395
 396
 397 class SubCommand(object):
 398     """A SubCommand template base class"""
 399     name = None  # Subcommand name
 400
 401     def shared_args(self, parser):
 402         parser.add_argument("--quiet", action="store_true",
 403                             help="Run quietly unless an error occurred")
 404
 405     def args(self, parser):
 406         """Setup argument parser"""
 407         pass
 408
 409     def run(self, args, argv):
 410         """Run command.
 411         args: parsed argument by argument parser.
 412         argv: remaining arguments from sys.argv.
 413         """
 414         pass
 415
 416
 417 class RunCommand(SubCommand):
 418     """Invoke docker run and take care of cleaning up"""
 419     name = "run"
 420
 421     def args(self, parser):
 422         parser.add_argument("--keep", action="store_true",
 423                             help="Don't remove image when command completes")
 424         parser.add_argument("--run-as-current-user", action="store_true",
 425                             help="Run container using the current user's uid")
 426
 427     def run(self, args, argv):
 428         return Docker().run(argv, args.keep, quiet=args.quiet,
 429                             as_user=args.run_as_current_user)
 430
 431
 432 class BuildCommand(SubCommand):
 433     """ Build docker image out of a dockerfile. Arg: <tag> <dockerfile>"""
 434     name = "build"
 435
 436     def args(self, parser):
 437         parser.add_argument("--include-executable", "-e",
 438                             help="""Specify a binary that will be copied to the
 439                             container together with all its dependent
 440                             libraries""")
 441         parser.add_argument("--skip-binfmt",
 442                             action="store_true",
 443                             help="""Skip binfmt entry check (used for testing)""")
 444         parser.add_argument("--extra-files", nargs='*',
 445                             help="""Specify files that will be copied in the
 446                             Docker image, fulfilling the ADD directive from the
 447                             Dockerfile""")
 448         parser.add_argument("--add-current-user", "-u", dest="user",
 449                             action="store_true",
 450                             help="Add the current user to image's passwd")
 451         parser.add_argument("--registry", "-r",
 452                             help="cache from docker registry")
 453         parser.add_argument("-t", dest="tag",
 454                             help="Image Tag")
 455         parser.add_argument("-f", dest="dockerfile",
 456                             help="Dockerfile name")
 457
 458     def run(self, args, argv):
 459         dockerfile = _read_dockerfile(args.dockerfile)
 460         tag = args.tag
 461
 462         dkr = Docker()
 463         if "--no-cache" not in argv and \
 464            dkr.image_matches_dockerfile(tag, dockerfile):
 465             if not args.quiet:
 466                 print("Image is up to date.")
 467         else:
 468             # Create a docker context directory for the build
 469             docker_dir = tempfile.mkdtemp(prefix="docker_build")
 470
 471             # Validate binfmt_misc will work
 472             if args.skip_binfmt:
 473                 qpath = args.include_executable
 474             elif args.include_executable:
 475                 qpath, enabled = _check_binfmt_misc(args.include_executable)
 476                 if not enabled:
 477                     return 1
 478
 479             # Is there a .pre file to run in the build context?
 480             docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
 481             if os.path.exists(docker_pre):
 482                 stdout = DEVNULL if args.quiet else None
 483                 rc = subprocess.call(os.path.realpath(docker_pre),
 484                                      cwd=docker_dir, stdout=stdout)
 485                 if rc == 3:
 486                     print("Skip")
 487                     return 0
 488                 elif rc != 0:
 489                     print("%s exited with code %d" % (docker_pre, rc))
 490                     return 1
 491
 492             # Copy any extra files into the Docker context. These can be
 493             # included by the use of the ADD directive in the Dockerfile.
 494             cksum = []
 495             if args.include_executable:
 496                 # FIXME: there is no checksum of this executable and the linked
 497                 # libraries, once the image built any change of this executable
 498                 # or any library won't trigger another build.
 499                 _copy_binary_with_libs(args.include_executable,
 500                                        qpath, docker_dir)
 501
 502             for filename in args.extra_files or []:
 503                 _copy_with_mkdir(filename, docker_dir)
 504                 cksum += [(filename, _file_checksum(filename))]
 505
 506             argv += ["--build-arg=" + k.lower() + "=" + v
 507                      for k, v in os.environ.items()
 508                      if k.lower() in FILTERED_ENV_NAMES]
 509             dkr.build_image(tag, docker_dir, dockerfile,
 510                             quiet=args.quiet, user=args.user,
 511                             argv=argv, registry=args.registry,
 512                             extra_files_cksum=cksum)
 513
 514             rmtree(docker_dir)
 515
 516         return 0
 517
 518
 519 class UpdateCommand(SubCommand):
 520     """ Update a docker image with new executables. Args: <tag> <executable>"""
 521     name = "update"
 522
 523     def args(self, parser):
 524         parser.add_argument("tag",
 525                             help="Image Tag")
 526         parser.add_argument("executable",
 527                             help="Executable to copy")
 528
 529     def run(self, args, argv):
 530         # Create a temporary tarball with our whole build context and
 531         # dockerfile for the update
 532         tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
 533         tmp_tar = TarFile(fileobj=tmp, mode='w')
 534
 535         # Add the executable to the tarball, using the current
 536         # configured binfmt_misc path. If we don't get a path then we
 537         # only need the support libraries copied
 538         ff, enabled = _check_binfmt_misc(args.executable)
 539
 540         if not enabled:
 541             print("binfmt_misc not enabled, update disabled")
 542             return 1
 543
 544         if ff:
 545             tmp_tar.add(args.executable, arcname=ff)
 546
 547         # Add any associated libraries
 548         libs = _get_so_libs(args.executable)
 549         if libs:
 550             for l in libs:
 551                 tmp_tar.add(os.path.realpath(l), arcname=l)
 552
 553         # Create a Docker buildfile
 554         df = StringIO()
 555         df.write(u"FROM %s\n" % args.tag)
 556         df.write(u"ADD . /\n")
 557
 558         df_bytes = BytesIO(bytes(df.getvalue(), "UTF-8"))
 559
 560         df_tar = TarInfo(name="Dockerfile")
 561         df_tar.size = df_bytes.getbuffer().nbytes
 562         tmp_tar.addfile(df_tar, fileobj=df_bytes)
 563
 564         tmp_tar.close()
 565
 566         # reset the file pointers
 567         tmp.flush()
 568         tmp.seek(0)
 569
 570         # Run the build with our tarball context
 571         dkr = Docker()
 572         dkr.update_image(args.tag, tmp, quiet=args.quiet)
 573
 574         return 0
 575
 576
 577 class CleanCommand(SubCommand):
 578     """Clean up docker instances"""
 579     name = "clean"
 580
 581     def run(self, args, argv):
 582         Docker().clean()
 583         return 0
 584
 585
 586 class ImagesCommand(SubCommand):
 587     """Run "docker images" command"""
 588     name = "images"
 589
 590     def run(self, args, argv):
 591         return Docker().command("images", argv, args.quiet)
 592
 593
 594 class ProbeCommand(SubCommand):
 595     """Probe if we can run docker automatically"""
 596     name = "probe"
 597
 598     def run(self, args, argv):
 599         try:
 600             docker = Docker()
 601             if docker._command[0] == "docker":
 602                 print("docker")
 603             elif docker._command[0] == "sudo":
 604                 print("sudo docker")
 605             elif docker._command[0] == "podman":
 606                 print("podman")
 607         except Exception:
 608             print("no")
 609
 610         return
 611
 612
 613 class CcCommand(SubCommand):
 614     """Compile sources with cc in images"""
 615     name = "cc"
 616
 617     def args(self, parser):
 618         parser.add_argument("--image", "-i", required=True,
 619                             help="The docker image in which to run cc")
 620         parser.add_argument("--cc", default="cc",
 621                             help="The compiler executable to call")
 622         parser.add_argument("--source-path", "-s", nargs="*", dest="paths",
 623                             help="""Extra paths to (ro) mount into container for
 624                             reading sources""")
 625
 626     def run(self, args, argv):
 627         if argv and argv[0] == "--":
 628             argv = argv[1:]
 629         cwd = os.getcwd()
 630         cmd = ["-w", cwd,
 631                "-v", "%s:%s:rw" % (cwd, cwd)]
 632         if args.paths:
 633             for p in args.paths:
 634                 cmd += ["-v", "%s:%s:ro,z" % (p, p)]
 635         cmd += [args.image, args.cc]
 636         cmd += argv
 637         return Docker().run(cmd, False, quiet=args.quiet,
 638                             as_user=True)
 639
 640
 641 class CheckCommand(SubCommand):
 642     """Check if we need to re-build a docker image out of a dockerfile.
 643     Arguments: <tag> <dockerfile>"""
 644     name = "check"
 645
 646     def args(self, parser):
 647         parser.add_argument("tag",
 648                             help="Image Tag")
 649         parser.add_argument("dockerfile", default=None,
 650                             help="Dockerfile name", nargs='?')
 651         parser.add_argument("--checktype", choices=["checksum", "age"],
 652                             default="checksum", help="check type")
 653         parser.add_argument("--olderthan", default=60, type=int,
 654                             help="number of minutes")
 655
 656     def run(self, args, argv):
 657         tag = args.tag
 658
 659         try:
 660             dkr = Docker()
 661         except subprocess.CalledProcessError:
 662             print("Docker not set up")
 663             return 1
 664
 665         info = dkr.inspect_tag(tag)
 666         if info is None:
 667             print("Image does not exist")
 668             return 1
 669
 670         if args.checktype == "checksum":
 671             if not args.dockerfile:
 672                 print("Need a dockerfile for tag:%s" % (tag))
 673                 return 1
 674
 675             dockerfile = _read_dockerfile(args.dockerfile)
 676
 677             if dkr.image_matches_dockerfile(tag, dockerfile):
 678                 if not args.quiet:
 679                     print("Image is up to date")
 680                 return 0
 681             else:
 682                 print("Image needs updating")
 683                 return 1
 684         elif args.checktype == "age":
 685             timestr = dkr.get_image_creation_time(info).split(".")[0]
 686             created = datetime.strptime(timestr, "%Y-%m-%dT%H:%M:%S")
 687             past = datetime.now() - timedelta(minutes=args.olderthan)
 688             if created < past:
 689                 print ("Image created @ %s more than %d minutes old" %
 690                        (timestr, args.olderthan))
 691                 return 1
 692             else:
 693                 if not args.quiet:
 694                     print ("Image less than %d minutes old" % (args.olderthan))
 695                 return 0
 696
 697
 698 def main():
 699     global USE_ENGINE
 700
 701     parser = argparse.ArgumentParser(description="A Docker helper",
 702                                      usage="%s <subcommand> ..." %
 703                                      os.path.basename(sys.argv[0]))
 704     parser.add_argument("--engine", type=EngineEnum.argparse, choices=list(EngineEnum),
 705                         help="specify which container engine to use")
 706     subparsers = parser.add_subparsers(title="subcommands", help=None)
 707     for cls in SubCommand.__subclasses__():
 708         cmd = cls()
 709         subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
 710         cmd.shared_args(subp)
 711         cmd.args(subp)
 712         subp.set_defaults(cmdobj=cmd)
 713     args, argv = parser.parse_known_args()
 714     if args.engine:
 715         USE_ENGINE = args.engine
 716     return args.cmdobj.run(args, argv)
 717
 718
 719 if __name__ == "__main__":
 720     sys.exit(main())