admin/containers/scripted_gmx_docker_builds.py

   1 #!/usr/bin/env python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2020, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """
  37 Generates a set of docker images used for running GROMACS CI on Gitlab.
  38 The images are prepared according to a selection of build configuration targets
  39 that hope to cover a broad enough scope of different possible systems,
  40 allowing us to check compiler types and versions, as well as libraries used
  41 for accelerators and parallel communication systems. Each combinations is
  42 described as an entry in the build_configs dictionary, with the script
  43 analysing the logic and adding build stages as needed.
  44
  45 Based on the example script provided by the NVidia HPCCM repository.
  46
  47 Authors:
  48     * Paul Bauer <paul.bauer.q@gmail.com>
  49     * Eric Irrgang <ericirrgang@gmail.com>
  50     * Joe Jordan <e.jjordan12@gmail.com>
  51
  52 Usage::
  53
  54     $ python3 scripted_gmx_docker_builds.py --help
  55     $ python3 scripted_gmx_docker_builds.py --format docker > Dockerfile && docker build .
  56     $ python3 scripted_gmx_docker_builds.py | docker build -
  57
  58 """
  59
  60 import argparse
  61 import collections
  62 import typing
  63 from distutils.version import StrictVersion
  64
  65 import hpccm
  66 import hpccm.config
  67 from hpccm.building_blocks.base import bb_base
  68
  69 try:
  70     import utility
  71 except ImportError:
  72     raise RuntimeError(
  73         'This module assumes availability of supporting modules in the same directory. Add the directory to '
  74         'PYTHONPATH or invoke Python from within the module directory so module location can be resolved.')
  75
  76 # Basic packages for all final images.
  77 _common_packages = ['build-essential',
  78                     'ca-certificates',
  79                     'ccache',
  80                     'git',
  81                     'gnupg',
  82                     'libfftw3-dev',
  83                     'libhwloc-dev',
  84                     'liblapack-dev',
  85                     'libx11-dev',
  86                     'moreutils',
  87                     'ninja-build',
  88                     'rsync',
  89                     'valgrind',
  90                     'vim',
  91                     'wget',
  92                     'xsltproc']
  93
  94 # Extra packages needed to build Python installations from source.
  95 _python_extra_packages = ['build-essential',
  96                           'ca-certificates',
  97                           'ccache',
  98                           'curl',
  99                           'git',
 100                           'libbz2-dev',
 101                           'libffi-dev',
 102                           'liblzma-dev',
 103                           'libncurses5-dev',
 104                           'libncursesw5-dev',
 105                           'libreadline-dev',
 106                           'libsqlite3-dev',
 107                           'libssl-dev',
 108                           'llvm',
 109                           'python-openssl',
 110                           'vim',
 111                           'wget',
 112                           'zlib1g-dev']
 113
 114 # Extra packages needed for images for building documentation.
 115 _docs_extra_packages = ['autoconf',
 116                         'automake',
 117                         'autopoint',
 118                         'autotools-dev',
 119                         'bison',
 120                         'flex',
 121                         'ghostscript',
 122                         'graphviz',
 123                         'help2man',
 124                         'imagemagick',
 125                         'libtool',
 126                         'linkchecker',
 127                         'mscgen',
 128                         'm4',
 129                         'texinfo',
 130                         'texlive-latex-base',
 131                         'texlive-latex-extra',
 132                         'texlive-fonts-recommended',
 133                         'texlive-fonts-extra']
 134
 135 # Supported Python versions for maintained branches.
 136 _python_versions = ['3.6.10', '3.7.7', '3.8.2']
 137
 138 # Parse command line arguments
 139 parser = argparse.ArgumentParser(description='GROMACS CI image creation script', parents=[utility.parser])
 140
 141 parser.add_argument('--format', type=str, default='docker',
 142                     choices=['docker', 'singularity'],
 143                     help='Container specification format (default: docker)')
 144 parser.add_argument('--venvs', nargs='*', type=str, default=_python_versions,
 145                     help='List of Python versions ("major.minor.patch") for which to install venvs. '
 146                          'Default: {}'.format(' '.join(_python_versions)))
 147
 148
 149 def base_image_tag(args) -> str:
 150     # Check if we use CUDA images or plain linux images
 151     if args.cuda is not None:
 152         cuda_version_tag = 'nvidia/cuda:' + args.cuda + '-devel'
 153         if args.centos is not None:
 154             cuda_version_tag += '-centos' + args.centos
 155         elif args.ubuntu is not None:
 156             cuda_version_tag += '-ubuntu' + args.ubuntu
 157         else:
 158             raise RuntimeError('Logic error: no Linux distribution selected.')
 159
 160         base_image_tag = cuda_version_tag
 161     else:
 162         if args.centos is not None:
 163             base_image_tag = 'centos:centos' + args.centos
 164         elif args.ubuntu is not None:
 165             base_image_tag = 'ubuntu:' + args.ubuntu
 166         else:
 167             raise RuntimeError('Logic error: no Linux distribution selected.')
 168     return base_image_tag
 169
 170
 171 def get_llvm_packages(args) -> typing.Iterable[str]:
 172     # If we use the package version of LLVM, we need to install extra packages for it.
 173     if (args.llvm is not None) and (args.tsan is None):
 174         return ['libomp-dev',
 175                 'clang-format-' + str(args.llvm),
 176                 'clang-tidy-' + str(args.llvm)]
 177     else:
 178         return []
 179
 180
 181 def get_compiler(args, tsan_stage: hpccm.Stage = None) -> bb_base:
 182     # Compiler
 183     if args.icc is not None:
 184         raise RuntimeError('Intel compiler toolchain recipe not implemented yet')
 185
 186     if args.llvm is not None:
 187         # Build our own version instead to get TSAN + OMP
 188         if args.tsan is not None:
 189             if tsan_stage is not None:
 190                 compiler = tsan_stage.runtime(_from='tsan')
 191             else:
 192                 raise RuntimeError('No TSAN stage!')
 193         # Build the default compiler if we don't need special support
 194         else:
 195             compiler = hpccm.building_blocks.llvm(extra_repository=True, version=args.llvm)
 196
 197     elif (args.gcc is not None):
 198         compiler = hpccm.building_blocks.gnu(extra_repository=True,
 199                                              version=args.gcc,
 200                                              fortran=False)
 201     else:
 202         raise RuntimeError('Logic error: no compiler toolchain selected.')
 203     return compiler
 204
 205
 206 def get_mpi(args, compiler):
 207     # If needed, add MPI to the image
 208     if args.mpi is not None:
 209         if args.mpi == 'openmpi':
 210             use_cuda = False
 211             if args.cuda is not None:
 212                 use_cuda = True
 213
 214             if hasattr(compiler, 'toolchain'):
 215                 return hpccm.building_blocks.openmpi(toolchain=compiler.toolchain, cuda=use_cuda, infiniband=False)
 216             else:
 217                 raise RuntimeError('compiler is not an HPCCM compiler building block!')
 218
 219         elif args.mpi == 'impi':
 220             raise RuntimeError('Intel MPI recipe not implemented yet.')
 221         else:
 222             raise RuntimeError('Requested unknown MPI implementation.')
 223     else:
 224         return None
 225
 226
 227 def get_opencl(args):
 228     # Add OpenCL environment if needed
 229     if (args.opencl is not None):
 230         if args.opencl == 'nvidia':
 231             if (args.cuda is None):
 232                 raise RuntimeError('Need Nvidia environment for Nvidia OpenCL image')
 233
 234             return hpccm.building_blocks.packages(ospackages=['nvidia-opencl-dev'])
 235
 236         elif args.opencl == 'intel':
 237             return hpccm.building_blocks.packages(
 238                     apt_ppas=['ppa:intel-opencl/intel-opencl'],
 239                     ospackages=['opencl-headers', 'ocl-icd-libopencl1',
 240                                 'ocl-icd-opencl-dev', 'intel-opencl-icd'])
 241
 242         elif args.opencl == 'amd':
 243             # libelf1 is a necessary dependency for something in the ROCm stack,
 244             # which they should set up, but seem to have omitted.
 245             return hpccm.building_blocks.packages(
 246                     apt_keys=['http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key'],
 247                     apt_repositories=['deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main'],
 248                     ospackages=['ocl-icd-libopencl1', 'ocl-icd-opencl-dev', 'opencl-headers', 'libelf1', 'rocm-opencl'])
 249     else:
 250         return None
 251
 252
 253 def get_clfft(args):
 254     if (args.clfft is not None):
 255         return hpccm.building_blocks.generic_cmake(
 256             repository='https://github.com/clMathLibraries/clFFT.git',
 257             prefix='/usr/local', recursive=True, branch=args.clfft, directory='clFFT/src')
 258     else:
 259         return None
 260
 261
 262 def add_tsan_stage(input_args, output_stages: typing.Mapping[str, hpccm.Stage]):
 263     """Isolate the expensive TSAN preparation stage.
 264
 265     This is a very expensive stage, but has few and disjoint dependencies, and
 266     its output is easily compartmentalized (/usr/local) so we can isolate this
 267     build stage to maximize build cache hits and reduce rebuild time, bookkeeping,
 268     and final image size.
 269     """
 270     if not isinstance(output_stages, collections.abc.MutableMapping):
 271         raise RuntimeError('Need output_stages container.')
 272     tsan_stage = hpccm.Stage()
 273     tsan_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='tsan')
 274
 275     tsan_stage += hpccm.building_blocks.packages(ospackages=['git', 'ca-certificates', 'build-essential', 'cmake'])
 276     # CMake will get duplicated later, but this is an expensive image, and it isn't worth optimizing
 277     # out that duplication...
 278     tsan_stage += hpccm.building_blocks.python(python3=True, python2=False, devel=False)
 279
 280     compiler_branch = 'release_' + str(input_args.llvm) + '0'
 281     tsan_stage += hpccm.building_blocks.generic_cmake(
 282         repository='https://git.llvm.org/git/llvm.git',
 283         prefix='/usr/local', recursive=True, branch=compiler_branch,
 284         cmake_opts=['-D CMAKE_BUILD_TYPE=Release', '-D LLVM_ENABLE_PROJECTS="clang;openmp;clang-tools-extra"',
 285                     '-D LIBOMP_TSAN_SUPPORT=on'],
 286         preconfigure=['export branch=' + compiler_branch,
 287                       '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxx.git)',
 288                       '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/libcxxabi.git)',
 289                       '(cd projects; git clone --depth=1 --branch $branch https://git.llvm.org/git/compiler-rt.git)',
 290                       '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/openmp.git)',
 291                       '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang.git)',
 292                       '(cd ..; git clone --depth=1 --branch $branch https://git.llvm.org/git/clang-tools-extra.git)'],
 293         postinstall=['ln -s /usr/local/bin/clang++ /usr/local/bin/clang++-' + str(input_args.llvm),
 294                      'ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-' + str(input_args.llvm),
 295                      'ln -s /usr/local/bin/clang-tidy /usr/local/bin/clang-tidy-' + str(input_args.llvm),
 296                      'ln -s /usr/local/libexec/c++-analyzer /usr/local/bin/c++-analyzer-' + str(input_args.llvm)])
 297     output_stages['tsan'] = tsan_stage
 298
 299
 300 def prepare_venv(version: StrictVersion) -> typing.Sequence[str]:
 301     """Get shell commands to set up the venv for the requested Python version."""
 302     major = version.version[0]
 303     minor = version.version[1]  # type: int
 304
 305     pyenv = '$HOME/.pyenv/bin/pyenv'
 306
 307     py_ver = '{}.{}'.format(major, minor)
 308     venv_path = '$HOME/venv/py{}'.format(py_ver)
 309     commands = ['$({pyenv} prefix `{pyenv} whence python{py_ver}`)/bin/python -m venv {path}'.format(
 310         pyenv=pyenv,
 311         py_ver=py_ver,
 312         path=venv_path
 313     )]
 314
 315     commands.append('{path}/bin/python -m pip install --upgrade pip setuptools'.format(
 316         path=venv_path
 317     ))
 318     # Install dependencies for building and testing gmxapi Python package.
 319     # WARNING: Please keep this list synchronized with python_packaging/requirements-test.txt
 320     # TODO: Get requirements.txt from an input argument.
 321     commands.append("""{path}/bin/python -m pip install --upgrade \
 322             'cmake>=3.13' \
 323             'flake8>=3.7.7' \
 324             'mpi4py>=3.0.3' \
 325             'networkx>=2.0' \
 326             'numpy>=1' \
 327             'pip>=10.1' \
 328             'pytest>=3.9' \
 329             'setuptools>=42' \
 330             'scikit-build>=0.10'""".format(path=venv_path))
 331
 332     # TODO: Remove 'importlib_resources' dependency when Python >=3.7 is required.
 333     if minor == 6:
 334         commands.append("""{path}/bin/python -m pip install --upgrade \
 335                 'importlib_resources'""".format(path=venv_path))
 336
 337     return commands
 338
 339
 340 def add_python_stages(building_blocks: typing.Mapping[str, bb_base],
 341                       input_args,
 342                       output_stages: typing.MutableMapping[str, hpccm.Stage]):
 343     """Add the stage(s) necessary for the requested venvs.
 344
 345     One intermediate build stage is created for each venv (see --venv option).
 346
 347     Each stage partially populates Python installations and venvs in the home
 348     directory. The home directory is collected by the 'pyenv' stage for use by
 349     the main build stage.
 350     """
 351     if len(input_args.venvs) < 1:
 352         raise RuntimeError('No venvs to build...')
 353     if output_stages is None or not isinstance(output_stages, collections.abc.Mapping):
 354         raise RuntimeError('Need a container for output stages.')
 355
 356     # Main Python stage that collects the environments from individual stages.
 357     # We collect the stages individually, rather than chaining them, because the
 358     # copy is a bit slow and wastes local Docker image space for each filesystem
 359     # layer.
 360     pyenv_stage = hpccm.Stage()
 361     pyenv_stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as='pyenv')
 362     pyenv_stage += building_blocks['compiler']
 363     pyenv_stage += building_blocks['mpi']
 364     pyenv_stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
 365
 366     for version in [StrictVersion(py_ver) for py_ver in sorted(input_args.venvs)]:
 367         stage_name = 'py' + str(version)
 368         stage = hpccm.Stage()
 369         stage += hpccm.primitives.baseimage(image=base_image_tag(input_args), _as=stage_name)
 370         stage += building_blocks['compiler']
 371         stage += building_blocks['mpi']
 372         stage += hpccm.building_blocks.packages(ospackages=_python_extra_packages)
 373
 374         # TODO: Use a non-root user for testing and Python virtual environments.
 375         stage += hpccm.primitives.shell(commands=[
 376             'curl https://pyenv.run | bash',
 377             """echo 'export PYENV_ROOT="$HOME/.pyenv"' >> $HOME/.bashrc""",
 378             """echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> $HOME/.bashrc""",
 379             """echo 'eval "$(pyenv init -)"' >> $HOME/.bashrc""",
 380             """echo 'eval "$(pyenv virtualenv-init -)"' >> $HOME/.bashrc"""])
 381         pyenv = '$HOME/.pyenv/bin/pyenv'
 382         commands = ['PYTHON_CONFIGURE_OPTS="--enable-shared" {pyenv} install -s {version}'.format(
 383             pyenv=pyenv,
 384             version=str(version))]
 385         stage += hpccm.primitives.shell(commands=commands)
 386
 387         commands = prepare_venv(version)
 388         stage += hpccm.primitives.shell(commands=commands)
 389
 390         # TODO: Update user home directory.
 391         pyenv_stage += hpccm.primitives.copy(_from=stage_name, _mkdir=True, src=['/root/'],
 392                                              dest='/root')
 393
 394         # Add the intermediate build stage to the sequence
 395         output_stages[stage_name] = stage
 396
 397     # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured.
 398     # # `version` should be a system installation or pyenv environment (or pyenv-virtualenv)
 399     # # with the dependencies for all of the Python aspects of CMake-driven builds.
 400     # commands = '{pyenv} global {version}'.format(
 401     #             pyenv=pyenv,
 402     #             version=...)
 403     # pyenv_stage += hpccm.primitives.shell(commands=commands)
 404
 405     # Add the aggregating build stage to the sequence. This allows the main stage to copy
 406     # the files in a single stage, potentially reducing the overall output image size.
 407     output_stages['pyenv'] = pyenv_stage
 408
 409
 410 def add_documentation_dependencies(input_args,
 411                                    output_stages: typing.MutableMapping[str, hpccm.Stage]):
 412     """Add appropriate layers according to doxygen input arguments."""
 413     if input_args.doxygen is None:
 414         return
 415     output_stages['main'] += hpccm.primitives.shell(
 416         commands=['sed -i \'/\"XPS\"/d;/\"PDF\"/d;/\"PS\"/d;/\"EPS\"/d;/disable ghostscript format types/d\' /etc/ImageMagick-6/policy.xml'])
 417     output_stages['main'] += hpccm.building_blocks.pip(pip='pip3', packages=['sphinx==1.6.1'])
 418     if input_args.doxygen == '1.8.5':
 419         doxygen_commit = 'ed4ed873ab0e7f15116e2052119a6729d4589f7a'
 420         output_stages['main'] += hpccm.building_blocks.generic_autotools(
 421             repository='https://github.com/westes/flex.git',
 422             commit='f7788a9a0ecccdc953ed12043ccb59ca25714018',
 423             prefix='/tmp/install-of-flex',
 424             configure_opts=['--disable-shared'],
 425             preconfigure=['./autogen.sh'])
 426         output_stages['main'] += hpccm.building_blocks.generic_autotools(
 427             repository='https://github.com/doxygen/doxygen.git',
 428             commit=doxygen_commit,
 429             prefix='',
 430             configure_opts=[
 431                 '--flex /tmp/install-of-flex/bin/flex',
 432                 '--static'])
 433     else:
 434         version = input_args.doxygen
 435         archive_name = 'doxygen-{}.linux.bin.tar.gz'.format(version)
 436         archive_url = 'https://sourceforge.net/projects/doxygen/files/rel-{}/{}'.format(
 437             version,
 438             archive_name
 439         )
 440         binary_path = 'doxygen-{}/bin/doxygen'.format(version)
 441         commands = [
 442             'mkdir doxygen && cd doxygen',
 443             'wget {}'.format(archive_url),
 444             'tar xf {} {}'.format(archive_name, binary_path),
 445             'cp {} /usr/local/bin/'.format(binary_path),
 446             'cd .. && rm -rf doxygen'
 447         ]
 448         output_stages['main'] += hpccm.primitives.shell(commands=commands)
 449
 450
 451 def build_stages(args) -> typing.Iterable[hpccm.Stage]:
 452     """Define and sequence the stages for the recipe corresponding to *args*."""
 453
 454     # A Dockerfile or Singularity recipe can have multiple build stages.
 455     # The main build stage can copy files from previous stages, though only
 456     # the last stage is included in the tagged output image. This means that
 457     # large or expensive sets of build instructions can be isolated in
 458     # local/temporary images, but all of the stages need to be output by this
 459     # script, and need to occur in the correct order, so we create a sequence
 460     # object early in this function.
 461     stages = collections.OrderedDict()
 462
 463     # If we need the TSAN compilers, the early build is more involved.
 464     if args.llvm is not None and args.tsan is not None:
 465         add_tsan_stage(input_args=args, output_stages=stages)
 466
 467     # Building blocks are chunks of container-builder instructions that can be
 468     # copied to any build stage with the addition operator.
 469     building_blocks = collections.OrderedDict()
 470
 471     # These are the most expensive and most reusable layers, so we put them first.
 472     building_blocks['compiler'] = get_compiler(args, tsan_stage=stages.get('tsan'))
 473     building_blocks['mpi'] = get_mpi(args, building_blocks['compiler'])
 474
 475     # Install additional packages early in the build to optimize Docker build layer cache.
 476     os_packages = _common_packages + get_llvm_packages(args)
 477     if args.doxygen is not None:
 478         os_packages += _docs_extra_packages
 479     building_blocks['ospackages'] = hpccm.building_blocks.packages(ospackages=os_packages)
 480
 481     building_blocks['cmake'] = hpccm.building_blocks.cmake(eula=True, version=args.cmake)
 482     building_blocks['opencl'] = get_opencl(args)
 483     building_blocks['clfft'] = get_clfft(args)
 484
 485     # Add Python environments to MPI images, only, so we don't have to worry
 486     # about whether to install mpi4py.
 487     if args.mpi is not None and len(args.venvs) > 0:
 488         add_python_stages(building_blocks=building_blocks, input_args=args, output_stages=stages)
 489
 490     # Create the stage from which the targeted image will be tagged.
 491     stages['main'] = hpccm.Stage()
 492
 493     stages['main'] += hpccm.primitives.baseimage(image=base_image_tag(args))
 494     for bb in building_blocks.values():
 495         if bb is not None:
 496             stages['main'] += bb
 497
 498     # We always add Python3 and Pip
 499     stages['main'] += hpccm.building_blocks.python(python3=True, python2=False, devel=True)
 500     stages['main'] += hpccm.building_blocks.pip(upgrade=True, pip='pip3',
 501                                                 packages=['pytest', 'networkx', 'numpy'])
 502
 503     # Add documentation requirements (doxygen and sphinx + misc).
 504     if args.doxygen is not None:
 505         add_documentation_dependencies(args, stages)
 506
 507     if 'pyenv' in stages and stages['pyenv'] is not None:
 508         stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/.pyenv/'],
 509                                                 dest='/root/.pyenv')
 510         stages['main'] += hpccm.primitives.copy(_from='pyenv', _mkdir=True, src=['/root/venv/'],
 511                                                 dest='/root/venv')
 512         # TODO: Update user home directory.
 513         # TODO: If we activate pyenv for login shells, the `global` "version" should be full-featured.
 514         # stages['main'] += hpccm.primitives.copy(_from='pyenv', src=['/root/.bashrc'],
 515         #                                         dest='/root/')
 516
 517     # Make sure that `python` resolves to something.
 518     stages['main'] += hpccm.primitives.shell(commands=['test -x /usr/bin/python || '
 519                                                        'update-alternatives --install /usr/bin/python python /usr/bin/python3 1 && '
 520                                                        '/usr/bin/python --version'])
 521
 522     # Note that the list of stages should be sorted in dependency order.
 523     for build_stage in stages.values():
 524         if build_stage is not None:
 525             yield build_stage
 526
 527
 528 if __name__ == '__main__':
 529     args = parser.parse_args()
 530
 531     # Set container specification output format
 532     hpccm.config.set_container_format(args.format)
 533
 534     container_recipe = build_stages(args)
 535
 536     # Output container specification
 537     for stage in container_recipe:
 538         print(stage)