3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """Docker image and docker container classes.
19 In Docker terminology image is a read-only layer that never changes.
20 Container is created once you start a process in Docker from an Image. Container
21 consists of read-write layer, plus information about the parent Image, plus
22 some additional information like its unique ID, networking configuration,
24 For more information refer to http://docs.docker.io/.
26 Mapping to Docker CLI:
27 Image is a result of "docker build path/to/Dockerfile" command.
28 Container is a result of "docker run image_tag" command.
29 ImageOptions and ContainerOptions allow to pass parameters to these commands.
31 Versions 1.9 and 1.10 of docker remote API are supported.
34 from collections
import namedtuple
48 _SUCCESSFUL_BUILD_PATTERN
= re
.compile(r
'Successfully built ([a-zA-Z0-9]{12})')
50 _ERROR_LOG_TMPL
= 'Build Error: {error}.'
51 _ERROR_LOG_DETAILED_TMPL
= _ERROR_LOG_TMPL
+ ' Detail: {detail}'
55 class ImageOptions(namedtuple('ImageOptionsT',
56 ['dockerfile_dir', 'tag', 'nocache', 'rm'])):
57 """Options for building Docker Images."""
59 def __new__(cls
, dockerfile_dir
=None, tag
=None, nocache
=False, rm
=True):
60 """This method is redefined to provide default values for namedtuple.
63 dockerfile_dir: str, Path to the directory with the Dockerfile. If it is
64 None, no build is needed. We will be looking for the existing image
65 with the specified tag and raise an error if it does not exist.
66 tag: str, Repository name (and optionally a tag) to be applied to the
67 image in case of successful build. If dockerfile_dir is None, tag
68 is used for lookup of an image.
69 nocache: boolean, True if cache should not be used when building the
71 rm: boolean, True if intermediate images should be removed after a
72 successful build. Default value is set to True because this is the
73 default value used by "docker build" command.
78 return super(ImageOptions
, cls
).__new
__(
79 cls
, dockerfile_dir
=dockerfile_dir
, tag
=tag
, nocache
=nocache
, rm
=rm
)
82 class ContainerOptions(namedtuple('ContainerOptionsT',
83 ['image_opts', 'port', 'port_bindings',
84 'environment', 'volumes', 'volumes_from',
86 """Options for creating and running Docker Containers."""
88 def __new__(cls
, image_opts
=None, port
=None, port_bindings
=None,
89 environment
=None, volumes
=None, volumes_from
=None, name
=None):
90 """This method is redefined to provide default values for namedtuple.
93 image_opts: ImageOptions, properties of underlying Docker Image.
94 port: int, Primary port that the process inside of a container is
95 listening on. If this port is not part of the port bindings
96 specified, a default binding will be added for this port.
97 port_bindings: dict, Port bindings for exposing multiple ports. If the
98 only binding needed is the default binding of just one port this
100 environment: dict, Environment variables.
101 volumes: dict, Volumes to mount from the host system.
102 volumes_from: list, Volumes from the specified container(s).
103 name: str, Name of a container. Needed for data containers.
106 ContainerOptions object.
108 return super(ContainerOptions
, cls
).__new
__(
109 cls
, image_opts
=image_opts
, port
=port
, port_bindings
=port_bindings
,
110 environment
=environment
, volumes
=volumes
, volumes_from
=volumes_from
,
114 class Error(Exception):
115 """Base exception for containers module."""
118 class ImageError(Error
):
119 """Image related errors."""
122 class ContainerError(Error
):
123 """Container related erorrs."""
126 class DockerDaemonConnectionError(Error
):
127 """Raised if the docker client can't connect to the docker daemon."""
130 class BaseImage(object):
131 """Abstract base class for Docker images."""
133 def __init__(self
, docker_client
, image_opts
):
134 """Initializer for BaseImage.
137 docker_client: an object of docker.Client class to communicate with a
139 image_opts: an instance of ImageOptions class describing the parameters
140 passed to docker commands.
143 DockerDaemonConnectionError: If the docker daemon isn't responding.
145 self
._docker
_client
= docker_client
146 self
._image
_opts
= image_opts
150 self
._docker
_client
.ping()
151 except requests
.exceptions
.ConnectionError
:
152 raise DockerDaemonConnectionError(
153 'Couldn\'t connect to the docker daemon at %s. Please check that '
154 'the docker daemon is running and that you have specified the '
155 'correct docker host.' % self
._docker
_client
.base_url
)
158 """Calls "docker build" if needed."""
159 raise NotImplementedError
162 """Calls "docker rmi" if needed."""
163 raise NotImplementedError
167 """Returns 64 hexadecimal digit string identifying the image."""
168 # Might also be a first 12-characters shortcut.
173 """Returns image tag string."""
174 return self
._image
_opts
.tag
177 """Makes BaseImage usable with "with" statement."""
181 # pylint: disable=redefined-builtin
182 def __exit__(self
, type, value
, traceback
):
183 """Makes BaseImage usable with "with" statement."""
187 """Makes sure that build artifacts are cleaned up."""
191 class Image(BaseImage
):
192 """Docker image that requires building and should be removed afterwards."""
194 def __init__(self
, docker_client
, image_opts
):
195 """Initializer for Image.
198 docker_client: an object of docker.Client class to communicate with a
200 image_opts: an instance of ImageOptions class that must have
201 dockerfile_dir set. image_id will be returned by "docker build"
205 ImageError: if dockerfile_dir is not set.
207 if not image_opts
.dockerfile_dir
:
208 raise ImageError('dockerfile_dir for images that require building '
211 super(Image
, self
).__init
__(docker_client
, image_opts
)
214 """Calls "docker build".
217 ImageError: if the image could not be built.
219 logging
.info('Building image %s...', self
.tag
)
221 build_res
= self
._docker
_client
.build(
222 path
=self
._image
_opts
.dockerfile_dir
,
224 quiet
=False, fileobj
=None, nocache
=self
._image
_opts
.nocache
,
225 rm
=self
._image
_opts
.rm
)
227 log_lines
= [json
.loads(x
.strip()) for x
in build_res
]
230 logging
.error('Error building docker image %s [with no output]', self
.tag
)
233 def _FormatBuildLog(lines
):
236 return ('Full Image Build Log:\n%s' %
237 ''.join(l
.get(_STREAM
) for l
in lines
))
239 success_message
= log_lines
[-1].get(_STREAM
)
241 m
= _SUCCESSFUL_BUILD_PATTERN
.match(success_message
)
243 # The build was successful.
244 self
._id
= m
.group(1)
245 logging
.info('Image %s built, id = %s', self
.tag
, self
.id)
246 logging
.debug(_FormatBuildLog(log_lines
))
249 logging
.error('Error building docker image %s', self
.tag
)
251 # Last log line usually contains error details if not a success message.
252 err_line
= log_lines
[-1]
253 error
= err_line
.get('error')
254 error_detail
= err_line
.get('errorDetail')
256 error_detail
= error_detail
.get('message')
258 stop
= len(log_lines
)
259 if error
or error_detail
:
260 el
= (_ERROR_LOG_TMPL
if error
== error_detail
261 else _ERROR_LOG_DETAILED_TMPL
).format(error
=error
,
266 logging
.error(_FormatBuildLog(itertools
.islice(log_lines
, stop
)))
270 """Calls "docker rmi"."""
273 self
._docker
_client
.remove_image(self
.id)
274 except docker
.errors
.APIError
as e
:
275 logging
.warning('Image %s (id=%s) cannot be removed: %s. Try cleaning '
276 'up old containers that can be listed with '
277 '"docker ps -a" and removing the image again with '
278 '"docker rmi IMAGE_ID".',
279 self
.tag
, self
.id, e
)
283 class PrebuiltImage(BaseImage
):
284 """Prebuilt Docker image. Build and Remove functions are noops."""
286 def __init__(self
, docker_client
, image_opts
):
287 """Initializer for PrebuiltImage.
290 docker_client: an object of docker.Client class to communicate with a
292 image_opts: an instance of ImageOptions class that must have
293 dockerfile_dir not set and tag set.
296 ImageError: if image_opts.dockerfile_dir is set or
297 image_opts.tag is not set.
299 if image_opts
.dockerfile_dir
:
300 raise ImageError('dockerfile_dir for PrebuiltImage must not be set.')
302 if not image_opts
.tag
:
303 raise ImageError('PrebuiltImage must have tag specified to find '
306 super(PrebuiltImage
, self
).__init
__(docker_client
, image_opts
)
309 """Searches for pre-built image with specified tag.
312 ImageError: if image with this tag was not found.
314 logging
.info('Looking for image_id for image with tag %s', self
.tag
)
315 images
= self
._docker
_client
.images(
316 name
=self
.tag
, quiet
=True, all
=False, viz
=False)
319 raise ImageError('Image with tag %s was not found' % self
.tag
)
321 # TODO: check if it's possible to have more than one image returned.
325 """Unassigns image_id only, does not remove the image as we don't own it."""
329 def CreateImage(docker_client
, image_opts
):
330 """Creates an new object to represent Docker image.
333 docker_client: an object of docker.Client class to communicate with a
335 image_opts: an instance of ImageOptions class.
338 New object, subclass of BaseImage class.
340 image
= Image
if image_opts
.dockerfile_dir
else PrebuiltImage
341 return image(docker_client
, image_opts
)
344 def GetDockerHost(docker_client
):
345 parsed_url
= urlparse
.urlparse(docker_client
.base_url
)
347 # Socket url schemes look like: unix:// or http+unix://.
348 # If the user is running docker locally and connecting over a socket, we
349 # should just use localhost.
350 if 'unix' in parsed_url
.scheme
:
352 return parsed_url
.hostname
355 class Container(object):
356 """Docker Container."""
358 def __init__(self
, docker_client
, container_opts
):
359 """Initializer for Container.
362 docker_client: an object of docker.Client class to communicate with a
364 container_opts: an instance of ContainerOptions class.
366 self
._docker
_client
= docker_client
367 self
._container
_opts
= container_opts
369 self
._image
= CreateImage(docker_client
, container_opts
.image_opts
)
371 self
._host
= GetDockerHost(self
._docker
_client
)
372 self
._container
_host
= None
374 # Port bindings will be set to a dictionary mapping exposed ports
375 # to the interface they are bound to. This will be populated from
376 # the container options passed when the container is started.
377 self
._port
_bindings
= None
379 # Use the daemon flag in case we leak these threads.
380 self
._logs
_listener
= threading
.Thread(target
=self
._ListenToLogs
)
381 self
._logs
_listener
.daemon
= True
384 """Builds an image (if necessary) and runs a container.
387 ContainerError: if container_id is already set, i.e. container is already
391 raise ContainerError('Trying to start already running container.')
395 logging
.info('Creating container...')
396 port_bindings
= self
._container
_opts
.port_bindings
or {}
397 if self
._container
_opts
.port
:
398 # Add primary port to port bindings if not already specified.
399 # Setting its value to None lets docker pick any available port.
400 port_bindings
[self
._container
_opts
.port
] = port_bindings
.get(
401 self
._container
_opts
.port
)
403 self
._id
= self
._docker
_client
.create_container(
404 image
=self
._image
.id, hostname
=None, user
=None, detach
=True,
406 tty
=False, mem_limit
=0,
407 ports
=port_bindings
.keys(),
408 volumes
=(self
._container
_opts
.volumes
.keys()
409 if self
._container
_opts
.volumes
else None),
410 environment
=self
._container
_opts
.environment
,
412 network_disabled
=False,
414 # create_container returns a dict sometimes.
415 if isinstance(self
.id, dict):
416 self
._id
= self
.id.get('Id')
417 logging
.info('Container %s created.', self
.id)
419 self
._docker
_client
.start(
421 port_bindings
=port_bindings
,
422 binds
=self
._container
_opts
.volumes
,
423 # In the newer API version volumes_from got moved from
424 # create_container to start. In older version volumes_from option was
425 # completely broken therefore we support only passing volumes_from
427 volumes_from
=self
._container
_opts
.volumes_from
)
429 self
._logs
_listener
.start()
431 if not port_bindings
:
435 container_info
= self
._docker
_client
.inspect_container(self
._id
)
436 network_settings
= container_info
['NetworkSettings']
437 self
._container
_host
= network_settings
['IPAddress']
438 self
._port
_bindings
= {
439 port
: int(network_settings
['Ports']['%d/tcp' % port
][0]['HostPort'])
440 for port
in port_bindings
444 """Stops a running container, removes it and underlying image if needed."""
446 self
._docker
_client
.kill(self
.id)
447 self
._docker
_client
.remove_container(self
.id, v
=False,
452 def PortBinding(self
, port
):
453 """Get the host binding of a container port.
456 port: Port inside container.
459 Port on the host system mapped to the given port inside of
462 return self
._port
_bindings
.get(port
)
466 """Host the container can be reached at by the host (i.e. client) system."""
471 """Port (on the host system) mapped to the port inside of the container."""
472 return self
._port
_bindings
[self
._container
_opts
.port
]
476 """An address the container can be reached at by the host system."""
477 return '%s:%d' % (self
.host
, self
.port
)
481 """Returns 64 hexadecimal digit string identifying the container."""
485 def container_addr(self
):
486 """An address the container can be reached at by another container."""
487 return '%s:%d' % (self
._container
_host
, self
._container
_opts
.port
)
491 """String, identifying a container. Required for data containers."""
492 return self
._container
_opts
.name
494 def _ListenToLogs(self
):
495 """Logs all output from the docker container.
497 The docker.Client.logs method returns a generator that yields log lines.
498 This method iterates over that generator and outputs those log lines to
499 the devappserver2 logs.
501 log_lines
= self
._docker
_client
.logs(container
=self
.id, stream
=True)
502 for line
in log_lines
:
504 logging
.debug('Container: %s: %s', self
.id[0:12], line
)
507 """Makes Container usable with "with" statement."""
511 # pylint: disable=redefined-builtin
512 def __exit__(self
, type, value
, traceback
):
513 """Makes Container usable with "with" statement."""
517 """Makes sure that all build and run artifacts are cleaned up."""