python/google/appengine/tools/devappserver2/vm_runtime_proxy.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Manages a VM Runtime process running inside of a docker container."""
  18
  19 import logging
  20 import os
  21 import socket
  22
  23 import google
  24
  25 from google.appengine.tools.devappserver2 import application_configuration
  26 from google.appengine.tools.devappserver2 import http_proxy
  27 from google.appengine.tools.devappserver2 import instance
  28 from google.appengine.tools.docker import containers
  29
  30
  31 _DOCKER_IMAGE_NAME_FORMAT = '{display}.{module}.{version}'
  32 _DOCKER_CONTAINER_NAME_FORMAT = 'google.appengine.{image_name}.{minor_version}'
  33
  34
  35 class Error(Exception):
  36   """Base class for errors in this module."""
  37
  38
  39 class InvalidEnvVariableError(Error):
  40   """Raised if an environment variable name or value cannot be supported."""
  41
  42
  43 class VersionError(Error):
  44   """Raised if no version is specified in application configuration file."""
  45
  46
  47 def _GetPortToPublish(port):
  48   """Checks if given port is available.
  49
  50   Useful for publishing debug ports when it's more convenient to bind to
  51   the same address on each container restart.
  52
  53   Args:
  54     port: int, Port to check.
  55
  56   Returns:
  57     given port if it is available, None if it is already taken (then any
  58         random available port will be selected by Dockerd).
  59   """
  60   sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  61   try:
  62     sock.bind(('', port))
  63     sock.close()
  64     return port
  65   except socket.error:
  66     logging.warning('Requested debug port %d is already in use. '
  67                     'Will use another available port.', port)
  68   return None
  69
  70
  71 class VMRuntimeProxy(instance.RuntimeProxy):
  72   """Manages a VM Runtime process running inside of a docker container."""
  73
  74   DEFAULT_DEBUG_PORT = 5005
  75
  76   def __init__(self, docker_client, runtime_config_getter,
  77                module_configuration, default_port=8080, port_bindings=None,
  78                additional_environment=None):
  79     """Initializer for VMRuntimeProxy.
  80
  81     Args:
  82       docker_client: docker.Client object to communicate with Docker daemon.
  83       runtime_config_getter: A function that can be called without arguments
  84           and returns the runtime_config_pb2.Config containing the configuration
  85           for the runtime.
  86       module_configuration: An application_configuration.ModuleConfiguration
  87           instance respresenting the configuration of the module that owns the
  88           runtime.
  89       default_port: int, main port inside of the container that instance is
  90           listening on.
  91       port_bindings: dict, Additional port bindings from the container.
  92       additional_environment: doct, Additional environment variables to pass
  93           to the container.
  94     """
  95     super(VMRuntimeProxy, self).__init__()
  96
  97     self._runtime_config_getter = runtime_config_getter
  98     self._module_configuration = module_configuration
  99     self._docker_client = docker_client
 100     self._default_port = default_port
 101     self._port_bindings = port_bindings
 102     self._additional_environment = additional_environment
 103     self._container = None
 104     self._proxy = None
 105
 106   def handle(self, environ, start_response, url_map, match, request_id,
 107              request_type):
 108     """Serves request by forwarding it to application instance via HttpProxy.
 109
 110     Args:
 111       environ: An environ dict for the request as defined in PEP-333.
 112       start_response: A function with semantics defined in PEP-333.
 113       url_map: An appinfo.URLMap instance containing the configuration for the
 114           handler matching this request.
 115       match: A re.MatchObject containing the result of the matched URL pattern.
 116       request_id: A unique string id associated with the request.
 117       request_type: The type of the request. See instance.*_REQUEST module
 118           constants.
 119
 120     Returns:
 121       Generator of sequence of strings containing the body of the HTTP response.
 122
 123     Raises:
 124       InvalidEnvVariableError: if user tried to redefine any of the reserved
 125           environment variables.
 126     """
 127     return self._proxy.handle(environ, start_response, url_map, match,
 128                               request_id, request_type)
 129
 130   def _get_instance_logs(self):
 131     # TODO: Handle docker container's logs
 132     return ''
 133
 134   def _instance_died_unexpectedly(self):
 135     # TODO: Check if container is still up and running
 136     return False
 137
 138   def _escape_domain(self, application_external_name):
 139     return application_external_name.replace(':', '.')
 140
 141   def start(self, dockerfile_dir=None):
 142     runtime_config = self._runtime_config_getter()
 143
 144     if not self._module_configuration.major_version:
 145       logging.error('Version needs to be specified in your application '
 146                     'configuration file.')
 147       raise VersionError()
 148
 149     if not dockerfile_dir:
 150       dockerfile_dir = self._module_configuration.application_root
 151
 152     # api_host set to 'localhost' won't be accessible from a docker container
 153     # because container will have it's own 'localhost'.
 154     # 10.0.2.2 is a special network setup by virtualbox to connect from the
 155     # guest to the host.
 156     api_host = runtime_config.api_host
 157     if runtime_config.api_host in ('0.0.0.0', 'localhost'):
 158       api_host = '10.0.2.2'
 159
 160     image_name = _DOCKER_IMAGE_NAME_FORMAT.format(
 161         # Escape domain if it is present.
 162         display=self._escape_domain(
 163             self._module_configuration.application_external_name),
 164         module=self._module_configuration.module_name,
 165         version=self._module_configuration.major_version)
 166
 167     port_bindings = self._port_bindings if self._port_bindings else {}
 168     port_bindings.setdefault(self._default_port, None)
 169     debug_port = None
 170
 171     environment = {
 172         'API_HOST': api_host,
 173         'API_PORT': runtime_config.api_port,
 174         'GAE_LONG_APP_ID': self._module_configuration.application_external_name,
 175         'GAE_PARTITION': self._module_configuration.partition,
 176         'GAE_MODULE_NAME': self._module_configuration.module_name,
 177         'GAE_MODULE_VERSION': self._module_configuration.major_version,
 178         'GAE_MINOR_VERSION': self._module_configuration.minor_version,
 179         'GAE_MODULE_INSTANCE': runtime_config.instance_id,
 180         'GAE_SERVER_PORT': runtime_config.server_port,
 181         'MODULE_YAML_PATH': os.path.basename(
 182             self._module_configuration.config_path)
 183     }
 184     if self._additional_environment:
 185       environment.update(self._additional_environment)
 186
 187     # Handle user defined environment variables
 188     if self._module_configuration.env_variables:
 189       ev = (environment.viewkeys() &
 190             self._module_configuration.env_variables.viewkeys())
 191       if ev:
 192         raise InvalidEnvVariableError(
 193             'Environment variables [%s] are reserved for App Engine use' %
 194             ', '.join(ev))
 195
 196       environment.update(self._module_configuration.env_variables)
 197
 198       # Publish debug port if running in Debug mode.
 199       if self._module_configuration.env_variables.get('DBG_ENABLE'):
 200         debug_port = int(self._module_configuration.env_variables.get(
 201             'DBG_PORT', self.DEFAULT_DEBUG_PORT))
 202         environment['DBG_PORT'] = debug_port
 203         port_bindings[debug_port] = _GetPortToPublish(debug_port)
 204
 205     external_logs_path = os.path.join(
 206         '/var/log/app_engine',
 207         self._escape_domain(
 208             self._module_configuration.application_external_name),
 209         self._module_configuration.module_name,
 210         self._module_configuration.major_version,
 211         runtime_config.instance_id)
 212     container_name = _DOCKER_CONTAINER_NAME_FORMAT.format(
 213         image_name=image_name,
 214         minor_version=self._module_configuration.minor_version)
 215     self._container = containers.Container(
 216         self._docker_client,
 217         containers.ContainerOptions(
 218             image_opts=containers.ImageOptions(
 219                 dockerfile_dir=dockerfile_dir,
 220                 tag=image_name,
 221                 nocache=False),
 222             port=self._default_port,
 223             port_bindings=port_bindings,
 224             environment=environment,
 225             volumes={
 226                 external_logs_path: {'bind': '/var/log/app_engine'}
 227             },
 228             name=container_name
 229         ))
 230
 231     self._container.Start()
 232
 233     # Print the debug information before connecting to the container
 234     # as debugging might break the runtime during initialization, and
 235     # connecting the debugger is required to start processing requests.
 236     if debug_port:
 237       logging.info('To debug module {module} attach to {host}:{port}'.format(
 238           module=self._module_configuration.module_name,
 239           host=self.ContainerHost(),
 240           port=self.PortBinding(debug_port)))
 241
 242     self._proxy = http_proxy.HttpProxy(
 243         host=self._container.host, port=self._container.port,
 244         instance_died_unexpectedly=self._instance_died_unexpectedly,
 245         instance_logs_getter=self._get_instance_logs,
 246         error_handler_file=application_configuration.get_app_error_file(
 247             self._module_configuration))
 248     self._proxy.wait_for_connection()
 249
 250   def quit(self):
 251     """Kills running container and removes it."""
 252     self._container.Stop()
 253
 254   def PortBinding(self, port):
 255     """Get the host binding of a container port.
 256
 257     Args:
 258       port: Port inside container.
 259
 260     Returns:
 261       Port on the host system mapped to the given port inside of
 262           the container.
 263     """
 264     return self._container.PortBinding(port)
 265
 266   def ContainerHost(self):
 267     """Get the host IP address of the container.
 268
 269     Returns:
 270       IP address on the host system for accessing the container.
 271     """
 272     return self._container.host