python/google/appengine/tools/devappserver2/vm_runtime_proxy.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Manages a VM Runtime process running inside of a docker container.
  18 """
  19
  20 import logging
  21 import socket
  22
  23 import google
  24 import docker
  25
  26 from google.appengine.tools.devappserver2 import application_configuration
  27 from google.appengine.tools.devappserver2 import http_proxy
  28 from google.appengine.tools.devappserver2 import instance
  29 from google.appengine.tools.docker import containers
  30
  31
  32 class VMRuntimeProxy(instance.RuntimeProxy):
  33   """Manages a VM Runtime process running inside of a docker container"""
  34
  35   def __init__(self, docker_client, runtime_config_getter,
  36                module_configuration):
  37     """Initializer for VMRuntimeProxy.
  38
  39     Args:
  40       docker_client: docker.Client object to communicate with Docker daemon.
  41       runtime_config_getter: A function that can be called without arguments
  42           and returns the runtime_config_pb2.Config containing the configuration
  43           for the runtime.
  44       module_configuration: An application_configuration.ModuleConfiguration
  45           instance respresenting the configuration of the module that owns the
  46           runtime.
  47     """
  48     super(VMRuntimeProxy, self).__init__()
  49
  50     self._runtime_config_getter = runtime_config_getter
  51     self._module_configuration = module_configuration
  52     self._docker_client = docker_client
  53     self._container = None
  54     self._proxy = None
  55
  56   def handle(self, environ, start_response, url_map, match, request_id,
  57              request_type):
  58     """Serves this request by forwarding it to application instance
  59     via HttpProxy.
  60
  61     Args:
  62       environ: An environ dict for the request as defined in PEP-333.
  63       start_response: A function with semantics defined in PEP-333.
  64       url_map: An appinfo.URLMap instance containing the configuration for the
  65           handler matching this request.
  66       match: A re.MatchObject containing the result of the matched URL pattern.
  67       request_id: A unique string id associated with the request.
  68       request_type: The type of the request. See instance.*_REQUEST module
  69           constants.
  70
  71     Yields:
  72       A sequence of strings containing the body of the HTTP response.
  73     """
  74     return self._proxy.handle(environ, start_response, url_map, match,
  75                               request_id, request_type)
  76
  77   def _get_instance_logs(self):
  78     # TODO: Handle docker container's logs
  79     return ''
  80
  81   def _instance_died_unexpectedly(self):
  82     # TODO: Check if container is still up and running
  83     return False
  84
  85   def start(self):
  86     runtime_config = self._runtime_config_getter()
  87
  88     # api_host set to 'localhost' won't be accessible from a docker container
  89     # because container will have it's own 'localhost'.
  90     # TODO: this works only when /etc/hosts is configured properly.
  91     api_host = socket.gethostbyname(socket.gethostname()) if (
  92         runtime_config.api_host == '0.0.0.0') else runtime_config.api_host
  93
  94     # Must be HTTP_PORT from apphosting/ext/vmruntime/vmservice.py
  95     # TODO: update apphosting/ext/vmruntime/vmservice.py to use
  96     # env var set here.
  97     PORT = 8080
  98
  99     self._container = containers.Container(
 100         self._docker_client,
 101         containers.ContainerOptions(
 102             image_opts=containers.ImageOptions(
 103                 dockerfile_dir=self._module_configuration.application_root,
 104                 tag='vm.%(RUNTIME)s.%(APP_ID)s.%(MODULE)s.%(VERSION)s' % {
 105                     'APP_ID': self._module_configuration.application,
 106                     'MODULE': self._module_configuration.module_name,
 107                     'RUNTIME': self._module_configuration.effective_runtime,
 108                     'VERSION': self._module_configuration.major_version},
 109                 nocache=False,
 110                 image_id=None),
 111             port=PORT,
 112             environment={
 113                 'API_HOST': api_host,
 114                 'API_PORT': runtime_config.api_port,
 115                 'GAE_LONG_APP_ID':
 116                     self._module_configuration.application_external_name,
 117                 'GAE_PARTITION': self._module_configuration.partition,
 118                 'GAE_MODULE_NAME': self._module_configuration.module_name,
 119                 'GAE_MODULE_VERSION': self._module_configuration.major_version,
 120                 'GAE_MINOR_VERSION': self._module_configuration.minor_version,
 121                 'GAE_MODULE_INSTANCE': runtime_config.instance_id},
 122             volumes={'/var/log/app_engine/app': '/var/log/app_engine/app:rw'}))
 123
 124     self._container.Start()
 125
 126     self._proxy = http_proxy.HttpProxy(
 127         host=self._container.host, port=self._container.port,
 128         instance_died_unexpectedly=self._instance_died_unexpectedly,
 129         instance_logs_getter=self._get_instance_logs,
 130         error_handler_file=application_configuration.get_app_error_file(
 131             self._module_configuration))
 132
 133   def quit(self):
 134     """Kills running container and removes it."""
 135     self._container.Stop()
 136
 137
 138 class VMRuntimeInstanceFactory(instance.InstanceFactory):
 139   """A factory that creates new VM runtime Instances."""
 140
 141   SUPPORTS_INTERACTIVE_REQUESTS = True
 142   FILE_CHANGE_INSTANCE_RESTART_POLICY = instance.ALWAYS
 143
 144   # Timeout of HTTP request from docker-py client to docker daemon, in seconds.
 145   DOCKER_D_REQUEST_TIMEOUT = 60
 146
 147   def __init__(self, request_data, runtime_config_getter, module_configuration):
 148     """Initializer for VMRuntimeInstanceFactory.
 149
 150     Args:
 151       request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
 152           with request information for use by API stubs.
 153       runtime_config_getter: A function that can be called without arguments
 154           and returns the runtime_config_pb2.Config containing the configuration
 155           for the runtime.
 156       module_configuration: An application_configuration.ModuleConfiguration
 157           instance representing the configuration of the module that owns the
 158           runtime.
 159     """
 160     assert runtime_config_getter().vm_config.HasField('docker_daemon_url'), (
 161         'VM runtime requires docker_daemon_url to be specified')
 162     super(VMRuntimeInstanceFactory, self).__init__(
 163         request_data,
 164         8 if runtime_config_getter().threadsafe else 1, 10)
 165     self._runtime_config_getter = runtime_config_getter
 166     self._module_configuration = module_configuration
 167     docker_daemon_url = runtime_config_getter().vm_config.docker_daemon_url
 168     self._docker_client = docker.Client(base_url=docker_daemon_url,
 169                                         version='1.6',
 170                                         timeout=self.DOCKER_D_REQUEST_TIMEOUT)
 171     if not self._docker_client:
 172       logging.error('Couldn\'t connect to docker daemon on %s' %
 173                     docker_daemon_url)
 174
 175   def new_instance(self, instance_id, expect_ready_request=False):
 176     """Create and return a new Instance.
 177
 178     Args:
 179       instance_id: A string or integer representing the unique (per module) id
 180           of the instance.
 181       expect_ready_request: If True then the instance will be sent a special
 182           request (i.e. /_ah/warmup or /_ah/start) before it can handle external
 183           requests.
 184
 185     Returns:
 186       The newly created instance.Instance.
 187     """
 188
 189     def runtime_config_getter():
 190       runtime_config = self._runtime_config_getter()
 191       runtime_config.instance_id = str(instance_id)
 192       return runtime_config
 193
 194     proxy = VMRuntimeProxy(self._docker_client,
 195                            runtime_config_getter,
 196                            self._module_configuration)
 197     return instance.Instance(self.request_data,
 198                              instance_id,
 199                              proxy,
 200                              self.max_concurrent_requests,
 201                              self.max_background_threads,
 202                              expect_ready_request)