python/google/appengine/tools/devappserver2/vm_runtime_proxy_go.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Manages a Go VM Runtime process running inside of a docker container.
  18 """
  19
  20 import logging
  21 import os
  22 import shutil
  23 import tempfile
  24
  25 import google
  26
  27 from google.appengine.tools.devappserver2 import go_application
  28 from google.appengine.tools.devappserver2 import instance
  29 from google.appengine.tools.devappserver2 import vm_runtime_proxy
  30
  31 DEBUG_PORT = 5858
  32 VM_SERVICE_PORT = 8181
  33 # TODO: Remove this when classic Go SDK is gone.
  34 DEFAULT_DOCKER_FILE = """FROM google/appengine-go
  35
  36 RUN apt-get install --no-install-recommends -y -q \
  37     curl build-essential git mercurial bzr
  38 RUN mkdir /goroot && curl https://storage.googleapis.com/golang/go1.2.2.linux-amd64.tar.gz | tar xvzf - -C /goroot --strip-components=1
  39 RUN mkdir /gopath
  40
  41 ENV GOROOT /goroot
  42 ENV GOPATH /gopath
  43 ENV PATH $PATH:$GOROOT/bin:$GOPATH/bin
  44
  45 # TODO: Remove next line once google/appengine-go image updates.
  46 WORKDIR /app
  47
  48 ADD . /app
  49 RUN /bin/bash /app/_ah/build.sh
  50 """
  51
  52 # Where to look for go-app-builder, which is needed for copying
  53 # into the Docker image for building the Go App Engine app.
  54 # There is no need to add '.exe' here because it is always a Linux executable.
  55 _GO_APP_BUILDER = os.path.join(
  56     go_application.GOROOT, 'pkg', 'tool', 'docker-gab')
  57
  58
  59 class GoVMRuntimeProxy(instance.RuntimeProxy):
  60   """Manages a Go VM Runtime process running inside of a docker container.
  61
  62   The Go VM Runtime forwards all requests to the Go application instance.
  63   """
  64
  65   def __init__(self, docker_client, runtime_config_getter,
  66                module_configuration):
  67     """Initializer for VMRuntimeProxy.
  68
  69     Args:
  70       docker_client: docker.Client object to communicate with Docker daemon.
  71       runtime_config_getter: A function that can be called without arguments
  72           and returns the runtime_config_pb2.Config containing the configuration
  73           for the runtime.
  74       module_configuration: An application_configuration.ModuleConfiguration
  75           instance respresenting the configuration of the module that owns the
  76           runtime.
  77     """
  78     super(GoVMRuntimeProxy, self).__init__()
  79     self._runtime_config_getter = runtime_config_getter
  80     self._module_configuration = module_configuration
  81     port_bindings = {
  82         DEBUG_PORT: None,
  83         VM_SERVICE_PORT: None,
  84     }
  85     self._vm_runtime_proxy = vm_runtime_proxy.VMRuntimeProxy(
  86         docker_client=docker_client,
  87         runtime_config_getter=runtime_config_getter,
  88         module_configuration=module_configuration,
  89         port_bindings=port_bindings)
  90
  91   def handle(self, environ, start_response, url_map, match, request_id,
  92              request_type):
  93     """Handle request to Go runtime.
  94
  95     Serves this request by forwarding to the Go application instance via
  96     HttpProxy.
  97
  98     Args:
  99       environ: An environ dict for the request as defined in PEP-333.
 100       start_response: A function with semantics defined in PEP-333.
 101       url_map: An appinfo.URLMap instance containing the configuration for the
 102           handler matching this request.
 103       match: A re.MatchObject containing the result of the matched URL pattern.
 104       request_id: A unique string id associated with the request.
 105       request_type: The type of the request. See instance.*_REQUEST module
 106           constants.
 107
 108     Yields:
 109       A sequence of strings containing the body of the HTTP response.
 110     """
 111
 112     it = self._vm_runtime_proxy.handle(environ, start_response, url_map,
 113                                        match, request_id, request_type)
 114     for data in it:
 115       yield data
 116
 117   def start(self):
 118     logging.info('Starting Go VM Deployment process')
 119
 120     try:
 121       application_dir = os.path.abspath(
 122           self._module_configuration.application_root)
 123
 124       with TempDir('go_deployment_dir') as dst_deployment_dir:
 125         build_go_docker_image_source(
 126             application_dir, dst_deployment_dir, _GO_APP_BUILDER,
 127             self._module_configuration.nobuild_files,
 128             self._module_configuration.skip_files)
 129
 130         self._vm_runtime_proxy.start(dockerfile_dir=dst_deployment_dir)
 131
 132       logging.info(
 133           'GoVM vmservice available at http://127.0.0.1:%s/ !',
 134           self._vm_runtime_proxy.PortBinding(VM_SERVICE_PORT))
 135
 136     except Exception as e:
 137       logging.info('Go VM Deployment process failed: %s', str(e))
 138       raise
 139
 140   def quit(self):
 141     self._vm_runtime_proxy.quit()
 142
 143
 144 def _write_dockerfile(dst_dir):
 145   """Writes Dockerfile to named directory if one does not exist.
 146
 147   Args:
 148     dst_dir: string name of destination directory.
 149   """
 150   dst_dockerfile = os.path.join(dst_dir, 'Dockerfile')
 151   if not os.path.exists(dst_dockerfile):
 152     with open(dst_dockerfile, 'w') as fd:
 153       fd.write(DEFAULT_DOCKER_FILE)
 154
 155
 156 class TempDir(object):
 157   """Creates a temporary directory."""
 158
 159   def __init__(self, prefix=''):
 160     self._temp_dir = None
 161     self._prefix = prefix
 162
 163   def __enter__(self):
 164     self._temp_dir = tempfile.mkdtemp(self._prefix)
 165     return self._temp_dir
 166
 167   def __exit__(self, *_):
 168     shutil.rmtree(self._temp_dir, ignore_errors=True)
 169
 170
 171 def _copytree(src, dst, skip_files, symlinks=False):
 172   """Copies src tree to dst (except those matching skip_files).
 173
 174   Args:
 175     src: string name of source directory to copy from.
 176     dst: string name of destination directory to copy to.
 177     skip_files: RegexStr of files to skip from appinfo.py.
 178     symlinks: optional bool determines if symbolic links are followed.
 179   """
 180   # Ignore files that match the skip_files RegexStr.
 181   # TODO: skip_files expects the full path relative to the app root, so
 182   # this may need fixing.
 183   def ignored_files(unused_dir, filenames):
 184     return [filename for filename in filenames if skip_files.match(filename)]
 185
 186   for item in os.listdir(src):
 187     s = os.path.join(src, item)
 188     if skip_files.match(item):
 189       logging.info('skipping file %s', s)
 190       continue
 191     d = os.path.join(dst, item)
 192     if os.path.isdir(s):
 193       shutil.copytree(s, d, symlinks, ignore=ignored_files)
 194     else:
 195       shutil.copy2(s, d)
 196
 197
 198 def build_go_docker_image_source(
 199     application_dir, dst_deployment_dir, go_app_builder,
 200     nobuild_files, skip_files):
 201   """Builds the Docker image source in preparation for building.
 202
 203   Steps:
 204     copy the application to dst_deployment_dir (follow symlinks)
 205     copy used parts of $GOPATH to dst_deployment_dir
 206     copy or create a Dockerfile in dst_deployment_dir
 207
 208   Args:
 209     application_dir: string pathname of application directory.
 210     dst_deployment_dir: string pathname of temporary deployment directory.
 211     go_app_builder: string pathname of docker-gab executable.
 212     nobuild_files: regexp identifying which files to not build.
 213     skip_files: regexp identifying which files to omit from app.
 214   """
 215   try:
 216     _copytree(application_dir, dst_deployment_dir, skip_files)
 217   except shutil.Error as e:
 218     logging.error('Error copying tree: %s', e)
 219     for src, unused_dst, unused_error in e.args[0]:
 220       if os.path.islink(src):
 221         linkto = os.readlink(src)
 222         if not os.path.exists(linkto):
 223           logging.error('Dangling symlink in Go project. '
 224                         'Path %s links to %s', src, os.readlink(src))
 225     raise
 226   except OSError as e:
 227     logging.error('Failed to copy dir: %s', e.strerror)
 228     raise
 229
 230   extras = go_application.get_app_extras_for_vm(
 231       application_dir, nobuild_files, skip_files)
 232   for dest, src in extras:
 233     try:
 234       dest = os.path.join(dst_deployment_dir, dest)
 235       dirname = os.path.dirname(dest)
 236       if not os.path.exists(dirname):
 237         os.makedirs(dirname)
 238       shutil.copy(src, dest)
 239     except OSError as e:
 240       logging.error('Failed to copy %s to %s', src, dest)
 241       raise
 242
 243   # Make the _ah subdirectory for the app engine tools.
 244   ah_dir = os.path.join(dst_deployment_dir, '_ah')
 245   try:
 246     os.mkdir(ah_dir)
 247   except OSError as e:
 248     logging.error('Failed to create %s: %s', ah_dir, e.strerror)
 249     raise
 250
 251   # Copy gab.
 252   try:
 253     gab_dest = os.path.join(ah_dir, 'gab')
 254     shutil.copy(go_app_builder, gab_dest)
 255   except OSError as e:
 256     logging.error('Failed to copy %s to %s', go_app_builder, gab_dest)
 257     raise
 258
 259   # Write build script.
 260   gab_args = [
 261       '/app/_ah/gab',
 262       '-app_base', '/app',
 263       '-arch', '6',
 264       '-dynamic',
 265       '-goroot', '/goroot',
 266       '-nobuild_files', '^' + str(nobuild_files),
 267       '-unsafe',
 268       '-binary_name', '_ah_exe',
 269       '-work_dir', '/tmp/work',
 270       '-vm',
 271   ]
 272   gab_args.extend(go_application.list_go_files(
 273       application_dir, nobuild_files, skip_files))
 274   gab_args.extend([x[0] for x in extras])
 275   dst_build = os.path.join(ah_dir, 'build.sh')
 276   lines = [
 277       '#!/bin/bash',
 278       'set -e',
 279       'mkdir -p /tmp/work',
 280       'chmod a+x /app/_ah/gab',
 281       # Without this line, Windows errors "text file busy".
 282       'shasum /app/_ah/gab',
 283       ' '.join(gab_args),
 284       'mv /tmp/work/_ah_exe /app/_ah/exe',
 285       'rm -rf /tmp/work',
 286       'echo Done.',
 287   ]
 288   with open(dst_build, 'wb') as fd:
 289     fd.write('\n'.join(lines) + '\n')
 290   os.chmod(dst_build, 0777)
 291
 292   # TODO: Remove this when classic Go SDK is gone.
 293   # Write default Dockerfile if none found.
 294   _write_dockerfile(dst_deployment_dir)
 295   # Also write the default Dockerfile if not found in the app dir.
 296   _write_dockerfile(application_dir)