python/google/appengine/tools/yaml_translator.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Performs XML-to-YAML translation.
  18
  19   TranslateXmlToYaml(): performs xml-to-yaml translation with
  20   string inputs and outputs
  21   AppYamlTranslator: Class that facilitates xml-to-yaml translation
  22 """
  23
  24 import os
  25 import re
  26
  27 from google.appengine.tools import app_engine_web_xml_parser as aewxp
  28 from google.appengine.tools import handler_generator
  29 from google.appengine.tools import web_xml_parser
  30 from google.appengine.tools.app_engine_web_xml_parser import AppEngineConfigException
  31
  32
  33 NO_API_VERSION = 'none'
  34
  35
  36 def TranslateXmlToYaml(app_engine_web_xml_str,
  37                        web_xml_str,
  38                        has_jsps):
  39   """Does xml-string to yaml-string translation, given each separate file text.
  40
  41   Processes each xml string into an object representing the xml,
  42   and passes these to the translator.
  43
  44   Args:
  45     app_engine_web_xml_str: text from app_engine_web.xml
  46     web_xml_str: text from web.xml
  47     has_jsps: true if the app has any *.jsp files
  48
  49   Returns:
  50     The full text of the app.yaml generated from the xml files.
  51
  52   Raises:
  53     AppEngineConfigException: raised in processing stage for illegal XML.
  54   """
  55   aewx_parser = aewxp.AppEngineWebXmlParser()
  56   web_parser = web_xml_parser.WebXmlParser()
  57   app_engine_web_xml = aewx_parser.ProcessXml(app_engine_web_xml_str)
  58   web_xml = web_parser.ProcessXml(web_xml_str, has_jsps)
  59   translator = AppYamlTranslator(app_engine_web_xml, web_xml, [], '1.0')
  60   return translator.GetYaml()
  61
  62
  63 def TranslateXmlToYamlForDevAppServer(app_engine_web_xml_str,
  64                                       web_xml_str,
  65                                       has_jsps,
  66                                       war_root):
  67   """Does xml-string to yaml-string translation, given each separate file text.
  68
  69   Processes each xml string into an object representing the xml,
  70   and passes these to the translator. This variant is used in the Dev App Server
  71   context, where files are served directly from the input war directory, unlike
  72   the appcfg case where they are copied or linked into a parallel hierarchy.
  73   This means that there is no __static__ directory containing exactly the files
  74   that are supposed to be served statically.
  75
  76   Args:
  77     app_engine_web_xml_str: text from app_engine_web.xml
  78     web_xml_str: text from web.xml
  79     has_jsps: true if the app has any *.jsp files
  80     war_root: the path to the root directory of the war hierarchy
  81
  82   Returns:
  83     The full text of the app.yaml generated from the xml files.
  84
  85   Raises:
  86     AppEngineConfigException: raised in processing stage for illegal XML.
  87   """
  88   aewx_parser = aewxp.AppEngineWebXmlParser()
  89   web_parser = web_xml_parser.WebXmlParser()
  90   app_engine_web_xml = aewx_parser.ProcessXml(app_engine_web_xml_str)
  91   web_xml = web_parser.ProcessXml(web_xml_str, has_jsps)
  92   translator = AppYamlTranslatorForDevAppServer(
  93       app_engine_web_xml, web_xml, war_root)
  94   return translator.GetYaml()
  95
  96
  97 class AppYamlTranslator(object):
  98   """Object that contains relevant information for generating app.yaml.
  99
 100   Attributes:
 101     app_engine_web_xml: AppEngineWebXml object containing relevant information
 102       from appengine-web.xml
 103   """
 104
 105   def __init__(self,
 106                app_engine_web_xml,
 107                web_xml,
 108                static_files,
 109                api_version):
 110
 111     self.app_engine_web_xml = app_engine_web_xml
 112     self.web_xml = web_xml
 113     self.static_files = static_files
 114     self.api_version = api_version
 115
 116   def GetRuntime(self):
 117     return 'java7'
 118
 119   def GetYaml(self):
 120     """Returns full yaml text."""
 121     self.VerifyRequiredEntriesPresent()
 122     stmnt_list = self.TranslateBasicEntries()
 123     stmnt_list += self.TranslateAutomaticScaling()
 124     stmnt_list += self.TranslateBasicScaling()
 125     stmnt_list += self.TranslateManualScaling()
 126     stmnt_list += self.TranslatePrecompilationEnabled()
 127     stmnt_list += self.TranslateInboundServices()
 128     stmnt_list += self.TranslateAdminConsolePages()
 129     stmnt_list += self.TranslateApiConfig()
 130     stmnt_list += self.TranslatePagespeed()
 131     stmnt_list += self.TranslateEnvVariables()
 132     stmnt_list += self.TranslateVmSettings()
 133     stmnt_list += self.TranslateErrorHandlers()
 134     stmnt_list += self.TranslateApiVersion()
 135     stmnt_list += self.TranslateHandlers()
 136     return '\n'.join(stmnt_list) + '\n'
 137
 138   def SanitizeForYaml(self, the_string):
 139     return "'%s'" % the_string.replace("'", "''")
 140
 141   def TranslateBasicEntries(self):
 142     """Produces yaml for entries requiring little formatting."""
 143     basic_statements = []
 144
 145     for entry_name, field in [
 146         ('application', self.app_engine_web_xml.app_id),
 147         ('source_language', self.app_engine_web_xml.source_language),
 148         ('module', self.app_engine_web_xml.module),
 149         ('version', self.app_engine_web_xml.version_id)]:
 150       if field:
 151         basic_statements.append(
 152             '%s: %s' % (entry_name, self.SanitizeForYaml(field)))
 153     for entry_name, field in [
 154         ('runtime', self.GetRuntime()),
 155         ('vm', self.app_engine_web_xml.vm),
 156         ('threadsafe', self.app_engine_web_xml.threadsafe),
 157         ('instance_class', self.app_engine_web_xml.instance_class),
 158         ('auto_id_policy', self.app_engine_web_xml.auto_id_policy),
 159         ('code_lock', self.app_engine_web_xml.codelock)]:
 160       if field:
 161         basic_statements.append('%s: %s' % (entry_name, field))
 162     return basic_statements
 163
 164   def TranslateAutomaticScaling(self):
 165     """Translates automatic scaling settings to yaml."""
 166     if not self.app_engine_web_xml.automatic_scaling:
 167       return []
 168     statements = ['automatic_scaling:']
 169     for setting in ['min_pending_latency',
 170                     'max_pending_latency',
 171                     'min_idle_instances',
 172                     'max_idle_instances']:
 173       value = getattr(self.app_engine_web_xml.automatic_scaling, setting)
 174       if value:
 175         statements.append('  %s: %s' % (setting, value))
 176     return statements
 177
 178   def TranslateBasicScaling(self):
 179     if not self.app_engine_web_xml.basic_scaling:
 180       return []
 181     statements = ['basic_scaling:']
 182     statements.append('  max_instances: ' +
 183                       self.app_engine_web_xml.basic_scaling.max_instances)
 184     if self.app_engine_web_xml.basic_scaling.idle_timeout:
 185       statements.append('  idle_timeout: ' +
 186                         self.app_engine_web_xml.basic_scaling.idle_timeout)
 187     return statements
 188
 189   def TranslateManualScaling(self):
 190     if not self.app_engine_web_xml.manual_scaling:
 191       return []
 192
 193     statements = ['manual_scaling:']
 194     statements.append('  instances: ' +
 195                       self.app_engine_web_xml.manual_scaling.instances)
 196     return statements
 197
 198   def TranslatePrecompilationEnabled(self):
 199     if self.app_engine_web_xml.precompilation_enabled:
 200       return ['derived_file_type:', '- java_precompiled']
 201     return []
 202
 203   def TranslateAdminConsolePages(self):
 204     if not self.app_engine_web_xml.admin_console_pages:
 205       return []
 206     statements = ['admin_console:', '  pages:']
 207     for admin_console_page in self.app_engine_web_xml.admin_console_pages:
 208       statements.append('  - name: %s' % admin_console_page.name)
 209       statements.append('    url: %s' % admin_console_page.url)
 210     return statements
 211
 212   def TranslateApiConfig(self):
 213
 214     if not self.app_engine_web_xml.api_config:
 215       return []
 216     return ['api_config:', '  url: %s' % self.app_engine_web_xml.api_config.url,
 217             '  script: unused']
 218
 219   def TranslateApiVersion(self):
 220     return ['api_version: %s' % self.SanitizeForYaml(
 221         self.api_version or NO_API_VERSION)]
 222
 223   def TranslatePagespeed(self):
 224     """Translates pagespeed settings in appengine-web.xml to yaml."""
 225     pagespeed = self.app_engine_web_xml.pagespeed
 226     if not pagespeed:
 227       return []
 228     statements = ['pagespeed:']
 229     for title, urls in [('domains_to_rewrite', pagespeed.domains_to_rewrite),
 230                         ('url_blacklist', pagespeed.url_blacklist),
 231                         ('enabled_rewriters', pagespeed.enabled_rewriters),
 232                         ('disabled_rewriters', pagespeed.disabled_rewriters)]:
 233       if urls:
 234         statements.append('  %s:' % title)
 235         statements += ['  - %s' % url for url in urls]
 236     return statements
 237
 238   def TranslateEnvVariables(self):
 239     if not self.app_engine_web_xml.env_variables:
 240       return []
 241
 242     variables = self.app_engine_web_xml.env_variables
 243     statements = ['env_variables:']
 244     for name, value in sorted(variables.iteritems()):
 245       statements.append(
 246           '  %s: %s' % (
 247               self.SanitizeForYaml(name), self.SanitizeForYaml(value)))
 248     return statements
 249
 250   def TranslateVmSettings(self):
 251     """Translates VM settings in appengine-web.xml to yaml."""
 252     if not self.app_engine_web_xml.vm:
 253       return []
 254
 255     settings = self.app_engine_web_xml.vm_settings or {}
 256     settings['has_docker_image'] = 'True'
 257     statements = ['vm_settings:']
 258     for name in sorted(settings):
 259       statements.append(
 260           '  %s: %s' % (
 261               self.SanitizeForYaml(name), self.SanitizeForYaml(settings[name])))
 262     return statements
 263
 264   def TranslateVmHealthCheck(self):
 265     """Translates <vm-health-check> in appengine-web.xml to yaml."""
 266     vm_health_check = self.app_engine_web_xml.vm_health_check
 267     if not vm_health_check:
 268       return []
 269
 270     statements = ['vm_health_check:']
 271     for attr in ('enable_health_check', 'check_interval_sec', 'timeout_sec',
 272                  'unhealthy_threshold', 'healthy_threshold',
 273                  'restart_threshold', 'host'):
 274       value = getattr(vm_health_check, attr, None)
 275       if value is not None:
 276         statements.append('  %s: %s' % (attr, value))
 277     return statements
 278
 279   def TranslateInboundServices(self):
 280     services = self.app_engine_web_xml.inbound_services
 281     if not services:
 282       return []
 283
 284     statements = ['inbound_services:']
 285     for service in sorted(services):
 286       statements.append('- %s' % service)
 287     return statements
 288
 289   def TranslateErrorHandlers(self):
 290     """Translates error handlers specified in appengine-web.xml to yaml."""
 291     if not self.app_engine_web_xml.static_error_handlers:
 292       return []
 293     statements = ['error_handlers:']
 294     for error_handler in self.app_engine_web_xml.static_error_handlers:
 295
 296       path = self.ErrorHandlerPath(error_handler)
 297       statements.append('- file: %s' % path)
 298       if error_handler.code:
 299         statements.append('  error_code: %s' % error_handler.code)
 300       mime_type = self.web_xml.GetMimeTypeForPath(error_handler.name)
 301       if mime_type:
 302         statements.append('  mime_type: %s' % mime_type)
 303
 304     return statements
 305
 306   def ErrorHandlerPath(self, error_handler):
 307     """Returns the relative path name for the given error handler.
 308
 309     Args:
 310       error_handler: an app_engine_web_xml.ErrorHandler.
 311
 312     Returns:
 313       the relative path name for the handler.
 314
 315     Raises:
 316       AppEngineConfigException: if the named file is not an existing static
 317         file.
 318     """
 319     name = error_handler.name
 320     if not name.startswith('/'):
 321       name = '/' + name
 322     path = '__static__' + name
 323     if path not in self.static_files:
 324       raise AppEngineConfigException(
 325           'No static file found for error handler: %s, out of %s' %
 326           (name, self.static_files))
 327     return path
 328
 329   def TranslateHandlers(self):
 330     return handler_generator.GenerateYamlHandlersList(
 331         self.app_engine_web_xml,
 332         self.web_xml,
 333         self.static_files)
 334
 335   def VerifyRequiredEntriesPresent(self):
 336     required = {
 337         'runtime': self.GetRuntime(),
 338         'threadsafe': self.app_engine_web_xml.threadsafe_value_provided,
 339     }
 340     missing = [field for (field, value) in required.items() if not value]
 341     if missing:
 342       raise AppEngineConfigException('Missing required fields: %s' %
 343                                      ', '.join(missing))
 344
 345
 346 def _XmlPatternToRegEx(xml_pattern):
 347   r"""Translates an appengine-web.xml pattern into a regular expression.
 348
 349   Specially, this applies to the patterns that appear in the <include> and
 350   <exclude> elements inside <static-files>. They look like '/**.png' or
 351   '/stylesheets/*.css', and are translated into expressions like
 352   '^/.*\.png$' or '^/stylesheets/.*\.css$'.
 353
 354   Args:
 355     xml_pattern: a string like '/**.png'
 356
 357   Returns:
 358     a compiled regular expression like re.compile('^/.*\.png$').
 359   """
 360   result = ['^']
 361   while xml_pattern:
 362     if xml_pattern.startswith('**'):
 363       result.append(r'.*')
 364       xml_pattern = xml_pattern[1:]
 365     elif xml_pattern.startswith('*'):
 366       result.append(r'[^/]*')
 367     elif xml_pattern.startswith('/'):
 368
 369
 370       result.append('/')
 371     else:
 372       result.append(re.escape(xml_pattern[0]))
 373     xml_pattern = xml_pattern[1:]
 374   result.append('$')
 375   return re.compile(''.join(result))
 376
 377
 378 class AppYamlTranslatorForDevAppServer(AppYamlTranslator):
 379   """Subclass of AppYamlTranslator specialized for the Dev App Server case.
 380
 381   The key difference is that static files are served directly from the war
 382   directory, which means that the app.yaml patterns we define must cover
 383   exactly those files in that directory hierarchy that are supposed to be static
 384   while not covering any files that are not supposed to be static.
 385
 386   Attributes:
 387     war_root: the root directory of the war hierarchy.
 388     static_urls: a list of two-item tuples where the first item is a URL that
 389       should be served statically and the second item corresponds to the
 390       <include> element that caused that URL to be included.
 391   """
 392
 393   def __init__(self,
 394                app_engine_web_xml,
 395                web_xml,
 396                war_root):
 397     super(AppYamlTranslatorForDevAppServer, self).__init__(
 398         app_engine_web_xml, web_xml, [], '1.0')
 399     self.war_root = war_root
 400     self.static_urls = self.IncludedStaticUrls()
 401
 402   def IncludedStaticUrls(self):
 403     """Returns the URLs that should be resolved statically for this app.
 404
 405     The result includes a URL for every file in the war hierarchy that is
 406     covered by one of the <include> elements for <static-files> and not covered
 407     by any of the <exclude> elements.
 408
 409     Returns:
 410       a list of two-item tuples where the first item is a URL that should be
 411       served statically and the second item corresponds to the <include>
 412       element that caused that URL to be included.
 413     """
 414
 415
 416
 417
 418
 419
 420     includes = self.app_engine_web_xml.static_file_includes
 421     if not includes:
 422
 423
 424
 425       includes = [aewxp.StaticFileInclude('**', None, {})]
 426     excludes = self.app_engine_web_xml.static_file_excludes
 427     files = os.listdir(self.war_root)
 428     web_inf_name = os.path.normcase('WEB-INF')
 429     files = [f for f in files if os.path.normcase(f) != web_inf_name]
 430     static_urls = []
 431     includes_and_res = [(include, _XmlPatternToRegEx(include.pattern))
 432                         for include in includes]
 433     exclude_res = [_XmlPatternToRegEx(exclude) for exclude in excludes]
 434     self.ComputeIncludedStaticUrls(
 435         static_urls, self.war_root, '/', files, includes_and_res, exclude_res)
 436     return static_urls
 437
 438
 439
 440
 441   def ComputeIncludedStaticUrls(
 442       self,
 443       static_urls, dirpath, url_prefix, files, includes_and_res, exclude_res):
 444     """Compute the URLs that should be resolved statically.
 445
 446     This recursive method is called for the war directory and every
 447     subdirectory except the top-level WEB-INF directory. If we have arrived
 448     at the directory <war-root>/foo/bar then dirpath will be <war-root>/foo/bar
 449     and url_prefix will be /foo/bar.
 450
 451     Args:
 452       static_urls: a list to be filled with the result, two-item tuples where
 453         the first item is a URL and the second is a parsed <include> element.
 454       dirpath: the path to the directory inside the war hierarchy that we have
 455         reached at this point in the recursion.
 456       url_prefix: the URL prefix that we have reached at this point in the
 457         recursion.
 458       files: the contents of the dirpath directory, minus the WEB-INF directory
 459         if dirpath is the war directory itself.
 460       includes_and_res: a list of two-item tuples where the first item is a
 461         parsed <include> element and the second item is a compiled regular
 462         expression corresponding to the path= pattern from that element.
 463       exclude_res: a list of compiled regular expressions corresponding to the
 464         path= patterns from <exclude> elements.
 465     """
 466     for f in files:
 467       path = os.path.join(dirpath, f)
 468       if os.path.isfile(path):
 469         url = url_prefix + f
 470         if not any(exclude_re.search(url) for exclude_re in exclude_res):
 471           for include, include_re in includes_and_res:
 472             if include_re.search(url):
 473               static_urls.append((url, include))
 474               break
 475       else:
 476         self.ComputeIncludedStaticUrls(
 477             static_urls, path, url_prefix + f + '/', os.listdir(path),
 478             includes_and_res, exclude_res)
 479
 480   def TranslateHandlers(self):
 481     return handler_generator.GenerateYamlHandlersListForDevAppServer(
 482         self.app_engine_web_xml,
 483         self.web_xml,
 484         self.static_urls)
 485
 486   def ErrorHandlerPath(self, error_handler):
 487     name = error_handler.name
 488     if name.startswith('/'):
 489       name = name[1:]
 490     if name not in self.static_files:
 491       raise AppEngineConfigException(
 492           'No static file found for error handler: %s, out of %s' %
 493           (name, self.static_files))
 494     return name