python/google/appengine/tools/yaml_translator.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Performs XML-to-YAML translation.
  18
  19   TranslateXmlToYaml(): performs xml-to-yaml translation with
  20   string inputs and outputs
  21   AppYamlTranslator: Class that facilitates xml-to-yaml translation
  22 """
  23
  24 import os
  25 import re
  26
  27 from google.appengine.tools import app_engine_web_xml_parser as aewxp
  28 from google.appengine.tools import handler_generator
  29 from google.appengine.tools import web_xml_parser
  30 from google.appengine.tools.app_engine_web_xml_parser import AppEngineConfigException
  31
  32
  33 NO_API_VERSION = 'none'
  34
  35
  36 def TranslateXmlToYaml(app_engine_web_xml_str,
  37                        web_xml_str,
  38                        has_jsps):
  39   """Does xml-string to yaml-string translation, given each separate file text.
  40
  41   Processes each xml string into an object representing the xml,
  42   and passes these to the translator.
  43
  44   Args:
  45     app_engine_web_xml_str: text from app_engine_web.xml
  46     web_xml_str: text from web.xml
  47     has_jsps: true if the app has any *.jsp files
  48
  49   Returns:
  50     The full text of the app.yaml generated from the xml files.
  51
  52   Raises:
  53     AppEngineConfigException: raised in processing stage for illegal XML.
  54   """
  55   aewx_parser = aewxp.AppEngineWebXmlParser()
  56   web_parser = web_xml_parser.WebXmlParser()
  57   app_engine_web_xml = aewx_parser.ProcessXml(app_engine_web_xml_str)
  58   web_xml = web_parser.ProcessXml(web_xml_str, has_jsps)
  59   translator = AppYamlTranslator(app_engine_web_xml, web_xml, [], '1.0')
  60   return translator.GetYaml()
  61
  62
  63 def TranslateXmlToYamlForDevAppServer(app_engine_web_xml_str,
  64                                       web_xml_str,
  65                                       has_jsps,
  66                                       war_root):
  67   """Does xml-string to yaml-string translation, given each separate file text.
  68
  69   Processes each xml string into an object representing the xml,
  70   and passes these to the translator. This variant is used in the Dev App Server
  71   context, where files are served directly from the input war directory, unlike
  72   the appcfg case where they are copied or linked into a parallel hierarchy.
  73   This means that there is no __static__ directory containing exactly the files
  74   that are supposed to be served statically.
  75
  76   Args:
  77     app_engine_web_xml_str: text from app_engine_web.xml
  78     web_xml_str: text from web.xml
  79     has_jsps: true if the app has any *.jsp files
  80     war_root: the path to the root directory of the war hierarchy
  81
  82   Returns:
  83     The full text of the app.yaml generated from the xml files.
  84
  85   Raises:
  86     AppEngineConfigException: raised in processing stage for illegal XML.
  87   """
  88   aewx_parser = aewxp.AppEngineWebXmlParser()
  89   web_parser = web_xml_parser.WebXmlParser()
  90   app_engine_web_xml = aewx_parser.ProcessXml(app_engine_web_xml_str)
  91   web_xml = web_parser.ProcessXml(web_xml_str, has_jsps)
  92   translator = AppYamlTranslatorForDevAppServer(
  93       app_engine_web_xml, web_xml, war_root)
  94   return translator.GetYaml()
  95
  96
  97 class AppYamlTranslator(object):
  98   """Object that contains relevant information for generating app.yaml.
  99
 100   Attributes:
 101     app_engine_web_xml: AppEngineWebXml object containing relevant information
 102       from appengine-web.xml
 103   """
 104
 105   def __init__(self,
 106                app_engine_web_xml,
 107                web_xml,
 108                static_files,
 109                api_version):
 110
 111     self.app_engine_web_xml = app_engine_web_xml
 112     self.web_xml = web_xml
 113     self.static_files = static_files
 114     self.api_version = api_version
 115
 116   def GetRuntime(self):
 117     return 'java7'
 118
 119   def GetYaml(self):
 120     """Returns full yaml text."""
 121     self.VerifyRequiredEntriesPresent()
 122     stmnt_list = self.TranslateBasicEntries()
 123     stmnt_list += self.TranslateAutomaticScaling()
 124     stmnt_list += self.TranslateBasicScaling()
 125     stmnt_list += self.TranslateManualScaling()
 126     stmnt_list += self.TranslatePrecompilationEnabled()
 127     stmnt_list += self.TranslateInboundServices()
 128     stmnt_list += self.TranslateAdminConsolePages()
 129     stmnt_list += self.TranslateApiConfig()
 130     stmnt_list += self.TranslatePagespeed()
 131     stmnt_list += self.TranslateEnvVariables()
 132     stmnt_list += self.TranslateVmSettings()
 133     stmnt_list += self.TranslateErrorHandlers()
 134     stmnt_list += self.TranslateApiVersion()
 135     stmnt_list += self.TranslateHandlers()
 136     return '\n'.join(stmnt_list) + '\n'
 137
 138   def SanitizeForYaml(self, the_string):
 139     return "'%s'" % the_string.replace("'", "''")
 140
 141   def TranslateBasicEntries(self):
 142     """Produces yaml for entries requiring little formatting."""
 143     basic_statements = []
 144
 145     for entry_name, field in [
 146         ('application', self.app_engine_web_xml.app_id),
 147         ('source_language', self.app_engine_web_xml.source_language),
 148         ('module', self.app_engine_web_xml.module),
 149         ('version', self.app_engine_web_xml.version_id)]:
 150       if field:
 151         basic_statements.append(
 152             '%s: %s' % (entry_name, self.SanitizeForYaml(field)))
 153     for entry_name, field in [
 154         ('runtime', self.GetRuntime()),
 155         ('vm', self.app_engine_web_xml.vm),
 156         ('threadsafe', self.app_engine_web_xml.threadsafe),
 157         ('instance_class', self.app_engine_web_xml.instance_class),
 158         ('auto_id_policy', self.app_engine_web_xml.auto_id_policy),
 159         ('code_lock', self.app_engine_web_xml.codelock)]:
 160       if field:
 161         basic_statements.append('%s: %s' % (entry_name, field))
 162     return basic_statements
 163
 164   def TranslateAutomaticScaling(self):
 165     """Translates automatic scaling settings to yaml."""
 166     if not self.app_engine_web_xml.automatic_scaling:
 167       return []
 168     statements = ['automatic_scaling:']
 169     for setting in ['min_pending_latency',
 170                     'max_pending_latency',
 171                     'min_idle_instances',
 172                     'max_idle_instances']:
 173       value = getattr(self.app_engine_web_xml.automatic_scaling, setting)
 174       if value:
 175         statements.append('  %s: %s' % (setting, value))
 176     return statements
 177
 178   def TranslateBasicScaling(self):
 179     if not self.app_engine_web_xml.basic_scaling:
 180       return []
 181     statements = ['basic_scaling:']
 182     statements.append('  max_instances: ' +
 183                       self.app_engine_web_xml.basic_scaling.max_instances)
 184     if self.app_engine_web_xml.basic_scaling.idle_timeout:
 185       statements.append('  idle_timeout: ' +
 186                         self.app_engine_web_xml.basic_scaling.idle_timeout)
 187     return statements
 188
 189   def TranslateManualScaling(self):
 190     if not self.app_engine_web_xml.manual_scaling:
 191       return []
 192
 193     statements = ['manual_scaling:']
 194     statements.append('  instances: ' +
 195                       self.app_engine_web_xml.manual_scaling.instances)
 196     return statements
 197
 198   def TranslatePrecompilationEnabled(self):
 199     if self.app_engine_web_xml.precompilation_enabled:
 200       return ['derived_file_type:', '- java_precompiled']
 201     return []
 202
 203   def TranslateAdminConsolePages(self):
 204     if not self.app_engine_web_xml.admin_console_pages:
 205       return []
 206     statements = ['admin_console:', '  pages:']
 207     for admin_console_page in self.app_engine_web_xml.admin_console_pages:
 208       statements.append('  - name: %s' % admin_console_page.name)
 209       statements.append('    url: %s' % admin_console_page.url)
 210     return statements
 211
 212   def TranslateApiConfig(self):
 213
 214     if not self.app_engine_web_xml.api_config:
 215       return []
 216     return ['api_config:', '  url: %s' % self.app_engine_web_xml.api_config.url,
 217             '  script: unused']
 218
 219   def TranslateApiVersion(self):
 220     return ['api_version: %s' % self.SanitizeForYaml(
 221         self.api_version or NO_API_VERSION)]
 222
 223   def TranslatePagespeed(self):
 224     """Translates pagespeed settings in appengine-web.xml to yaml."""
 225     pagespeed = self.app_engine_web_xml.pagespeed
 226     if not pagespeed:
 227       return []
 228     statements = ['pagespeed:']
 229     for title, urls in [('domains_to_rewrite', pagespeed.domains_to_rewrite),
 230                         ('url_blacklist', pagespeed.url_blacklist),
 231                         ('enabled_rewriters', pagespeed.enabled_rewriters),
 232                         ('disabled_rewriters', pagespeed.disabled_rewriters)]:
 233       if urls:
 234         statements.append('  %s:' % title)
 235         statements += ['  - %s' % url for url in urls]
 236     return statements
 237
 238   def TranslateEnvVariables(self):
 239     if (not self.app_engine_web_xml.vm or
 240         not self.app_engine_web_xml.env_variables):
 241       return []
 242
 243     variables = self.app_engine_web_xml.env_variables
 244     statements = ['env_variables:']
 245     for name, value in sorted(variables.iteritems()):
 246       statements.append(
 247           '  %s: %s' % (
 248               self.SanitizeForYaml(name), self.SanitizeForYaml(value)))
 249     return statements
 250
 251   def TranslateVmSettings(self):
 252     """Translates VM settings in appengine-web.xml to yaml."""
 253     if not self.app_engine_web_xml.vm:
 254       return []
 255
 256     settings = self.app_engine_web_xml.vm_settings or {}
 257     settings['has_docker_image'] = 'True'
 258     statements = ['vm_settings:']
 259     for name in sorted(settings):
 260       statements.append(
 261           '  %s: %s' % (
 262               self.SanitizeForYaml(name), self.SanitizeForYaml(settings[name])))
 263     return statements
 264
 265   def TranslateVmHealthCheck(self):
 266     """Translates <vm-health-check> in appengine-web.xml to yaml."""
 267     vm_health_check = self.app_engine_web_xml.vm_health_check
 268     if not vm_health_check:
 269       return []
 270
 271     statements = ['vm_health_check:']
 272     for attr in ('enable_health_check', 'check_interval_sec', 'timeout_sec',
 273                  'unhealthy_threshold', 'healthy_threshold',
 274                  'restart_threshold', 'host'):
 275       value = getattr(vm_health_check, attr, None)
 276       if value is not None:
 277         statements.append('  %s: %s' % (attr, value))
 278     return statements
 279
 280   def TranslateInboundServices(self):
 281     services = self.app_engine_web_xml.inbound_services
 282     if not services:
 283       return []
 284
 285     statements = ['inbound_services:']
 286     for service in sorted(services):
 287       statements.append('- %s' % service)
 288     return statements
 289
 290   def TranslateErrorHandlers(self):
 291     """Translates error handlers specified in appengine-web.xml to yaml."""
 292     if not self.app_engine_web_xml.static_error_handlers:
 293       return []
 294     statements = ['error_handlers:']
 295     for error_handler in self.app_engine_web_xml.static_error_handlers:
 296
 297       path = self.ErrorHandlerPath(error_handler)
 298       statements.append('- file: %s' % path)
 299       if error_handler.code:
 300         statements.append('  error_code: %s' % error_handler.code)
 301       mime_type = self.web_xml.GetMimeTypeForPath(error_handler.name)
 302       if mime_type:
 303         statements.append('  mime_type: %s' % mime_type)
 304
 305     return statements
 306
 307   def ErrorHandlerPath(self, error_handler):
 308     """Returns the relative path name for the given error handler.
 309
 310     Args:
 311       error_handler: an app_engine_web_xml.ErrorHandler.
 312
 313     Returns:
 314       the relative path name for the handler.
 315
 316     Raises:
 317       AppEngineConfigException: if the named file is not an existing static
 318         file.
 319     """
 320     name = error_handler.name
 321     if not name.startswith('/'):
 322       name = '/' + name
 323     path = '__static__' + name
 324     if path not in self.static_files:
 325       raise AppEngineConfigException(
 326           'No static file found for error handler: %s, out of %s' %
 327           (name, self.static_files))
 328     return path
 329
 330   def TranslateHandlers(self):
 331     return handler_generator.GenerateYamlHandlersList(
 332         self.app_engine_web_xml,
 333         self.web_xml,
 334         self.static_files)
 335
 336   def VerifyRequiredEntriesPresent(self):
 337     required = {
 338         'runtime': self.GetRuntime(),
 339         'threadsafe': self.app_engine_web_xml.threadsafe_value_provided,
 340     }
 341     missing = [field for (field, value) in required.items() if not value]
 342     if missing:
 343       raise AppEngineConfigException('Missing required fields: %s' %
 344                                      ', '.join(missing))
 345
 346
 347 def _XmlPatternToRegEx(xml_pattern):
 348   r"""Translates an appengine-web.xml pattern into a regular expression.
 349
 350   Specially, this applies to the patterns that appear in the <include> and
 351   <exclude> elements inside <static-files>. They look like '/**.png' or
 352   '/stylesheets/*.css', and are translated into expressions like
 353   '^/.*\.png$' or '^/stylesheets/.*\.css$'.
 354
 355   Args:
 356     xml_pattern: a string like '/**.png'
 357
 358   Returns:
 359     a compiled regular expression like re.compile('^/.*\.png$').
 360   """
 361   result = ['^']
 362   while xml_pattern:
 363     if xml_pattern.startswith('**'):
 364       result.append(r'.*')
 365       xml_pattern = xml_pattern[1:]
 366     elif xml_pattern.startswith('*'):
 367       result.append(r'[^/]*')
 368     elif xml_pattern.startswith('/'):
 369
 370
 371       result.append('/')
 372     else:
 373       result.append(re.escape(xml_pattern[0]))
 374     xml_pattern = xml_pattern[1:]
 375   result.append('$')
 376   return re.compile(''.join(result))
 377
 378
 379 class AppYamlTranslatorForDevAppServer(AppYamlTranslator):
 380   """Subclass of AppYamlTranslator specialized for the Dev App Server case.
 381
 382   The key difference is that static files are served directly from the war
 383   directory, which means that the app.yaml patterns we define must cover
 384   exactly those files in that directory hierarchy that are supposed to be static
 385   while not covering any files that are not supposed to be static.
 386
 387   Attributes:
 388     war_root: the root directory of the war hierarchy.
 389     static_urls: a list of two-item tuples where the first item is a URL that
 390       should be served statically and the second item corresponds to the
 391       <include> element that caused that URL to be included.
 392   """
 393
 394   def __init__(self,
 395                app_engine_web_xml,
 396                web_xml,
 397                war_root):
 398     super(AppYamlTranslatorForDevAppServer, self).__init__(
 399         app_engine_web_xml, web_xml, [], '1.0')
 400     self.war_root = war_root
 401     self.static_urls = self.IncludedStaticUrls()
 402
 403   def IncludedStaticUrls(self):
 404     """Returns the URLs that should be resolved statically for this app.
 405
 406     The result includes a URL for every file in the war hierarchy that is
 407     covered by one of the <include> elements for <static-files> and not covered
 408     by any of the <exclude> elements.
 409
 410     Returns:
 411       a list of two-item tuples where the first item is a URL that should be
 412       served statically and the second item corresponds to the <include>
 413       element that caused that URL to be included.
 414     """
 415
 416
 417
 418
 419
 420
 421     includes = self.app_engine_web_xml.static_file_includes
 422     if not includes:
 423
 424
 425
 426       includes = [aewxp.StaticFileInclude('**', None, {})]
 427     excludes = self.app_engine_web_xml.static_file_excludes
 428     files = os.listdir(self.war_root)
 429     web_inf_name = os.path.normcase('WEB-INF')
 430     files = [f for f in files if os.path.normcase(f) != web_inf_name]
 431     static_urls = []
 432     includes_and_res = [(include, _XmlPatternToRegEx(include.pattern))
 433                         for include in includes]
 434     exclude_res = [_XmlPatternToRegEx(exclude) for exclude in excludes]
 435     self.ComputeIncludedStaticUrls(
 436         static_urls, self.war_root, '/', files, includes_and_res, exclude_res)
 437     return static_urls
 438
 439
 440
 441
 442   def ComputeIncludedStaticUrls(
 443       self,
 444       static_urls, dirpath, url_prefix, files, includes_and_res, exclude_res):
 445     """Compute the URLs that should be resolved statically.
 446
 447     This recursive method is called for the war directory and every
 448     subdirectory except the top-level WEB-INF directory. If we have arrived
 449     at the directory <war-root>/foo/bar then dirpath will be <war-root>/foo/bar
 450     and url_prefix will be /foo/bar.
 451
 452     Args:
 453       static_urls: a list to be filled with the result, two-item tuples where
 454         the first item is a URL and the second is a parsed <include> element.
 455       dirpath: the path to the directory inside the war hierarchy that we have
 456         reached at this point in the recursion.
 457       url_prefix: the URL prefix that we have reached at this point in the
 458         recursion.
 459       files: the contents of the dirpath directory, minus the WEB-INF directory
 460         if dirpath is the war directory itself.
 461       includes_and_res: a list of two-item tuples where the first item is a
 462         parsed <include> element and the second item is a compiled regular
 463         expression corresponding to the path= pattern from that element.
 464       exclude_res: a list of compiled regular expressions corresponding to the
 465         path= patterns from <exclude> elements.
 466     """
 467     for f in files:
 468       path = os.path.join(dirpath, f)
 469       if os.path.isfile(path):
 470         url = url_prefix + f
 471         if not any(exclude_re.search(url) for exclude_re in exclude_res):
 472           for include, include_re in includes_and_res:
 473             if include_re.search(url):
 474               static_urls.append((url, include))
 475               break
 476       else:
 477         self.ComputeIncludedStaticUrls(
 478             static_urls, path, url_prefix + f + '/', os.listdir(path),
 479             includes_and_res, exclude_res)
 480
 481   def TranslateHandlers(self):
 482     return handler_generator.GenerateYamlHandlersListForDevAppServer(
 483         self.app_engine_web_xml,
 484         self.web_xml,
 485         self.static_urls)
 486
 487   def ErrorHandlerPath(self, error_handler):
 488     name = error_handler.name
 489     if name.startswith('/'):
 490       name = name[1:]
 491     if name not in self.static_files:
 492       raise AppEngineConfigException(
 493           'No static file found for error handler: %s, out of %s' %
 494           (name, self.static_files))
 495     return name