3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """Performs XML-to-YAML translation.
19 TranslateXmlToYaml(): performs xml-to-yaml translation with
20 string inputs and outputs
21 AppYamlTranslator: Class that facilitates xml-to-yaml translation
27 from google
.appengine
.tools
import app_engine_web_xml_parser
as aewxp
28 from google
.appengine
.tools
import handler_generator
29 from google
.appengine
.tools
import web_xml_parser
30 from google
.appengine
.tools
.app_engine_web_xml_parser
import AppEngineConfigException
33 NO_API_VERSION
= 'none'
36 def TranslateXmlToYaml(app_engine_web_xml_str
,
39 """Does xml-string to yaml-string translation, given each separate file text.
41 Processes each xml string into an object representing the xml,
42 and passes these to the translator.
45 app_engine_web_xml_str: text from app_engine_web.xml
46 web_xml_str: text from web.xml
47 has_jsps: true if the app has any *.jsp files
50 The full text of the app.yaml generated from the xml files.
53 AppEngineConfigException: raised in processing stage for illegal XML.
55 aewx_parser
= aewxp
.AppEngineWebXmlParser()
56 web_parser
= web_xml_parser
.WebXmlParser()
57 app_engine_web_xml
= aewx_parser
.ProcessXml(app_engine_web_xml_str
)
58 web_xml
= web_parser
.ProcessXml(web_xml_str
, has_jsps
)
59 translator
= AppYamlTranslator(app_engine_web_xml
, web_xml
, [], '1.0')
60 return translator
.GetYaml()
63 def TranslateXmlToYamlForDevAppServer(app_engine_web_xml_str
,
67 """Does xml-string to yaml-string translation, given each separate file text.
69 Processes each xml string into an object representing the xml,
70 and passes these to the translator. This variant is used in the Dev App Server
71 context, where files are served directly from the input war directory, unlike
72 the appcfg case where they are copied or linked into a parallel hierarchy.
73 This means that there is no __static__ directory containing exactly the files
74 that are supposed to be served statically.
77 app_engine_web_xml_str: text from app_engine_web.xml
78 web_xml_str: text from web.xml
79 has_jsps: true if the app has any *.jsp files
80 war_root: the path to the root directory of the war hierarchy
83 The full text of the app.yaml generated from the xml files.
86 AppEngineConfigException: raised in processing stage for illegal XML.
88 aewx_parser
= aewxp
.AppEngineWebXmlParser()
89 web_parser
= web_xml_parser
.WebXmlParser()
90 app_engine_web_xml
= aewx_parser
.ProcessXml(app_engine_web_xml_str
)
91 web_xml
= web_parser
.ProcessXml(web_xml_str
, has_jsps
)
92 translator
= AppYamlTranslatorForDevAppServer(
93 app_engine_web_xml
, web_xml
, war_root
)
94 return translator
.GetYaml()
97 class AppYamlTranslator(object):
98 """Object that contains relevant information for generating app.yaml.
101 app_engine_web_xml: AppEngineWebXml object containing relevant information
102 from appengine-web.xml
111 self
.app_engine_web_xml
= app_engine_web_xml
112 self
.web_xml
= web_xml
113 self
.static_files
= static_files
114 self
.api_version
= api_version
116 def GetRuntime(self
):
120 """Returns full yaml text."""
121 self
.VerifyRequiredEntriesPresent()
122 stmnt_list
= self
.TranslateBasicEntries()
123 stmnt_list
+= self
.TranslateAutomaticScaling()
124 stmnt_list
+= self
.TranslateBasicScaling()
125 stmnt_list
+= self
.TranslateManualScaling()
126 stmnt_list
+= self
.TranslatePrecompilationEnabled()
127 stmnt_list
+= self
.TranslateInboundServices()
128 stmnt_list
+= self
.TranslateAdminConsolePages()
129 stmnt_list
+= self
.TranslateApiConfig()
130 stmnt_list
+= self
.TranslatePagespeed()
131 stmnt_list
+= self
.TranslateEnvVariables()
132 stmnt_list
+= self
.TranslateVmSettings()
133 stmnt_list
+= self
.TranslateErrorHandlers()
134 stmnt_list
+= self
.TranslateApiVersion()
135 stmnt_list
+= self
.TranslateHandlers()
136 return '\n'.join(stmnt_list
) + '\n'
138 def SanitizeForYaml(self
, the_string
):
139 return "'%s'" % the_string
.replace("'", "''")
141 def TranslateBasicEntries(self
):
142 """Produces yaml for entries requiring little formatting."""
143 basic_statements
= []
145 for entry_name
, field
in [
146 ('application', self
.app_engine_web_xml
.app_id
),
147 ('source_language', self
.app_engine_web_xml
.source_language
),
148 ('module', self
.app_engine_web_xml
.module
),
149 ('version', self
.app_engine_web_xml
.version_id
)]:
151 basic_statements
.append(
152 '%s: %s' % (entry_name
, self
.SanitizeForYaml(field
)))
153 for entry_name
, field
in [
154 ('runtime', self
.GetRuntime()),
155 ('vm', self
.app_engine_web_xml
.vm
),
156 ('threadsafe', self
.app_engine_web_xml
.threadsafe
),
157 ('instance_class', self
.app_engine_web_xml
.instance_class
),
158 ('auto_id_policy', self
.app_engine_web_xml
.auto_id_policy
),
159 ('code_lock', self
.app_engine_web_xml
.codelock
)]:
161 basic_statements
.append('%s: %s' % (entry_name
, field
))
162 return basic_statements
164 def TranslateAutomaticScaling(self
):
165 """Translates automatic scaling settings to yaml."""
166 if not self
.app_engine_web_xml
.automatic_scaling
:
168 statements
= ['automatic_scaling:']
169 for setting
in ['min_pending_latency',
170 'max_pending_latency',
171 'min_idle_instances',
172 'max_idle_instances']:
173 value
= getattr(self
.app_engine_web_xml
.automatic_scaling
, setting
)
175 statements
.append(' %s: %s' % (setting
, value
))
178 def TranslateBasicScaling(self
):
179 if not self
.app_engine_web_xml
.basic_scaling
:
181 statements
= ['basic_scaling:']
182 statements
.append(' max_instances: ' +
183 self
.app_engine_web_xml
.basic_scaling
.max_instances
)
184 if self
.app_engine_web_xml
.basic_scaling
.idle_timeout
:
185 statements
.append(' idle_timeout: ' +
186 self
.app_engine_web_xml
.basic_scaling
.idle_timeout
)
189 def TranslateManualScaling(self
):
190 if not self
.app_engine_web_xml
.manual_scaling
:
193 statements
= ['manual_scaling:']
194 statements
.append(' instances: ' +
195 self
.app_engine_web_xml
.manual_scaling
.instances
)
198 def TranslatePrecompilationEnabled(self
):
199 if self
.app_engine_web_xml
.precompilation_enabled
:
200 return ['derived_file_type:', '- java_precompiled']
203 def TranslateAdminConsolePages(self
):
204 if not self
.app_engine_web_xml
.admin_console_pages
:
206 statements
= ['admin_console:', ' pages:']
207 for admin_console_page
in self
.app_engine_web_xml
.admin_console_pages
:
208 statements
.append(' - name: %s' % admin_console_page
.name
)
209 statements
.append(' url: %s' % admin_console_page
.url
)
212 def TranslateApiConfig(self
):
214 if not self
.app_engine_web_xml
.api_config
:
216 return ['api_config:', ' url: %s' % self
.app_engine_web_xml
.api_config
.url
,
219 def TranslateApiVersion(self
):
220 return ['api_version: %s' % self
.SanitizeForYaml(
221 self
.api_version
or NO_API_VERSION
)]
223 def TranslatePagespeed(self
):
224 """Translates pagespeed settings in appengine-web.xml to yaml."""
225 pagespeed
= self
.app_engine_web_xml
.pagespeed
228 statements
= ['pagespeed:']
229 for title
, urls
in [('domains_to_rewrite', pagespeed
.domains_to_rewrite
),
230 ('url_blacklist', pagespeed
.url_blacklist
),
231 ('enabled_rewriters', pagespeed
.enabled_rewriters
),
232 ('disabled_rewriters', pagespeed
.disabled_rewriters
)]:
234 statements
.append(' %s:' % title
)
235 statements
+= [' - %s' % url
for url
in urls
]
238 def TranslateEnvVariables(self
):
239 if not self
.app_engine_web_xml
.env_variables
:
242 variables
= self
.app_engine_web_xml
.env_variables
243 statements
= ['env_variables:']
244 for name
, value
in sorted(variables
.iteritems()):
247 self
.SanitizeForYaml(name
), self
.SanitizeForYaml(value
)))
250 def TranslateVmSettings(self
):
251 """Translates VM settings in appengine-web.xml to yaml."""
252 if not self
.app_engine_web_xml
.vm
:
255 settings
= self
.app_engine_web_xml
.vm_settings
or {}
256 settings
['has_docker_image'] = 'True'
257 statements
= ['vm_settings:']
258 for name
in sorted(settings
):
261 self
.SanitizeForYaml(name
), self
.SanitizeForYaml(settings
[name
])))
264 def TranslateVmHealthCheck(self
):
265 """Translates <vm-health-check> in appengine-web.xml to yaml."""
266 vm_health_check
= self
.app_engine_web_xml
.vm_health_check
267 if not vm_health_check
:
270 statements
= ['vm_health_check:']
271 for attr
in ('enable_health_check', 'check_interval_sec', 'timeout_sec',
272 'unhealthy_threshold', 'healthy_threshold',
273 'restart_threshold', 'host'):
274 value
= getattr(vm_health_check
, attr
, None)
275 if value
is not None:
276 statements
.append(' %s: %s' % (attr
, value
))
279 def TranslateInboundServices(self
):
280 services
= self
.app_engine_web_xml
.inbound_services
284 statements
= ['inbound_services:']
285 for service
in sorted(services
):
286 statements
.append('- %s' % service
)
289 def TranslateErrorHandlers(self
):
290 """Translates error handlers specified in appengine-web.xml to yaml."""
291 if not self
.app_engine_web_xml
.static_error_handlers
:
293 statements
= ['error_handlers:']
294 for error_handler
in self
.app_engine_web_xml
.static_error_handlers
:
296 path
= self
.ErrorHandlerPath(error_handler
)
297 statements
.append('- file: %s' % path
)
298 if error_handler
.code
:
299 statements
.append(' error_code: %s' % error_handler
.code
)
300 mime_type
= self
.web_xml
.GetMimeTypeForPath(error_handler
.name
)
302 statements
.append(' mime_type: %s' % mime_type
)
306 def ErrorHandlerPath(self
, error_handler
):
307 """Returns the relative path name for the given error handler.
310 error_handler: an app_engine_web_xml.ErrorHandler.
313 the relative path name for the handler.
316 AppEngineConfigException: if the named file is not an existing static
319 name
= error_handler
.name
320 if not name
.startswith('/'):
322 path
= '__static__' + name
323 if path
not in self
.static_files
:
324 raise AppEngineConfigException(
325 'No static file found for error handler: %s, out of %s' %
326 (name
, self
.static_files
))
329 def TranslateHandlers(self
):
330 return handler_generator
.GenerateYamlHandlersList(
331 self
.app_engine_web_xml
,
335 def VerifyRequiredEntriesPresent(self
):
337 'runtime': self
.GetRuntime(),
338 'threadsafe': self
.app_engine_web_xml
.threadsafe_value_provided
,
340 missing
= [field
for (field
, value
) in required
.items() if not value
]
342 raise AppEngineConfigException('Missing required fields: %s' %
346 def _XmlPatternToRegEx(xml_pattern
):
347 r
"""Translates an appengine-web.xml pattern into a regular expression.
349 Specially, this applies to the patterns that appear in the <include> and
350 <exclude> elements inside <static-files>. They look like '/**.png' or
351 '/stylesheets/*.css', and are translated into expressions like
352 '^/.*\.png$' or '^/stylesheets/.*\.css$'.
355 xml_pattern: a string like '/**.png'
358 a compiled regular expression like re.compile('^/.*\.png$').
362 if xml_pattern
.startswith('**'):
364 xml_pattern
= xml_pattern
[1:]
365 elif xml_pattern
.startswith('*'):
366 result
.append(r
'[^/]*')
367 elif xml_pattern
.startswith('/'):
372 result
.append(re
.escape(xml_pattern
[0]))
373 xml_pattern
= xml_pattern
[1:]
375 return re
.compile(''.join(result
))
378 class AppYamlTranslatorForDevAppServer(AppYamlTranslator
):
379 """Subclass of AppYamlTranslator specialized for the Dev App Server case.
381 The key difference is that static files are served directly from the war
382 directory, which means that the app.yaml patterns we define must cover
383 exactly those files in that directory hierarchy that are supposed to be static
384 while not covering any files that are not supposed to be static.
387 war_root: the root directory of the war hierarchy.
388 static_urls: a list of two-item tuples where the first item is a URL that
389 should be served statically and the second item corresponds to the
390 <include> element that caused that URL to be included.
397 super(AppYamlTranslatorForDevAppServer
, self
).__init
__(
398 app_engine_web_xml
, web_xml
, [], '1.0')
399 self
.war_root
= war_root
400 self
.static_urls
= self
.IncludedStaticUrls()
402 def IncludedStaticUrls(self
):
403 """Returns the URLs that should be resolved statically for this app.
405 The result includes a URL for every file in the war hierarchy that is
406 covered by one of the <include> elements for <static-files> and not covered
407 by any of the <exclude> elements.
410 a list of two-item tuples where the first item is a URL that should be
411 served statically and the second item corresponds to the <include>
412 element that caused that URL to be included.
420 includes
= self
.app_engine_web_xml
.static_file_includes
425 includes
= [aewxp
.StaticFileInclude('**', None, {})]
426 excludes
= self
.app_engine_web_xml
.static_file_excludes
427 files
= os
.listdir(self
.war_root
)
428 web_inf_name
= os
.path
.normcase('WEB-INF')
429 files
= [f
for f
in files
if os
.path
.normcase(f
) != web_inf_name
]
431 includes_and_res
= [(include
, _XmlPatternToRegEx(include
.pattern
))
432 for include
in includes
]
433 exclude_res
= [_XmlPatternToRegEx(exclude
) for exclude
in excludes
]
434 self
.ComputeIncludedStaticUrls(
435 static_urls
, self
.war_root
, '/', files
, includes_and_res
, exclude_res
)
441 def ComputeIncludedStaticUrls(
443 static_urls
, dirpath
, url_prefix
, files
, includes_and_res
, exclude_res
):
444 """Compute the URLs that should be resolved statically.
446 This recursive method is called for the war directory and every
447 subdirectory except the top-level WEB-INF directory. If we have arrived
448 at the directory <war-root>/foo/bar then dirpath will be <war-root>/foo/bar
449 and url_prefix will be /foo/bar.
452 static_urls: a list to be filled with the result, two-item tuples where
453 the first item is a URL and the second is a parsed <include> element.
454 dirpath: the path to the directory inside the war hierarchy that we have
455 reached at this point in the recursion.
456 url_prefix: the URL prefix that we have reached at this point in the
458 files: the contents of the dirpath directory, minus the WEB-INF directory
459 if dirpath is the war directory itself.
460 includes_and_res: a list of two-item tuples where the first item is a
461 parsed <include> element and the second item is a compiled regular
462 expression corresponding to the path= pattern from that element.
463 exclude_res: a list of compiled regular expressions corresponding to the
464 path= patterns from <exclude> elements.
467 path
= os
.path
.join(dirpath
, f
)
468 if os
.path
.isfile(path
):
470 if not any(exclude_re
.search(url
) for exclude_re
in exclude_res
):
471 for include
, include_re
in includes_and_res
:
472 if include_re
.search(url
):
473 static_urls
.append((url
, include
))
476 self
.ComputeIncludedStaticUrls(
477 static_urls
, path
, url_prefix
+ f
+ '/', os
.listdir(path
),
478 includes_and_res
, exclude_res
)
480 def TranslateHandlers(self
):
481 return handler_generator
.GenerateYamlHandlersListForDevAppServer(
482 self
.app_engine_web_xml
,
486 def ErrorHandlerPath(self
, error_handler
):
487 name
= error_handler
.name
488 if name
.startswith('/'):
490 if name
not in self
.static_files
:
491 raise AppEngineConfigException(
492 'No static file found for error handler: %s, out of %s' %
493 (name
, self
.static_files
))