1 // Copyright 2008 Google Inc. All Rights Reserved.
3 package com
.google
.appengine
.tools
.admin
;
5 import com
.google
.apphosting
.utils
.config
.AppEngineConfigException
;
6 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
;
7 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
.AdminConsolePage
;
8 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
.ApiConfig
;
9 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
.ErrorHandler
;
10 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
.Pagespeed
;
11 import com
.google
.apphosting
.utils
.config
.BackendsXml
;
12 import com
.google
.apphosting
.utils
.config
.WebXml
;
13 import com
.google
.apphosting
.utils
.config
.WebXml
.SecurityConstraint
;
14 import com
.google
.apphosting
.utils
.glob
.ConflictResolver
;
15 import com
.google
.apphosting
.utils
.glob
.Glob
;
16 import com
.google
.apphosting
.utils
.glob
.GlobFactory
;
17 import com
.google
.apphosting
.utils
.glob
.GlobIntersector
;
18 import com
.google
.apphosting
.utils
.glob
.LongestPatternConflictResolver
;
19 import com
.google
.common
.collect
.Maps
;
21 import net
.sourceforge
.yamlbeans
.YamlConfig
;
22 import net
.sourceforge
.yamlbeans
.YamlException
;
23 import net
.sourceforge
.yamlbeans
.YamlWriter
;
25 import java
.io
.StringWriter
;
26 import java
.util
.ArrayList
;
27 import java
.util
.Collection
;
28 import java
.util
.Collections
;
29 import java
.util
.HashMap
;
30 import java
.util
.List
;
35 * Generates {@code app.yaml} files suitable for uploading as part of
36 * a Google App Engine application.
39 public class AppYamlTranslator
{
40 private static final String NO_API_VERSION
= "none";
42 private static final ConflictResolver RESOLVER
=
43 new LongestPatternConflictResolver();
45 private static final String DYNAMIC_PROPERTY
= "dynamic";
46 private static final String STATIC_PROPERTY
= "static";
47 private static final String WELCOME_FILES
= "welcome";
48 private static final String TRANSPORT_GUARANTEE_PROPERTY
= "transportGuarantee";
49 private static final String REQUIRED_ROLE_PROPERTY
= "requiredRole";
50 private static final String EXPIRATION_PROPERTY
= "expiration";
51 private static final String HTTP_HEADERS_PROPERTY
= "http_headers";
52 private static final String API_ENDPOINT_REGEX
= "/_ah/spi/*";
54 private static final String
[] PROPERTIES
= new String
[] {
58 TRANSPORT_GUARANTEE_PROPERTY
,
59 REQUIRED_ROLE_PROPERTY
,
63 private static final int MAX_HANDLERS
= 100;
65 private final AppEngineWebXml appEngineWebXml
;
66 private final WebXml webXml
;
67 private final BackendsXml backendsXml
;
68 private final String apiVersion
;
69 private final Set
<String
> staticFiles
;
71 public AppYamlTranslator(AppEngineWebXml appEngineWebXml
,
73 BackendsXml backendsXml
,
75 Set
<String
> staticFiles
,
76 ApiConfig apiConfig
) {
77 this.appEngineWebXml
= appEngineWebXml
;
79 this.backendsXml
= backendsXml
;
80 this.apiVersion
= apiVersion
;
81 this.staticFiles
= staticFiles
;
84 public String
getYaml() {
85 StringBuilder builder
= new StringBuilder();
86 translateAppEngineWebXml(builder
);
87 translateApiVersion(builder
);
88 translateWebXml(builder
);
89 return builder
.toString();
92 private void appendIfNotNull(StringBuilder builder
, String tag
, Object value
) {
95 builder
.append(value
);
101 * Returns the String that is used to identify the App Engine runtime
102 * on which the uploaded app should run. Normally this String is
103 * "java". But during periods of transition we may need to support
104 * multiple runtime labels.
106 * Currently we are in transition from Java 6 to Java 7 and
107 * we have two runtimes: "java" and "java7". This method
108 * will return "java" unless the System Property
109 * "com.google.apphosting.runtime.use_java7" is "true".
111 private String
getRuntimeLabel() {
112 return (Boolean
.getBoolean(AppCfg
.USE_JAVA7_SYSTEM_PROP
) ?
"java7" : "java");
115 private void translateAppEngineWebXml(StringBuilder builder
) {
116 builder
.append("application: '" + appEngineWebXml
.getAppId() + "'\n");
117 builder
.append("runtime: " + getRuntimeLabel() + "\n");
118 if (appEngineWebXml
.getSourceLanguage() != null) {
119 builder
.append("source_language: '" + appEngineWebXml
.getSourceLanguage() + "'\n");
122 if (appEngineWebXml
.getMajorVersionId() != null) {
123 builder
.append("version: '" + appEngineWebXml
.getMajorVersionId() + "'\n");
126 if (appEngineWebXml
.getServer() != null) {
127 builder
.append("server: '" + appEngineWebXml
.getServer() + "'\n");
130 if (appEngineWebXml
.getInstanceClass() != null) {
131 builder
.append("instance_class: " + appEngineWebXml
.getInstanceClass() + "\n");
134 if (!appEngineWebXml
.getAutomaticScaling().isEmpty()) {
135 builder
.append("automatic_scaling:\n");
136 AppEngineWebXml
.AutomaticScaling settings
= appEngineWebXml
.getAutomaticScaling();
137 appendIfNotNull(builder
, " min_pending_latency: ", settings
.getMinPendingLatency());
138 appendIfNotNull(builder
, " max_pending_latency: ", settings
.getMaxPendingLatency());
139 appendIfNotNull(builder
, " min_idle_instances: ", settings
.getMinIdleInstances());
140 appendIfNotNull(builder
, " max_idle_instances: ", settings
.getMaxIdleInstances());
143 if (!appEngineWebXml
.getManualScaling().isEmpty()) {
144 builder
.append("manual_scaling:\n");
145 AppEngineWebXml
.ManualScaling settings
= appEngineWebXml
.getManualScaling();
146 builder
.append(" instances: " + settings
.getInstances() + "\n");
149 if (!appEngineWebXml
.getBasicScaling().isEmpty()) {
150 builder
.append("basic_scaling:\n");
151 AppEngineWebXml
.BasicScaling settings
= appEngineWebXml
.getBasicScaling();
152 builder
.append(" max_instances: " + settings
.getMaxInstances() + "\n");
153 appendIfNotNull(builder
, " idle_timeout: ", settings
.getIdleTimeout());
156 Collection
<String
> services
= appEngineWebXml
.getInboundServices();
157 if (!services
.isEmpty()) {
158 builder
.append("inbound_services:\n");
159 for (String service
: services
) {
160 builder
.append("- " + service
+ "\n");
164 if (appEngineWebXml
.getPrecompilationEnabled()) {
165 builder
.append("derived_file_type:\n");
166 builder
.append("- java_precompiled\n");
169 if (appEngineWebXml
.getThreadsafe()) {
170 builder
.append("threadsafe: True\n");
173 if (appEngineWebXml
.getCodeLock()) {
174 builder
.append("code_lock: True\n");
177 List
<AdminConsolePage
> adminConsolePages
= appEngineWebXml
.getAdminConsolePages();
178 if (!adminConsolePages
.isEmpty()) {
179 builder
.append("admin_console:\n");
180 builder
.append(" pages:\n");
181 for (AdminConsolePage page
: adminConsolePages
) {
182 builder
.append(" - name: " + page
.getName() + "\n");
183 builder
.append(" url: " + page
.getUrl() + "\n");
187 List
<ErrorHandler
> errorHandlers
= appEngineWebXml
.getErrorHandlers();
188 if (!errorHandlers
.isEmpty()) {
189 builder
.append("error_handlers:\n");
190 for (ErrorHandler handler
: errorHandlers
) {
191 String fileName
= handler
.getFile();
192 if (!fileName
.startsWith("/")) {
193 fileName
= "/" + fileName
;
195 if (!staticFiles
.contains("__static__" + fileName
)) {
196 throw new AppEngineConfigException("No static file found for error handler: " +
197 fileName
+ ", out of " + staticFiles
);
199 builder
.append("- file: __static__" + fileName
+ "\n");
200 if (handler
.getErrorCode() != null) {
201 builder
.append(" error_code: " + handler
.getErrorCode() + "\n");
203 String mimeType
= webXml
.getMimeTypeForPath(handler
.getFile());
204 if (mimeType
!= null) {
205 builder
.append(" mime_type: " + mimeType
+ "\n");
210 if (backendsXml
!= null) {
211 builder
.append(backendsXml
.toYaml());
214 ApiConfig apiConfig
= appEngineWebXml
.getApiConfig();
215 if (apiConfig
!= null) {
216 builder
.append("api_config:\n");
217 builder
.append(" url: " + apiConfig
.getUrl() + "\n");
218 builder
.append(" script: unused\n");
221 if (appEngineWebXml
.getPagespeed() != null) {
222 builder
.append("pagespeed:\n");
223 appendPagespeed(appEngineWebXml
.getPagespeed(), builder
, 2);
228 * Append the given Pagespeed node as YAML to the given StringBuilder.
229 * @param pagespeed The Pagespeed instance to append as YAML.
230 * @param builder The StringBuilder to append to.
231 * @param indent The number of spaces to indent the pagespeed YAML.
233 public static void appendPagespeed(Pagespeed pagespeed
, StringBuilder builder
, int indent
) {
234 if (pagespeed
!= null && !pagespeed
.isEmpty()) {
235 Map
<String
, List
<String
>> config
= Maps
.newTreeMap();
236 putListInMapIfNotEmpty(config
, "url_blacklist", pagespeed
.getUrlBlacklist());
237 putListInMapIfNotEmpty(config
, "domains_to_rewrite", pagespeed
.getDomainsToRewrite());
238 putListInMapIfNotEmpty(config
, "enabled_rewriters", pagespeed
.getEnabledRewriters());
239 putListInMapIfNotEmpty(config
, "disabled_rewriters", pagespeed
.getDisabledRewriters());
240 appendObjectAsYaml(builder
, config
, indent
);
245 * Adds the list to the given map, using the specified name, if the list is non-null and not
248 private static void putListInMapIfNotEmpty(Map
<String
, List
<String
>> map
, String name
,
249 List
<String
> values
) {
250 if (values
!= null && !values
.isEmpty()) {
251 map
.put(name
, values
);
256 * Appends the given collection to the StringBuilder as YAML, indenting each emitted line by
259 private static void appendObjectAsYaml(
260 StringBuilder builder
, Object collection
, int numIndentSpaces
) {
261 StringBuilder prefixBuilder
= new StringBuilder();
262 for (int i
= 0; i
< numIndentSpaces
; ++i
) {
263 prefixBuilder
.append(' ');
265 final String indentPrefix
= prefixBuilder
.toString();
267 StringWriter stringWriter
= new StringWriter();
268 YamlConfig yamlConfig
= new YamlConfig();
269 yamlConfig
.writeConfig
.setIndentSize(2);
270 yamlConfig
.writeConfig
.setWriteRootTags(false);
272 YamlWriter writer
= new YamlWriter(stringWriter
, yamlConfig
);
274 writer
.write(collection
);
276 } catch (YamlException e
) {
277 throw new AppEngineConfigException("Unable to generate YAML.", e
);
280 for (String line
: stringWriter
.toString().split("\n")) {
281 builder
.append(indentPrefix
);
282 builder
.append(line
);
283 builder
.append("\n");
287 private void translateApiVersion(StringBuilder builder
) {
288 if (apiVersion
== null) {
289 builder
.append("api_version: '" + NO_API_VERSION
+ "'\n");
291 builder
.append("api_version: '" + apiVersion
+ "'\n");
295 private void translateWebXml(StringBuilder builder
) {
296 builder
.append("handlers:\n");
298 AbstractHandlerGenerator staticGenerator
= null;
299 if (staticFiles
.isEmpty()) {
300 staticGenerator
= new EmptyHandlerGenerator();
302 staticGenerator
= new StaticHandlerGenerator(appEngineWebXml
.getPublicRoot());
305 DynamicHandlerGenerator dynamicGenerator
=
306 new DynamicHandlerGenerator(webXml
.getFallThroughToRuntime());
307 if (staticGenerator
.size() + dynamicGenerator
.size() > MAX_HANDLERS
) {
308 dynamicGenerator
= new DynamicHandlerGenerator(true);
311 staticGenerator
.translate(builder
);
312 dynamicGenerator
.translate(builder
);
315 class StaticHandlerGenerator
extends AbstractHandlerGenerator
{
316 private final String root
;
318 public StaticHandlerGenerator(String root
) {
323 protected Map
<String
, Object
> getWelcomeProperties() {
324 List
<String
> staticWelcomeFiles
= new ArrayList
<String
>();
325 for (String welcomeFile
: webXml
.getWelcomeFiles()) {
326 for (String staticFile
: staticFiles
) {
327 if (staticFile
.endsWith("/" + welcomeFile
)) {
328 staticWelcomeFiles
.add(welcomeFile
);
333 return Collections
.<String
,Object
>singletonMap(WELCOME_FILES
, staticWelcomeFiles
);
337 protected void addPatterns(GlobIntersector intersector
) {
338 List
<AppEngineWebXml
.StaticFileInclude
> includes
= appEngineWebXml
.getStaticFileIncludes();
339 if (includes
.isEmpty()) {
340 intersector
.addGlob(GlobFactory
.createGlob("/*", STATIC_PROPERTY
, true));
342 for (AppEngineWebXml
.StaticFileInclude include
: includes
) {
343 String pattern
= include
.getPattern().replaceAll("\\*\\*", "*");
344 if (!pattern
.startsWith("/")) {
345 pattern
= "/" + pattern
;
347 Map
<String
, Object
> props
= new HashMap
<String
, Object
>();
348 props
.put(STATIC_PROPERTY
, true);
349 if (include
.getExpiration() != null) {
350 props
.put(EXPIRATION_PROPERTY
, include
.getExpiration());
352 if (include
.getHttpHeaders() != null) {
353 props
.put(HTTP_HEADERS_PROPERTY
, include
.getHttpHeaders());
356 intersector
.addGlob(GlobFactory
.createGlob(pattern
, props
));
362 public void translateGlob(StringBuilder builder
, Glob glob
) {
363 String regex
= glob
.getRegularExpression().pattern();
364 if (!root
.equals("")) {
365 if (regex
.startsWith(root
)){
366 regex
= regex
.substring(root
.length(), regex
.length());
369 @SuppressWarnings("unchecked")
370 List
<String
> welcomeFiles
=
371 (List
<String
>) glob
.getProperty(WELCOME_FILES
, RESOLVER
);
372 if (welcomeFiles
!= null) {
373 for (String welcomeFile
: welcomeFiles
) {
374 builder
.append("- url: (" + regex
+ ")\n");
375 builder
.append(" static_files: __static__" + root
+ "\\1" + welcomeFile
+ "\n");
376 builder
.append(" upload: __NOT_USED__\n");
377 builder
.append(" require_matching_file: True\n");
378 translateHandlerOptions(builder
, glob
);
379 translateAdditionalStaticOptions(builder
, glob
);
382 Boolean isStatic
= (Boolean
) glob
.getProperty(STATIC_PROPERTY
, RESOLVER
);
383 if (isStatic
!= null && isStatic
.booleanValue()) {
384 builder
.append("- url: (" + regex
+ ")\n");
385 builder
.append(" static_files: __static__" + root
+ "\\1\n");
386 builder
.append(" upload: __NOT_USED__\n");
387 builder
.append(" require_matching_file: True\n");
388 translateHandlerOptions(builder
, glob
);
389 translateAdditionalStaticOptions(builder
, glob
);
394 private void translateAdditionalStaticOptions(StringBuilder builder
, Glob glob
)
395 throws AppEngineConfigException
{
396 String expiration
= (String
) glob
.getProperty(EXPIRATION_PROPERTY
, RESOLVER
);
397 if (expiration
!= null) {
398 builder
.append(" expiration: " + expiration
+ "\n");
401 @SuppressWarnings("unchecked")
402 Map
<String
, String
> httpHeaders
=
403 (Map
<String
, String
>) glob
.getProperty(HTTP_HEADERS_PROPERTY
, RESOLVER
);
404 if (httpHeaders
!= null && !httpHeaders
.isEmpty()) {
405 builder
.append(" http_headers:\n");
406 appendObjectAsYaml(builder
, httpHeaders
, 4);
412 * According to the example In section 12.2.2 of Servlet Spec 3.0 , /baz/* should also match /baz,
413 * so add an additional glob for that.
415 private static void extendMeaningOfTrailingStar(
416 GlobIntersector intersector
, String pattern
, String property
, Object value
) {
417 if (pattern
.endsWith("/*") && pattern
.length() > 2) {
419 GlobFactory
.createGlob(pattern
.substring(0, pattern
.length() - 2), property
, value
));
423 class DynamicHandlerGenerator
extends AbstractHandlerGenerator
{
424 private final List
<String
> patterns
;
425 private boolean fallthrough
;
426 private boolean hasJsps
;
428 DynamicHandlerGenerator(boolean alwaysFallthrough
) {
429 fallthrough
= alwaysFallthrough
;
430 patterns
= new ArrayList
<String
>();
431 for (String servletPattern
: webXml
.getServletPatterns()) {
432 if (servletPattern
.equals("/") || servletPattern
.equals("/*")) {
434 } else if (servletPattern
.equals(API_ENDPOINT_REGEX
)) {
435 hasApiEndpoint
= true;
436 } else if (servletPattern
.endsWith(".jsp")) {
439 patterns
.add(servletPattern
);
445 protected Map
<String
, Object
> getWelcomeProperties() {
449 return Collections
.<String
,Object
>singletonMap(DYNAMIC_PROPERTY
, true);
454 protected void addPatterns(GlobIntersector intersector
) {
456 intersector
.addGlob(GlobFactory
.createGlob(
458 DYNAMIC_PROPERTY
, true));
460 for (String servletPattern
: patterns
) {
461 intersector
.addGlob(GlobFactory
.createGlob(
463 DYNAMIC_PROPERTY
, true));
464 extendMeaningOfTrailingStar(intersector
, servletPattern
, DYNAMIC_PROPERTY
, true);
467 intersector
.addGlob(GlobFactory
.createGlob(
469 DYNAMIC_PROPERTY
, true));
471 intersector
.addGlob(GlobFactory
.createGlob(
473 DYNAMIC_PROPERTY
, true));
478 public void translateGlob(StringBuilder builder
, Glob glob
) {
479 String regex
= glob
.getRegularExpression().pattern();
481 Boolean isDynamic
= (Boolean
) glob
.getProperty(DYNAMIC_PROPERTY
, RESOLVER
);
482 if (isDynamic
!= null && isDynamic
.booleanValue()) {
483 builder
.append("- url: " + regex
+ "\n");
484 builder
.append(" script: unused\n");
485 translateHandlerOptions(builder
, glob
);
491 * An {@code AbstractHandlerGenerator} that returns no globs.
493 class EmptyHandlerGenerator
extends AbstractHandlerGenerator
{
495 protected void addPatterns(GlobIntersector intersector
) {
499 protected void translateGlob(StringBuilder builder
, Glob glob
) {
503 protected Map
<String
, Object
> getWelcomeProperties() {
504 return Collections
.emptyMap();
508 abstract class AbstractHandlerGenerator
{
509 private List
<Glob
> globs
= null;
510 protected boolean hasApiEndpoint
;
513 return getGlobPatterns().size();
516 public void translate(StringBuilder builder
) {
517 for (Glob glob
: getGlobPatterns()) {
518 translateGlob(builder
, glob
);
522 abstract protected void addPatterns(GlobIntersector intersector
);
523 abstract protected void translateGlob(StringBuilder builder
, Glob glob
);
526 * @returns a map of welcome properties to apply to the welcome
527 * file entries, or {@code null} if no welcome file entries are
530 abstract protected Map
<String
, Object
> getWelcomeProperties();
532 protected List
<Glob
> getGlobPatterns() {
534 GlobIntersector intersector
= new GlobIntersector();
535 addPatterns(intersector
);
536 addSecurityConstraints(intersector
);
537 addWelcomeFiles(intersector
);
539 globs
= intersector
.getIntersection();
540 removeNearDuplicates(globs
);
541 if (hasApiEndpoint
) {
542 globs
.add(GlobFactory
.createGlob(API_ENDPOINT_REGEX
, DYNAMIC_PROPERTY
, true));
548 protected void addWelcomeFiles(GlobIntersector intersector
) {
549 Map
<String
, Object
> welcomeProperties
= getWelcomeProperties();
550 if (welcomeProperties
!= null) {
551 intersector
.addGlob(GlobFactory
.createGlob("/", welcomeProperties
));
552 intersector
.addGlob(GlobFactory
.createGlob("/*/", welcomeProperties
));
556 protected void addSecurityConstraints(GlobIntersector intersector
) {
557 for (SecurityConstraint constraint
: webXml
.getSecurityConstraints()) {
558 for (String pattern
: constraint
.getUrlPatterns()) {
559 intersector
.addGlob(GlobFactory
.createGlob(
561 TRANSPORT_GUARANTEE_PROPERTY
,
562 constraint
.getTransportGuarantee()));
563 extendMeaningOfTrailingStar(intersector
, pattern
, TRANSPORT_GUARANTEE_PROPERTY
,
564 constraint
.getTransportGuarantee());
565 intersector
.addGlob(GlobFactory
.createGlob(
567 REQUIRED_ROLE_PROPERTY
,
568 constraint
.getRequiredRole()));
569 extendMeaningOfTrailingStar(
570 intersector
, pattern
, REQUIRED_ROLE_PROPERTY
, constraint
.getRequiredRole());
575 protected void translateHandlerOptions(StringBuilder builder
, Glob glob
) {
576 SecurityConstraint
.RequiredRole requiredRole
=
577 (SecurityConstraint
.RequiredRole
) glob
.getProperty(REQUIRED_ROLE_PROPERTY
, RESOLVER
);
578 if (requiredRole
== null) {
579 requiredRole
= SecurityConstraint
.RequiredRole
.NONE
;
581 switch (requiredRole
) {
583 builder
.append(" login: optional\n");
586 builder
.append(" login: required\n");
589 builder
.append(" login: admin\n");
593 SecurityConstraint
.TransportGuarantee transportGuarantee
=
594 (SecurityConstraint
.TransportGuarantee
) glob
.getProperty(TRANSPORT_GUARANTEE_PROPERTY
,
596 if (transportGuarantee
== null) {
597 transportGuarantee
= SecurityConstraint
.TransportGuarantee
.NONE
;
599 switch (transportGuarantee
) {
601 if (appEngineWebXml
.getSslEnabled()) {
602 builder
.append(" secure: optional\n");
604 builder
.append(" secure: never\n");
609 if (!appEngineWebXml
.getSslEnabled()) {
610 throw new AppEngineConfigException(
611 "SSL must be enabled in appengine-web.xml to use transport-guarantee");
613 builder
.append(" secure: always\n");
617 String pattern
= glob
.getRegularExpression().pattern();
618 String id
= webXml
.getHandlerIdForPattern(pattern
);
620 if (appEngineWebXml
.isApiEndpoint(id
)) {
621 builder
.append(" api_endpoint: True\n");
626 private void removeNearDuplicates(List
<Glob
> globs
) {
627 for (int i
= 0; i
< globs
.size(); i
++) {
628 Glob topGlob
= globs
.get(i
);
629 for (int j
= i
+ 1; j
< globs
.size(); j
++) {
630 Glob bottomGlob
= globs
.get(j
);
631 if (bottomGlob
.matchesAll(topGlob
)) {
632 if (propertiesMatch(topGlob
, bottomGlob
)) {
642 private boolean propertiesMatch(Glob glob1
, Glob glob2
) {
643 for (String property
: PROPERTIES
) {
644 Object value1
= glob1
.getProperty(property
, RESOLVER
);
645 Object value2
= glob2
.getProperty(property
, RESOLVER
);
646 if (value1
!= value2
&& (value1
== null || !value1
.equals(value2
))) {