Version 1.7.4
[gae.git] / java / src / main / com / google / appengine / tools / admin / AppYamlTranslator.java
blobb1b22b406eca153addf65c646c451a42fafa24f1
1 // Copyright 2008 Google Inc. All Rights Reserved.
3 package com.google.appengine.tools.admin;
5 import com.google.apphosting.utils.config.AppEngineConfigException;
6 import com.google.apphosting.utils.config.AppEngineWebXml;
7 import com.google.apphosting.utils.config.AppEngineWebXml.AdminConsolePage;
8 import com.google.apphosting.utils.config.AppEngineWebXml.ApiConfig;
9 import com.google.apphosting.utils.config.AppEngineWebXml.ErrorHandler;
10 import com.google.apphosting.utils.config.AppEngineWebXml.Pagespeed;
11 import com.google.apphosting.utils.config.BackendsXml;
12 import com.google.apphosting.utils.config.WebXml;
13 import com.google.apphosting.utils.config.WebXml.SecurityConstraint;
14 import com.google.apphosting.utils.glob.ConflictResolver;
15 import com.google.apphosting.utils.glob.Glob;
16 import com.google.apphosting.utils.glob.GlobFactory;
17 import com.google.apphosting.utils.glob.GlobIntersector;
18 import com.google.apphosting.utils.glob.LongestPatternConflictResolver;
19 import com.google.common.collect.Maps;
21 import net.sourceforge.yamlbeans.YamlConfig;
22 import net.sourceforge.yamlbeans.YamlException;
23 import net.sourceforge.yamlbeans.YamlWriter;
25 import java.io.StringWriter;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.Collections;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Set;
34 /**
35 * Generates {@code app.yaml} files suitable for uploading as part of
36 * a Google App Engine application.
39 public class AppYamlTranslator {
40 private static final String NO_API_VERSION = "none";
42 private static final ConflictResolver RESOLVER =
43 new LongestPatternConflictResolver();
45 private static final String DYNAMIC_PROPERTY = "dynamic";
46 private static final String STATIC_PROPERTY = "static";
47 private static final String WELCOME_FILES = "welcome";
48 private static final String TRANSPORT_GUARANTEE_PROPERTY = "transportGuarantee";
49 private static final String REQUIRED_ROLE_PROPERTY = "requiredRole";
50 private static final String EXPIRATION_PROPERTY = "expiration";
51 private static final String HTTP_HEADERS_PROPERTY = "http_headers";
52 private static final String API_ENDPOINT_REGEX = "/_ah/spi/*";
54 private static final String[] PROPERTIES = new String[] {
55 DYNAMIC_PROPERTY,
56 STATIC_PROPERTY,
57 WELCOME_FILES,
58 TRANSPORT_GUARANTEE_PROPERTY,
59 REQUIRED_ROLE_PROPERTY,
60 EXPIRATION_PROPERTY,
63 private static final int MAX_HANDLERS = 100;
65 private final AppEngineWebXml appEngineWebXml;
66 private final WebXml webXml;
67 private final BackendsXml backendsXml;
68 private final String apiVersion;
69 private final Set<String> staticFiles;
71 public AppYamlTranslator(AppEngineWebXml appEngineWebXml,
72 WebXml webXml,
73 BackendsXml backendsXml,
74 String apiVersion,
75 Set<String> staticFiles,
76 ApiConfig apiConfig) {
77 this.appEngineWebXml = appEngineWebXml;
78 this.webXml = webXml;
79 this.backendsXml = backendsXml;
80 this.apiVersion = apiVersion;
81 this.staticFiles = staticFiles;
84 public String getYaml() {
85 StringBuilder builder = new StringBuilder();
86 translateAppEngineWebXml(builder);
87 translateApiVersion(builder);
88 translateWebXml(builder);
89 return builder.toString();
92 private void appendIfNotNull(StringBuilder builder, String tag, Object value) {
93 if (value != null) {
94 builder.append(tag);
95 builder.append(value);
96 builder.append("\n");
101 * Returns the String that is used to identify the App Engine runtime
102 * on which the uploaded app should run. Normally this String is
103 * "java". But during periods of transition we may need to support
104 * multiple runtime labels.
105 * <p>
106 * Currently we are in transition from Java 6 to Java 7 and
107 * we have two runtimes: "java" and "java7". This method
108 * will return "java" unless the System Property
109 * "com.google.apphosting.runtime.use_java7" is "true".
111 private String getRuntimeLabel() {
112 return (Boolean.getBoolean(AppCfg.USE_JAVA7_SYSTEM_PROP) ? "java7" : "java");
115 private void translateAppEngineWebXml(StringBuilder builder) {
116 builder.append("application: '" + appEngineWebXml.getAppId() + "'\n");
117 builder.append("runtime: " + getRuntimeLabel() + "\n");
118 if (appEngineWebXml.getSourceLanguage() != null) {
119 builder.append("source_language: '" + appEngineWebXml.getSourceLanguage() + "'\n");
122 if (appEngineWebXml.getMajorVersionId() != null) {
123 builder.append("version: '" + appEngineWebXml.getMajorVersionId() + "'\n");
126 if (appEngineWebXml.getServer() != null) {
127 builder.append("server: '" + appEngineWebXml.getServer() + "'\n");
130 if (appEngineWebXml.getInstanceClass() != null) {
131 builder.append("instance_class: " + appEngineWebXml.getInstanceClass() + "\n");
134 if (!appEngineWebXml.getAutomaticScaling().isEmpty()) {
135 builder.append("automatic_scaling:\n");
136 AppEngineWebXml.AutomaticScaling settings = appEngineWebXml.getAutomaticScaling();
137 appendIfNotNull(builder, " min_pending_latency: ", settings.getMinPendingLatency());
138 appendIfNotNull(builder, " max_pending_latency: ", settings.getMaxPendingLatency());
139 appendIfNotNull(builder, " min_idle_instances: ", settings.getMinIdleInstances());
140 appendIfNotNull(builder, " max_idle_instances: ", settings.getMaxIdleInstances());
143 if (!appEngineWebXml.getManualScaling().isEmpty()) {
144 builder.append("manual_scaling:\n");
145 AppEngineWebXml.ManualScaling settings = appEngineWebXml.getManualScaling();
146 builder.append(" instances: " + settings.getInstances() + "\n");
149 if (!appEngineWebXml.getBasicScaling().isEmpty()) {
150 builder.append("basic_scaling:\n");
151 AppEngineWebXml.BasicScaling settings = appEngineWebXml.getBasicScaling();
152 builder.append(" max_instances: " + settings.getMaxInstances() + "\n");
153 appendIfNotNull(builder, " idle_timeout: ", settings.getIdleTimeout());
156 Collection<String> services = appEngineWebXml.getInboundServices();
157 if (!services.isEmpty()) {
158 builder.append("inbound_services:\n");
159 for (String service : services) {
160 builder.append("- " + service + "\n");
164 if (appEngineWebXml.getPrecompilationEnabled()) {
165 builder.append("derived_file_type:\n");
166 builder.append("- java_precompiled\n");
169 if (appEngineWebXml.getThreadsafe()) {
170 builder.append("threadsafe: True\n");
173 if (appEngineWebXml.getCodeLock()) {
174 builder.append("code_lock: True\n");
177 List<AdminConsolePage> adminConsolePages = appEngineWebXml.getAdminConsolePages();
178 if (!adminConsolePages.isEmpty()) {
179 builder.append("admin_console:\n");
180 builder.append(" pages:\n");
181 for (AdminConsolePage page : adminConsolePages) {
182 builder.append(" - name: " + page.getName() + "\n");
183 builder.append(" url: " + page.getUrl() + "\n");
187 List<ErrorHandler> errorHandlers = appEngineWebXml.getErrorHandlers();
188 if (!errorHandlers.isEmpty()) {
189 builder.append("error_handlers:\n");
190 for (ErrorHandler handler : errorHandlers) {
191 String fileName = handler.getFile();
192 if (!fileName.startsWith("/")) {
193 fileName = "/" + fileName;
195 if (!staticFiles.contains("__static__" + fileName)) {
196 throw new AppEngineConfigException("No static file found for error handler: " +
197 fileName + ", out of " + staticFiles);
199 builder.append("- file: __static__" + fileName + "\n");
200 if (handler.getErrorCode() != null) {
201 builder.append(" error_code: " + handler.getErrorCode() + "\n");
203 String mimeType = webXml.getMimeTypeForPath(handler.getFile());
204 if (mimeType != null) {
205 builder.append(" mime_type: " + mimeType + "\n");
210 if (backendsXml != null) {
211 builder.append(backendsXml.toYaml());
214 ApiConfig apiConfig = appEngineWebXml.getApiConfig();
215 if (apiConfig != null) {
216 builder.append("api_config:\n");
217 builder.append(" url: " + apiConfig.getUrl() + "\n");
218 builder.append(" script: unused\n");
221 if (appEngineWebXml.getPagespeed() != null) {
222 builder.append("pagespeed:\n");
223 appendPagespeed(appEngineWebXml.getPagespeed(), builder, 2);
228 * Append the given Pagespeed node as YAML to the given StringBuilder.
229 * @param pagespeed The Pagespeed instance to append as YAML.
230 * @param builder The StringBuilder to append to.
231 * @param indent The number of spaces to indent the pagespeed YAML.
233 public static void appendPagespeed(Pagespeed pagespeed, StringBuilder builder, int indent) {
234 if (pagespeed != null && !pagespeed.isEmpty()) {
235 Map<String, List<String>> config = Maps.newTreeMap();
236 putListInMapIfNotEmpty(config, "url_blacklist", pagespeed.getUrlBlacklist());
237 putListInMapIfNotEmpty(config, "domains_to_rewrite", pagespeed.getDomainsToRewrite());
238 putListInMapIfNotEmpty(config, "enabled_rewriters", pagespeed.getEnabledRewriters());
239 putListInMapIfNotEmpty(config, "disabled_rewriters", pagespeed.getDisabledRewriters());
240 appendObjectAsYaml(builder, config, indent);
245 * Adds the list to the given map, using the specified name, if the list is non-null and not
246 * empty.
248 private static void putListInMapIfNotEmpty(Map<String, List<String>> map, String name,
249 List<String> values) {
250 if (values != null && !values.isEmpty()) {
251 map.put(name, values);
256 * Appends the given collection to the StringBuilder as YAML, indenting each emitted line by
257 * numIndentSpaces.
259 private static void appendObjectAsYaml(
260 StringBuilder builder, Object collection, int numIndentSpaces) {
261 StringBuilder prefixBuilder = new StringBuilder();
262 for (int i = 0; i < numIndentSpaces; ++i) {
263 prefixBuilder.append(' ');
265 final String indentPrefix = prefixBuilder.toString();
267 StringWriter stringWriter = new StringWriter();
268 YamlConfig yamlConfig = new YamlConfig();
269 yamlConfig.writeConfig.setIndentSize(2);
270 yamlConfig.writeConfig.setWriteRootTags(false);
272 YamlWriter writer = new YamlWriter(stringWriter, yamlConfig);
273 try {
274 writer.write(collection);
275 writer.close();
276 } catch (YamlException e) {
277 throw new AppEngineConfigException("Unable to generate YAML.", e);
280 for (String line : stringWriter.toString().split("\n")) {
281 builder.append(indentPrefix);
282 builder.append(line);
283 builder.append("\n");
287 private void translateApiVersion(StringBuilder builder) {
288 if (apiVersion == null) {
289 builder.append("api_version: '" + NO_API_VERSION + "'\n");
290 } else {
291 builder.append("api_version: '" + apiVersion + "'\n");
295 private void translateWebXml(StringBuilder builder) {
296 builder.append("handlers:\n");
298 AbstractHandlerGenerator staticGenerator = null;
299 if (staticFiles.isEmpty()) {
300 staticGenerator = new EmptyHandlerGenerator();
301 } else {
302 staticGenerator = new StaticHandlerGenerator(appEngineWebXml.getPublicRoot());
305 DynamicHandlerGenerator dynamicGenerator =
306 new DynamicHandlerGenerator(webXml.getFallThroughToRuntime());
307 if (staticGenerator.size() + dynamicGenerator.size() > MAX_HANDLERS) {
308 dynamicGenerator = new DynamicHandlerGenerator(true);
311 staticGenerator.translate(builder);
312 dynamicGenerator.translate(builder);
315 class StaticHandlerGenerator extends AbstractHandlerGenerator {
316 private final String root;
318 public StaticHandlerGenerator(String root) {
319 this.root = root;
322 @Override
323 protected Map<String, Object> getWelcomeProperties() {
324 List<String> staticWelcomeFiles = new ArrayList<String>();
325 for (String welcomeFile : webXml.getWelcomeFiles()) {
326 for (String staticFile : staticFiles) {
327 if (staticFile.endsWith("/" + welcomeFile)) {
328 staticWelcomeFiles.add(welcomeFile);
329 break;
333 return Collections.<String,Object>singletonMap(WELCOME_FILES, staticWelcomeFiles);
336 @Override
337 protected void addPatterns(GlobIntersector intersector) {
338 List<AppEngineWebXml.StaticFileInclude> includes = appEngineWebXml.getStaticFileIncludes();
339 if (includes.isEmpty()) {
340 intersector.addGlob(GlobFactory.createGlob("/*", STATIC_PROPERTY, true));
341 } else {
342 for (AppEngineWebXml.StaticFileInclude include : includes) {
343 String pattern = include.getPattern().replaceAll("\\*\\*", "*");
344 if (!pattern.startsWith("/")) {
345 pattern = "/" + pattern;
347 Map<String, Object> props = new HashMap<String, Object>();
348 props.put(STATIC_PROPERTY, true);
349 if (include.getExpiration() != null) {
350 props.put(EXPIRATION_PROPERTY, include.getExpiration());
352 if (include.getHttpHeaders() != null) {
353 props.put(HTTP_HEADERS_PROPERTY, include.getHttpHeaders());
356 intersector.addGlob(GlobFactory.createGlob(pattern, props));
361 @Override
362 public void translateGlob(StringBuilder builder, Glob glob) {
363 String regex = glob.getRegularExpression().pattern();
364 if (!root.equals("")) {
365 if (regex.startsWith(root)){
366 regex = regex.substring(root.length(), regex.length());
369 @SuppressWarnings("unchecked")
370 List<String> welcomeFiles =
371 (List<String>) glob.getProperty(WELCOME_FILES, RESOLVER);
372 if (welcomeFiles != null) {
373 for (String welcomeFile : welcomeFiles) {
374 builder.append("- url: (" + regex + ")\n");
375 builder.append(" static_files: __static__" + root + "\\1" + welcomeFile + "\n");
376 builder.append(" upload: __NOT_USED__\n");
377 builder.append(" require_matching_file: True\n");
378 translateHandlerOptions(builder, glob);
379 translateAdditionalStaticOptions(builder, glob);
381 } else {
382 Boolean isStatic = (Boolean) glob.getProperty(STATIC_PROPERTY, RESOLVER);
383 if (isStatic != null && isStatic.booleanValue()) {
384 builder.append("- url: (" + regex + ")\n");
385 builder.append(" static_files: __static__" + root + "\\1\n");
386 builder.append(" upload: __NOT_USED__\n");
387 builder.append(" require_matching_file: True\n");
388 translateHandlerOptions(builder, glob);
389 translateAdditionalStaticOptions(builder, glob);
394 private void translateAdditionalStaticOptions(StringBuilder builder, Glob glob)
395 throws AppEngineConfigException {
396 String expiration = (String) glob.getProperty(EXPIRATION_PROPERTY, RESOLVER);
397 if (expiration != null) {
398 builder.append(" expiration: " + expiration + "\n");
401 @SuppressWarnings("unchecked")
402 Map<String, String> httpHeaders =
403 (Map<String, String>) glob.getProperty(HTTP_HEADERS_PROPERTY, RESOLVER);
404 if (httpHeaders != null && !httpHeaders.isEmpty()) {
405 builder.append(" http_headers:\n");
406 appendObjectAsYaml(builder, httpHeaders, 4);
412 * According to the example In section 12.2.2 of Servlet Spec 3.0 , /baz/* should also match /baz,
413 * so add an additional glob for that.
415 private static void extendMeaningOfTrailingStar(
416 GlobIntersector intersector, String pattern, String property, Object value) {
417 if (pattern.endsWith("/*") && pattern.length() > 2) {
418 intersector.addGlob(
419 GlobFactory.createGlob(pattern.substring(0, pattern.length() - 2), property, value));
423 class DynamicHandlerGenerator extends AbstractHandlerGenerator {
424 private final List<String> patterns;
425 private boolean fallthrough;
426 private boolean hasJsps;
428 DynamicHandlerGenerator(boolean alwaysFallthrough) {
429 fallthrough = alwaysFallthrough;
430 patterns = new ArrayList<String>();
431 for (String servletPattern : webXml.getServletPatterns()) {
432 if (servletPattern.equals("/") || servletPattern.equals("/*")) {
433 fallthrough = true;
434 } else if (servletPattern.equals(API_ENDPOINT_REGEX)) {
435 hasApiEndpoint = true;
436 } else if (servletPattern.endsWith(".jsp")) {
437 hasJsps = true;
438 } else {
439 patterns.add(servletPattern);
444 @Override
445 protected Map<String, Object> getWelcomeProperties() {
446 if (fallthrough) {
447 return null;
448 } else {
449 return Collections.<String,Object>singletonMap(DYNAMIC_PROPERTY, true);
453 @Override
454 protected void addPatterns(GlobIntersector intersector) {
455 if (fallthrough) {
456 intersector.addGlob(GlobFactory.createGlob(
457 "/*",
458 DYNAMIC_PROPERTY, true));
459 } else {
460 for (String servletPattern : patterns) {
461 intersector.addGlob(GlobFactory.createGlob(
462 servletPattern,
463 DYNAMIC_PROPERTY, true));
464 extendMeaningOfTrailingStar(intersector, servletPattern, DYNAMIC_PROPERTY, true);
466 if (hasJsps) {
467 intersector.addGlob(GlobFactory.createGlob(
468 "*.jsp",
469 DYNAMIC_PROPERTY, true));
471 intersector.addGlob(GlobFactory.createGlob(
472 "/_ah/*",
473 DYNAMIC_PROPERTY, true));
477 @Override
478 public void translateGlob(StringBuilder builder, Glob glob) {
479 String regex = glob.getRegularExpression().pattern();
481 Boolean isDynamic = (Boolean) glob.getProperty(DYNAMIC_PROPERTY, RESOLVER);
482 if (isDynamic != null && isDynamic.booleanValue()) {
483 builder.append("- url: " + regex + "\n");
484 builder.append(" script: unused\n");
485 translateHandlerOptions(builder, glob);
491 * An {@code AbstractHandlerGenerator} that returns no globs.
493 class EmptyHandlerGenerator extends AbstractHandlerGenerator {
494 @Override
495 protected void addPatterns(GlobIntersector intersector) {
498 @Override
499 protected void translateGlob(StringBuilder builder, Glob glob) {
502 @Override
503 protected Map<String, Object> getWelcomeProperties() {
504 return Collections.emptyMap();
508 abstract class AbstractHandlerGenerator {
509 private List<Glob> globs = null;
510 protected boolean hasApiEndpoint;
512 public int size() {
513 return getGlobPatterns().size();
516 public void translate(StringBuilder builder) {
517 for (Glob glob : getGlobPatterns()) {
518 translateGlob(builder, glob);
522 abstract protected void addPatterns(GlobIntersector intersector);
523 abstract protected void translateGlob(StringBuilder builder, Glob glob);
526 * @returns a map of welcome properties to apply to the welcome
527 * file entries, or {@code null} if no welcome file entries are
528 * necessary.
530 abstract protected Map<String, Object> getWelcomeProperties();
532 protected List<Glob> getGlobPatterns() {
533 if (globs == null) {
534 GlobIntersector intersector = new GlobIntersector();
535 addPatterns(intersector);
536 addSecurityConstraints(intersector);
537 addWelcomeFiles(intersector);
539 globs = intersector.getIntersection();
540 removeNearDuplicates(globs);
541 if (hasApiEndpoint) {
542 globs.add(GlobFactory.createGlob(API_ENDPOINT_REGEX, DYNAMIC_PROPERTY, true));
545 return globs;
548 protected void addWelcomeFiles(GlobIntersector intersector) {
549 Map<String, Object> welcomeProperties = getWelcomeProperties();
550 if (welcomeProperties != null) {
551 intersector.addGlob(GlobFactory.createGlob("/", welcomeProperties));
552 intersector.addGlob(GlobFactory.createGlob("/*/", welcomeProperties));
556 protected void addSecurityConstraints(GlobIntersector intersector) {
557 for (SecurityConstraint constraint : webXml.getSecurityConstraints()) {
558 for (String pattern : constraint.getUrlPatterns()) {
559 intersector.addGlob(GlobFactory.createGlob(
560 pattern,
561 TRANSPORT_GUARANTEE_PROPERTY,
562 constraint.getTransportGuarantee()));
563 extendMeaningOfTrailingStar(intersector, pattern, TRANSPORT_GUARANTEE_PROPERTY,
564 constraint.getTransportGuarantee());
565 intersector.addGlob(GlobFactory.createGlob(
566 pattern,
567 REQUIRED_ROLE_PROPERTY,
568 constraint.getRequiredRole()));
569 extendMeaningOfTrailingStar(
570 intersector, pattern, REQUIRED_ROLE_PROPERTY, constraint.getRequiredRole());
575 protected void translateHandlerOptions(StringBuilder builder, Glob glob) {
576 SecurityConstraint.RequiredRole requiredRole =
577 (SecurityConstraint.RequiredRole) glob.getProperty(REQUIRED_ROLE_PROPERTY, RESOLVER);
578 if (requiredRole == null) {
579 requiredRole = SecurityConstraint.RequiredRole.NONE;
581 switch (requiredRole) {
582 case NONE:
583 builder.append(" login: optional\n");
584 break;
585 case ANY_USER:
586 builder.append(" login: required\n");
587 break;
588 case ADMIN:
589 builder.append(" login: admin\n");
590 break;
593 SecurityConstraint.TransportGuarantee transportGuarantee =
594 (SecurityConstraint.TransportGuarantee) glob.getProperty(TRANSPORT_GUARANTEE_PROPERTY,
595 RESOLVER);
596 if (transportGuarantee == null) {
597 transportGuarantee = SecurityConstraint.TransportGuarantee.NONE;
599 switch (transportGuarantee) {
600 case NONE:
601 if (appEngineWebXml.getSslEnabled()) {
602 builder.append(" secure: optional\n");
603 } else {
604 builder.append(" secure: never\n");
606 break;
607 case INTEGRAL:
608 case CONFIDENTIAL:
609 if (!appEngineWebXml.getSslEnabled()) {
610 throw new AppEngineConfigException(
611 "SSL must be enabled in appengine-web.xml to use transport-guarantee");
613 builder.append(" secure: always\n");
614 break;
617 String pattern = glob.getRegularExpression().pattern();
618 String id = webXml.getHandlerIdForPattern(pattern);
619 if (id != null) {
620 if (appEngineWebXml.isApiEndpoint(id)) {
621 builder.append(" api_endpoint: True\n");
626 private void removeNearDuplicates(List<Glob> globs) {
627 for (int i = 0; i < globs.size(); i++) {
628 Glob topGlob = globs.get(i);
629 for (int j = i + 1; j < globs.size(); j++) {
630 Glob bottomGlob = globs.get(j);
631 if (bottomGlob.matchesAll(topGlob)) {
632 if (propertiesMatch(topGlob, bottomGlob)) {
633 globs.remove(i);
634 i--;
636 break;
642 private boolean propertiesMatch(Glob glob1, Glob glob2) {
643 for (String property : PROPERTIES) {
644 Object value1 = glob1.getProperty(property, RESOLVER);
645 Object value2 = glob2.getProperty(property, RESOLVER);
646 if (value1 != value2 && (value1 == null || !value1.equals(value2))) {
647 return false;
650 return true;