java/src/main/com/google/appengine/tools/admin/Application.java

   1 // Copyright 2008 Google Inc. All Rights Reserved.
   2
   3 package com.google.appengine.tools.admin;
   4
   5 import static java.nio.charset.StandardCharsets.UTF_8;
   6
   7 import com.google.appengine.tools.admin.AppAdminFactory.ApplicationProcessingOptions;
   8 import com.google.appengine.tools.admin.RepoInfo.SourceContext;
   9 import com.google.appengine.tools.info.SdkImplInfo;
  10 import com.google.appengine.tools.info.SdkInfo;
  11 import com.google.appengine.tools.info.Version;
  12 import com.google.appengine.tools.util.ApiVersionFinder;
  13 import com.google.appengine.tools.util.FileIterator;
  14 import com.google.appengine.tools.util.JarSplitter;
  15 import com.google.appengine.tools.util.JarTool;
  16 import com.google.apphosting.utils.config.AppEngineConfigException;
  17 import com.google.apphosting.utils.config.AppEngineWebXml;
  18 import com.google.apphosting.utils.config.AppEngineWebXmlReader;
  19 import com.google.apphosting.utils.config.AppYamlProcessor;
  20 import com.google.apphosting.utils.config.BackendsXml;
  21 import com.google.apphosting.utils.config.BackendsXmlReader;
  22 import com.google.apphosting.utils.config.BackendsYamlReader;
  23 import com.google.apphosting.utils.config.CronXml;
  24 import com.google.apphosting.utils.config.CronXmlReader;
  25 import com.google.apphosting.utils.config.CronYamlReader;
  26 import com.google.apphosting.utils.config.DispatchXml;
  27 import com.google.apphosting.utils.config.DispatchXmlReader;
  28 import com.google.apphosting.utils.config.DispatchYamlReader;
  29 import com.google.apphosting.utils.config.DosXml;
  30 import com.google.apphosting.utils.config.DosXmlReader;
  31 import com.google.apphosting.utils.config.DosYamlReader;
  32 import com.google.apphosting.utils.config.GenerationDirectory;
  33 import com.google.apphosting.utils.config.IndexesXml;
  34 import com.google.apphosting.utils.config.IndexesXmlReader;
  35 import com.google.apphosting.utils.config.QueueXml;
  36 import com.google.apphosting.utils.config.QueueXmlReader;
  37 import com.google.apphosting.utils.config.QueueYamlReader;
  38 import com.google.apphosting.utils.config.WebXml;
  39 import com.google.apphosting.utils.config.WebXmlReader;
  40 import com.google.apphosting.utils.config.XmlUtils;
  41 import com.google.common.collect.ImmutableSet;
  42 import com.google.common.collect.Lists;
  43 import com.google.common.collect.Sets;
  44 import com.google.common.io.Files;
  45
  46 import org.mortbay.io.Buffer;
  47 import org.mortbay.jetty.MimeTypes;
  48 import org.w3c.dom.Document;
  49 import org.w3c.dom.Node;
  50 import org.w3c.dom.NodeList;
  51 import org.xml.sax.SAXException;
  52
  53 import java.io.DataInputStream;
  54 import java.io.File;
  55 import java.io.FileInputStream;
  56 import java.io.FileNotFoundException;
  57 import java.io.FileOutputStream;
  58 import java.io.FileWriter;
  59 import java.io.IOException;
  60 import java.io.InputStream;
  61 import java.io.PrintWriter;
  62 import java.net.URL;
  63 import java.net.URLConnection;
  64 import java.util.ArrayList;
  65 import java.util.Arrays;
  66 import java.util.HashSet;
  67 import java.util.List;
  68 import java.util.Set;
  69 import java.util.jar.JarEntry;
  70 import java.util.jar.JarInputStream;
  71 import java.util.logging.Level;
  72 import java.util.logging.Logger;
  73 import java.util.regex.Pattern;
  74
  75 import javax.activation.FileTypeMap;
  76 import javax.tools.JavaCompiler;
  77 import javax.tools.JavaFileObject;
  78 import javax.tools.StandardJavaFileManager;
  79 import javax.tools.ToolProvider;
  80 import javax.xml.parsers.DocumentBuilder;
  81 import javax.xml.parsers.DocumentBuilderFactory;
  82 import javax.xml.parsers.ParserConfigurationException;
  83 import javax.xml.transform.OutputKeys;
  84 import javax.xml.transform.Transformer;
  85 import javax.xml.transform.TransformerException;
  86 import javax.xml.transform.TransformerFactory;
  87 import javax.xml.transform.dom.DOMSource;
  88 import javax.xml.transform.stream.StreamResult;
  89
  90 /**
  91  * An App Engine application. You can {@link #readApplication read} an
  92  * {@code Application} from a path, and
  93  * {@link com.google.appengine.tools.admin.AppAdminFactory#createAppAdmin create}
  94  * an {@link com.google.appengine.tools.admin.AppAdmin} to upload, create
  95  * indexes, or otherwise manage it.
  96  *
  97  */
  98 public class Application implements GenericApplication {
  99
 100   private static final int MAX_COMPILED_JSP_JAR_SIZE = 1024 * 1024 * 5;
 101   private static final String COMPILED_JSP_JAR_NAME_PREFIX = "_ah_compiled_jsps";
 102
 103   private static final int MAX_CLASSES_JAR_SIZE = 1024 * 1024 * 5;
 104   private static final String CLASSES_JAR_NAME_PREFIX = "_ah_webinf_classes";
 105
 106   private static final String JAVA_7_RUNTIME_ID = "java7";
 107   private static final ImmutableSet<String> ALLOWED_RUNTIME_IDS = ImmutableSet.of(
 108       JAVA_7_RUNTIME_ID);
 109
 110   private static final String BETA_SOURCE_REFERENCE_KEY = "source_reference";
 111
 112   private static Pattern JSP_REGEX = Pattern.compile(".*\\.jspx?");
 113
 114   /** If available, this is set to a program to make symlinks, e.g. /bin/ln */
 115   private static File ln = Utility.findLink();
 116   private static File sdkDocsDir;
 117   public static synchronized File getSdkDocsDir(){
 118     if (null == sdkDocsDir){
 119       sdkDocsDir = new File(SdkInfo.getSdkRoot(), "docs");
 120     }
 121     return sdkDocsDir;
 122   }
 123
 124   private static Version sdkVersion;
 125   public static synchronized Version getSdkVersion() {
 126     if (null == sdkVersion) {
 127       sdkVersion = SdkInfo.getLocalVersion();
 128     }
 129     return sdkVersion;
 130   }
 131
 132   private static final String STAGEDIR_PREFIX = "appcfg";
 133
 134   private static final Logger logger = Logger.getLogger(Application.class.getName());
 135
 136   private static final MimeTypes mimeTypes = new MimeTypes();
 137
 138   private AppEngineWebXml appEngineWebXml;
 139   private WebXml webXml;
 140   private CronXml cronXml;
 141   private DispatchXml dispatchXml;
 142   private DosXml dosXml;
 143   private String pagespeedYaml;
 144   private QueueXml queueXml;
 145   private IndexesXml indexesXml;
 146   private BackendsXml backendsXml;
 147   private File baseDir;
 148   private File stageDir;
 149   private File externalResourceDir;
 150   private String apiVersion;
 151   private String appYaml;
 152   private SourceContext sourceContext;
 153
 154   private UpdateListener listener;
 155   private PrintWriter detailsWriter;
 156   private int updateProgress = 0;
 157   private int progressAmount = 0;
 158
 159   protected Application(){
 160   }
 161
 162   /**
 163    * Builds a normalized path for the given directory in which
 164    * forward slashes are used as the file separator on all platforms.
 165    * @param dir A directory
 166    * @return The normalized path
 167    */
 168   private static String buildNormalizedPath(File dir) {
 169     String normalizedPath = dir.getPath();
 170     if (File.separatorChar == '\\') {
 171       normalizedPath = normalizedPath.replace('\\', '/');
 172     }
 173     return normalizedPath;
 174   }
 175
 176   private Application(String explodedPath, String appId, String module, String appVersion) {
 177     this.baseDir = new File(explodedPath);
 178     explodedPath = buildNormalizedPath(baseDir);
 179     File webinf = new File(baseDir, "WEB-INF");
 180     if (!webinf.getName().equals("WEB-INF")) {
 181       throw new AppEngineConfigException("WEB-INF directory must be capitalized.");
 182     }
 183
 184     String webinfPath = webinf.getPath();
 185     AppEngineWebXmlReader aewebReader = new AppEngineWebXmlReader(explodedPath);
 186     WebXmlReader webXmlReader = new WebXmlReader(explodedPath);
 187     AppYamlProcessor.convert(webinf, aewebReader.getFilename(), webXmlReader.getFilename());
 188
 189     if (new File(aewebReader.getFilename()).exists()) {
 190       XmlUtils.validateXml(
 191           aewebReader.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd"));
 192     }
 193     appEngineWebXml = aewebReader.readAppEngineWebXml();
 194     appEngineWebXml.setSourcePrefix(explodedPath);
 195     if (appId == null) {
 196       if (appEngineWebXml.getAppId() == null) {
 197         throw new AppEngineConfigException(
 198             "No app id supplied and XML files have no <application> element");
 199       }
 200     } else {
 201       appEngineWebXml.setAppId(appId);
 202     }
 203     if (module != null) {
 204       appEngineWebXml.setModule(module);
 205     }
 206     if (appVersion != null) {
 207       appEngineWebXml.setMajorVersionId(appVersion);
 208     }
 209
 210     sourceContext = new RepoInfo(baseDir).getSourceContext();
 211     if (sourceContext != null) {
 212       String sourceRef = sourceContext.getRevisionId();
 213       if (sourceContext.getRepositoryUrl() != null) {
 214         sourceRef = sourceContext.getRepositoryUrl() + "#" + sourceRef;
 215       }
 216       appEngineWebXml.addBetaSetting(BETA_SOURCE_REFERENCE_KEY, sourceRef);
 217     }
 218
 219     webXml = webXmlReader.readWebXml();
 220     webXml.validate();
 221
 222     CronXmlReader cronReader = new CronXmlReader(explodedPath);
 223     if (new File(cronReader.getFilename()).exists()) {
 224       XmlUtils.validateXml(cronReader.getFilename(), new File(getSdkDocsDir(), "cron.xsd"));
 225     }
 226     cronXml = cronReader.readCronXml();
 227     if (cronXml == null) {
 228       CronYamlReader cronYaml = new CronYamlReader(webinfPath);
 229       cronXml = cronYaml.parse();
 230     }
 231
 232     QueueXmlReader queueReader = new QueueXmlReader(explodedPath);
 233     if (new File(queueReader.getFilename()).exists()) {
 234       XmlUtils.validateXml(queueReader.getFilename(), new File(getSdkDocsDir(), "queue.xsd"));
 235     }
 236     queueXml = queueReader.readQueueXml();
 237     if (queueXml == null) {
 238       QueueYamlReader queueYaml = new QueueYamlReader(webinfPath);
 239       queueXml = queueYaml.parse();
 240     }
 241
 242     DispatchXmlReader dispatchXmlReader = new DispatchXmlReader(explodedPath,
 243         DispatchXmlReader.DEFAULT_RELATIVE_FILENAME);
 244     if (new File(dispatchXmlReader.getFilename()).exists()) {
 245       XmlUtils.validateXml(
 246           dispatchXmlReader.getFilename(), new File(getSdkDocsDir(), "dispatch.xsd"));
 247     }
 248     dispatchXml = dispatchXmlReader.readDispatchXml();
 249     if (dispatchXml == null) {
 250       DispatchYamlReader dispatchYamlReader = new DispatchYamlReader(webinfPath);
 251       dispatchXml = dispatchYamlReader.parse();
 252     }
 253
 254     DosXmlReader dosReader = new DosXmlReader(explodedPath);
 255     if (new File(dosReader.getFilename()).exists()) {
 256       XmlUtils.validateXml(dosReader.getFilename(), new File(getSdkDocsDir(), "dos.xsd"));
 257     }
 258     dosXml = dosReader.readDosXml();
 259     if (dosXml == null) {
 260       DosYamlReader dosYaml = new DosYamlReader(webinfPath);
 261       dosXml = dosYaml.parse();
 262     }
 263
 264     if (getAppEngineWebXml().getPagespeed() != null) {
 265       StringBuilder pagespeedYamlBuilder = new StringBuilder();
 266       AppYamlTranslator.appendPagespeed(
 267           getAppEngineWebXml().getPagespeed(), pagespeedYamlBuilder, 0);
 268       pagespeedYaml = pagespeedYamlBuilder.toString();
 269     }
 270
 271     IndexesXmlReader indexReader = new IndexesXmlReader(explodedPath);
 272     File datastoreSchema = new File(getSdkDocsDir(), "datastore-indexes.xsd");
 273     if (new File(indexReader.getFilename()).exists()) {
 274       XmlUtils.validateXml(indexReader.getFilename(), datastoreSchema);
 275     }
 276     indexesXml = indexReader.readIndexesXml();
 277
 278     BackendsXmlReader backendsReader = new BackendsXmlReader(explodedPath);
 279     if (new File(backendsReader.getFilename()).exists()) {
 280       XmlUtils.validateXml(backendsReader.getFilename(), new File(getSdkDocsDir(), "backends.xsd"));
 281     }
 282     backendsXml = backendsReader.readBackendsXml();
 283     if (backendsXml == null) {
 284       BackendsYamlReader backendsYaml = new BackendsYamlReader(webinfPath);
 285       backendsXml = backendsYaml.parse();
 286     }
 287   }
 288
 289   /**
 290    * Reads the App Engine application from {@code path}. The path may either
 291    * be a WAR file or the root of an exploded WAR directory.
 292    *
 293    * @param path a not {@code null} path.
 294    *
 295    * @throws IOException if an error occurs while trying to read the
 296    * {@code Application}.
 297    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 298    * {@code Application's} appengine-web.xml file is malformed.
 299    */
 300   public static Application readApplication(String path)
 301       throws IOException {
 302     return new Application(path, null, null, null);
 303   }
 304
 305   /**
 306    * Sets the external resource directory. Call this method before invoking
 307    * {@link #createStagingDirectory(ApplicationProcessingOptions, ResourceLimits)}.
 308    * <p>
 309    * The external resource directory is a directory outside of the war directory where additional
 310    * files live. These files will be copied into the staging directory during an upload, after the
 311    * war directory is copied there. Consequently if there are any name collisions the files in the
 312    * external resource directory will win.
 313    *
 314    * @param path a not {@code null} path to an existing directory.
 315    *
 316    * @throws IllegalArgumentException If {@code path} does not refer to an existing
 317    *         directory.
 318    */
 319   public void setExternalResourceDir(String path) {
 320     if (path == null) {
 321       throw new NullPointerException("path is null");
 322     }
 323     if (stageDir != null) {
 324       throw new IllegalStateException(
 325           "This method must be invoked prior to createStagingDirectory()");
 326     }
 327     File dir = new File(path);
 328     if (!dir.exists()) {
 329       throw new IllegalArgumentException("path does not exist: " + path);
 330     }
 331     if (!dir.isDirectory()) {
 332       throw new IllegalArgumentException(path + " is not a directory.");
 333     }
 334     this.externalResourceDir = dir;
 335   }
 336
 337   /**
 338    * Reads the App Engine application from {@code path}. The path may either
 339    * be a WAR file or the root of an exploded WAR directory.
 340    *
 341    * @param path a not {@code null} path.
 342    * @param appId if non-null, use this as an application id override.
 343    * @param module if non-null, use this as a module id override.
 344    * @param appVersion if non-null, use this as an application version override.
 345    *
 346    * @throws IOException if an error occurs while trying to read the
 347    * {@code Application}.
 348    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 349    * {@code Application's} appengine-web.xml file is malformed.
 350    */
 351   public static Application readApplication(String path,
 352       String appId,
 353       String module,
 354       String appVersion) throws IOException {
 355     return new Application(path, appId, module, appVersion);
 356   }
 357
 358   /**
 359    * Returns the application identifier, from the AppEngineWebXml config
 360    * @return application identifier
 361    */
 362   @Override
 363   public String getAppId() {
 364     return appEngineWebXml.getAppId();
 365   }
 366
 367   /**
 368    * Returns the application version, from the AppEngineWebXml config
 369    * @return application version
 370    */
 371   @Override
 372   public String getVersion() {
 373     return appEngineWebXml.getMajorVersionId();
 374   }
 375
 376   @Override
 377   public String getSourceLanguage() {
 378     return appEngineWebXml.getSourceLanguage();
 379   }
 380
 381   @Override
 382   public String getModule() {
 383     return appEngineWebXml.getModule();
 384   }
 385
 386   @Override
 387   public String getInstanceClass() {
 388     return appEngineWebXml.getInstanceClass();
 389   }
 390
 391   @Override
 392   public boolean isPrecompilationEnabled() {
 393     return appEngineWebXml.getPrecompilationEnabled();
 394   }
 395
 396   @Override
 397   public List<ErrorHandler> getErrorHandlers() {
 398     class ErrorHandlerImpl implements ErrorHandler {
 399       private final AppEngineWebXml.ErrorHandler errorHandler;
 400       public ErrorHandlerImpl(AppEngineWebXml.ErrorHandler errorHandler) {
 401         this.errorHandler = errorHandler;
 402       }
 403       @Override
 404       public String getFile() {
 405         return "__static__/" + errorHandler.getFile();
 406       }
 407       @Override
 408       public String getErrorCode() {
 409         return errorHandler.getErrorCode();
 410       }
 411       @Override
 412       public String getMimeType() {
 413         return getMimeTypeIfStatic(getFile());
 414       }
 415     }
 416     List<ErrorHandler> errorHandlers = new ArrayList<ErrorHandler>();
 417     for (AppEngineWebXml.ErrorHandler errorHandler: appEngineWebXml.getErrorHandlers()) {
 418       errorHandlers.add(new ErrorHandlerImpl(errorHandler));
 419     }
 420     return errorHandlers;
 421   }
 422
 423   @Override
 424   public String getMimeTypeIfStatic(String path) {
 425     if (!path.contains("__static__/")) {
 426       return null;
 427     }
 428     String mimeType = webXml.getMimeTypeForPath(path);
 429     if (mimeType != null) {
 430       return mimeType;
 431     }
 432     return guessContentTypeFromName(path);
 433   }
 434
 435   /**
 436    * @param fileName path of a file with extension
 437    * @return the mimetype of the file (or application/octect-stream if not recognized)
 438    */
 439   public static String guessContentTypeFromName(String fileName) {
 440     String defaultValue = "application/octet-stream";
 441     try {
 442       Buffer buffer = mimeTypes.getMimeByExtension(fileName);
 443       if (buffer != null) {
 444         return new String(buffer.asArray());
 445       }
 446       String lowerName = fileName.toLowerCase();
 447       if (lowerName.endsWith(".json")) {
 448         return "application/json";
 449       }
 450       FileTypeMap typeMap = FileTypeMap.getDefaultFileTypeMap();
 451       String ret = typeMap.getContentType(fileName);
 452       if (ret != null) {
 453         return ret;
 454       }
 455       ret = URLConnection.guessContentTypeFromName(fileName);
 456       if (ret != null) {
 457         return ret;
 458       }
 459       return defaultValue;
 460     } catch (Throwable t) {
 461       logger.log(Level.WARNING, "Error identify mimetype for " + fileName, t);
 462       return defaultValue;
 463     }
 464   }
 465   /**
 466    * Returns the AppEngineWebXml describing the application.
 467    *
 468    * @return a not {@code null} deployment descriptor
 469    */
 470   public AppEngineWebXml getAppEngineWebXml() {
 471     return appEngineWebXml;
 472   }
 473
 474   /**
 475    * Returns the AppEngineWebXml with application and version removed
 476    *
 477    * @return a not {@code null} deployment descriptor
 478    */
 479   public AppEngineWebXml getScrubbedAppEngineWebXml() {
 480     AppEngineWebXml scrubbedAppEngineWebXml = appEngineWebXml.clone();
 481     scrubbedAppEngineWebXml.setAppId(null);
 482     scrubbedAppEngineWebXml.setMajorVersionId(null);
 483     return scrubbedAppEngineWebXml;
 484   }
 485
 486   /**
 487    * Returns the CronXml describing the applications' cron jobs.
 488    * @return a cron descriptor, possibly empty or {@code null}
 489    */
 490   @Override
 491   public CronXml getCronXml() {
 492     return cronXml;
 493   }
 494
 495   /**
 496    * Returns the QueueXml describing the applications' task queues.
 497    * @return a queue descriptor, possibly empty or {@code null}
 498    */
 499   @Override
 500   public QueueXml getQueueXml() {
 501     return queueXml;
 502   }
 503
 504   @Override
 505   public DispatchXml getDispatchXml() {
 506     return dispatchXml;
 507   }
 508
 509   /**
 510    * Returns the DosXml describing the applications' DoS entries.
 511    * @return a dos descriptor, possibly empty or {@code null}
 512    */
 513   @Override
 514   public DosXml getDosXml() {
 515     return dosXml;
 516   }
 517
 518   /**
 519    * Returns the pagespeed.yaml describing the applications' PageSpeed configuration.
 520    * @return a pagespeed.yaml string, possibly empty or {@code null}
 521    */
 522   @Override
 523   public String getPagespeedYaml() {
 524     return pagespeedYaml;
 525   }
 526
 527   /**
 528    * Returns the IndexesXml describing the applications' indexes.
 529    * @return a index descriptor, possibly empty or {@code null}
 530    */
 531   @Override
 532   public IndexesXml getIndexesXml() {
 533     return indexesXml;
 534   }
 535
 536   /**
 537    * Returns the WebXml describing the applications' servlets and generic web
 538    * application information.
 539    *
 540    * @return a WebXml descriptor, possibly empty but not {@code null}
 541    */
 542   public WebXml getWebXml() {
 543     return webXml;
 544   }
 545
 546   @Override
 547   public BackendsXml getBackendsXml() {
 548     return backendsXml;
 549   }
 550
 551   /**
 552    * Returns the desired API version for the current application, or
 553    * {@code "none"} if no API version was used.
 554    *
 555    * @throws IllegalStateException if createStagingDirectory has not been called.
 556    */
 557   @Override
 558   public String getApiVersion() {
 559     if (apiVersion == null) {
 560       throw new IllegalStateException("Must call createStagingDirectory first.");
 561     }
 562     return apiVersion;
 563   }
 564
 565   /**
 566    * Returns a path to an exploded WAR directory for the application.
 567    * This may be a temporary directory.
 568    *
 569    * @return a not {@code null} path pointing to a directory
 570    */
 571   @Override
 572   public String getPath() {
 573     return baseDir.getAbsolutePath();
 574   }
 575
 576   /**
 577    * Returns the staging directory, or {@code null} if none has been created.
 578    */
 579   @Override
 580   public File getStagingDir() {
 581     return stageDir;
 582   }
 583
 584   @Override
 585   public void resetProgress() {
 586     updateProgress = 0;
 587     progressAmount = 0;
 588   }
 589
 590   /**
 591    * Creates a new staging directory, if needed, or returns the existing one
 592    * if already created.
 593    *
 594    * @param opts User-specified options for processing the application.
 595    * @return staging directory
 596    * @throws IOException
 597    */
 598   @Override
 599   public File createStagingDirectory(ApplicationProcessingOptions opts,
 600       ResourceLimits resourceLimits) throws IOException {
 601     if (stageDir != null) {
 602       return stageDir;
 603     }
 604
 605     int i = 0;
 606     while (stageDir == null && i++ < 3) {
 607       try {
 608         stageDir = File.createTempFile(STAGEDIR_PREFIX, null);
 609       } catch (IOException ex) {
 610         continue;
 611       }
 612       stageDir.delete();
 613       if (!stageDir.mkdir()) {
 614         stageDir = null;
 615       }
 616     }
 617     if (i == 3) {
 618       throw new IOException("Couldn't create a temporary directory in 3 tries.");
 619     }
 620     statusUpdate("Created staging directory at: '" + stageDir.getPath() + "'", 20);
 621
 622     String runtime = getRuntime(opts);
 623     return populateStagingDirectory(opts, resourceLimits, false, runtime);
 624   }
 625
 626   /**
 627    * Populates and creates (if necessary) a user specified, staging directory
 628    *
 629    * @param opts User-specified options for processing the application.
 630    * @param resourceLimits Various resource limits provided by the cloud.
 631    * @param stagingDir User-specified staging directory (must be empty or not exist)
 632    * @return staging directory
 633    * @throws IOException if an error occurs trying to create or populate the staging directory
 634    */
 635   @Override
 636   public File createStagingDirectory(ApplicationProcessingOptions opts,
 637       ResourceLimits resourceLimits, File stagingDir) throws IOException {
 638     if (!stagingDir.exists()) {
 639       if (!stagingDir.mkdir()) {
 640         throw new IOException("Could not create staging directory at " + stagingDir.getPath());
 641       }
 642     }
 643
 644     stageDir = stagingDir;
 645     ln = null;
 646
 647     String runtime = getRuntime(opts);
 648     populateStagingDirectory(opts, resourceLimits, true, runtime);
 649     copyOrLinkDirectories(GenerationDirectory.getGenerationDirectory(stageDir), stageDir, runtime);
 650     return stageDir;
 651   }
 652
 653   private File populateStagingDirectory(ApplicationProcessingOptions opts,
 654       ResourceLimits resourceLimits, boolean writeScrubbedAppYaml, String runtime)
 655       throws IOException {
 656     File staticDir = new File(stageDir, "__static__");
 657     staticDir.mkdir();
 658     copyOrLink(baseDir, stageDir, staticDir, false, opts, runtime);
 659     if (externalResourceDir != null) {
 660       String previousPrefix = appEngineWebXml.getSourcePrefix();
 661       String newPrefix = buildNormalizedPath(externalResourceDir);
 662       try {
 663         appEngineWebXml.setSourcePrefix(newPrefix);
 664         copyOrLink(externalResourceDir, stageDir, staticDir, false, opts, runtime);
 665       } finally {
 666         appEngineWebXml.setSourcePrefix(previousPrefix);
 667       }
 668     }
 669
 670     apiVersion = findApiVersion(stageDir, true);
 671
 672     if (opts.isCompileJspsSet()) {
 673       compileJsps(stageDir, opts, runtime);
 674     }
 675
 676     if (opts.isQuickstart()) {
 677       try {
 678         createQuickstartWebXml(opts);
 679         webXml = new WebXmlReader(stageDir.getAbsolutePath(), "/WEB-INF/min-quickstart-web.xml")
 680             .readWebXml();
 681       } catch (SAXException | ParserConfigurationException | TransformerException e) {
 682         throw new IOException(e);
 683       }
 684     }
 685
 686     appYaml = generateAppYaml(stageDir, runtime, appEngineWebXml);
 687
 688     if (GenerationDirectory.getGenerationDirectory(stageDir).mkdirs()) {
 689       writePreparedYamlFile("app", writeScrubbedAppYaml
 690           ? generateAppYaml(stageDir, runtime, getScrubbedAppEngineWebXml()) : appYaml);
 691       writePreparedYamlFile("backends", backendsXml == null ? null : backendsXml.toYaml());
 692       writePreparedYamlFile("index", indexesXml.size() == 0 ? null : indexesXml.toYaml());
 693       writePreparedYamlFile("cron", cronXml == null ? null : cronXml.toYaml());
 694       writePreparedYamlFile("queue", queueXml == null ? null : queueXml.toYaml());
 695       writePreparedYamlFile("dos", dosXml == null ? null : dosXml.toYaml());
 696     }
 697
 698     int maxJarSize = (int) resourceLimits.maxFileSize();
 699
 700     if (opts.isSplitJarsSet()) {
 701       splitJars(new File(new File(stageDir, "WEB-INF"), "lib"),
 702                 maxJarSize, opts.getJarSplittingExcludes());
 703     }
 704
 705     return stageDir;
 706   }
 707
 708   @Override
 709   public void exportRepoInfoFile() {
 710     File target = new File(stageDir, "WEB-INF/classes/source-context.json");
 711     if (target.exists()) {
 712       return;
 713     }
 714
 715     if (sourceContext == null || sourceContext.getJson() == null) {
 716       return;
 717     }
 718
 719     try {
 720       Files.write(sourceContext.getJson(), target, UTF_8);
 721     } catch (IOException ex) {
 722       logger.log(Level.FINE, "Failed to write git repository information file.", ex);
 723       return;
 724     }
 725
 726     statusUpdate("Generated git repository information file.");
 727   }
 728
 729   /**
 730    * Write yaml file to generation subdirectory within stage directory.
 731    */
 732   private void writePreparedYamlFile(String yamlName, String yamlString) throws IOException {
 733     File f = new File(GenerationDirectory.getGenerationDirectory(stageDir), yamlName + ".yaml");
 734     if (yamlString != null  && f.createNewFile()) {
 735       FileWriter fw = new FileWriter(f);
 736       fw.write(yamlString);
 737       fw.close();
 738     }
 739   }
 740
 741   private static String findApiVersion(File baseDir, boolean deleteApiJars) {
 742     ApiVersionFinder finder = new ApiVersionFinder();
 743
 744     String foundApiVersion = null;
 745     File webInf = new File(baseDir, "WEB-INF");
 746     File libDir = new File(webInf, "lib");
 747     for (File file : new FileIterator(libDir)) {
 748       if (file.getPath().endsWith(".jar")) {
 749         try {
 750           String apiVersion = finder.findApiVersion(file);
 751           if (apiVersion != null) {
 752             if (foundApiVersion == null) {
 753               foundApiVersion = apiVersion;
 754             } else if (!foundApiVersion.equals(apiVersion)) {
 755               logger.warning("Warning: found duplicate API version: " + foundApiVersion +
 756                              ", using " + apiVersion);
 757             }
 758             if (deleteApiJars) {
 759               if (!file.delete()) {
 760                  logger.log(Level.SEVERE, "Could not delete API jar: " + file);
 761               }
 762             }
 763           }
 764         } catch (IOException ex) {
 765           logger.log(Level.WARNING, "Could not identify API version in " + file, ex);
 766         }
 767       }
 768     }
 769
 770     if (foundApiVersion == null) {
 771       foundApiVersion = "none";
 772     }
 773     return foundApiVersion;
 774   }
 775
 776   /**
 777    * Returns the runtime id to use in the generated app.yaml.
 778    *
 779    * This method returns {@code "java7"}, unless an explicit runtime id was specified
 780    * using the {@code -r} option.
 781    *
 782    * Before accepting an explicit runtime id, this method validates it against the list of
 783    * supported Java runtimes (currently only {@code "java7"}), unless validation was turned
 784    * off using the {@code --allowAnyRuntimes} option.
 785    */
 786   private String getRuntime(ApplicationProcessingOptions opts) {
 787     boolean vm = appEngineWebXml.getUseVm() || appEngineWebXml.getEnv().equals("2");
 788     if (vm && new File(baseDir, "Dockerfile").exists()) {
 789       return "custom";
 790     }
 791     String runtime = opts.getRuntime();
 792     if (runtime != null) {
 793       if (!opts.isAllowAnyRuntime() && !ALLOWED_RUNTIME_IDS.contains(runtime)) {
 794         throw new AppEngineConfigException("Invalid runtime id: " + runtime + ". Valid " +
 795             "runtime id: java7.");
 796       }
 797       return runtime;
 798     }
 799     return JAVA_7_RUNTIME_ID;
 800   }
 801
 802   private static final String JSPC_MAIN = "com.google.appengine.tools.development.LocalJspC";
 803
 804   private void compileJsps(File stage, ApplicationProcessingOptions opts, String runtime)
 805       throws IOException {
 806     statusUpdate("Scanning for jsp files.");
 807
 808     if (matchingFileExists(new File(stage.getPath()), JSP_REGEX)) {
 809       statusUpdate("Compiling jsp files.");
 810
 811       File webInf = new File(stage, "WEB-INF");
 812
 813       for (File file : SdkImplInfo.getUserJspLibFiles()) {
 814         copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()), runtime);
 815       }
 816       for (File file : SdkImplInfo.getSharedJspLibFiles()) {
 817         copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()), runtime);
 818       }
 819
 820       File classes = new File(webInf, "classes");
 821       File generatedWebXml = new File(webInf, "generated_web.xml");
 822       File tempDir = Files.createTempDir();
 823       String classpath = getJspClasspath(classes, tempDir);
 824
 825       String javaCmd = opts.getJavaExecutable().getPath();
 826       String[] args = new String[] {
 827         javaCmd,
 828         "-classpath", classpath,
 829         JSPC_MAIN,
 830         "-uriroot", stage.getPath(),
 831         "-p", "org.apache.jsp",
 832         "-l", "-v",
 833         "-webinc", generatedWebXml.getPath(),
 834         "-d", tempDir.getPath(),
 835         "-javaEncoding", opts.getCompileEncoding(),
 836       };
 837       Process jspc = startProcess(args);
 838
 839       int status = 1;
 840       try {
 841         status = jspc.waitFor();
 842       } catch (InterruptedException ex) { }
 843
 844       if (status != 0) {
 845         detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
 846         throw new JspCompilationException("Failed to compile jsp files.",
 847                                           JspCompilationException.Source.JASPER);
 848       }
 849
 850       compileJavaFiles(classpath, webInf, tempDir, opts, runtime);
 851
 852       webXml = new WebXmlReader(stage.getPath()).readWebXml();
 853     }
 854   }
 855
 856   private void compileJavaFiles(String classpath, File webInf, File jspClassDir,
 857       ApplicationProcessingOptions opts, String runtime) throws IOException {
 858
 859     JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
 860     if (compiler == null) {
 861       throw new RuntimeException(
 862           "Cannot get the System Java Compiler. Please use a JDK, not a JRE.");
 863     }
 864     StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
 865
 866     ArrayList<File> files = new ArrayList<File>();
 867     for (File f : new FileIterator(jspClassDir)) {
 868       if (f.getPath().toLowerCase().endsWith(".java")) {
 869         files.add(f);
 870       }
 871     }
 872     if (files.isEmpty()) {
 873       return;
 874     }
 875     List<String> optionList = new ArrayList<String>();
 876     optionList.addAll(Arrays.asList("-classpath", classpath.toString()));
 877     optionList.addAll(Arrays.asList("-d", jspClassDir.getPath()));
 878     optionList.addAll(Arrays.asList("-encoding", opts.getCompileEncoding()));
 879     if (runtime.equals(JAVA_7_RUNTIME_ID)) {
 880       optionList.addAll(Arrays.asList("-source", "7"));
 881       optionList.addAll(Arrays.asList("-target", "7"));
 882     }
 883
 884     Iterable<? extends JavaFileObject> compilationUnits =
 885         fileManager.getJavaFileObjectsFromFiles(files);
 886     boolean success = compiler.getTask(
 887         null, fileManager, null, optionList, null, compilationUnits).call();
 888     fileManager.close();
 889
 890     if (!success) {
 891       throw new JspCompilationException("Failed to compile the generated JSP java files.",
 892           JspCompilationException.Source.JSPC);
 893     }
 894     if (opts.isJarJSPsSet()) {
 895       zipJasperGeneratedFiles(webInf, jspClassDir);
 896     } else {
 897       copyOrLinkDirectories(jspClassDir, new File(webInf, "classes"), runtime);
 898     }
 899     if (opts.isDeleteJSPs()) {
 900       for (File f : new FileIterator(webInf.getParentFile())) {
 901         if (f.getPath().toLowerCase().endsWith(".jsp")) {
 902           f.delete();
 903         }
 904       }
 905     }
 906     if (opts.isJarClassesSet()) {
 907       zipWebInfClassesFiles(webInf);
 908
 909     }
 910   }
 911
 912   private void zipJasperGeneratedFiles(File webInfDir, File jspClassDir) throws IOException {
 913     Set<String> fileTypesToExclude = ImmutableSet.of(".java");
 914     File libDir = new File(webInfDir, "lib");
 915     JarTool jarTool = new JarTool(
 916         COMPILED_JSP_JAR_NAME_PREFIX, jspClassDir, libDir, MAX_COMPILED_JSP_JAR_SIZE,
 917         fileTypesToExclude);
 918     jarTool.run();
 919     recursiveDelete(jspClassDir);
 920   }
 921
 922   private void zipWebInfClassesFiles(File webInfDir) throws IOException {
 923     File libDir = new File(webInfDir, "lib");
 924     File classesDir = new File(webInfDir, "classes");
 925     JarTool jarTool = new JarTool(
 926         CLASSES_JAR_NAME_PREFIX, classesDir, libDir, MAX_CLASSES_JAR_SIZE,
 927         null);
 928     jarTool.run();
 929     recursiveDelete(classesDir);
 930     classesDir.mkdir();
 931   }
 932
 933   private String getJspClasspath(File classDir, File genDir) {
 934     StringBuilder classpath = new StringBuilder();
 935     for (URL lib : SdkImplInfo.getImplLibs()) {
 936       classpath.append(lib.getPath());
 937       classpath.append(File.pathSeparatorChar);
 938     }
 939     for (File lib : SdkInfo.getSharedLibFiles()) {
 940       classpath.append(lib.getPath());
 941       classpath.append(File.pathSeparatorChar);
 942     }
 943
 944     classpath.append(classDir.getPath());
 945     classpath.append(File.pathSeparatorChar);
 946     classpath.append(genDir.getPath());
 947     classpath.append(File.pathSeparatorChar);
 948
 949     for (File f : new FileIterator(new File(classDir.getParentFile(), "lib"))) {
 950       String filename = f.getPath().toLowerCase();
 951       if (filename.endsWith(".jar") || filename.endsWith(".zip")) {
 952         classpath.append(f.getPath());
 953         classpath.append(File.pathSeparatorChar);
 954       }
 955     }
 956
 957     return classpath.toString();
 958   }
 959
 960   private Process startProcess(String... args) throws IOException {
 961     ProcessBuilder builder = new ProcessBuilder(args);
 962     Process proc = builder.redirectErrorStream(true).start();
 963     logger.fine(formatCommand(builder.command()));
 964     new Thread(new OutputPump(proc.getInputStream(), detailsWriter)).start();
 965     return proc;
 966   }
 967
 968   private String formatCommand(Iterable<String> args) {
 969     StringBuilder command = new StringBuilder();
 970     for (String chunk : args) {
 971       command.append(chunk);
 972       command.append(" ");
 973     }
 974     return command.toString();
 975   }
 976
 977   /**
 978    * Scans a given directory tree, testing whether any file matches a given
 979    * pattern.
 980    *
 981    * @param dir the directory under which to scan
 982    * @param regex the pattern to look for
 983    * @returns Returns {@code true} on the first instance of such a file,
 984    *   {@code false} otherwise.
 985    */
 986   private static boolean matchingFileExists(File dir, Pattern regex) {
 987     for (File file : dir.listFiles()) {
 988       if (file.isDirectory()) {
 989         if (matchingFileExists(file, regex)) {
 990           return true;
 991         }
 992       } else {
 993         if (regex.matcher(file.getName()).matches()) {
 994           return true;
 995         }
 996       }
 997     }
 998     return false;
 999   }
1000
1001   /**
1002    * Invokes the JarSplitter code on any jar files found in {@code dir}.  Any
1003    * jars larger than {@code max} will be split into fragments of at most that
1004    * size.
1005    * @param dir the directory to search, recursively
1006    * @param max the maximum allowed size
1007    * @param excludes a set of suffixes to exclude.
1008    * @throws IOException on filesystem errors.
1009    */
1010   private static void splitJars(File dir, int max, Set<String> excludes) throws IOException {
1011     String children[] = dir.list();
1012     if (children == null) {
1013       return;
1014     }
1015     for (String name : children) {
1016       File subfile = new File(dir, name);
1017       if (subfile.isDirectory()) {
1018         splitJars(subfile, max, excludes);
1019       } else if (name.endsWith(".jar")) {
1020         if (subfile.length() > max) {
1021           new JarSplitter(subfile, dir, max, false, 4, excludes).run();
1022           subfile.delete();
1023         }
1024       }
1025     }
1026   }
1027
1028   private static final Pattern SKIP_FILES = Pattern.compile(
1029       "^(.*/)?((#.*#)|(.*~)|(.*/RCS/.*)|)$");
1030
1031   /**
1032    * Copies files from the app to the upload staging directory, or makes
1033    * symlinks instead if supported.  Puts the files into the correct places for
1034    * static vs. resource files, recursively.
1035    *
1036    * @param sourceDir application war dir, or on recursion a subdirectory of it
1037    * @param resDir staging resource dir, or on recursion a subdirectory matching
1038    *    the subdirectory in {@code sourceDir}
1039    * @param staticDir staging {@code __static__} dir, or an appropriate recursive
1040    *    subdirectory
1041    * @param forceResource if all files should be considered resource files
1042    * @param opts processing options, used primarily for handling of *.jsp files
1043    * @throws FileNotFoundException
1044    * @throws IOException
1045    */
1046   private void copyOrLink(File sourceDir, File resDir, File staticDir, boolean forceResource,
1047       ApplicationProcessingOptions opts, String runtime)
1048     throws FileNotFoundException, IOException {
1049
1050     for (String name : sourceDir.list()) {
1051       File file = new File(sourceDir, name);
1052
1053       String path = file.getPath();
1054       if (File.separatorChar == '\\') {
1055         path = path.replace('\\', '/');
1056       }
1057
1058       if (file.getName().startsWith(".") ||
1059           file.equals(GenerationDirectory.getGenerationDirectory(baseDir))) {
1060         continue;
1061       }
1062
1063       if (file.isDirectory()) {
1064         if (file.getName().equals("WEB-INF")) {
1065           copyOrLink(file, new File(resDir, name), new File(staticDir, name), true, opts, runtime);
1066         } else {
1067           copyOrLink(file, new File(resDir, name), new File(staticDir, name), forceResource,
1068               opts, runtime);
1069         }
1070       } else {
1071         if (SKIP_FILES.matcher(path).matches()) {
1072           continue;
1073         }
1074
1075         if (forceResource || appEngineWebXml.includesResource(path) ||
1076             (opts.isCompileJspsSet() && name.toLowerCase().endsWith(".jsp"))) {
1077           copyOrLinkFile(file, new File(resDir, name), runtime);
1078         }
1079         if (!forceResource && appEngineWebXml.includesStatic(path)) {
1080           copyOrLinkFile(file, new File(staticDir, name), runtime);
1081         }
1082       }
1083     }
1084   }
1085
1086   /**
1087    * Attempts to symlink a single file, or copies it if symlinking is either
1088    * unsupported or fails.
1089    *
1090    * @param source source file
1091    * @param dest destination file
1092    * @throws FileNotFoundException
1093    * @throws IOException
1094    */
1095   private void copyOrLinkFile(File source, File dest, String runtime)
1096       throws FileNotFoundException, IOException {
1097     if (runtime.equals(JAVA_7_RUNTIME_ID)) {
1098       checkJavaVersion(source, 7);
1099     }
1100     dest.getParentFile().mkdirs();
1101     if (ln != null && !source.getName().endsWith("web.xml")) {
1102
1103       try {
1104         dest.delete();
1105       } catch (Exception e) {
1106         System.err.println("Warning: We tried to delete " + dest.getPath());
1107         System.err.println("in order to create a symlink from " + source.getPath());
1108         System.err.println("but the delete failed with message: " + e.getMessage());
1109       }
1110
1111       Process link = startProcess(ln.getAbsolutePath(), "-s",
1112                                   source.getAbsolutePath(),
1113                                   dest.getAbsolutePath());
1114       try {
1115         int stat = link.waitFor();
1116         if (stat == 0) {
1117           return;
1118         }
1119         System.err.println(ln.getAbsolutePath() + " returned status " + stat
1120             + ", copying instead...");
1121       } catch (InterruptedException ex) {
1122         System.err.println(ln.getAbsolutePath() + " was interrupted, copying instead...");
1123       }
1124       if (dest.delete()) {
1125         System.err.println("ln failed but symlink was created, removed: " + dest.getAbsolutePath());
1126       }
1127     }
1128     try (FileInputStream inStream = new FileInputStream(source);
1129          FileOutputStream outStream = new FileOutputStream(dest)) {
1130       byte buffer[] = new byte[1024];
1131       int readlen = inStream.read(buffer);
1132       while (readlen > 0) {
1133         outStream.write(buffer, 0, readlen);
1134         readlen = inStream.read(buffer);
1135       }
1136     }
1137   }
1138
1139   /** Copy (or link) one directory into another one.
1140    */
1141   private void copyOrLinkDirectories(File sourceDir, File destination, String runtime)
1142       throws IOException {
1143
1144     for (String name : sourceDir.list()) {
1145       File file = new File(sourceDir, name);
1146       if (file.isDirectory()) {
1147         copyOrLinkDirectories(file, new File(destination, name), runtime);
1148       } else {
1149         copyOrLinkFile(file, new File(destination, name), runtime);
1150       }
1151     }
1152   }
1153
1154   private void checkJavaVersion(File file, int maxVersion) {
1155     String name = file.getName();
1156     try {
1157       if (name.endsWith(".class")) {
1158         checkJavaClassVersion(file, maxVersion);
1159       } else if (name.endsWith(".jar")) {
1160         checkJavaJarVersion(file, maxVersion);
1161       }
1162     } catch (IOException e) {
1163     }
1164   }
1165
1166   private void checkJavaClassVersion(File classFile, int maxVersion) throws IOException {
1167     try (InputStream inputStream = new FileInputStream(classFile)) {
1168       checkJavaVersion(inputStream, maxVersion, classFile.getPath());
1169     }
1170   }
1171
1172   private void checkJavaJarVersion(File jarFile, int maxVersion) throws IOException {
1173     try (JarInputStream jarInputStream = new JarInputStream(new FileInputStream(jarFile))) {
1174       while (true) {
1175         JarEntry jarEntry = jarInputStream.getNextJarEntry();
1176         if (jarEntry == null) {
1177           return;
1178         }
1179         if (jarEntry.getName().endsWith(".class")) {
1180           checkJavaVersion(jarInputStream, maxVersion, jarEntry.getName() + " in " + jarFile);
1181           return;
1182         }
1183       }
1184     }
1185   }
1186
1187   private void checkJavaVersion(
1188       InputStream inputStream, int maxVersion, String what) throws IOException {
1189     DataInputStream in = new DataInputStream(inputStream);
1190     if (in.readInt() == 0xcafebabe) {
1191       in.readShort();
1192       int majorVersion = in.readShort();
1193       int actualVersion = majorVersion - 44;
1194       if (actualVersion > maxVersion) {
1195         throw new IllegalArgumentException(
1196             String.format("Class file is Java %d but max supported is Java %d: %s",
1197                 actualVersion, maxVersion, what));
1198       }
1199     }
1200   }
1201
1202   /** deletes the staging directory, if one was created. */
1203   @Override
1204   public void cleanStagingDirectory() {
1205     if (stageDir != null) {
1206       recursiveDelete(stageDir);
1207     }
1208   }
1209
1210   /** Recursive directory deletion. */
1211   public static void recursiveDelete(File dead) {
1212     String[] files = dead.list();
1213     if (files != null) {
1214       for (String name : files) {
1215         recursiveDelete(new File(dead, name));
1216       }
1217     }
1218     dead.delete();
1219   }
1220
1221   @Override
1222   public void setListener(UpdateListener l) {
1223     listener = l;
1224   }
1225
1226   @Override
1227   public void setDetailsWriter(PrintWriter detailsWriter) {
1228     this.detailsWriter = detailsWriter;
1229   }
1230
1231   @Override
1232   public void statusUpdate(String message, int amount) {
1233     updateProgress += progressAmount;
1234     if (updateProgress > 99) {
1235       updateProgress = 99;
1236     }
1237     progressAmount = amount;
1238     if (listener != null) {
1239       listener.onProgress(new UpdateProgressEvent(
1240                               Thread.currentThread(), message, updateProgress));
1241     }
1242   }
1243
1244   @Override
1245   public void statusUpdate(String message) {
1246     int amount = progressAmount / 4;
1247     updateProgress += amount;
1248     if (updateProgress > 99) {
1249       updateProgress = 99;
1250     }
1251     progressAmount -= amount;
1252     if (listener != null) {
1253       listener.onProgress(new UpdateProgressEvent(
1254                               Thread.currentThread(), message, updateProgress));
1255     }
1256   }
1257
1258   private String generateAppYaml(File stageDir, String runtime, AppEngineWebXml aeWebXml) {
1259     Set<String> staticFiles = new HashSet<String>();
1260     for (File f : new FileIterator(new File(stageDir, "__static__"))) {
1261       staticFiles.add(Utility.calculatePath(f, stageDir));
1262     }
1263
1264     AppYamlTranslator translator =
1265         new AppYamlTranslator(aeWebXml, getWebXml(), getBackendsXml(),
1266                               getApiVersion(), staticFiles, null, runtime, getSdkVersion());
1267     String yaml = translator.getYaml();
1268     logger.fine("Generated app.yaml file:\n" + yaml);
1269     return yaml;
1270   }
1271
1272   /**
1273    * Returns the app.yaml string.
1274    *
1275    * @throws IllegalStateException if createStagingDirectory has not been called.
1276    */
1277   @Override
1278   public String getAppYaml() {
1279     if (appYaml == null) {
1280       throw new IllegalStateException("Must call createStagingDirectory first.");
1281     }
1282     return appYaml;
1283   }
1284
1285   /**
1286    * Generates a quickstart-web.xml. Minimizes and saves in min-quickstart-web.xml
1287    * @return Relative path to min-quickstart-web.xml
1288    */
1289   private void createQuickstartWebXml(ApplicationProcessingOptions opts)
1290       throws IOException, SAXException, ParserConfigurationException, TransformerException {
1291     String javaCmd = opts.getJavaExecutable().getPath();
1292
1293     String quickstartJar = new File(SdkInfo.getSdkRoot(),
1294         "lib/java-managed-vm/appengine-java-vmruntime/quickstartgenerator.jar").getAbsolutePath();
1295
1296     String[] args = {
1297         javaCmd,
1298         "-jar", quickstartJar,
1299         stageDir.getAbsolutePath()
1300     };
1301     Process quickstartProcess = startProcess(args);
1302
1303     int status;
1304     try {
1305       status = quickstartProcess.waitFor();
1306     } catch (InterruptedException ex) {
1307       status = 1;
1308     }
1309
1310     if (status != 0) {
1311       detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
1312       throw new RuntimeException("Failed to generate quickstart-web.xml.");
1313     }
1314
1315     File webDefaultXml = new File(SdkInfo.getSdkRoot() + "/lib/jetty-base-sdk/etc/webdefault.xml");
1316     File quickstartXml = new File(stageDir, "/WEB-INF/quickstart-web.xml");
1317     File minimizedQuickstartXml = new File(stageDir, "/WEB-INF/min-quickstart-web.xml");
1318
1319     Document quickstartDoc = getFilteredQuickstartDoc(quickstartXml, webDefaultXml);
1320
1321     Transformer transformer = TransformerFactory.newInstance().newTransformer();
1322     transformer.setOutputProperty(OutputKeys.INDENT, "yes");
1323     StreamResult result = new StreamResult(new FileWriter(minimizedQuickstartXml));
1324     DOMSource source = new DOMSource(quickstartDoc);
1325     transformer.transform(source, result);
1326   }
1327
1328   /**
1329    * Removes mappings from quickstart-web.xml that come from webdefault.xml.
1330    *
1331    * The quickstart-web.xml generated by the quickstartgenerator process includes
1332    * the contents of the user's web.xml, entries derived from Java annotations,
1333    * and entries derived from the contents of webdefault.xml. All of those are
1334    * appropriate for the Java web server. But when generating an app.yaml for
1335    * appcfg or dev_appserver, the webdefault.xml entries are not appropriate, since
1336    * app.yaml should only reflect what is specific to the user's app. So this
1337    * method returns a modified min-quickstart-web Document from which the webdefault.xml
1338    * entries have been removed. Specifically, we look at the <url-pattern> inside
1339    * every <servlet-mapping> or <filter-mapping> element in webdefault.xml to
1340    * determine default patterns; then we look at those elements inside
1341    * quickstart-web.xml and remove any whose <url-pattern> is one of the default
1342    * patterns.
1343    *
1344    * @return a filtered quickstart Document object appropriate for translation to app.yaml
1345    */
1346   static Document getFilteredQuickstartDoc(File quickstartXml, File webDefaultXml)
1347       throws ParserConfigurationException, IOException, SAXException {
1348
1349     DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
1350     DocumentBuilder webDefaultDocBuilder = docBuilderFactory.newDocumentBuilder();
1351     Document webDefaultDoc = webDefaultDocBuilder.parse(webDefaultXml);
1352     DocumentBuilder quickstartDocBuilder = docBuilderFactory.newDocumentBuilder();
1353     Document quickstartDoc = quickstartDocBuilder.parse(quickstartXml);
1354     final Set<String> tagsToExamine = ImmutableSet.of("filter-mapping", "servlet-mapping");
1355     final String urlPatternTag = "url-pattern";
1356
1357     Set<String> defaultRoots = Sets.newHashSet();
1358     List<Node> nodesToRemove = Lists.newArrayList();
1359
1360     webDefaultDoc.getDocumentElement().normalize();
1361     NodeList webDefaultChildren = webDefaultDoc.getDocumentElement()
1362         .getElementsByTagName(urlPatternTag);
1363     for (int i = 0; i < webDefaultChildren.getLength(); i++) {
1364       Node child = webDefaultChildren.item(i);
1365       if (tagsToExamine.contains(child.getParentNode().getNodeName())) {
1366         String url = child.getTextContent().trim();
1367         if (url.startsWith("/")) {
1368           defaultRoots.add(url);
1369         }
1370       }
1371     }
1372
1373     quickstartDoc.getDocumentElement().normalize();
1374     NodeList quickstartChildren = quickstartDoc.getDocumentElement()
1375         .getElementsByTagName(urlPatternTag);
1376     for (int i = 0; i < quickstartChildren.getLength(); i++) {
1377       Node child = quickstartChildren.item(i);
1378       if (tagsToExamine.contains(child.getParentNode().getNodeName())) {
1379         String url = child.getTextContent().trim();
1380         if (defaultRoots.contains(url)) {
1381           nodesToRemove.add(child.getParentNode());
1382         }
1383       }
1384     }
1385     for (Node node : nodesToRemove) {
1386       quickstartDoc.getDocumentElement().removeChild(node);
1387     }
1388
1389     return quickstartDoc;
1390   }
1391 }