java/src/main/com/google/appengine/tools/admin/Application.java

   1 // Copyright 2008 Google Inc. All Rights Reserved.
   2
   3 package com.google.appengine.tools.admin;
   4
   5 import static java.nio.charset.StandardCharsets.UTF_8;
   6
   7 import com.google.appengine.tools.admin.AppAdminFactory.ApplicationProcessingOptions;
   8 import com.google.appengine.tools.admin.RepoInfo.SourceContext;
   9 import com.google.appengine.tools.info.SdkImplInfo;
  10 import com.google.appengine.tools.info.SdkInfo;
  11 import com.google.appengine.tools.info.Version;
  12 import com.google.appengine.tools.util.ApiVersionFinder;
  13 import com.google.appengine.tools.util.FileIterator;
  14 import com.google.appengine.tools.util.JarSplitter;
  15 import com.google.appengine.tools.util.JarTool;
  16 import com.google.apphosting.utils.config.AppEngineConfigException;
  17 import com.google.apphosting.utils.config.AppEngineWebXml;
  18 import com.google.apphosting.utils.config.AppEngineWebXmlReader;
  19 import com.google.apphosting.utils.config.AppYamlProcessor;
  20 import com.google.apphosting.utils.config.BackendsXml;
  21 import com.google.apphosting.utils.config.BackendsXmlReader;
  22 import com.google.apphosting.utils.config.BackendsYamlReader;
  23 import com.google.apphosting.utils.config.CronXml;
  24 import com.google.apphosting.utils.config.CronXmlReader;
  25 import com.google.apphosting.utils.config.CronYamlReader;
  26 import com.google.apphosting.utils.config.DispatchXml;
  27 import com.google.apphosting.utils.config.DispatchXmlReader;
  28 import com.google.apphosting.utils.config.DispatchYamlReader;
  29 import com.google.apphosting.utils.config.DosXml;
  30 import com.google.apphosting.utils.config.DosXmlReader;
  31 import com.google.apphosting.utils.config.DosYamlReader;
  32 import com.google.apphosting.utils.config.GenerationDirectory;
  33 import com.google.apphosting.utils.config.IndexesXml;
  34 import com.google.apphosting.utils.config.IndexesXmlReader;
  35 import com.google.apphosting.utils.config.QueueXml;
  36 import com.google.apphosting.utils.config.QueueXmlReader;
  37 import com.google.apphosting.utils.config.QueueYamlReader;
  38 import com.google.apphosting.utils.config.WebXml;
  39 import com.google.apphosting.utils.config.WebXmlReader;
  40 import com.google.apphosting.utils.config.XmlUtils;
  41 import com.google.common.collect.ImmutableSet;
  42 import com.google.common.collect.Lists;
  43 import com.google.common.collect.Sets;
  44 import com.google.common.io.Files;
  45
  46 import org.mortbay.io.Buffer;
  47 import org.mortbay.jetty.MimeTypes;
  48 import org.w3c.dom.Document;
  49 import org.w3c.dom.Node;
  50 import org.w3c.dom.NodeList;
  51 import org.xml.sax.SAXException;
  52
  53 import java.io.DataInputStream;
  54 import java.io.File;
  55 import java.io.FileInputStream;
  56 import java.io.FileNotFoundException;
  57 import java.io.FileOutputStream;
  58 import java.io.FileWriter;
  59 import java.io.IOException;
  60 import java.io.InputStream;
  61 import java.io.PrintWriter;
  62 import java.net.URL;
  63 import java.net.URLConnection;
  64 import java.util.ArrayList;
  65 import java.util.Arrays;
  66 import java.util.HashSet;
  67 import java.util.List;
  68 import java.util.Set;
  69 import java.util.jar.JarEntry;
  70 import java.util.jar.JarInputStream;
  71 import java.util.logging.Level;
  72 import java.util.logging.Logger;
  73 import java.util.regex.Pattern;
  74
  75 import javax.activation.FileTypeMap;
  76 import javax.tools.JavaCompiler;
  77 import javax.tools.JavaFileObject;
  78 import javax.tools.StandardJavaFileManager;
  79 import javax.tools.ToolProvider;
  80 import javax.xml.parsers.DocumentBuilder;
  81 import javax.xml.parsers.DocumentBuilderFactory;
  82 import javax.xml.parsers.ParserConfigurationException;
  83 import javax.xml.transform.OutputKeys;
  84 import javax.xml.transform.Transformer;
  85 import javax.xml.transform.TransformerException;
  86 import javax.xml.transform.TransformerFactory;
  87 import javax.xml.transform.dom.DOMSource;
  88 import javax.xml.transform.stream.StreamResult;
  89
  90 /**
  91  * An App Engine application. You can {@link #readApplication read} an
  92  * {@code Application} from a path, and
  93  * {@link com.google.appengine.tools.admin.AppAdminFactory#createAppAdmin create}
  94  * an {@link com.google.appengine.tools.admin.AppAdmin} to upload, create
  95  * indexes, or otherwise manage it.
  96  *
  97  */
  98 public class Application implements GenericApplication {
  99
 100   private static final int MAX_COMPILED_JSP_JAR_SIZE = 1024 * 1024 * 5;
 101   private static final String COMPILED_JSP_JAR_NAME_PREFIX = "_ah_compiled_jsps";
 102
 103   private static final int MAX_CLASSES_JAR_SIZE = 1024 * 1024 * 5;
 104   private static final String CLASSES_JAR_NAME_PREFIX = "_ah_webinf_classes";
 105
 106   private static final String JAVA_7_RUNTIME_ID = "java7";
 107   private static final ImmutableSet<String> ALLOWED_RUNTIME_IDS = ImmutableSet.of(
 108       JAVA_7_RUNTIME_ID);
 109
 110   private static final String BETA_SOURCE_REFERENCE_KEY = "source_reference";
 111
 112   private static final Pattern JSP_REGEX = Pattern.compile(".*\\.jspx?");
 113
 114   static final String DEFAULT_WEB_XML_CONTENT =
 115       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
 116           + "<web-app version=\"3.1\" "
 117           + "xmlns=\"http://xmlns.jcp.org/xml/ns/javaee\" "
 118           + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" "
 119           + "xsi:schemaLocation=\"http://xmlns.jcp.org/xml/ns/javaee "
 120           + "http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd\">"
 121           + "</web-app>";
 122
 123   /** If available, this is set to a program to make symlinks, e.g. /bin/ln */
 124   private static File ln = Utility.findLink();
 125   private static File sdkDocsDir;
 126   public static synchronized File getSdkDocsDir(){
 127     if (null == sdkDocsDir){
 128       sdkDocsDir = new File(SdkInfo.getSdkRoot(), "docs");
 129     }
 130     return sdkDocsDir;
 131   }
 132
 133   private static Version sdkVersion;
 134   public static synchronized Version getSdkVersion() {
 135     if (null == sdkVersion) {
 136       sdkVersion = SdkInfo.getLocalVersion();
 137     }
 138     return sdkVersion;
 139   }
 140
 141   private static final String STAGEDIR_PREFIX = "appcfg";
 142
 143   private static final Logger logger = Logger.getLogger(Application.class.getName());
 144
 145   private static final MimeTypes mimeTypes = new MimeTypes();
 146
 147   private AppEngineWebXml appEngineWebXml;
 148   private WebXml webXml;
 149   private CronXml cronXml;
 150   private DispatchXml dispatchXml;
 151   private DosXml dosXml;
 152   private String pagespeedYaml;
 153   private QueueXml queueXml;
 154   private IndexesXml indexesXml;
 155   private BackendsXml backendsXml;
 156   private File baseDir;
 157   private File stageDir;
 158   private File externalResourceDir;
 159   private String apiVersion;
 160   private String appYaml;
 161   private SourceContext sourceContext;
 162
 163   private UpdateListener listener;
 164   private PrintWriter detailsWriter;
 165   private int updateProgress = 0;
 166   private int progressAmount = 0;
 167
 168   protected Application(){
 169   }
 170
 171   /**
 172    * Builds a normalized path for the given directory in which
 173    * forward slashes are used as the file separator on all platforms.
 174    * @param dir A directory
 175    * @return The normalized path
 176    */
 177   private static String buildNormalizedPath(File dir) {
 178     String normalizedPath = dir.getPath();
 179     if (File.separatorChar == '\\') {
 180       normalizedPath = normalizedPath.replace('\\', '/');
 181     }
 182     return normalizedPath;
 183   }
 184
 185   private Application(String explodedPath, String appId, String module, String appVersion,
 186        RepoInfo.SourceContext sourceContext) {
 187     this.baseDir = new File(explodedPath);
 188     explodedPath = buildNormalizedPath(baseDir);
 189     File webinf = new File(baseDir, "WEB-INF");
 190     if (!webinf.getName().equals("WEB-INF")) {
 191       throw new AppEngineConfigException("WEB-INF directory must be capitalized.");
 192     }
 193
 194     String webinfPath = webinf.getPath();
 195     AppEngineWebXmlReader aewebReader = new AppEngineWebXmlReader(explodedPath);
 196     WebXmlReader webXmlReader = new WebXmlReader(explodedPath);
 197     AppYamlProcessor.convert(webinf, aewebReader.getFilename(), webXmlReader.getFilename());
 198
 199     File webXmlFile = new File(webinfPath, "web.xml");
 200     if (!webXmlFile.exists()) {
 201       writeDefaultWebXml(webXmlFile);
 202     }
 203
 204     if (new File(aewebReader.getFilename()).exists()) {
 205       XmlUtils.validateXml(
 206           aewebReader.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd"));
 207     }
 208     appEngineWebXml = aewebReader.readAppEngineWebXml();
 209     appEngineWebXml.setSourcePrefix(explodedPath);
 210     if (appId == null) {
 211       if (appEngineWebXml.getAppId() == null) {
 212         throw new AppEngineConfigException(
 213             "No app id supplied and XML files have no <application> element");
 214       }
 215     } else {
 216       appEngineWebXml.setAppId(appId);
 217     }
 218     if (module != null) {
 219       appEngineWebXml.setModule(module);
 220     }
 221     if (appVersion != null) {
 222       appEngineWebXml.setMajorVersionId(appVersion);
 223     }
 224
 225     if (sourceContext == null) {
 226       sourceContext = new RepoInfo(baseDir).getSourceContext();
 227       if (sourceContext != null) {
 228         String sourceRef = sourceContext.getRevisionId();
 229         if (sourceContext.getRepositoryUrl() != null) {
 230           sourceRef = sourceContext.getRepositoryUrl() + "#" + sourceRef;
 231         }
 232         appEngineWebXml.addBetaSetting(BETA_SOURCE_REFERENCE_KEY, sourceRef);
 233       }
 234     }
 235     this.sourceContext = sourceContext;
 236
 237     webXml = webXmlReader.readWebXml();
 238     webXml.validate();
 239
 240     CronXmlReader cronReader = new CronXmlReader(explodedPath);
 241     if (new File(cronReader.getFilename()).exists()) {
 242       XmlUtils.validateXml(cronReader.getFilename(), new File(getSdkDocsDir(), "cron.xsd"));
 243     }
 244     cronXml = cronReader.readCronXml();
 245     if (cronXml == null) {
 246       CronYamlReader cronYaml = new CronYamlReader(webinfPath);
 247       cronXml = cronYaml.parse();
 248     }
 249
 250     QueueXmlReader queueReader = new QueueXmlReader(explodedPath);
 251     if (new File(queueReader.getFilename()).exists()) {
 252       XmlUtils.validateXml(queueReader.getFilename(), new File(getSdkDocsDir(), "queue.xsd"));
 253     }
 254     queueXml = queueReader.readQueueXml();
 255     if (queueXml == null) {
 256       QueueYamlReader queueYaml = new QueueYamlReader(webinfPath);
 257       queueXml = queueYaml.parse();
 258     }
 259
 260     DispatchXmlReader dispatchXmlReader = new DispatchXmlReader(explodedPath,
 261         DispatchXmlReader.DEFAULT_RELATIVE_FILENAME);
 262     if (new File(dispatchXmlReader.getFilename()).exists()) {
 263       XmlUtils.validateXml(
 264           dispatchXmlReader.getFilename(), new File(getSdkDocsDir(), "dispatch.xsd"));
 265     }
 266     dispatchXml = dispatchXmlReader.readDispatchXml();
 267     if (dispatchXml == null) {
 268       DispatchYamlReader dispatchYamlReader = new DispatchYamlReader(webinfPath);
 269       dispatchXml = dispatchYamlReader.parse();
 270     }
 271
 272     DosXmlReader dosReader = new DosXmlReader(explodedPath);
 273     if (new File(dosReader.getFilename()).exists()) {
 274       XmlUtils.validateXml(dosReader.getFilename(), new File(getSdkDocsDir(), "dos.xsd"));
 275     }
 276     dosXml = dosReader.readDosXml();
 277     if (dosXml == null) {
 278       DosYamlReader dosYaml = new DosYamlReader(webinfPath);
 279       dosXml = dosYaml.parse();
 280     }
 281
 282     if (getAppEngineWebXml().getPagespeed() != null) {
 283       StringBuilder pagespeedYamlBuilder = new StringBuilder();
 284       AppYamlTranslator.appendPagespeed(
 285           getAppEngineWebXml().getPagespeed(), pagespeedYamlBuilder, 0);
 286       pagespeedYaml = pagespeedYamlBuilder.toString();
 287     }
 288
 289     IndexesXmlReader indexReader = new IndexesXmlReader(explodedPath);
 290     File datastoreSchema = new File(getSdkDocsDir(), "datastore-indexes.xsd");
 291     if (new File(indexReader.getFilename()).exists()) {
 292       XmlUtils.validateXml(indexReader.getFilename(), datastoreSchema);
 293     }
 294     indexesXml = indexReader.readIndexesXml();
 295
 296     BackendsXmlReader backendsReader = new BackendsXmlReader(explodedPath);
 297     if (new File(backendsReader.getFilename()).exists()) {
 298       XmlUtils.validateXml(backendsReader.getFilename(), new File(getSdkDocsDir(), "backends.xsd"));
 299     }
 300     backendsXml = backendsReader.readBackendsXml();
 301     if (backendsXml == null) {
 302       BackendsYamlReader backendsYaml = new BackendsYamlReader(webinfPath);
 303       backendsXml = backendsYaml.parse();
 304     }
 305   }
 306
 307   /**
 308    * Reads the App Engine application from {@code path}. The path may either
 309    * be a WAR file or the root of an exploded WAR directory.
 310    *
 311    * @param path a not {@code null} path.
 312    *
 313    * @throws IOException if an error occurs while trying to read the
 314    * {@code Application}.
 315    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 316    * {@code Application's} appengine-web.xml file is malformed.
 317    */
 318   public static Application readApplication(String path)
 319       throws IOException {
 320     return readApplication(path, null);
 321   }
 322
 323   /**
 324    * Reads the App Engine application from {@code path}. The path may either
 325    * be a WAR file or the root of an exploded WAR directory.
 326    *
 327    * @param path a not {@code null} path.
 328    * @param sourceContext an explicit RepoInfo.SourceContext. If {@code null}, the source
 329    *     context will be inferred from the current directory.
 330    *
 331    * @throws IOException if an error occurs while trying to read the
 332    * {@code Application}.
 333    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 334    * {@code Application's} appengine-web.xml file is malformed.
 335    */
 336   public static Application readApplication(String path, SourceContext sourceContext)
 337       throws IOException {
 338     return new Application(path, null, null, null, sourceContext);
 339   }
 340
 341   /**
 342    * Sets the external resource directory. Call this method before invoking
 343    * {@link #createStagingDirectory(ApplicationProcessingOptions, ResourceLimits)}.
 344    * <p>
 345    * The external resource directory is a directory outside of the war directory where additional
 346    * files live. These files will be copied into the staging directory during an upload, after the
 347    * war directory is copied there. Consequently if there are any name collisions the files in the
 348    * external resource directory will win.
 349    *
 350    * @param path a not {@code null} path to an existing directory.
 351    *
 352    * @throws IllegalArgumentException If {@code path} does not refer to an existing
 353    *         directory.
 354    */
 355   public void setExternalResourceDir(String path) {
 356     if (path == null) {
 357       throw new NullPointerException("path is null");
 358     }
 359     if (stageDir != null) {
 360       throw new IllegalStateException(
 361           "This method must be invoked prior to createStagingDirectory()");
 362     }
 363     File dir = new File(path);
 364     if (!dir.exists()) {
 365       throw new IllegalArgumentException("path does not exist: " + path);
 366     }
 367     if (!dir.isDirectory()) {
 368       throw new IllegalArgumentException(path + " is not a directory.");
 369     }
 370     this.externalResourceDir = dir;
 371   }
 372
 373   /**
 374    * Reads the App Engine application from {@code path}. The path may either
 375    * be a WAR file or the root of an exploded WAR directory.
 376    *
 377    * @param path a not {@code null} path.
 378    * @param appId if non-null, use this as an application id override.
 379    * @param module if non-null, use this as a module id override.
 380    * @param appVersion if non-null, use this as an application version override.
 381    *
 382    * @throws IOException if an error occurs while trying to read the
 383    * {@code Application}.
 384    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 385    * {@code Application's} appengine-web.xml file is malformed.
 386    */
 387   public static Application readApplication(String path,
 388       String appId,
 389       String module,
 390       String appVersion) throws IOException {
 391     return new Application(path, appId, module, appVersion, null);
 392   }
 393
 394   /**
 395    * Returns the application identifier, from the AppEngineWebXml config
 396    * @return application identifier
 397    */
 398   @Override
 399   public String getAppId() {
 400     return appEngineWebXml.getAppId();
 401   }
 402
 403   /**
 404    * Returns the application version, from the AppEngineWebXml config
 405    * @return application version
 406    */
 407   @Override
 408   public String getVersion() {
 409     return appEngineWebXml.getMajorVersionId();
 410   }
 411
 412   @Override
 413   public String getSourceLanguage() {
 414     return appEngineWebXml.getSourceLanguage();
 415   }
 416
 417   @Override
 418   public String getModule() {
 419     if (appEngineWebXml.getModule() != null) {
 420       return appEngineWebXml.getModule();
 421
 422     } else {
 423       return appEngineWebXml.getService();
 424     }
 425   }
 426
 427   @Override
 428   public String getInstanceClass() {
 429     return appEngineWebXml.getInstanceClass();
 430   }
 431
 432   @Override
 433   public boolean isPrecompilationEnabled() {
 434     return appEngineWebXml.getPrecompilationEnabled();
 435   }
 436
 437   @Override
 438   public List<ErrorHandler> getErrorHandlers() {
 439     class ErrorHandlerImpl implements ErrorHandler {
 440       private final AppEngineWebXml.ErrorHandler errorHandler;
 441       public ErrorHandlerImpl(AppEngineWebXml.ErrorHandler errorHandler) {
 442         this.errorHandler = errorHandler;
 443       }
 444       @Override
 445       public String getFile() {
 446         return "__static__/" + errorHandler.getFile();
 447       }
 448       @Override
 449       public String getErrorCode() {
 450         return errorHandler.getErrorCode();
 451       }
 452       @Override
 453       public String getMimeType() {
 454         return getMimeTypeIfStatic(getFile());
 455       }
 456     }
 457     List<ErrorHandler> errorHandlers = new ArrayList<ErrorHandler>();
 458     for (AppEngineWebXml.ErrorHandler errorHandler : appEngineWebXml.getErrorHandlers()) {
 459       errorHandlers.add(new ErrorHandlerImpl(errorHandler));
 460     }
 461     return errorHandlers;
 462   }
 463
 464   @Override
 465   public String getMimeTypeIfStatic(String path) {
 466     if (!path.contains("__static__/")) {
 467       return null;
 468     }
 469     String mimeType = webXml.getMimeTypeForPath(path);
 470     if (mimeType != null) {
 471       return mimeType;
 472     }
 473     return guessContentTypeFromName(path);
 474   }
 475
 476   /**
 477    * @param fileName path of a file with extension
 478    * @return the mimetype of the file (or application/octect-stream if not recognized)
 479    */
 480   public static String guessContentTypeFromName(String fileName) {
 481     String defaultValue = "application/octet-stream";
 482     try {
 483       Buffer buffer = mimeTypes.getMimeByExtension(fileName);
 484       if (buffer != null) {
 485         return new String(buffer.asArray());
 486       }
 487       String lowerName = fileName.toLowerCase();
 488       if (lowerName.endsWith(".json")) {
 489         return "application/json";
 490       }
 491       FileTypeMap typeMap = FileTypeMap.getDefaultFileTypeMap();
 492       String ret = typeMap.getContentType(fileName);
 493       if (ret != null) {
 494         return ret;
 495       }
 496       ret = URLConnection.guessContentTypeFromName(fileName);
 497       if (ret != null) {
 498         return ret;
 499       }
 500       return defaultValue;
 501     } catch (Throwable t) {
 502       logger.log(Level.WARNING, "Error identify mimetype for " + fileName, t);
 503       return defaultValue;
 504     }
 505   }
 506   /**
 507    * Returns the AppEngineWebXml describing the application.
 508    *
 509    * @return a not {@code null} deployment descriptor
 510    */
 511   public AppEngineWebXml getAppEngineWebXml() {
 512     return appEngineWebXml;
 513   }
 514
 515   /**
 516    * Returns the AppEngineWebXml with application and version removed
 517    *
 518    * @return a not {@code null} deployment descriptor
 519    */
 520   public AppEngineWebXml getScrubbedAppEngineWebXml() {
 521     AppEngineWebXml scrubbedAppEngineWebXml = appEngineWebXml.clone();
 522     scrubbedAppEngineWebXml.setAppId(null);
 523     scrubbedAppEngineWebXml.setMajorVersionId(null);
 524     return scrubbedAppEngineWebXml;
 525   }
 526
 527   /**
 528    * Returns the CronXml describing the applications' cron jobs.
 529    * @return a cron descriptor, possibly empty or {@code null}
 530    */
 531   @Override
 532   public CronXml getCronXml() {
 533     return cronXml;
 534   }
 535
 536   /**
 537    * Returns the QueueXml describing the applications' task queues.
 538    * @return a queue descriptor, possibly empty or {@code null}
 539    */
 540   @Override
 541   public QueueXml getQueueXml() {
 542     return queueXml;
 543   }
 544
 545   @Override
 546   public DispatchXml getDispatchXml() {
 547     return dispatchXml;
 548   }
 549
 550   /**
 551    * Returns the DosXml describing the applications' DoS entries.
 552    * @return a dos descriptor, possibly empty or {@code null}
 553    */
 554   @Override
 555   public DosXml getDosXml() {
 556     return dosXml;
 557   }
 558
 559   /**
 560    * Returns the pagespeed.yaml describing the applications' PageSpeed configuration.
 561    * @return a pagespeed.yaml string, possibly empty or {@code null}
 562    */
 563   @Override
 564   public String getPagespeedYaml() {
 565     return pagespeedYaml;
 566   }
 567
 568   /**
 569    * Returns the IndexesXml describing the applications' indexes.
 570    * @return a index descriptor, possibly empty or {@code null}
 571    */
 572   @Override
 573   public IndexesXml getIndexesXml() {
 574     return indexesXml;
 575   }
 576
 577   /**
 578    * Returns the WebXml describing the applications' servlets and generic web
 579    * application information.
 580    *
 581    * @return a WebXml descriptor, possibly empty but not {@code null}
 582    */
 583   public WebXml getWebXml() {
 584     return webXml;
 585   }
 586
 587   @Override
 588   public BackendsXml getBackendsXml() {
 589     return backendsXml;
 590   }
 591
 592   /**
 593    * Returns the desired API version for the current application, or
 594    * {@code "none"} if no API version was used.
 595    *
 596    * @throws IllegalStateException if createStagingDirectory has not been called.
 597    */
 598   @Override
 599   public String getApiVersion() {
 600     if (apiVersion == null) {
 601       throw new IllegalStateException("Must call createStagingDirectory first.");
 602     }
 603     return apiVersion;
 604   }
 605
 606   /**
 607    * Returns a path to an exploded WAR directory for the application.
 608    * This may be a temporary directory.
 609    *
 610    * @return a not {@code null} path pointing to a directory
 611    */
 612   @Override
 613   public String getPath() {
 614     return baseDir.getAbsolutePath();
 615   }
 616
 617   /**
 618    * Returns the staging directory, or {@code null} if none has been created.
 619    */
 620   @Override
 621   public File getStagingDir() {
 622     return stageDir;
 623   }
 624
 625   @Override
 626   public void resetProgress() {
 627     updateProgress = 0;
 628     progressAmount = 0;
 629   }
 630
 631   /**
 632    * Creates a new staging directory, if needed, or returns the existing one
 633    * if already created.
 634    *
 635    * @param opts User-specified options for processing the application.
 636    * @return staging directory
 637    * @throws IOException
 638    */
 639   @Override
 640   public File createStagingDirectory(ApplicationProcessingOptions opts,
 641       ResourceLimits resourceLimits) throws IOException {
 642     if (stageDir != null) {
 643       return stageDir;
 644     }
 645
 646     int i = 0;
 647     while (stageDir == null && i++ < 3) {
 648       try {
 649         stageDir = File.createTempFile(STAGEDIR_PREFIX, null);
 650       } catch (IOException ex) {
 651         continue;
 652       }
 653       stageDir.delete();
 654       if (!stageDir.mkdir()) {
 655         stageDir = null;
 656       }
 657     }
 658     if (i == 3) {
 659       throw new IOException("Couldn't create a temporary directory in 3 tries.");
 660     }
 661     statusUpdate("Created staging directory at: '" + stageDir.getPath() + "'", 20);
 662
 663     String runtime = getRuntime(opts);
 664     return populateStagingDirectory(opts, resourceLimits, false, runtime);
 665   }
 666
 667   /**
 668    * Populates and creates (if necessary) a user specified, staging directory
 669    *
 670    * @param opts User-specified options for processing the application.
 671    * @param resourceLimits Various resource limits provided by the cloud.
 672    * @param stagingDir User-specified staging directory (must be empty or not exist)
 673    * @return staging directory
 674    * @throws IOException if an error occurs trying to create or populate the staging directory
 675    */
 676   @Override
 677   public File createStagingDirectory(ApplicationProcessingOptions opts,
 678       ResourceLimits resourceLimits, File stagingDir) throws IOException {
 679     if (!stagingDir.exists()) {
 680       if (!stagingDir.mkdir()) {
 681         throw new IOException("Could not create staging directory at " + stagingDir.getPath());
 682       }
 683     }
 684
 685     stageDir = stagingDir;
 686     ln = null;
 687
 688     String runtime = getRuntime(opts);
 689     populateStagingDirectory(opts, resourceLimits, true, runtime);
 690     copyOrLinkDirectories(GenerationDirectory.getGenerationDirectory(stageDir), stageDir, runtime);
 691     return stageDir;
 692   }
 693
 694   private File populateStagingDirectory(ApplicationProcessingOptions opts,
 695       ResourceLimits resourceLimits, boolean writeScrubbedAppYaml, String runtime)
 696       throws IOException {
 697     File staticDir = new File(stageDir, "__static__");
 698     staticDir.mkdir();
 699     copyOrLink(baseDir, stageDir, staticDir, false, opts, runtime);
 700     if (externalResourceDir != null) {
 701       String previousPrefix = appEngineWebXml.getSourcePrefix();
 702       String newPrefix = buildNormalizedPath(externalResourceDir);
 703       try {
 704         appEngineWebXml.setSourcePrefix(newPrefix);
 705         copyOrLink(externalResourceDir, stageDir, staticDir, false, opts, runtime);
 706       } finally {
 707         appEngineWebXml.setSourcePrefix(previousPrefix);
 708       }
 709     }
 710
 711     apiVersion = findApiVersion(stageDir, true);
 712
 713     if (opts.isCompileJspsSet()) {
 714       compileJsps(stageDir, opts, runtime);
 715     }
 716
 717     if (opts.isQuickstart()) {
 718       try {
 719         createQuickstartWebXml(opts);
 720         webXml = new WebXmlReader(stageDir.getAbsolutePath(), "/WEB-INF/min-quickstart-web.xml")
 721             .readWebXml();
 722       } catch (SAXException | ParserConfigurationException | TransformerException e) {
 723         throw new IOException(e);
 724       }
 725     }
 726
 727     appYaml = generateAppYaml(stageDir, runtime, appEngineWebXml);
 728
 729     if (GenerationDirectory.getGenerationDirectory(stageDir).mkdirs()) {
 730       writePreparedYamlFile("app", writeScrubbedAppYaml
 731           ? generateAppYaml(stageDir, runtime, getScrubbedAppEngineWebXml()) : appYaml);
 732       writePreparedYamlFile("backends", backendsXml == null ? null : backendsXml.toYaml());
 733       writePreparedYamlFile("index", indexesXml.size() == 0 ? null : indexesXml.toYaml());
 734       writePreparedYamlFile("cron", cronXml == null ? null : cronXml.toYaml());
 735       writePreparedYamlFile("queue", queueXml == null ? null : queueXml.toYaml());
 736       writePreparedYamlFile("dos", dosXml == null ? null : dosXml.toYaml());
 737     }
 738
 739     int maxJarSize = (int) resourceLimits.maxFileSize();
 740
 741     if (opts.isSplitJarsSet()) {
 742       splitJars(new File(new File(stageDir, "WEB-INF"), "lib"),
 743                 maxJarSize, opts.getJarSplittingExcludes());
 744     }
 745
 746     exportRepoInfoFile();
 747
 748     return stageDir;
 749   }
 750
 751   @Override
 752   public void exportRepoInfoFile() {
 753     File target = new File(stageDir, "WEB-INF/classes/source-context.json");
 754     if (target.exists()) {
 755       return;
 756     }
 757
 758     if (sourceContext == null || sourceContext.getJson() == null) {
 759       return;
 760     }
 761
 762     try {
 763       target.getParentFile().mkdirs();
 764       Files.write(sourceContext.getJson(), target, UTF_8);
 765     } catch (IOException ex) {
 766       logger.log(Level.FINE, "Failed to write git repository information file.", ex);
 767       return;
 768     }
 769
 770     statusUpdate("Generated git repository information file.");
 771   }
 772
 773   /**
 774    * Write yaml file to generation subdirectory within stage directory.
 775    */
 776   private void writePreparedYamlFile(String yamlName, String yamlString) throws IOException {
 777     File f = new File(GenerationDirectory.getGenerationDirectory(stageDir), yamlName + ".yaml");
 778     if (yamlString != null  && f.createNewFile()) {
 779       FileWriter fw = new FileWriter(f);
 780       fw.write(yamlString);
 781       fw.close();
 782     }
 783   }
 784
 785   private static String findApiVersion(File baseDir, boolean deleteApiJars) {
 786     ApiVersionFinder finder = new ApiVersionFinder();
 787
 788     String foundApiVersion = null;
 789     File webInf = new File(baseDir, "WEB-INF");
 790     File libDir = new File(webInf, "lib");
 791     for (File file : new FileIterator(libDir)) {
 792       if (file.getPath().endsWith(".jar")) {
 793         try {
 794           String apiVersion = finder.findApiVersion(file);
 795           if (apiVersion != null) {
 796             if (foundApiVersion == null) {
 797               foundApiVersion = apiVersion;
 798             } else if (!foundApiVersion.equals(apiVersion)) {
 799               logger.warning("Warning: found duplicate API version: " + foundApiVersion +
 800                              ", using " + apiVersion);
 801             }
 802             if (deleteApiJars) {
 803               if (!file.delete()) {
 804                  logger.log(Level.SEVERE, "Could not delete API jar: " + file);
 805               }
 806             }
 807           }
 808         } catch (IOException ex) {
 809           logger.log(Level.WARNING, "Could not identify API version in " + file, ex);
 810         }
 811       }
 812     }
 813
 814     if (foundApiVersion == null) {
 815       foundApiVersion = "none";
 816     }
 817     return foundApiVersion;
 818   }
 819
 820   /**
 821    * Returns the runtime id to use in the generated app.yaml.
 822    *
 823    * This method returns {@code "java7"}, unless an explicit runtime id was specified
 824    * using the {@code -r} option.
 825    *
 826    * Before accepting an explicit runtime id, this method validates it against the list of
 827    * supported Java runtimes (currently only {@code "java7"}), unless validation was turned
 828    * off using the {@code --allowAnyRuntimes} option.
 829    */
 830   private String getRuntime(ApplicationProcessingOptions opts) {
 831     boolean vm = appEngineWebXml.getUseVm() || appEngineWebXml.getEnv().equals("2");
 832     if (vm && new File(baseDir, "Dockerfile").exists()) {
 833       return "custom";
 834     }
 835     String runtime = opts.getRuntime();
 836     if (runtime != null) {
 837       if (!opts.isAllowAnyRuntime() && !ALLOWED_RUNTIME_IDS.contains(runtime)) {
 838         throw new AppEngineConfigException("Invalid runtime id: " + runtime + ". Valid " +
 839             "runtime id: java7.");
 840       }
 841       return runtime;
 842     }
 843     return JAVA_7_RUNTIME_ID;
 844   }
 845
 846   private static final String JSPC_MAIN = "com.google.appengine.tools.development.LocalJspC";
 847
 848   private void compileJsps(File stage, ApplicationProcessingOptions opts, String runtime)
 849       throws IOException {
 850     statusUpdate("Scanning for jsp files.");
 851
 852     if (matchingFileExists(new File(stage.getPath()), JSP_REGEX)) {
 853       statusUpdate("Compiling jsp files.");
 854
 855       File webInf = new File(stage, "WEB-INF");
 856
 857       for (File file : SdkImplInfo.getUserJspLibFiles()) {
 858         copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()), runtime);
 859       }
 860       for (File file : SdkImplInfo.getSharedJspLibFiles()) {
 861         copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()), runtime);
 862       }
 863
 864       File classes = new File(webInf, "classes");
 865       File generatedWebXml = new File(webInf, "generated_web.xml");
 866       File tempDir = Files.createTempDir();
 867       String classpath = getJspClasspath(classes, tempDir);
 868
 869       String javaCmd = opts.getJavaExecutable().getPath();
 870       String[] args = new String[] {
 871         javaCmd,
 872         "-classpath", classpath,
 873         JSPC_MAIN,
 874         "-uriroot", stage.getPath(),
 875         "-p", "org.apache.jsp",
 876         "-l", "-v",
 877         "-webinc", generatedWebXml.getPath(),
 878         "-d", tempDir.getPath(),
 879         "-javaEncoding", opts.getCompileEncoding(),
 880       };
 881       Process jspc = startProcess(args);
 882
 883       int status = 1;
 884       try {
 885         status = jspc.waitFor();
 886       } catch (InterruptedException ex) { }
 887
 888       if (status != 0) {
 889         detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
 890         throw new JspCompilationException("Failed to compile jsp files.",
 891                                           JspCompilationException.Source.JASPER);
 892       }
 893
 894       compileJavaFiles(classpath, webInf, tempDir, opts, runtime);
 895
 896       webXml = new WebXmlReader(stage.getPath()).readWebXml();
 897     }
 898   }
 899
 900   private void compileJavaFiles(String classpath, File webInf, File jspClassDir,
 901       ApplicationProcessingOptions opts, String runtime) throws IOException {
 902
 903     JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
 904     if (compiler == null) {
 905       throw new RuntimeException(
 906           "Cannot get the System Java Compiler. Please use a JDK, not a JRE.");
 907     }
 908     StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
 909
 910     ArrayList<File> files = new ArrayList<File>();
 911     for (File f : new FileIterator(jspClassDir)) {
 912       if (f.getPath().toLowerCase().endsWith(".java")) {
 913         files.add(f);
 914       }
 915     }
 916     if (files.isEmpty()) {
 917       return;
 918     }
 919     List<String> optionList = new ArrayList<String>();
 920     optionList.addAll(Arrays.asList("-classpath", classpath.toString()));
 921     optionList.addAll(Arrays.asList("-d", jspClassDir.getPath()));
 922     optionList.addAll(Arrays.asList("-encoding", opts.getCompileEncoding()));
 923     if (runtime.equals(JAVA_7_RUNTIME_ID)) {
 924       optionList.addAll(Arrays.asList("-source", "7"));
 925       optionList.addAll(Arrays.asList("-target", "7"));
 926     }
 927
 928     Iterable<? extends JavaFileObject> compilationUnits =
 929         fileManager.getJavaFileObjectsFromFiles(files);
 930     boolean success = compiler.getTask(
 931         null, fileManager, null, optionList, null, compilationUnits).call();
 932     fileManager.close();
 933
 934     if (!success) {
 935       throw new JspCompilationException("Failed to compile the generated JSP java files.",
 936           JspCompilationException.Source.JSPC);
 937     }
 938     if (opts.isJarJSPsSet()) {
 939       zipJasperGeneratedFiles(webInf, jspClassDir);
 940     } else {
 941       copyOrLinkDirectories(jspClassDir, new File(webInf, "classes"), runtime);
 942     }
 943     if (opts.isDeleteJSPs()) {
 944       for (File f : new FileIterator(webInf.getParentFile())) {
 945         if (f.getPath().toLowerCase().endsWith(".jsp")) {
 946           f.delete();
 947         }
 948       }
 949     }
 950     if (opts.isJarClassesSet()) {
 951       zipWebInfClassesFiles(webInf);
 952
 953     }
 954   }
 955
 956   private void zipJasperGeneratedFiles(File webInfDir, File jspClassDir) throws IOException {
 957     Set<String> fileTypesToExclude = ImmutableSet.of(".java");
 958     File libDir = new File(webInfDir, "lib");
 959     JarTool jarTool = new JarTool(
 960         COMPILED_JSP_JAR_NAME_PREFIX, jspClassDir, libDir, MAX_COMPILED_JSP_JAR_SIZE,
 961         fileTypesToExclude);
 962     jarTool.run();
 963     recursiveDelete(jspClassDir);
 964   }
 965
 966   private void zipWebInfClassesFiles(File webInfDir) throws IOException {
 967     File libDir = new File(webInfDir, "lib");
 968     File classesDir = new File(webInfDir, "classes");
 969     JarTool jarTool = new JarTool(
 970         CLASSES_JAR_NAME_PREFIX, classesDir, libDir, MAX_CLASSES_JAR_SIZE,
 971         null);
 972     jarTool.run();
 973     recursiveDelete(classesDir);
 974     classesDir.mkdir();
 975   }
 976
 977   private String getJspClasspath(File classDir, File genDir) {
 978     StringBuilder classpath = new StringBuilder();
 979     for (URL lib : SdkImplInfo.getImplLibs()) {
 980       classpath.append(lib.getPath());
 981       classpath.append(File.pathSeparatorChar);
 982     }
 983     for (File lib : SdkInfo.getSharedLibFiles()) {
 984       classpath.append(lib.getPath());
 985       classpath.append(File.pathSeparatorChar);
 986     }
 987
 988     classpath.append(classDir.getPath());
 989     classpath.append(File.pathSeparatorChar);
 990     classpath.append(genDir.getPath());
 991     classpath.append(File.pathSeparatorChar);
 992
 993     for (File f : new FileIterator(new File(classDir.getParentFile(), "lib"))) {
 994       String filename = f.getPath().toLowerCase();
 995       if (filename.endsWith(".jar") || filename.endsWith(".zip")) {
 996         classpath.append(f.getPath());
 997         classpath.append(File.pathSeparatorChar);
 998       }
 999     }
1000
1001     return classpath.toString();
1002   }
1003
1004   private Process startProcess(String... args) throws IOException {
1005     ProcessBuilder builder = new ProcessBuilder(args);
1006     Process proc = builder.redirectErrorStream(true).start();
1007     logger.fine(formatCommand(builder.command()));
1008     new Thread(new OutputPump(proc.getInputStream(), detailsWriter)).start();
1009     return proc;
1010   }
1011
1012   private String formatCommand(Iterable<String> args) {
1013     StringBuilder command = new StringBuilder();
1014     for (String chunk : args) {
1015       command.append(chunk);
1016       command.append(" ");
1017     }
1018     return command.toString();
1019   }
1020
1021   /**
1022    * Scans a given directory tree, testing whether any file matches a given
1023    * pattern.
1024    *
1025    * @param dir the directory under which to scan
1026    * @param regex the pattern to look for
1027    * @returns Returns {@code true} on the first instance of such a file,
1028    *   {@code false} otherwise.
1029    */
1030   private static boolean matchingFileExists(File dir, Pattern regex) {
1031     for (File file : dir.listFiles()) {
1032       if (file.isDirectory()) {
1033         if (matchingFileExists(file, regex)) {
1034           return true;
1035         }
1036       } else {
1037         if (regex.matcher(file.getName()).matches()) {
1038           return true;
1039         }
1040       }
1041     }
1042     return false;
1043   }
1044
1045   /**
1046    * Invokes the JarSplitter code on any jar files found in {@code dir}.  Any
1047    * jars larger than {@code max} will be split into fragments of at most that
1048    * size.
1049    * @param dir the directory to search, recursively
1050    * @param max the maximum allowed size
1051    * @param excludes a set of suffixes to exclude.
1052    * @throws IOException on filesystem errors.
1053    */
1054   private static void splitJars(File dir, int max, Set<String> excludes) throws IOException {
1055     String[] children = dir.list();
1056     if (children == null) {
1057       return;
1058     }
1059     for (String name : children) {
1060       File subfile = new File(dir, name);
1061       if (subfile.isDirectory()) {
1062         splitJars(subfile, max, excludes);
1063       } else if (name.endsWith(".jar")) {
1064         if (subfile.length() > max) {
1065           new JarSplitter(subfile, dir, max, false, 4, excludes).run();
1066           subfile.delete();
1067         }
1068       }
1069     }
1070   }
1071
1072   private static final Pattern SKIP_FILES = Pattern.compile(
1073       "^(.*/)?((#.*#)|(.*~)|(.*/RCS/.*)|)$");
1074
1075   /**
1076    * Copies files from the app to the upload staging directory, or makes
1077    * symlinks instead if supported.  Puts the files into the correct places for
1078    * static vs. resource files, recursively.
1079    *
1080    * @param sourceDir application war dir, or on recursion a subdirectory of it
1081    * @param resDir staging resource dir, or on recursion a subdirectory matching
1082    *    the subdirectory in {@code sourceDir}
1083    * @param staticDir staging {@code __static__} dir, or an appropriate recursive
1084    *    subdirectory
1085    * @param forceResource if all files should be considered resource files
1086    * @param opts processing options, used primarily for handling of *.jsp files
1087    * @throws FileNotFoundException
1088    * @throws IOException
1089    */
1090   private void copyOrLink(File sourceDir, File resDir, File staticDir, boolean forceResource,
1091       ApplicationProcessingOptions opts, String runtime)
1092     throws FileNotFoundException, IOException {
1093
1094     for (String name : sourceDir.list()) {
1095       File file = new File(sourceDir, name);
1096
1097       String path = file.getPath();
1098       if (File.separatorChar == '\\') {
1099         path = path.replace('\\', '/');
1100       }
1101
1102       if (file.getName().startsWith(".") ||
1103           file.equals(GenerationDirectory.getGenerationDirectory(baseDir))) {
1104         continue;
1105       }
1106
1107       if (file.isDirectory()) {
1108         if (file.getName().equals("WEB-INF")) {
1109           copyOrLink(file, new File(resDir, name), new File(staticDir, name), true, opts, runtime);
1110         } else {
1111           copyOrLink(file, new File(resDir, name), new File(staticDir, name), forceResource,
1112               opts, runtime);
1113         }
1114       } else {
1115         if (SKIP_FILES.matcher(path).matches()) {
1116           continue;
1117         }
1118
1119         if (forceResource || appEngineWebXml.includesResource(path) ||
1120             (opts.isCompileJspsSet() && name.toLowerCase().endsWith(".jsp"))) {
1121           copyOrLinkFile(file, new File(resDir, name), runtime);
1122         }
1123         if (!forceResource && appEngineWebXml.includesStatic(path)) {
1124           copyOrLinkFile(file, new File(staticDir, name), runtime);
1125         }
1126       }
1127     }
1128   }
1129
1130   /**
1131    * Attempts to symlink a single file, or copies it if symlinking is either
1132    * unsupported or fails.
1133    *
1134    * @param source source file
1135    * @param dest destination file
1136    * @throws FileNotFoundException
1137    * @throws IOException
1138    */
1139   private void copyOrLinkFile(File source, File dest, String runtime)
1140       throws FileNotFoundException, IOException {
1141     if (runtime.equals(JAVA_7_RUNTIME_ID)) {
1142       checkJavaVersion(source, 7);
1143     }
1144     dest.getParentFile().mkdirs();
1145     if (ln != null && !source.getName().endsWith("web.xml")) {
1146
1147       try {
1148         dest.delete();
1149       } catch (Exception e) {
1150         System.err.println("Warning: We tried to delete " + dest.getPath());
1151         System.err.println("in order to create a symlink from " + source.getPath());
1152         System.err.println("but the delete failed with message: " + e.getMessage());
1153       }
1154
1155       Process link = startProcess(ln.getAbsolutePath(), "-s",
1156                                   source.getAbsolutePath(),
1157                                   dest.getAbsolutePath());
1158       try {
1159         int stat = link.waitFor();
1160         if (stat == 0) {
1161           return;
1162         }
1163         System.err.println(ln.getAbsolutePath() + " returned status " + stat
1164             + ", copying instead...");
1165       } catch (InterruptedException ex) {
1166         System.err.println(ln.getAbsolutePath() + " was interrupted, copying instead...");
1167       }
1168       if (dest.delete()) {
1169         System.err.println("ln failed but symlink was created, removed: " + dest.getAbsolutePath());
1170       }
1171     }
1172     try (FileInputStream inStream = new FileInputStream(source);
1173          FileOutputStream outStream = new FileOutputStream(dest)) {
1174       byte[] buffer = new byte[1024];
1175       int readlen = inStream.read(buffer);
1176       while (readlen > 0) {
1177         outStream.write(buffer, 0, readlen);
1178         readlen = inStream.read(buffer);
1179       }
1180     }
1181   }
1182
1183   /** Copy (or link) one directory into another one.
1184    */
1185   private void copyOrLinkDirectories(File sourceDir, File destination, String runtime)
1186       throws IOException {
1187
1188     for (String name : sourceDir.list()) {
1189       File file = new File(sourceDir, name);
1190       if (file.isDirectory()) {
1191         copyOrLinkDirectories(file, new File(destination, name), runtime);
1192       } else {
1193         copyOrLinkFile(file, new File(destination, name), runtime);
1194       }
1195     }
1196   }
1197
1198   private void checkJavaVersion(File file, int maxVersion) {
1199     String name = file.getName();
1200     try {
1201       if (name.endsWith(".class")) {
1202         checkJavaClassVersion(file, maxVersion);
1203       } else if (name.endsWith(".jar")) {
1204         checkJavaJarVersion(file, maxVersion);
1205       }
1206     } catch (IOException e) {
1207     }
1208   }
1209
1210   private void checkJavaClassVersion(File classFile, int maxVersion) throws IOException {
1211     try (InputStream inputStream = new FileInputStream(classFile)) {
1212       checkJavaVersion(inputStream, maxVersion, classFile.getPath());
1213     }
1214   }
1215
1216   private void checkJavaJarVersion(File jarFile, int maxVersion) throws IOException {
1217     try (JarInputStream jarInputStream = new JarInputStream(new FileInputStream(jarFile))) {
1218       while (true) {
1219         JarEntry jarEntry = jarInputStream.getNextJarEntry();
1220         if (jarEntry == null) {
1221           return;
1222         }
1223         if (jarEntry.getName().endsWith(".class")) {
1224           checkJavaVersion(jarInputStream, maxVersion, jarEntry.getName() + " in " + jarFile);
1225           return;
1226         }
1227       }
1228     }
1229   }
1230
1231   private void checkJavaVersion(
1232       InputStream inputStream, int maxVersion, String what) throws IOException {
1233     DataInputStream in = new DataInputStream(inputStream);
1234     if (in.readInt() == 0xcafebabe) {
1235       in.readShort();
1236       int majorVersion = in.readShort();
1237       int actualVersion = majorVersion - 44;
1238       if (actualVersion > maxVersion) {
1239         throw new IllegalArgumentException(
1240             String.format("Class file is Java %d but max supported is Java %d: %s",
1241                 actualVersion, maxVersion, what));
1242       }
1243     }
1244   }
1245
1246   /** deletes the staging directory, if one was created. */
1247   @Override
1248   public void cleanStagingDirectory() {
1249     if (stageDir != null) {
1250       recursiveDelete(stageDir);
1251     }
1252   }
1253
1254   /** Recursive directory deletion. */
1255   public static void recursiveDelete(File dead) {
1256     String[] files = dead.list();
1257     if (files != null) {
1258       for (String name : files) {
1259         recursiveDelete(new File(dead, name));
1260       }
1261     }
1262     dead.delete();
1263   }
1264
1265   @Override
1266   public void setListener(UpdateListener l) {
1267     listener = l;
1268   }
1269
1270   @Override
1271   public void setDetailsWriter(PrintWriter detailsWriter) {
1272     this.detailsWriter = detailsWriter;
1273   }
1274
1275   @Override
1276   public void statusUpdate(String message, int amount) {
1277     updateProgress += progressAmount;
1278     if (updateProgress > 99) {
1279       updateProgress = 99;
1280     }
1281     progressAmount = amount;
1282     if (listener != null) {
1283       listener.onProgress(new UpdateProgressEvent(
1284                               Thread.currentThread(), message, updateProgress));
1285     }
1286   }
1287
1288   @Override
1289   public void statusUpdate(String message) {
1290     int amount = progressAmount / 4;
1291     updateProgress += amount;
1292     if (updateProgress > 99) {
1293       updateProgress = 99;
1294     }
1295     progressAmount -= amount;
1296     if (listener != null) {
1297       listener.onProgress(new UpdateProgressEvent(
1298                               Thread.currentThread(), message, updateProgress));
1299     }
1300   }
1301
1302   private String generateAppYaml(File stageDir, String runtime, AppEngineWebXml aeWebXml) {
1303     Set<String> staticFiles = new HashSet<String>();
1304     for (File f : new FileIterator(new File(stageDir, "__static__"))) {
1305       staticFiles.add(Utility.calculatePath(f, stageDir));
1306     }
1307
1308     AppYamlTranslator translator =
1309         new AppYamlTranslator(aeWebXml, getWebXml(), getBackendsXml(),
1310                               getApiVersion(), staticFiles, null, runtime, getSdkVersion());
1311     String yaml = translator.getYaml();
1312     logger.fine("Generated app.yaml file:\n" + yaml);
1313     return yaml;
1314   }
1315
1316   /**
1317    * Returns the app.yaml string.
1318    *
1319    * @throws IllegalStateException if createStagingDirectory has not been called.
1320    */
1321   @Override
1322   public String getAppYaml() {
1323     if (appYaml == null) {
1324       throw new IllegalStateException("Must call createStagingDirectory first.");
1325     }
1326     return appYaml;
1327   }
1328
1329   private void writeDefaultWebXml(File webXmlFile) {
1330     try {
1331       Files.write(DEFAULT_WEB_XML_CONTENT, webXmlFile, UTF_8);
1332     } catch (IOException e) {
1333       throw new AppEngineConfigException("Error: "
1334           + "Could not autogenerate file " + webXmlFile.getAbsolutePath());
1335     }
1336   }
1337
1338   /**
1339    * Generates a quickstart-web.xml. Minimizes and saves in min-quickstart-web.xml
1340    * @return Relative path to min-quickstart-web.xml
1341    */
1342   private void createQuickstartWebXml(ApplicationProcessingOptions opts)
1343       throws IOException, SAXException, ParserConfigurationException, TransformerException {
1344     String javaCmd = opts.getJavaExecutable().getPath();
1345
1346     String quickstartJar = new File(SdkInfo.getSdkRoot(),
1347         "lib/java-managed-vm/appengine-java-vmruntime/quickstartgenerator.jar").getAbsolutePath();
1348
1349     String[] args = {
1350         javaCmd,
1351         "-jar", quickstartJar,
1352         stageDir.getAbsolutePath()
1353     };
1354     Process quickstartProcess = startProcess(args);
1355
1356     int status;
1357     try {
1358       status = quickstartProcess.waitFor();
1359     } catch (InterruptedException ex) {
1360       status = 1;
1361     }
1362
1363     if (status != 0) {
1364       detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
1365       throw new RuntimeException("Failed to generate quickstart-web.xml.");
1366     }
1367
1368     File webDefaultXml = new File(SdkInfo.getSdkRoot() + "/lib/jetty-base-sdk/etc/webdefault.xml");
1369     File quickstartXml = new File(stageDir, "/WEB-INF/quickstart-web.xml");
1370     File minimizedQuickstartXml = new File(stageDir, "/WEB-INF/min-quickstart-web.xml");
1371
1372     Document quickstartDoc = getFilteredQuickstartDoc(quickstartXml, webDefaultXml);
1373
1374     Transformer transformer = TransformerFactory.newInstance().newTransformer();
1375     transformer.setOutputProperty(OutputKeys.INDENT, "yes");
1376     StreamResult result = new StreamResult(new FileWriter(minimizedQuickstartXml));
1377     DOMSource source = new DOMSource(quickstartDoc);
1378     transformer.transform(source, result);
1379   }
1380
1381   /**
1382    * Removes mappings from quickstart-web.xml that come from webdefault.xml.
1383    *
1384    * The quickstart-web.xml generated by the quickstartgenerator process includes
1385    * the contents of the user's web.xml, entries derived from Java annotations,
1386    * and entries derived from the contents of webdefault.xml. All of those are
1387    * appropriate for the Java web server. But when generating an app.yaml for
1388    * appcfg or dev_appserver, the webdefault.xml entries are not appropriate, since
1389    * app.yaml should only reflect what is specific to the user's app. So this
1390    * method returns a modified min-quickstart-web Document from which the webdefault.xml
1391    * entries have been removed. Specifically, we look at the <url-pattern> inside
1392    * every <servlet-mapping> or <filter-mapping> element in webdefault.xml to
1393    * determine default patterns; then we look at those elements inside
1394    * quickstart-web.xml and remove any whose <url-pattern> is one of the default
1395    * patterns.
1396    *
1397    * @return a filtered quickstart Document object appropriate for translation to app.yaml
1398    */
1399   static Document getFilteredQuickstartDoc(File quickstartXml, File webDefaultXml)
1400       throws ParserConfigurationException, IOException, SAXException {
1401
1402     DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
1403     DocumentBuilder webDefaultDocBuilder = docBuilderFactory.newDocumentBuilder();
1404     Document webDefaultDoc = webDefaultDocBuilder.parse(webDefaultXml);
1405     DocumentBuilder quickstartDocBuilder = docBuilderFactory.newDocumentBuilder();
1406     Document quickstartDoc = quickstartDocBuilder.parse(quickstartXml);
1407     final Set<String> tagsToExamine = ImmutableSet.of("filter-mapping", "servlet-mapping");
1408     final String urlPatternTag = "url-pattern";
1409
1410     Set<String> defaultRoots = Sets.newHashSet();
1411     List<Node> nodesToRemove = Lists.newArrayList();
1412
1413     webDefaultDoc.getDocumentElement().normalize();
1414     NodeList webDefaultChildren = webDefaultDoc.getDocumentElement()
1415         .getElementsByTagName(urlPatternTag);
1416     for (int i = 0; i < webDefaultChildren.getLength(); i++) {
1417       Node child = webDefaultChildren.item(i);
1418       if (tagsToExamine.contains(child.getParentNode().getNodeName())) {
1419         String url = child.getTextContent().trim();
1420         if (url.startsWith("/")) {
1421           defaultRoots.add(url);
1422         }
1423       }
1424     }
1425
1426     quickstartDoc.getDocumentElement().normalize();
1427     NodeList quickstartChildren = quickstartDoc.getDocumentElement()
1428         .getElementsByTagName(urlPatternTag);
1429     for (int i = 0; i < quickstartChildren.getLength(); i++) {
1430       Node child = quickstartChildren.item(i);
1431       if (tagsToExamine.contains(child.getParentNode().getNodeName())) {
1432         String url = child.getTextContent().trim();
1433         if (defaultRoots.contains(url)) {
1434           nodesToRemove.add(child.getParentNode());
1435         }
1436       }
1437     }
1438     for (Node node : nodesToRemove) {
1439       quickstartDoc.getDocumentElement().removeChild(node);
1440     }
1441
1442     return quickstartDoc;
1443   }
1444 }