Revision created by MOE tool push_codebase.

[gae.git] / java / src / main / com / google / appengine / tools / admin / Application.java

// Copyright 2008 Google Inc. All Rights Reserved.

package com.google.appengine.tools.admin;

import com.google.appengine.tools.admin.AppAdminFactory.ApplicationProcessingOptions;
import com.google.appengine.tools.info.SdkImplInfo;
import com.google.appengine.tools.info.SdkInfo;
import com.google.appengine.tools.info.Version;
import com.google.appengine.tools.util.ApiVersionFinder;
import com.google.appengine.tools.util.FileIterator;
import com.google.appengine.tools.util.JarSplitter;
import com.google.appengine.tools.util.JarTool;
import com.google.apphosting.utils.config.AppEngineConfigException;
import com.google.apphosting.utils.config.AppEngineWebXml;
import com.google.apphosting.utils.config.AppEngineWebXmlReader;
import com.google.apphosting.utils.config.AppYamlProcessor;
import com.google.apphosting.utils.config.BackendsXml;
import com.google.apphosting.utils.config.BackendsXmlReader;
import com.google.apphosting.utils.config.BackendsYamlReader;
import com.google.apphosting.utils.config.CronXml;
import com.google.apphosting.utils.config.CronXmlReader;
import com.google.apphosting.utils.config.CronYamlReader;
import com.google.apphosting.utils.config.DispatchXml;
import com.google.apphosting.utils.config.DispatchXmlReader;
import com.google.apphosting.utils.config.DispatchYamlReader;
import com.google.apphosting.utils.config.DosXml;
import com.google.apphosting.utils.config.DosXmlReader;
import com.google.apphosting.utils.config.DosYamlReader;
import com.google.apphosting.utils.config.GenerationDirectory;
import com.google.apphosting.utils.config.IndexesXml;
import com.google.apphosting.utils.config.IndexesXmlReader;
import com.google.apphosting.utils.config.QueueXml;
import com.google.apphosting.utils.config.QueueXmlReader;
import com.google.apphosting.utils.config.QueueYamlReader;
import com.google.apphosting.utils.config.WebXml;
import com.google.apphosting.utils.config.WebXmlReader;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;

import org.mortbay.io.Buffer;
import org.mortbay.jetty.MimeTypes;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;

import javax.activation.FileTypeMap;
import javax.tools.JavaCompiler;
import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.SchemaFactory;

/**
 * An App Engine application. You can {@link #readApplication read} an
 * {@code Application} from a path, and
 * {@link com.google.appengine.tools.admin.AppAdminFactory#createAppAdmin create}
 * an {@link com.google.appengine.tools.admin.AppAdmin} to upload, create
 * indexes, or otherwise manage it.
 *
 */
public class Application implements GenericApplication {

  private static final int MAX_COMPILED_JSP_JAR_SIZE = 1024 * 1024 * 5;
  private static final String COMPILED_JSP_JAR_NAME_PREFIX = "_ah_compiled_jsps";

  private static final int MAX_CLASSES_JAR_SIZE = 1024 * 1024 * 5;
  private static final String CLASSES_JAR_NAME_PREFIX = "_ah_webinf_classes";

  private static final String JAVA_7_RUNTIME_ID = "java7";
  private static final ImmutableSet<String> ALLOWED_RUNTIME_IDS = ImmutableSet.of(
      JAVA_7_RUNTIME_ID);

  private static Pattern JSP_REGEX = Pattern.compile(".*\\.jspx?");

  /** If available, this is set to a program to make symlinks, e.g. /bin/ln */
  private static File ln = Utility.findLink();
  private static File sdkDocsDir;
  public static synchronized File getSdkDocsDir(){
    if (null == sdkDocsDir){
      sdkDocsDir = new File(SdkInfo.getSdkRoot(), "docs");
    }
    return sdkDocsDir;
  }

  private static Version sdkVersion;
  public static synchronized Version getSdkVersion() {
    if (null == sdkVersion) {
      sdkVersion = SdkInfo.getLocalVersion();
    }
    return sdkVersion;
  }

  private static final String STAGEDIR_PREFIX = "appcfg";

  private static final Logger logger = Logger.getLogger(Application.class.getName());

  private static final MimeTypes mimeTypes = new MimeTypes();

  private AppEngineWebXml appEngineWebXml;
  private WebXml webXml;
  private CronXml cronXml;
  private DispatchXml dispatchXml;
  private DosXml dosXml;
  private String pagespeedYaml;
  private QueueXml queueXml;
  private IndexesXml indexesXml;
  private BackendsXml backendsXml;
  private File baseDir;
  private File stageDir;
  private File externalResourceDir;
  private String apiVersion;
  private String appYaml;

  private UpdateListener listener;
  private PrintWriter detailsWriter;
  private int updateProgress = 0;
  private int progressAmount = 0;

  protected Application(){
  }

  /**
   * Builds a normalized path for the given directory in which
   * forward slashes are used as the file separator on all platforms.
   * @param dir A directory
   * @return The normalized path
   */
  private static String buildNormalizedPath(File dir) {
    String normalizedPath = dir.getPath();
    if (File.separatorChar == '\\') {
      normalizedPath = normalizedPath.replace('\\', '/');
    }
    return normalizedPath;
  }

  private Application(String explodedPath, String appId, String module, String appVersion) {
    this.baseDir = new File(explodedPath);
    explodedPath = buildNormalizedPath(baseDir);
    File webinf = new File(baseDir, "WEB-INF");
    if (!webinf.getName().equals("WEB-INF")) {
      throw new AppEngineConfigException("WEB-INF directory must be capitalized.");
    }

    String webinfPath = webinf.getPath();
    AppEngineWebXmlReader aewebReader = new AppEngineWebXmlReader(explodedPath);
    WebXmlReader webXmlReader = new WebXmlReader(explodedPath);
    AppYamlProcessor.convert(webinf, aewebReader.getFilename(), webXmlReader.getFilename());

    validateXml(aewebReader.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd"));
    appEngineWebXml = aewebReader.readAppEngineWebXml();
    appEngineWebXml.setSourcePrefix(explodedPath);
    if (appId == null) {
      if (appEngineWebXml.getAppId() == null) {
        throw new AppEngineConfigException(
            "No app id supplied and XML files have no <application> element");
      }
    } else {
      appEngineWebXml.setAppId(appId);
    }
    if (module != null) {
      appEngineWebXml.setModule(module);
    }
    if (appVersion != null) {
      appEngineWebXml.setMajorVersionId(appVersion);
    }

    webXml = webXmlReader.readWebXml();
    webXml.validate();

    CronXmlReader cronReader = new CronXmlReader(explodedPath);
    validateXml(cronReader.getFilename(), new File(getSdkDocsDir(), "cron.xsd"));
    cronXml = cronReader.readCronXml();
    if (cronXml == null) {
      CronYamlReader cronYaml = new CronYamlReader(webinfPath);
      cronXml = cronYaml.parse();
    }

    QueueXmlReader queueReader = new QueueXmlReader(explodedPath);
    validateXml(queueReader.getFilename(), new File(getSdkDocsDir(), "queue.xsd"));
    queueXml = queueReader.readQueueXml();
    if (queueXml == null) {
      QueueYamlReader queueYaml = new QueueYamlReader(webinfPath);
      queueXml = queueYaml.parse();
    }

    DispatchXmlReader dispatchXmlReader = new DispatchXmlReader(explodedPath,
        DispatchXmlReader.DEFAULT_RELATIVE_FILENAME);
    validateXml(dispatchXmlReader.getFilename(), new File(getSdkDocsDir(), "dispatch.xsd"));
    dispatchXml = dispatchXmlReader.readDispatchXml();
    if (dispatchXml == null) {
      DispatchYamlReader dispatchYamlReader = new DispatchYamlReader(webinfPath);
      dispatchXml = dispatchYamlReader.parse();
    }

    DosXmlReader dosReader = new DosXmlReader(explodedPath);
    validateXml(dosReader.getFilename(), new File(getSdkDocsDir(), "dos.xsd"));
    dosXml = dosReader.readDosXml();
    if (dosXml == null) {
      DosYamlReader dosYaml = new DosYamlReader(webinfPath);
      dosXml = dosYaml.parse();
    }

    if (getAppEngineWebXml().getPagespeed() != null) {
      StringBuilder pagespeedYamlBuilder = new StringBuilder();
      AppYamlTranslator.appendPagespeed(
          getAppEngineWebXml().getPagespeed(), pagespeedYamlBuilder, 0);
      pagespeedYaml = pagespeedYamlBuilder.toString();
    }

    IndexesXmlReader indexReader = new IndexesXmlReader(explodedPath);
    File datastoreSchema = new File(getSdkDocsDir(), "datastore-indexes.xsd");
    validateXml(indexReader.getFilename(), datastoreSchema);
    indexesXml = indexReader.readIndexesXml();

    BackendsXmlReader backendsReader = new BackendsXmlReader(explodedPath);
    validateXml(backendsReader.getFilename(), new File(getSdkDocsDir(), "backends.xsd"));
    backendsXml = backendsReader.readBackendsXml();
    if (backendsXml == null) {
      BackendsYamlReader backendsYaml = new BackendsYamlReader(webinfPath);
      backendsXml = backendsYaml.parse();
    }
  }

  /**
   * Reads the App Engine application from {@code path}. The path may either
   * be a WAR file or the root of an exploded WAR directory.
   *
   * @param path a not {@code null} path.
   *
   * @throws IOException if an error occurs while trying to read the
   * {@code Application}.
   * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
   * {@code Application's} appengine-web.xml file is malformed.
   */
  public static Application readApplication(String path)
      throws IOException {
    return new Application(path, null, null, null);
  }

  /**
   * Sets the external resource directory. Call this method before invoking
   * {@link #createStagingDirectory(ApplicationProcessingOptions, ResourceLimits)}.
   * <p>
   * The external resource directory is a directory outside of the war directory where additional
   * files live. These files will be copied into the staging directory during an upload, after the
   * war directory is copied there. Consequently if there are any name collisions the files in the
   * external resource directory will win.
   *
   * @param path a not {@code null} path to an existing directory.
   *
   * @throws IllegalArgumentException If {@code path} does not refer to an existing
   *         directory.
   */
  public void setExternalResourceDir(String path) {
    if (path == null) {
      throw new NullPointerException("path is null");
    }
    if (stageDir != null) {
      throw new IllegalStateException(
          "This method must be invoked prior to createStagingDirectory()");
    }
    File dir = new File(path);
    if (!dir.exists()) {
      throw new IllegalArgumentException("path does not exist: " + path);
    }
    if (!dir.isDirectory()) {
      throw new IllegalArgumentException(path + " is not a directory.");
    }
    this.externalResourceDir = dir;
  }

  /**
   * Reads the App Engine application from {@code path}. The path may either
   * be a WAR file or the root of an exploded WAR directory.
   *
   * @param path a not {@code null} path.
   * @param appId if non-null, use this as an application id override.
   * @param module if non-null, use this as a module id override.
   * @param appVersion if non-null, use this as an application version override.
   *
   * @throws IOException if an error occurs while trying to read the
   * {@code Application}.
   * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
   * {@code Application's} appengine-web.xml file is malformed.
   */
  public static Application readApplication(String path,
      String appId,
      String module,
      String appVersion) throws IOException {
    return new Application(path, appId, module, appVersion);
  }

  /**
   * Returns the application identifier, from the AppEngineWebXml config
   * @return application identifier
   */
  @Override
  public String getAppId() {
    return appEngineWebXml.getAppId();
  }

  /**
   * Returns the application version, from the AppEngineWebXml config
   * @return application version
   */
  @Override
  public String getVersion() {
    return appEngineWebXml.getMajorVersionId();
  }

  @Override
  public String getSourceLanguage() {
    return appEngineWebXml.getSourceLanguage();
  }

  @Override
  public String getModule() {
    return appEngineWebXml.getModule();
  }

  @Override
  public String getInstanceClass() {
    return appEngineWebXml.getInstanceClass();
  }

  @Override
  public boolean isPrecompilationEnabled() {
    return appEngineWebXml.getPrecompilationEnabled();
  }

  @Override
  public List<ErrorHandler> getErrorHandlers() {
    class ErrorHandlerImpl implements ErrorHandler {
      private final AppEngineWebXml.ErrorHandler errorHandler;
      public ErrorHandlerImpl(AppEngineWebXml.ErrorHandler errorHandler) {
        this.errorHandler = errorHandler;
      }
      @Override
      public String getFile() {
        return "__static__/" + errorHandler.getFile();
      }
      @Override
      public String getErrorCode() {
        return errorHandler.getErrorCode();
      }
      @Override
      public String getMimeType() {
        return getMimeTypeIfStatic(getFile());
      }
    }
    List<ErrorHandler> errorHandlers = new ArrayList<ErrorHandler>();
    for (AppEngineWebXml.ErrorHandler errorHandler: appEngineWebXml.getErrorHandlers()) {
      errorHandlers.add(new ErrorHandlerImpl(errorHandler));
    }
    return errorHandlers;
  }

  @Override
  public String getMimeTypeIfStatic(String path) {
    if (!path.contains("__static__/")) {
      return null;
    }
    String mimeType = webXml.getMimeTypeForPath(path);
    if (mimeType != null) {
      return mimeType;
    }
    return guessContentTypeFromName(path);
  }

  /**
   * @param fileName path of a file with extension
   * @return the mimetype of the file (or application/octect-stream if not recognized)
   */
  public static String guessContentTypeFromName(String fileName) {
    String defaultValue = "application/octet-stream";
    try {
      Buffer buffer = mimeTypes.getMimeByExtension(fileName);
      if (buffer != null) {
        return new String(buffer.asArray());
      }
      String lowerName = fileName.toLowerCase();
      if (lowerName.endsWith(".json")) {
        return "application/json";
      }
      FileTypeMap typeMap = FileTypeMap.getDefaultFileTypeMap();
      String ret = typeMap.getContentType(fileName);
      if (ret != null) {
        return ret;
      }
      ret = URLConnection.guessContentTypeFromName(fileName);
      if (ret != null) {
        return ret;
      }
      return defaultValue;
    } catch (Throwable t) {
      logger.log(Level.WARNING, "Error identify mimetype for " + fileName, t);
      return defaultValue;
    }
  }
  /**
   * Returns the AppEngineWebXml describing the application.
   *
   * @return a not {@code null} deployment descriptor
   */
  public AppEngineWebXml getAppEngineWebXml() {
    return appEngineWebXml;
  }

  /**
   * Returns the AppEngineWebXml with application and version removed
   *
   * @return a not {@code null} deployment descriptor
   */
  public AppEngineWebXml getScrubbedAppEngineWebXml() {
    AppEngineWebXml scrubbedAppEngineWebXml = appEngineWebXml.clone();
    scrubbedAppEngineWebXml.setAppId(null);
    scrubbedAppEngineWebXml.setMajorVersionId(null);
    return scrubbedAppEngineWebXml;
  }

  /**
   * Returns the CronXml describing the applications' cron jobs.
   * @return a cron descriptor, possibly empty or {@code null}
   */
  @Override
  public CronXml getCronXml() {
    return cronXml;
  }

  /**
   * Returns the QueueXml describing the applications' task queues.
   * @return a queue descriptor, possibly empty or {@code null}
   */
  @Override
  public QueueXml getQueueXml() {
    return queueXml;
  }

  @Override
  public DispatchXml getDispatchXml() {
    return dispatchXml;
  }

  /**
   * Returns the DosXml describing the applications' DoS entries.
   * @return a dos descriptor, possibly empty or {@code null}
   */
  @Override
  public DosXml getDosXml() {
    return dosXml;
  }

  /**
   * Returns the pagespeed.yaml describing the applications' PageSpeed configuration.
   * @return a pagespeed.yaml string, possibly empty or {@code null}
   */
  @Override
  public String getPagespeedYaml() {
    return pagespeedYaml;
  }

  /**
   * Returns the IndexesXml describing the applications' indexes.
   * @return a index descriptor, possibly empty or {@code null}
   */
  @Override
  public IndexesXml getIndexesXml() {
    return indexesXml;
  }

  /**
   * Returns the WebXml describing the applications' servlets and generic web
   * application information.
   *
   * @return a WebXml descriptor, possibly empty but not {@code null}
   */
  public WebXml getWebXml() {
    return webXml;
  }

  @Override
  public BackendsXml getBackendsXml() {
    return backendsXml;
  }

  /**
   * Returns the desired API version for the current application, or
   * {@code "none"} if no API version was used.
   *
   * @throws IllegalStateException if createStagingDirectory has not been called.
   */
  @Override
  public String getApiVersion() {
    if (apiVersion == null) {
      throw new IllegalStateException("Must call createStagingDirectory first.");
    }
    return apiVersion;
  }

  /**
   * Returns a path to an exploded WAR directory for the application.
   * This may be a temporary directory.
   *
   * @return a not {@code null} path pointing to a directory
   */
  @Override
  public String getPath() {
    return baseDir.getAbsolutePath();
  }

  /**
   * Returns the staging directory, or {@code null} if none has been created.
   */
  @Override
  public File getStagingDir() {
    return stageDir;
  }

  @Override
  public void resetProgress() {
    updateProgress = 0;
    progressAmount = 0;
  }

  /**
   * Creates a new staging directory, if needed, or returns the existing one
   * if already created.
   *
   * @param opts User-specified options for processing the application.
   * @return staging directory
   * @throws IOException
   */
  @Override
  public File createStagingDirectory(ApplicationProcessingOptions opts,
      ResourceLimits resourceLimits) throws IOException {
    if (stageDir != null) {
      return stageDir;
    }

    int i = 0;
    while (stageDir == null && i++ < 3) {
      try {
        stageDir = File.createTempFile(STAGEDIR_PREFIX, null);
      } catch (IOException ex) {
        continue;
      }
      stageDir.delete();
      if (!stageDir.mkdir()) {
        stageDir = null;
      }
    }
    if (i == 3) {
      throw new IOException("Couldn't create a temporary directory in 3 tries.");
    }
    statusUpdate("Created staging directory at: '" + stageDir.getPath() + "'", 20);

    return populateStagingDirectory(opts, resourceLimits, false);
  }

  /**
   * Populates and creates (if necessary) a user specified, staging directory
   *
   * @param opts User-specified options for processing the application.
   * @param resourceLimits Various resource limits provided by the cloud.
   * @param stagingDir User-specified staging directory (must be empty or not exist)
   * @return staging directory
   * @throws IOException if an error occurs trying to create or populate the staging directory
   */
  @Override
  public File createStagingDirectory(ApplicationProcessingOptions opts,
      ResourceLimits resourceLimits, File stagingDir) throws IOException {
    if (!stagingDir.exists()) {
      if (!stagingDir.mkdir()) {
        throw new IOException("Could not create staging directory at " + stagingDir.getPath());
      }
    }

    stageDir = stagingDir;
    ln = null;

    populateStagingDirectory(opts, resourceLimits, true);
    copyOrLinkDirectories(GenerationDirectory.getGenerationDirectory(stageDir), stageDir);
    return stageDir;
  }

  private File populateStagingDirectory(ApplicationProcessingOptions opts,
      ResourceLimits resourceLimits, boolean writeScrubbedAppYaml) throws IOException {
    File staticDir = new File(stageDir, "__static__");
    staticDir.mkdir();
    copyOrLink(baseDir, stageDir, staticDir, false, opts);
    if (externalResourceDir != null) {
      String previousPrefix = appEngineWebXml.getSourcePrefix();
      String newPrefix = buildNormalizedPath(externalResourceDir);
      try {
        appEngineWebXml.setSourcePrefix(newPrefix);
        copyOrLink(externalResourceDir, stageDir, staticDir, false, opts);
      } finally {
        appEngineWebXml.setSourcePrefix(previousPrefix);
      }
    }

    apiVersion = findApiVersion(stageDir, true);

    String runtime = getRuntime(opts);

    if (opts.isCompileJspsSet()) {
      compileJsps(stageDir, opts);
    }

    if (opts.isQuickstart()) {
      try {
        createQuickstartWebXml(opts);
        webXml = new WebXmlReader(stageDir.getAbsolutePath(), "/WEB-INF/min-quickstart-web.xml")
            .readWebXml();
      } catch (SAXException | ParserConfigurationException | TransformerException e) {
        throw new IOException(e);
      }
    }

    appYaml = generateAppYaml(stageDir, runtime, appEngineWebXml);

    if (GenerationDirectory.getGenerationDirectory(stageDir).mkdirs()) {
      writePreparedYamlFile("app", writeScrubbedAppYaml
          ? generateAppYaml(stageDir, runtime, getScrubbedAppEngineWebXml()) : appYaml);
      writePreparedYamlFile("backends", backendsXml == null ? null : backendsXml.toYaml());
      writePreparedYamlFile("index", indexesXml.size() == 0 ? null : indexesXml.toYaml());
      writePreparedYamlFile("cron", cronXml == null ? null : cronXml.toYaml());
      writePreparedYamlFile("queue", queueXml == null ? null : queueXml.toYaml());
      writePreparedYamlFile("dos", dosXml == null ? null : dosXml.toYaml());
    }

    int maxJarSize = (int) resourceLimits.maxFileSize();

    if (opts.isSplitJarsSet()) {
      splitJars(new File(new File(stageDir, "WEB-INF"), "lib"),
                maxJarSize, opts.getJarSplittingExcludes());
    }

    return stageDir;
  }

  @Override
  public void exportRepoInfoFile() {
    File target = new File(stageDir, "WEB-INF/classes/source-context.json");
    if (target.exists()) {
      return;
    }

    RepoInfo repoInfo = new RepoInfo(baseDir);
    if (!repoInfo.generate(target)) {
      return;
    }

    statusUpdate("Generated git repository information file.");
  }

  /**
   * Write yaml file to generation subdirectory within stage directory.
   */
  private void writePreparedYamlFile(String yamlName, String yamlString) throws IOException {
    File f = new File(GenerationDirectory.getGenerationDirectory(stageDir), yamlName + ".yaml");
    if (yamlString != null  && f.createNewFile()) {
      FileWriter fw = new FileWriter(f);
      fw.write(yamlString);
      fw.close();
    }
  }

  private static String findApiVersion(File baseDir, boolean deleteApiJars) {
    ApiVersionFinder finder = new ApiVersionFinder();

    String foundApiVersion = null;
    File webInf = new File(baseDir, "WEB-INF");
    File libDir = new File(webInf, "lib");
    for (File file : new FileIterator(libDir)) {
      if (file.getPath().endsWith(".jar")) {
        try {
          String apiVersion = finder.findApiVersion(file);
          if (apiVersion != null) {
            if (foundApiVersion == null) {
              foundApiVersion = apiVersion;
            } else if (!foundApiVersion.equals(apiVersion)) {
              logger.warning("Warning: found duplicate API version: " + foundApiVersion +
                             ", using " + apiVersion);
            }
            if (deleteApiJars) {
              if (!file.delete()) {
                 logger.log(Level.SEVERE, "Could not delete API jar: " + file);
              }
            }
          }
        } catch (IOException ex) {
          logger.log(Level.WARNING, "Could not identify API version in " + file, ex);
        }
      }
    }

    if (foundApiVersion == null) {
      foundApiVersion = "none";
    }
    return foundApiVersion;
  }

  /**
   * Returns the runtime id to use in the generated app.yaml.
   *
   * This method returns {@code "java7"}, unless an explicit runtime id was specified
   * using the {@code -r} option.
   *
   * Before accepting an explicit runtime id, this method validates it against the list of
   * supported Java runtimes (currently only {@code "java7"}), unless validation was turned
   * off using the {@code --allowAnyRuntimes} option.
   */
  private String getRuntime(ApplicationProcessingOptions opts) {
    String runtime = opts.getRuntime();
    if (runtime != null) {
      if (!opts.isAllowAnyRuntime() && !ALLOWED_RUNTIME_IDS.contains(runtime)) {
        throw new AppEngineConfigException("Invalid runtime id: " + runtime + ". Valid " +
            "runtime id: java7.");
      }
      return runtime;
    }
    return JAVA_7_RUNTIME_ID;
  }

  /**
   * Validates a given XML document against a given schema.
   *
   * @param xmlFilename filename with XML document
   * @param schema XSD schema to validate with
   *
   * @throws AppEngineConfigException for malformed XML, or IO errors
   */
  private static void validateXml(String xmlFilename, File schema) {
    File xml = new File(xmlFilename);
    if (!xml.exists()) {
      return;
    }
    try {
      SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
      try {
        factory.newSchema(schema).newValidator().validate(
            new StreamSource(new FileInputStream(xml)));
      } catch (SAXException ex) {
        throw new AppEngineConfigException("XML error validating " +
            xml.getPath() + " against " + schema.getPath(), ex);
      }
    } catch (IOException ex) {
      throw new AppEngineConfigException("IO error validating " +
          xml.getPath() + " against " + schema.getPath(), ex);
    }
  }

  private static final String JSPC_MAIN = "com.google.appengine.tools.development.LocalJspC";

  private void compileJsps(File stage, ApplicationProcessingOptions opts)
      throws IOException {
    statusUpdate("Scanning for jsp files.");

    if (matchingFileExists(new File(stage.getPath()), JSP_REGEX)) {
      statusUpdate("Compiling jsp files.");

      File webInf = new File(stage, "WEB-INF");

      for (File file : SdkImplInfo.getUserJspLibFiles()) {
        copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()));
      }
      for (File file : SdkImplInfo.getSharedJspLibFiles()) {
        copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()));
      }

      File classes = new File(webInf, "classes");
      File generatedWebXml = new File(webInf, "generated_web.xml");
      File tempDir = Files.createTempDir();
      String classpath = getJspClasspath(classes, tempDir);

      String javaCmd = opts.getJavaExecutable().getPath();
      String[] args = new String[] {
        javaCmd,
        "-classpath", classpath,
        JSPC_MAIN,
        "-uriroot", stage.getPath(),
        "-p", "org.apache.jsp",
        "-l", "-v",
        "-webinc", generatedWebXml.getPath(),
        "-d", tempDir.getPath(),
        "-javaEncoding", opts.getCompileEncoding(),
      };
      Process jspc = startProcess(args);

      int status = 1;
      try {
        status = jspc.waitFor();
      } catch (InterruptedException ex) { }

      if (status != 0) {
        detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
        throw new JspCompilationException("Failed to compile jsp files.",
                                          JspCompilationException.Source.JASPER);
      }

      compileJavaFiles(classpath, webInf, tempDir, opts);

      webXml = new WebXmlReader(stage.getPath()).readWebXml();
    }
  }

  private void compileJavaFiles(String classpath, File webInf, File jspClassDir,
      ApplicationProcessingOptions opts) throws IOException {

    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
    if (compiler == null) {
      throw new RuntimeException(
          "Cannot get the System Java Compiler. Please use a JDK, not a JRE.");
    }
    StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);

    ArrayList<File> files = new ArrayList<File>();
    for (File f : new FileIterator(jspClassDir)) {
      if (f.getPath().toLowerCase().endsWith(".java")) {
        files.add(f);
      }
    }
    if (files.isEmpty()) {
      return;
    }
    List<String> optionList = new ArrayList<String>();
    optionList.addAll(Arrays.asList("-classpath", classpath.toString()));
    optionList.addAll(Arrays.asList("-d", jspClassDir.getPath()));
    optionList.addAll(Arrays.asList("-encoding", opts.getCompileEncoding()));
    optionList.addAll(Arrays.asList("-source", "7"));
    optionList.addAll(Arrays.asList("-target", "7"));

    Iterable<? extends JavaFileObject> compilationUnits =
        fileManager.getJavaFileObjectsFromFiles(files);
    boolean success = compiler.getTask(
        null, fileManager, null, optionList, null, compilationUnits).call();
    fileManager.close();

    if (!success) {
      throw new JspCompilationException("Failed to compile the generated JSP java files.",
          JspCompilationException.Source.JSPC);
    }
    if (opts.isJarJSPsSet()) {
      zipJasperGeneratedFiles(webInf, jspClassDir);
    } else {
      copyOrLinkDirectories(jspClassDir, new File(webInf, "classes"));
    }
    if (opts.isDeleteJSPs()) {
      for (File f : new FileIterator(webInf.getParentFile())) {
        if (f.getPath().toLowerCase().endsWith(".jsp")) {
          f.delete();
        }
      }
    }
    if (opts.isJarClassesSet()) {
      zipWebInfClassesFiles(webInf);

    }
  }

  private void zipJasperGeneratedFiles(File webInfDir, File jspClassDir) throws IOException {
    Set<String> fileTypesToExclude = ImmutableSet.of(".java");
    File libDir = new File(webInfDir, "lib");
    JarTool jarTool = new JarTool(
        COMPILED_JSP_JAR_NAME_PREFIX, jspClassDir, libDir, MAX_COMPILED_JSP_JAR_SIZE,
        fileTypesToExclude);
    jarTool.run();
    recursiveDelete(jspClassDir);
  }

  private void zipWebInfClassesFiles(File webInfDir) throws IOException {
    File libDir = new File(webInfDir, "lib");
    File classesDir = new File(webInfDir, "classes");
    JarTool jarTool = new JarTool(
        CLASSES_JAR_NAME_PREFIX, classesDir, libDir, MAX_CLASSES_JAR_SIZE,
        null);
    jarTool.run();
    recursiveDelete(classesDir);
    classesDir.mkdir();
  }

  private String getJspClasspath(File classDir, File genDir) {
    StringBuilder classpath = new StringBuilder();
    for (URL lib : SdkImplInfo.getImplLibs()) {
      classpath.append(lib.getPath());
      classpath.append(File.pathSeparatorChar);
    }
    for (File lib : SdkInfo.getSharedLibFiles()) {
      classpath.append(lib.getPath());
      classpath.append(File.pathSeparatorChar);
    }

    classpath.append(classDir.getPath());
    classpath.append(File.pathSeparatorChar);
    classpath.append(genDir.getPath());
    classpath.append(File.pathSeparatorChar);

    for (File f : new FileIterator(new File(classDir.getParentFile(), "lib"))) {
      String filename = f.getPath().toLowerCase();
      if (filename.endsWith(".jar") || filename.endsWith(".zip")) {
        classpath.append(f.getPath());
        classpath.append(File.pathSeparatorChar);
      }
    }

    return classpath.toString();
  }

  private Process startProcess(String... args) throws IOException {
    ProcessBuilder builder = new ProcessBuilder(args);
    Process proc = builder.redirectErrorStream(true).start();
    logger.fine(formatCommand(builder.command()));
    new Thread(new OutputPump(proc.getInputStream(), detailsWriter)).start();
    return proc;
  }

  private String formatCommand(Iterable<String> args) {
    StringBuilder command = new StringBuilder();
    for (String chunk : args) {
      command.append(chunk);
      command.append(" ");
    }
    return command.toString();
  }

  /**
   * Scans a given directory tree, testing whether any file matches a given
   * pattern.
   *
   * @param dir the directory under which to scan
   * @param regex the pattern to look for
   * @returns Returns {@code true} on the first instance of such a file,
   *   {@code false} otherwise.
   */
  private static boolean matchingFileExists(File dir, Pattern regex) {
    for (File file : dir.listFiles()) {
      if (file.isDirectory()) {
        if (matchingFileExists(file, regex)) {
          return true;
        }
      } else {
        if (regex.matcher(file.getName()).matches()) {
          return true;
        }
      }
    }
    return false;
  }

  /**
   * Invokes the JarSplitter code on any jar files found in {@code dir}.  Any
   * jars larger than {@code max} will be split into fragments of at most that
   * size.
   * @param dir the directory to search, recursively
   * @param max the maximum allowed size
   * @param excludes a set of suffixes to exclude.
   * @throws IOException on filesystem errors.
   */
  private static void splitJars(File dir, int max, Set<String> excludes) throws IOException {
    String children[] = dir.list();
    if (children == null) {
      return;
    }
    for (String name : children) {
      File subfile = new File(dir, name);
      if (subfile.isDirectory()) {
        splitJars(subfile, max, excludes);
      } else if (name.endsWith(".jar")) {
        if (subfile.length() > max) {
          new JarSplitter(subfile, dir, max, false, 4, excludes).run();
          subfile.delete();
        }
      }
    }
  }

  private static final Pattern SKIP_FILES = Pattern.compile(
      "^(.*/)?((#.*#)|(.*~)|(.*/RCS/.*)|)$");

  /**
   * Copies files from the app to the upload staging directory, or makes
   * symlinks instead if supported.  Puts the files into the correct places for
   * static vs. resource files, recursively.
   *
   * @param sourceDir application war dir, or on recursion a subdirectory of it
   * @param resDir staging resource dir, or on recursion a subdirectory matching
   *    the subdirectory in {@code sourceDir}
   * @param staticDir staging {@code __static__} dir, or an appropriate recursive
   *    subdirectory
   * @param forceResource if all files should be considered resource files
   * @param opts processing options, used primarily for handling of *.jsp files
   * @throws FileNotFoundException
   * @throws IOException
   */
  private void copyOrLink(File sourceDir, File resDir, File staticDir, boolean forceResource,
      ApplicationProcessingOptions opts)
    throws FileNotFoundException, IOException {

    for (String name : sourceDir.list()) {
      File file = new File(sourceDir, name);

      String path = file.getPath();
      if (File.separatorChar == '\\') {
        path = path.replace('\\', '/');
      }

      if (file.getName().startsWith(".") ||
          file.equals(GenerationDirectory.getGenerationDirectory(baseDir))) {
        continue;
      }

      if (file.isDirectory()) {
        if (file.getName().equals("WEB-INF")) {
          copyOrLink(file, new File(resDir, name), new File(staticDir, name), true, opts);
        } else {
          copyOrLink(file, new File(resDir, name), new File(staticDir, name), forceResource,
              opts);
        }
      } else {
        if (SKIP_FILES.matcher(path).matches()) {
          continue;
        }

        if (forceResource || appEngineWebXml.includesResource(path) ||
            (opts.isCompileJspsSet() && name.toLowerCase().endsWith(".jsp"))) {
          copyOrLinkFile(file, new File(resDir, name));
        }
        if (!forceResource && appEngineWebXml.includesStatic(path)) {
          copyOrLinkFile(file, new File(staticDir, name));
        }
      }
    }
  }

  /**
   * Attempts to symlink a single file, or copies it if symlinking is either
   * unsupported or fails.
   *
   * @param source source file
   * @param dest destination file
   * @throws FileNotFoundException
   * @throws IOException
   */
  private void copyOrLinkFile(File source, File dest)
      throws FileNotFoundException, IOException {
    dest.getParentFile().mkdirs();
    if (ln != null && !source.getName().endsWith("web.xml")) {

      try {
        dest.delete();
      } catch (Exception e) {
        System.err.println("Warning: We tried to delete " + dest.getPath());
        System.err.println("in order to create a symlink from " + source.getPath());
        System.err.println("but the delete failed with message: " + e.getMessage());
      }

      Process link = startProcess(ln.getAbsolutePath(), "-s",
                                  source.getAbsolutePath(),
                                  dest.getAbsolutePath());
      try {
        int stat = link.waitFor();
        if (stat == 0) {
          return;
        }
        System.err.println(ln.getAbsolutePath() + " returned status " + stat
            + ", copying instead...");
      } catch (InterruptedException ex) {
        System.err.println(ln.getAbsolutePath() + " was interrupted, copying instead...");
      }
      if (dest.delete()) {
        System.err.println("ln failed but symlink was created, removed: " + dest.getAbsolutePath());
      }
    }
    byte buffer[] = new byte[1024];
    int readlen;
    FileInputStream inStream = new FileInputStream(source);
    FileOutputStream outStream = new FileOutputStream(dest);
    try {
      readlen = inStream.read(buffer);
      while (readlen > 0) {
        outStream.write(buffer, 0, readlen);
        readlen = inStream.read(buffer);
      }
    } finally {
      try {
        inStream.close();
      } catch (IOException ex) {
      }
      try {
        outStream.close();
      } catch (IOException ex) {
      }
    }
  }
  /** Copy (or link) one directory into another one.
   */
  private void copyOrLinkDirectories(File sourceDir, File destination)
      throws IOException {

    for (String name : sourceDir.list()) {
      File file = new File(sourceDir, name);
      if (file.isDirectory()) {
        copyOrLinkDirectories(file, new File(destination, name));
      } else {
        copyOrLinkFile(file, new File(destination, name));
      }
    }
  }

  /** deletes the staging directory, if one was created. */
  @Override
  public void cleanStagingDirectory() {
    if (stageDir != null) {
      recursiveDelete(stageDir);
    }
  }

  /** Recursive directory deletion. */
  public static void recursiveDelete(File dead) {
    String[] files = dead.list();
    if (files != null) {
      for (String name : files) {
        recursiveDelete(new File(dead, name));
      }
    }
    dead.delete();
  }

  @Override
  public void setListener(UpdateListener l) {
    listener = l;
  }

  @Override
  public void setDetailsWriter(PrintWriter detailsWriter) {
    this.detailsWriter = detailsWriter;
  }

  @Override
  public void statusUpdate(String message, int amount) {
    updateProgress += progressAmount;
    if (updateProgress > 99) {
      updateProgress = 99;
    }
    progressAmount = amount;
    if (listener != null) {
      listener.onProgress(new UpdateProgressEvent(
                              Thread.currentThread(), message, updateProgress));
    }
  }

  @Override
  public void statusUpdate(String message) {
    int amount = progressAmount / 4;
    updateProgress += amount;
    if (updateProgress > 99) {
      updateProgress = 99;
    }
    progressAmount -= amount;
    if (listener != null) {
      listener.onProgress(new UpdateProgressEvent(
                              Thread.currentThread(), message, updateProgress));
    }
  }

  private String generateAppYaml(File stageDir, String runtime, AppEngineWebXml aeWebXml) {
    Set<String> staticFiles = new HashSet<String>();
    for (File f : new FileIterator(new File(stageDir, "__static__"))) {
      staticFiles.add(Utility.calculatePath(f, stageDir));
    }

    AppYamlTranslator translator =
        new AppYamlTranslator(aeWebXml, getWebXml(), getBackendsXml(),
                              getApiVersion(), staticFiles, null, runtime, getSdkVersion());
    String yaml = translator.getYaml();
    logger.fine("Generated app.yaml file:\n" + yaml);
    return yaml;
  }

  /**
   * Returns the app.yaml string.
   *
   * @throws IllegalStateException if createStagingDirectory has not been called.
   */
  @Override
  public String getAppYaml() {
    if (appYaml == null) {
      throw new IllegalStateException("Must call createStagingDirectory first.");
    }
    return appYaml;
  }

  /**
   * Generates a quickstart-web.xml. Minimizes and saves in min-quickstart-web.xml
   * @return Relative path to min-quickstart-web.xml
   */
  private void createQuickstartWebXml(ApplicationProcessingOptions opts)
      throws IOException, SAXException, ParserConfigurationException, TransformerException {
    String javaCmd = opts.getJavaExecutable().getPath();

    String quickstartJar = new File(SdkInfo.getSdkRoot(),
        "lib/java-managed-vm/appengine-java-vmruntime/quickstartgenerator.jar").getAbsolutePath();

    String[] args = {
        javaCmd,
        "-jar", quickstartJar,
        stageDir.getAbsolutePath()
    };
    Process quickstartProcess = startProcess(args);

    int status;
    try {
      status = quickstartProcess.waitFor();
    } catch (InterruptedException ex) {
      status = 1;
    }

    if (status != 0) {
      detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
      throw new RuntimeException("Failed to generate quickstart-web.xml.");
    }

    File webDefaultXml = new File(SdkInfo.getSdkRoot() + "/lib/jetty-base-sdk/etc/webdefault.xml");
    File quickstartXml = new File(stageDir, "/WEB-INF/quickstart-web.xml");
    File minimizedQuickstartXml = new File(stageDir, "/WEB-INF/min-quickstart-web.xml");

    Document quickstartDoc = getFilteredQuickstartDoc(quickstartXml, webDefaultXml);

    Transformer transformer = TransformerFactory.newInstance().newTransformer();
    transformer.setOutputProperty(OutputKeys.INDENT, "yes");
    StreamResult result = new StreamResult(new FileWriter(minimizedQuickstartXml));
    DOMSource source = new DOMSource(quickstartDoc);
    transformer.transform(source, result);
  }

  /**
   * Removes mappings from quickstart-web.xml that come from webdefault.xml.
   *
   * The quickstart-web.xml generated by the quickstartgenerator process includes
   * the contents of the user's web.xml, entries derived from Java annotations,
   * and entries derived from the contents of webdefault.xml. All of those are
   * appropriate for the Java web server. But when generating an app.yaml for
   * appcfg or dev_appserver, the webdefault.xml entries are not appropriate, since
   * app.yaml should only reflect what is specific to the user's app. So this
   * method returns a modified min-quickstart-web Document from which the webdefault.xml
   * entries have been removed. Specifically, we look at the <url-pattern> inside
   * every <servlet-mapping> or <filter-mapping> element in webdefault.xml to
   * determine default patterns; then we look at those elements inside
   * quickstart-web.xml and remove any whose <url-pattern> is one of the default
   * patterns.
   *
   * @return a filtered quickstart Document object appropriate for translation to app.yaml
   */
  static Document getFilteredQuickstartDoc(File quickstartXml, File webDefaultXml)
      throws ParserConfigurationException, IOException, SAXException {

    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
    DocumentBuilder webDefaultDocBuilder = docBuilderFactory.newDocumentBuilder();
    Document webDefaultDoc = webDefaultDocBuilder.parse(webDefaultXml);
    DocumentBuilder quickstartDocBuilder = docBuilderFactory.newDocumentBuilder();
    Document quickstartDoc = quickstartDocBuilder.parse(quickstartXml);
    final Set<String> tagsToExamine = ImmutableSet.of("filter-mapping", "servlet-mapping");
    final String urlPatternTag = "url-pattern";

    Set<String> defaultRoots = Sets.newHashSet();
    List<Node> nodesToRemove = Lists.newArrayList();

    webDefaultDoc.getDocumentElement().normalize();
    NodeList webDefaultChildren = webDefaultDoc.getDocumentElement()
        .getElementsByTagName(urlPatternTag);
    for (int i = 0; i < webDefaultChildren.getLength(); i++) {
      Node child = webDefaultChildren.item(i);
      if (tagsToExamine.contains(child.getParentNode().getNodeName())) {
        String url = child.getTextContent().trim();
        if (url.startsWith("/")) {
          defaultRoots.add(url);
        }
      }
    }

    quickstartDoc.getDocumentElement().normalize();
    NodeList quickstartChildren = quickstartDoc.getDocumentElement()
        .getElementsByTagName(urlPatternTag);
    for (int i = 0; i < quickstartChildren.getLength(); i++) {
      Node child = quickstartChildren.item(i);
      if (tagsToExamine.contains(child.getParentNode().getNodeName())) {
        String url = child.getTextContent().trim();
        if (defaultRoots.contains(url)) {
          nodesToRemove.add(child.getParentNode());
        }
      }
    }
    for (Node node : nodesToRemove) {
      quickstartDoc.getDocumentElement().removeChild(node);
    }

    return quickstartDoc;
  }
}