java/src/main/com/google/appengine/tools/admin/Application.java

   1 // Copyright 2008 Google Inc. All Rights Reserved.
   2
   3 package com.google.appengine.tools.admin;
   4
   5 import com.google.appengine.tools.admin.AppAdminFactory.ApplicationProcessingOptions;
   6 import com.google.appengine.tools.info.SdkImplInfo;
   7 import com.google.appengine.tools.info.SdkInfo;
   8 import com.google.appengine.tools.plugins.AppYamlProcessor;
   9 import com.google.appengine.tools.plugins.SDKPluginManager;
  10 import com.google.appengine.tools.plugins.SDKRuntimePlugin;
  11 import com.google.appengine.tools.util.ApiVersionFinder;
  12 import com.google.appengine.tools.util.FileIterator;
  13 import com.google.appengine.tools.util.JarSplitter;
  14 import com.google.appengine.tools.util.JarTool;
  15 import com.google.apphosting.utils.config.AppEngineConfigException;
  16 import com.google.apphosting.utils.config.AppEngineWebXml;
  17 import com.google.apphosting.utils.config.AppEngineWebXmlReader;
  18 import com.google.apphosting.utils.config.BackendsXml;
  19 import com.google.apphosting.utils.config.BackendsXmlReader;
  20 import com.google.apphosting.utils.config.BackendsYamlReader;
  21 import com.google.apphosting.utils.config.CronXml;
  22 import com.google.apphosting.utils.config.CronXmlReader;
  23 import com.google.apphosting.utils.config.CronYamlReader;
  24 import com.google.apphosting.utils.config.DosXml;
  25 import com.google.apphosting.utils.config.DosXmlReader;
  26 import com.google.apphosting.utils.config.DosYamlReader;
  27 import com.google.apphosting.utils.config.GenerationDirectory;
  28 import com.google.apphosting.utils.config.IndexesXml;
  29 import com.google.apphosting.utils.config.IndexesXmlReader;
  30 import com.google.apphosting.utils.config.QueueXml;
  31 import com.google.apphosting.utils.config.QueueXmlReader;
  32 import com.google.apphosting.utils.config.QueueYamlReader;
  33 import com.google.apphosting.utils.config.WebXml;
  34 import com.google.apphosting.utils.config.WebXmlReader;
  35 import com.google.common.collect.ImmutableSet;
  36 import com.google.common.io.Files;
  37
  38 import org.mortbay.io.Buffer;
  39 import org.mortbay.jetty.MimeTypes;
  40 import org.xml.sax.SAXException;
  41
  42 import java.io.DataInputStream;
  43 import java.io.File;
  44 import java.io.FileInputStream;
  45 import java.io.FileNotFoundException;
  46 import java.io.FileOutputStream;
  47 import java.io.FileWriter;
  48 import java.io.IOException;
  49 import java.io.PrintWriter;
  50 import java.net.URL;
  51 import java.net.URLConnection;
  52 import java.util.ArrayList;
  53 import java.util.Arrays;
  54 import java.util.HashSet;
  55 import java.util.List;
  56 import java.util.Set;
  57 import java.util.logging.Level;
  58 import java.util.logging.Logger;
  59 import java.util.regex.Pattern;
  60
  61 import javax.activation.FileTypeMap;
  62 import javax.tools.JavaCompiler;
  63 import javax.tools.JavaFileObject;
  64 import javax.tools.StandardJavaFileManager;
  65 import javax.tools.ToolProvider;
  66 import javax.xml.XMLConstants;
  67 import javax.xml.transform.stream.StreamSource;
  68 import javax.xml.validation.SchemaFactory;
  69
  70 /**
  71  * An App Engine application. You can {@link #readApplication read} an
  72  * {@code Application} from a path, and
  73  * {@link com.google.appengine.tools.admin.AppAdminFactory#createAppAdmin create}
  74  * an {@link com.google.appengine.tools.admin.AppAdmin} to upload, create
  75  * indexes, or otherwise manage it.
  76  *
  77  */
  78 public class Application implements GenericApplication {
  79
  80   private static final int MAX_COMPILED_JSP_JAR_SIZE = 1024 * 1024 * 5;
  81   private static final String COMPILED_JSP_JAR_NAME_PREFIX = "_ah_compiled_jsps";
  82
  83   private static final int MAX_CLASSES_JAR_SIZE = 1024 * 1024 * 5;
  84   private static final String CLASSES_JAR_NAME_PREFIX = "_ah_webinf_classes";
  85
  86   private static Pattern JSP_REGEX = Pattern.compile(".*\\.jspx?");
  87
  88   /** If available, this is set to a program to make symlinks, e.g. /bin/ln */
  89   private static File ln = Utility.findLink();
  90   private static File sdkDocsDir;
  91   private static synchronized File getSdkDocsDir(){
  92     if (null == sdkDocsDir){
  93       sdkDocsDir = new File(SdkInfo.getSdkRoot(), "docs");
  94     }
  95     return sdkDocsDir;
  96   }
  97
  98   private static final String STAGEDIR_PREFIX = "appcfg";
  99
 100   private static final Logger logger = Logger.getLogger(Application.class.getName());
 101
 102   private static final MimeTypes mimeTypes = new MimeTypes();
 103
 104   private AppEngineWebXml appEngineWebXml;
 105   private WebXml webXml;
 106   private CronXml cronXml;
 107   private DosXml dosXml;
 108   private String pagespeedYaml;
 109   private QueueXml queueXml;
 110   private IndexesXml indexesXml;
 111   private BackendsXml backendsXml;
 112   private File baseDir;
 113   private File stageDir;
 114   private File externalResourceDir;
 115   private String apiVersion;
 116   private String appYaml;
 117
 118   private UpdateListener listener;
 119   private PrintWriter detailsWriter;
 120   private int updateProgress = 0;
 121   private int progressAmount = 0;
 122
 123   protected Application(){
 124   }
 125
 126   /**
 127    * Builds a normalized path for the given directory in which
 128    * forward slashes are used as the file separator on all platforms.
 129    * @param dir A directory
 130    * @return The normalized path
 131    */
 132   private static String buildNormalizedPath(File dir) {
 133     String normalizedPath = dir.getPath();
 134     if (File.separatorChar == '\\') {
 135       normalizedPath = normalizedPath.replace('\\', '/');
 136     }
 137     return normalizedPath;
 138   }
 139
 140   private Application(String explodedPath, String appId, String appVersion) {
 141     this.baseDir = new File(explodedPath);
 142     explodedPath = buildNormalizedPath(baseDir);
 143     File webinf = new File(baseDir, "WEB-INF");
 144     if (!webinf.getName().equals("WEB-INF")) {
 145       throw new AppEngineConfigException("WEB-INF directory must be capitalized.");
 146     }
 147
 148     String webinfPath = webinf.getPath();
 149     AppEngineWebXmlReader aewebReader = new AppEngineWebXmlReader(explodedPath);
 150     WebXmlReader webXmlReader = new WebXmlReader(explodedPath);
 151     AppYamlProcessor.convert(webinf, aewebReader.getFilename(), webXmlReader.getFilename());
 152
 153     validateXml(aewebReader.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd"));
 154     appEngineWebXml = aewebReader.readAppEngineWebXml();
 155     appEngineWebXml.setSourcePrefix(explodedPath);
 156     if (appId != null) {
 157       appEngineWebXml.setAppId(appId);
 158     }
 159     if (appVersion != null) {
 160       appEngineWebXml.setMajorVersionId(appVersion);
 161     }
 162
 163     webXml = webXmlReader.readWebXml();
 164     webXml.validate();
 165
 166     CronXmlReader cronReader = new CronXmlReader(explodedPath);
 167     validateXml(cronReader.getFilename(), new File(getSdkDocsDir(), "cron.xsd"));
 168     cronXml = cronReader.readCronXml();
 169     if (cronXml == null) {
 170       CronYamlReader cronYaml = new CronYamlReader(webinfPath);
 171       cronXml = cronYaml.parse();
 172     }
 173
 174     QueueXmlReader queueReader = new QueueXmlReader(explodedPath);
 175     validateXml(queueReader.getFilename(), new File(getSdkDocsDir(), "queue.xsd"));
 176     queueXml = queueReader.readQueueXml();
 177     if (queueXml == null) {
 178       QueueYamlReader queueYaml = new QueueYamlReader(webinfPath);
 179       queueXml = queueYaml.parse();
 180     }
 181
 182     DosXmlReader dosReader = new DosXmlReader(explodedPath);
 183     validateXml(dosReader.getFilename(), new File(getSdkDocsDir(), "dos.xsd"));
 184     dosXml = dosReader.readDosXml();
 185     if (dosXml == null) {
 186       DosYamlReader dosYaml = new DosYamlReader(webinfPath);
 187       dosXml = dosYaml.parse();
 188     }
 189
 190     if (getAppEngineWebXml().getPagespeed() != null) {
 191       StringBuilder pagespeedYamlBuilder = new StringBuilder();
 192       AppYamlTranslator.appendPagespeed(
 193           getAppEngineWebXml().getPagespeed(), pagespeedYamlBuilder, 0);
 194       pagespeedYaml = pagespeedYamlBuilder.toString();
 195     }
 196
 197     IndexesXmlReader indexReader = new IndexesXmlReader(explodedPath);
 198     validateXml(indexReader.getFilename(), new File(getSdkDocsDir(), "datastore-indexes.xsd"));
 199     indexesXml = indexReader.readIndexesXml();
 200
 201     BackendsXmlReader backendsReader = new BackendsXmlReader(explodedPath);
 202     validateXml(backendsReader.getFilename(), new File(getSdkDocsDir(), "backends.xsd"));
 203     backendsXml = backendsReader.readBackendsXml();
 204     if (backendsXml == null) {
 205       BackendsYamlReader backendsYaml = new BackendsYamlReader(webinfPath);
 206       backendsXml = backendsYaml.parse();
 207     }
 208   }
 209
 210   /**
 211    * Reads the App Engine application from {@code path}. The path may either
 212    * be a WAR file or the root of an exploded WAR directory.
 213    *
 214    * @param path a not {@code null} path.
 215    *
 216    * @throws IOException if an error occurs while trying to read the
 217    * {@code Application}.
 218    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 219    * {@code Application's} appengine-web.xml file is malformed.
 220    */
 221   public static Application readApplication(String path)
 222       throws IOException {
 223     return new Application(path, null, null);
 224   }
 225
 226   /**
 227    * Sets the external resource directory. Call this method before invoking
 228    * {@link #createStagingDirectory(ApplicationProcessingOptions, ResourceLimits)}.
 229    * <p>
 230    * The external resource directory is a directory outside of the war directory where additional
 231    * files live. These files will be copied into the staging directory during an upload, after the
 232    * war directory is copied there. Consequently if there are any name collisions the files in the
 233    * external resource directory will win.
 234    *
 235    * @param path a not {@code null} path to an existing directory.
 236    *
 237    * @throws IllegalArgumentException If {@code path} does not refer to an existing
 238    *         directory.
 239    */
 240   public void setExternalResourceDir(String path) {
 241     if (path == null) {
 242       throw new NullPointerException("path is null");
 243     }
 244     if (stageDir != null) {
 245       throw new IllegalStateException(
 246           "This method must be invoked prior to createStagingDirectory()");
 247     }
 248     File dir = new File(path);
 249     if (!dir.exists()) {
 250       throw new IllegalArgumentException("path does not exist: " + path);
 251     }
 252     if (!dir.isDirectory()) {
 253       throw new IllegalArgumentException(path + " is not a directory.");
 254     }
 255     this.externalResourceDir = dir;
 256   }
 257
 258   /**
 259    * Reads the App Engine application from {@code path}. The path may either
 260    * be a WAR file or the root of an exploded WAR directory.
 261    *
 262    * @param path a not {@code null} path.
 263    * @param appId if non-null, use this as an application id override.
 264    * @param appVersion if non-null, use this as an application version override.
 265    *
 266    * @throws IOException if an error occurs while trying to read the
 267    * {@code Application}.
 268    * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
 269    * {@code Application's} appengine-web.xml file is malformed.
 270    */
 271   public static Application readApplication(String path, String appId, String appVersion)
 272       throws IOException {
 273     return new Application(path, appId, appVersion);
 274   }
 275
 276   /**
 277    * Returns the application identifier, from the AppEngineWebXml config
 278    * @return application identifier
 279    */
 280   @Override
 281   public String getAppId() {
 282     return appEngineWebXml.getAppId();
 283   }
 284
 285   /**
 286    * Returns the application version, from the AppEngineWebXml config
 287    * @return application version
 288    */
 289   @Override
 290   public String getVersion() {
 291     return appEngineWebXml.getMajorVersionId();
 292   }
 293
 294   @Override
 295   public String getSourceLanguage() {
 296     return appEngineWebXml.getSourceLanguage();
 297   }
 298
 299   @Override
 300   public String getServer() {
 301     return appEngineWebXml.getServer();
 302   }
 303
 304   @Override
 305   public String getInstanceClass() {
 306     return appEngineWebXml.getInstanceClass();
 307   }
 308
 309   @Override
 310   public boolean isPrecompilationEnabled() {
 311     return appEngineWebXml.getPrecompilationEnabled();
 312   }
 313
 314   @Override
 315   public List<ErrorHandler> getErrorHandlers() {
 316     class ErrorHandlerImpl implements ErrorHandler {
 317       private AppEngineWebXml.ErrorHandler errorHandler;
 318       public ErrorHandlerImpl(AppEngineWebXml.ErrorHandler errorHandler) {
 319         this.errorHandler = errorHandler;
 320       }
 321       @Override
 322       public String getFile() {
 323         return "__static__/" + errorHandler.getFile();
 324       }
 325       @Override
 326       public String getErrorCode() {
 327         return errorHandler.getErrorCode();
 328       }
 329       @Override
 330       public String getMimeType() {
 331         return getMimeTypeIfStatic(getFile());
 332       }
 333     }
 334     List<ErrorHandler> errorHandlers = new ArrayList<ErrorHandler>();
 335     for (AppEngineWebXml.ErrorHandler errorHandler: appEngineWebXml.getErrorHandlers()) {
 336       errorHandlers.add(new ErrorHandlerImpl(errorHandler));
 337     }
 338     return errorHandlers;
 339   }
 340
 341   @Override
 342   public String getMimeTypeIfStatic(String path) {
 343     if (!path.contains("__static__/")) {
 344       return null;
 345     }
 346     String mimeType = webXml.getMimeTypeForPath(path);
 347     if (mimeType != null) {
 348       return mimeType;
 349     }
 350     return guessContentTypeFromName(path);
 351   }
 352
 353   /**
 354    * @param fileName path of a file with extension
 355    * @return the mimetype of the file (or application/octect-stream if not recognized)
 356    */
 357   public static String guessContentTypeFromName(String fileName) {
 358     String defaultValue = "application/octet-stream";
 359     try {
 360       Buffer buffer = mimeTypes.getMimeByExtension(fileName);
 361       if (buffer != null) {
 362         return new String(buffer.asArray());
 363       }
 364       String lowerName = fileName.toLowerCase();
 365       if (lowerName.endsWith(".json")) {
 366         return "application/json";
 367       }
 368       FileTypeMap typeMap = FileTypeMap.getDefaultFileTypeMap();
 369       String ret = typeMap.getContentType(fileName);
 370       if (ret != null) {
 371         return ret;
 372       }
 373       ret = URLConnection.guessContentTypeFromName(fileName);
 374       if (ret != null) {
 375         return ret;
 376       }
 377       return defaultValue;
 378     } catch (Throwable t) {
 379       logger.log(Level.WARNING, "Error identify mimetype for " + fileName, t);
 380       return defaultValue;
 381     }
 382   }
 383   /**
 384    * Returns the AppEngineWebXml describing the application.
 385    *
 386    * @return a not {@code null} deployment descriptor
 387    */
 388   public AppEngineWebXml getAppEngineWebXml() {
 389     return appEngineWebXml;
 390   }
 391
 392   /**
 393    * Returns the CronXml describing the applications' cron jobs.
 394    * @return a cron descriptor, possibly empty or {@code null}
 395    */
 396   @Override
 397   public CronXml getCronXml() {
 398     return cronXml;
 399   }
 400
 401   /**
 402    * Returns the QueueXml describing the applications' task queues.
 403    * @return a queue descriptor, possibly empty or {@code null}
 404    */
 405   @Override
 406   public QueueXml getQueueXml() {
 407     return queueXml;
 408   }
 409
 410   /**
 411    * Returns the DosXml describing the applications' DoS entries.
 412    * @return a dos descriptor, possibly empty or {@code null}
 413    */
 414   @Override
 415   public DosXml getDosXml() {
 416     return dosXml;
 417   }
 418
 419   /**
 420    * Returns the pagespeed.yaml describing the applications' PageSpeed configuration.
 421    * @return a pagespeed.yaml string, possibly empty or {@code null}
 422    */
 423   @Override
 424   public String getPagespeedYaml() {
 425     return pagespeedYaml;
 426   }
 427
 428   /**
 429    * Returns the IndexesXml describing the applications' indexes.
 430    * @return a index descriptor, possibly empty or {@code null}
 431    */
 432   @Override
 433   public IndexesXml getIndexesXml() {
 434     return indexesXml;
 435   }
 436
 437   /**
 438    * Returns the WebXml describing the applications' servlets and generic web
 439    * application information.
 440    *
 441    * @return a WebXml descriptor, possibly empty but not {@code null}
 442    */
 443   public WebXml getWebXml() {
 444     return webXml;
 445   }
 446
 447   @Override
 448   public BackendsXml getBackendsXml() {
 449     return backendsXml;
 450   }
 451
 452   /**
 453    * Returns the desired API version for the current application, or
 454    * {@code "none"} if no API version was used.
 455    *
 456    * @throws IllegalStateException if createStagingDirectory has not been called.
 457    */
 458   @Override
 459   public String getApiVersion() {
 460     if (apiVersion == null) {
 461       throw new IllegalStateException("Must call createStagingDirectory first.");
 462     }
 463     return apiVersion;
 464   }
 465
 466   /**
 467    * Returns a path to an exploded WAR directory for the application.
 468    * This may be a temporary directory.
 469    *
 470    * @return a not {@code null} path pointing to a directory
 471    */
 472   @Override
 473   public String getPath() {
 474     return baseDir.getAbsolutePath();
 475   }
 476
 477   /**
 478    * Returns the staging directory, or {@code null} if none has been created.
 479    */
 480   @Override
 481   public File getStagingDir() {
 482     return stageDir;
 483   }
 484
 485   @Override
 486   public void resetProgress() {
 487     updateProgress = 0;
 488     progressAmount = 0;
 489   }
 490
 491   /**
 492    * Creates a new staging directory, if needed, or returns the existing one
 493    * if already created.
 494    *
 495    * @param opts User-specified options for processing the application.
 496    * @return staging directory
 497    * @throws IOException
 498    */
 499   @Override
 500   public File createStagingDirectory(ApplicationProcessingOptions opts,
 501       ResourceLimits resourceLimits) throws IOException {
 502     if (stageDir != null) {
 503       return stageDir;
 504     }
 505
 506     int i = 0;
 507     while (stageDir == null && i++ < 3) {
 508       try {
 509         stageDir = File.createTempFile(STAGEDIR_PREFIX, null);
 510       } catch (IOException ex) {
 511         continue;
 512       }
 513       stageDir.delete();
 514       if (stageDir.mkdir() == false) {
 515         stageDir = null;
 516       }
 517     }
 518     if (i == 3) {
 519       throw new IOException("Couldn't create a temporary directory in 3 tries.");
 520     }
 521     statusUpdate("Created staging directory at: '" + stageDir.getPath() + "'", 20);
 522
 523     File staticDir = new File(stageDir, "__static__");
 524     staticDir.mkdir();
 525     copyOrLink(baseDir, stageDir, staticDir, false, opts);
 526     if (externalResourceDir != null) {
 527       String previousPrefix = appEngineWebXml.getSourcePrefix();
 528       String newPrefix = buildNormalizedPath(externalResourceDir);
 529       try {
 530         appEngineWebXml.setSourcePrefix(newPrefix);
 531         copyOrLink(externalResourceDir, stageDir, staticDir, false, opts);
 532       } finally {
 533         appEngineWebXml.setSourcePrefix(previousPrefix);
 534       }
 535     }
 536     if (!Boolean.getBoolean(AppCfg.USE_JAVA7_SYSTEM_PROP)) {
 537       CheckJava7Classes verifier = new CheckJava7Classes(new File(stageDir, "WEB-INF/classes"));
 538       if (verifier.isJava7App()) {
 539         throw new RuntimeException(
 540             "The application contains Java 7 classes, but the --use_java7 flag has not been set.");
 541       }
 542     }
 543
 544     if (opts.isCompileJspsSet()) {
 545       compileJsps(stageDir, opts);
 546     }
 547
 548     apiVersion = findApiVersion(stageDir, true);
 549
 550     appYaml = generateAppYaml(stageDir);
 551
 552     if (GenerationDirectory.getGenerationDirectory(stageDir).mkdirs()) {
 553       writePreparedYamlFile("app", appYaml);
 554       writePreparedYamlFile("backends", backendsXml == null ? null : backendsXml.toYaml());
 555       writePreparedYamlFile("index", indexesXml.size() == 0 ? null : indexesXml.toYaml());
 556       writePreparedYamlFile("cron", cronXml == null ? null : cronXml.toYaml());
 557       writePreparedYamlFile("queue", queueXml == null ? null : queueXml.toYaml());
 558       writePreparedYamlFile("dos", dosXml == null ? null : dosXml.toYaml());
 559     }
 560
 561     int maxJarSize = (int) resourceLimits.maxFileSize();
 562
 563     if (opts.isSplitJarsSet()) {
 564       splitJars(new File(new File(stageDir, "WEB-INF"), "lib"),
 565                 maxJarSize, opts.getJarSplittingExcludes());
 566     }
 567
 568     if (getSourceLanguage() != null) {
 569       SDKRuntimePlugin runtimePlugin = SDKPluginManager.findRuntimePlugin(getSourceLanguage());
 570       if (runtimePlugin != null) {
 571         runtimePlugin.processStagingDirectory(stageDir);
 572       }
 573     }
 574
 575     return stageDir;
 576   }
 577
 578   /**
 579    * Write yaml file to generation subdirectory within stage directory.
 580    */
 581   private void writePreparedYamlFile(String yamlName, String yamlString) throws IOException {
 582     File f = new File(GenerationDirectory.getGenerationDirectory(stageDir), yamlName + ".yaml");
 583     if (yamlString != null  && f.createNewFile()) {
 584       FileWriter fw = new FileWriter(f);
 585       fw.write(yamlString);
 586       fw.close();
 587     }
 588   }
 589
 590   private static String findApiVersion(File baseDir, boolean deleteApiJars) {
 591     ApiVersionFinder finder = new ApiVersionFinder();
 592
 593     String foundApiVersion = null;
 594     File webInf = new File(baseDir, "WEB-INF");
 595     File libDir = new File(webInf, "lib");
 596     for (File file : new FileIterator(libDir)) {
 597       if (file.getPath().endsWith(".jar")) {
 598         try {
 599           String apiVersion = finder.findApiVersion(file);
 600           if (apiVersion != null) {
 601             if (foundApiVersion == null) {
 602               foundApiVersion = apiVersion;
 603             } else if (!foundApiVersion.equals(apiVersion)) {
 604               logger.warning("Warning: found duplicate API version: " + foundApiVersion +
 605                              ", using " + apiVersion);
 606             }
 607             if (deleteApiJars) {
 608               file.delete();
 609             }
 610           }
 611         } catch (IOException ex) {
 612           logger.log(Level.WARNING, "Could not identify API version in " + file, ex);
 613         }
 614       }
 615     }
 616
 617     if (foundApiVersion == null) {
 618       foundApiVersion = "none";
 619     }
 620     return foundApiVersion;
 621   }
 622
 623   /**
 624    * Validates a given XML document against a given schema.
 625    *
 626    * @param xmlFilename filename with XML document
 627    * @param schema XSD schema to validate with
 628    *
 629    * @throws AppEngineConfigException for malformed XML, or IO errors
 630    */
 631   private static void validateXml(String xmlFilename, File schema) {
 632     File xml = new File(xmlFilename);
 633     if (!xml.exists()) {
 634       return;
 635     }
 636     try {
 637       SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
 638       try {
 639         factory.newSchema(schema).newValidator().validate(
 640             new StreamSource(new FileInputStream(xml)));
 641       } catch (SAXException ex) {
 642         throw new AppEngineConfigException("XML error validating " +
 643             xml.getPath() + " against " + schema.getPath(), ex);
 644       }
 645     } catch (IOException ex) {
 646       throw new AppEngineConfigException("IO error validating " +
 647           xml.getPath() + " against " + schema.getPath(), ex);
 648     }
 649   }
 650
 651   private static final String JSPC_MAIN = "com.google.appengine.tools.development.LocalJspC";
 652
 653   private void compileJsps(File stage, ApplicationProcessingOptions opts)
 654       throws IOException {
 655     statusUpdate("Scanning for jsp files.");
 656
 657     if (matchingFileExists(new File(stage.getPath()), JSP_REGEX)) {
 658       statusUpdate("Compiling jsp files.");
 659
 660       File webInf = new File(stage, "WEB-INF");
 661
 662       for (File file : SdkImplInfo.getUserJspLibFiles()) {
 663         copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()));
 664       }
 665       for (File file : SdkImplInfo.getSharedJspLibFiles()) {
 666         copyOrLinkFile(file, new File(new File(webInf, "lib"), file.getName()));
 667       }
 668
 669       File classes = new File(webInf, "classes");
 670       File generatedWebXml = new File(webInf, "generated_web.xml");
 671       File tempDir = Files.createTempDir();
 672       String classpath = getJspClasspath(classes, tempDir);
 673
 674       String javaCmd = opts.getJavaExecutable().getPath();
 675       String[] args = new String[] {
 676         javaCmd,
 677         "-classpath", classpath,
 678         "-D" + AppCfg.USE_JAVA7_SYSTEM_PROP + "=" +
 679             System.getProperty(AppCfg.USE_JAVA7_SYSTEM_PROP, "false"),
 680         JSPC_MAIN,
 681         "-uriroot", stage.getPath(),
 682         "-p", "org.apache.jsp",
 683         "-l", "-v",
 684         "-webinc", generatedWebXml.getPath(),
 685         "-d", tempDir.getPath(),
 686         "-javaEncoding", opts.getCompileEncoding(),
 687       };
 688       Process jspc = startProcess(args);
 689
 690       int status = 1;
 691       try {
 692         status = jspc.waitFor();
 693       } catch (InterruptedException ex) { }
 694
 695       if (status != 0) {
 696         detailsWriter.println("Error while executing: " + formatCommand(Arrays.asList(args)));
 697         throw new JspCompilationException("Failed to compile jsp files.",
 698                                           JspCompilationException.Source.JASPER);
 699       }
 700
 701       compileJavaFiles(classpath, webInf, tempDir, opts);
 702
 703       webXml = new WebXmlReader(stage.getPath()).readWebXml();
 704
 705     }
 706   }
 707
 708   private void compileJavaFiles(String classpath, File webInf, File jspClassDir,
 709       ApplicationProcessingOptions opts) throws IOException {
 710
 711     JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
 712     if (compiler == null) {
 713       throw new RuntimeException(
 714           "Cannot get the System Java Compiler. Please use a JDK, not a JRE.");
 715     }
 716     StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
 717
 718     ArrayList<File> files = new ArrayList<File>();
 719     for (File f : new FileIterator(jspClassDir)) {
 720       if (f.getPath().toLowerCase().endsWith(".java")) {
 721         files.add(f);
 722       }
 723     }
 724     if (files.size() == 0) {
 725       return;
 726     }
 727     List<String> optionList = new ArrayList<String>();
 728     optionList.addAll(Arrays.asList("-classpath", classpath.toString()));
 729     optionList.addAll(Arrays.asList("-d", jspClassDir.getPath()));
 730     optionList.addAll(Arrays.asList("-encoding", opts.getCompileEncoding()));
 731     if (System.getProperty(AppCfg.USE_JAVA7_SYSTEM_PROP, "false").equals("false")) {
 732       optionList.addAll(Arrays.asList("-source", "6"));
 733       optionList.addAll(Arrays.asList("-target", "6"));
 734     }
 735
 736     Iterable<? extends JavaFileObject> compilationUnits =
 737         fileManager.getJavaFileObjectsFromFiles(files);
 738     boolean success = compiler.getTask(
 739         null, fileManager, null, optionList, null, compilationUnits).call();
 740     fileManager.close();
 741
 742     if (!success) {
 743       throw new JspCompilationException("Failed to compile the generated JSP java files.",
 744           JspCompilationException.Source.JSPC);
 745     }
 746     if (opts.isJarJSPsSet()) {
 747       zipJasperGeneratedFiles(webInf, jspClassDir);
 748     } else {
 749       copyOrLinkDirectories(jspClassDir, new File(webInf, "classes"));
 750     }
 751     if (opts.isDeleteJSPs()) {
 752       for (File f : new FileIterator(webInf.getParentFile())) {
 753         if (f.getPath().toLowerCase().endsWith(".jsp")) {
 754           f.delete();
 755         }
 756       }
 757     }
 758     if (opts.isJarClassesSet()) {
 759       zipWebInfClassesFiles(webInf);
 760
 761     }
 762   }
 763
 764   private void zipJasperGeneratedFiles(File webInfDir, File jspClassDir) throws IOException {
 765     Set<String> fileTypesToExclude = ImmutableSet.of(".java");
 766     File libDir = new File(webInfDir, "lib");
 767     JarTool jarTool = new JarTool(
 768         COMPILED_JSP_JAR_NAME_PREFIX, jspClassDir, libDir, MAX_COMPILED_JSP_JAR_SIZE,
 769         fileTypesToExclude);
 770     jarTool.run();
 771     recursiveDelete(jspClassDir);
 772   }
 773
 774   private void zipWebInfClassesFiles(File webInfDir) throws IOException {
 775     File libDir = new File(webInfDir, "lib");
 776     File classesDir = new File(webInfDir, "classes");
 777     JarTool jarTool = new JarTool(
 778         CLASSES_JAR_NAME_PREFIX, classesDir, libDir, MAX_CLASSES_JAR_SIZE,
 779         null);
 780     jarTool.run();
 781     recursiveDelete(classesDir);
 782     classesDir.mkdir();
 783   }
 784
 785   private String getJspClasspath(File classDir, File genDir) {
 786     StringBuilder classpath = new StringBuilder();
 787     for (URL lib : SdkImplInfo.getImplLibs()) {
 788       classpath.append(lib.getPath());
 789       classpath.append(File.pathSeparatorChar);
 790     }
 791     for (File lib : SdkInfo.getSharedLibFiles()) {
 792       classpath.append(lib.getPath());
 793       classpath.append(File.pathSeparatorChar);
 794     }
 795
 796     classpath.append(classDir.getPath());
 797     classpath.append(File.pathSeparatorChar);
 798     classpath.append(genDir.getPath());
 799     classpath.append(File.pathSeparatorChar);
 800
 801     for (File f : new FileIterator(new File(classDir.getParentFile(), "lib"))) {
 802       String filename = f.getPath().toLowerCase();
 803       if (filename.endsWith(".jar") || filename.endsWith(".zip")) {
 804         classpath.append(f.getPath());
 805         classpath.append(File.pathSeparatorChar);
 806       }
 807     }
 808
 809     return classpath.toString();
 810   }
 811
 812   private Process startProcess(String... args) throws IOException {
 813     ProcessBuilder builder = new ProcessBuilder(args);
 814     Process proc = builder.redirectErrorStream(true).start();
 815     logger.fine(formatCommand(builder.command()));
 816     new Thread(new OutputPump(proc.getInputStream(), detailsWriter)).start();
 817     return proc;
 818   }
 819
 820   private String formatCommand(Iterable<String> args) {
 821     StringBuilder command = new StringBuilder();
 822     for (String chunk : args) {
 823       command.append(chunk);
 824       command.append(" ");
 825     }
 826     return command.toString();
 827   }
 828
 829   /**
 830    * Scans a given directory tree, testing whether any file matches a given
 831    * pattern.
 832    *
 833    * @param dir the directory under which to scan
 834    * @param regex the pattern to look for
 835    * @returns Returns {@code true} on the first instance of such a file,
 836    *   {@code false} otherwise.
 837    */
 838   private static boolean matchingFileExists(File dir, Pattern regex) {
 839     for (File file : dir.listFiles()) {
 840       if (file.isDirectory()) {
 841         if (matchingFileExists(file, regex)) {
 842           return true;
 843         }
 844       } else {
 845         if (regex.matcher(file.getName()).matches()) {
 846           return true;
 847         }
 848       }
 849     }
 850     return false;
 851   }
 852
 853   /**
 854    * Invokes the JarSplitter code on any jar files found in {@code dir}.  Any
 855    * jars larger than {@code max} will be split into fragments of at most that
 856    * size.
 857    * @param dir the directory to search, recursively
 858    * @param max the maximum allowed size
 859    * @param excludes a set of suffixes to exclude.
 860    * @throws IOException on filesystem errors.
 861    */
 862   private static void splitJars(File dir, int max, Set<String> excludes) throws IOException {
 863     String children[] = dir.list();
 864     if (children == null) {
 865       return;
 866     }
 867     for (String name : children) {
 868       File subfile = new File(dir, name);
 869       if (subfile.isDirectory()) {
 870         splitJars(subfile, max, excludes);
 871       } else if (name.endsWith(".jar")) {
 872         if (subfile.length() > max) {
 873           new JarSplitter(subfile, dir, max, false, 4, excludes).run();
 874           subfile.delete();
 875         }
 876       }
 877     }
 878   }
 879
 880   private static final Pattern SKIP_FILES = Pattern.compile(
 881       "^(.*/)?((#.*#)|(.*~)|(.*/RCS/.*)|)$");
 882
 883   /**
 884    * Copies files from the app to the upload staging directory, or makes
 885    * symlinks instead if supported.  Puts the files into the correct places for
 886    * static vs. resource files, recursively.
 887    *
 888    * @param sourceDir application war dir, or on recursion a subdirectory of it
 889    * @param resDir staging resource dir, or on recursion a subdirectory matching
 890    *    the subdirectory in {@code sourceDir}
 891    * @param staticDir staging {@code __static__} dir, or an appropriate recursive
 892    *    subdirectory
 893    * @param forceResource if all files should be considered resource files
 894    * @param opts processing options, used primarily for handling of *.jsp files
 895    * @throws FileNotFoundException
 896    * @throws IOException
 897    */
 898   private void copyOrLink(File sourceDir, File resDir, File staticDir, boolean forceResource,
 899       ApplicationProcessingOptions opts)
 900     throws FileNotFoundException, IOException {
 901
 902     for (String name : sourceDir.list()) {
 903       File file = new File(sourceDir, name);
 904
 905       String path = file.getPath();
 906       if (File.separatorChar == '\\') {
 907         path = path.replace('\\', '/');
 908       }
 909
 910       if (file.getName().startsWith(".") ||
 911           file.equals(GenerationDirectory.getGenerationDirectory(baseDir))) {
 912         continue;
 913       }
 914
 915       if (file.isDirectory()) {
 916         if (file.getName().equals("WEB-INF")) {
 917           copyOrLink(file, new File(resDir, name), new File(staticDir, name), true, opts);
 918         } else {
 919           copyOrLink(file, new File(resDir, name), new File(staticDir, name), forceResource,
 920               opts);
 921         }
 922       } else {
 923         if (SKIP_FILES.matcher(path).matches()) {
 924           continue;
 925         }
 926
 927         if (forceResource || appEngineWebXml.includesResource(path) ||
 928             (opts.isCompileJspsSet() && name.toLowerCase().endsWith(".jsp"))) {
 929           copyOrLinkFile(file, new File(resDir, name));
 930         }
 931         if (!forceResource && appEngineWebXml.includesStatic(path)) {
 932           copyOrLinkFile(file, new File(staticDir, name));
 933         }
 934       }
 935     }
 936   }
 937
 938   /**
 939    * Attempts to symlink a single file, or copies it if symlinking is either
 940    * unsupported or fails.
 941    *
 942    * @param source source file
 943    * @param dest destination file
 944    * @throws FileNotFoundException
 945    * @throws IOException
 946    */
 947   private void copyOrLinkFile(File source, File dest)
 948       throws FileNotFoundException, IOException {
 949     dest.getParentFile().mkdirs();
 950     if (ln != null && !source.getName().endsWith("web.xml")) {
 951
 952       try {
 953         dest.delete();
 954       } catch (Exception e) {
 955         System.err.println("Warning: We tried to delete " + dest.getPath());
 956         System.err.println("in order to create a symlink from " + source.getPath());
 957         System.err.println("but the delete failed with message: " + e.getMessage());
 958       }
 959
 960       Process link = startProcess(ln.getAbsolutePath(), "-s",
 961                                   source.getAbsolutePath(),
 962                                   dest.getAbsolutePath());
 963       try {
 964         int stat = link.waitFor();
 965         if (stat == 0) {
 966           return;
 967         }
 968         System.err.println(ln.getAbsolutePath() + " returned status " + stat
 969             + ", copying instead...");
 970       } catch (InterruptedException ex) {
 971         System.err.println(ln.getAbsolutePath() + " was interrupted, copying instead...");
 972       }
 973       if (dest.delete()) {
 974         System.err.println("ln failed but symlink was created, removed: " + dest.getAbsolutePath());
 975       }
 976     }
 977     byte buffer[] = new byte[1024];
 978     int readlen;
 979     FileInputStream inStream = new FileInputStream(source);
 980     FileOutputStream outStream = new FileOutputStream(dest);
 981     try {
 982       readlen = inStream.read(buffer);
 983       while (readlen > 0) {
 984         outStream.write(buffer, 0, readlen);
 985         readlen = inStream.read(buffer);
 986       }
 987     } finally {
 988       try {
 989         inStream.close();
 990       } catch (IOException ex) {
 991       }
 992       try {
 993         outStream.close();
 994       } catch (IOException ex) {
 995       }
 996     }
 997   }
 998   /** Copy (or link) one directory into another one.
 999    */
1000   private void copyOrLinkDirectories(File sourceDir, File destination)
1001       throws IOException {
1002
1003     for (String name : sourceDir.list()) {
1004       File file = new File(sourceDir, name);
1005       if (file.isDirectory()) {
1006         copyOrLinkDirectories(file, new File(destination, name));
1007       } else {
1008         copyOrLinkFile(file, new File(destination, name));
1009       }
1010     }
1011   }
1012
1013   /** deletes the staging directory, if one was created. */
1014   @Override
1015   public void cleanStagingDirectory() {
1016     if (stageDir != null) {
1017       recursiveDelete(stageDir);
1018     }
1019   }
1020
1021   /** Recursive directory deletion. */
1022   public static void recursiveDelete(File dead) {
1023     String[] files = dead.list();
1024     if (files != null) {
1025       for (String name : files) {
1026         recursiveDelete(new File(dead, name));
1027       }
1028     }
1029     dead.delete();
1030   }
1031
1032   @Override
1033   public void setListener(UpdateListener l) {
1034     listener = l;
1035   }
1036
1037   @Override
1038   public void setDetailsWriter(PrintWriter detailsWriter) {
1039     this.detailsWriter = detailsWriter;
1040   }
1041
1042   @Override
1043   public void statusUpdate(String message, int amount) {
1044     updateProgress += progressAmount;
1045     if (updateProgress > 99) {
1046       updateProgress = 99;
1047     }
1048     progressAmount = amount;
1049     if (listener != null) {
1050       listener.onProgress(new UpdateProgressEvent(
1051                               Thread.currentThread(), message, updateProgress));
1052     }
1053   }
1054
1055   @Override
1056   public void statusUpdate(String message) {
1057     int amount = progressAmount / 4;
1058     updateProgress += amount;
1059     if (updateProgress > 99) {
1060       updateProgress = 99;
1061     }
1062     progressAmount -= amount;
1063     if (listener != null) {
1064       listener.onProgress(new UpdateProgressEvent(
1065                               Thread.currentThread(), message, updateProgress));
1066     }
1067   }
1068
1069   private String generateAppYaml(File stageDir) {
1070     Set<String> staticFiles = new HashSet<String>();
1071     for (File f : new FileIterator(new File(stageDir, "__static__"))) {
1072       staticFiles.add(Utility.calculatePath(f, stageDir));
1073     }
1074
1075     AppYamlTranslator translator =
1076         new AppYamlTranslator(getAppEngineWebXml(), getWebXml(), getBackendsXml(),
1077                               getApiVersion(), staticFiles, null);
1078     String yaml = translator.getYaml();
1079     logger.fine("Generated app.yaml file:\n" + yaml);
1080     return yaml;
1081   }
1082
1083   /**
1084    * Returns the app.yaml string.
1085    *
1086    * @throws IllegalStateException if createStagingDirectory has not been called.
1087    */
1088   @Override
1089   public String getAppYaml() {
1090     if (appYaml == null) {
1091       throw new IllegalStateException("Must call createStagingDirectory first.");
1092     }
1093     return appYaml;
1094   }
1095
1096   /**
1097    * Utility to check if an application contains Java7 bytes code class.
1098    * It only checks the first class available.
1099    */
1100   static class CheckJava7Classes {
1101     boolean oneClassFound = false;
1102     boolean classIs51 = false;
1103     File root;
1104
1105     public CheckJava7Classes(File directory) {
1106       root = directory;
1107     }
1108
1109     /**
1110      *
1111      * @return true if the app contains a java7 class
1112      */
1113     public boolean isJava7App() {
1114       listFilesInDirectory(root);
1115       return classIs51;
1116     }
1117
1118     private void listFilesInDirectory(File dir) {
1119       File[] files = dir.listFiles();
1120       if (files != null) {
1121         for (File f : files) {
1122           if (f.isDirectory()) {
1123             listFilesInDirectory(f);
1124           } else {
1125             processFile(f);
1126           }
1127           if (oneClassFound) {
1128             break;
1129           }
1130         }
1131       }
1132     }
1133
1134     private void processFile(File f) {
1135       if (!f.getAbsolutePath().endsWith(".class")) {
1136         return;
1137       }
1138       DataInputStream in = null;
1139       try {
1140         in = new DataInputStream(new FileInputStream(f));
1141         int magic = in.readInt();
1142         if (magic != 0xcafebabe) {
1143           return;
1144         }
1145         int minor = in.readUnsignedShort();
1146         int major = in.readUnsignedShort();
1147         oneClassFound = true;
1148         classIs51 = major == 51;
1149
1150       } catch (IOException e) {
1151       } finally {
1152         if (in != null) {
1153           try {
1154             in.close();
1155           } catch (IOException ex) {
1156           }
1157         }
1158       }
1159     }
1160   }
1161 }