1 // Copyright 2008 Google Inc. All Rights Reserved.
3 package com
.google
.appengine
.tools
.admin
;
5 import com
.google
.appengine
.tools
.admin
.AppAdminFactory
.ApplicationProcessingOptions
;
6 import com
.google
.appengine
.tools
.info
.SdkImplInfo
;
7 import com
.google
.appengine
.tools
.info
.SdkInfo
;
8 import com
.google
.appengine
.tools
.plugins
.AppYamlProcessor
;
9 import com
.google
.appengine
.tools
.plugins
.SDKPluginManager
;
10 import com
.google
.appengine
.tools
.plugins
.SDKRuntimePlugin
;
11 import com
.google
.appengine
.tools
.util
.ApiVersionFinder
;
12 import com
.google
.appengine
.tools
.util
.FileIterator
;
13 import com
.google
.appengine
.tools
.util
.JarSplitter
;
14 import com
.google
.appengine
.tools
.util
.JarTool
;
15 import com
.google
.apphosting
.utils
.config
.AppEngineConfigException
;
16 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
;
17 import com
.google
.apphosting
.utils
.config
.AppEngineWebXmlReader
;
18 import com
.google
.apphosting
.utils
.config
.BackendsXml
;
19 import com
.google
.apphosting
.utils
.config
.BackendsXmlReader
;
20 import com
.google
.apphosting
.utils
.config
.BackendsYamlReader
;
21 import com
.google
.apphosting
.utils
.config
.CronXml
;
22 import com
.google
.apphosting
.utils
.config
.CronXmlReader
;
23 import com
.google
.apphosting
.utils
.config
.CronYamlReader
;
24 import com
.google
.apphosting
.utils
.config
.DosXml
;
25 import com
.google
.apphosting
.utils
.config
.DosXmlReader
;
26 import com
.google
.apphosting
.utils
.config
.DosYamlReader
;
27 import com
.google
.apphosting
.utils
.config
.GenerationDirectory
;
28 import com
.google
.apphosting
.utils
.config
.IndexesXml
;
29 import com
.google
.apphosting
.utils
.config
.IndexesXmlReader
;
30 import com
.google
.apphosting
.utils
.config
.QueueXml
;
31 import com
.google
.apphosting
.utils
.config
.QueueXmlReader
;
32 import com
.google
.apphosting
.utils
.config
.QueueYamlReader
;
33 import com
.google
.apphosting
.utils
.config
.WebXml
;
34 import com
.google
.apphosting
.utils
.config
.WebXmlReader
;
35 import com
.google
.common
.collect
.ImmutableSet
;
36 import com
.google
.common
.io
.Files
;
38 import org
.mortbay
.io
.Buffer
;
39 import org
.mortbay
.jetty
.MimeTypes
;
40 import org
.xml
.sax
.SAXException
;
42 import java
.io
.DataInputStream
;
44 import java
.io
.FileInputStream
;
45 import java
.io
.FileNotFoundException
;
46 import java
.io
.FileOutputStream
;
47 import java
.io
.FileWriter
;
48 import java
.io
.IOException
;
49 import java
.io
.InputStream
;
50 import java
.io
.PrintWriter
;
52 import java
.net
.URLConnection
;
53 import java
.util
.ArrayList
;
54 import java
.util
.Arrays
;
55 import java
.util
.Enumeration
;
56 import java
.util
.HashSet
;
57 import java
.util
.List
;
59 import java
.util
.jar
.JarEntry
;
60 import java
.util
.jar
.JarFile
;
61 import java
.util
.logging
.Level
;
62 import java
.util
.logging
.Logger
;
63 import java
.util
.regex
.Pattern
;
65 import javax
.activation
.FileTypeMap
;
66 import javax
.tools
.JavaCompiler
;
67 import javax
.tools
.JavaFileObject
;
68 import javax
.tools
.StandardJavaFileManager
;
69 import javax
.tools
.ToolProvider
;
70 import javax
.xml
.XMLConstants
;
71 import javax
.xml
.transform
.stream
.StreamSource
;
72 import javax
.xml
.validation
.SchemaFactory
;
75 * An App Engine application. You can {@link #readApplication read} an
76 * {@code Application} from a path, and
77 * {@link com.google.appengine.tools.admin.AppAdminFactory#createAppAdmin create}
78 * an {@link com.google.appengine.tools.admin.AppAdmin} to upload, create
79 * indexes, or otherwise manage it.
82 public class Application
implements GenericApplication
{
84 private static final int MAX_COMPILED_JSP_JAR_SIZE
= 1024 * 1024 * 5;
85 private static final String COMPILED_JSP_JAR_NAME_PREFIX
= "_ah_compiled_jsps";
87 private static final int MAX_CLASSES_JAR_SIZE
= 1024 * 1024 * 5;
88 private static final String CLASSES_JAR_NAME_PREFIX
= "_ah_webinf_classes";
90 private static Pattern JSP_REGEX
= Pattern
.compile(".*\\.jspx?");
92 /** If available, this is set to a program to make symlinks, e.g. /bin/ln */
93 private static File ln
= Utility
.findLink();
94 private static File sdkDocsDir
;
95 private static synchronized File
getSdkDocsDir(){
96 if (null == sdkDocsDir
){
97 sdkDocsDir
= new File(SdkInfo
.getSdkRoot(), "docs");
102 private static final String STAGEDIR_PREFIX
= "appcfg";
104 private static final Logger logger
= Logger
.getLogger(Application
.class.getName());
106 private static final MimeTypes mimeTypes
= new MimeTypes();
108 private AppEngineWebXml appEngineWebXml
;
109 private WebXml webXml
;
110 private CronXml cronXml
;
111 private DosXml dosXml
;
112 private String pagespeedYaml
;
113 private QueueXml queueXml
;
114 private IndexesXml indexesXml
;
115 private BackendsXml backendsXml
;
116 private File baseDir
;
117 private File stageDir
;
118 private File externalResourceDir
;
119 private String apiVersion
;
120 private String appYaml
;
122 private UpdateListener listener
;
123 private PrintWriter detailsWriter
;
124 private int updateProgress
= 0;
125 private int progressAmount
= 0;
127 protected Application(){
131 * Builds a normalized path for the given directory in which
132 * forward slashes are used as the file separator on all platforms.
133 * @param dir A directory
134 * @return The normalized path
136 private static String
buildNormalizedPath(File dir
) {
137 String normalizedPath
= dir
.getPath();
138 if (File
.separatorChar
== '\\') {
139 normalizedPath
= normalizedPath
.replace('\\', '/');
141 return normalizedPath
;
144 private Application(String explodedPath
, String appId
, String server
, String appVersion
) {
145 this.baseDir
= new File(explodedPath
);
146 explodedPath
= buildNormalizedPath(baseDir
);
147 File webinf
= new File(baseDir
, "WEB-INF");
148 if (!webinf
.getName().equals("WEB-INF")) {
149 throw new AppEngineConfigException("WEB-INF directory must be capitalized.");
152 String webinfPath
= webinf
.getPath();
153 AppEngineWebXmlReader aewebReader
= new AppEngineWebXmlReader(explodedPath
);
154 WebXmlReader webXmlReader
= new WebXmlReader(explodedPath
);
155 AppYamlProcessor
.convert(webinf
, aewebReader
.getFilename(), webXmlReader
.getFilename());
157 validateXml(aewebReader
.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd"));
158 appEngineWebXml
= aewebReader
.readAppEngineWebXml();
159 appEngineWebXml
.setSourcePrefix(explodedPath
);
161 appEngineWebXml
.setAppId(appId
);
163 if (server
!= null) {
164 appEngineWebXml
.setServer(server
);
166 if (appVersion
!= null) {
167 appEngineWebXml
.setMajorVersionId(appVersion
);
170 webXml
= webXmlReader
.readWebXml();
173 CronXmlReader cronReader
= new CronXmlReader(explodedPath
);
174 validateXml(cronReader
.getFilename(), new File(getSdkDocsDir(), "cron.xsd"));
175 cronXml
= cronReader
.readCronXml();
176 if (cronXml
== null) {
177 CronYamlReader cronYaml
= new CronYamlReader(webinfPath
);
178 cronXml
= cronYaml
.parse();
181 QueueXmlReader queueReader
= new QueueXmlReader(explodedPath
);
182 validateXml(queueReader
.getFilename(), new File(getSdkDocsDir(), "queue.xsd"));
183 queueXml
= queueReader
.readQueueXml();
184 if (queueXml
== null) {
185 QueueYamlReader queueYaml
= new QueueYamlReader(webinfPath
);
186 queueXml
= queueYaml
.parse();
189 DosXmlReader dosReader
= new DosXmlReader(explodedPath
);
190 validateXml(dosReader
.getFilename(), new File(getSdkDocsDir(), "dos.xsd"));
191 dosXml
= dosReader
.readDosXml();
192 if (dosXml
== null) {
193 DosYamlReader dosYaml
= new DosYamlReader(webinfPath
);
194 dosXml
= dosYaml
.parse();
197 if (getAppEngineWebXml().getPagespeed() != null) {
198 StringBuilder pagespeedYamlBuilder
= new StringBuilder();
199 AppYamlTranslator
.appendPagespeed(
200 getAppEngineWebXml().getPagespeed(), pagespeedYamlBuilder
, 0);
201 pagespeedYaml
= pagespeedYamlBuilder
.toString();
204 IndexesXmlReader indexReader
= new IndexesXmlReader(explodedPath
);
205 validateXml(indexReader
.getFilename(), new File(getSdkDocsDir(), "datastore-indexes.xsd"));
206 indexesXml
= indexReader
.readIndexesXml();
208 BackendsXmlReader backendsReader
= new BackendsXmlReader(explodedPath
);
209 validateXml(backendsReader
.getFilename(), new File(getSdkDocsDir(), "backends.xsd"));
210 backendsXml
= backendsReader
.readBackendsXml();
211 if (backendsXml
== null) {
212 BackendsYamlReader backendsYaml
= new BackendsYamlReader(webinfPath
);
213 backendsXml
= backendsYaml
.parse();
218 * Reads the App Engine application from {@code path}. The path may either
219 * be a WAR file or the root of an exploded WAR directory.
221 * @param path a not {@code null} path.
223 * @throws IOException if an error occurs while trying to read the
224 * {@code Application}.
225 * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
226 * {@code Application's} appengine-web.xml file is malformed.
228 public static Application
readApplication(String path
)
230 return new Application(path
, null, null, null);
234 * Sets the external resource directory. Call this method before invoking
235 * {@link #createStagingDirectory(ApplicationProcessingOptions, ResourceLimits)}.
237 * The external resource directory is a directory outside of the war directory where additional
238 * files live. These files will be copied into the staging directory during an upload, after the
239 * war directory is copied there. Consequently if there are any name collisions the files in the
240 * external resource directory will win.
242 * @param path a not {@code null} path to an existing directory.
244 * @throws IllegalArgumentException If {@code path} does not refer to an existing
247 public void setExternalResourceDir(String path
) {
249 throw new NullPointerException("path is null");
251 if (stageDir
!= null) {
252 throw new IllegalStateException(
253 "This method must be invoked prior to createStagingDirectory()");
255 File dir
= new File(path
);
257 throw new IllegalArgumentException("path does not exist: " + path
);
259 if (!dir
.isDirectory()) {
260 throw new IllegalArgumentException(path
+ " is not a directory.");
262 this.externalResourceDir
= dir
;
266 * Reads the App Engine application from {@code path}. The path may either
267 * be a WAR file or the root of an exploded WAR directory.
269 * @param path a not {@code null} path.
270 * @param appId if non-null, use this as an application id override.
271 * @param server if non-null, use this as an server id override.
272 * @param appVersion if non-null, use this as an application version override.
274 * @throws IOException if an error occurs while trying to read the
275 * {@code Application}.
276 * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
277 * {@code Application's} appengine-web.xml file is malformed.
279 public static Application
readApplication(String path
,
282 String appVersion
) throws IOException
{
283 return new Application(path
, appId
, server
, appVersion
);
287 * Returns the application identifier, from the AppEngineWebXml config
288 * @return application identifier
291 public String
getAppId() {
292 return appEngineWebXml
.getAppId();
296 * Returns the application version, from the AppEngineWebXml config
297 * @return application version
300 public String
getVersion() {
301 return appEngineWebXml
.getMajorVersionId();
305 public String
getSourceLanguage() {
306 return appEngineWebXml
.getSourceLanguage();
310 public String
getServer() {
311 return appEngineWebXml
.getServer();
315 public String
getInstanceClass() {
316 return appEngineWebXml
.getInstanceClass();
320 public boolean isPrecompilationEnabled() {
321 return appEngineWebXml
.getPrecompilationEnabled();
325 public List
<ErrorHandler
> getErrorHandlers() {
326 class ErrorHandlerImpl
implements ErrorHandler
{
327 private AppEngineWebXml
.ErrorHandler errorHandler
;
328 public ErrorHandlerImpl(AppEngineWebXml
.ErrorHandler errorHandler
) {
329 this.errorHandler
= errorHandler
;
332 public String
getFile() {
333 return "__static__/" + errorHandler
.getFile();
336 public String
getErrorCode() {
337 return errorHandler
.getErrorCode();
340 public String
getMimeType() {
341 return getMimeTypeIfStatic(getFile());
344 List
<ErrorHandler
> errorHandlers
= new ArrayList
<ErrorHandler
>();
345 for (AppEngineWebXml
.ErrorHandler errorHandler
: appEngineWebXml
.getErrorHandlers()) {
346 errorHandlers
.add(new ErrorHandlerImpl(errorHandler
));
348 return errorHandlers
;
352 public String
getMimeTypeIfStatic(String path
) {
353 if (!path
.contains("__static__/")) {
356 String mimeType
= webXml
.getMimeTypeForPath(path
);
357 if (mimeType
!= null) {
360 return guessContentTypeFromName(path
);
364 * @param fileName path of a file with extension
365 * @return the mimetype of the file (or application/octect-stream if not recognized)
367 public static String
guessContentTypeFromName(String fileName
) {
368 String defaultValue
= "application/octet-stream";
370 Buffer buffer
= mimeTypes
.getMimeByExtension(fileName
);
371 if (buffer
!= null) {
372 return new String(buffer
.asArray());
374 String lowerName
= fileName
.toLowerCase();
375 if (lowerName
.endsWith(".json")) {
376 return "application/json";
378 FileTypeMap typeMap
= FileTypeMap
.getDefaultFileTypeMap();
379 String ret
= typeMap
.getContentType(fileName
);
383 ret
= URLConnection
.guessContentTypeFromName(fileName
);
388 } catch (Throwable t
) {
389 logger
.log(Level
.WARNING
, "Error identify mimetype for " + fileName
, t
);
394 * Returns the AppEngineWebXml describing the application.
396 * @return a not {@code null} deployment descriptor
398 public AppEngineWebXml
getAppEngineWebXml() {
399 return appEngineWebXml
;
403 * Returns the CronXml describing the applications' cron jobs.
404 * @return a cron descriptor, possibly empty or {@code null}
407 public CronXml
getCronXml() {
412 * Returns the QueueXml describing the applications' task queues.
413 * @return a queue descriptor, possibly empty or {@code null}
416 public QueueXml
getQueueXml() {
421 * Returns the DosXml describing the applications' DoS entries.
422 * @return a dos descriptor, possibly empty or {@code null}
425 public DosXml
getDosXml() {
430 * Returns the pagespeed.yaml describing the applications' PageSpeed configuration.
431 * @return a pagespeed.yaml string, possibly empty or {@code null}
434 public String
getPagespeedYaml() {
435 return pagespeedYaml
;
439 * Returns the IndexesXml describing the applications' indexes.
440 * @return a index descriptor, possibly empty or {@code null}
443 public IndexesXml
getIndexesXml() {
448 * Returns the WebXml describing the applications' servlets and generic web
449 * application information.
451 * @return a WebXml descriptor, possibly empty but not {@code null}
453 public WebXml
getWebXml() {
458 public BackendsXml
getBackendsXml() {
463 * Returns the desired API version for the current application, or
464 * {@code "none"} if no API version was used.
466 * @throws IllegalStateException if createStagingDirectory has not been called.
469 public String
getApiVersion() {
470 if (apiVersion
== null) {
471 throw new IllegalStateException("Must call createStagingDirectory first.");
477 * Returns a path to an exploded WAR directory for the application.
478 * This may be a temporary directory.
480 * @return a not {@code null} path pointing to a directory
483 public String
getPath() {
484 return baseDir
.getAbsolutePath();
488 * Returns the staging directory, or {@code null} if none has been created.
491 public File
getStagingDir() {
496 public void resetProgress() {
502 * Creates a new staging directory, if needed, or returns the existing one
503 * if already created.
505 * @param opts User-specified options for processing the application.
506 * @return staging directory
507 * @throws IOException
510 public File
createStagingDirectory(ApplicationProcessingOptions opts
,
511 ResourceLimits resourceLimits
) throws IOException
{
512 if (stageDir
!= null) {
517 while (stageDir
== null && i
++ < 3) {
519 stageDir
= File
.createTempFile(STAGEDIR_PREFIX
, null);
520 } catch (IOException ex
) {
524 if (stageDir
.mkdir() == false) {
529 throw new IOException("Couldn't create a temporary directory in 3 tries.");
531 statusUpdate("Created staging directory at: '" + stageDir
.getPath() + "'", 20);
533 File staticDir
= new File(stageDir
, "__static__");
535 copyOrLink(baseDir
, stageDir
, staticDir
, false, opts
);
536 if (externalResourceDir
!= null) {
537 String previousPrefix
= appEngineWebXml
.getSourcePrefix();
538 String newPrefix
= buildNormalizedPath(externalResourceDir
);
540 appEngineWebXml
.setSourcePrefix(newPrefix
);
541 copyOrLink(externalResourceDir
, stageDir
, staticDir
, false, opts
);
543 appEngineWebXml
.setSourcePrefix(previousPrefix
);
546 if (!Boolean
.getBoolean(AppCfg
.USE_JAVA7_SYSTEM_PROP
)) {
547 CheckJava7Classes verifier
= new CheckJava7Classes(new File(stageDir
, "WEB-INF"));
548 if (verifier
.isJava7App()) {
549 System
.setProperty(AppCfg
.USE_JAVA7_SYSTEM_PROP
, "true");
552 statusUpdate("Using java7 runtime: " + Boolean
.getBoolean(AppCfg
.USE_JAVA7_SYSTEM_PROP
));
554 if (opts
.isCompileJspsSet()) {
555 compileJsps(stageDir
, opts
);
558 apiVersion
= findApiVersion(stageDir
, true);
560 appYaml
= generateAppYaml(stageDir
);
562 if (GenerationDirectory
.getGenerationDirectory(stageDir
).mkdirs()) {
563 writePreparedYamlFile("app", appYaml
);
564 writePreparedYamlFile("backends", backendsXml
== null ?
null : backendsXml
.toYaml());
565 writePreparedYamlFile("index", indexesXml
.size() == 0 ?
null : indexesXml
.toYaml());
566 writePreparedYamlFile("cron", cronXml
== null ?
null : cronXml
.toYaml());
567 writePreparedYamlFile("queue", queueXml
== null ?
null : queueXml
.toYaml());
568 writePreparedYamlFile("dos", dosXml
== null ?
null : dosXml
.toYaml());
571 int maxJarSize
= (int) resourceLimits
.maxFileSize();
573 if (opts
.isSplitJarsSet()) {
574 splitJars(new File(new File(stageDir
, "WEB-INF"), "lib"),
575 maxJarSize
, opts
.getJarSplittingExcludes());
578 if (getSourceLanguage() != null) {
579 SDKRuntimePlugin runtimePlugin
= SDKPluginManager
.findRuntimePlugin(getSourceLanguage());
580 if (runtimePlugin
!= null) {
581 runtimePlugin
.processStagingDirectory(stageDir
);
589 * Write yaml file to generation subdirectory within stage directory.
591 private void writePreparedYamlFile(String yamlName
, String yamlString
) throws IOException
{
592 File f
= new File(GenerationDirectory
.getGenerationDirectory(stageDir
), yamlName
+ ".yaml");
593 if (yamlString
!= null && f
.createNewFile()) {
594 FileWriter fw
= new FileWriter(f
);
595 fw
.write(yamlString
);
600 private static String
findApiVersion(File baseDir
, boolean deleteApiJars
) {
601 ApiVersionFinder finder
= new ApiVersionFinder();
603 String foundApiVersion
= null;
604 File webInf
= new File(baseDir
, "WEB-INF");
605 File libDir
= new File(webInf
, "lib");
606 for (File file
: new FileIterator(libDir
)) {
607 if (file
.getPath().endsWith(".jar")) {
609 String apiVersion
= finder
.findApiVersion(file
);
610 if (apiVersion
!= null) {
611 if (foundApiVersion
== null) {
612 foundApiVersion
= apiVersion
;
613 } else if (!foundApiVersion
.equals(apiVersion
)) {
614 logger
.warning("Warning: found duplicate API version: " + foundApiVersion
+
615 ", using " + apiVersion
);
621 } catch (IOException ex
) {
622 logger
.log(Level
.WARNING
, "Could not identify API version in " + file
, ex
);
627 if (foundApiVersion
== null) {
628 foundApiVersion
= "none";
630 return foundApiVersion
;
634 * Validates a given XML document against a given schema.
636 * @param xmlFilename filename with XML document
637 * @param schema XSD schema to validate with
639 * @throws AppEngineConfigException for malformed XML, or IO errors
641 private static void validateXml(String xmlFilename
, File schema
) {
642 File xml
= new File(xmlFilename
);
647 SchemaFactory factory
= SchemaFactory
.newInstance(XMLConstants
.W3C_XML_SCHEMA_NS_URI
);
649 factory
.newSchema(schema
).newValidator().validate(
650 new StreamSource(new FileInputStream(xml
)));
651 } catch (SAXException ex
) {
652 throw new AppEngineConfigException("XML error validating " +
653 xml
.getPath() + " against " + schema
.getPath(), ex
);
655 } catch (IOException ex
) {
656 throw new AppEngineConfigException("IO error validating " +
657 xml
.getPath() + " against " + schema
.getPath(), ex
);
661 private static final String JSPC_MAIN
= "com.google.appengine.tools.development.LocalJspC";
663 private void compileJsps(File stage
, ApplicationProcessingOptions opts
)
665 statusUpdate("Scanning for jsp files.");
667 if (matchingFileExists(new File(stage
.getPath()), JSP_REGEX
)) {
668 statusUpdate("Compiling jsp files.");
670 File webInf
= new File(stage
, "WEB-INF");
672 for (File file
: SdkImplInfo
.getUserJspLibFiles()) {
673 copyOrLinkFile(file
, new File(new File(webInf
, "lib"), file
.getName()));
675 for (File file
: SdkImplInfo
.getSharedJspLibFiles()) {
676 copyOrLinkFile(file
, new File(new File(webInf
, "lib"), file
.getName()));
679 File classes
= new File(webInf
, "classes");
680 File generatedWebXml
= new File(webInf
, "generated_web.xml");
681 File tempDir
= Files
.createTempDir();
682 String classpath
= getJspClasspath(classes
, tempDir
);
684 String javaCmd
= opts
.getJavaExecutable().getPath();
685 String
[] args
= new String
[] {
687 "-classpath", classpath
,
688 "-D" + AppCfg
.USE_JAVA7_SYSTEM_PROP
+ "=" +
689 System
.getProperty(AppCfg
.USE_JAVA7_SYSTEM_PROP
, "false"),
691 "-uriroot", stage
.getPath(),
692 "-p", "org.apache.jsp",
694 "-webinc", generatedWebXml
.getPath(),
695 "-d", tempDir
.getPath(),
696 "-javaEncoding", opts
.getCompileEncoding(),
698 Process jspc
= startProcess(args
);
702 status
= jspc
.waitFor();
703 } catch (InterruptedException ex
) { }
706 detailsWriter
.println("Error while executing: " + formatCommand(Arrays
.asList(args
)));
707 throw new JspCompilationException("Failed to compile jsp files.",
708 JspCompilationException
.Source
.JASPER
);
711 compileJavaFiles(classpath
, webInf
, tempDir
, opts
);
713 webXml
= new WebXmlReader(stage
.getPath()).readWebXml();
718 private void compileJavaFiles(String classpath
, File webInf
, File jspClassDir
,
719 ApplicationProcessingOptions opts
) throws IOException
{
721 JavaCompiler compiler
= ToolProvider
.getSystemJavaCompiler();
722 if (compiler
== null) {
723 throw new RuntimeException(
724 "Cannot get the System Java Compiler. Please use a JDK, not a JRE.");
726 StandardJavaFileManager fileManager
= compiler
.getStandardFileManager(null, null, null);
728 ArrayList
<File
> files
= new ArrayList
<File
>();
729 for (File f
: new FileIterator(jspClassDir
)) {
730 if (f
.getPath().toLowerCase().endsWith(".java")) {
734 if (files
.size() == 0) {
737 List
<String
> optionList
= new ArrayList
<String
>();
738 optionList
.addAll(Arrays
.asList("-classpath", classpath
.toString()));
739 optionList
.addAll(Arrays
.asList("-d", jspClassDir
.getPath()));
740 optionList
.addAll(Arrays
.asList("-encoding", opts
.getCompileEncoding()));
741 if (System
.getProperty(AppCfg
.USE_JAVA7_SYSTEM_PROP
, "false").equals("false")) {
742 optionList
.addAll(Arrays
.asList("-source", "6"));
743 optionList
.addAll(Arrays
.asList("-target", "6"));
746 Iterable
<?
extends JavaFileObject
> compilationUnits
=
747 fileManager
.getJavaFileObjectsFromFiles(files
);
748 boolean success
= compiler
.getTask(
749 null, fileManager
, null, optionList
, null, compilationUnits
).call();
753 throw new JspCompilationException("Failed to compile the generated JSP java files.",
754 JspCompilationException
.Source
.JSPC
);
756 if (opts
.isJarJSPsSet()) {
757 zipJasperGeneratedFiles(webInf
, jspClassDir
);
759 copyOrLinkDirectories(jspClassDir
, new File(webInf
, "classes"));
761 if (opts
.isDeleteJSPs()) {
762 for (File f
: new FileIterator(webInf
.getParentFile())) {
763 if (f
.getPath().toLowerCase().endsWith(".jsp")) {
768 if (opts
.isJarClassesSet()) {
769 zipWebInfClassesFiles(webInf
);
774 private void zipJasperGeneratedFiles(File webInfDir
, File jspClassDir
) throws IOException
{
775 Set
<String
> fileTypesToExclude
= ImmutableSet
.of(".java");
776 File libDir
= new File(webInfDir
, "lib");
777 JarTool jarTool
= new JarTool(
778 COMPILED_JSP_JAR_NAME_PREFIX
, jspClassDir
, libDir
, MAX_COMPILED_JSP_JAR_SIZE
,
781 recursiveDelete(jspClassDir
);
784 private void zipWebInfClassesFiles(File webInfDir
) throws IOException
{
785 File libDir
= new File(webInfDir
, "lib");
786 File classesDir
= new File(webInfDir
, "classes");
787 JarTool jarTool
= new JarTool(
788 CLASSES_JAR_NAME_PREFIX
, classesDir
, libDir
, MAX_CLASSES_JAR_SIZE
,
791 recursiveDelete(classesDir
);
795 private String
getJspClasspath(File classDir
, File genDir
) {
796 StringBuilder classpath
= new StringBuilder();
797 for (URL lib
: SdkImplInfo
.getImplLibs()) {
798 classpath
.append(lib
.getPath());
799 classpath
.append(File
.pathSeparatorChar
);
801 for (File lib
: SdkInfo
.getSharedLibFiles()) {
802 classpath
.append(lib
.getPath());
803 classpath
.append(File
.pathSeparatorChar
);
806 classpath
.append(classDir
.getPath());
807 classpath
.append(File
.pathSeparatorChar
);
808 classpath
.append(genDir
.getPath());
809 classpath
.append(File
.pathSeparatorChar
);
811 for (File f
: new FileIterator(new File(classDir
.getParentFile(), "lib"))) {
812 String filename
= f
.getPath().toLowerCase();
813 if (filename
.endsWith(".jar") || filename
.endsWith(".zip")) {
814 classpath
.append(f
.getPath());
815 classpath
.append(File
.pathSeparatorChar
);
819 return classpath
.toString();
822 private Process
startProcess(String
... args
) throws IOException
{
823 ProcessBuilder builder
= new ProcessBuilder(args
);
824 Process proc
= builder
.redirectErrorStream(true).start();
825 logger
.fine(formatCommand(builder
.command()));
826 new Thread(new OutputPump(proc
.getInputStream(), detailsWriter
)).start();
830 private String
formatCommand(Iterable
<String
> args
) {
831 StringBuilder command
= new StringBuilder();
832 for (String chunk
: args
) {
833 command
.append(chunk
);
836 return command
.toString();
840 * Scans a given directory tree, testing whether any file matches a given
843 * @param dir the directory under which to scan
844 * @param regex the pattern to look for
845 * @returns Returns {@code true} on the first instance of such a file,
846 * {@code false} otherwise.
848 private static boolean matchingFileExists(File dir
, Pattern regex
) {
849 for (File file
: dir
.listFiles()) {
850 if (file
.isDirectory()) {
851 if (matchingFileExists(file
, regex
)) {
855 if (regex
.matcher(file
.getName()).matches()) {
864 * Invokes the JarSplitter code on any jar files found in {@code dir}. Any
865 * jars larger than {@code max} will be split into fragments of at most that
867 * @param dir the directory to search, recursively
868 * @param max the maximum allowed size
869 * @param excludes a set of suffixes to exclude.
870 * @throws IOException on filesystem errors.
872 private static void splitJars(File dir
, int max
, Set
<String
> excludes
) throws IOException
{
873 String children
[] = dir
.list();
874 if (children
== null) {
877 for (String name
: children
) {
878 File subfile
= new File(dir
, name
);
879 if (subfile
.isDirectory()) {
880 splitJars(subfile
, max
, excludes
);
881 } else if (name
.endsWith(".jar")) {
882 if (subfile
.length() > max
) {
883 new JarSplitter(subfile
, dir
, max
, false, 4, excludes
).run();
890 private static final Pattern SKIP_FILES
= Pattern
.compile(
891 "^(.*/)?((#.*#)|(.*~)|(.*/RCS/.*)|)$");
894 * Copies files from the app to the upload staging directory, or makes
895 * symlinks instead if supported. Puts the files into the correct places for
896 * static vs. resource files, recursively.
898 * @param sourceDir application war dir, or on recursion a subdirectory of it
899 * @param resDir staging resource dir, or on recursion a subdirectory matching
900 * the subdirectory in {@code sourceDir}
901 * @param staticDir staging {@code __static__} dir, or an appropriate recursive
903 * @param forceResource if all files should be considered resource files
904 * @param opts processing options, used primarily for handling of *.jsp files
905 * @throws FileNotFoundException
906 * @throws IOException
908 private void copyOrLink(File sourceDir
, File resDir
, File staticDir
, boolean forceResource
,
909 ApplicationProcessingOptions opts
)
910 throws FileNotFoundException
, IOException
{
912 for (String name
: sourceDir
.list()) {
913 File file
= new File(sourceDir
, name
);
915 String path
= file
.getPath();
916 if (File
.separatorChar
== '\\') {
917 path
= path
.replace('\\', '/');
920 if (file
.getName().startsWith(".") ||
921 file
.equals(GenerationDirectory
.getGenerationDirectory(baseDir
))) {
925 if (file
.isDirectory()) {
926 if (file
.getName().equals("WEB-INF")) {
927 copyOrLink(file
, new File(resDir
, name
), new File(staticDir
, name
), true, opts
);
929 copyOrLink(file
, new File(resDir
, name
), new File(staticDir
, name
), forceResource
,
933 if (SKIP_FILES
.matcher(path
).matches()) {
937 if (forceResource
|| appEngineWebXml
.includesResource(path
) ||
938 (opts
.isCompileJspsSet() && name
.toLowerCase().endsWith(".jsp"))) {
939 copyOrLinkFile(file
, new File(resDir
, name
));
941 if (!forceResource
&& appEngineWebXml
.includesStatic(path
)) {
942 copyOrLinkFile(file
, new File(staticDir
, name
));
949 * Attempts to symlink a single file, or copies it if symlinking is either
950 * unsupported or fails.
952 * @param source source file
953 * @param dest destination file
954 * @throws FileNotFoundException
955 * @throws IOException
957 private void copyOrLinkFile(File source
, File dest
)
958 throws FileNotFoundException
, IOException
{
959 dest
.getParentFile().mkdirs();
960 if (ln
!= null && !source
.getName().endsWith("web.xml")) {
964 } catch (Exception e
) {
965 System
.err
.println("Warning: We tried to delete " + dest
.getPath());
966 System
.err
.println("in order to create a symlink from " + source
.getPath());
967 System
.err
.println("but the delete failed with message: " + e
.getMessage());
970 Process link
= startProcess(ln
.getAbsolutePath(), "-s",
971 source
.getAbsolutePath(),
972 dest
.getAbsolutePath());
974 int stat
= link
.waitFor();
978 System
.err
.println(ln
.getAbsolutePath() + " returned status " + stat
979 + ", copying instead...");
980 } catch (InterruptedException ex
) {
981 System
.err
.println(ln
.getAbsolutePath() + " was interrupted, copying instead...");
984 System
.err
.println("ln failed but symlink was created, removed: " + dest
.getAbsolutePath());
987 byte buffer
[] = new byte[1024];
989 FileInputStream inStream
= new FileInputStream(source
);
990 FileOutputStream outStream
= new FileOutputStream(dest
);
992 readlen
= inStream
.read(buffer
);
993 while (readlen
> 0) {
994 outStream
.write(buffer
, 0, readlen
);
995 readlen
= inStream
.read(buffer
);
1000 } catch (IOException ex
) {
1004 } catch (IOException ex
) {
1008 /** Copy (or link) one directory into another one.
1010 private void copyOrLinkDirectories(File sourceDir
, File destination
)
1011 throws IOException
{
1013 for (String name
: sourceDir
.list()) {
1014 File file
= new File(sourceDir
, name
);
1015 if (file
.isDirectory()) {
1016 copyOrLinkDirectories(file
, new File(destination
, name
));
1018 copyOrLinkFile(file
, new File(destination
, name
));
1023 /** deletes the staging directory, if one was created. */
1025 public void cleanStagingDirectory() {
1026 if (stageDir
!= null) {
1027 recursiveDelete(stageDir
);
1031 /** Recursive directory deletion. */
1032 public static void recursiveDelete(File dead
) {
1033 String
[] files
= dead
.list();
1034 if (files
!= null) {
1035 for (String name
: files
) {
1036 recursiveDelete(new File(dead
, name
));
1043 public void setListener(UpdateListener l
) {
1048 public void setDetailsWriter(PrintWriter detailsWriter
) {
1049 this.detailsWriter
= detailsWriter
;
1053 public void statusUpdate(String message
, int amount
) {
1054 updateProgress
+= progressAmount
;
1055 if (updateProgress
> 99) {
1056 updateProgress
= 99;
1058 progressAmount
= amount
;
1059 if (listener
!= null) {
1060 listener
.onProgress(new UpdateProgressEvent(
1061 Thread
.currentThread(), message
, updateProgress
));
1066 public void statusUpdate(String message
) {
1067 int amount
= progressAmount
/ 4;
1068 updateProgress
+= amount
;
1069 if (updateProgress
> 99) {
1070 updateProgress
= 99;
1072 progressAmount
-= amount
;
1073 if (listener
!= null) {
1074 listener
.onProgress(new UpdateProgressEvent(
1075 Thread
.currentThread(), message
, updateProgress
));
1079 private String
generateAppYaml(File stageDir
) {
1080 Set
<String
> staticFiles
= new HashSet
<String
>();
1081 for (File f
: new FileIterator(new File(stageDir
, "__static__"))) {
1082 staticFiles
.add(Utility
.calculatePath(f
, stageDir
));
1085 AppYamlTranslator translator
=
1086 new AppYamlTranslator(getAppEngineWebXml(), getWebXml(), getBackendsXml(),
1087 getApiVersion(), staticFiles
, null);
1088 String yaml
= translator
.getYaml();
1089 logger
.fine("Generated app.yaml file:\n" + yaml
);
1094 * Returns the app.yaml string.
1096 * @throws IllegalStateException if createStagingDirectory has not been called.
1099 public String
getAppYaml() {
1100 if (appYaml
== null) {
1101 throw new IllegalStateException("Must call createStagingDirectory first.");
1107 * Utility to check if an application contains Java7 bytes code class.
1108 * It only checks the first class available.
1110 static class CheckJava7Classes
{
1111 boolean oneClassFound
= false;
1112 boolean classRequiresJava7
= false;
1115 public CheckJava7Classes(File webInfDirectory
) {
1116 webInf
= webInfDirectory
;
1121 * @return true if the app contains a java7 class.
1123 public boolean isJava7App() {
1124 processClassesInDirectory(new File(webInf
, "classes"));
1125 if (!classRequiresJava7
) {
1126 processJarsInDirectory(new File(webInf
, "lib"));
1128 return classRequiresJava7
;
1131 private void processClassesInDirectory(File dir
) {
1132 File
[] files
= dir
.listFiles();
1133 if (files
!= null) {
1134 for (File f
: files
) {
1135 if (oneClassFound
) {
1138 if (f
.isDirectory()) {
1139 processClassesInDirectory(f
);
1140 } else if (f
.getName().endsWith(".class")) {
1141 oneClassFound
= true;
1143 processStream(new FileInputStream(f
));
1144 } catch (IOException e
) {
1145 logger
.severe(e
.getMessage());
1153 * Check for all jars in the given directory (WEB-INF/lib).
1155 private void processJarsInDirectory(File webInfLib
) {
1156 File
[] files
= webInfLib
.listFiles();
1157 if (files
!= null) {
1158 for (File f
: files
) {
1159 if (f
.getName().endsWith(".jar")) {
1162 } catch (IOException e
) {
1163 logger
.severe(e
.getMessage());
1165 if (classRequiresJava7
) {
1173 private void processJar(File f
) throws IOException
{
1174 JarFile jarFile
= new JarFile(f
);
1175 Enumeration
<JarEntry
> entries
= jarFile
.entries();
1176 while (entries
.hasMoreElements()) {
1177 JarEntry entry
= entries
.nextElement();
1178 if (entry
.getName().endsWith(".class")) {
1179 processStream(jarFile
.getInputStream(entry
));
1185 private void processStream(InputStream is
) {
1186 DataInputStream in
= null;
1188 in
= new DataInputStream(is
);
1189 int magic
= in
.readInt();
1190 if (magic
!= 0xcafebabe) {
1193 int minor
= in
.readUnsignedShort();
1194 int major
= in
.readUnsignedShort();
1195 classRequiresJava7
= major
== 51;
1197 } catch (IOException e
) {
1202 } catch (IOException ex
) {