1 // Copyright 2008 Google Inc. All Rights Reserved.
3 package com
.google
.appengine
.tools
.admin
;
5 import com
.google
.appengine
.tools
.admin
.AppAdminFactory
.ApplicationProcessingOptions
;
6 import com
.google
.appengine
.tools
.info
.SdkImplInfo
;
7 import com
.google
.appengine
.tools
.info
.SdkInfo
;
8 import com
.google
.appengine
.tools
.info
.Version
;
9 import com
.google
.appengine
.tools
.plugins
.AppYamlProcessor
;
10 import com
.google
.appengine
.tools
.plugins
.SDKPluginManager
;
11 import com
.google
.appengine
.tools
.plugins
.SDKRuntimePlugin
;
12 import com
.google
.appengine
.tools
.util
.ApiVersionFinder
;
13 import com
.google
.appengine
.tools
.util
.FileIterator
;
14 import com
.google
.appengine
.tools
.util
.JarSplitter
;
15 import com
.google
.appengine
.tools
.util
.JarTool
;
16 import com
.google
.apphosting
.utils
.config
.AppEngineConfigException
;
17 import com
.google
.apphosting
.utils
.config
.AppEngineWebXml
;
18 import com
.google
.apphosting
.utils
.config
.AppEngineWebXmlReader
;
19 import com
.google
.apphosting
.utils
.config
.BackendsXml
;
20 import com
.google
.apphosting
.utils
.config
.BackendsXmlReader
;
21 import com
.google
.apphosting
.utils
.config
.BackendsYamlReader
;
22 import com
.google
.apphosting
.utils
.config
.CronXml
;
23 import com
.google
.apphosting
.utils
.config
.CronXmlReader
;
24 import com
.google
.apphosting
.utils
.config
.CronYamlReader
;
25 import com
.google
.apphosting
.utils
.config
.DispatchXml
;
26 import com
.google
.apphosting
.utils
.config
.DispatchXmlReader
;
27 import com
.google
.apphosting
.utils
.config
.DispatchYamlReader
;
28 import com
.google
.apphosting
.utils
.config
.DosXml
;
29 import com
.google
.apphosting
.utils
.config
.DosXmlReader
;
30 import com
.google
.apphosting
.utils
.config
.DosYamlReader
;
31 import com
.google
.apphosting
.utils
.config
.GenerationDirectory
;
32 import com
.google
.apphosting
.utils
.config
.IndexesXml
;
33 import com
.google
.apphosting
.utils
.config
.IndexesXmlReader
;
34 import com
.google
.apphosting
.utils
.config
.QueueXml
;
35 import com
.google
.apphosting
.utils
.config
.QueueXmlReader
;
36 import com
.google
.apphosting
.utils
.config
.QueueYamlReader
;
37 import com
.google
.apphosting
.utils
.config
.WebXml
;
38 import com
.google
.apphosting
.utils
.config
.WebXmlReader
;
39 import com
.google
.common
.collect
.ImmutableSet
;
40 import com
.google
.common
.io
.Files
;
42 import org
.mortbay
.io
.Buffer
;
43 import org
.mortbay
.jetty
.MimeTypes
;
44 import org
.xml
.sax
.SAXException
;
47 import java
.io
.FileInputStream
;
48 import java
.io
.FileNotFoundException
;
49 import java
.io
.FileOutputStream
;
50 import java
.io
.FileWriter
;
51 import java
.io
.IOException
;
52 import java
.io
.PrintWriter
;
54 import java
.net
.URLConnection
;
55 import java
.util
.ArrayList
;
56 import java
.util
.Arrays
;
57 import java
.util
.HashSet
;
58 import java
.util
.List
;
60 import java
.util
.logging
.Level
;
61 import java
.util
.logging
.Logger
;
62 import java
.util
.regex
.Pattern
;
64 import javax
.activation
.FileTypeMap
;
65 import javax
.tools
.JavaCompiler
;
66 import javax
.tools
.JavaFileObject
;
67 import javax
.tools
.StandardJavaFileManager
;
68 import javax
.tools
.ToolProvider
;
69 import javax
.xml
.XMLConstants
;
70 import javax
.xml
.transform
.stream
.StreamSource
;
71 import javax
.xml
.validation
.SchemaFactory
;
74 * An App Engine application. You can {@link #readApplication read} an
75 * {@code Application} from a path, and
76 * {@link com.google.appengine.tools.admin.AppAdminFactory#createAppAdmin create}
77 * an {@link com.google.appengine.tools.admin.AppAdmin} to upload, create
78 * indexes, or otherwise manage it.
81 public class Application
implements GenericApplication
{
83 private static final int MAX_COMPILED_JSP_JAR_SIZE
= 1024 * 1024 * 5;
84 private static final String COMPILED_JSP_JAR_NAME_PREFIX
= "_ah_compiled_jsps";
86 private static final int MAX_CLASSES_JAR_SIZE
= 1024 * 1024 * 5;
87 private static final String CLASSES_JAR_NAME_PREFIX
= "_ah_webinf_classes";
89 private static final String JAVA_7_RUNTIME_ID
= "java7";
90 private static final ImmutableSet
<String
> ALLOWED_RUNTIME_IDS
= ImmutableSet
.of(
93 private static Pattern JSP_REGEX
= Pattern
.compile(".*\\.jspx?");
95 /** If available, this is set to a program to make symlinks, e.g. /bin/ln */
96 private static File ln
= Utility
.findLink();
97 private static File sdkDocsDir
;
98 public static synchronized File
getSdkDocsDir(){
99 if (null == sdkDocsDir
){
100 sdkDocsDir
= new File(SdkInfo
.getSdkRoot(), "docs");
105 private static Version sdkVersion
;
106 public static synchronized Version
getSdkVersion() {
107 if (null == sdkVersion
) {
108 sdkVersion
= SdkInfo
.getLocalVersion();
113 private static final String STAGEDIR_PREFIX
= "appcfg";
115 private static final Logger logger
= Logger
.getLogger(Application
.class.getName());
117 private static final MimeTypes mimeTypes
= new MimeTypes();
119 private AppEngineWebXml appEngineWebXml
;
120 private WebXml webXml
;
121 private CronXml cronXml
;
122 private DispatchXml dispatchXml
;
123 private DosXml dosXml
;
124 private String pagespeedYaml
;
125 private QueueXml queueXml
;
126 private IndexesXml indexesXml
;
127 private BackendsXml backendsXml
;
128 private File baseDir
;
129 private File stageDir
;
130 private File externalResourceDir
;
131 private String apiVersion
;
132 private String appYaml
;
134 private UpdateListener listener
;
135 private PrintWriter detailsWriter
;
136 private int updateProgress
= 0;
137 private int progressAmount
= 0;
139 protected Application(){
143 * Builds a normalized path for the given directory in which
144 * forward slashes are used as the file separator on all platforms.
145 * @param dir A directory
146 * @return The normalized path
148 private static String
buildNormalizedPath(File dir
) {
149 String normalizedPath
= dir
.getPath();
150 if (File
.separatorChar
== '\\') {
151 normalizedPath
= normalizedPath
.replace('\\', '/');
153 return normalizedPath
;
156 private Application(String explodedPath
, String appId
, String module
, String appVersion
) {
157 this.baseDir
= new File(explodedPath
);
158 explodedPath
= buildNormalizedPath(baseDir
);
159 File webinf
= new File(baseDir
, "WEB-INF");
160 if (!webinf
.getName().equals("WEB-INF")) {
161 throw new AppEngineConfigException("WEB-INF directory must be capitalized.");
164 String webinfPath
= webinf
.getPath();
165 AppEngineWebXmlReader aewebReader
= new AppEngineWebXmlReader(explodedPath
);
166 WebXmlReader webXmlReader
= new WebXmlReader(explodedPath
);
167 AppYamlProcessor
.convert(webinf
, aewebReader
.getFilename(), webXmlReader
.getFilename());
169 validateXml(aewebReader
.getFilename(), new File(getSdkDocsDir(), "appengine-web.xsd"));
170 appEngineWebXml
= aewebReader
.readAppEngineWebXml();
171 appEngineWebXml
.setSourcePrefix(explodedPath
);
173 if (appEngineWebXml
.getAppId() == null) {
174 throw new AppEngineConfigException(
175 "No app id supplied and XML files have no <application> element");
178 appEngineWebXml
.setAppId(appId
);
180 if (module
!= null) {
181 appEngineWebXml
.setModule(module
);
183 if (appVersion
!= null) {
184 appEngineWebXml
.setMajorVersionId(appVersion
);
187 webXml
= webXmlReader
.readWebXml();
190 CronXmlReader cronReader
= new CronXmlReader(explodedPath
);
191 validateXml(cronReader
.getFilename(), new File(getSdkDocsDir(), "cron.xsd"));
192 cronXml
= cronReader
.readCronXml();
193 if (cronXml
== null) {
194 CronYamlReader cronYaml
= new CronYamlReader(webinfPath
);
195 cronXml
= cronYaml
.parse();
198 QueueXmlReader queueReader
= new QueueXmlReader(explodedPath
);
199 validateXml(queueReader
.getFilename(), new File(getSdkDocsDir(), "queue.xsd"));
200 queueXml
= queueReader
.readQueueXml();
201 if (queueXml
== null) {
202 QueueYamlReader queueYaml
= new QueueYamlReader(webinfPath
);
203 queueXml
= queueYaml
.parse();
206 DispatchXmlReader dispatchXmlReader
= new DispatchXmlReader(explodedPath
,
207 DispatchXmlReader
.DEFAULT_RELATIVE_FILENAME
);
208 validateXml(dispatchXmlReader
.getFilename(), new File(getSdkDocsDir(), "dispatch.xsd"));
209 dispatchXml
= dispatchXmlReader
.readDispatchXml();
210 if (dispatchXml
== null) {
211 DispatchYamlReader dispatchYamlReader
= new DispatchYamlReader(webinfPath
);
212 dispatchXml
= dispatchYamlReader
.parse();
215 DosXmlReader dosReader
= new DosXmlReader(explodedPath
);
216 validateXml(dosReader
.getFilename(), new File(getSdkDocsDir(), "dos.xsd"));
217 dosXml
= dosReader
.readDosXml();
218 if (dosXml
== null) {
219 DosYamlReader dosYaml
= new DosYamlReader(webinfPath
);
220 dosXml
= dosYaml
.parse();
223 if (getAppEngineWebXml().getPagespeed() != null) {
224 StringBuilder pagespeedYamlBuilder
= new StringBuilder();
225 AppYamlTranslator
.appendPagespeed(
226 getAppEngineWebXml().getPagespeed(), pagespeedYamlBuilder
, 0);
227 pagespeedYaml
= pagespeedYamlBuilder
.toString();
230 IndexesXmlReader indexReader
= new IndexesXmlReader(explodedPath
);
231 File datastoreSchema
= new File(getSdkDocsDir(), "datastore-indexes.xsd");
232 validateXml(indexReader
.getFilename(), datastoreSchema
);
233 indexesXml
= indexReader
.readIndexesXml();
235 BackendsXmlReader backendsReader
= new BackendsXmlReader(explodedPath
);
236 validateXml(backendsReader
.getFilename(), new File(getSdkDocsDir(), "backends.xsd"));
237 backendsXml
= backendsReader
.readBackendsXml();
238 if (backendsXml
== null) {
239 BackendsYamlReader backendsYaml
= new BackendsYamlReader(webinfPath
);
240 backendsXml
= backendsYaml
.parse();
245 * Reads the App Engine application from {@code path}. The path may either
246 * be a WAR file or the root of an exploded WAR directory.
248 * @param path a not {@code null} path.
250 * @throws IOException if an error occurs while trying to read the
251 * {@code Application}.
252 * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
253 * {@code Application's} appengine-web.xml file is malformed.
255 public static Application
readApplication(String path
)
257 return new Application(path
, null, null, null);
261 * Sets the external resource directory. Call this method before invoking
262 * {@link #createStagingDirectory(ApplicationProcessingOptions, ResourceLimits)}.
264 * The external resource directory is a directory outside of the war directory where additional
265 * files live. These files will be copied into the staging directory during an upload, after the
266 * war directory is copied there. Consequently if there are any name collisions the files in the
267 * external resource directory will win.
269 * @param path a not {@code null} path to an existing directory.
271 * @throws IllegalArgumentException If {@code path} does not refer to an existing
274 public void setExternalResourceDir(String path
) {
276 throw new NullPointerException("path is null");
278 if (stageDir
!= null) {
279 throw new IllegalStateException(
280 "This method must be invoked prior to createStagingDirectory()");
282 File dir
= new File(path
);
284 throw new IllegalArgumentException("path does not exist: " + path
);
286 if (!dir
.isDirectory()) {
287 throw new IllegalArgumentException(path
+ " is not a directory.");
289 this.externalResourceDir
= dir
;
293 * Reads the App Engine application from {@code path}. The path may either
294 * be a WAR file or the root of an exploded WAR directory.
296 * @param path a not {@code null} path.
297 * @param appId if non-null, use this as an application id override.
298 * @param module if non-null, use this as a module id override.
299 * @param appVersion if non-null, use this as an application version override.
301 * @throws IOException if an error occurs while trying to read the
302 * {@code Application}.
303 * @throws com.google.apphosting.utils.config.AppEngineConfigException if the
304 * {@code Application's} appengine-web.xml file is malformed.
306 public static Application
readApplication(String path
,
309 String appVersion
) throws IOException
{
310 return new Application(path
, appId
, module
, appVersion
);
314 * Returns the application identifier, from the AppEngineWebXml config
315 * @return application identifier
318 public String
getAppId() {
319 return appEngineWebXml
.getAppId();
323 * Returns the application version, from the AppEngineWebXml config
324 * @return application version
327 public String
getVersion() {
328 return appEngineWebXml
.getMajorVersionId();
332 public String
getSourceLanguage() {
333 return appEngineWebXml
.getSourceLanguage();
337 public String
getModule() {
338 return appEngineWebXml
.getModule();
342 public String
getInstanceClass() {
343 return appEngineWebXml
.getInstanceClass();
347 public boolean isPrecompilationEnabled() {
348 return appEngineWebXml
.getPrecompilationEnabled();
352 public List
<ErrorHandler
> getErrorHandlers() {
353 class ErrorHandlerImpl
implements ErrorHandler
{
354 private final AppEngineWebXml
.ErrorHandler errorHandler
;
355 public ErrorHandlerImpl(AppEngineWebXml
.ErrorHandler errorHandler
) {
356 this.errorHandler
= errorHandler
;
359 public String
getFile() {
360 return "__static__/" + errorHandler
.getFile();
363 public String
getErrorCode() {
364 return errorHandler
.getErrorCode();
367 public String
getMimeType() {
368 return getMimeTypeIfStatic(getFile());
371 List
<ErrorHandler
> errorHandlers
= new ArrayList
<ErrorHandler
>();
372 for (AppEngineWebXml
.ErrorHandler errorHandler
: appEngineWebXml
.getErrorHandlers()) {
373 errorHandlers
.add(new ErrorHandlerImpl(errorHandler
));
375 return errorHandlers
;
379 public String
getMimeTypeIfStatic(String path
) {
380 if (!path
.contains("__static__/")) {
383 String mimeType
= webXml
.getMimeTypeForPath(path
);
384 if (mimeType
!= null) {
387 return guessContentTypeFromName(path
);
391 * @param fileName path of a file with extension
392 * @return the mimetype of the file (or application/octect-stream if not recognized)
394 public static String
guessContentTypeFromName(String fileName
) {
395 String defaultValue
= "application/octet-stream";
397 Buffer buffer
= mimeTypes
.getMimeByExtension(fileName
);
398 if (buffer
!= null) {
399 return new String(buffer
.asArray());
401 String lowerName
= fileName
.toLowerCase();
402 if (lowerName
.endsWith(".json")) {
403 return "application/json";
405 FileTypeMap typeMap
= FileTypeMap
.getDefaultFileTypeMap();
406 String ret
= typeMap
.getContentType(fileName
);
410 ret
= URLConnection
.guessContentTypeFromName(fileName
);
415 } catch (Throwable t
) {
416 logger
.log(Level
.WARNING
, "Error identify mimetype for " + fileName
, t
);
421 * Returns the AppEngineWebXml describing the application.
423 * @return a not {@code null} deployment descriptor
425 public AppEngineWebXml
getAppEngineWebXml() {
426 return appEngineWebXml
;
430 * Returns the CronXml describing the applications' cron jobs.
431 * @return a cron descriptor, possibly empty or {@code null}
434 public CronXml
getCronXml() {
439 * Returns the QueueXml describing the applications' task queues.
440 * @return a queue descriptor, possibly empty or {@code null}
443 public QueueXml
getQueueXml() {
448 public DispatchXml
getDispatchXml() {
453 * Returns the DosXml describing the applications' DoS entries.
454 * @return a dos descriptor, possibly empty or {@code null}
457 public DosXml
getDosXml() {
462 * Returns the pagespeed.yaml describing the applications' PageSpeed configuration.
463 * @return a pagespeed.yaml string, possibly empty or {@code null}
466 public String
getPagespeedYaml() {
467 return pagespeedYaml
;
471 * Returns the IndexesXml describing the applications' indexes.
472 * @return a index descriptor, possibly empty or {@code null}
475 public IndexesXml
getIndexesXml() {
480 * Returns the WebXml describing the applications' servlets and generic web
481 * application information.
483 * @return a WebXml descriptor, possibly empty but not {@code null}
485 public WebXml
getWebXml() {
490 public BackendsXml
getBackendsXml() {
495 * Returns the desired API version for the current application, or
496 * {@code "none"} if no API version was used.
498 * @throws IllegalStateException if createStagingDirectory has not been called.
501 public String
getApiVersion() {
502 if (apiVersion
== null) {
503 throw new IllegalStateException("Must call createStagingDirectory first.");
509 * Returns a path to an exploded WAR directory for the application.
510 * This may be a temporary directory.
512 * @return a not {@code null} path pointing to a directory
515 public String
getPath() {
516 return baseDir
.getAbsolutePath();
520 * Returns the staging directory, or {@code null} if none has been created.
523 public File
getStagingDir() {
528 public void resetProgress() {
534 * Creates a new staging directory, if needed, or returns the existing one
535 * if already created.
537 * @param opts User-specified options for processing the application.
538 * @return staging directory
539 * @throws IOException
542 public File
createStagingDirectory(ApplicationProcessingOptions opts
,
543 ResourceLimits resourceLimits
) throws IOException
{
544 if (stageDir
!= null) {
549 while (stageDir
== null && i
++ < 3) {
551 stageDir
= File
.createTempFile(STAGEDIR_PREFIX
, null);
552 } catch (IOException ex
) {
556 if (!stageDir
.mkdir()) {
561 throw new IOException("Couldn't create a temporary directory in 3 tries.");
563 statusUpdate("Created staging directory at: '" + stageDir
.getPath() + "'", 20);
565 return populateStagingDirectory(opts
, resourceLimits
);
569 * Populates and creates (if necessary) a user specified, staging directory
571 * @param opts User-specified options for processing the application.
572 * @param resourceLimits Various resource limits provided by the cloud.
573 * @param stagingDir User-specified staging directory (must be empty or not exist)
574 * @return staging directory
575 * @throws IOException if an error occurs trying to create or populate the staging directory
578 public File
createStagingDirectory(ApplicationProcessingOptions opts
,
579 ResourceLimits resourceLimits
, File stagingDir
) throws IOException
{
580 if (!stagingDir
.exists()) {
581 if (!stagingDir
.mkdir()) {
582 throw new IOException("Could not create staging directory at " + stagingDir
.getPath());
586 stageDir
= stagingDir
;
588 return populateStagingDirectory(opts
, resourceLimits
);
591 private File
populateStagingDirectory(ApplicationProcessingOptions opts
,
592 ResourceLimits resourceLimits
) throws IOException
{
593 File staticDir
= new File(stageDir
, "__static__");
595 copyOrLink(baseDir
, stageDir
, staticDir
, false, opts
);
596 if (externalResourceDir
!= null) {
597 String previousPrefix
= appEngineWebXml
.getSourcePrefix();
598 String newPrefix
= buildNormalizedPath(externalResourceDir
);
600 appEngineWebXml
.setSourcePrefix(newPrefix
);
601 copyOrLink(externalResourceDir
, stageDir
, staticDir
, false, opts
);
603 appEngineWebXml
.setSourcePrefix(previousPrefix
);
607 apiVersion
= findApiVersion(stageDir
, true);
609 String runtime
= getRuntime(opts
);
611 if (opts
.isCompileJspsSet()) {
612 compileJsps(stageDir
, opts
);
615 appYaml
= generateAppYaml(stageDir
, runtime
);
617 if (GenerationDirectory
.getGenerationDirectory(stageDir
).mkdirs()) {
618 writePreparedYamlFile("app", appYaml
);
619 writePreparedYamlFile("backends", backendsXml
== null ?
null : backendsXml
.toYaml());
620 writePreparedYamlFile("index", indexesXml
.size() == 0 ?
null : indexesXml
.toYaml());
621 writePreparedYamlFile("cron", cronXml
== null ?
null : cronXml
.toYaml());
622 writePreparedYamlFile("queue", queueXml
== null ?
null : queueXml
.toYaml());
623 writePreparedYamlFile("dos", dosXml
== null ?
null : dosXml
.toYaml());
626 int maxJarSize
= (int) resourceLimits
.maxFileSize();
628 if (opts
.isSplitJarsSet()) {
629 splitJars(new File(new File(stageDir
, "WEB-INF"), "lib"),
630 maxJarSize
, opts
.getJarSplittingExcludes());
633 if (getSourceLanguage() != null) {
634 SDKRuntimePlugin runtimePlugin
= SDKPluginManager
.findRuntimePlugin(getSourceLanguage());
635 if (runtimePlugin
!= null) {
636 runtimePlugin
.processStagingDirectory(stageDir
);
644 * Write yaml file to generation subdirectory within stage directory.
646 private void writePreparedYamlFile(String yamlName
, String yamlString
) throws IOException
{
647 File f
= new File(GenerationDirectory
.getGenerationDirectory(stageDir
), yamlName
+ ".yaml");
648 if (yamlString
!= null && f
.createNewFile()) {
649 FileWriter fw
= new FileWriter(f
);
650 fw
.write(yamlString
);
655 private static String
findApiVersion(File baseDir
, boolean deleteApiJars
) {
656 ApiVersionFinder finder
= new ApiVersionFinder();
658 String foundApiVersion
= null;
659 File webInf
= new File(baseDir
, "WEB-INF");
660 File libDir
= new File(webInf
, "lib");
661 for (File file
: new FileIterator(libDir
)) {
662 if (file
.getPath().endsWith(".jar")) {
664 String apiVersion
= finder
.findApiVersion(file
);
665 if (apiVersion
!= null) {
666 if (foundApiVersion
== null) {
667 foundApiVersion
= apiVersion
;
668 } else if (!foundApiVersion
.equals(apiVersion
)) {
669 logger
.warning("Warning: found duplicate API version: " + foundApiVersion
+
670 ", using " + apiVersion
);
673 if (!file
.delete()) {
674 logger
.log(Level
.SEVERE
, "Could not delete API jar: " + file
);
678 } catch (IOException ex
) {
679 logger
.log(Level
.WARNING
, "Could not identify API version in " + file
, ex
);
684 if (foundApiVersion
== null) {
685 foundApiVersion
= "none";
687 return foundApiVersion
;
691 * Returns the runtime id to use in the generated app.yaml.
693 * This method returns {@code "java7"}, unless an explicit runtime id was specified
694 * using the {@code -r} option.
696 * Before accepting an explicit runtime id, this method validates it against the list of
697 * supported Java runtimes (currently only {@code "java7"}), unless validation was turned
698 * off using the {@code --allowAnyRuntimes} option.
700 private String
getRuntime(ApplicationProcessingOptions opts
) {
701 String runtime
= opts
.getRuntime();
702 if (runtime
!= null) {
703 if (!opts
.isAllowAnyRuntime() && !ALLOWED_RUNTIME_IDS
.contains(runtime
)) {
704 throw new AppEngineConfigException("Invalid runtime id: " + runtime
+ ". Valid " +
705 "runtime id: java7.");
709 return JAVA_7_RUNTIME_ID
;
713 * Validates a given XML document against a given schema.
715 * @param xmlFilename filename with XML document
716 * @param schema XSD schema to validate with
718 * @throws AppEngineConfigException for malformed XML, or IO errors
720 private static void validateXml(String xmlFilename
, File schema
) {
721 File xml
= new File(xmlFilename
);
726 SchemaFactory factory
= SchemaFactory
.newInstance(XMLConstants
.W3C_XML_SCHEMA_NS_URI
);
728 factory
.newSchema(schema
).newValidator().validate(
729 new StreamSource(new FileInputStream(xml
)));
730 } catch (SAXException ex
) {
731 throw new AppEngineConfigException("XML error validating " +
732 xml
.getPath() + " against " + schema
.getPath(), ex
);
734 } catch (IOException ex
) {
735 throw new AppEngineConfigException("IO error validating " +
736 xml
.getPath() + " against " + schema
.getPath(), ex
);
740 private static final String JSPC_MAIN
= "com.google.appengine.tools.development.LocalJspC";
742 private void compileJsps(File stage
, ApplicationProcessingOptions opts
)
744 statusUpdate("Scanning for jsp files.");
746 if (matchingFileExists(new File(stage
.getPath()), JSP_REGEX
)) {
747 statusUpdate("Compiling jsp files.");
749 File webInf
= new File(stage
, "WEB-INF");
751 for (File file
: SdkImplInfo
.getUserJspLibFiles()) {
752 copyOrLinkFile(file
, new File(new File(webInf
, "lib"), file
.getName()));
754 for (File file
: SdkImplInfo
.getSharedJspLibFiles()) {
755 copyOrLinkFile(file
, new File(new File(webInf
, "lib"), file
.getName()));
758 File classes
= new File(webInf
, "classes");
759 File generatedWebXml
= new File(webInf
, "generated_web.xml");
760 File tempDir
= Files
.createTempDir();
761 String classpath
= getJspClasspath(classes
, tempDir
);
763 String javaCmd
= opts
.getJavaExecutable().getPath();
764 String
[] args
= new String
[] {
766 "-classpath", classpath
,
768 "-uriroot", stage
.getPath(),
769 "-p", "org.apache.jsp",
771 "-webinc", generatedWebXml
.getPath(),
772 "-d", tempDir
.getPath(),
773 "-javaEncoding", opts
.getCompileEncoding(),
775 Process jspc
= startProcess(args
);
779 status
= jspc
.waitFor();
780 } catch (InterruptedException ex
) { }
783 detailsWriter
.println("Error while executing: " + formatCommand(Arrays
.asList(args
)));
784 throw new JspCompilationException("Failed to compile jsp files.",
785 JspCompilationException
.Source
.JASPER
);
788 compileJavaFiles(classpath
, webInf
, tempDir
, opts
);
790 webXml
= new WebXmlReader(stage
.getPath()).readWebXml();
795 private void compileJavaFiles(String classpath
, File webInf
, File jspClassDir
,
796 ApplicationProcessingOptions opts
) throws IOException
{
798 JavaCompiler compiler
= ToolProvider
.getSystemJavaCompiler();
799 if (compiler
== null) {
800 throw new RuntimeException(
801 "Cannot get the System Java Compiler. Please use a JDK, not a JRE.");
803 StandardJavaFileManager fileManager
= compiler
.getStandardFileManager(null, null, null);
805 ArrayList
<File
> files
= new ArrayList
<File
>();
806 for (File f
: new FileIterator(jspClassDir
)) {
807 if (f
.getPath().toLowerCase().endsWith(".java")) {
811 if (files
.isEmpty()) {
814 List
<String
> optionList
= new ArrayList
<String
>();
815 optionList
.addAll(Arrays
.asList("-classpath", classpath
.toString()));
816 optionList
.addAll(Arrays
.asList("-d", jspClassDir
.getPath()));
817 optionList
.addAll(Arrays
.asList("-encoding", opts
.getCompileEncoding()));
819 Iterable
<?
extends JavaFileObject
> compilationUnits
=
820 fileManager
.getJavaFileObjectsFromFiles(files
);
821 boolean success
= compiler
.getTask(
822 null, fileManager
, null, optionList
, null, compilationUnits
).call();
826 throw new JspCompilationException("Failed to compile the generated JSP java files.",
827 JspCompilationException
.Source
.JSPC
);
829 if (opts
.isJarJSPsSet()) {
830 zipJasperGeneratedFiles(webInf
, jspClassDir
);
832 copyOrLinkDirectories(jspClassDir
, new File(webInf
, "classes"));
834 if (opts
.isDeleteJSPs()) {
835 for (File f
: new FileIterator(webInf
.getParentFile())) {
836 if (f
.getPath().toLowerCase().endsWith(".jsp")) {
841 if (opts
.isJarClassesSet()) {
842 zipWebInfClassesFiles(webInf
);
847 private void zipJasperGeneratedFiles(File webInfDir
, File jspClassDir
) throws IOException
{
848 Set
<String
> fileTypesToExclude
= ImmutableSet
.of(".java");
849 File libDir
= new File(webInfDir
, "lib");
850 JarTool jarTool
= new JarTool(
851 COMPILED_JSP_JAR_NAME_PREFIX
, jspClassDir
, libDir
, MAX_COMPILED_JSP_JAR_SIZE
,
854 recursiveDelete(jspClassDir
);
857 private void zipWebInfClassesFiles(File webInfDir
) throws IOException
{
858 File libDir
= new File(webInfDir
, "lib");
859 File classesDir
= new File(webInfDir
, "classes");
860 JarTool jarTool
= new JarTool(
861 CLASSES_JAR_NAME_PREFIX
, classesDir
, libDir
, MAX_CLASSES_JAR_SIZE
,
864 recursiveDelete(classesDir
);
868 private String
getJspClasspath(File classDir
, File genDir
) {
869 StringBuilder classpath
= new StringBuilder();
870 for (URL lib
: SdkImplInfo
.getImplLibs()) {
871 classpath
.append(lib
.getPath());
872 classpath
.append(File
.pathSeparatorChar
);
874 for (File lib
: SdkInfo
.getSharedLibFiles()) {
875 classpath
.append(lib
.getPath());
876 classpath
.append(File
.pathSeparatorChar
);
879 classpath
.append(classDir
.getPath());
880 classpath
.append(File
.pathSeparatorChar
);
881 classpath
.append(genDir
.getPath());
882 classpath
.append(File
.pathSeparatorChar
);
884 for (File f
: new FileIterator(new File(classDir
.getParentFile(), "lib"))) {
885 String filename
= f
.getPath().toLowerCase();
886 if (filename
.endsWith(".jar") || filename
.endsWith(".zip")) {
887 classpath
.append(f
.getPath());
888 classpath
.append(File
.pathSeparatorChar
);
892 return classpath
.toString();
895 private Process
startProcess(String
... args
) throws IOException
{
896 ProcessBuilder builder
= new ProcessBuilder(args
);
897 Process proc
= builder
.redirectErrorStream(true).start();
898 logger
.fine(formatCommand(builder
.command()));
899 new Thread(new OutputPump(proc
.getInputStream(), detailsWriter
)).start();
903 private String
formatCommand(Iterable
<String
> args
) {
904 StringBuilder command
= new StringBuilder();
905 for (String chunk
: args
) {
906 command
.append(chunk
);
909 return command
.toString();
913 * Scans a given directory tree, testing whether any file matches a given
916 * @param dir the directory under which to scan
917 * @param regex the pattern to look for
918 * @returns Returns {@code true} on the first instance of such a file,
919 * {@code false} otherwise.
921 private static boolean matchingFileExists(File dir
, Pattern regex
) {
922 for (File file
: dir
.listFiles()) {
923 if (file
.isDirectory()) {
924 if (matchingFileExists(file
, regex
)) {
928 if (regex
.matcher(file
.getName()).matches()) {
937 * Invokes the JarSplitter code on any jar files found in {@code dir}. Any
938 * jars larger than {@code max} will be split into fragments of at most that
940 * @param dir the directory to search, recursively
941 * @param max the maximum allowed size
942 * @param excludes a set of suffixes to exclude.
943 * @throws IOException on filesystem errors.
945 private static void splitJars(File dir
, int max
, Set
<String
> excludes
) throws IOException
{
946 String children
[] = dir
.list();
947 if (children
== null) {
950 for (String name
: children
) {
951 File subfile
= new File(dir
, name
);
952 if (subfile
.isDirectory()) {
953 splitJars(subfile
, max
, excludes
);
954 } else if (name
.endsWith(".jar")) {
955 if (subfile
.length() > max
) {
956 new JarSplitter(subfile
, dir
, max
, false, 4, excludes
).run();
963 private static final Pattern SKIP_FILES
= Pattern
.compile(
964 "^(.*/)?((#.*#)|(.*~)|(.*/RCS/.*)|)$");
967 * Copies files from the app to the upload staging directory, or makes
968 * symlinks instead if supported. Puts the files into the correct places for
969 * static vs. resource files, recursively.
971 * @param sourceDir application war dir, or on recursion a subdirectory of it
972 * @param resDir staging resource dir, or on recursion a subdirectory matching
973 * the subdirectory in {@code sourceDir}
974 * @param staticDir staging {@code __static__} dir, or an appropriate recursive
976 * @param forceResource if all files should be considered resource files
977 * @param opts processing options, used primarily for handling of *.jsp files
978 * @throws FileNotFoundException
979 * @throws IOException
981 private void copyOrLink(File sourceDir
, File resDir
, File staticDir
, boolean forceResource
,
982 ApplicationProcessingOptions opts
)
983 throws FileNotFoundException
, IOException
{
985 for (String name
: sourceDir
.list()) {
986 File file
= new File(sourceDir
, name
);
988 String path
= file
.getPath();
989 if (File
.separatorChar
== '\\') {
990 path
= path
.replace('\\', '/');
993 if (file
.getName().startsWith(".") ||
994 file
.equals(GenerationDirectory
.getGenerationDirectory(baseDir
))) {
998 if (file
.isDirectory()) {
999 if (file
.getName().equals("WEB-INF")) {
1000 copyOrLink(file
, new File(resDir
, name
), new File(staticDir
, name
), true, opts
);
1002 copyOrLink(file
, new File(resDir
, name
), new File(staticDir
, name
), forceResource
,
1006 if (SKIP_FILES
.matcher(path
).matches()) {
1010 if (forceResource
|| appEngineWebXml
.includesResource(path
) ||
1011 (opts
.isCompileJspsSet() && name
.toLowerCase().endsWith(".jsp"))) {
1012 copyOrLinkFile(file
, new File(resDir
, name
));
1014 if (!forceResource
&& appEngineWebXml
.includesStatic(path
)) {
1015 copyOrLinkFile(file
, new File(staticDir
, name
));
1022 * Attempts to symlink a single file, or copies it if symlinking is either
1023 * unsupported or fails.
1025 * @param source source file
1026 * @param dest destination file
1027 * @throws FileNotFoundException
1028 * @throws IOException
1030 private void copyOrLinkFile(File source
, File dest
)
1031 throws FileNotFoundException
, IOException
{
1032 dest
.getParentFile().mkdirs();
1033 if (ln
!= null && !source
.getName().endsWith("web.xml")) {
1037 } catch (Exception e
) {
1038 System
.err
.println("Warning: We tried to delete " + dest
.getPath());
1039 System
.err
.println("in order to create a symlink from " + source
.getPath());
1040 System
.err
.println("but the delete failed with message: " + e
.getMessage());
1043 Process link
= startProcess(ln
.getAbsolutePath(), "-s",
1044 source
.getAbsolutePath(),
1045 dest
.getAbsolutePath());
1047 int stat
= link
.waitFor();
1051 System
.err
.println(ln
.getAbsolutePath() + " returned status " + stat
1052 + ", copying instead...");
1053 } catch (InterruptedException ex
) {
1054 System
.err
.println(ln
.getAbsolutePath() + " was interrupted, copying instead...");
1056 if (dest
.delete()) {
1057 System
.err
.println("ln failed but symlink was created, removed: " + dest
.getAbsolutePath());
1060 byte buffer
[] = new byte[1024];
1062 FileInputStream inStream
= new FileInputStream(source
);
1063 FileOutputStream outStream
= new FileOutputStream(dest
);
1065 readlen
= inStream
.read(buffer
);
1066 while (readlen
> 0) {
1067 outStream
.write(buffer
, 0, readlen
);
1068 readlen
= inStream
.read(buffer
);
1073 } catch (IOException ex
) {
1077 } catch (IOException ex
) {
1081 /** Copy (or link) one directory into another one.
1083 private void copyOrLinkDirectories(File sourceDir
, File destination
)
1084 throws IOException
{
1086 for (String name
: sourceDir
.list()) {
1087 File file
= new File(sourceDir
, name
);
1088 if (file
.isDirectory()) {
1089 copyOrLinkDirectories(file
, new File(destination
, name
));
1091 copyOrLinkFile(file
, new File(destination
, name
));
1096 /** deletes the staging directory, if one was created. */
1098 public void cleanStagingDirectory() {
1099 if (stageDir
!= null) {
1100 recursiveDelete(stageDir
);
1104 /** Recursive directory deletion. */
1105 public static void recursiveDelete(File dead
) {
1106 String
[] files
= dead
.list();
1107 if (files
!= null) {
1108 for (String name
: files
) {
1109 recursiveDelete(new File(dead
, name
));
1116 public void setListener(UpdateListener l
) {
1121 public void setDetailsWriter(PrintWriter detailsWriter
) {
1122 this.detailsWriter
= detailsWriter
;
1126 public void statusUpdate(String message
, int amount
) {
1127 updateProgress
+= progressAmount
;
1128 if (updateProgress
> 99) {
1129 updateProgress
= 99;
1131 progressAmount
= amount
;
1132 if (listener
!= null) {
1133 listener
.onProgress(new UpdateProgressEvent(
1134 Thread
.currentThread(), message
, updateProgress
));
1139 public void statusUpdate(String message
) {
1140 int amount
= progressAmount
/ 4;
1141 updateProgress
+= amount
;
1142 if (updateProgress
> 99) {
1143 updateProgress
= 99;
1145 progressAmount
-= amount
;
1146 if (listener
!= null) {
1147 listener
.onProgress(new UpdateProgressEvent(
1148 Thread
.currentThread(), message
, updateProgress
));
1152 private String
generateAppYaml(File stageDir
, String runtime
) {
1153 Set
<String
> staticFiles
= new HashSet
<String
>();
1154 for (File f
: new FileIterator(new File(stageDir
, "__static__"))) {
1155 staticFiles
.add(Utility
.calculatePath(f
, stageDir
));
1158 AppYamlTranslator translator
=
1159 new AppYamlTranslator(getAppEngineWebXml(), getWebXml(), getBackendsXml(),
1160 getApiVersion(), staticFiles
, null, runtime
, getSdkVersion());
1161 String yaml
= translator
.getYaml();
1162 logger
.fine("Generated app.yaml file:\n" + yaml
);
1167 * Returns the app.yaml string.
1169 * @throws IllegalStateException if createStagingDirectory has not been called.
1172 public String
getAppYaml() {
1173 if (appYaml
== null) {
1174 throw new IllegalStateException("Must call createStagingDirectory first.");