1 // Copyright 2009 Google Inc. All rights reserved.
3 package com
.google
.appengine
.tools
.admin
;
5 import com
.google
.appengine
.tools
.admin
.GenericApplication
.ErrorHandler
;
6 import com
.google
.appengine
.tools
.util
.FileIterator
;
7 import com
.google
.common
.annotations
.VisibleForTesting
;
8 import com
.google
.common
.base
.Joiner
;
9 import com
.google
.common
.base
.Optional
;
10 import com
.google
.common
.base
.Preconditions
;
11 import com
.google
.common
.collect
.ArrayListMultimap
;
12 import com
.google
.common
.collect
.ImmutableMap
;
13 import com
.google
.common
.collect
.Multimap
;
14 import com
.google
.common
.hash
.Hashing
;
15 import com
.google
.common
.io
.BaseEncoding
;
16 import com
.google
.common
.io
.ByteSource
;
17 import com
.google
.common
.io
.Files
;
19 import net
.sourceforge
.yamlbeans
.YamlException
;
20 import net
.sourceforge
.yamlbeans
.YamlReader
;
23 import java
.io
.IOException
;
24 import java
.io
.StringReader
;
25 import java
.net
.HttpURLConnection
;
26 import java
.text
.MessageFormat
;
27 import java
.util
.ArrayList
;
28 import java
.util
.Arrays
;
29 import java
.util
.Collection
;
30 import java
.util
.Collections
;
31 import java
.util
.HashMap
;
32 import java
.util
.List
;
34 import java
.util
.Objects
;
35 import java
.util
.TreeMap
;
36 import java
.util
.TreeSet
;
37 import java
.util
.concurrent
.Callable
;
38 import java
.util
.logging
.Logger
;
39 import java
.util
.regex
.Pattern
;
42 * Uploads a new appversion to the hosting service.
45 public class AppVersionUpload
{
47 * Don't try to precompile more than this number of files in one request.
49 private static final int MAX_FILES_PER_PRECOMPILE
= 50;
51 private static final String YAML_EMPTY_STRING
= "null";
53 private static final String PRECOMPILATION_FAILED_WARNING_MESSAGE
=
54 "Precompilation failed. Consider retrying the update later, or add"
55 + " <precompilation-enabled>false</precompilation-enabled> to your appengine-web.xml"
56 + " to disable precompilation.";
58 private static final Logger logger
= Logger
.getLogger(AppVersionUpload
.class.getName());
61 * Status codes that can be returned by the /api/isconfigupdated endpoint. These indicate whether
62 * the app's Google Cloud Endpoints API configuration has been updated (if there is one).
65 enum EndpointsServingStatus
{
70 private final String parseName
;
72 EndpointsServingStatus(String parseName
) {
73 this.parseName
= parseName
;
76 static EndpointsServingStatus
parse(String value
) {
77 for (EndpointsServingStatus status
: EndpointsServingStatus
.values()) {
78 if (value
.equalsIgnoreCase(status
.parseName
)) {
82 throw new IllegalArgumentException("Value is not a recognized EndpointsServingStatus:"
88 * Combined serving status and error message. The result returned by the server typically includes
89 * both of these pieces of information. It's useful to have them packaged together.
92 static class EndpointsStatusAndMessage
{
93 public final EndpointsServingStatus status
;
94 public final String errorMessage
;
96 EndpointsStatusAndMessage(String status
, String errorMessage
) {
97 this.status
= EndpointsServingStatus
.parse(status
);
98 this.errorMessage
= errorMessage
;
101 EndpointsStatusAndMessage(EndpointsServingStatus status
) {
102 this.status
= status
;
103 this.errorMessage
= null;
107 public boolean equals(Object otherObj
) {
108 if (!(otherObj
instanceof EndpointsStatusAndMessage
)) {
112 EndpointsStatusAndMessage other
= (EndpointsStatusAndMessage
) otherObj
;
113 if (this.status
!= other
.status
) {
116 if ((errorMessage
== null && other
.errorMessage
!= null)
117 || (errorMessage
!= null && other
.errorMessage
== null)) {
120 if (errorMessage
!= null && !errorMessage
.equals(other
.errorMessage
)) {
127 public int hashCode() {
128 return Objects
.hash(status
, errorMessage
);
132 protected ServerConnection connection
;
133 protected GenericApplication app
;
134 protected final String backend
;
135 private boolean inTransaction
= false;
136 private Map
<String
, FileInfo
> files
= new HashMap
<String
, FileInfo
>();
137 private boolean deployed
= false;
138 private boolean started
= false;
139 private boolean checkConfigUpdated
= false;
140 private final UploadBatcher fileBatcher
;
141 private final UploadBatcher blobBatcher
;
142 private ClientDeploySender clientDeploySender
;
143 private SleepIfShouldRetry sleepIfShouldRetry
;
145 public AppVersionUpload(ServerConnection connection
, GenericApplication app
) {
146 this(connection
, app
, null, true);
150 * Create a new {@link AppVersionUpload} instance that can deploy a new
151 * versions of {@code app} via {@code connection}.
153 * @param connection to connect to the server
154 * @param app that contains the code to be deployed
155 * @param backend if supplied and non-{@code null}, a particular backend is
158 public AppVersionUpload(ServerConnection connection
, GenericApplication app
,
159 String backend
, boolean batchMode
) {
160 this.connection
= connection
;
162 this.backend
= backend
;
163 this.clientDeploySender
= new NoLoggingClientDeploySender(connection
);
164 fileBatcher
= new UploadBatcher("file", batchMode
);
165 blobBatcher
= new UploadBatcher("blob", batchMode
);
166 sleepIfShouldRetry
= new DefaultSleepAndRetry();
170 * Get an AppVersionUpload that already has the started flag set to true.
173 static AppVersionUpload
getStartedAppForTesting(ServerConnection connection
,
174 GenericApplication app
) {
175 AppVersionUpload upload
= new AppVersionUpload(connection
, app
);
176 upload
.started
= true;
181 * Uploads a new appversion to the server.
183 * @throws LocalIOException if a problem occurs with local files.
184 * @throws RemoteIOException if a problem occurs communicating with the server.
186 public void doUpload(ResourceLimits resourceLimits
, boolean updateGlobalConfigurations
,
187 boolean failOnPrecompilationError
, boolean ignoreEndpointsFailures
,
188 ClientDeploySender clientDeploySender
)
189 throws LocalIOException
, RemoteIOException
{
191 ClientDeploySender originalClientDeploySender
= this.clientDeploySender
;
192 this.clientDeploySender
= Preconditions
.checkNotNull(clientDeploySender
);
194 uploadFilesTransaction(resourceLimits
, failOnPrecompilationError
, ignoreEndpointsFailures
);
196 this.clientDeploySender
= originalClientDeploySender
;
198 if (updateGlobalConfigurations
) {
204 reportIfSkippingDispatchConfiguration();
205 if (getPagespeedYaml() != null) {
206 reportPageSpeedServiceDeprecation();
209 reportSkippingGlobalConfiguration();
213 private void uploadFilesTransaction(ResourceLimits resourceLimits
,
214 boolean failOnPrecompilationError
, boolean ignoreEndpointsFailures
) throws LocalIOException
,
218 File basepath
= getBasepath();
219 scanFiles(basepath
, resourceLimits
);
220 Collection
<FileInfo
> missingFiles
= beginTransaction(resourceLimits
);
221 uploadFiles(failOnPrecompilationError
, basepath
, missingFiles
);
222 commit(ignoreEndpointsFailures
);
223 clientDeploySender
.logClientDeploy(true, addVersionToArgs());
227 } catch (HttpIoException e
) {
228 if (e
.isSlaError()) {
229 clientDeploySender
.logClientDeploy(false, addVersionToArgs());
232 } catch (RuntimeException e
) {
233 clientDeploySender
.logClientDeploy(false, addVersionToArgs());
238 private void uploadFiles(boolean failOnPrecompilationError
, File basepath
,
239 Collection
<FileInfo
> missingFiles
)
240 throws LocalIOException
, RemoteIOException
{
242 app
.statusUpdate("Uploading " + missingFiles
.size() + " files.", 50);
243 if (!missingFiles
.isEmpty()) {
245 int quarter
= Math
.max(1, missingFiles
.size() / 4);
246 for (FileInfo missingFile
: missingFiles
) {
247 logger
.fine("Uploading file '" + missingFile
+ "'");
248 uploadFile(missingFile
);
249 if (++numFiles
% quarter
== 0) {
250 app
.statusUpdate("Uploaded " + numFiles
+ " files.");
254 uploadErrorHandlers(app
.getErrorHandlers(), basepath
);
255 if (app
.isPrecompilationEnabled()) {
256 precompile(failOnPrecompilationError
);
262 private void scanFiles(File basepath
, ResourceLimits resourceLimits
)
263 throws LocalIOException
{
265 app
.statusUpdate("Scanning files on local disk.", 20);
267 long resourceTotal
= 0;
268 List
<Pattern
> skipFiles
= loadSkipFiles(app
.getAppYaml());
269 for (File f
: new FileIterator(basepath
)) {
270 if (shouldSkip(f
.getName(), skipFiles
)) {
273 FileInfo fileInfo
= new FileInfo(f
, basepath
);
274 fileInfo
.setMimeType(app
);
276 logger
.fine("Processing file '" + f
+ "'.");
277 long maxFileBlobSize
= fileInfo
.mimeType
!= null ?
278 resourceLimits
.maxBlobSize() : resourceLimits
.maxFileSize();
279 if (f
.length() > maxFileBlobSize
) {
281 if (f
.getName().toLowerCase().endsWith(".jar")) {
282 message
= "Jar " + f
.getPath() + " is too large. Consider "
283 + "using --enable_jar_splitting.";
285 message
= "File " + f
.getPath() + " is too large (limit "
286 + maxFileBlobSize
+ " bytes).";
288 throw new LocalIOException(message
);
290 resourceTotal
+= addFile(fileInfo
);
292 if (++numFiles
% 250 == 0) {
293 app
.statusUpdate("Scanned " + numFiles
+ " files.");
296 if (numFiles
> resourceLimits
.maxFileCount()) {
297 throw new LocalIOException("Applications are limited to "
298 + resourceLimits
.maxFileCount() + " files, you have " + numFiles
301 if (resourceTotal
> resourceLimits
.maxTotalFileSize()) {
302 throw new LocalIOException("Applications are limited to "
303 + resourceLimits
.maxTotalFileSize() + " bytes of resource files, "
304 + "you have " + resourceTotal
+ ".");
308 private void reportSkippingGlobalConfiguration() {
309 TreeSet
<String
> skipSet
= new TreeSet
<String
>();
310 if (app
.getIndexesXml() != null) {
311 skipSet
.add("indexes.xml");
313 if (app
.getCronXml() != null) {
314 skipSet
.add("cron.xml");
316 if (app
.getQueueXml() != null) {
317 skipSet
.add("queue.xml");
319 if (app
.getDispatchXml() != null) {
320 skipSet
.add("dispatch.xml");
322 if (app
.getDosXml() != null) {
323 skipSet
.add("dos.xml");
325 if (app
.getPagespeedYaml() != null) {
326 skipSet
.add("pagespeed");
328 if (!skipSet
.isEmpty()) {
329 app
.statusUpdate("Skipping global configurations: " + Joiner
.on(", ").join(skipSet
));
333 private void reportIfSkippingDispatchConfiguration() {
334 if (app
.getDispatchXml() != null) {
336 "Skipping dispatch.xml - consider running \"appcfg.sh update_dispatch <war-dir>\"");
340 private void reportPageSpeedServiceDeprecation() {
341 logger
.warning("This application contains PageSpeed related configurations,"
342 + " which is deprecated! Those configurations will stop working after "
343 + "December 1, 2015. Read "
344 + "https://cloud.google.com/appengine/docs/adminconsole/pagespeed#disabling-pagespeed"
345 + " to learn how to disable PageSpeed.");
348 private void uploadErrorHandlers(List
<ErrorHandler
> errorHandlers
, File basepath
)
349 throws LocalIOException
, RemoteIOException
{
350 if (!errorHandlers
.isEmpty()) {
351 app
.statusUpdate("Uploading " + errorHandlers
.size() + " file(s) "
352 + "for static error handlers.");
353 for (ErrorHandler handler
: errorHandlers
) {
354 File file
= new File(basepath
, handler
.getFile());
355 FileInfo info
= new FileInfo(file
, basepath
);
356 String error
= FileInfo
.checkValidFilename(info
.path
);
358 throw new LocalIOException("Could not find static error handler: " + error
);
360 info
.mimeType
= handler
.getMimeType();
361 String errorType
= handler
.getErrorCode();
362 if (errorType
== null) {
363 errorType
= "default";
365 send("/api/appversion/adderrorblob", info
.file
, info
.mimeType
, "path",
372 interface SleepIfShouldRetry
{
374 * If precompilation should be retried given the number of errors so far then sleep and return
375 * true; otherwise return false.
376 * @param errorCount the number of precompilation errors seen so far.
377 * @return true if precompilation should be tried.
379 boolean sleepIfShouldRetry(int errorCount
);
382 private static class DefaultSleepAndRetry
implements SleepIfShouldRetry
{
383 @Override public boolean sleepIfShouldRetry(int errorCount
) {
384 if (errorCount
> 3) {
389 } catch (InterruptedException e
) {
397 void setSleepIfShouldRetry(SleepIfShouldRetry sleepAndRetry
) {
398 this.sleepIfShouldRetry
= sleepAndRetry
;
401 public void precompile(boolean failOnPrecompilationError
) throws RemoteIOException
{
402 app
.statusUpdate("Initializing precompilation...");
403 List
<String
> filesToCompile
= new ArrayList
<String
>();
405 boolean containsGoFiles
= false;
406 for (String f
: this.files
.keySet()) {
407 boolean isGoFile
= f
.toLowerCase().endsWith(".go");
408 if (isGoFile
&& !containsGoFiles
) {
409 containsGoFiles
= true;
411 if (isGoFile
|| f
.toLowerCase().endsWith(".py")) {
412 filesToCompile
.add(f
);
415 Collections
.sort(filesToCompile
);
416 if (containsGoFiles
) {
417 failOnPrecompilationError
= true;
423 filesToCompile
.addAll(sendPrecompileRequest(Collections
.<String
>emptyList()));
425 } catch (RemoteIOException ex
) {
427 if (!sleepIfShouldRetry
.sleepIfShouldRetry(errorCount
)) {
428 if (failOnPrecompilationError
) {
429 throw precompilationFailedException("", ex
);
431 logger
.warning(PRECOMPILATION_FAILED_WARNING_MESSAGE
);
439 while (!filesToCompile
.isEmpty()) {
441 if (precompileChunk(filesToCompile
)) {
444 } catch (RemoteIOException ex
) {
445 Collections
.shuffle(filesToCompile
);
447 if (!sleepIfShouldRetry
.sleepIfShouldRetry(errorCount
)) {
448 if (failOnPrecompilationError
) {
449 String messageFragment
= " with " + filesToCompile
.size() + " file(s) remaining";
450 throw precompilationFailedException(messageFragment
, ex
);
452 logger
.warning(PRECOMPILATION_FAILED_WARNING_MESSAGE
);
460 private static RemoteIOException
precompilationFailedException(
461 String messageFragment
, RemoteIOException cause
) {
462 String message
= "Precompilation failed" + messageFragment
+ ". Consider adding"
463 + " <precompilation-enabled>false</precompilation-enabled> to your appengine-web.xml"
464 + " and trying again.";
465 if (cause
instanceof HttpIoException
) {
466 HttpIoException httpCause
= (HttpIoException
) cause
;
467 return new HttpIoException(message
, httpCause
.getResponseCode(), httpCause
);
469 return RemoteIOException
.from(cause
, message
);
474 * Attempt to precompile up to {@code MAX_FILES_PER_PRECOMPILE} files from
475 * {@code filesToCompile}.
477 * @param filesToCompile a list of file names, which will be mutated to remove
478 * any files that were successfully compiled.
480 * @return true if filesToCompile was reduced in size (i.e. progress was
483 private boolean precompileChunk(List
<String
> filesToCompile
)
484 throws RemoteIOException
{
485 int filesLeft
= filesToCompile
.size();
486 if (filesLeft
== 0) {
487 app
.statusUpdate("Initializing precompilation...");
489 app
.statusUpdate(MessageFormat
.format(
490 "Precompiling... {0} file(s) left.", filesLeft
));
493 List
<String
> subset
=
495 .subList(0, Math
.min(filesLeft
, MAX_FILES_PER_PRECOMPILE
));
496 List
<String
> remainingFiles
= sendPrecompileRequest(subset
);
498 filesToCompile
.addAll(remainingFiles
);
499 return filesToCompile
.size() < filesLeft
;
502 private List
<String
> sendPrecompileRequest(List
<String
> filesToCompile
)
503 throws RemoteIOException
{
505 send("/api/appversion/precompile", Joiner
.on("\n").useForNull("null").join(filesToCompile
));
506 if (response
.length() > 0) {
507 return Arrays
.asList(response
.split("\n"));
509 return Collections
.emptyList();
513 public void updateIndexes() throws RemoteIOException
{
514 if (app
.getIndexesXml() != null) {
515 app
.statusUpdate("Uploading index definitions.");
516 send("/api/datastore/index/add", getIndexYaml());
521 public void updateCron() throws RemoteIOException
{
522 String yaml
= getCronYaml();
524 app
.statusUpdate("Uploading cron jobs.");
525 send("/api/datastore/cron/update", yaml
);
529 public void updateQueue() throws RemoteIOException
{
530 String yaml
= getQueueYaml();
532 app
.statusUpdate("Uploading task queues.");
533 send("/api/queue/update", yaml
);
537 public void updateDispatch() throws RemoteIOException
{
538 String yaml
= getDispatchYaml();
540 app
.statusUpdate("Uploading dispatch entries.");
541 send("/api/dispatch/update", yaml
);
545 public void updateDos() throws RemoteIOException
{
546 String yaml
= getDosYaml();
548 app
.statusUpdate("Uploading DoS entries.");
549 send("/api/dos/update", yaml
);
553 public void updatePagespeed() throws RemoteIOException
{
554 String yaml
= getPagespeedYaml();
556 app
.statusUpdate("Uploading PageSpeed entries.");
557 send("/api/appversion/updatepagespeed", yaml
);
560 send("/api/appversion/updatepagespeed", "");
561 } catch (HttpIoException exc
) {
562 if (exc
.getResponseCode() != HttpURLConnection
.HTTP_NOT_FOUND
) {
569 public void setDefaultVersion() throws IOException
{
570 String module
= app
.getModule();
571 String url
= "/api/appversion/setdefault";
572 if (module
!= null) {
573 String
[] modules
= module
.split(",");
574 if (modules
.length
> 1) {
575 app
.statusUpdate("Setting the default version of modules " + Joiner
.on(", ").join(modules
) +
576 " of application " + app
.getAppId() + " to " + app
.getVersion());
577 Multimap
<String
, String
> args
= ArrayListMultimap
.create();
578 args
.put("app_id", app
.getAppId());
579 args
.put("version", app
.getVersion());
580 for (String mod
: modules
) {
581 args
.put("module", mod
);
583 connection
.post(url
, "", args
);
586 app
.statusUpdate("Setting the default version of module " + module
+ " of application " +
587 app
.getAppId() + " to " + app
.getVersion());
590 app
.statusUpdate("Setting the default version of application " + app
.getAppId() +
591 " to " + app
.getVersion());
596 protected String
getIndexYaml() {
597 return app
.getIndexesXml().toYaml();
600 protected String
getCronYaml() {
601 if (app
.getCronXml() != null) {
602 return app
.getCronXml().toYaml();
608 protected String
getQueueYaml() {
609 if (app
.getQueueXml() != null) {
610 return app
.getQueueXml().toYaml();
616 protected String
getDispatchYaml() {
617 return app
.getDispatchXml() == null ?
null : app
.getDispatchXml().toYaml();
620 protected String
getDosYaml() {
621 if (app
.getDosXml() != null) {
622 return app
.getDosXml().toYaml();
628 protected String
getPagespeedYaml() {
629 return app
.getPagespeedYaml();
633 protected boolean getInTransaction() {
634 return this.inTransaction
;
638 protected void setInTransaction(boolean newValue
) {
639 this.inTransaction
= newValue
;
642 private File
getBasepath() {
643 File path
= app
.getStagingDir();
645 path
= new File(app
.getPath());
651 * Get the URL that the user would go to for their app's logs. This string is intended to be
652 * provided to the user, to show them where to go to find an error.
654 * @return A URL that the user can use to find their app's logs.
658 StringBuilder url
= new StringBuilder();
659 url
.append("https://appengine.google.com/logs?app_id=");
660 url
.append(app
.getAppId());
661 if (app
.getVersion() != null) {
662 url
.append("&version_id=");
663 if (app
.getModule() != null) {
664 url
.append(app
.getModule());
667 url
.append(app
.getVersion());
669 return url
.toString();
673 * Adds a file for uploading, returning the bytes counted against the total
677 * @return 0 for a static file, or file.length() for a resource file.
680 long addFile(FileInfo info
) {
682 throw new IllegalStateException("Already in a transaction.");
685 String error
= FileInfo
.checkValidFilename(info
.path
);
687 logger
.severe(error
);
691 files
.put(info
.path
, info
);
693 return info
.mimeType
!= null ?
0 : info
.file
.length();
697 * Parses the response from /api/appversion/create into a Map.
699 * @param response String returned from the /api/appversion/create call.
700 * @return YAML parsed into Map.
702 private ArrayList
<String
> validateBeginYaml(String response
) {
703 YamlReader yaml
= new YamlReader(new StringReader(response
));
705 Object obj
= yaml
.read();
707 @SuppressWarnings("unchecked")
708 Map
<String
, Object
> responseMap
= (Map
<String
, Object
>) obj
;
709 if (responseMap
!= null) {
710 obj
= responseMap
.get("warnings");
712 @SuppressWarnings("unchecked")
713 ArrayList
<String
> warnings
= (ArrayList
<String
>) obj
;
718 } catch (YamlException exc
) {
719 } catch (ClassCastException exc
) {
721 return new ArrayList
<String
>();
725 * Begins the transaction, returning a list of files that need uploading.
727 * All calls to addFile must be made before calling beginTransaction().
729 * @param resourceLimits is the collection of resource limits for AppCfg.
730 * @return A list of pathnames that should be uploaded using uploadFile()
731 * before calling commit().
734 Collection
<FileInfo
> beginTransaction(ResourceLimits resourceLimits
) throws RemoteIOException
{
736 throw new IllegalStateException("Already in a transaction.");
739 if (backend
== null) {
740 app
.statusUpdate("Initiating update.");
742 app
.statusUpdate("Initiating update of backend " + backend
+ ".");
744 String response
= send("/api/appversion/create", app
.getAppYaml());
745 ArrayList
<String
> warnings
= validateBeginYaml(response
);
746 for (String warning
: warnings
) {
747 app
.statusUpdate("WARNING: " + warning
);
749 inTransaction
= true;
750 Collection
<FileInfo
> blobsToClone
= new ArrayList
<FileInfo
>(files
.size());
751 Collection
<FileInfo
> filesToClone
= new ArrayList
<FileInfo
>(files
.size());
753 for (FileInfo f
: files
.values()) {
754 if (f
.mimeType
== null) {
761 TreeMap
<String
, FileInfo
> filesToUpload
= new TreeMap
<String
, FileInfo
>();
762 cloneFiles("/api/appversion/cloneblobs", blobsToClone
, "static",
763 filesToUpload
, resourceLimits
.maxFilesToClone());
764 cloneFiles("/api/appversion/clonefiles", filesToClone
, "application",
765 filesToUpload
, resourceLimits
.maxFilesToClone());
767 logger
.fine("Files to upload :");
768 for (FileInfo f
: filesToUpload
.values()) {
769 logger
.fine("\t" + f
);
772 this.files
= filesToUpload
;
773 return new ArrayList
<FileInfo
>(filesToUpload
.values());
776 private static final String LIST_DELIMITER
= "\n";
779 * Sends files to the given url.
781 * @param url server URL to use.
782 * @param filesParam List of files to clone.
783 * @param type Type of files ( "static" or "application")
784 * @param filesToUpload Files that need to be uploaded are added to this
786 * @param maxFilesToClone Max number of files to clone at a single time.
788 private void cloneFiles(String url
, Collection
<FileInfo
> filesParam
,
789 String type
, Map
<String
, FileInfo
> filesToUpload
, long maxFilesToClone
)
790 throws RemoteIOException
{
791 if (filesParam
.isEmpty()) {
794 app
.statusUpdate("Cloning " + filesParam
.size() + " " + type
+ " files.");
797 int remaining
= filesParam
.size();
798 ArrayList
<FileInfo
> chunk
= new ArrayList
<FileInfo
>((int) maxFilesToClone
);
799 for (FileInfo file
: filesParam
) {
801 if (--remaining
== 0 || chunk
.size() >= maxFilesToClone
) {
803 app
.statusUpdate("Cloned " + cloned
+ " files.");
805 String result
= send(url
, buildClonePayload(chunk
));
806 if (result
!= null && result
.length() > 0) {
807 for (String path
: result
.split(LIST_DELIMITER
)) {
808 if (path
== null || path
.length() == 0) {
811 FileInfo info
= this.files
.get(path
);
813 logger
.warning("Skipping " + path
+ ": missing FileInfo");
816 filesToUpload
.put(path
, info
);
819 cloned
+= chunk
.size();
826 * Uploads a file to the hosting service.
828 * Must only be called after beginTransaction(). The file provided must be on
829 * of those that were returned by beginTransaction();
831 * @param file FileInfo for the file to upload.
833 private void uploadFile(FileInfo file
) throws RemoteIOException
{
834 if (!inTransaction
) {
835 throw new IllegalStateException(
836 "beginTransaction() must be called before uploadFile().");
838 if (!files
.containsKey(file
.path
)) {
839 throw new IllegalArgumentException("File " + file
.path
840 + " is not in the list of files to be uploaded.");
843 files
.remove(file
.path
);
844 if (file
.mimeType
== null) {
845 fileBatcher
.addToBatch(file
);
847 blobBatcher
.addToBatch(file
);
852 * Commits the transaction, making the new app version available.
854 * All the files returned by beginTransaction must have been uploaded with
855 * uploadFile() before commit() may be called.
857 * @param ignoreEndpointsFailures True to ignore errors updating an Endpoints configuration, if
861 void commit(boolean ignoreEndpointsFailures
) throws RemoteIOException
{
864 boolean ready
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
866 public Boolean
call() throws Exception
{
874 logger
.severe("Version still not ready to serve, aborting.");
875 throw new RemoteIOException("Version not ready.");
878 boolean versionIsServing
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
880 public Boolean
call() throws Exception
{
884 if (!versionIsServing
) {
885 logger
.severe("Version still not serving, aborting.");
886 throw new RemoteIOException("Version not ready.");
888 if (checkConfigUpdated
) {
889 Optional
<EndpointsStatusAndMessage
> result
= retryWithBackoffOptional(1, 2, 60, 20,
890 new IsConfigUpdatedCallable());
891 checkEndpointsServingStatusResult(result
, ignoreEndpointsFailures
);
893 app
.statusUpdate("Closing update: new version is ready to start serving.");
894 inTransaction
= false;
895 } catch (RemoteIOException
| RuntimeException e
) {
897 } catch (Exception e
) {
898 throw new RuntimeException(e
);
903 * A Callable to check the isconfigserving endpoint to see if the Endpoints Configuration
904 * has been updated. This is intended for use with retryWithBackoffOptional.
906 class IsConfigUpdatedCallable
implements Callable
<Optional
<EndpointsStatusAndMessage
>> {
908 public Optional
<EndpointsStatusAndMessage
> call() throws Exception
{
909 EndpointsStatusAndMessage result
= isConfigUpdated();
910 return result
.status
== EndpointsServingStatus
.PENDING
911 ? Optional
.<EndpointsStatusAndMessage
>absent()
912 : Optional
.of(result
);
917 * Check the result of calling IsConfigUpdatedCallable. Failed values result in a
918 * RuntimeException being thrown.
920 * @param callResult The optional serving status to be checked. An empty value is treated the
921 * same as a PENDING value.
922 * @param ignoreEndpointsFailures True if failures to update the configuration should allow
923 * deployment to proceed, false if they should stop deployment. Either way, an error
924 * message is displayed if there's a problem updating the configuration.
927 void checkEndpointsServingStatusResult(
928 Optional
<EndpointsStatusAndMessage
> callResult
, boolean ignoreEndpointsFailures
) {
929 EndpointsStatusAndMessage configServingStatus
=
930 callResult
.or(new EndpointsStatusAndMessage(EndpointsServingStatus
.PENDING
));
931 if (configServingStatus
.status
!= EndpointsServingStatus
.SERVING
) {
932 String userMessage
= (configServingStatus
.errorMessage
== null)
933 ? String
.format("Check the app's AppEngine logs for errors: %s", getLogUrl())
934 : configServingStatus
.errorMessage
;
935 String errorMessage
= "Endpoints configuration not updated. " + userMessage
;
937 app
.statusUpdate(errorMessage
);
938 logger
.severe(errorMessage
);
940 app
.statusUpdate("See the deployment troubleshooting documentation for more information: "
941 + "https://developers.google.com/appengine/docs/java/endpoints/test_deploy"
942 + "#troubleshooting_a_deployment_failure");
944 if (ignoreEndpointsFailures
) {
945 app
.statusUpdate("Ignoring Endpoints failure and proceeding with update.");
947 throw new RuntimeException(errorMessage
);
953 * Deploys the new app version but does not make it default.
955 * All the files returned by beginTransaction must have been uploaded with
956 * uploadFile() before commit() may be called.
958 private void deploy() throws RemoteIOException
{
959 if (!inTransaction
) {
960 throw new IllegalStateException(
961 "beginTransaction() must be called before deploy().");
963 if (!files
.isEmpty()) {
964 throw new IllegalStateException(
965 "Some required files have not been uploaded.");
967 app
.statusUpdate("Deploying new version.", 20);
968 send("/api/appversion/deploy", "");
973 * Check if the new app version is ready to serve traffic.
975 * @return true if the server returned that the app is ready to serve.
977 private boolean isReady() throws IOException
{
979 throw new IllegalStateException(
980 "deploy() must be called before isReady()");
982 String result
= send("/api/appversion/isready", "");
983 return "1".equals(result
.trim());
986 private void startServing() throws IOException
{
988 throw new IllegalStateException(
989 "deploy() must be called before startServing()");
991 send("/api/appversion/startserving", "", "willcheckserving", "1");
996 protected Map
<String
, String
> parseIsServingResponse(String isServingResp
) {
997 ImmutableMap
.Builder
<String
, String
> result
= ImmutableMap
.builder();
998 if (isServingResp
.isEmpty()) {
999 return result
.build();
1003 YamlReader yamlReader
= new YamlReader(isServingResp
);
1004 @SuppressWarnings("unchecked")
1005 Map
<Object
, Object
> resultMap
= yamlReader
.read(Map
.class, String
.class);
1006 for (Object key
: resultMap
.keySet()) {
1007 result
.put((String
) key
, (String
) resultMap
.get(key
));
1009 } catch (YamlException e
) {
1010 logger
.severe("Unable to parse Yaml from response: " + result
);
1011 throw new RuntimeException(e
);
1013 return result
.build();
1016 private boolean isServing() throws IOException
{
1018 throw new IllegalStateException(
1019 "startServing() must be called before isServing().");
1021 String result
= send("/api/appversion/isserving", "", "new_serving_resp", "1");
1022 if ("1".equals(result
.trim()) || "0".equals(result
.trim())) {
1023 return "1".equals(result
.trim());
1026 Map
<String
, String
> resultMap
= parseIsServingResponse(result
.trim());
1027 if (resultMap
.containsKey("message") &&
1028 !YAML_EMPTY_STRING
.equals(resultMap
.get("message"))) {
1029 app
.statusUpdate(resultMap
.get("message"));
1031 if (resultMap
.containsKey("fatal") &&
1032 Boolean
.parseBoolean(resultMap
.get("fatal").toLowerCase())) {
1033 throw new RuntimeException(
1034 "Fatal problem encountered during deployment. Please refer to the logs" +
1035 " for more information.");
1037 if (resultMap
.containsKey("check_endpoints_config")) {
1038 checkConfigUpdated
= Boolean
.parseBoolean(resultMap
.get("check_endpoints_config"));
1040 if (resultMap
.containsKey("serving")) {
1041 return Boolean
.parseBoolean(resultMap
.get("serving"));
1043 throw new RuntimeException(
1044 "Fatal problem encountered during deployment. Unexpected response when " +
1045 "checking for serving status. Response: " + result
);
1050 Map
<String
, String
> parseIsConfigUpdatedResponse(String isConfigUpdatedResp
) {
1051 ImmutableMap
.Builder
<String
, String
> result
= ImmutableMap
.builder();
1053 YamlReader yamlReader
= new YamlReader(isConfigUpdatedResp
);
1054 @SuppressWarnings("unchecked")
1055 Map
<Object
, Object
> resultMap
= yamlReader
.read(Map
.class, String
.class);
1056 if (resultMap
== null) {
1057 return result
.build();
1060 for (Object key
: resultMap
.keySet()) {
1061 result
.put((String
) key
, (String
) resultMap
.get(key
));
1063 } catch (YamlException e
) {
1064 logger
.severe("Unable to parse Yaml from response: " + result
);
1065 throw new RuntimeException(e
);
1067 return result
.build();
1070 private EndpointsStatusAndMessage
isConfigUpdated() throws IOException
, IllegalArgumentException
{
1072 throw new IllegalStateException(
1073 "startServing() must be called before isConfigUpdated().");
1075 String result
= send("/api/isconfigupdated", "");
1077 Map
<String
, String
> resultMap
= parseIsConfigUpdatedResponse(result
.trim());
1078 if (resultMap
.containsKey("updatedDetail2")) {
1079 return new EndpointsStatusAndMessage(resultMap
.get("updatedDetail2"),
1080 resultMap
.get("errorMessage"));
1081 } else if (resultMap
.containsKey("updated")) {
1082 return Boolean
.parseBoolean(resultMap
.get("updated"))
1083 ?
new EndpointsStatusAndMessage(EndpointsServingStatus
.SERVING
)
1084 : new EndpointsStatusAndMessage(EndpointsServingStatus
.PENDING
);
1086 throw new RuntimeException(
1087 "Fatal problem encountered during deployment. Unexpected response when " +
1088 "checking for configuration update status. Response: " + result
);
1092 public void forceRollback() throws RemoteIOException
{
1093 app
.statusUpdate("Rolling back the update" + (this.backend
== null ?
"."
1094 : " on backend " + this.backend
+ "."));
1095 send("/api/appversion/rollback", "");
1098 private void rollback() throws RemoteIOException
{
1099 if (!inTransaction
) {
1106 String
send(String url
, String payload
, String
... args
)
1107 throws RemoteIOException
{
1109 return clientDeploySender
.send(url
, payload
, addVersionToArgs(args
));
1110 } catch (IOException e
) {
1111 throw RemoteIOException
.from(e
);
1116 String
send(String url
, File payload
, String mimeType
, String
... args
)
1117 throws RemoteIOException
{
1119 return clientDeploySender
.send(url
, payload
, mimeType
, addVersionToArgs(args
));
1120 } catch (IOException e
) {
1121 throw RemoteIOException
.from(e
);
1125 private String
[] addVersionToArgs(String
... args
) {
1126 List
<String
> result
= new ArrayList
<String
>();
1127 Collections
.addAll(result
, args
);
1128 result
.add("app_id");
1129 result
.add(app
.getAppId());
1130 if (backend
!= null) {
1131 result
.add("backend");
1132 result
.add(backend
);
1133 } else if (app
.getVersion() != null) {
1134 result
.add("version");
1135 result
.add(app
.getVersion());
1137 if (app
.getModule() != null) {
1138 result
.add("module");
1139 result
.add(app
.getModule());
1141 return result
.toArray(new String
[result
.size()]);
1145 * Calls a function multiple times until it returns true, backing off more and more each time.
1147 * @param initialDelay Inital delay after the first try, in seconds.
1148 * @param backoffFactor Delay will be multiplied by this factor after each
1150 * @param maxDelay Maximum delay factor.
1151 * @param maxTries Maximum number of tries.
1152 * @param callable Callable to call.
1153 * @return true if the Callable returned true in one of its tries.
1155 private boolean retryWithBackoff(double initialDelay
, double backoffFactor
,
1156 double maxDelay
, int maxTries
, final Callable
<Boolean
> callable
)
1158 Optional
<Boolean
> result
= retryWithBackoffOptional(
1159 initialDelay
, backoffFactor
, maxDelay
, maxTries
,
1160 new Callable
<Optional
<Boolean
>>() {
1162 public Optional
<Boolean
> call() throws Exception
{
1163 return callable
.call() ? Optional
.of(true) : Optional
.<Boolean
>absent();
1166 return result
.or(false);
1170 * Calls a function (with an optional return value) multiple times until it returns a value,
1171 * backing off more and more each time.
1173 * @param initialDelay Inital delay after the first try, in seconds.
1174 * @param backoffFactor Delay will be multiplied by this factor after each
1176 * @param maxDelay Maximum delay factor.
1177 * @param maxTries Maximum number of tries.
1178 * @param callable Callable to call.
1179 * @return the result of the last call to the Callable. If the optional Callable return value
1180 * never returns anything, the result will be an empty Optional.
1183 public <T
> Optional
<T
> retryWithBackoffOptional(double initialDelay
, double backoffFactor
,
1184 double maxDelay
, int maxTries
, Callable
<Optional
<T
>> callable
)
1186 long delayMillis
= (long) (initialDelay
* 1000);
1187 long maxDelayMillis
= (long) (maxDelay
* 1000);
1188 Optional
<T
> callResult
= callable
.call();
1189 if (callResult
.isPresent()) {
1192 while (maxTries
> 1) {
1193 app
.statusUpdate("Will check again in " + (delayMillis
/ 1000) + " seconds.");
1194 Thread
.sleep(delayMillis
);
1195 delayMillis
*= backoffFactor
;
1196 if (delayMillis
> maxDelayMillis
) {
1197 delayMillis
= maxDelayMillis
;
1200 callResult
= callable
.call();
1201 if (callResult
.isPresent()) {
1205 return Optional
.<T
>absent();
1208 private static final String TUPLE_DELIMITER
= "|";
1211 * Build the post body for a clone request.
1213 * @param files List of FileInfos for the files to clone.
1214 * @return A string containing the properly delimited tuples.
1216 private static String
buildClonePayload(Collection
<FileInfo
> files
) {
1217 StringBuffer data
= new StringBuffer();
1218 boolean first
= true;
1219 for (FileInfo file
: files
) {
1223 data
.append(LIST_DELIMITER
);
1225 data
.append(file
.path
);
1226 data
.append(TUPLE_DELIMITER
);
1227 data
.append(file
.hash
);
1228 if (file
.mimeType
!= null) {
1229 data
.append(TUPLE_DELIMITER
);
1230 data
.append(file
.mimeType
);
1234 return data
.toString();
1238 static String
getRuntime(String appYaml
) {
1239 String result
= "?";
1241 Map
<?
, ?
> yaml
= (Map
<?
, ?
>) new YamlReader(appYaml
).read();
1242 Object runtime
= yaml
.get("runtime");
1243 if (runtime
instanceof String
) {
1244 result
= (String
) runtime
;
1246 } catch (YamlException ex
) {
1247 logger
.severe(ex
.toString());
1253 static List
<Pattern
> loadSkipFiles(String appYaml
) {
1254 List
<Pattern
> skipFiles
= new ArrayList
<>();
1255 if (appYaml
== null) {
1259 Map
<?
, ?
> yaml
= (Map
<?
, ?
>) new YamlReader(appYaml
).read();
1260 List
<?
> skipFileList
= (List
<?
>) yaml
.get("skip_files");
1261 if (skipFileList
!= null) {
1262 for (Object skipFile
: skipFileList
) {
1263 skipFiles
.add(Pattern
.compile(skipFile
.toString()));
1266 } catch (YamlException ex
) {
1267 logger
.severe(ex
.toString());
1273 static boolean shouldSkip(String name
, List
<Pattern
> skipFiles
) {
1274 for (Pattern skipPattern
: skipFiles
) {
1275 if (skipPattern
.matcher(name
).matches()) {
1282 static class FileInfo
implements Comparable
<FileInfo
> {
1286 public String mimeType
;
1288 private FileInfo(String path
) {
1293 public FileInfo(File f
, File base
) throws LocalIOException
{
1295 this.path
= Utility
.calculatePath(f
, base
);
1296 this.hash
= calculateHash();
1300 static FileInfo
newForTesting(String path
) {
1301 return new FileInfo(path
);
1304 public void setMimeType(GenericApplication app
) {
1305 mimeType
= app
.getMimeTypeIfStatic(path
);
1309 public String
toString() {
1310 return (mimeType
== null ?
"" : mimeType
) + '\t' + hash
+ "\t" + path
;
1314 public int compareTo(FileInfo other
) {
1315 return path
.compareTo(other
.path
);
1319 public int hashCode() {
1320 return path
.hashCode();
1324 public boolean equals(Object obj
) {
1325 if (obj
instanceof FileInfo
) {
1326 return path
.equals(((FileInfo
) obj
).path
);
1331 private static final Pattern FILE_PATH_POSITIVE_RE
=
1332 Pattern
.compile("^[ 0-9a-zA-Z._+/@$-]{1,256}$");
1334 private static final Pattern FILE_PATH_NEGATIVE_RE_1
=
1335 Pattern
.compile("[.][.]|^[.]/|[.]$|/[.]/|^-|^_ah/|^/");
1337 private static final Pattern FILE_PATH_NEGATIVE_RE_2
=
1338 Pattern
.compile("//|/$");
1340 private static final Pattern FILE_PATH_NEGATIVE_RE_3
=
1341 Pattern
.compile("^ | $|/ | /");
1344 static String
checkValidFilename(String path
) {
1345 if (!FILE_PATH_POSITIVE_RE
.matcher(path
).matches()) {
1346 return "Invalid character in filename: " + path
;
1348 if (FILE_PATH_NEGATIVE_RE_1
.matcher(path
).find()) {
1349 return "Filname cannot contain '.' or '..' or start with '-', '_ah/' or '/' : " + path
;
1351 if (FILE_PATH_NEGATIVE_RE_2
.matcher(path
).find()) {
1352 return "Filname cannot have trailing / or contain //: " + path
;
1354 if (FILE_PATH_NEGATIVE_RE_3
.matcher(path
).find()) {
1355 return "Any spaces must be in the middle of a filename: '" + path
+ "'";
1360 private static final BaseEncoding SEPARATED_HEX
=
1361 BaseEncoding
.base16().lowerCase().withSeparator("_", 8);
1364 static String
calculateHash(ByteSource source
) throws IOException
{
1365 byte[] hash
= source
.hash(Hashing
.sha1()).asBytes();
1366 return SEPARATED_HEX
.encode(hash
);
1369 public String
calculateHash() throws LocalIOException
{
1371 return calculateHash(Files
.asByteSource(file
));
1372 } catch (IOException e
) {
1373 throw LocalIOException
.from(e
);
1378 class UploadBatcher
{
1380 static final int MAX_BATCH_SIZE
= 3200000;
1381 static final int MAX_BATCH_COUNT
= 100;
1382 static final int MAX_BATCH_FILE_SIZE
= 200000;
1383 static final int BATCH_OVERHEAD
= 500;
1388 boolean batching
= true;
1389 List
<FileInfo
> batch
= new ArrayList
<FileInfo
>();
1393 * @param what Either "file" or "blob" or "errorblob" indicating what kind of objects this
1394 * batcher uploads. Used in messages and URLs.
1395 * @param batching whether or not we want to really do batch.
1397 public UploadBatcher(String what
, boolean batching
) {
1399 this.singleUrl
= "/api/appversion/add" + what
;
1400 this.batchUrl
= singleUrl
+ "s";
1401 this.batching
= batching
;
1405 * Send the current batch on its way and reset the batch buffer when done
1407 public void sendBatch() throws IOException
{
1410 "Sending batch containing " + batch
.size() + " "+ what
+"(s) totaling " +
1411 batchSize
/ 1000 + "KB.");
1412 clientDeploySender
.sendBatch(batchUrl
, batch
, batchSize
, addVersionToArgs("", ""));
1413 batch
= new ArrayList
<>();
1418 * Flush the current batch.
1420 * This first attempts to send the batch as a single request; if that fails because the server
1421 * doesn"t support batching, the files are sent one by one, and self.batching is reset to
1424 * At the end, self.batch and self.batchSize are reset
1426 public void flush() throws RemoteIOException
{
1427 if (batch
.isEmpty()) {
1432 } catch (Exception e
) {
1433 app
.statusUpdate("Exception in flushing batch payload, so sending 1 by 1..."
1436 for (FileInfo fileInfo
: batch
) {
1437 send(singleUrl
, fileInfo
.file
, fileInfo
.mimeType
, "path", fileInfo
.path
);
1439 batch
= new ArrayList
<FileInfo
>();
1445 * Batch a file, possibly flushing first, or perhaps upload it directly.
1447 * Args: path: The name of the file. payload: The contents of the file. mime_type: The MIME
1448 * Content-type of the file, or None.
1450 * If mime_type is None, application/octet-stream is substituted. """
1452 public void addToBatch(FileInfo fileInfo
) throws RemoteIOException
{
1454 long size
= fileInfo
.file
.length();
1456 if (size
<= MAX_BATCH_FILE_SIZE
) {
1457 if ((batch
.size() >= MAX_BATCH_COUNT
) ||
1458 (batchSize
+ size
> MAX_BATCH_SIZE
)) {
1462 batch
.add(fileInfo
);
1463 batchSize
+= size
+ BATCH_OVERHEAD
;
1467 send(singleUrl
, fileInfo
.file
, fileInfo
.mimeType
, "path", fileInfo
.path
);