1 // Copyright 2009 Google Inc. All rights reserved.
3 package com
.google
.appengine
.tools
.admin
;
5 import com
.google
.appengine
.tools
.admin
.GenericApplication
.ErrorHandler
;
6 import com
.google
.appengine
.tools
.util
.FileIterator
;
7 import com
.google
.common
.annotations
.VisibleForTesting
;
8 import com
.google
.common
.base
.Joiner
;
9 import com
.google
.common
.base
.Optional
;
10 import com
.google
.common
.base
.Preconditions
;
11 import com
.google
.common
.collect
.ArrayListMultimap
;
12 import com
.google
.common
.collect
.ImmutableMap
;
13 import com
.google
.common
.collect
.Multimap
;
14 import com
.google
.common
.hash
.Hashing
;
15 import com
.google
.common
.io
.BaseEncoding
;
16 import com
.google
.common
.io
.ByteSource
;
17 import com
.google
.common
.io
.Files
;
19 import net
.sourceforge
.yamlbeans
.YamlException
;
20 import net
.sourceforge
.yamlbeans
.YamlReader
;
23 import java
.io
.IOException
;
24 import java
.io
.StringReader
;
25 import java
.net
.HttpURLConnection
;
26 import java
.text
.MessageFormat
;
27 import java
.util
.ArrayList
;
28 import java
.util
.Arrays
;
29 import java
.util
.Collection
;
30 import java
.util
.Collections
;
31 import java
.util
.HashMap
;
32 import java
.util
.List
;
34 import java
.util
.Objects
;
35 import java
.util
.TreeMap
;
36 import java
.util
.TreeSet
;
37 import java
.util
.concurrent
.Callable
;
38 import java
.util
.logging
.Logger
;
39 import java
.util
.regex
.Pattern
;
42 * Uploads a new appversion to the hosting service.
45 public class AppVersionUpload
{
47 * Don't try to precompile more than this number of files in one request.
49 private static final int MAX_FILES_PER_PRECOMPILE
= 50;
51 private static final String YAML_EMPTY_STRING
= "null";
53 private static final String PRECOMPILATION_FAILED_WARNING_MESSAGE
=
54 "Precompilation failed. Consider retrying the update later, or add"
55 + " <precompilation-enabled>false</precompilation-enabled> to your appengine-web.xml"
56 + " to disable precompilation.";
58 private static final Logger logger
= Logger
.getLogger(AppVersionUpload
.class.getName());
61 * Status codes that can be returned by the /api/isconfigupdated endpoint. These indicate whether
62 * the app's Google Cloud Endpoints API configuration has been updated (if there is one).
65 enum EndpointsServingStatus
{
70 private final String parseName
;
72 EndpointsServingStatus(String parseName
) {
73 this.parseName
= parseName
;
76 static EndpointsServingStatus
parse(String value
) {
77 for (EndpointsServingStatus status
: EndpointsServingStatus
.values()) {
78 if (value
.equalsIgnoreCase(status
.parseName
)) {
82 throw new IllegalArgumentException("Value is not a recognized EndpointsServingStatus:"
88 * Combined serving status and error message. The result returned by the server typically includes
89 * both of these pieces of information. It's useful to have them packaged together.
92 static class EndpointsStatusAndMessage
{
93 public final EndpointsServingStatus status
;
94 public final String errorMessage
;
96 EndpointsStatusAndMessage(String status
, String errorMessage
) {
97 this.status
= EndpointsServingStatus
.parse(status
);
98 this.errorMessage
= errorMessage
;
101 EndpointsStatusAndMessage(EndpointsServingStatus status
) {
102 this.status
= status
;
103 this.errorMessage
= null;
107 public boolean equals(Object otherObj
) {
108 if (!(otherObj
instanceof EndpointsStatusAndMessage
)) {
112 EndpointsStatusAndMessage other
= (EndpointsStatusAndMessage
) otherObj
;
113 if (this.status
!= other
.status
) {
116 if ((errorMessage
== null && other
.errorMessage
!= null)
117 || (errorMessage
!= null && other
.errorMessage
== null)) {
120 if (errorMessage
!= null && !errorMessage
.equals(other
.errorMessage
)) {
127 public int hashCode() {
128 return Objects
.hash(status
, errorMessage
);
132 protected ServerConnection connection
;
133 protected GenericApplication app
;
134 protected final String backend
;
135 private boolean inTransaction
= false;
136 private Map
<String
, FileInfo
> files
= new HashMap
<String
, FileInfo
>();
137 private boolean deployed
= false;
138 private boolean started
= false;
139 private boolean checkConfigUpdated
= false;
140 private final UploadBatcher fileBatcher
;
141 private final UploadBatcher blobBatcher
;
142 private ClientDeploySender clientDeploySender
;
143 private SleepIfShouldRetry sleepIfShouldRetry
;
145 public AppVersionUpload(ServerConnection connection
, GenericApplication app
) {
146 this(connection
, app
, null, true);
150 * Create a new {@link AppVersionUpload} instance that can deploy a new
151 * versions of {@code app} via {@code connection}.
153 * @param connection to connect to the server
154 * @param app that contains the code to be deployed
155 * @param backend if supplied and non-{@code null}, a particular backend is
158 public AppVersionUpload(ServerConnection connection
, GenericApplication app
,
159 String backend
, boolean batchMode
) {
160 this.connection
= connection
;
162 this.backend
= backend
;
163 this.clientDeploySender
= new NoLoggingClientDeploySender(connection
);
164 fileBatcher
= new UploadBatcher("file", batchMode
);
165 blobBatcher
= new UploadBatcher("blob", batchMode
);
166 sleepIfShouldRetry
= new DefaultSleepAndRetry();
170 * Get an AppVersionUpload that already has the started flag set to true.
173 static AppVersionUpload
getStartedAppForTesting(ServerConnection connection
,
174 GenericApplication app
) {
175 AppVersionUpload upload
= new AppVersionUpload(connection
, app
);
176 upload
.started
= true;
181 * Uploads a new appversion to the server.
183 * @throws LocalIOException if a problem occurs with local files.
184 * @throws RemoteIOException if a problem occurs communicating with the server.
186 public void doUpload(ResourceLimits resourceLimits
, boolean updateGlobalConfigurations
,
187 boolean failOnPrecompilationError
, boolean ignoreEndpointsFailures
,
188 ClientDeploySender clientDeploySender
)
189 throws LocalIOException
, RemoteIOException
{
191 ClientDeploySender originalClientDeploySender
= this.clientDeploySender
;
192 this.clientDeploySender
= Preconditions
.checkNotNull(clientDeploySender
);
194 uploadFilesTransaction(resourceLimits
, failOnPrecompilationError
, ignoreEndpointsFailures
);
196 this.clientDeploySender
= originalClientDeploySender
;
198 if (updateGlobalConfigurations
) {
204 reportIfSkippingDispatchConfiguration();
206 reportSkippingGlobalConfiguration();
210 private void uploadFilesTransaction(ResourceLimits resourceLimits
,
211 boolean failOnPrecompilationError
, boolean ignoreEndpointsFailures
) throws LocalIOException
,
215 File basepath
= getBasepath();
216 scanFiles(basepath
, resourceLimits
);
217 Collection
<FileInfo
> missingFiles
= beginTransaction(resourceLimits
);
218 uploadFiles(failOnPrecompilationError
, basepath
, missingFiles
);
219 commit(ignoreEndpointsFailures
);
220 clientDeploySender
.logClientDeploy(true, addVersionToArgs());
224 } catch (HttpIoException e
) {
225 if (e
.isSlaError()) {
226 clientDeploySender
.logClientDeploy(false, addVersionToArgs());
229 } catch (RuntimeException e
) {
230 clientDeploySender
.logClientDeploy(false, addVersionToArgs());
235 private void uploadFiles(boolean failOnPrecompilationError
, File basepath
,
236 Collection
<FileInfo
> missingFiles
)
237 throws LocalIOException
, RemoteIOException
{
239 app
.statusUpdate("Uploading " + missingFiles
.size() + " files.", 50);
240 if (!missingFiles
.isEmpty()) {
242 int quarter
= Math
.max(1, missingFiles
.size() / 4);
243 for (FileInfo missingFile
: missingFiles
) {
244 logger
.fine("Uploading file '" + missingFile
+ "'");
245 uploadFile(missingFile
);
246 if (++numFiles
% quarter
== 0) {
247 app
.statusUpdate("Uploaded " + numFiles
+ " files.");
251 uploadErrorHandlers(app
.getErrorHandlers(), basepath
);
252 if (app
.isPrecompilationEnabled()) {
253 precompile(failOnPrecompilationError
);
259 private void scanFiles(File basepath
, ResourceLimits resourceLimits
)
260 throws LocalIOException
{
262 app
.statusUpdate("Scanning files on local disk.", 20);
264 long resourceTotal
= 0;
265 List
<Pattern
> skipFiles
= loadSkipFiles(app
.getAppYaml());
266 for (File f
: new FileIterator(basepath
)) {
267 if (shouldSkip(f
.getName(), skipFiles
)) {
270 FileInfo fileInfo
= new FileInfo(f
, basepath
);
271 fileInfo
.setMimeType(app
);
273 logger
.fine("Processing file '" + f
+ "'.");
274 long maxFileBlobSize
= fileInfo
.mimeType
!= null ?
275 resourceLimits
.maxBlobSize() : resourceLimits
.maxFileSize();
276 if (f
.length() > maxFileBlobSize
) {
278 if (f
.getName().toLowerCase().endsWith(".jar")) {
279 message
= "Jar " + f
.getPath() + " is too large. Consider "
280 + "using --enable_jar_splitting.";
282 message
= "File " + f
.getPath() + " is too large (limit "
283 + maxFileBlobSize
+ " bytes).";
285 throw new LocalIOException(message
);
287 resourceTotal
+= addFile(fileInfo
);
289 if (++numFiles
% 250 == 0) {
290 app
.statusUpdate("Scanned " + numFiles
+ " files.");
293 if (numFiles
> resourceLimits
.maxFileCount()) {
294 throw new LocalIOException("Applications are limited to "
295 + resourceLimits
.maxFileCount() + " files, you have " + numFiles
298 if (resourceTotal
> resourceLimits
.maxTotalFileSize()) {
299 throw new LocalIOException("Applications are limited to "
300 + resourceLimits
.maxTotalFileSize() + " bytes of resource files, "
301 + "you have " + resourceTotal
+ ".");
305 private void reportSkippingGlobalConfiguration() {
306 TreeSet
<String
> skipSet
= new TreeSet
<String
>();
307 if (app
.getIndexesXml() != null) {
308 skipSet
.add("indexes.xml");
310 if (app
.getCronXml() != null) {
311 skipSet
.add("cron.xml");
313 if (app
.getQueueXml() != null) {
314 skipSet
.add("queue.xml");
316 if (app
.getDispatchXml() != null) {
317 skipSet
.add("dispatch.xml");
319 if (app
.getDosXml() != null) {
320 skipSet
.add("dos.xml");
322 if (app
.getPagespeedYaml() != null) {
323 skipSet
.add("pagespeed");
325 if (!skipSet
.isEmpty()) {
326 app
.statusUpdate("Skipping global configurations: " + Joiner
.on(", ").join(skipSet
));
330 private void reportIfSkippingDispatchConfiguration() {
331 if (app
.getDispatchXml() != null) {
333 "Skipping dispatch.xml - consider running \"appcfg.sh update_dispatch <war-dir>\"");
337 private void uploadErrorHandlers(List
<ErrorHandler
> errorHandlers
, File basepath
)
338 throws LocalIOException
, RemoteIOException
{
339 if (!errorHandlers
.isEmpty()) {
340 app
.statusUpdate("Uploading " + errorHandlers
.size() + " file(s) "
341 + "for static error handlers.");
342 for (ErrorHandler handler
: errorHandlers
) {
343 File file
= new File(basepath
, handler
.getFile());
344 FileInfo info
= new FileInfo(file
, basepath
);
345 String error
= FileInfo
.checkValidFilename(info
.path
);
347 throw new LocalIOException("Could not find static error handler: " + error
);
349 info
.mimeType
= handler
.getMimeType();
350 String errorType
= handler
.getErrorCode();
351 if (errorType
== null) {
352 errorType
= "default";
354 send("/api/appversion/adderrorblob", info
.file
, info
.mimeType
, "path",
361 interface SleepIfShouldRetry
{
363 * If precompilation should be retried given the number of errors so far then sleep and return
364 * true; otherwise return false.
365 * @param errorCount the number of precompilation errors seen so far.
366 * @return true if precompilation should be tried.
368 boolean sleepIfShouldRetry(int errorCount
);
371 private static class DefaultSleepAndRetry
implements SleepIfShouldRetry
{
372 @Override public boolean sleepIfShouldRetry(int errorCount
) {
373 if (errorCount
> 3) {
378 } catch (InterruptedException e
) {
386 void setSleepIfShouldRetry(SleepIfShouldRetry sleepAndRetry
) {
387 this.sleepIfShouldRetry
= sleepAndRetry
;
390 public void precompile(boolean failOnPrecompilationError
) throws RemoteIOException
{
391 app
.statusUpdate("Initializing precompilation...");
392 List
<String
> filesToCompile
= new ArrayList
<String
>();
394 boolean containsGoFiles
= false;
395 for (String f
: this.files
.keySet()) {
396 boolean isGoFile
= f
.toLowerCase().endsWith(".go");
397 if (isGoFile
&& !containsGoFiles
) {
398 containsGoFiles
= true;
400 if (isGoFile
|| f
.toLowerCase().endsWith(".py")) {
401 filesToCompile
.add(f
);
404 Collections
.sort(filesToCompile
);
405 if (containsGoFiles
) {
406 failOnPrecompilationError
= true;
412 filesToCompile
.addAll(sendPrecompileRequest(Collections
.<String
>emptyList()));
414 } catch (RemoteIOException ex
) {
416 if (!sleepIfShouldRetry
.sleepIfShouldRetry(errorCount
)) {
417 if (failOnPrecompilationError
) {
418 throw precompilationFailedException("", ex
);
420 logger
.warning(PRECOMPILATION_FAILED_WARNING_MESSAGE
);
428 while (!filesToCompile
.isEmpty()) {
430 if (precompileChunk(filesToCompile
)) {
433 } catch (RemoteIOException ex
) {
434 Collections
.shuffle(filesToCompile
);
436 if (!sleepIfShouldRetry
.sleepIfShouldRetry(errorCount
)) {
437 if (failOnPrecompilationError
) {
438 String messageFragment
= " with " + filesToCompile
.size() + " file(s) remaining";
439 throw precompilationFailedException(messageFragment
, ex
);
441 logger
.warning(PRECOMPILATION_FAILED_WARNING_MESSAGE
);
449 private static RemoteIOException
precompilationFailedException(
450 String messageFragment
, RemoteIOException cause
) {
451 String message
= "Precompilation failed" + messageFragment
+ ". Consider adding"
452 + " <precompilation-enabled>false</precompilation-enabled> to your appengine-web.xml"
453 + " and trying again.";
454 if (cause
instanceof HttpIoException
) {
455 HttpIoException httpCause
= (HttpIoException
) cause
;
456 return new HttpIoException(message
, httpCause
.getResponseCode(), httpCause
);
458 return RemoteIOException
.from(cause
, message
);
463 * Attempt to precompile up to {@code MAX_FILES_PER_PRECOMPILE} files from
464 * {@code filesToCompile}.
466 * @param filesToCompile a list of file names, which will be mutated to remove
467 * any files that were successfully compiled.
469 * @return true if filesToCompile was reduced in size (i.e. progress was
472 private boolean precompileChunk(List
<String
> filesToCompile
)
473 throws RemoteIOException
{
474 int filesLeft
= filesToCompile
.size();
475 if (filesLeft
== 0) {
476 app
.statusUpdate("Initializing precompilation...");
478 app
.statusUpdate(MessageFormat
.format(
479 "Precompiling... {0} file(s) left.", filesLeft
));
482 List
<String
> subset
=
484 .subList(0, Math
.min(filesLeft
, MAX_FILES_PER_PRECOMPILE
));
485 List
<String
> remainingFiles
= sendPrecompileRequest(subset
);
487 filesToCompile
.addAll(remainingFiles
);
488 return filesToCompile
.size() < filesLeft
;
491 private List
<String
> sendPrecompileRequest(List
<String
> filesToCompile
)
492 throws RemoteIOException
{
494 send("/api/appversion/precompile", Joiner
.on("\n").useForNull("null").join(filesToCompile
));
495 if (response
.length() > 0) {
496 return Arrays
.asList(response
.split("\n"));
498 return Collections
.emptyList();
502 public void updateIndexes() throws RemoteIOException
{
503 if (app
.getIndexesXml() != null) {
504 app
.statusUpdate("Uploading index definitions.");
505 send("/api/datastore/index/add", getIndexYaml());
510 public void updateCron() throws RemoteIOException
{
511 String yaml
= getCronYaml();
513 app
.statusUpdate("Uploading cron jobs.");
514 send("/api/datastore/cron/update", yaml
);
518 public void updateQueue() throws RemoteIOException
{
519 String yaml
= getQueueYaml();
521 app
.statusUpdate("Uploading task queues.");
522 send("/api/queue/update", yaml
);
526 public void updateDispatch() throws RemoteIOException
{
527 String yaml
= getDispatchYaml();
529 app
.statusUpdate("Uploading dispatch entries.");
530 send("/api/dispatch/update", yaml
);
534 public void updateDos() throws RemoteIOException
{
535 String yaml
= getDosYaml();
537 app
.statusUpdate("Uploading DoS entries.");
538 send("/api/dos/update", yaml
);
542 public void updatePagespeed() throws RemoteIOException
{
543 String yaml
= getPagespeedYaml();
545 app
.statusUpdate("Uploading PageSpeed entries.");
546 send("/api/appversion/updatepagespeed", yaml
);
549 send("/api/appversion/updatepagespeed", "");
550 } catch (HttpIoException exc
) {
551 if (exc
.getResponseCode() != HttpURLConnection
.HTTP_NOT_FOUND
) {
558 public void setDefaultVersion() throws IOException
{
559 String module
= app
.getModule();
560 String url
= "/api/appversion/setdefault";
561 if (module
!= null) {
562 String
[] modules
= module
.split(",");
563 if (modules
.length
> 1) {
564 app
.statusUpdate("Setting the default version of modules " + Joiner
.on(", ").join(modules
) +
565 " of application " + app
.getAppId() + " to " + app
.getVersion());
566 Multimap
<String
, String
> args
= ArrayListMultimap
.create();
567 args
.put("app_id", app
.getAppId());
568 args
.put("version", app
.getVersion());
569 for (String mod
: modules
) {
570 args
.put("module", mod
);
572 connection
.post(url
, "", args
);
575 app
.statusUpdate("Setting the default version of module " + module
+ " of application " +
576 app
.getAppId() + " to " + app
.getVersion());
579 app
.statusUpdate("Setting the default version of application " + app
.getAppId() +
580 " to " + app
.getVersion());
585 protected String
getIndexYaml() {
586 return app
.getIndexesXml().toYaml();
589 protected String
getCronYaml() {
590 if (app
.getCronXml() != null) {
591 return app
.getCronXml().toYaml();
597 protected String
getQueueYaml() {
598 if (app
.getQueueXml() != null) {
599 return app
.getQueueXml().toYaml();
605 protected String
getDispatchYaml() {
606 return app
.getDispatchXml() == null ?
null : app
.getDispatchXml().toYaml();
609 protected String
getDosYaml() {
610 if (app
.getDosXml() != null) {
611 return app
.getDosXml().toYaml();
617 protected String
getPagespeedYaml() {
618 return app
.getPagespeedYaml();
622 protected boolean getInTransaction() {
623 return this.inTransaction
;
627 protected void setInTransaction(boolean newValue
) {
628 this.inTransaction
= newValue
;
631 private File
getBasepath() {
632 File path
= app
.getStagingDir();
634 path
= new File(app
.getPath());
640 * Get the URL that the user would go to for their app's logs. This string is intended to be
641 * provided to the user, to show them where to go to find an error.
643 * @return A URL that the user can use to find their app's logs.
647 StringBuilder url
= new StringBuilder();
648 url
.append("https://appengine.google.com/logs?app_id=");
649 url
.append(app
.getAppId());
650 if (app
.getVersion() != null) {
651 url
.append("&version_id=");
652 if (app
.getModule() != null) {
653 url
.append(app
.getModule());
656 url
.append(app
.getVersion());
658 return url
.toString();
662 * Adds a file for uploading, returning the bytes counted against the total
666 * @return 0 for a static file, or file.length() for a resource file.
669 long addFile(FileInfo info
) {
671 throw new IllegalStateException("Already in a transaction.");
674 String error
= FileInfo
.checkValidFilename(info
.path
);
676 logger
.severe(error
);
680 files
.put(info
.path
, info
);
682 return info
.mimeType
!= null ?
0 : info
.file
.length();
686 * Parses the response from /api/appversion/create into a Map.
688 * @param response String returned from the /api/appversion/create call.
689 * @return YAML parsed into Map.
691 private ArrayList
<String
> validateBeginYaml(String response
) {
692 YamlReader yaml
= new YamlReader(new StringReader(response
));
694 Object obj
= yaml
.read();
696 @SuppressWarnings("unchecked")
697 Map
<String
, Object
> responseMap
= (Map
<String
, Object
>) obj
;
698 if (responseMap
!= null) {
699 obj
= responseMap
.get("warnings");
701 @SuppressWarnings("unchecked")
702 ArrayList
<String
> warnings
= (ArrayList
<String
>) obj
;
707 } catch (YamlException exc
) {
708 } catch (ClassCastException exc
) {
710 return new ArrayList
<String
>();
714 * Begins the transaction, returning a list of files that need uploading.
716 * All calls to addFile must be made before calling beginTransaction().
718 * @param resourceLimits is the collection of resource limits for AppCfg.
719 * @return A list of pathnames that should be uploaded using uploadFile()
720 * before calling commit().
723 Collection
<FileInfo
> beginTransaction(ResourceLimits resourceLimits
) throws RemoteIOException
{
725 throw new IllegalStateException("Already in a transaction.");
728 if (backend
== null) {
729 app
.statusUpdate("Initiating update.");
731 app
.statusUpdate("Initiating update of backend " + backend
+ ".");
733 String response
= send("/api/appversion/create", app
.getAppYaml());
734 ArrayList
<String
> warnings
= validateBeginYaml(response
);
735 for (String warning
: warnings
) {
736 app
.statusUpdate("WARNING: " + warning
);
738 inTransaction
= true;
739 Collection
<FileInfo
> blobsToClone
= new ArrayList
<FileInfo
>(files
.size());
740 Collection
<FileInfo
> filesToClone
= new ArrayList
<FileInfo
>(files
.size());
742 for (FileInfo f
: files
.values()) {
743 if (f
.mimeType
== null) {
750 TreeMap
<String
, FileInfo
> filesToUpload
= new TreeMap
<String
, FileInfo
>();
751 cloneFiles("/api/appversion/cloneblobs", blobsToClone
, "static",
752 filesToUpload
, resourceLimits
.maxFilesToClone());
753 cloneFiles("/api/appversion/clonefiles", filesToClone
, "application",
754 filesToUpload
, resourceLimits
.maxFilesToClone());
756 logger
.fine("Files to upload :");
757 for (FileInfo f
: filesToUpload
.values()) {
758 logger
.fine("\t" + f
);
761 this.files
= filesToUpload
;
762 return new ArrayList
<FileInfo
>(filesToUpload
.values());
765 private static final String LIST_DELIMITER
= "\n";
768 * Sends files to the given url.
770 * @param url server URL to use.
771 * @param filesParam List of files to clone.
772 * @param type Type of files ( "static" or "application")
773 * @param filesToUpload Files that need to be uploaded are added to this
775 * @param maxFilesToClone Max number of files to clone at a single time.
777 private void cloneFiles(String url
, Collection
<FileInfo
> filesParam
,
778 String type
, Map
<String
, FileInfo
> filesToUpload
, long maxFilesToClone
)
779 throws RemoteIOException
{
780 if (filesParam
.isEmpty()) {
783 app
.statusUpdate("Cloning " + filesParam
.size() + " " + type
+ " files.");
786 int remaining
= filesParam
.size();
787 ArrayList
<FileInfo
> chunk
= new ArrayList
<FileInfo
>((int) maxFilesToClone
);
788 for (FileInfo file
: filesParam
) {
790 if (--remaining
== 0 || chunk
.size() >= maxFilesToClone
) {
792 app
.statusUpdate("Cloned " + cloned
+ " files.");
794 String result
= send(url
, buildClonePayload(chunk
));
795 if (result
!= null && result
.length() > 0) {
796 for (String path
: result
.split(LIST_DELIMITER
)) {
797 if (path
== null || path
.length() == 0) {
800 FileInfo info
= this.files
.get(path
);
802 logger
.warning("Skipping " + path
+ ": missing FileInfo");
805 filesToUpload
.put(path
, info
);
808 cloned
+= chunk
.size();
815 * Uploads a file to the hosting service.
817 * Must only be called after beginTransaction(). The file provided must be on
818 * of those that were returned by beginTransaction();
820 * @param file FileInfo for the file to upload.
822 private void uploadFile(FileInfo file
) throws RemoteIOException
{
823 if (!inTransaction
) {
824 throw new IllegalStateException(
825 "beginTransaction() must be called before uploadFile().");
827 if (!files
.containsKey(file
.path
)) {
828 throw new IllegalArgumentException("File " + file
.path
829 + " is not in the list of files to be uploaded.");
832 files
.remove(file
.path
);
833 if (file
.mimeType
== null) {
834 fileBatcher
.addToBatch(file
);
836 blobBatcher
.addToBatch(file
);
841 * Commits the transaction, making the new app version available.
843 * All the files returned by beginTransaction must have been uploaded with
844 * uploadFile() before commit() may be called.
846 * @param ignoreEndpointsFailures True to ignore errors updating an Endpoints configuration, if
850 void commit(boolean ignoreEndpointsFailures
) throws RemoteIOException
{
853 boolean ready
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
855 public Boolean
call() throws Exception
{
863 logger
.severe("Version still not ready to serve, aborting.");
864 throw new RemoteIOException("Version not ready.");
867 boolean versionIsServing
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
869 public Boolean
call() throws Exception
{
873 if (!versionIsServing
) {
874 logger
.severe("Version still not serving, aborting.");
875 throw new RemoteIOException("Version not ready.");
877 if (checkConfigUpdated
) {
878 Optional
<EndpointsStatusAndMessage
> result
= retryWithBackoffOptional(1, 2, 60, 20,
879 new IsConfigUpdatedCallable());
880 checkEndpointsServingStatusResult(result
, ignoreEndpointsFailures
);
882 app
.statusUpdate("Closing update: new version is ready to start serving.");
883 inTransaction
= false;
884 } catch (RemoteIOException
| RuntimeException e
) {
886 } catch (Exception e
) {
887 throw new RuntimeException(e
);
892 * A Callable to check the isconfigserving endpoint to see if the Endpoints Configuration
893 * has been updated. This is intended for use with retryWithBackoffOptional.
895 class IsConfigUpdatedCallable
implements Callable
<Optional
<EndpointsStatusAndMessage
>> {
897 public Optional
<EndpointsStatusAndMessage
> call() throws Exception
{
898 EndpointsStatusAndMessage result
= isConfigUpdated();
899 return result
.status
== EndpointsServingStatus
.PENDING
900 ? Optional
.<EndpointsStatusAndMessage
>absent()
901 : Optional
.of(result
);
906 * Check the result of calling IsConfigUpdatedCallable. Failed values result in a
907 * RuntimeException being thrown.
909 * @param callResult The optional serving status to be checked. An empty value is treated the
910 * same as a PENDING value.
911 * @param ignoreEndpointsFailures True if failures to update the configuration should allow
912 * deployment to proceed, false if they should stop deployment. Either way, an error
913 * message is displayed if there's a problem updating the configuration.
916 void checkEndpointsServingStatusResult(
917 Optional
<EndpointsStatusAndMessage
> callResult
, boolean ignoreEndpointsFailures
) {
918 EndpointsStatusAndMessage configServingStatus
=
919 callResult
.or(new EndpointsStatusAndMessage(EndpointsServingStatus
.PENDING
));
920 if (configServingStatus
.status
!= EndpointsServingStatus
.SERVING
) {
921 String userMessage
= (configServingStatus
.errorMessage
== null)
922 ? String
.format("Check the app's AppEngine logs for errors: %s", getLogUrl())
923 : configServingStatus
.errorMessage
;
924 String errorMessage
= "Endpoints configuration not updated. " + userMessage
;
926 app
.statusUpdate(errorMessage
);
927 logger
.severe(errorMessage
);
929 app
.statusUpdate("See the deployment troubleshooting documentation for more information: "
930 + "https://developers.google.com/appengine/docs/java/endpoints/test_deploy"
931 + "#troubleshooting_a_deployment_failure");
933 if (ignoreEndpointsFailures
) {
934 app
.statusUpdate("Ignoring Endpoints failure and proceeding with update.");
936 throw new RuntimeException(errorMessage
);
942 * Deploys the new app version but does not make it default.
944 * All the files returned by beginTransaction must have been uploaded with
945 * uploadFile() before commit() may be called.
947 private void deploy() throws RemoteIOException
{
948 if (!inTransaction
) {
949 throw new IllegalStateException(
950 "beginTransaction() must be called before deploy().");
952 if (!files
.isEmpty()) {
953 throw new IllegalStateException(
954 "Some required files have not been uploaded.");
956 app
.statusUpdate("Deploying new version.", 20);
957 send("/api/appversion/deploy", "");
962 * Check if the new app version is ready to serve traffic.
964 * @return true if the server returned that the app is ready to serve.
966 private boolean isReady() throws IOException
{
968 throw new IllegalStateException(
969 "deploy() must be called before isReady()");
971 String result
= send("/api/appversion/isready", "");
972 return "1".equals(result
.trim());
975 private void startServing() throws IOException
{
977 throw new IllegalStateException(
978 "deploy() must be called before startServing()");
980 send("/api/appversion/startserving", "", "willcheckserving", "1");
985 protected Map
<String
, String
> parseIsServingResponse(String isServingResp
) {
986 ImmutableMap
.Builder
<String
, String
> result
= ImmutableMap
.builder();
987 if (isServingResp
.isEmpty()) {
988 return result
.build();
992 YamlReader yamlReader
= new YamlReader(isServingResp
);
993 @SuppressWarnings("unchecked")
994 Map
<Object
, Object
> resultMap
= yamlReader
.read(Map
.class, String
.class);
995 for (Object key
: resultMap
.keySet()) {
996 result
.put((String
) key
, (String
) resultMap
.get(key
));
998 } catch (YamlException e
) {
999 logger
.severe("Unable to parse Yaml from response: " + result
);
1000 throw new RuntimeException(e
);
1002 return result
.build();
1005 private boolean isServing() throws IOException
{
1007 throw new IllegalStateException(
1008 "startServing() must be called before isServing().");
1010 String result
= send("/api/appversion/isserving", "", "new_serving_resp", "1");
1011 if ("1".equals(result
.trim()) || "0".equals(result
.trim())) {
1012 return "1".equals(result
.trim());
1015 Map
<String
, String
> resultMap
= parseIsServingResponse(result
.trim());
1016 if (resultMap
.containsKey("message") &&
1017 !YAML_EMPTY_STRING
.equals(resultMap
.get("message"))) {
1018 app
.statusUpdate(resultMap
.get("message"));
1020 if (resultMap
.containsKey("fatal") &&
1021 Boolean
.parseBoolean(resultMap
.get("fatal").toLowerCase())) {
1022 throw new RuntimeException(
1023 "Fatal problem encountered during deployment. Please refer to the logs" +
1024 " for more information.");
1026 if (resultMap
.containsKey("check_endpoints_config")) {
1027 checkConfigUpdated
= Boolean
.parseBoolean(resultMap
.get("check_endpoints_config"));
1029 if (resultMap
.containsKey("serving")) {
1030 return Boolean
.parseBoolean(resultMap
.get("serving"));
1032 throw new RuntimeException(
1033 "Fatal problem encountered during deployment. Unexpected response when " +
1034 "checking for serving status. Response: " + result
);
1039 Map
<String
, String
> parseIsConfigUpdatedResponse(String isConfigUpdatedResp
) {
1040 ImmutableMap
.Builder
<String
, String
> result
= ImmutableMap
.builder();
1042 YamlReader yamlReader
= new YamlReader(isConfigUpdatedResp
);
1043 @SuppressWarnings("unchecked")
1044 Map
<Object
, Object
> resultMap
= yamlReader
.read(Map
.class, String
.class);
1045 if (resultMap
== null) {
1046 return result
.build();
1049 for (Object key
: resultMap
.keySet()) {
1050 result
.put((String
) key
, (String
) resultMap
.get(key
));
1052 } catch (YamlException e
) {
1053 logger
.severe("Unable to parse Yaml from response: " + result
);
1054 throw new RuntimeException(e
);
1056 return result
.build();
1059 private EndpointsStatusAndMessage
isConfigUpdated() throws IOException
, IllegalArgumentException
{
1061 throw new IllegalStateException(
1062 "startServing() must be called before isConfigUpdated().");
1064 String result
= send("/api/isconfigupdated", "");
1066 Map
<String
, String
> resultMap
= parseIsConfigUpdatedResponse(result
.trim());
1067 if (resultMap
.containsKey("updatedDetail2")) {
1068 return new EndpointsStatusAndMessage(resultMap
.get("updatedDetail2"),
1069 resultMap
.get("errorMessage"));
1070 } else if (resultMap
.containsKey("updated")) {
1071 return Boolean
.parseBoolean(resultMap
.get("updated"))
1072 ?
new EndpointsStatusAndMessage(EndpointsServingStatus
.SERVING
)
1073 : new EndpointsStatusAndMessage(EndpointsServingStatus
.PENDING
);
1075 throw new RuntimeException(
1076 "Fatal problem encountered during deployment. Unexpected response when " +
1077 "checking for configuration update status. Response: " + result
);
1081 public void forceRollback() throws RemoteIOException
{
1082 app
.statusUpdate("Rolling back the update" + (this.backend
== null ?
"."
1083 : " on backend " + this.backend
+ "."));
1084 send("/api/appversion/rollback", "");
1087 private void rollback() throws RemoteIOException
{
1088 if (!inTransaction
) {
1095 String
send(String url
, String payload
, String
... args
)
1096 throws RemoteIOException
{
1098 return clientDeploySender
.send(url
, payload
, addVersionToArgs(args
));
1099 } catch (IOException e
) {
1100 throw RemoteIOException
.from(e
);
1105 String
send(String url
, File payload
, String mimeType
, String
... args
)
1106 throws RemoteIOException
{
1108 return clientDeploySender
.send(url
, payload
, mimeType
, addVersionToArgs(args
));
1109 } catch (IOException e
) {
1110 throw RemoteIOException
.from(e
);
1114 private String
[] addVersionToArgs(String
... args
) {
1115 List
<String
> result
= new ArrayList
<String
>();
1116 Collections
.addAll(result
, args
);
1117 result
.add("app_id");
1118 result
.add(app
.getAppId());
1119 if (backend
!= null) {
1120 result
.add("backend");
1121 result
.add(backend
);
1122 } else if (app
.getVersion() != null) {
1123 result
.add("version");
1124 result
.add(app
.getVersion());
1126 if (app
.getModule() != null) {
1127 result
.add("module");
1128 result
.add(app
.getModule());
1130 return result
.toArray(new String
[result
.size()]);
1134 * Calls a function multiple times until it returns true, backing off more and more each time.
1136 * @param initialDelay Inital delay after the first try, in seconds.
1137 * @param backoffFactor Delay will be multiplied by this factor after each
1139 * @param maxDelay Maximum delay factor.
1140 * @param maxTries Maximum number of tries.
1141 * @param callable Callable to call.
1142 * @return true if the Callable returned true in one of its tries.
1144 private boolean retryWithBackoff(double initialDelay
, double backoffFactor
,
1145 double maxDelay
, int maxTries
, final Callable
<Boolean
> callable
)
1147 Optional
<Boolean
> result
= retryWithBackoffOptional(
1148 initialDelay
, backoffFactor
, maxDelay
, maxTries
,
1149 new Callable
<Optional
<Boolean
>>() {
1151 public Optional
<Boolean
> call() throws Exception
{
1152 return callable
.call() ? Optional
.of(true) : Optional
.<Boolean
>absent();
1155 return result
.or(false);
1159 * Calls a function (with an optional return value) multiple times until it returns a value,
1160 * backing off more and more each time.
1162 * @param initialDelay Inital delay after the first try, in seconds.
1163 * @param backoffFactor Delay will be multiplied by this factor after each
1165 * @param maxDelay Maximum delay factor.
1166 * @param maxTries Maximum number of tries.
1167 * @param callable Callable to call.
1168 * @return the result of the last call to the Callable. If the optional Callable return value
1169 * never returns anything, the result will be an empty Optional.
1172 public <T
> Optional
<T
> retryWithBackoffOptional(double initialDelay
, double backoffFactor
,
1173 double maxDelay
, int maxTries
, Callable
<Optional
<T
>> callable
)
1175 long delayMillis
= (long) (initialDelay
* 1000);
1176 long maxDelayMillis
= (long) (maxDelay
* 1000);
1177 Optional
<T
> callResult
= callable
.call();
1178 if (callResult
.isPresent()) {
1181 while (maxTries
> 1) {
1182 app
.statusUpdate("Will check again in " + (delayMillis
/ 1000) + " seconds.");
1183 Thread
.sleep(delayMillis
);
1184 delayMillis
*= backoffFactor
;
1185 if (delayMillis
> maxDelayMillis
) {
1186 delayMillis
= maxDelayMillis
;
1189 callResult
= callable
.call();
1190 if (callResult
.isPresent()) {
1194 return Optional
.<T
>absent();
1197 private static final String TUPLE_DELIMITER
= "|";
1200 * Build the post body for a clone request.
1202 * @param files List of FileInfos for the files to clone.
1203 * @return A string containing the properly delimited tuples.
1205 private static String
buildClonePayload(Collection
<FileInfo
> files
) {
1206 StringBuffer data
= new StringBuffer();
1207 boolean first
= true;
1208 for (FileInfo file
: files
) {
1212 data
.append(LIST_DELIMITER
);
1214 data
.append(file
.path
);
1215 data
.append(TUPLE_DELIMITER
);
1216 data
.append(file
.hash
);
1217 if (file
.mimeType
!= null) {
1218 data
.append(TUPLE_DELIMITER
);
1219 data
.append(file
.mimeType
);
1223 return data
.toString();
1227 static String
getRuntime(String appYaml
) {
1228 String result
= "?";
1230 Map
<?
, ?
> yaml
= (Map
<?
, ?
>) new YamlReader(appYaml
).read();
1231 Object runtime
= yaml
.get("runtime");
1232 if (runtime
instanceof String
) {
1233 result
= (String
) runtime
;
1235 } catch (YamlException ex
) {
1236 logger
.severe(ex
.toString());
1242 static List
<Pattern
> loadSkipFiles(String appYaml
) {
1243 List
<Pattern
> skipFiles
= new ArrayList
<>();
1244 if (appYaml
== null) {
1248 Map
<?
, ?
> yaml
= (Map
<?
, ?
>) new YamlReader(appYaml
).read();
1249 List
<?
> skipFileList
= (List
<?
>) yaml
.get("skip_files");
1250 if (skipFileList
!= null) {
1251 for (Object skipFile
: skipFileList
) {
1252 skipFiles
.add(Pattern
.compile(skipFile
.toString()));
1255 } catch (YamlException ex
) {
1256 logger
.severe(ex
.toString());
1262 static boolean shouldSkip(String name
, List
<Pattern
> skipFiles
) {
1263 for (Pattern skipPattern
: skipFiles
) {
1264 if (skipPattern
.matcher(name
).matches()) {
1271 static class FileInfo
implements Comparable
<FileInfo
> {
1275 public String mimeType
;
1277 private FileInfo(String path
) {
1282 public FileInfo(File f
, File base
) throws LocalIOException
{
1284 this.path
= Utility
.calculatePath(f
, base
);
1285 this.hash
= calculateHash();
1289 static FileInfo
newForTesting(String path
) {
1290 return new FileInfo(path
);
1293 public void setMimeType(GenericApplication app
) {
1294 mimeType
= app
.getMimeTypeIfStatic(path
);
1298 public String
toString() {
1299 return (mimeType
== null ?
"" : mimeType
) + '\t' + hash
+ "\t" + path
;
1303 public int compareTo(FileInfo other
) {
1304 return path
.compareTo(other
.path
);
1308 public int hashCode() {
1309 return path
.hashCode();
1313 public boolean equals(Object obj
) {
1314 if (obj
instanceof FileInfo
) {
1315 return path
.equals(((FileInfo
) obj
).path
);
1320 private static final Pattern FILE_PATH_POSITIVE_RE
=
1321 Pattern
.compile("^[ 0-9a-zA-Z._+/@$-]{1,256}$");
1323 private static final Pattern FILE_PATH_NEGATIVE_RE_1
=
1324 Pattern
.compile("[.][.]|^[.]/|[.]$|/[.]/|^-|^_ah/|^/");
1326 private static final Pattern FILE_PATH_NEGATIVE_RE_2
=
1327 Pattern
.compile("//|/$");
1329 private static final Pattern FILE_PATH_NEGATIVE_RE_3
=
1330 Pattern
.compile("^ | $|/ | /");
1333 static String
checkValidFilename(String path
) {
1334 if (!FILE_PATH_POSITIVE_RE
.matcher(path
).matches()) {
1335 return "Invalid character in filename: " + path
;
1337 if (FILE_PATH_NEGATIVE_RE_1
.matcher(path
).find()) {
1338 return "Filname cannot contain '.' or '..' or start with '-', '_ah/' or '/' : " + path
;
1340 if (FILE_PATH_NEGATIVE_RE_2
.matcher(path
).find()) {
1341 return "Filname cannot have trailing / or contain //: " + path
;
1343 if (FILE_PATH_NEGATIVE_RE_3
.matcher(path
).find()) {
1344 return "Any spaces must be in the middle of a filename: '" + path
+ "'";
1349 private static final BaseEncoding SEPARATED_HEX
=
1350 BaseEncoding
.base16().lowerCase().withSeparator("_", 8);
1353 static String
calculateHash(ByteSource source
) throws IOException
{
1354 byte[] hash
= source
.hash(Hashing
.sha1()).asBytes();
1355 return SEPARATED_HEX
.encode(hash
);
1358 public String
calculateHash() throws LocalIOException
{
1360 return calculateHash(Files
.asByteSource(file
));
1361 } catch (IOException e
) {
1362 throw LocalIOException
.from(e
);
1367 class UploadBatcher
{
1369 static final int MAX_BATCH_SIZE
= 3200000;
1370 static final int MAX_BATCH_COUNT
= 100;
1371 static final int MAX_BATCH_FILE_SIZE
= 200000;
1372 static final int BATCH_OVERHEAD
= 500;
1377 boolean batching
= true;
1378 List
<FileInfo
> batch
= new ArrayList
<FileInfo
>();
1382 * @param what Either "file" or "blob" or "errorblob" indicating what kind of objects this
1383 * batcher uploads. Used in messages and URLs.
1384 * @param batching whether or not we want to really do batch.
1386 public UploadBatcher(String what
, boolean batching
) {
1388 this.singleUrl
= "/api/appversion/add" + what
;
1389 this.batchUrl
= singleUrl
+ "s";
1390 this.batching
= batching
;
1394 * Send the current batch on its way and reset the batch buffer when done
1396 public void sendBatch() throws IOException
{
1399 "Sending batch containing " + batch
.size() + " "+ what
+"(s) totaling " +
1400 batchSize
/ 1000 + "KB.");
1401 clientDeploySender
.sendBatch(batchUrl
, batch
, batchSize
, addVersionToArgs("", ""));
1402 batch
= new ArrayList
<>();
1407 * Flush the current batch.
1409 * This first attempts to send the batch as a single request; if that fails because the server
1410 * doesn"t support batching, the files are sent one by one, and self.batching is reset to
1413 * At the end, self.batch and self.batchSize are reset
1415 public void flush() throws RemoteIOException
{
1416 if (batch
.isEmpty()) {
1421 } catch (Exception e
) {
1422 app
.statusUpdate("Exception in flushing batch payload, so sending 1 by 1..."
1425 for (FileInfo fileInfo
: batch
) {
1426 send(singleUrl
, fileInfo
.file
, fileInfo
.mimeType
, "path", fileInfo
.path
);
1428 batch
= new ArrayList
<FileInfo
>();
1434 * Batch a file, possibly flushing first, or perhaps upload it directly.
1436 * Args: path: The name of the file. payload: The contents of the file. mime_type: The MIME
1437 * Content-type of the file, or None.
1439 * If mime_type is None, application/octet-stream is substituted. """
1441 public void addToBatch(FileInfo fileInfo
) throws RemoteIOException
{
1443 long size
= fileInfo
.file
.length();
1445 if (size
<= MAX_BATCH_FILE_SIZE
) {
1446 if ((batch
.size() >= MAX_BATCH_COUNT
) ||
1447 (batchSize
+ size
> MAX_BATCH_SIZE
)) {
1451 batch
.add(fileInfo
);
1452 batchSize
+= size
+ BATCH_OVERHEAD
;
1456 send(singleUrl
, fileInfo
.file
, fileInfo
.mimeType
, "path", fileInfo
.path
);