1 // Copyright 2009 Google Inc. All rights reserved.
3 package com
.google
.appengine
.tools
.admin
;
5 import com
.google
.appengine
.tools
.admin
.GenericApplication
.ErrorHandler
;
6 import com
.google
.appengine
.tools
.util
.FileIterator
;
7 import com
.google
.common
.annotations
.VisibleForTesting
;
8 import com
.google
.common
.base
.Joiner
;
9 import com
.google
.common
.collect
.ImmutableMap
;
10 import com
.google
.common
.hash
.Hashing
;
11 import com
.google
.common
.io
.BaseEncoding
;
12 import com
.google
.common
.io
.ByteSource
;
13 import com
.google
.common
.io
.Files
;
15 import net
.sourceforge
.yamlbeans
.YamlException
;
16 import net
.sourceforge
.yamlbeans
.YamlReader
;
19 import java
.io
.IOException
;
20 import java
.io
.StringReader
;
21 import java
.net
.HttpURLConnection
;
22 import java
.text
.MessageFormat
;
23 import java
.util
.ArrayList
;
24 import java
.util
.Arrays
;
25 import java
.util
.Collection
;
26 import java
.util
.Collections
;
27 import java
.util
.HashMap
;
28 import java
.util
.List
;
30 import java
.util
.TreeMap
;
31 import java
.util
.TreeSet
;
32 import java
.util
.concurrent
.Callable
;
33 import java
.util
.logging
.Logger
;
34 import java
.util
.regex
.Pattern
;
37 * Uploads a new appversion to the hosting service.
40 public class AppVersionUpload
{
42 * Don't try to precompile more than this number of files in one request.
44 private static final int MAX_FILES_PER_PRECOMPILE
= 50;
46 private static final String YAML_EMPTY_STRING
= "null";
48 private static final String PRECOMPILATION_FAILED_WARNING_MESSAGE
=
49 "Precompilation failed. Consider retrying the update later, or add"
50 + " <precompilation-enabled>false</precompilation-enabled> to your appengine-web.xml"
51 + " to disable precompilation.";
53 protected ServerConnection connection
;
54 protected GenericApplication app
;
55 protected final String backend
;
56 private final Logger logger
= Logger
.getLogger(AppVersionUpload
.class.getName());
57 private boolean inTransaction
= false;
58 private Map
<String
, FileInfo
> files
= new HashMap
<String
, FileInfo
>();
59 private boolean deployed
= false;
60 private boolean started
= false;
61 private boolean checkConfigUpdated
= false;
62 private final UploadBatcher fileBatcher
;
63 private final UploadBatcher blobBatcher
;
65 public AppVersionUpload(ServerConnection connection
, GenericApplication app
) {
66 this(connection
, app
, null, Boolean
.TRUE
);
70 * Create a new {@link AppVersionUpload} instance that can deploy a new
71 * versions of {@code app} via {@code connection}.
73 * @param connection to connect to the server
74 * @param app that contains the code to be deployed
75 * @param backend if supplied and non-{@code null}, a particular backend is
78 public AppVersionUpload(ServerConnection connection
, GenericApplication app
,
79 String backend
, boolean batchMode
) {
80 this.connection
= connection
;
82 this.backend
= backend
;
83 fileBatcher
= new UploadBatcher("file", batchMode
);
84 blobBatcher
= new UploadBatcher("blob", batchMode
);
88 * Uploads a new appversion to the server.
90 * @throws IOException if a problem occurs in the upload.
92 public void doUpload(ResourceLimits resourceLimits
,
93 boolean updateGlobalConfigurations
, boolean failOnPrecompilationError
)
96 File basepath
= getBasepath();
98 app
.statusUpdate("Scanning files on local disk.", 20);
100 long resourceTotal
= 0;
101 for (File f
: new FileIterator(basepath
)) {
102 FileInfo fileInfo
= new FileInfo(f
, basepath
);
103 fileInfo
.setMimeType(app
);
104 logger
.fine("Processing file '" + f
+ "'.");
105 long maxFileBlobSize
= fileInfo
.mimeType
!= null ?
106 resourceLimits
.maxBlobSize() : resourceLimits
.maxFileSize();
107 if (f
.length() > maxFileBlobSize
) {
109 if (f
.getName().toLowerCase().endsWith(".jar")) {
110 message
= "Jar " + f
.getPath() + " is too large. Consider "
111 + "using --enable_jar_splitting.";
113 message
= "File " + f
.getPath() + " is too large (limit "
114 + maxFileBlobSize
+ " bytes).";
116 throw new IOException(message
);
118 resourceTotal
+= addFile(fileInfo
);
120 if (++numFiles
% 250 == 0) {
121 app
.statusUpdate("Scanned " + numFiles
+ " files.");
124 if (numFiles
> resourceLimits
.maxFileCount()) {
125 throw new IOException("Applications are limited to "
126 + resourceLimits
.maxFileCount() + " files, you have " + numFiles
129 if (resourceTotal
> resourceLimits
.maxTotalFileSize()) {
130 throw new IOException("Applications are limited to "
131 + resourceLimits
.maxTotalFileSize() + " bytes of resource files, "
132 + "you have " + resourceTotal
+ ".");
135 Collection
<FileInfo
> missingFiles
= beginTransaction(resourceLimits
);
136 app
.statusUpdate("Uploading " + missingFiles
.size() + " files.", 50);
137 if (missingFiles
.size() > 0) {
139 int quarter
= Math
.max(1, missingFiles
.size() / 4);
140 for (FileInfo missingFile
: missingFiles
) {
141 logger
.fine("Uploading file '" + missingFile
+ "'");
142 uploadFile(missingFile
);
143 if (++numFiles
% quarter
== 0) {
144 app
.statusUpdate("Uploaded " + numFiles
+ " files.");
148 uploadErrorHandlers(app
.getErrorHandlers(), basepath
);
149 if (app
.isPrecompilationEnabled()) {
150 precompile(failOnPrecompilationError
);
159 if (updateGlobalConfigurations
) {
167 reportSkippingGlobalConfiguration();
171 private void reportSkippingGlobalConfiguration() {
172 TreeSet
<String
> skipSet
= new TreeSet
<String
>();
173 if (app
.getIndexesXml() != null) {
174 skipSet
.add("indexes.xml");
176 if (app
.getCronXml() != null) {
177 skipSet
.add("cron.xml");
179 if (app
.getQueueXml() != null) {
180 skipSet
.add("queue.xml");
182 if (app
.getDispatchXml() != null) {
183 skipSet
.add("dispatch.xml");
185 if (app
.getDosXml() != null) {
186 skipSet
.add("dos.xml");
188 if (app
.getPagespeedYaml() != null) {
189 skipSet
.add("pagespeed");
191 if (!skipSet
.isEmpty()) {
192 app
.statusUpdate("Skipping global configurations: " + Joiner
.on(", ").join(skipSet
));
196 private void uploadErrorHandlers(List
<ErrorHandler
> errorHandlers
, File basepath
)
198 if (!errorHandlers
.isEmpty()) {
199 app
.statusUpdate("Uploading " + errorHandlers
.size() + " file(s) "
200 + "for static error handlers.");
201 for (ErrorHandler handler
: errorHandlers
) {
202 File file
= new File(basepath
, handler
.getFile());
203 FileInfo info
= new FileInfo(file
, basepath
);
204 String error
= FileInfo
.checkValidFilename(info
.path
);
206 throw new IOException("Could not find static error handler: " + error
);
208 info
.mimeType
= handler
.getMimeType();
209 String errorType
= handler
.getErrorCode();
210 if (errorType
== null) {
211 errorType
= "default";
213 send("/api/appversion/adderrorblob", info
.file
, info
.mimeType
, "path",
219 public void precompile(boolean failOnPrecompilationError
) throws IOException
{
220 app
.statusUpdate("Initializing precompilation...");
221 List
<String
> filesToCompile
= new ArrayList
<String
>();
223 boolean containsGoFiles
= false;
224 for (String f
: this.files
.keySet()) {
225 boolean isGoFile
= f
.toLowerCase().endsWith(".go");
226 if (isGoFile
&& !containsGoFiles
) {
227 containsGoFiles
= true;
229 if (isGoFile
|| f
.toLowerCase().endsWith(".py")) {
230 filesToCompile
.add(f
);
233 Collections
.sort(filesToCompile
);
234 if (containsGoFiles
) {
235 failOnPrecompilationError
= true;
241 filesToCompile
.addAll(sendPrecompileRequest(Collections
242 .<String
> emptyList()));
244 } catch (IOException ex
) {
245 if (errorCount
< 3) {
249 } catch (InterruptedException ex2
) {
251 new IOException("Interrupted during precompilation.");
256 if (failOnPrecompilationError
) {
259 "Precompilation failed. Consider adding <precompilation-enabled>false"
260 + "</precompilation-enabled> to your appengine-web.xml and trying again.");
264 logger
.warning(PRECOMPILATION_FAILED_WARNING_MESSAGE
);
272 IOException lastError
= null;
273 while (!filesToCompile
.isEmpty()) {
275 if (precompileChunk(filesToCompile
)) {
278 } catch (IOException ex
) {
281 Collections
.shuffle(filesToCompile
);
284 } catch (InterruptedException ex2
) {
286 new IOException("Interrupted during precompilation.");
292 if (errorCount
> 3) {
293 if (failOnPrecompilationError
) {
295 new IOException("Precompilation failed with "
296 + filesToCompile
.size() + " file(s) remaining. "
298 + " <precompilation-enabled>false</precompilation-enabled>"
299 + " to your " + "appengine-web.xml and trying again.");
300 ex2
.initCause(lastError
);
303 logger
.warning(PRECOMPILATION_FAILED_WARNING_MESSAGE
);
311 * Attempt to precompile up to {@code MAX_FILES_PER_PRECOMPILE} files from
312 * {@code filesToCompile}.
314 * @param filesToCompile a list of file names, which will be mutated to remove
315 * any files that were successfully compiled.
317 * @return true if filesToCompile was reduced in size (i.e. progress was
320 private boolean precompileChunk(List
<String
> filesToCompile
)
322 int filesLeft
= filesToCompile
.size();
323 if (filesLeft
== 0) {
324 app
.statusUpdate("Initializing precompilation...");
326 app
.statusUpdate(MessageFormat
.format(
327 "Precompiling... {0} file(s) left.", filesLeft
));
330 List
<String
> subset
=
332 .subList(0, Math
.min(filesLeft
, MAX_FILES_PER_PRECOMPILE
));
333 List
<String
> remainingFiles
= sendPrecompileRequest(subset
);
335 filesToCompile
.addAll(remainingFiles
);
336 return filesToCompile
.size() < filesLeft
;
339 private List
<String
> sendPrecompileRequest(List
<String
> filesToCompile
)
342 send("/api/appversion/precompile", Joiner
.on("\n").useForNull("null").join(filesToCompile
));
343 if (response
.length() > 0) {
344 return Arrays
.asList(response
.split("\n"));
346 return Collections
.emptyList();
350 public void updateIndexes() throws IOException
{
351 if (app
.getIndexesXml() != null) {
352 app
.statusUpdate("Uploading index definitions.");
353 send("/api/datastore/index/add", getIndexYaml());
358 public void updateCron() throws IOException
{
359 String yaml
= getCronYaml();
361 app
.statusUpdate("Uploading cron jobs.");
362 send("/api/datastore/cron/update", yaml
);
366 public void updateQueue() throws IOException
{
367 String yaml
= getQueueYaml();
369 app
.statusUpdate("Uploading task queues.");
370 send("/api/queue/update", yaml
);
374 public void updateDispatch() throws IOException
{
375 String yaml
= getDispatchYaml();
377 app
.statusUpdate("Uploading dispatch entries.");
378 send("/api/dispatch/update", yaml
);
382 public void updateDos() throws IOException
{
383 String yaml
= getDosYaml();
385 app
.statusUpdate("Uploading DoS entries.");
386 send("/api/dos/update", yaml
);
390 public void updatePagespeed() throws IOException
{
391 String yaml
= getPagespeedYaml();
393 app
.statusUpdate("Uploading PageSpeed entries.");
394 send("/api/appversion/updatepagespeed", yaml
);
397 send("/api/appversion/updatepagespeed", "");
398 } catch (HttpIoException exc
) {
399 if (exc
.getResponseCode() != HttpURLConnection
.HTTP_NOT_FOUND
) {
406 public void setDefaultVersion() throws IOException
{
407 app
.statusUpdate("Setting default version to " + app
.getVersion() + ".");
408 send("/api/appversion/setdefault", "");
411 protected String
getIndexYaml() {
412 return app
.getIndexesXml().toYaml();
415 protected String
getCronYaml() {
416 if (app
.getCronXml() != null) {
417 return app
.getCronXml().toYaml();
423 protected String
getQueueYaml() {
424 if (app
.getQueueXml() != null) {
425 return app
.getQueueXml().toYaml();
431 protected String
getDispatchYaml() {
432 return app
.getDispatchXml() == null ?
null : app
.getDispatchXml().toYaml();
435 protected String
getDosYaml() {
436 if (app
.getDosXml() != null) {
437 return app
.getDosXml().toYaml();
443 protected String
getPagespeedYaml() {
444 return app
.getPagespeedYaml();
448 protected boolean getInTransaction() {
449 return this.inTransaction
;
453 protected void setInTransaction(boolean newValue
) {
454 this.inTransaction
= newValue
;
457 private File
getBasepath() {
458 File path
= app
.getStagingDir();
460 path
= new File(app
.getPath());
466 * Adds a file for uploading, returning the bytes counted against the total
470 * @return 0 for a static file, or file.length() for a resource file.
471 * @throws IOException
474 long addFile(FileInfo info
) throws IOException
{
476 throw new IllegalStateException("Already in a transaction.");
479 String error
= FileInfo
.checkValidFilename(info
.path
);
481 logger
.severe(error
);
485 files
.put(info
.path
, info
);
487 return info
.mimeType
!= null ?
0 : info
.file
.length();
491 * Parses the response from /api/appversion/create into a Map.
493 * @param response String returned from the /api/appversion/create call.
494 * @return YAML parsed into Map.
496 private ArrayList
<String
> validateBeginYaml(String response
) {
497 YamlReader yaml
= new YamlReader(new StringReader(response
));
499 Object obj
= yaml
.read();
501 @SuppressWarnings("unchecked")
502 Map
<String
, Object
> responseMap
= (Map
<String
, Object
>) obj
;
503 if (responseMap
!= null) {
504 obj
= responseMap
.get("warnings");
506 @SuppressWarnings("unchecked")
507 ArrayList
<String
> warnings
= (ArrayList
<String
>) obj
;
512 } catch (YamlException exc
) {
513 } catch (ClassCastException exc
) {
515 return new ArrayList
<String
>();
519 * Begins the transaction, returning a list of files that need uploading.
521 * All calls to addFile must be made before calling beginTransaction().
523 * @param resourceLimits is the collection of resource limits for AppCfg.
524 * @return A list of pathnames that should be uploaded using uploadFile()
525 * before calling commit().
528 Collection
<FileInfo
> beginTransaction(ResourceLimits resourceLimits
)
531 throw new IllegalStateException("Already in a transaction.");
534 if (backend
== null) {
535 app
.statusUpdate("Initiating update.");
537 app
.statusUpdate("Initiating update of backend " + backend
+ ".");
539 String response
= send("/api/appversion/create", app
.getAppYaml());
540 ArrayList
<String
> warnings
= validateBeginYaml(response
);
541 for (String warning
: warnings
) {
542 app
.statusUpdate("WARNING: " + warning
);
544 inTransaction
= true;
545 Collection
<FileInfo
> blobsToClone
= new ArrayList
<FileInfo
>(files
.size());
546 Collection
<FileInfo
> filesToClone
= new ArrayList
<FileInfo
>(files
.size());
548 for (FileInfo f
: files
.values()) {
549 if (f
.mimeType
== null) {
556 TreeMap
<String
, FileInfo
> filesToUpload
= new TreeMap
<String
, FileInfo
>();
557 cloneFiles("/api/appversion/cloneblobs", blobsToClone
, "static",
558 filesToUpload
, resourceLimits
.maxFilesToClone());
559 cloneFiles("/api/appversion/clonefiles", filesToClone
, "application",
560 filesToUpload
, resourceLimits
.maxFilesToClone());
562 logger
.fine("Files to upload :");
563 for (FileInfo f
: filesToUpload
.values()) {
564 logger
.fine("\t" + f
);
567 this.files
= filesToUpload
;
568 return new ArrayList
<FileInfo
>(filesToUpload
.values());
571 private static final String LIST_DELIMITER
= "\n";
574 * Sends files to the given url.
576 * @param url server URL to use.
577 * @param filesParam List of files to clone.
578 * @param type Type of files ( "static" or "application")
579 * @param filesToUpload Files that need to be uploaded are added to this
581 * @param maxFilesToClone Max number of files to clone at a single time.
583 private void cloneFiles(String url
, Collection
<FileInfo
> filesParam
,
584 String type
, Map
<String
, FileInfo
> filesToUpload
, long maxFilesToClone
)
586 if (filesParam
.isEmpty()) {
589 app
.statusUpdate("Cloning " + filesParam
.size() + " " + type
+ " files.");
592 int remaining
= filesParam
.size();
593 ArrayList
<FileInfo
> chunk
= new ArrayList
<FileInfo
>((int) maxFilesToClone
);
594 for (FileInfo file
: filesParam
) {
596 if (--remaining
== 0 || chunk
.size() >= maxFilesToClone
) {
598 app
.statusUpdate("Cloned " + cloned
+ " files.");
600 String result
= send(url
, buildClonePayload(chunk
));
601 if (result
!= null && result
.length() > 0) {
602 for (String path
: result
.split(LIST_DELIMITER
)) {
603 if (path
== null || path
.length() == 0) {
606 FileInfo info
= this.files
.get(path
);
608 logger
.warning("Skipping " + path
+ ": missing FileInfo");
611 filesToUpload
.put(path
, info
);
614 cloned
+= chunk
.size();
621 * Uploads a file to the hosting service.
623 * Must only be called after beginTransaction(). The file provided must be on
624 * of those that were returned by beginTransaction();
626 * @param file FileInfo for the file to upload.
628 private void uploadFile(FileInfo file
) throws IOException
{
629 if (!inTransaction
) {
630 throw new IllegalStateException(
631 "beginTransaction() must be called before uploadFile().");
633 if (!files
.containsKey(file
.path
)) {
634 throw new IllegalArgumentException("File " + file
.path
635 + " is not in the list of files to be uploaded.");
638 files
.remove(file
.path
);
639 if (file
.mimeType
== null) {
640 fileBatcher
.addToBatch(file
);
642 blobBatcher
.addToBatch(file
);
647 * Commits the transaction, making the new app version available.
649 * All the files returned by beginTransaction must have been uploaded with
650 * uploadFile() before commit() may be called.
653 void commit() throws IOException
{
656 boolean ready
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
658 public Boolean
call() throws Exception
{
666 logger
.severe("Version still not ready to serve, aborting.");
667 throw new RuntimeException("Version not ready.");
670 boolean versionIsServing
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
672 public Boolean
call() throws Exception
{
676 if (!versionIsServing
) {
677 logger
.severe("Version still not serving, aborting.");
678 throw new RuntimeException("Version not ready.");
680 if (checkConfigUpdated
) {
681 boolean configIsUpdated
= retryWithBackoff(1, 2, 60, 20, new Callable
<Boolean
>() {
683 public Boolean
call() throws Exception
{
684 return isConfigUpdated();
687 if (!configIsUpdated
) {
688 logger
.severe("Endpoints configuration not updated, aborting.");
689 throw new RuntimeException("Endpoints configuration not updated.");
692 app
.statusUpdate("Closing update: new version is ready to start serving.");
693 inTransaction
= false;
694 } catch (IOException ioe
) {
696 } catch (RuntimeException e
) {
698 } catch (Exception e
) {
699 throw new RuntimeException(e
);
704 * Deploys the new app version but does not make it default.
706 * All the files returned by beginTransaction must have been uploaded with
707 * uploadFile() before commit() may be called.
709 private void deploy() throws IOException
{
710 if (!inTransaction
) {
711 throw new IllegalStateException(
712 "beginTransaction() must be called before deploy().");
714 if (files
.size() > 0) {
715 throw new IllegalStateException(
716 "Some required files have not been uploaded.");
718 app
.statusUpdate("Deploying new version.", 20);
719 send("/api/appversion/deploy", "");
724 * Check if the new app version is ready to serve traffic.
726 * @return true if the server returned that the app is ready to serve.
728 private boolean isReady() throws IOException
{
730 throw new IllegalStateException(
731 "deploy() must be called before isReady()");
733 String result
= send("/api/appversion/isready", "");
734 return "1".equals(result
.trim());
737 private void startServing() throws IOException
{
739 throw new IllegalStateException(
740 "deploy() must be called before startServing()");
742 send("/api/appversion/startserving", "", "willcheckserving", "1");
747 protected Map
<String
, String
> parseIsServingResponse(String isServingResp
) {
748 ImmutableMap
.Builder
<String
, String
> result
= ImmutableMap
.builder();
749 if (isServingResp
.isEmpty()) {
750 return result
.build();
754 YamlReader yamlReader
= new YamlReader(isServingResp
);
755 @SuppressWarnings("unchecked")
756 Map
<Object
, Object
> resultMap
= yamlReader
.read(Map
.class, String
.class);
757 for (Object key
: resultMap
.keySet()) {
758 result
.put((String
) key
, (String
) resultMap
.get(key
));
760 } catch (YamlException e
) {
761 logger
.severe("Unable to parse Yaml from response: " + result
);
762 throw new RuntimeException(e
);
764 return result
.build();
767 private boolean isServing() throws IOException
{
769 throw new IllegalStateException(
770 "startServing() must be called before isServing().");
772 String result
= send("/api/appversion/isserving", "", "new_serving_resp", "1");
773 if ("1".equals(result
.trim()) || "0".equals(result
.trim())) {
774 return "1".equals(result
.trim());
777 Map
<String
, String
> resultMap
= parseIsServingResponse(result
.trim());
778 if (resultMap
.containsKey("message") &&
779 !YAML_EMPTY_STRING
.equals(resultMap
.get("message"))) {
780 app
.statusUpdate(resultMap
.get("message"));
782 if (resultMap
.containsKey("fatal") &&
783 Boolean
.parseBoolean(resultMap
.get("fatal").toLowerCase())) {
784 throw new RuntimeException(
785 "Fatal problem encountered during deployment. Please refer to the logs" +
786 " for more information.");
788 if (resultMap
.containsKey("check_endpoints_config")) {
789 checkConfigUpdated
= Boolean
.parseBoolean(resultMap
.get("check_endpoints_config"));
791 if (resultMap
.containsKey("serving")) {
792 return Boolean
.parseBoolean(resultMap
.get("serving"));
794 throw new RuntimeException(
795 "Fatal problem encountered during deployment. Unexpected response when " +
796 "checking for serving status. Response: " + result
);
801 Map
<String
, String
> parseIsConfigUpdatedResponse(String isConfigUpdatedResp
) {
802 ImmutableMap
.Builder
<String
, String
> result
= ImmutableMap
.builder();
804 YamlReader yamlReader
= new YamlReader(isConfigUpdatedResp
);
805 @SuppressWarnings("unchecked")
806 Map
<Object
, Object
> resultMap
= yamlReader
.read(Map
.class, String
.class);
807 if (resultMap
== null) {
808 return result
.build();
811 for (Object key
: resultMap
.keySet()) {
812 result
.put((String
) key
, (String
) resultMap
.get(key
));
814 } catch (YamlException e
) {
815 logger
.severe("Unable to parse Yaml from response: " + result
);
816 throw new RuntimeException(e
);
818 return result
.build();
821 private boolean isConfigUpdated() throws IOException
{
823 throw new IllegalStateException(
824 "startServing() must be called before isConfigUpdated().");
826 String result
= send("/api/isconfigupdated", "");
828 Map
<String
, String
> resultMap
= parseIsConfigUpdatedResponse(result
.trim());
829 if (resultMap
.containsKey("updated")) {
830 return Boolean
.parseBoolean(resultMap
.get("updated"));
832 throw new RuntimeException(
833 "Fatal problem encountered during deployment. Unexpected response when " +
834 "checking for configuration update status. Response: " + result
);
838 public void forceRollback() throws IOException
{
839 app
.statusUpdate("Rolling back the update" + (this.backend
== null ?
"."
840 : " on backend " + this.backend
+ "."));
841 send("/api/appversion/rollback", "");
844 private void rollback() throws IOException
{
845 if (!inTransaction
) {
852 String
send(String url
, String payload
, String
... args
)
854 return connection
.post(url
, payload
, addVersionToArgs(args
));
858 String
send(String url
, File payload
, String mimeType
, String
... args
)
860 return connection
.post(url
, payload
, mimeType
, addVersionToArgs(args
));
863 private String
[] addVersionToArgs(String
... args
) {
864 List
<String
> result
= new ArrayList
<String
>();
865 result
.addAll(Arrays
.asList(args
));
866 result
.add("app_id");
867 result
.add(app
.getAppId());
868 if (backend
!= null) {
869 result
.add("backend");
871 } else if (app
.getVersion() != null) {
872 result
.add("version");
873 result
.add(app
.getVersion());
875 if (app
.getModule() != null) {
876 result
.add("module");
877 result
.add(app
.getModule());
879 return result
.toArray(new String
[result
.size()]);
883 * Calls a function multiple times, backing off more and more each time.
885 * @param initialDelay Inital delay after the first try, in seconds.
886 * @param backoffFactor Delay will be multiplied by this factor after each
888 * @param maxDelay Maximum delay factor.
889 * @param maxTries Maximum number of tries.
890 * @param callable Callable to call.
891 * @return true if the Callable returned true in one of its tries.
893 private boolean retryWithBackoff(double initialDelay
, double backoffFactor
,
894 double maxDelay
, int maxTries
, Callable
<Boolean
> callable
)
896 long delayMillis
= (long) (initialDelay
* 1000);
897 long maxDelayMillis
= (long) (maxDelay
* 1000);
898 if (callable
.call()) {
901 while (maxTries
> 1) {
902 app
.statusUpdate("Will check again in " + (delayMillis
/ 1000)
904 Thread
.sleep(delayMillis
);
905 delayMillis
*= backoffFactor
;
906 if (delayMillis
> maxDelayMillis
) {
907 delayMillis
= maxDelayMillis
;
910 if (callable
.call()) {
917 private static final String TUPLE_DELIMITER
= "|";
920 * Build the post body for a clone request.
922 * @param files List of FileInfos for the files to clone.
923 * @return A string containing the properly delimited tuples.
925 private static String
buildClonePayload(Collection
<FileInfo
> files
) {
926 StringBuffer data
= new StringBuffer();
927 boolean first
= true;
928 for (FileInfo file
: files
) {
932 data
.append(LIST_DELIMITER
);
934 data
.append(file
.path
);
935 data
.append(TUPLE_DELIMITER
);
936 data
.append(file
.hash
);
937 if (file
.mimeType
!= null) {
938 data
.append(TUPLE_DELIMITER
);
939 data
.append(file
.mimeType
);
943 return data
.toString();
946 static class FileInfo
implements Comparable
<FileInfo
> {
950 public String mimeType
;
952 private FileInfo(String path
) {
957 public FileInfo(File f
, File base
) throws IOException
{
959 this.path
= Utility
.calculatePath(f
, base
);
960 this.hash
= calculateHash();
964 static FileInfo
newForTesting(String path
) {
965 return new FileInfo(path
);
968 public void setMimeType(GenericApplication app
) {
969 mimeType
= app
.getMimeTypeIfStatic(path
);
973 public String
toString() {
974 return (mimeType
== null ?
"" : mimeType
) + '\t' + hash
+ "\t" + path
;
978 public int compareTo(FileInfo other
) {
979 return path
.compareTo(other
.path
);
983 public int hashCode() {
984 return path
.hashCode();
988 public boolean equals(Object obj
) {
989 if (obj
instanceof FileInfo
) {
990 return path
.equals(((FileInfo
) obj
).path
);
995 private static final Pattern FILE_PATH_POSITIVE_RE
=
996 Pattern
.compile("^[ 0-9a-zA-Z._+/@$-]{1,256}$");
998 private static final Pattern FILE_PATH_NEGATIVE_RE_1
=
999 Pattern
.compile("[.][.]|^[.]/|[.]$|/[.]/|^-|^_ah/|^/");
1001 private static final Pattern FILE_PATH_NEGATIVE_RE_2
=
1002 Pattern
.compile("//|/$");
1004 private static final Pattern FILE_PATH_NEGATIVE_RE_3
=
1005 Pattern
.compile("^ | $|/ | /");
1008 static String
checkValidFilename(String path
) {
1009 if (!FILE_PATH_POSITIVE_RE
.matcher(path
).matches()) {
1010 return "Invalid character in filename: " + path
;
1012 if (FILE_PATH_NEGATIVE_RE_1
.matcher(path
).find()) {
1013 return "Filname cannot contain '.' or '..' or start with '-', '_ah/' or '/' : " + path
;
1015 if (FILE_PATH_NEGATIVE_RE_2
.matcher(path
).find()) {
1016 return "Filname cannot have trailing / or contain //: " + path
;
1018 if (FILE_PATH_NEGATIVE_RE_3
.matcher(path
).find()) {
1019 return "Any spaces must be in the middle of a filename: '" + path
+ "'";
1024 private static final BaseEncoding SEPARATED_HEX
=
1025 BaseEncoding
.base16().lowerCase().withSeparator("_", 8);
1028 static String
calculateHash(ByteSource source
) throws IOException
{
1029 byte[] hash
= source
.hash(Hashing
.sha1()).asBytes();
1030 return SEPARATED_HEX
.encode(hash
);
1033 public String
calculateHash() throws IOException
{
1034 return calculateHash(Files
.asByteSource(file
));
1038 class UploadBatcher
{
1040 private static final int MAX_BATCH_SIZE
= 3200000;
1041 private static final int MAX_BATCH_COUNT
= 100;
1042 private static final int MAX_BATCH_FILE_SIZE
= 200000;
1043 private static final int BATCH_OVERHEAD
= 500;
1048 boolean batching
= true;
1049 List
<FileInfo
> batch
= new ArrayList
<FileInfo
>();
1053 * @param what Either "file" or "blob" or "errorblob" indicating what kind of objects this
1054 * batcher uploads. Used in messages and URLs.
1055 * @param batching whether or not we want to really do batch.
1057 public UploadBatcher(String what
, boolean batching
) {
1059 this.singleUrl
= "/api/appversion/add" + what
;
1060 this.batchUrl
= singleUrl
+ "s";
1061 this.batching
= batching
;
1065 * Send the current batch on its way and reset the batrch buffer when done
1067 public void sendBatch() throws IOException
{
1070 "Sending batch containing " + batch
.size() + " "+ what
+"(s) totaling " +
1071 batchSize
/ 1000 + "KB.");
1072 connection
.post(batchUrl
, batch
, addVersionToArgs("", ""));
1073 batch
= new ArrayList
<FileInfo
>();
1078 * """Flush the current batch.
1080 * This first attempts to send the batch as a single request; if that fails because the server
1081 * doesn"t support batching, the files are sent one by one, and self.batching is reset to
1084 * At the end, self.batch and self.batchSize are reset
1086 public void flush() throws IOException
{
1087 if (batch
.isEmpty()) {
1092 } catch (Exception e
) {
1093 app
.statusUpdate("Exception in flushing batch payload, so sending 1 by 1..."
1096 for (FileInfo fileInfo
: batch
) {
1097 send(singleUrl
, fileInfo
.file
, fileInfo
.mimeType
, "path", fileInfo
.path
);
1099 batch
= new ArrayList
<FileInfo
>();
1105 * Batch a file, possibly flushing first, or perhaps upload it directly.
1107 * Args: path: The name of the file. payload: The contents of the file. mime_type: The MIME
1108 * Content-type of the file, or None.
1110 * If mime_type is None, application/octet-stream is substituted. """
1112 public void addToBatch(FileInfo fileInfo
) throws IOException
{
1114 long size
= fileInfo
.file
.length();
1116 if (size
<= MAX_BATCH_FILE_SIZE
) {
1117 if ((batch
.size() >= MAX_BATCH_COUNT
) ||
1118 (batchSize
+ size
> MAX_BATCH_SIZE
)) {
1122 batch
.add(fileInfo
);
1123 batchSize
+= size
+ BATCH_OVERHEAD
;
1127 send(singleUrl
, fileInfo
.file
, fileInfo
.mimeType
, "path", fileInfo
.path
);