java/src/main/com/google/appengine/tools/admin/AppVersionUpload.java

   1 // Copyright 2009 Google Inc. All rights reserved.
   2
   3 package com.google.appengine.tools.admin;
   4
   5 import com.google.appengine.tools.admin.GenericApplication.ErrorHandler;
   6 import com.google.appengine.tools.util.FileIterator;
   7 import com.google.common.annotations.VisibleForTesting;
   8 import com.google.common.base.Join;
   9
  10 import java.io.File;
  11 import java.io.FileInputStream;
  12 import java.io.IOException;
  13 import java.io.InputStream;
  14 import java.net.HttpURLConnection;
  15 import java.security.MessageDigest;
  16 import java.security.NoSuchAlgorithmException;
  17 import java.text.MessageFormat;
  18 import java.util.ArrayList;
  19 import java.util.Arrays;
  20 import java.util.Collection;
  21 import java.util.Collections;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.TreeMap;
  26 import java.util.concurrent.Callable;
  27 import java.util.logging.Logger;
  28 import java.util.regex.Pattern;
  29
  30 /**
  31  * Uploads a new appversion to the hosting service.
  32  *
  33  */
  34 public class AppVersionUpload {
  35   /**
  36    * Don't try to precompile more than this number of files in one request.
  37    */
  38   private static final int MAX_FILES_PER_PRECOMPILE = 50;
  39
  40   protected ServerConnection connection;
  41   protected GenericApplication app;
  42   protected final String backend;
  43   private final Logger logger = Logger.getLogger(AppVersionUpload.class.getName());
  44   private boolean inTransaction = false;
  45   private Map<String, FileInfo> files = new HashMap<String, FileInfo>();
  46   private boolean deployed = false;
  47   private final UploadBatcher fileBatcher;
  48   private final UploadBatcher blobBatcher;
  49
  50   public AppVersionUpload(ServerConnection connection, GenericApplication app) {
  51     this(connection, app, null, Boolean.TRUE);
  52   }
  53
  54   /**
  55    * Create a new {@link AppVersionUpload} instance that can deploy a new
  56    * versions of {@code app} via {@code connection}.
  57    *
  58    * @param connection to connect to the server
  59    * @param app that contains the code to be deployed
  60    * @param backend if supplied and non-{@code null}, a particular backend is
  61    *        being updated
  62    */
  63   public AppVersionUpload(ServerConnection connection, GenericApplication app,
  64       String backend, boolean batchMode) {
  65     this.connection = connection;
  66     this.app = app;
  67     this.backend = backend;
  68     fileBatcher = new UploadBatcher("file", batchMode);
  69     blobBatcher = new UploadBatcher("blob", batchMode);
  70   }
  71
  72   /***
  73    * Uploads a new appversion to the server.
  74    *
  75    * @throws IOException if a problem occurs in the upload.
  76    */
  77   public void doUpload(ResourceLimits resourceLimits) throws IOException {
  78     try {
  79       File basepath = getBasepath();
  80
  81       app.statusUpdate("Scanning files on local disk.", 20);
  82       int numFiles = 0;
  83       long resourceTotal = 0;
  84       for (File f : new FileIterator(basepath)) {
  85         FileInfo fileInfo = new FileInfo(f, basepath);
  86         fileInfo.setMimeType(app);
  87         logger.fine("Processing file '" + f + "'.");
  88         long maxFileBlobSize = fileInfo.mimeType != null ?
  89             resourceLimits.maxBlobSize() : resourceLimits.maxFileSize();
  90         if (f.length() > maxFileBlobSize) {
  91           String message;
  92           if (f.getName().toLowerCase().endsWith(".jar")) {
  93             message = "Jar " + f.getPath() + " is too large. Consider "
  94                 + "using --enable_jar_splitting.";
  95           } else {
  96             message = "File " + f.getPath() + " is too large (limit "
  97                 + maxFileBlobSize + " bytes).";
  98           }
  99           throw new IOException(message);
 100         }
 101         resourceTotal += addFile(fileInfo);
 102
 103         if (++numFiles % 250 == 0) {
 104           app.statusUpdate("Scanned " + numFiles + " files.");
 105         }
 106       }
 107       if (numFiles > resourceLimits.maxFileCount()) {
 108         throw new IOException("Applications are limited to "
 109             + resourceLimits.maxFileCount() + " files, you have " + numFiles
 110             + ".");
 111       }
 112       if (resourceTotal > resourceLimits.maxTotalFileSize()) {
 113         throw new IOException("Applications are limited to "
 114             + resourceLimits.maxTotalFileSize() + " bytes of resource files, "
 115             + "you have " + resourceTotal + ".");
 116       }
 117
 118       Collection<FileInfo> missingFiles = beginTransaction(resourceLimits);
 119       app.statusUpdate("Uploading " + missingFiles.size() + " files.", 50);
 120       if (missingFiles.size() > 0) {
 121         numFiles = 0;
 122         int quarter = Math.max(1, missingFiles.size() / 4);
 123         for (FileInfo missingFile : missingFiles) {
 124           logger.fine("Uploading file '" + missingFile + "'");
 125           uploadFile(missingFile);
 126           if (++numFiles % quarter == 0) {
 127             app.statusUpdate("Uploaded " + numFiles + " files.");
 128           }
 129         }
 130       }
 131       uploadErrorHandlers(app.getErrorHandlers(), basepath);
 132       if (app.isPrecompilationEnabled()) {
 133         precompile();
 134       }
 135       fileBatcher.flush();
 136       blobBatcher.flush();
 137       commit();
 138     } finally {
 139       rollback();
 140     }
 141
 142     updateIndexes();
 143     updateCron();
 144     updateQueue();
 145     updateDos();
 146     updatePagespeed();
 147   }
 148
 149   private void uploadErrorHandlers(List<ErrorHandler> errorHandlers, File basepath)
 150     throws IOException {
 151     if (!errorHandlers.isEmpty()) {
 152       app.statusUpdate("Uploading " + errorHandlers.size() + " file(s) "
 153           + "for static error handlers.");
 154       for (ErrorHandler handler : errorHandlers) {
 155         File file = new File(basepath, handler.getFile());
 156         FileInfo info = new FileInfo(file, basepath);
 157         String error = info.checkValidFilename();
 158         if (error != null) {
 159           throw new IOException("Could not find static error handler: " + error);
 160         }
 161         info.mimeType = handler.getMimeType();
 162         String errorType = handler.getErrorCode();
 163         if (errorType == null) {
 164           errorType = "default";
 165         }
 166         send("/api/appversion/adderrorblob", info.file, info.mimeType, "path",
 167             errorType);
 168       }
 169     }
 170   }
 171
 172   public void precompile() throws IOException {
 173     app.statusUpdate("Initializing precompilation...");
 174     List<String> filesToCompile = new ArrayList<String>();
 175
 176     int errorCount = 0;
 177     while (true) {
 178       try {
 179         filesToCompile.addAll(sendPrecompileRequest(Collections
 180             .<String> emptyList()));
 181         break;
 182       } catch (IOException ex) {
 183         if (errorCount < 3) {
 184           errorCount++;
 185           try {
 186             Thread.sleep(1000);
 187           } catch (InterruptedException ex2) {
 188             IOException ex3 =
 189                 new IOException("Interrupted during precompilation.");
 190             ex3.initCause(ex2);
 191             throw ex3;
 192           }
 193         } else {
 194           IOException ex2 =
 195               new IOException(
 196                   "Precompilation failed.  Consider adding <precompilation-enabled>false"
 197                       + "</precompilation-enabled> to your appengine-web.xml and trying again.");
 198           ex2.initCause(ex);
 199           throw ex2;
 200         }
 201       }
 202     }
 203
 204     errorCount = 0;
 205     IOException lastError = null;
 206     while (!filesToCompile.isEmpty()) {
 207       try {
 208         if (precompileChunk(filesToCompile)) {
 209           errorCount = 0;
 210         }
 211       } catch (IOException ex) {
 212         lastError = ex;
 213         errorCount++;
 214         Collections.shuffle(filesToCompile);
 215         try {
 216           Thread.sleep(1000);
 217         } catch (InterruptedException ex2) {
 218           IOException ex3 =
 219               new IOException("Interrupted during precompilation.");
 220           ex3.initCause(ex2);
 221           throw ex3;
 222         }
 223       }
 224
 225       if (errorCount > 3) {
 226         IOException ex2 =
 227             new IOException("Precompilation failed with "
 228                 + filesToCompile.size() + " file(s) remaining.  "
 229                 + "Consider adding"
 230                 + " <precompilation-enabled>false</precompilation-enabled>"
 231                 + " to your " + "appengine-web.xml and trying again.");
 232         ex2.initCause(lastError);
 233         throw ex2;
 234       }
 235     }
 236   }
 237
 238   /**
 239    * Attempt to precompile up to {@code MAX_FILES_PER_PRECOMPILE} files from
 240    * {@code filesToCompile}.
 241    *
 242    * @param filesToCompile a list of file names, which will be mutated to remove
 243    *        any files that were successfully compiled.
 244    *
 245    * @return true if filesToCompile was reduced in size (i.e. progress was
 246    *         made).
 247    */
 248   private boolean precompileChunk(List<String> filesToCompile)
 249       throws IOException {
 250     int filesLeft = filesToCompile.size();
 251     if (filesLeft == 0) {
 252       app.statusUpdate("Initializing precompilation...");
 253     } else {
 254       app.statusUpdate(MessageFormat.format(
 255           "Precompiling... {0} file(s) left.", filesLeft));
 256     }
 257
 258     List<String> subset =
 259         filesToCompile
 260             .subList(0, Math.min(filesLeft, MAX_FILES_PER_PRECOMPILE));
 261     List<String> remainingFiles = sendPrecompileRequest(subset);
 262     subset.clear();
 263     filesToCompile.addAll(remainingFiles);
 264     return filesToCompile.size() < filesLeft;
 265   }
 266
 267   private List<String> sendPrecompileRequest(List<String> filesToCompile)
 268       throws IOException {
 269     String response =
 270         send("/api/appversion/precompile", Join.join("\n", filesToCompile));
 271     if (response.length() > 0) {
 272       return Arrays.asList(response.split("\n"));
 273     } else {
 274       return Collections.emptyList();
 275     }
 276   }
 277
 278   public void updateIndexes() throws IOException {
 279     if (app.getIndexesXml() != null) {
 280       app.statusUpdate("Uploading index definitions.");
 281       send("/api/datastore/index/add", getIndexYaml());
 282     }
 283
 284   }
 285
 286   public void updateCron() throws IOException {
 287     String yaml = getCronYaml();
 288     if (yaml != null) {
 289       app.statusUpdate("Uploading cron jobs.");
 290       send("/api/datastore/cron/update", yaml);
 291     }
 292   }
 293
 294   public void updateQueue() throws IOException {
 295     String yaml = getQueueYaml();
 296     if (yaml != null) {
 297       app.statusUpdate("Uploading task queues.");
 298       send("/api/queue/update", yaml);
 299     }
 300   }
 301
 302   public void updateDos() throws IOException {
 303     String yaml = getDosYaml();
 304     if (yaml != null) {
 305       app.statusUpdate("Uploading DoS entries.");
 306       send("/api/dos/update", yaml);
 307     }
 308   }
 309
 310   public void updatePagespeed() throws IOException {
 311     String yaml = getPagespeedYaml();
 312     if (yaml != null) {
 313       app.statusUpdate("Uploading PageSpeed entries.");
 314       send("/api/appversion/updatepagespeed", yaml);
 315     } else {
 316       try {
 317         send("/api/appversion/updatepagespeed", "");
 318       } catch (HttpIoException exc) {
 319         if (exc.getResponseCode() != HttpURLConnection.HTTP_NOT_FOUND) {
 320           throw exc;
 321         }
 322       }
 323     }
 324   }
 325
 326   public void setDefaultVersion() throws IOException {
 327     app.statusUpdate("Setting default version to " + app.getVersion() + ".");
 328     send("/api/appversion/setdefault", "");
 329   }
 330
 331   protected String getIndexYaml() {
 332     return app.getIndexesXml().toYaml();
 333   }
 334
 335   protected String getCronYaml() {
 336     if (app.getCronXml() != null) {
 337       return app.getCronXml().toYaml();
 338     } else {
 339       return null;
 340     }
 341   }
 342
 343   protected String getQueueYaml() {
 344     if (app.getQueueXml() != null) {
 345       return app.getQueueXml().toYaml();
 346     } else {
 347       return null;
 348     }
 349   }
 350
 351   protected String getDosYaml() {
 352     if (app.getDosXml() != null) {
 353       return app.getDosXml().toYaml();
 354     } else {
 355       return null;
 356     }
 357   }
 358
 359   protected String getPagespeedYaml() {
 360     return app.getPagespeedYaml();
 361   }
 362
 363   private File getBasepath() {
 364     File path = app.getStagingDir();
 365     if (path == null) {
 366       path = new File(app.getPath());
 367     }
 368     return path;
 369   }
 370
 371   /**
 372    * Adds a file for uploading, returning the bytes counted against the total
 373    * resource quota.
 374    *
 375    * @param info
 376    * @return 0 for a static file, or file.length() for a resource file.
 377    * @throws IOException
 378    */
 379   private long addFile(FileInfo info) throws IOException {
 380     if (inTransaction) {
 381       throw new IllegalStateException("Already in a transaction.");
 382     }
 383
 384     String error = info.checkValidFilename();
 385     if (error != null) {
 386       logger.severe(error);
 387       return 0;
 388     }
 389
 390     files.put(info.path, info);
 391
 392     return info.mimeType != null ? 0 : info.file.length();
 393   }
 394
 395   /**
 396    * Begins the transaction, returning a list of files that need uploading.
 397    *
 398    * All calls to addFile must be made before calling beginTransaction().
 399    *
 400    * @param resourceLimits is the collection of resource limits for AppCfg.
 401    * @return A list of pathnames that should be uploaded using uploadFile()
 402    *         before calling commit().
 403    */
 404   private Collection<FileInfo> beginTransaction(ResourceLimits resourceLimits)
 405       throws IOException {
 406     if (inTransaction) {
 407       throw new IllegalStateException("Already in a transaction.");
 408     }
 409
 410     if (backend == null) {
 411       app.statusUpdate("Initiating update.");
 412     } else {
 413       app.statusUpdate("Initiating update of backend " + backend + ".");
 414     }
 415     send("/api/appversion/create", app.getAppYaml());
 416     inTransaction = true;
 417     Collection<FileInfo> blobsToClone = new ArrayList<FileInfo>(files.size());
 418     Collection<FileInfo> filesToClone = new ArrayList<FileInfo>(files.size());
 419
 420     for (FileInfo f : files.values()) {
 421       if (f.mimeType == null) {
 422         filesToClone.add(f);
 423       } else {
 424         blobsToClone.add(f);
 425       }
 426     }
 427
 428     TreeMap<String, FileInfo> filesToUpload = new TreeMap<String, FileInfo>();
 429     cloneFiles("/api/appversion/cloneblobs", blobsToClone, "static",
 430         filesToUpload, resourceLimits.maxFilesToClone());
 431     cloneFiles("/api/appversion/clonefiles", filesToClone, "application",
 432         filesToUpload, resourceLimits.maxFilesToClone());
 433
 434     logger.fine("Files to upload :");
 435     for (FileInfo f : filesToUpload.values()) {
 436       logger.fine("\t" + f);
 437     }
 438
 439     this.files = filesToUpload;
 440     return new ArrayList<FileInfo>(filesToUpload.values());
 441   }
 442
 443   private static final String LIST_DELIMITER = "\n";
 444
 445   /**
 446    * Sends files to the given url.
 447    *
 448    * @param url server URL to use.
 449    * @param filesParam List of files to clone.
 450    * @param type Type of files ( "static" or "application")
 451    * @param filesToUpload Files that need to be uploaded are added to this
 452    *        Collection.
 453    * @param maxFilesToClone Max number of files to clone at a single time.
 454    */
 455   private void cloneFiles(String url, Collection<FileInfo> filesParam,
 456       String type, Map<String, FileInfo> filesToUpload, long maxFilesToClone)
 457       throws IOException {
 458     if (filesParam.isEmpty()) {
 459       return;
 460     }
 461     app.statusUpdate("Cloning " + filesParam.size() + " " + type + " files.");
 462
 463     int cloned = 0;
 464     int remaining = filesParam.size();
 465     ArrayList<FileInfo> chunk = new ArrayList<FileInfo>((int) maxFilesToClone);
 466     for (FileInfo file : filesParam) {
 467       chunk.add(file);
 468       if (--remaining == 0 || chunk.size() >= maxFilesToClone) {
 469         if (cloned > 0) {
 470           app.statusUpdate("Cloned " + cloned + " files.");
 471         }
 472         String result = send(url, buildClonePayload(chunk));
 473         if (result != null && result.length() > 0) {
 474           for (String path : result.split(LIST_DELIMITER)) {
 475             if (path == null || path.length() == 0) {
 476               continue;
 477             }
 478             FileInfo info = this.files.get(path);
 479             if (info == null) {
 480               logger.warning("Skipping " + path + ": missing FileInfo");
 481               continue;
 482             }
 483             filesToUpload.put(path, info);
 484           }
 485         }
 486         cloned += chunk.size();
 487         chunk.clear();
 488       }
 489     }
 490   }
 491
 492   /**
 493    * Uploads a file to the hosting service.
 494    *
 495    * Must only be called after beginTransaction(). The file provided must be on
 496    * of those that were returned by beginTransaction();
 497    *
 498    * @param file FileInfo for the file to upload.
 499    */
 500   private void uploadFile(FileInfo file) throws IOException {
 501     if (!inTransaction) {
 502       throw new IllegalStateException(
 503           "beginTransaction() must be called before uploadFile().");
 504     }
 505     if (!files.containsKey(file.path)) {
 506       throw new IllegalArgumentException("File " + file.path
 507           + " is not in the list of files to be uploaded.");
 508     }
 509
 510     files.remove(file.path);
 511     if (file.mimeType == null) {
 512       fileBatcher.addToBatch(file);
 513     } else {
 514       blobBatcher.addToBatch(file);
 515     }
 516   }
 517
 518   /**
 519    * Commits the transaction, making the new app version available.
 520    *
 521    * All the files returned by beginTransaction must have been uploaded with
 522    * uploadFile() before commit() may be called.
 523    */
 524   private void commit() throws IOException {
 525     deploy();
 526     try {
 527       boolean ready = retryWithBackoff(1, 2, 60, 20, new Callable<Boolean>() {
 528         @Override
 529         public Boolean call() throws Exception {
 530           return isReady();
 531         }
 532       });
 533
 534       if (ready) {
 535         startServing();
 536       } else {
 537         logger.severe("Version still not ready to serve, aborting.");
 538         throw new RuntimeException("Version not ready.");
 539       }
 540     } catch (IOException ioe) {
 541       throw ioe;
 542     } catch (RuntimeException e) {
 543       throw e;
 544     } catch (Exception e) {
 545       throw new RuntimeException(e);
 546     }
 547   }
 548
 549   /**
 550    * Deploys the new app version but does not make it default.
 551    *
 552    * All the files returned by beginTransaction must have been uploaded with
 553    * uploadFile() before commit() may be called.
 554    */
 555   private void deploy() throws IOException {
 556     if (!inTransaction) {
 557       throw new IllegalStateException(
 558           "beginTransaction() must be called before uploadFile().");
 559     }
 560     if (files.size() > 0) {
 561       throw new IllegalStateException(
 562           "Some required files have not been uploaded.");
 563     }
 564     app.statusUpdate("Deploying new version.", 20);
 565     send("/api/appversion/deploy", "");
 566     deployed = true;
 567   }
 568
 569   /**
 570    * Check if the new app version is ready to serve traffic.
 571    *
 572    * @return true if the server returned that the app is ready to serve.
 573    */
 574   private boolean isReady() throws IOException {
 575     if (!deployed) {
 576       throw new IllegalStateException(
 577           "deploy() must be called before isReady()");
 578     }
 579     String result = send("/api/appversion/isready", "");
 580     return "1".equals(result.trim());
 581   }
 582
 583   private void startServing() throws IOException {
 584     if (!deployed) {
 585       throw new IllegalStateException(
 586           "deploy() must be called before startServing()");
 587     }
 588     app.statusUpdate("Closing update: new version is ready to start serving.");
 589     send("/api/appversion/startserving", "");
 590     inTransaction = false;
 591   }
 592
 593   public void forceRollback() throws IOException {
 594     app.statusUpdate("Rolling back the update" + this.backend == null ? "."
 595         : " on backend " + this.backend + ".");
 596     send("/api/appversion/rollback", "");
 597   }
 598
 599   private void rollback() throws IOException {
 600     if (!inTransaction) {
 601       return;
 602     }
 603     forceRollback();
 604   }
 605
 606   @VisibleForTesting
 607   String send(String url, String payload, String... args)
 608       throws IOException {
 609     return connection.post(url, payload, addVersionToArgs(args));
 610   }
 611
 612   @VisibleForTesting
 613   String send(String url, File payload, String mimeType, String... args)
 614       throws IOException {
 615     return connection.post(url, payload, mimeType, addVersionToArgs(args));
 616   }
 617
 618   private String[] addVersionToArgs(String... args) {
 619     List<String> result = new ArrayList<String>();
 620     result.addAll(Arrays.asList(args));
 621     result.add("app_id");
 622     result.add(app.getAppId());
 623     if (backend != null) {
 624       result.add("backend");
 625       result.add(backend);
 626     } else if (app.getVersion() != null) {
 627       result.add("version");
 628       result.add(app.getVersion());
 629     }
 630     if (app.getServer() != null) {
 631       result.add("server");
 632       result.add(app.getServer());
 633     }
 634     return result.toArray(new String[result.size()]);
 635   }
 636
 637   /**
 638    * Calls a function multiple times, backing off more and more each time.
 639    *
 640    * @param initialDelay Inital delay after the first try, in seconds.
 641    * @param backoffFactor Delay will be multiplied by this factor after each
 642    *        try.
 643    * @param maxDelay Maximum delay factor.
 644    * @param maxTries Maximum number of tries.
 645    * @param callable Callable to call.
 646    * @return true if the Callable returned true in one of its tries.
 647    */
 648   private boolean retryWithBackoff(double initialDelay, double backoffFactor,
 649       double maxDelay, int maxTries, Callable<Boolean> callable)
 650       throws Exception {
 651     long delayMillis = (long) (initialDelay * 1000);
 652     long maxDelayMillis = (long) (maxDelay * 1000);
 653     if (callable.call()) {
 654       return true;
 655     }
 656     while (maxTries > 1) {
 657       app.statusUpdate("Will check again in " + (delayMillis / 1000)
 658           + " seconds.");
 659       Thread.sleep(delayMillis);
 660       delayMillis *= backoffFactor;
 661       if (delayMillis > maxDelayMillis) {
 662         delayMillis = maxDelayMillis;
 663       }
 664       maxTries--;
 665       if (callable.call()) {
 666         return true;
 667       }
 668     }
 669     return false;
 670   }
 671
 672   private static final String TUPLE_DELIMITER = "|";
 673
 674   /**
 675    * Build the post body for a clone request.
 676    *
 677    * @param files List of FileInfos for the files to clone.
 678    * @return A string containing the properly delimited tuples.
 679    */
 680   private static String buildClonePayload(Collection<FileInfo> files) {
 681     StringBuffer data = new StringBuffer();
 682     boolean first = true;
 683     for (FileInfo file : files) {
 684       if (first) {
 685         first = false;
 686       } else {
 687         data.append(LIST_DELIMITER);
 688       }
 689       data.append(file.path);
 690       data.append(TUPLE_DELIMITER);
 691       data.append(file.hash);
 692       if (file.mimeType != null) {
 693         data.append(TUPLE_DELIMITER);
 694         data.append(file.mimeType);
 695       }
 696     }
 697
 698     return data.toString();
 699   }
 700
 701   static class FileInfo implements Comparable<FileInfo> {
 702     public File file;
 703     public String path;
 704     public String hash;
 705     public String mimeType;
 706
 707     public FileInfo(File f, File base) throws IOException {
 708       this.file = f;
 709       this.path = Utility.calculatePath(f, base);
 710       this.hash = calculateHash();
 711     }
 712
 713     public void setMimeType(GenericApplication app) {
 714       mimeType = app.getMimeTypeIfStatic(path);
 715     }
 716
 717     @Override
 718     public String toString() {
 719       return (mimeType == null ? "" : mimeType) + '\t' + hash + "\t" + path;
 720     }
 721
 722     @Override
 723     public int compareTo(FileInfo other) {
 724       return path.compareTo(other.path);
 725     }
 726
 727     @Override
 728     public int hashCode() {
 729       return path.hashCode();
 730     }
 731
 732     @Override
 733     public boolean equals(Object obj) {
 734       if (obj instanceof FileInfo) {
 735         return path.equals(((FileInfo) obj).path);
 736       }
 737       return false;
 738     }
 739
 740     private static final Pattern FILE_PATH_POSITIVE_RE =
 741         Pattern.compile("^[ 0-9a-zA-Z._+/$-]{1,256}$");
 742
 743     private static final Pattern FILE_PATH_NEGATIVE_RE_1 =
 744         Pattern.compile("[.][.]|^[.]/|[.]$|/[.]/|^-");
 745
 746     private static final Pattern FILE_PATH_NEGATIVE_RE_2 =
 747         Pattern.compile("//|/$");
 748
 749     private static final Pattern FILE_PATH_NEGATIVE_RE_3 =
 750         Pattern.compile("^ | $|/ | /");
 751
 752     private String checkValidFilename() {
 753       if (!FILE_PATH_POSITIVE_RE.matcher(path).matches()) {
 754         return "Invalid character in filename: " + path;
 755       }
 756       if (FILE_PATH_NEGATIVE_RE_1.matcher(path).find()) {
 757         return "Filname cannot contain '.' or '..' or start with '-': " + path;
 758       }
 759       if (FILE_PATH_NEGATIVE_RE_2.matcher(path).find()) {
 760         return "Filname cannot have trailing / or contain //: " + path;
 761       }
 762       if (FILE_PATH_NEGATIVE_RE_3.matcher(path).find()) {
 763         return "Any spaces must be in the middle of a filename: '" + path + "'";
 764       }
 765       return null;
 766     }
 767
 768     private static final char[] HEX =
 769         {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd',
 770             'e', 'f'};
 771
 772     public String calculateHash() throws IOException {
 773       InputStream s = new FileInputStream(file);
 774       byte[] buf = new byte[4096];
 775       try {
 776         MessageDigest digest = MessageDigest.getInstance("SHA-1");
 777         for (int numRead; (numRead = s.read(buf)) != -1;) {
 778           digest.update(buf, 0, numRead);
 779         }
 780         StringBuffer hashValue = new StringBuffer(40);
 781         int i = 0;
 782         for (byte b : digest.digest()) {
 783           if ((i > 0) && ((i % 4) == 0)) {
 784             hashValue.append('_');
 785           }
 786           hashValue.append(HEX[(b >> 4) & 0xf]);
 787           hashValue.append(HEX[b & 0xf]);
 788           ++i;
 789         }
 790
 791         return hashValue.toString();
 792       } catch (NoSuchAlgorithmException e) {
 793         throw new RuntimeException(e);
 794       } finally {
 795         try {
 796           s.close();
 797         } catch (IOException ex) {
 798           ;
 799         }
 800       }
 801     }
 802   }
 803
 804   class UploadBatcher {
 805
 806     private static final int MAX_BATCH_SIZE = 3200000;
 807     private static final int MAX_BATCH_COUNT = 100;
 808     private static final int MAX_BATCH_FILE_SIZE = 200000;
 809     private static final int BATCH_OVERHEAD = 500;
 810
 811     String what;
 812     String singleUrl;
 813     String batchUrl;
 814     boolean batching = true;
 815     List<FileInfo> batch = new ArrayList<FileInfo>();
 816     long batchSize = 0;
 817
 818     /**
 819      * @param what     Either "file" or "blob" or "errorblob" indicating what kind of objects this
 820      *                 batcher uploads. Used in messages and URLs.
 821      * @param batching whether or not we want to really do batch.
 822      */
 823     public UploadBatcher(String what, boolean batching) {
 824       this.what = what;
 825       this.singleUrl = "/api/appversion/add" + what;
 826       this.batchUrl = singleUrl + "s";
 827       this.batching = batching;
 828     }
 829
 830     /**
 831      * Send the current batch on its way and reset the batrch buffer when done
 832      */
 833     public void sendBatch() throws IOException {
 834
 835       app.statusUpdate(
 836           "Sending batch containing " + batch.size() + " "+ what +"(s) totaling " +
 837               batchSize / 1000 + "KB.");
 838       connection.post(batchUrl, batch, addVersionToArgs("", ""));
 839       batch = new ArrayList<FileInfo>();
 840       batchSize = 0;
 841     }
 842
 843     /**
 844      * """Flush the current batch.
 845      *
 846      * This first attempts to send the batch as a single request; if that fails because the server
 847      * doesn"t support batching, the files are sent one by one, and self.batching is reset to
 848      * False.
 849      *
 850      * At the end, self.batch and self.batchSize are reset
 851      */
 852     public void flush() throws IOException {
 853       if (batch.isEmpty()) {
 854         return;
 855       }
 856       try {
 857         sendBatch();
 858       } catch (Exception e) {
 859         app.statusUpdate("Exception in flushing batch payload, so sending 1 by 1..."
 860             + e.getMessage());
 861         batching = false;
 862         for (FileInfo fileInfo : batch) {
 863           send(singleUrl, fileInfo.file, fileInfo.mimeType, "path", fileInfo.path);
 864         }
 865         batch = new ArrayList<FileInfo>();
 866         batchSize = 0;
 867       }
 868     }
 869
 870     /**
 871      * Batch a file, possibly flushing first, or perhaps upload it directly.
 872      *
 873      * Args: path: The name of the file. payload: The contents of the file. mime_type: The MIME
 874      * Content-type of the file, or None.
 875      *
 876      * If mime_type is None, application/octet-stream is substituted. """
 877      */
 878     public void addToBatch(FileInfo fileInfo) throws IOException {
 879
 880       long size = fileInfo.file.length();
 881
 882       if (size <= MAX_BATCH_FILE_SIZE) {
 883         if ((batch.size() >= MAX_BATCH_COUNT) ||
 884             (batchSize + size > MAX_BATCH_SIZE)) {
 885           flush();
 886         }
 887         if (batching) {
 888           batch.add(fileInfo);
 889           batchSize += size + BATCH_OVERHEAD;
 890           return;
 891         }
 892       }
 893       send(singleUrl, fileInfo.file, fileInfo.mimeType, "path", fileInfo.path);
 894
 895     }
 896   }
 897
 898 }