org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java

   1 /*
   2  * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
   3  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   4  *
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  * - Redistributions of source code must retain the above copyright
  12  *   notice, this list of conditions and the following disclaimer.
  13  *
  14  * - Redistributions in binary form must reproduce the above
  15  *   copyright notice, this list of conditions and the following
  16  *   disclaimer in the documentation and/or other materials provided
  17  *   with the distribution.
  18  *
  19  * - Neither the name of the Git Development Community nor the
  20  *   names of its contributors may be used to endorse or promote
  21  *   products derived from this software without specific prior
  22  *   written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  25  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  26  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  31  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 package org.spearce.jgit.transport;
  40
  41 import java.io.EOFException;
  42 import java.io.File;
  43 import java.io.FileOutputStream;
  44 import java.io.IOException;
  45 import java.io.InputStream;
  46 import java.io.RandomAccessFile;
  47 import java.security.MessageDigest;
  48 import java.util.ArrayList;
  49 import java.util.Arrays;
  50 import java.util.HashMap;
  51 import java.util.List;
  52 import java.util.zip.CRC32;
  53 import java.util.zip.DataFormatException;
  54 import java.util.zip.Deflater;
  55 import java.util.zip.Inflater;
  56
  57 import org.spearce.jgit.errors.CorruptObjectException;
  58 import org.spearce.jgit.errors.MissingObjectException;
  59 import org.spearce.jgit.lib.BinaryDelta;
  60 import org.spearce.jgit.lib.Constants;
  61 import org.spearce.jgit.lib.InflaterCache;
  62 import org.spearce.jgit.lib.MutableObjectId;
  63 import org.spearce.jgit.lib.ObjectId;
  64 import org.spearce.jgit.lib.ObjectIdMap;
  65 import org.spearce.jgit.lib.ObjectLoader;
  66 import org.spearce.jgit.lib.PackIndexWriter;
  67 import org.spearce.jgit.lib.ProgressMonitor;
  68 import org.spearce.jgit.lib.Repository;
  69 import org.spearce.jgit.util.NB;
  70
  71 /** Indexes Git pack files for local use. */
  72 public class IndexPack {
  73         /** Progress message when reading raw data from the pack. */
  74         public static final String PROGRESS_DOWNLOAD = "Receiving objects";
  75
  76         /** Progress message when computing names of delta compressed objects. */
  77         public static final String PROGRESS_RESOLVE_DELTA = "Resolving deltas";
  78
  79         /**
  80          * Size of the internal stream buffer.
  81          * <p>
  82          * If callers are going to be supplying IndexPack a BufferedInputStream they
  83          * should use this buffer size as the size of the buffer for that
  84          * BufferedInputStream, and any other its may be wrapping. This way the
  85          * buffers will cascade efficiently and only the IndexPack buffer will be
  86          * receiving the bulk of the data stream.
  87          */
  88         public static final int BUFFER_SIZE = 8192;
  89
  90         /**
  91          * Create an index pack instance to load a new pack into a repository.
  92          * <p>
  93          * The received pack data and generated index will be saved to temporary
  94          * files within the repository's <code>objects</code> directory. To use the
  95          * data contained within them call {@link #renameAndOpenPack()} once the
  96          * indexing is complete.
  97          *
  98          * @param db
  99          *            the repository that will receive the new pack.
 100          * @param is
 101          *            stream to read the pack data from. If the stream is buffered
 102          *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
 103          * @return a new index pack instance.
 104          * @throws IOException
 105          *             a temporary file could not be created.
 106          */
 107         public static IndexPack create(final Repository db, final InputStream is)
 108                         throws IOException {
 109                 final String suffix = ".pack";
 110                 final File objdir = db.getObjectsDirectory();
 111                 final File tmp = File.createTempFile("incoming_", suffix, objdir);
 112                 final String n = tmp.getName();
 113                 final File base;
 114
 115                 base = new File(objdir, n.substring(0, n.length() - suffix.length()));
 116                 final IndexPack ip = new IndexPack(db, is, base);
 117                 ip.setIndexVersion(db.getConfig().getCore().getPackIndexVersion());
 118                 return ip;
 119         }
 120
 121         private final Repository repo;
 122
 123         private Inflater inflater;
 124
 125         private final MessageDigest objectDigest;
 126
 127         private final MutableObjectId tempObjectId;
 128
 129         private InputStream in;
 130
 131         private byte[] buf;
 132
 133         private long bBase;
 134
 135         private int bOffset;
 136
 137         private int bAvail;
 138
 139         private boolean fixThin;
 140
 141         private int outputVersion;
 142
 143         private final File dstPack;
 144
 145         private final File dstIdx;
 146
 147         private long objectCount;
 148
 149         private PackedObjectInfo[] entries;
 150
 151         private int deltaCount;
 152
 153         private int entryCount;
 154
 155         private final CRC32 crc = new CRC32();
 156
 157         private ObjectIdMap<ArrayList<UnresolvedDelta>> baseById;
 158
 159         private HashMap<Long, ArrayList<UnresolvedDelta>> baseByPos;
 160
 161         private byte[] objectData;
 162
 163         private MessageDigest packDigest;
 164
 165         private RandomAccessFile packOut;
 166
 167         private byte[] packcsum;
 168
 169         /** If {@link #fixThin} this is the last byte of the original checksum. */
 170         private long originalEOF;
 171
 172         /**
 173          * Create a new pack indexer utility.
 174          *
 175          * @param db
 176          * @param src
 177          *            stream to read the pack data from. If the stream is buffered
 178          *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
 179          * @param dstBase
 180          * @throws IOException
 181          *             the output packfile could not be created.
 182          */
 183         public IndexPack(final Repository db, final InputStream src,
 184                         final File dstBase) throws IOException {
 185                 repo = db;
 186                 in = src;
 187                 inflater = InflaterCache.get();
 188                 buf = new byte[BUFFER_SIZE];
 189                 objectData = new byte[BUFFER_SIZE];
 190                 objectDigest = Constants.newMessageDigest();
 191                 tempObjectId = new MutableObjectId();
 192                 packDigest = Constants.newMessageDigest();
 193
 194                 if (dstBase != null) {
 195                         final File dir = dstBase.getParentFile();
 196                         final String nam = dstBase.getName();
 197                         dstPack = new File(dir, nam + ".pack");
 198                         dstIdx = new File(dir, nam + ".idx");
 199                         packOut = new RandomAccessFile(dstPack, "rw");
 200                         packOut.setLength(0);
 201                 } else {
 202                         dstPack = null;
 203                         dstIdx = null;
 204                 }
 205         }
 206
 207         /**
 208          * Set the pack index file format version this instance will create.
 209          *
 210          * @param version
 211          *            the version to write. The special version 0 designates the
 212          *            oldest (most compatible) format available for the objects.
 213          * @see PackIndexWriter
 214          */
 215         public void setIndexVersion(final int version) {
 216                 outputVersion = version;
 217         }
 218
 219         /**
 220          * Configure this index pack instance to make a thin pack complete.
 221          * <p>
 222          * Thin packs are sometimes used during network transfers to allow a delta
 223          * to be sent without a base object. Such packs are not permitted on disk.
 224          * They can be fixed by copying the base object onto the end of the pack.
 225          *
 226          * @param fix
 227          *            true to enable fixing a thin pack.
 228          */
 229         public void setFixThin(final boolean fix) {
 230                 fixThin = fix;
 231         }
 232
 233         /**
 234          * Consume data from the input stream until the packfile is indexed.
 235          *
 236          * @param progress
 237          *            progress feedback
 238          *
 239          * @throws IOException
 240          */
 241         public void index(final ProgressMonitor progress) throws IOException {
 242                 progress.start(2 /* tasks */);
 243                 try {
 244                         try {
 245                                 readPackHeader();
 246
 247                                 entries = new PackedObjectInfo[(int) objectCount];
 248                                 baseById = new ObjectIdMap<ArrayList<UnresolvedDelta>>();
 249                                 baseByPos = new HashMap<Long, ArrayList<UnresolvedDelta>>();
 250
 251                                 progress.beginTask(PROGRESS_DOWNLOAD, (int) objectCount);
 252                                 for (int done = 0; done < objectCount; done++) {
 253                                         indexOneObject();
 254                                         progress.update(1);
 255                                         if (progress.isCancelled())
 256                                                 throw new IOException("Download cancelled");
 257                                 }
 258                                 readPackFooter();
 259                                 endInput();
 260                                 progress.endTask();
 261                                 if (deltaCount > 0) {
 262                                         if (packOut == null)
 263                                                 throw new IOException("need packOut");
 264                                         resolveDeltas(progress);
 265                                         if (entryCount < objectCount) {
 266                                                 if (!fixThin) {
 267                                                         throw new IOException("pack has "
 268                                                                         + (objectCount - entryCount)
 269                                                                         + " unresolved deltas");
 270                                                 }
 271                                                 fixThinPack(progress);
 272                                         }
 273                                 }
 274                                 if (packOut != null)
 275                                         packOut.getChannel().force(true);
 276
 277                                 packDigest = null;
 278                                 baseById = null;
 279                                 baseByPos = null;
 280
 281                                 if (dstIdx != null)
 282                                         writeIdx();
 283
 284                         } finally {
 285                                 final Inflater inf = inflater;
 286                                 inflater = null;
 287                                 InflaterCache.release(inf);
 288
 289                                 progress.endTask();
 290                                 if (packOut != null)
 291                                         packOut.close();
 292                         }
 293
 294                         if (dstPack != null)
 295                                 dstPack.setReadOnly();
 296                         if (dstIdx != null)
 297                                 dstIdx.setReadOnly();
 298                 } catch (IOException err) {
 299                         if (dstPack != null)
 300                                 dstPack.delete();
 301                         if (dstIdx != null)
 302                                 dstIdx.delete();
 303                         throw err;
 304                 }
 305         }
 306
 307         private void resolveDeltas(final ProgressMonitor progress)
 308                         throws IOException {
 309                 progress.beginTask(PROGRESS_RESOLVE_DELTA, deltaCount);
 310                 final int last = entryCount;
 311                 for (int i = 0; i < last; i++) {
 312                         final int before = entryCount;
 313                         resolveDeltas(entries[i]);
 314                         progress.update(entryCount - before);
 315                         if (progress.isCancelled())
 316                                 throw new IOException("Download cancelled during indexing");
 317                 }
 318                 progress.endTask();
 319         }
 320
 321         private void resolveDeltas(final PackedObjectInfo oe) throws IOException {
 322                 final int oldCRC = oe.getCRC();
 323                 if (baseById.containsKey(oe)
 324                                 || baseByPos.containsKey(new Long(oe.getOffset())))
 325                         resolveDeltas(oe.getOffset(), oldCRC, Constants.OBJ_BAD, null, oe);
 326         }
 327
 328         private void resolveDeltas(final long pos, final int oldCRC, int type,
 329                         byte[] data, PackedObjectInfo oe) throws IOException {
 330                 crc.reset();
 331                 position(pos);
 332                 int c = readFromFile();
 333                 final int typeCode = (c >> 4) & 7;
 334                 long sz = c & 15;
 335                 int shift = 4;
 336                 while ((c & 0x80) != 0) {
 337                         c = readFromFile();
 338                         sz += (c & 0x7f) << shift;
 339                         shift += 7;
 340                 }
 341
 342                 switch (typeCode) {
 343                 case Constants.OBJ_COMMIT:
 344                 case Constants.OBJ_TREE:
 345                 case Constants.OBJ_BLOB:
 346                 case Constants.OBJ_TAG:
 347                         type = typeCode;
 348                         data = inflateFromFile((int) sz);
 349                         break;
 350                 case Constants.OBJ_OFS_DELTA: {
 351                         c = readFromFile() & 0xff;
 352                         while ((c & 128) != 0)
 353                                 c = readFromFile() & 0xff;
 354                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 355                         break;
 356                 }
 357                 case Constants.OBJ_REF_DELTA: {
 358                         crc.update(buf, fillFromFile(20), 20);
 359                         use(20);
 360                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 361                         break;
 362                 }
 363                 default:
 364                         throw new IOException("Unknown object type " + typeCode + ".");
 365                 }
 366
 367                 final int crc32 = (int) crc.getValue();
 368                 if (oldCRC != crc32)
 369                         throw new IOException("Corruption detected re-reading at " + pos);
 370                 if (oe == null) {
 371                         objectDigest.update(Constants.encodedTypeString(type));
 372                         objectDigest.update((byte) ' ');
 373                         objectDigest.update(Constants.encodeASCII(data.length));
 374                         objectDigest.update((byte) 0);
 375                         objectDigest.update(data);
 376                         tempObjectId.fromRaw(objectDigest.digest(), 0);
 377
 378                         verifyNoCollision(type, data);
 379                         oe = new PackedObjectInfo(pos, crc32, tempObjectId);
 380                         entries[entryCount++] = oe;
 381                 }
 382
 383                 resolveChildDeltas(pos, type, data, oe);
 384         }
 385
 386         private void resolveChildDeltas(final long pos, int type, byte[] data,
 387                         PackedObjectInfo oe) throws IOException {
 388                 final ArrayList<UnresolvedDelta> a = baseById.remove(oe);
 389                 final ArrayList<UnresolvedDelta> b = baseByPos.remove(new Long(pos));
 390                 int ai = 0, bi = 0;
 391                 if (a != null && b != null) {
 392                         while (ai < a.size() && bi < b.size()) {
 393                                 final UnresolvedDelta ad = a.get(ai);
 394                                 final UnresolvedDelta bd = b.get(bi);
 395                                 if (ad.position < bd.position) {
 396                                         resolveDeltas(ad.position, ad.crc, type, data, null);
 397                                         ai++;
 398                                 } else {
 399                                         resolveDeltas(bd.position, bd.crc, type, data, null);
 400                                         bi++;
 401                                 }
 402                         }
 403                 }
 404                 if (a != null)
 405                         while (ai < a.size()) {
 406                                 final UnresolvedDelta ad = a.get(ai++);
 407                                 resolveDeltas(ad.position, ad.crc, type, data, null);
 408                         }
 409                 if (b != null)
 410                         while (bi < b.size()) {
 411                                 final UnresolvedDelta bd = b.get(bi++);
 412                                 resolveDeltas(bd.position, bd.crc, type, data, null);
 413                         }
 414         }
 415
 416         private void fixThinPack(final ProgressMonitor progress) throws IOException {
 417                 growEntries();
 418
 419                 packDigest.reset();
 420                 originalEOF = packOut.length() - 20;
 421                 final Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
 422                 long end = originalEOF;
 423                 for (final ObjectId baseId : new ArrayList<ObjectId>(baseById.keySet())) {
 424                         final ObjectLoader ldr = repo.openObject(baseId);
 425                         if (ldr == null)
 426                                 continue;
 427                         final byte[] data = ldr.getBytes();
 428                         final int typeCode = ldr.getType();
 429                         final PackedObjectInfo oe;
 430
 431                         crc.reset();
 432                         packOut.seek(end);
 433                         writeWhole(def, typeCode, data);
 434                         oe = new PackedObjectInfo(end, (int) crc.getValue(), baseId);
 435                         entries[entryCount++] = oe;
 436                         end = packOut.getFilePointer();
 437
 438                         resolveChildDeltas(oe.getOffset(), typeCode, data, oe);
 439                         if (progress.isCancelled())
 440                                 throw new IOException("Download cancelled during indexing");
 441                 }
 442                 def.end();
 443
 444                 if (!baseById.isEmpty()) {
 445                         final ObjectId need = baseById.keySet().iterator().next();
 446                         throw new MissingObjectException(need, "delta base");
 447                 }
 448
 449                 fixHeaderFooter(packcsum, packDigest.digest());
 450         }
 451
 452         private void writeWhole(final Deflater def, final int typeCode,
 453                         final byte[] data) throws IOException {
 454                 int sz = data.length;
 455                 int hdrlen = 0;
 456                 buf[hdrlen++] = (byte) ((typeCode << 4) | sz & 15);
 457                 sz >>>= 4;
 458                 while (sz > 0) {
 459                         buf[hdrlen - 1] |= 0x80;
 460                         buf[hdrlen++] = (byte) (sz & 0x7f);
 461                         sz >>>= 7;
 462                 }
 463                 packDigest.update(buf, 0, hdrlen);
 464                 crc.update(buf, 0, hdrlen);
 465                 packOut.write(buf, 0, hdrlen);
 466                 def.reset();
 467                 def.setInput(data);
 468                 def.finish();
 469                 while (!def.finished()) {
 470                         final int datlen = def.deflate(buf);
 471                         packDigest.update(buf, 0, datlen);
 472                         crc.update(buf, 0, datlen);
 473                         packOut.write(buf, 0, datlen);
 474                 }
 475         }
 476
 477         private void fixHeaderFooter(final byte[] origcsum, final byte[] tailcsum)
 478                         throws IOException {
 479                 final MessageDigest origDigest = Constants.newMessageDigest();
 480                 final MessageDigest tailDigest = Constants.newMessageDigest();
 481                 long origRemaining = originalEOF;
 482
 483                 packOut.seek(0);
 484                 bAvail = 0;
 485                 bOffset = 0;
 486                 fillFromFile(12);
 487
 488                 {
 489                         final int origCnt = (int) Math.min(bAvail, origRemaining);
 490                         origDigest.update(buf, 0, origCnt);
 491                         origRemaining -= origCnt;
 492                         if (origRemaining == 0)
 493                                 tailDigest.update(buf, origCnt, bAvail - origCnt);
 494                 }
 495
 496                 NB.encodeInt32(buf, 8, entryCount);
 497                 packOut.seek(0);
 498                 packOut.write(buf, 0, 12);
 499                 packOut.seek(bAvail);
 500
 501                 packDigest.reset();
 502                 packDigest.update(buf, 0, bAvail);
 503                 for (;;) {
 504                         final int n = packOut.read(buf);
 505                         if (n < 0)
 506                                 break;
 507                         if (origRemaining != 0) {
 508                                 final int origCnt = (int) Math.min(n, origRemaining);
 509                                 origDigest.update(buf, 0, origCnt);
 510                                 origRemaining -= origCnt;
 511                                 if (origRemaining == 0)
 512                                         tailDigest.update(buf, origCnt, n - origCnt);
 513                         } else
 514                                 tailDigest.update(buf, 0, n);
 515
 516                         packDigest.update(buf, 0, n);
 517                 }
 518
 519                 if (!Arrays.equals(origDigest.digest(), origcsum)
 520                                 || !Arrays.equals(tailDigest.digest(), tailcsum))
 521                         throw new IOException("Pack corrupted while writing to filesystem");
 522
 523                 packcsum = packDigest.digest();
 524                 packOut.write(packcsum);
 525         }
 526
 527         private void growEntries() {
 528                 final PackedObjectInfo[] ne;
 529
 530                 ne = new PackedObjectInfo[(int) objectCount + baseById.size()];
 531                 System.arraycopy(entries, 0, ne, 0, entryCount);
 532                 entries = ne;
 533         }
 534
 535         private void writeIdx() throws IOException {
 536                 Arrays.sort(entries, 0, entryCount);
 537                 List<PackedObjectInfo> list = Arrays.asList(entries);
 538                 if (entryCount < entries.length)
 539                         list = list.subList(0, entryCount);
 540
 541                 final FileOutputStream os = new FileOutputStream(dstIdx);
 542                 try {
 543                         final PackIndexWriter iw;
 544                         if (outputVersion <= 0)
 545                                 iw = PackIndexWriter.createOldestPossible(os, list);
 546                         else
 547                                 iw = PackIndexWriter.createVersion(os, outputVersion);
 548                         iw.write(list, packcsum);
 549                         os.getChannel().force(true);
 550                 } finally {
 551                         os.close();
 552                 }
 553         }
 554
 555         private void readPackHeader() throws IOException {
 556                 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
 557                 final int p = fillFromInput(hdrln);
 558                 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
 559                         if (buf[p + k] != Constants.PACK_SIGNATURE[k])
 560                                 throw new IOException("Not a PACK file.");
 561
 562                 final long vers = NB.decodeUInt32(buf, p + 4);
 563                 if (vers != 2 && vers != 3)
 564                         throw new IOException("Unsupported pack version " + vers + ".");
 565                 objectCount = NB.decodeUInt32(buf, p + 8);
 566                 use(hdrln);
 567         }
 568
 569         private void readPackFooter() throws IOException {
 570                 sync();
 571                 final byte[] cmpcsum = packDigest.digest();
 572                 final int c = fillFromInput(20);
 573                 packcsum = new byte[20];
 574                 System.arraycopy(buf, c, packcsum, 0, 20);
 575                 use(20);
 576                 if (packOut != null)
 577                         packOut.write(packcsum);
 578
 579                 if (!Arrays.equals(cmpcsum, packcsum))
 580                         throw new CorruptObjectException("Packfile checksum incorrect.");
 581         }
 582
 583         // Cleanup all resources associated with our input parsing.
 584         private void endInput() {
 585                 in = null;
 586                 objectData = null;
 587         }
 588
 589         // Read one entire object or delta from the input.
 590         private void indexOneObject() throws IOException {
 591                 final long pos = position();
 592
 593                 crc.reset();
 594                 int c = readFromInput();
 595                 final int typeCode = (c >> 4) & 7;
 596                 long sz = c & 15;
 597                 int shift = 4;
 598                 while ((c & 0x80) != 0) {
 599                         c = readFromInput();
 600                         sz += (c & 0x7f) << shift;
 601                         shift += 7;
 602                 }
 603
 604                 switch (typeCode) {
 605                 case Constants.OBJ_COMMIT:
 606                 case Constants.OBJ_TREE:
 607                 case Constants.OBJ_BLOB:
 608                 case Constants.OBJ_TAG:
 609                         whole(typeCode, pos, sz);
 610                         break;
 611                 case Constants.OBJ_OFS_DELTA: {
 612                         c = readFromInput();
 613                         long ofs = c & 127;
 614                         while ((c & 128) != 0) {
 615                                 ofs += 1;
 616                                 c = readFromInput();
 617                                 ofs <<= 7;
 618                                 ofs += (c & 127);
 619                         }
 620                         final Long base = new Long(pos - ofs);
 621                         ArrayList<UnresolvedDelta> r = baseByPos.get(base);
 622                         if (r == null) {
 623                                 r = new ArrayList<UnresolvedDelta>(8);
 624                                 baseByPos.put(base, r);
 625                         }
 626                         skipInflateFromInput(sz);
 627                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 628                         deltaCount++;
 629                         break;
 630                 }
 631                 case Constants.OBJ_REF_DELTA: {
 632                         c = fillFromInput(20);
 633                         crc.update(buf, c, 20);
 634                         final ObjectId base = ObjectId.fromRaw(buf, c);
 635                         use(20);
 636                         ArrayList<UnresolvedDelta> r = baseById.get(base);
 637                         if (r == null) {
 638                                 r = new ArrayList<UnresolvedDelta>(8);
 639                                 baseById.put(base, r);
 640                         }
 641                         skipInflateFromInput(sz);
 642                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 643                         deltaCount++;
 644                         break;
 645                 }
 646                 default:
 647                         throw new IOException("Unknown object type " + typeCode + ".");
 648                 }
 649         }
 650
 651         private void whole(final int type, final long pos, final long sz)
 652                         throws IOException {
 653                 final byte[] data = inflateFromInput(sz);
 654                 objectDigest.update(Constants.encodedTypeString(type));
 655                 objectDigest.update((byte) ' ');
 656                 objectDigest.update(Constants.encodeASCII(sz));
 657                 objectDigest.update((byte) 0);
 658                 objectDigest.update(data);
 659                 tempObjectId.fromRaw(objectDigest.digest(), 0);
 660
 661                 verifyNoCollision(type, data);
 662                 final int crc32 = (int) crc.getValue();
 663                 entries[entryCount++] = new PackedObjectInfo(pos, crc32, tempObjectId);
 664         }
 665
 666         private void verifyNoCollision(final int type, final byte[] data)
 667                         throws IOException {
 668                 final ObjectLoader ldr = repo.openObject(tempObjectId);
 669                 if (ldr != null) {
 670                         final byte[] existingData = ldr.getCachedBytes();
 671                         if (ldr.getType() != type || !Arrays.equals(data, existingData)) {
 672                                 throw new IOException("collision in " + tempObjectId.name());
 673                         }
 674                 }
 675         }
 676
 677         // Current position of {@link #bOffset} within the entire file.
 678         private long position() {
 679                 return bBase + bOffset;
 680         }
 681
 682         private void position(final long pos) throws IOException {
 683                 packOut.seek(pos);
 684                 bBase = pos;
 685                 bOffset = 0;
 686                 bAvail = 0;
 687         }
 688
 689         // Consume exactly one byte from the buffer and return it.
 690         private int readFromInput() throws IOException {
 691                 if (bAvail == 0)
 692                         fillFromInput(1);
 693                 bAvail--;
 694                 final int b = buf[bOffset++] & 0xff;
 695                 crc.update(b);
 696                 return b;
 697         }
 698
 699         // Consume exactly one byte from the buffer and return it.
 700         private int readFromFile() throws IOException {
 701                 if (bAvail == 0)
 702                         fillFromFile(1);
 703                 bAvail--;
 704                 final int b = buf[bOffset++] & 0xff;
 705                 crc.update(b);
 706                 return b;
 707         }
 708
 709         // Consume cnt bytes from the buffer.
 710         private void use(final int cnt) {
 711                 bOffset += cnt;
 712                 bAvail -= cnt;
 713         }
 714
 715         // Ensure at least need bytes are available in in {@link #buf}.
 716         private int fillFromInput(final int need) throws IOException {
 717                 while (bAvail < need) {
 718                         int next = bOffset + bAvail;
 719                         int free = buf.length - next;
 720                         if (free + bAvail < need) {
 721                                 sync();
 722                                 next = bAvail;
 723                                 free = buf.length - next;
 724                         }
 725                         next = in.read(buf, next, free);
 726                         if (next <= 0)
 727                                 throw new EOFException("Packfile is truncated.");
 728                         bAvail += next;
 729                 }
 730                 return bOffset;
 731         }
 732
 733         // Ensure at least need bytes are available in in {@link #buf}.
 734         private int fillFromFile(final int need) throws IOException {
 735                 if (bAvail < need) {
 736                         int next = bOffset + bAvail;
 737                         int free = buf.length - next;
 738                         if (free + bAvail < need) {
 739                                 if (bAvail > 0)
 740                                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 741                                 bOffset = 0;
 742                                 next = bAvail;
 743                                 free = buf.length - next;
 744                         }
 745                         next = packOut.read(buf, next, free);
 746                         if (next <= 0)
 747                                 throw new EOFException("Packfile is truncated.");
 748                         bAvail += next;
 749                 }
 750                 return bOffset;
 751         }
 752
 753         // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
 754         private void sync() throws IOException {
 755                 packDigest.update(buf, 0, bOffset);
 756                 if (packOut != null)
 757                         packOut.write(buf, 0, bOffset);
 758                 if (bAvail > 0)
 759                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 760                 bBase += bOffset;
 761                 bOffset = 0;
 762         }
 763
 764         private void skipInflateFromInput(long sz) throws IOException {
 765                 final Inflater inf = inflater;
 766                 try {
 767                         final byte[] dst = objectData;
 768                         int n = 0;
 769                         int p = -1;
 770                         while (!inf.finished()) {
 771                                 if (inf.needsInput()) {
 772                                         if (p >= 0) {
 773                                                 crc.update(buf, p, bAvail);
 774                                                 use(bAvail);
 775                                         }
 776                                         p = fillFromInput(1);
 777                                         inf.setInput(buf, p, bAvail);
 778                                 }
 779
 780                                 int free = dst.length - n;
 781                                 if (free < 8) {
 782                                         sz -= n;
 783                                         n = 0;
 784                                         free = dst.length;
 785                                 }
 786                                 n += inf.inflate(dst, n, free);
 787                         }
 788                         if (n != sz)
 789                                 throw new DataFormatException("wrong decompressed length");
 790                         n = bAvail - inf.getRemaining();
 791                         if (n > 0) {
 792                                 crc.update(buf, p, n);
 793                                 use(n);
 794                         }
 795                 } catch (DataFormatException dfe) {
 796                         throw corrupt(dfe);
 797                 } finally {
 798                         inf.reset();
 799                 }
 800         }
 801
 802         private byte[] inflateFromInput(final long sz) throws IOException {
 803                 final byte[] dst = new byte[(int) sz];
 804                 final Inflater inf = inflater;
 805                 try {
 806                         int n = 0;
 807                         int p = -1;
 808                         while (!inf.finished()) {
 809                                 if (inf.needsInput()) {
 810                                         if (p >= 0) {
 811                                                 crc.update(buf, p, bAvail);
 812                                                 use(bAvail);
 813                                         }
 814                                         p = fillFromInput(1);
 815                                         inf.setInput(buf, p, bAvail);
 816                                 }
 817
 818                                 n += inf.inflate(dst, n, dst.length - n);
 819                         }
 820                         if (n != sz)
 821                                 throw new DataFormatException("wrong decompressed length");
 822                         n = bAvail - inf.getRemaining();
 823                         if (n > 0) {
 824                                 crc.update(buf, p, n);
 825                                 use(n);
 826                         }
 827                         return dst;
 828                 } catch (DataFormatException dfe) {
 829                         throw corrupt(dfe);
 830                 } finally {
 831                         inf.reset();
 832                 }
 833         }
 834
 835         private byte[] inflateFromFile(final int sz) throws IOException {
 836                 final Inflater inf = inflater;
 837                 try {
 838                         final byte[] dst = new byte[sz];
 839                         int n = 0;
 840                         int p = -1;
 841                         while (!inf.finished()) {
 842                                 if (inf.needsInput()) {
 843                                         if (p >= 0) {
 844                                                 crc.update(buf, p, bAvail);
 845                                                 use(bAvail);
 846                                         }
 847                                         p = fillFromFile(1);
 848                                         inf.setInput(buf, p, bAvail);
 849                                 }
 850                                 n += inf.inflate(dst, n, sz - n);
 851                         }
 852                         n = bAvail - inf.getRemaining();
 853                         if (n > 0) {
 854                                 crc.update(buf, p, n);
 855                                 use(n);
 856                         }
 857                         return dst;
 858                 } catch (DataFormatException dfe) {
 859                         throw corrupt(dfe);
 860                 } finally {
 861                         inf.reset();
 862                 }
 863         }
 864
 865         private static CorruptObjectException corrupt(final DataFormatException dfe) {
 866                 return new CorruptObjectException("Packfile corruption detected: "
 867                                 + dfe.getMessage());
 868         }
 869
 870         private static class UnresolvedDelta {
 871                 final long position;
 872
 873                 final int crc;
 874
 875                 UnresolvedDelta(final long headerOffset, final int crc32) {
 876                         position = headerOffset;
 877                         crc = crc32;
 878                 }
 879         }
 880
 881         /**
 882          * Rename the pack to it's final name and location and open it.
 883          * <p>
 884          * If the call completes successfully the repository this IndexPack instance
 885          * was created with will have the objects in the pack available for reading
 886          * and use, without needing to scan for packs.
 887          *
 888          * @throws IOException
 889          *             The pack could not be inserted into the repository's objects
 890          *             directory. The pack no longer exists on disk, as it was
 891          *             removed prior to throwing the exception to the caller.
 892          */
 893         public void renameAndOpenPack() throws IOException {
 894                 final MessageDigest d = Constants.newMessageDigest();
 895                 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
 896                 for (int i = 0; i < entryCount; i++) {
 897                         final PackedObjectInfo oe = entries[i];
 898                         oe.copyRawTo(oeBytes, 0);
 899                         d.update(oeBytes);
 900                 }
 901
 902                 final String name = ObjectId.fromRaw(d.digest()).name();
 903                 final File packDir = new File(repo.getObjectsDirectory(), "pack");
 904                 final File finalPack = new File(packDir, "pack-" + name + ".pack");
 905                 final File finalIdx = new File(packDir, "pack-" + name + ".idx");
 906
 907                 if (finalPack.exists()) {
 908                         // If the pack is already present we should never replace it.
 909                         //
 910                         cleanupTemporaryFiles();
 911                         return;
 912                 }
 913
 914                 if (!dstPack.renameTo(finalPack)) {
 915                         cleanupTemporaryFiles();
 916                         throw new IOException("Cannot move pack to " + finalPack);
 917                 }
 918
 919                 if (!dstIdx.renameTo(finalIdx)) {
 920                         cleanupTemporaryFiles();
 921                         if (!finalPack.delete())
 922                                 finalPack.deleteOnExit();
 923                         throw new IOException("Cannot move index to " + finalIdx);
 924                 }
 925
 926                 try {
 927                         repo.openPack(finalPack, finalIdx);
 928                 } catch (IOException err) {
 929                         finalPack.delete();
 930                         finalIdx.delete();
 931                         throw err;
 932                 }
 933         }
 934
 935         private void cleanupTemporaryFiles() {
 936                 if (!dstIdx.delete())
 937                         dstIdx.deleteOnExit();
 938                 if (!dstPack.delete())
 939                         dstPack.deleteOnExit();
 940         }
 941 }