org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java

   1 /*
   2  * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
   3  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   4  *
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  * - Redistributions of source code must retain the above copyright
  12  *   notice, this list of conditions and the following disclaimer.
  13  *
  14  * - Redistributions in binary form must reproduce the above
  15  *   copyright notice, this list of conditions and the following
  16  *   disclaimer in the documentation and/or other materials provided
  17  *   with the distribution.
  18  *
  19  * - Neither the name of the Git Development Community nor the
  20  *   names of its contributors may be used to endorse or promote
  21  *   products derived from this software without specific prior
  22  *   written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  25  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  26  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  31  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 package org.spearce.jgit.transport;
  40
  41 import java.io.EOFException;
  42 import java.io.File;
  43 import java.io.FileOutputStream;
  44 import java.io.IOException;
  45 import java.io.InputStream;
  46 import java.io.RandomAccessFile;
  47 import java.security.MessageDigest;
  48 import java.util.ArrayList;
  49 import java.util.Arrays;
  50 import java.util.HashMap;
  51 import java.util.List;
  52 import java.util.zip.CRC32;
  53 import java.util.zip.DataFormatException;
  54 import java.util.zip.Deflater;
  55 import java.util.zip.Inflater;
  56
  57 import org.spearce.jgit.errors.CorruptObjectException;
  58 import org.spearce.jgit.errors.MissingObjectException;
  59 import org.spearce.jgit.lib.BinaryDelta;
  60 import org.spearce.jgit.lib.Constants;
  61 import org.spearce.jgit.lib.InflaterCache;
  62 import org.spearce.jgit.lib.MutableObjectId;
  63 import org.spearce.jgit.lib.ObjectId;
  64 import org.spearce.jgit.lib.ObjectIdMap;
  65 import org.spearce.jgit.lib.ObjectLoader;
  66 import org.spearce.jgit.lib.PackIndexWriter;
  67 import org.spearce.jgit.lib.ProgressMonitor;
  68 import org.spearce.jgit.lib.Repository;
  69 import org.spearce.jgit.util.NB;
  70
  71 /** Indexes Git pack files for local use. */
  72 public class IndexPack {
  73         /** Progress message when reading raw data from the pack. */
  74         public static final String PROGRESS_DOWNLOAD = "Receiving objects";
  75
  76         /** Progress message when computing names of delta compressed objects. */
  77         public static final String PROGRESS_RESOLVE_DELTA = "Resolving deltas";
  78
  79         private static final int BUFFER_SIZE = 2048;
  80
  81         /**
  82          * Create an index pack instance to load a new pack into a repository.
  83          * <p>
  84          * The received pack data and generated index will be saved to temporary
  85          * files within the repository's <code>objects</code> directory. To use
  86          * the data contained within them call {@link #renameAndOpenPack()} once the
  87          * indexing is complete.
  88          *
  89          * @param db
  90          *            the repository that will receive the new pack.
  91          * @param is
  92          *            stream to read the pack data from.
  93          * @return a new index pack instance.
  94          * @throws IOException
  95          *             a temporary file could not be created.
  96          */
  97         public static IndexPack create(final Repository db, final InputStream is)
  98                         throws IOException {
  99                 final String suffix = ".pack";
 100                 final File objdir = db.getObjectsDirectory();
 101                 final File tmp = File.createTempFile("incoming_", suffix, objdir);
 102                 final String n = tmp.getName();
 103                 final File base;
 104
 105                 base = new File(objdir, n.substring(0, n.length() - suffix.length()));
 106                 final IndexPack ip = new IndexPack(db, is, base);
 107                 ip.setIndexVersion(db.getConfig().getCore().getPackIndexVersion());
 108                 return ip;
 109         }
 110
 111         private final Repository repo;
 112
 113         private Inflater inflater;
 114
 115         private final MessageDigest objectDigest;
 116
 117         private final MutableObjectId tempObjectId;
 118
 119         private InputStream in;
 120
 121         private byte[] buf;
 122
 123         private long bBase;
 124
 125         private int bOffset;
 126
 127         private int bAvail;
 128
 129         private boolean fixThin;
 130
 131         private int outputVersion;
 132
 133         private final File dstPack;
 134
 135         private final File dstIdx;
 136
 137         private long objectCount;
 138
 139         private PackedObjectInfo[] entries;
 140
 141         private int deltaCount;
 142
 143         private int entryCount;
 144
 145         private final CRC32 crc = new CRC32();
 146
 147         private ObjectIdMap<ArrayList<UnresolvedDelta>> baseById;
 148
 149         private HashMap<Long, ArrayList<UnresolvedDelta>> baseByPos;
 150
 151         private byte[] objectData;
 152
 153         private MessageDigest packDigest;
 154
 155         private RandomAccessFile packOut;
 156
 157         private byte[] packcsum;
 158
 159         /** If {@link #fixThin} this is the last byte of the original checksum. */
 160         private long originalEOF;
 161
 162         /**
 163          * Create a new pack indexer utility.
 164          *
 165          * @param db
 166          * @param src
 167          * @param dstBase
 168          * @throws IOException
 169          *             the output packfile could not be created.
 170          */
 171         public IndexPack(final Repository db, final InputStream src,
 172                         final File dstBase) throws IOException {
 173                 repo = db;
 174                 in = src;
 175                 inflater = InflaterCache.get();
 176                 buf = new byte[BUFFER_SIZE];
 177                 objectData = new byte[BUFFER_SIZE];
 178                 objectDigest = Constants.newMessageDigest();
 179                 tempObjectId = new MutableObjectId();
 180                 packDigest = Constants.newMessageDigest();
 181
 182                 if (dstBase != null) {
 183                         final File dir = dstBase.getParentFile();
 184                         final String nam = dstBase.getName();
 185                         dstPack = new File(dir, nam + ".pack");
 186                         dstIdx = new File(dir, nam + ".idx");
 187                         packOut = new RandomAccessFile(dstPack, "rw");
 188                         packOut.setLength(0);
 189                 } else {
 190                         dstPack = null;
 191                         dstIdx = null;
 192                 }
 193         }
 194
 195         /**
 196          * Set the pack index file format version this instance will create.
 197          *
 198          * @param version
 199          *            the version to write. The special version 0 designates the
 200          *            oldest (most compatible) format available for the objects.
 201          * @see PackIndexWriter
 202          */
 203         public void setIndexVersion(final int version) {
 204                 outputVersion = version;
 205         }
 206
 207         /**
 208          * Configure this index pack instance to make a thin pack complete.
 209          * <p>
 210          * Thin packs are sometimes used during network transfers to allow a delta
 211          * to be sent without a base object. Such packs are not permitted on disk.
 212          * They can be fixed by copying the base object onto the end of the pack.
 213          *
 214          * @param fix
 215          *            true to enable fixing a thin pack.
 216          */
 217         public void setFixThin(final boolean fix) {
 218                 fixThin = fix;
 219         }
 220
 221         /**
 222          * Consume data from the input stream until the packfile is indexed.
 223          *
 224          * @param progress
 225          *            progress feedback
 226          *
 227          * @throws IOException
 228          */
 229         public void index(final ProgressMonitor progress) throws IOException {
 230                 progress.start(2 /* tasks */);
 231                 try {
 232                         try {
 233                                 readPackHeader();
 234
 235                                 entries = new PackedObjectInfo[(int) objectCount];
 236                                 baseById = new ObjectIdMap<ArrayList<UnresolvedDelta>>();
 237                                 baseByPos = new HashMap<Long, ArrayList<UnresolvedDelta>>();
 238
 239                                 progress.beginTask(PROGRESS_DOWNLOAD, (int) objectCount);
 240                                 for (int done = 0; done < objectCount; done++) {
 241                                         indexOneObject();
 242                                         progress.update(1);
 243                                         if (progress.isCancelled())
 244                                                 throw new IOException("Download cancelled");
 245                                 }
 246                                 readPackFooter();
 247                                 endInput();
 248                                 progress.endTask();
 249                                 if (deltaCount > 0) {
 250                                         if (packOut == null)
 251                                                 throw new IOException("need packOut");
 252                                         resolveDeltas(progress);
 253                                         if (entryCount < objectCount) {
 254                                                 if (!fixThin) {
 255                                                         throw new IOException("pack has "
 256                                                                         + (objectCount - entryCount)
 257                                                                         + " unresolved deltas");
 258                                                 }
 259                                                 fixThinPack(progress);
 260                                         }
 261                                 }
 262                                 if (packOut != null)
 263                                         packOut.getChannel().force(true);
 264
 265                                 packDigest = null;
 266                                 baseById = null;
 267                                 baseByPos = null;
 268
 269                                 if (dstIdx != null)
 270                                         writeIdx();
 271
 272                         } finally {
 273                                 final Inflater inf = inflater;
 274                                 inflater = null;
 275                                 InflaterCache.release(inf);
 276
 277                                 progress.endTask();
 278                                 if (packOut != null)
 279                                         packOut.close();
 280                         }
 281
 282                         if (dstPack != null)
 283                                 dstPack.setReadOnly();
 284                         if (dstIdx != null)
 285                                 dstIdx.setReadOnly();
 286                 } catch (IOException err) {
 287                         if (dstPack != null)
 288                                 dstPack.delete();
 289                         if (dstIdx != null)
 290                                 dstIdx.delete();
 291                         throw err;
 292                 }
 293         }
 294
 295         private void resolveDeltas(final ProgressMonitor progress)
 296                         throws IOException {
 297                 progress.beginTask(PROGRESS_RESOLVE_DELTA, deltaCount);
 298                 final int last = entryCount;
 299                 for (int i = 0; i < last; i++) {
 300                         final int before = entryCount;
 301                         resolveDeltas(entries[i]);
 302                         progress.update(entryCount - before);
 303                         if (progress.isCancelled())
 304                                 throw new IOException("Download cancelled during indexing");
 305                 }
 306                 progress.endTask();
 307         }
 308
 309         private void resolveDeltas(final PackedObjectInfo oe) throws IOException {
 310                 final int oldCRC = oe.getCRC();
 311                 if (baseById.containsKey(oe)
 312                                 || baseByPos.containsKey(new Long(oe.getOffset())))
 313                         resolveDeltas(oe.getOffset(), oldCRC, Constants.OBJ_BAD, null, oe);
 314         }
 315
 316         private void resolveDeltas(final long pos, final int oldCRC, int type,
 317                         byte[] data, PackedObjectInfo oe) throws IOException {
 318                 crc.reset();
 319                 position(pos);
 320                 int c = readFromFile();
 321                 final int typeCode = (c >> 4) & 7;
 322                 long sz = c & 15;
 323                 int shift = 4;
 324                 while ((c & 0x80) != 0) {
 325                         c = readFromFile();
 326                         sz += (c & 0x7f) << shift;
 327                         shift += 7;
 328                 }
 329
 330                 switch (typeCode) {
 331                 case Constants.OBJ_COMMIT:
 332                 case Constants.OBJ_TREE:
 333                 case Constants.OBJ_BLOB:
 334                 case Constants.OBJ_TAG:
 335                         type = typeCode;
 336                         data = inflateFromFile((int) sz);
 337                         break;
 338                 case Constants.OBJ_OFS_DELTA: {
 339                         c = readFromFile() & 0xff;
 340                         while ((c & 128) != 0)
 341                                 c = readFromFile() & 0xff;
 342                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 343                         break;
 344                 }
 345                 case Constants.OBJ_REF_DELTA: {
 346                         crc.update(buf, fillFromFile(20), 20);
 347                         use(20);
 348                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 349                         break;
 350                 }
 351                 default:
 352                         throw new IOException("Unknown object type " + typeCode + ".");
 353                 }
 354
 355                 final int crc32 = (int) crc.getValue();
 356                 if (oldCRC != crc32)
 357                         throw new IOException("Corruption detected re-reading at " + pos);
 358                 if (oe == null) {
 359                         objectDigest.update(Constants.encodedTypeString(type));
 360                         objectDigest.update((byte) ' ');
 361                         objectDigest.update(Constants.encodeASCII(data.length));
 362                         objectDigest.update((byte) 0);
 363                         objectDigest.update(data);
 364                         tempObjectId.fromRaw(objectDigest.digest(), 0);
 365
 366                         oe = new PackedObjectInfo(pos, crc32, tempObjectId);
 367                         entries[entryCount++] = oe;
 368                 }
 369
 370                 resolveChildDeltas(pos, type, data, oe);
 371         }
 372
 373         private void resolveChildDeltas(final long pos, int type, byte[] data,
 374                         PackedObjectInfo oe) throws IOException {
 375                 final ArrayList<UnresolvedDelta> a = baseById.remove(oe);
 376                 final ArrayList<UnresolvedDelta> b = baseByPos.remove(new Long(pos));
 377                 int ai = 0, bi = 0;
 378                 if (a != null && b != null) {
 379                         while (ai < a.size() && bi < b.size()) {
 380                                 final UnresolvedDelta ad = a.get(ai);
 381                                 final UnresolvedDelta bd = b.get(bi);
 382                                 if (ad.position < bd.position) {
 383                                         resolveDeltas(ad.position, ad.crc, type, data, null);
 384                                         ai++;
 385                                 } else {
 386                                         resolveDeltas(bd.position, bd.crc, type, data, null);
 387                                         bi++;
 388                                 }
 389                         }
 390                 }
 391                 if (a != null)
 392                         while (ai < a.size()) {
 393                                 final UnresolvedDelta ad = a.get(ai++);
 394                                 resolveDeltas(ad.position, ad.crc, type, data, null);
 395                         }
 396                 if (b != null)
 397                         while (bi < b.size()) {
 398                                 final UnresolvedDelta bd = b.get(bi++);
 399                                 resolveDeltas(bd.position, bd.crc, type, data, null);
 400                         }
 401         }
 402
 403         private void fixThinPack(final ProgressMonitor progress) throws IOException {
 404                 growEntries();
 405
 406                 originalEOF = packOut.length() - 20;
 407                 final Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
 408                 long end = originalEOF;
 409                 for (final ObjectId baseId : new ArrayList<ObjectId>(baseById.keySet())) {
 410                         final ObjectLoader ldr = repo.openObject(baseId);
 411                         if (ldr == null)
 412                                 continue;
 413                         final byte[] data = ldr.getBytes();
 414                         final int typeCode = ldr.getType();
 415                         final PackedObjectInfo oe;
 416
 417                         crc.reset();
 418                         packOut.seek(end);
 419                         writeWhole(def, typeCode, data);
 420                         oe = new PackedObjectInfo(end, (int) crc.getValue(), baseId);
 421                         entries[entryCount++] = oe;
 422                         end = packOut.getFilePointer();
 423
 424                         resolveChildDeltas(oe.getOffset(), typeCode, data, oe);
 425                         if (progress.isCancelled())
 426                                 throw new IOException("Download cancelled during indexing");
 427                 }
 428                 def.end();
 429
 430                 if (!baseById.isEmpty()) {
 431                         final ObjectId need = baseById.keySet().iterator().next();
 432                         throw new MissingObjectException(need, "delta base");
 433                 }
 434
 435                 fixHeaderFooter();
 436         }
 437
 438         private void writeWhole(final Deflater def, final int typeCode,
 439                         final byte[] data) throws IOException {
 440                 int sz = data.length;
 441                 int hdrlen = 0;
 442                 buf[hdrlen++] = (byte) ((typeCode << 4) | sz & 15);
 443                 sz >>>= 4;
 444                 while (sz > 0) {
 445                         buf[hdrlen - 1] |= 0x80;
 446                         buf[hdrlen++] = (byte) (sz & 0x7f);
 447                         sz >>>= 7;
 448                 }
 449                 crc.update(buf, 0, hdrlen);
 450                 packOut.write(buf, 0, hdrlen);
 451                 def.reset();
 452                 def.setInput(data);
 453                 def.finish();
 454                 while (!def.finished()) {
 455                         final int datlen = def.deflate(buf);
 456                         crc.update(buf, 0, datlen);
 457                         packOut.write(buf, 0, datlen);
 458                 }
 459         }
 460
 461         private void fixHeaderFooter() throws IOException {
 462                 final MessageDigest origDigest = Constants.newMessageDigest();
 463                 long origRemaining = originalEOF - 12;
 464
 465                 packOut.seek(0);
 466                 if (packOut.read(buf, 0, 12) != 12)
 467                         throw new IOException("Cannot re-read pack header to fix count");
 468                 origDigest.update(buf, 0, 12);
 469                 NB.encodeInt32(buf, 8, entryCount);
 470                 packOut.seek(0);
 471                 packOut.write(buf, 0, 12);
 472
 473                 packDigest.reset();
 474                 packDigest.update(buf, 0, 12);
 475                 for (;;) {
 476                         final int n = packOut.read(buf);
 477                         if (n < 0)
 478                                 break;
 479                         if (origRemaining > 0) {
 480                                 origDigest.update(buf, 0, (int) Math.min(n, origRemaining));
 481                                 origRemaining -= n;
 482                         }
 483                         packDigest.update(buf, 0, n);
 484                 }
 485
 486                 if (!Arrays.equals(origDigest.digest(), packcsum))
 487                         throw new IOException("Pack corrupted while writing to filesystem");
 488
 489                 packcsum = packDigest.digest();
 490                 packOut.write(packcsum);
 491         }
 492
 493         private void growEntries() {
 494                 final PackedObjectInfo[] ne;
 495
 496                 ne = new PackedObjectInfo[(int) objectCount + baseById.size()];
 497                 System.arraycopy(entries, 0, ne, 0, entryCount);
 498                 entries = ne;
 499         }
 500
 501         private void writeIdx() throws IOException {
 502                 Arrays.sort(entries, 0, entryCount);
 503                 List<PackedObjectInfo> list = Arrays.asList(entries);
 504                 if (entryCount < entries.length)
 505                         list = list.subList(0, entryCount);
 506
 507                 final FileOutputStream os = new FileOutputStream(dstIdx);
 508                 try {
 509                         final PackIndexWriter iw;
 510                         if (outputVersion <= 0)
 511                                 iw = PackIndexWriter.createOldestPossible(os, list);
 512                         else
 513                                 iw = PackIndexWriter.createVersion(os, outputVersion);
 514                         iw.write(list, packcsum);
 515                         os.getChannel().force(true);
 516                 } finally {
 517                         os.close();
 518                 }
 519         }
 520
 521         private void readPackHeader() throws IOException {
 522                 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
 523                 final int p = fillFromInput(hdrln);
 524                 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
 525                         if (buf[p + k] != Constants.PACK_SIGNATURE[k])
 526                                 throw new IOException("Not a PACK file.");
 527
 528                 final long vers = NB.decodeUInt32(buf, p + 4);
 529                 if (vers != 2 && vers != 3)
 530                         throw new IOException("Unsupported pack version " + vers + ".");
 531                 objectCount = NB.decodeUInt32(buf, p + 8);
 532                 use(hdrln);
 533         }
 534
 535         private void readPackFooter() throws IOException {
 536                 sync();
 537                 final byte[] cmpcsum = packDigest.digest();
 538                 final int c = fillFromInput(20);
 539                 packcsum = new byte[20];
 540                 System.arraycopy(buf, c, packcsum, 0, 20);
 541                 use(20);
 542                 if (packOut != null)
 543                         packOut.write(packcsum);
 544
 545                 if (!Arrays.equals(cmpcsum, packcsum))
 546                         throw new CorruptObjectException("Packfile checksum incorrect.");
 547         }
 548
 549         // Cleanup all resources associated with our input parsing.
 550         private void endInput() {
 551                 in = null;
 552                 objectData = null;
 553         }
 554
 555         // Read one entire object or delta from the input.
 556         private void indexOneObject() throws IOException {
 557                 final long pos = position();
 558
 559                 crc.reset();
 560                 int c = readFromInput();
 561                 final int typeCode = (c >> 4) & 7;
 562                 long sz = c & 15;
 563                 int shift = 4;
 564                 while ((c & 0x80) != 0) {
 565                         c = readFromInput();
 566                         sz += (c & 0x7f) << shift;
 567                         shift += 7;
 568                 }
 569
 570                 switch (typeCode) {
 571                 case Constants.OBJ_COMMIT:
 572                 case Constants.OBJ_TREE:
 573                 case Constants.OBJ_BLOB:
 574                 case Constants.OBJ_TAG:
 575                         whole(typeCode, pos, sz);
 576                         break;
 577                 case Constants.OBJ_OFS_DELTA: {
 578                         c = readFromInput();
 579                         long ofs = c & 127;
 580                         while ((c & 128) != 0) {
 581                                 ofs += 1;
 582                                 c = readFromInput();
 583                                 ofs <<= 7;
 584                                 ofs += (c & 127);
 585                         }
 586                         final Long base = new Long(pos - ofs);
 587                         ArrayList<UnresolvedDelta> r = baseByPos.get(base);
 588                         if (r == null) {
 589                                 r = new ArrayList<UnresolvedDelta>(8);
 590                                 baseByPos.put(base, r);
 591                         }
 592                         inflateFromInput(false);
 593                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 594                         deltaCount++;
 595                         break;
 596                 }
 597                 case Constants.OBJ_REF_DELTA: {
 598                         c = fillFromInput(20);
 599                         crc.update(buf, c, 20);
 600                         final ObjectId base = ObjectId.fromRaw(buf, c);
 601                         use(20);
 602                         ArrayList<UnresolvedDelta> r = baseById.get(base);
 603                         if (r == null) {
 604                                 r = new ArrayList<UnresolvedDelta>(8);
 605                                 baseById.put(base, r);
 606                         }
 607                         inflateFromInput(false);
 608                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 609                         deltaCount++;
 610                         break;
 611                 }
 612                 default:
 613                         throw new IOException("Unknown object type " + typeCode + ".");
 614                 }
 615         }
 616
 617         private void whole(final int type, final long pos, final long sz)
 618                         throws IOException {
 619                 objectDigest.update(Constants.encodedTypeString(type));
 620                 objectDigest.update((byte) ' ');
 621                 objectDigest.update(Constants.encodeASCII(sz));
 622                 objectDigest.update((byte) 0);
 623                 inflateFromInput(true);
 624                 tempObjectId.fromRaw(objectDigest.digest(), 0);
 625
 626                 final int crc32 = (int) crc.getValue();
 627                 entries[entryCount++] = new PackedObjectInfo(pos, crc32, tempObjectId);
 628         }
 629
 630         // Current position of {@link #bOffset} within the entire file.
 631         private long position() {
 632                 return bBase + bOffset;
 633         }
 634
 635         private void position(final long pos) throws IOException {
 636                 packOut.seek(pos);
 637                 bBase = pos;
 638                 bOffset = 0;
 639                 bAvail = 0;
 640         }
 641
 642         // Consume exactly one byte from the buffer and return it.
 643         private int readFromInput() throws IOException {
 644                 if (bAvail == 0)
 645                         fillFromInput(1);
 646                 bAvail--;
 647                 final int b = buf[bOffset++] & 0xff;
 648                 crc.update(b);
 649                 return b;
 650         }
 651
 652         // Consume exactly one byte from the buffer and return it.
 653         private int readFromFile() throws IOException {
 654                 if (bAvail == 0)
 655                         fillFromFile(1);
 656                 bAvail--;
 657                 final int b = buf[bOffset++] & 0xff;
 658                 crc.update(b);
 659                 return b;
 660         }
 661
 662         // Consume cnt bytes from the buffer.
 663         private void use(final int cnt) {
 664                 bOffset += cnt;
 665                 bAvail -= cnt;
 666         }
 667
 668         // Ensure at least need bytes are available in in {@link #buf}.
 669         private int fillFromInput(final int need) throws IOException {
 670                 while (bAvail < need) {
 671                         int next = bOffset + bAvail;
 672                         int free = buf.length - next;
 673                         if (free + bAvail < need) {
 674                                 sync();
 675                                 next = bAvail;
 676                                 free = buf.length - next;
 677                         }
 678                         next = in.read(buf, next, free);
 679                         if (next <= 0)
 680                                 throw new EOFException("Packfile is truncated.");
 681                         bAvail += next;
 682                 }
 683                 return bOffset;
 684         }
 685
 686         // Ensure at least need bytes are available in in {@link #buf}.
 687         private int fillFromFile(final int need) throws IOException {
 688                 if (bAvail < need) {
 689                         int next = bOffset + bAvail;
 690                         int free = buf.length - next;
 691                         if (free + bAvail < need) {
 692                                 if (bAvail > 0)
 693                                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 694                                 bOffset = 0;
 695                                 next = bAvail;
 696                                 free = buf.length - next;
 697                         }
 698                         next = packOut.read(buf, next, free);
 699                         if (next <= 0)
 700                                 throw new EOFException("Packfile is truncated.");
 701                         bAvail += next;
 702                 }
 703                 return bOffset;
 704         }
 705
 706         // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
 707         private void sync() throws IOException {
 708                 packDigest.update(buf, 0, bOffset);
 709                 if (packOut != null)
 710                         packOut.write(buf, 0, bOffset);
 711                 if (bAvail > 0)
 712                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 713                 bBase += bOffset;
 714                 bOffset = 0;
 715         }
 716
 717         private void inflateFromInput(final boolean digest) throws IOException {
 718                 final Inflater inf = inflater;
 719                 try {
 720                         final byte[] dst = objectData;
 721                         int n = 0;
 722                         int p = -1;
 723                         while (!inf.finished()) {
 724                                 if (inf.needsInput()) {
 725                                         if (p >= 0) {
 726                                                 crc.update(buf, p, bAvail);
 727                                                 use(bAvail);
 728                                         }
 729                                         p = fillFromInput(1);
 730                                         inf.setInput(buf, p, bAvail);
 731                                 }
 732
 733                                 int free = dst.length - n;
 734                                 if (free < 8) {
 735                                         if (digest)
 736                                                 objectDigest.update(dst, 0, n);
 737                                         n = 0;
 738                                         free = dst.length;
 739                                 }
 740
 741                                 n += inf.inflate(dst, n, free);
 742                         }
 743                         if (digest)
 744                                 objectDigest.update(dst, 0, n);
 745                         n = bAvail - inf.getRemaining();
 746                         if (n > 0) {
 747                                 crc.update(buf, p, n);
 748                                 use(n);
 749                         }
 750                 } catch (DataFormatException dfe) {
 751                         throw corrupt(dfe);
 752                 } finally {
 753                         inf.reset();
 754                 }
 755         }
 756
 757         private byte[] inflateFromFile(final int sz) throws IOException {
 758                 final Inflater inf = inflater;
 759                 try {
 760                         final byte[] dst = new byte[sz];
 761                         int n = 0;
 762                         int p = -1;
 763                         while (!inf.finished()) {
 764                                 if (inf.needsInput()) {
 765                                         if (p >= 0) {
 766                                                 crc.update(buf, p, bAvail);
 767                                                 use(bAvail);
 768                                         }
 769                                         p = fillFromFile(1);
 770                                         inf.setInput(buf, p, bAvail);
 771                                 }
 772                                 n += inf.inflate(dst, n, sz - n);
 773                         }
 774                         n = bAvail - inf.getRemaining();
 775                         if (n > 0) {
 776                                 crc.update(buf, p, n);
 777                                 use(n);
 778                         }
 779                         return dst;
 780                 } catch (DataFormatException dfe) {
 781                         throw corrupt(dfe);
 782                 } finally {
 783                         inf.reset();
 784                 }
 785         }
 786
 787         private static CorruptObjectException corrupt(final DataFormatException dfe) {
 788                 return new CorruptObjectException("Packfile corruption detected: "
 789                                 + dfe.getMessage());
 790         }
 791
 792         private static class UnresolvedDelta {
 793                 final long position;
 794
 795                 final int crc;
 796
 797                 UnresolvedDelta(final long headerOffset, final int crc32) {
 798                         position = headerOffset;
 799                         crc = crc32;
 800                 }
 801         }
 802
 803         /**
 804          * Rename the pack to it's final name and location and open it.
 805          * <p>
 806          * If the call completes successfully the repository this IndexPack instance
 807          * was created with will have the objects in the pack available for reading
 808          * and use, without needing to scan for packs.
 809          *
 810          * @throws IOException
 811          *             The pack could not be inserted into the repository's objects
 812          *             directory. The pack no longer exists on disk, as it was
 813          *             removed prior to throwing the exception to the caller.
 814          */
 815         public void renameAndOpenPack() throws IOException {
 816                 final MessageDigest d = Constants.newMessageDigest();
 817                 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
 818                 for (int i = 0; i < entryCount; i++) {
 819                         final PackedObjectInfo oe = entries[i];
 820                         oe.copyRawTo(oeBytes, 0);
 821                         d.update(oeBytes);
 822                 }
 823
 824                 final ObjectId name = ObjectId.fromRaw(d.digest());
 825                 final File packDir = new File(repo.getObjectsDirectory(), "pack");
 826                 final File finalPack = new File(packDir, "pack-" + name + ".pack");
 827                 final File finalIdx = new File(packDir, "pack-" + name + ".idx");
 828
 829                 if (finalPack.exists()) {
 830                         // If the pack is already present we should never replace it.
 831                         //
 832                         cleanupTemporaryFiles();
 833                         return;
 834                 }
 835
 836                 if (!dstPack.renameTo(finalPack)) {
 837                         cleanupTemporaryFiles();
 838                         throw new IOException("Cannot move pack to " + finalPack);
 839                 }
 840
 841                 if (!dstIdx.renameTo(finalIdx)) {
 842                         cleanupTemporaryFiles();
 843                         if (!finalPack.delete())
 844                                 finalPack.deleteOnExit();
 845                         throw new IOException("Cannot move index to " + finalIdx);
 846                 }
 847
 848                 try {
 849                         repo.openPack(finalPack, finalIdx);
 850                 } catch (IOException err) {
 851                         finalPack.delete();
 852                         finalIdx.delete();
 853                         throw err;
 854                 }
 855         }
 856
 857         private void cleanupTemporaryFiles() {
 858                 if (!dstIdx.delete())
 859                         dstIdx.deleteOnExit();
 860                 if (!dstPack.delete())
 861                         dstPack.deleteOnExit();
 862         }
 863 }