org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java

   1 /*
   2  * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
   3  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   4  *
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  * - Redistributions of source code must retain the above copyright
  12  *   notice, this list of conditions and the following disclaimer.
  13  *
  14  * - Redistributions in binary form must reproduce the above
  15  *   copyright notice, this list of conditions and the following
  16  *   disclaimer in the documentation and/or other materials provided
  17  *   with the distribution.
  18  *
  19  * - Neither the name of the Git Development Community nor the
  20  *   names of its contributors may be used to endorse or promote
  21  *   products derived from this software without specific prior
  22  *   written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  25  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  26  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  31  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 package org.spearce.jgit.transport;
  40
  41 import java.io.EOFException;
  42 import java.io.File;
  43 import java.io.FileOutputStream;
  44 import java.io.IOException;
  45 import java.io.InputStream;
  46 import java.io.RandomAccessFile;
  47 import java.security.MessageDigest;
  48 import java.util.ArrayList;
  49 import java.util.Arrays;
  50 import java.util.HashMap;
  51 import java.util.List;
  52 import java.util.zip.CRC32;
  53 import java.util.zip.DataFormatException;
  54 import java.util.zip.Deflater;
  55 import java.util.zip.Inflater;
  56
  57 import org.spearce.jgit.errors.CorruptObjectException;
  58 import org.spearce.jgit.errors.MissingObjectException;
  59 import org.spearce.jgit.lib.AnyObjectId;
  60 import org.spearce.jgit.lib.BinaryDelta;
  61 import org.spearce.jgit.lib.Constants;
  62 import org.spearce.jgit.lib.InflaterCache;
  63 import org.spearce.jgit.lib.MutableObjectId;
  64 import org.spearce.jgit.lib.ObjectChecker;
  65 import org.spearce.jgit.lib.ObjectId;
  66 import org.spearce.jgit.lib.ObjectIdMap;
  67 import org.spearce.jgit.lib.ObjectLoader;
  68 import org.spearce.jgit.lib.PackIndexWriter;
  69 import org.spearce.jgit.lib.ProgressMonitor;
  70 import org.spearce.jgit.lib.Repository;
  71 import org.spearce.jgit.lib.WindowCursor;
  72 import org.spearce.jgit.util.NB;
  73
  74 /** Indexes Git pack files for local use. */
  75 public class IndexPack {
  76         /** Progress message when reading raw data from the pack. */
  77         public static final String PROGRESS_DOWNLOAD = "Receiving objects";
  78
  79         /** Progress message when computing names of delta compressed objects. */
  80         public static final String PROGRESS_RESOLVE_DELTA = "Resolving deltas";
  81
  82         /**
  83          * Size of the internal stream buffer.
  84          * <p>
  85          * If callers are going to be supplying IndexPack a BufferedInputStream they
  86          * should use this buffer size as the size of the buffer for that
  87          * BufferedInputStream, and any other its may be wrapping. This way the
  88          * buffers will cascade efficiently and only the IndexPack buffer will be
  89          * receiving the bulk of the data stream.
  90          */
  91         public static final int BUFFER_SIZE = 8192;
  92
  93         /**
  94          * Create an index pack instance to load a new pack into a repository.
  95          * <p>
  96          * The received pack data and generated index will be saved to temporary
  97          * files within the repository's <code>objects</code> directory. To use the
  98          * data contained within them call {@link #renameAndOpenPack()} once the
  99          * indexing is complete.
 100          *
 101          * @param db
 102          *            the repository that will receive the new pack.
 103          * @param is
 104          *            stream to read the pack data from. If the stream is buffered
 105          *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
 106          * @return a new index pack instance.
 107          * @throws IOException
 108          *             a temporary file could not be created.
 109          */
 110         public static IndexPack create(final Repository db, final InputStream is)
 111                         throws IOException {
 112                 final String suffix = ".pack";
 113                 final File objdir = db.getObjectsDirectory();
 114                 final File tmp = File.createTempFile("incoming_", suffix, objdir);
 115                 final String n = tmp.getName();
 116                 final File base;
 117
 118                 base = new File(objdir, n.substring(0, n.length() - suffix.length()));
 119                 final IndexPack ip = new IndexPack(db, is, base);
 120                 ip.setIndexVersion(db.getConfig().getCore().getPackIndexVersion());
 121                 return ip;
 122         }
 123
 124         private final Repository repo;
 125
 126         private Inflater inflater;
 127
 128         private final MessageDigest objectDigest;
 129
 130         private final MutableObjectId tempObjectId;
 131
 132         private InputStream in;
 133
 134         private byte[] buf;
 135
 136         private long bBase;
 137
 138         private int bOffset;
 139
 140         private int bAvail;
 141
 142         private ObjectChecker objCheck;
 143
 144         private boolean fixThin;
 145
 146         private int outputVersion;
 147
 148         private final File dstPack;
 149
 150         private final File dstIdx;
 151
 152         private long objectCount;
 153
 154         private PackedObjectInfo[] entries;
 155
 156         private int deltaCount;
 157
 158         private int entryCount;
 159
 160         private final CRC32 crc = new CRC32();
 161
 162         private ObjectIdMap<ArrayList<UnresolvedDelta>> baseById;
 163
 164         private HashMap<Long, ArrayList<UnresolvedDelta>> baseByPos;
 165
 166         private byte[] objectData;
 167
 168         private MessageDigest packDigest;
 169
 170         private RandomAccessFile packOut;
 171
 172         private byte[] packcsum;
 173
 174         /** If {@link #fixThin} this is the last byte of the original checksum. */
 175         private long originalEOF;
 176
 177         private WindowCursor readCurs;
 178
 179         /**
 180          * Create a new pack indexer utility.
 181          *
 182          * @param db
 183          * @param src
 184          *            stream to read the pack data from. If the stream is buffered
 185          *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
 186          * @param dstBase
 187          * @throws IOException
 188          *             the output packfile could not be created.
 189          */
 190         public IndexPack(final Repository db, final InputStream src,
 191                         final File dstBase) throws IOException {
 192                 repo = db;
 193                 in = src;
 194                 inflater = InflaterCache.get();
 195                 readCurs = new WindowCursor();
 196                 buf = new byte[BUFFER_SIZE];
 197                 objectData = new byte[BUFFER_SIZE];
 198                 objectDigest = Constants.newMessageDigest();
 199                 tempObjectId = new MutableObjectId();
 200                 packDigest = Constants.newMessageDigest();
 201
 202                 if (dstBase != null) {
 203                         final File dir = dstBase.getParentFile();
 204                         final String nam = dstBase.getName();
 205                         dstPack = new File(dir, nam + ".pack");
 206                         dstIdx = new File(dir, nam + ".idx");
 207                         packOut = new RandomAccessFile(dstPack, "rw");
 208                         packOut.setLength(0);
 209                 } else {
 210                         dstPack = null;
 211                         dstIdx = null;
 212                 }
 213         }
 214
 215         /**
 216          * Set the pack index file format version this instance will create.
 217          *
 218          * @param version
 219          *            the version to write. The special version 0 designates the
 220          *            oldest (most compatible) format available for the objects.
 221          * @see PackIndexWriter
 222          */
 223         public void setIndexVersion(final int version) {
 224                 outputVersion = version;
 225         }
 226
 227         /**
 228          * Configure this index pack instance to make a thin pack complete.
 229          * <p>
 230          * Thin packs are sometimes used during network transfers to allow a delta
 231          * to be sent without a base object. Such packs are not permitted on disk.
 232          * They can be fixed by copying the base object onto the end of the pack.
 233          *
 234          * @param fix
 235          *            true to enable fixing a thin pack.
 236          */
 237         public void setFixThin(final boolean fix) {
 238                 fixThin = fix;
 239         }
 240
 241         /**
 242          * Configure the checker used to validate received objects.
 243          * <p>
 244          * Usually object checking isn't necessary, as Git implementations only
 245          * create valid objects in pack files. However, additional checking may be
 246          * useful if processing data from an untrusted source.
 247          *
 248          * @param oc
 249          *            the checker instance; null to disable object checking.
 250          */
 251         public void setObjectChecker(final ObjectChecker oc) {
 252                 objCheck = oc;
 253         }
 254
 255         /**
 256          * Configure the checker used to validate received objects.
 257          * <p>
 258          * Usually object checking isn't necessary, as Git implementations only
 259          * create valid objects in pack files. However, additional checking may be
 260          * useful if processing data from an untrusted source.
 261          * <p>
 262          * This is shorthand for:
 263          *
 264          * <pre>
 265          * setObjectChecker(on ? new ObjectChecker() : null);
 266          * </pre>
 267          *
 268          * @param on
 269          *            true to enable the default checker; false to disable it.
 270          */
 271         public void setObjectChecking(final boolean on) {
 272                 setObjectChecker(on ? new ObjectChecker() : null);
 273         }
 274
 275         /**
 276          * Consume data from the input stream until the packfile is indexed.
 277          *
 278          * @param progress
 279          *            progress feedback
 280          *
 281          * @throws IOException
 282          */
 283         public void index(final ProgressMonitor progress) throws IOException {
 284                 progress.start(2 /* tasks */);
 285                 try {
 286                         try {
 287                                 readPackHeader();
 288
 289                                 entries = new PackedObjectInfo[(int) objectCount];
 290                                 baseById = new ObjectIdMap<ArrayList<UnresolvedDelta>>();
 291                                 baseByPos = new HashMap<Long, ArrayList<UnresolvedDelta>>();
 292
 293                                 progress.beginTask(PROGRESS_DOWNLOAD, (int) objectCount);
 294                                 for (int done = 0; done < objectCount; done++) {
 295                                         indexOneObject();
 296                                         progress.update(1);
 297                                         if (progress.isCancelled())
 298                                                 throw new IOException("Download cancelled");
 299                                 }
 300                                 readPackFooter();
 301                                 endInput();
 302                                 progress.endTask();
 303                                 if (deltaCount > 0) {
 304                                         if (packOut == null)
 305                                                 throw new IOException("need packOut");
 306                                         resolveDeltas(progress);
 307                                         if (entryCount < objectCount) {
 308                                                 if (!fixThin) {
 309                                                         throw new IOException("pack has "
 310                                                                         + (objectCount - entryCount)
 311                                                                         + " unresolved deltas");
 312                                                 }
 313                                                 fixThinPack(progress);
 314                                         }
 315                                 }
 316                                 if (packOut != null)
 317                                         packOut.getChannel().force(true);
 318
 319                                 packDigest = null;
 320                                 baseById = null;
 321                                 baseByPos = null;
 322
 323                                 if (dstIdx != null)
 324                                         writeIdx();
 325
 326                         } finally {
 327                                 try {
 328                                         InflaterCache.release(inflater);
 329                                 } finally {
 330                                         inflater = null;
 331                                 }
 332                                 readCurs = WindowCursor.release(readCurs);
 333
 334                                 progress.endTask();
 335                                 if (packOut != null)
 336                                         packOut.close();
 337                         }
 338
 339                         if (dstPack != null)
 340                                 dstPack.setReadOnly();
 341                         if (dstIdx != null)
 342                                 dstIdx.setReadOnly();
 343                 } catch (IOException err) {
 344                         if (dstPack != null)
 345                                 dstPack.delete();
 346                         if (dstIdx != null)
 347                                 dstIdx.delete();
 348                         throw err;
 349                 }
 350         }
 351
 352         private void resolveDeltas(final ProgressMonitor progress)
 353                         throws IOException {
 354                 progress.beginTask(PROGRESS_RESOLVE_DELTA, deltaCount);
 355                 final int last = entryCount;
 356                 for (int i = 0; i < last; i++) {
 357                         final int before = entryCount;
 358                         resolveDeltas(entries[i]);
 359                         progress.update(entryCount - before);
 360                         if (progress.isCancelled())
 361                                 throw new IOException("Download cancelled during indexing");
 362                 }
 363                 progress.endTask();
 364         }
 365
 366         private void resolveDeltas(final PackedObjectInfo oe) throws IOException {
 367                 final int oldCRC = oe.getCRC();
 368                 if (baseById.containsKey(oe)
 369                                 || baseByPos.containsKey(new Long(oe.getOffset())))
 370                         resolveDeltas(oe.getOffset(), oldCRC, Constants.OBJ_BAD, null, oe);
 371         }
 372
 373         private void resolveDeltas(final long pos, final int oldCRC, int type,
 374                         byte[] data, PackedObjectInfo oe) throws IOException {
 375                 crc.reset();
 376                 position(pos);
 377                 int c = readFromFile();
 378                 final int typeCode = (c >> 4) & 7;
 379                 long sz = c & 15;
 380                 int shift = 4;
 381                 while ((c & 0x80) != 0) {
 382                         c = readFromFile();
 383                         sz += (c & 0x7f) << shift;
 384                         shift += 7;
 385                 }
 386
 387                 switch (typeCode) {
 388                 case Constants.OBJ_COMMIT:
 389                 case Constants.OBJ_TREE:
 390                 case Constants.OBJ_BLOB:
 391                 case Constants.OBJ_TAG:
 392                         type = typeCode;
 393                         data = inflateFromFile((int) sz);
 394                         break;
 395                 case Constants.OBJ_OFS_DELTA: {
 396                         c = readFromFile() & 0xff;
 397                         while ((c & 128) != 0)
 398                                 c = readFromFile() & 0xff;
 399                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 400                         break;
 401                 }
 402                 case Constants.OBJ_REF_DELTA: {
 403                         crc.update(buf, fillFromFile(20), 20);
 404                         use(20);
 405                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 406                         break;
 407                 }
 408                 default:
 409                         throw new IOException("Unknown object type " + typeCode + ".");
 410                 }
 411
 412                 final int crc32 = (int) crc.getValue();
 413                 if (oldCRC != crc32)
 414                         throw new IOException("Corruption detected re-reading at " + pos);
 415                 if (oe == null) {
 416                         objectDigest.update(Constants.encodedTypeString(type));
 417                         objectDigest.update((byte) ' ');
 418                         objectDigest.update(Constants.encodeASCII(data.length));
 419                         objectDigest.update((byte) 0);
 420                         objectDigest.update(data);
 421                         tempObjectId.fromRaw(objectDigest.digest(), 0);
 422
 423                         verifySafeObject(tempObjectId, type, data);
 424                         oe = new PackedObjectInfo(pos, crc32, tempObjectId);
 425                         entries[entryCount++] = oe;
 426                 }
 427
 428                 resolveChildDeltas(pos, type, data, oe);
 429         }
 430
 431         private void resolveChildDeltas(final long pos, int type, byte[] data,
 432                         PackedObjectInfo oe) throws IOException {
 433                 final ArrayList<UnresolvedDelta> a = baseById.remove(oe);
 434                 final ArrayList<UnresolvedDelta> b = baseByPos.remove(new Long(pos));
 435                 int ai = 0, bi = 0;
 436                 if (a != null && b != null) {
 437                         while (ai < a.size() && bi < b.size()) {
 438                                 final UnresolvedDelta ad = a.get(ai);
 439                                 final UnresolvedDelta bd = b.get(bi);
 440                                 if (ad.position < bd.position) {
 441                                         resolveDeltas(ad.position, ad.crc, type, data, null);
 442                                         ai++;
 443                                 } else {
 444                                         resolveDeltas(bd.position, bd.crc, type, data, null);
 445                                         bi++;
 446                                 }
 447                         }
 448                 }
 449                 if (a != null)
 450                         while (ai < a.size()) {
 451                                 final UnresolvedDelta ad = a.get(ai++);
 452                                 resolveDeltas(ad.position, ad.crc, type, data, null);
 453                         }
 454                 if (b != null)
 455                         while (bi < b.size()) {
 456                                 final UnresolvedDelta bd = b.get(bi++);
 457                                 resolveDeltas(bd.position, bd.crc, type, data, null);
 458                         }
 459         }
 460
 461         private void fixThinPack(final ProgressMonitor progress) throws IOException {
 462                 growEntries();
 463
 464                 packDigest.reset();
 465                 originalEOF = packOut.length() - 20;
 466                 final Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
 467                 long end = originalEOF;
 468                 for (final ObjectId baseId : new ArrayList<ObjectId>(baseById.keySet())) {
 469                         final ObjectLoader ldr = repo.openObject(readCurs, baseId);
 470                         if (ldr == null)
 471                                 continue;
 472                         final byte[] data = ldr.getBytes();
 473                         final int typeCode = ldr.getType();
 474                         final PackedObjectInfo oe;
 475
 476                         crc.reset();
 477                         packOut.seek(end);
 478                         writeWhole(def, typeCode, data);
 479                         oe = new PackedObjectInfo(end, (int) crc.getValue(), baseId);
 480                         entries[entryCount++] = oe;
 481                         end = packOut.getFilePointer();
 482
 483                         resolveChildDeltas(oe.getOffset(), typeCode, data, oe);
 484                         if (progress.isCancelled())
 485                                 throw new IOException("Download cancelled during indexing");
 486                 }
 487                 def.end();
 488
 489                 if (!baseById.isEmpty()) {
 490                         final ObjectId need = baseById.keySet().iterator().next();
 491                         throw new MissingObjectException(need, "delta base");
 492                 }
 493
 494                 fixHeaderFooter(packcsum, packDigest.digest());
 495         }
 496
 497         private void writeWhole(final Deflater def, final int typeCode,
 498                         final byte[] data) throws IOException {
 499                 int sz = data.length;
 500                 int hdrlen = 0;
 501                 buf[hdrlen++] = (byte) ((typeCode << 4) | sz & 15);
 502                 sz >>>= 4;
 503                 while (sz > 0) {
 504                         buf[hdrlen - 1] |= 0x80;
 505                         buf[hdrlen++] = (byte) (sz & 0x7f);
 506                         sz >>>= 7;
 507                 }
 508                 packDigest.update(buf, 0, hdrlen);
 509                 crc.update(buf, 0, hdrlen);
 510                 packOut.write(buf, 0, hdrlen);
 511                 def.reset();
 512                 def.setInput(data);
 513                 def.finish();
 514                 while (!def.finished()) {
 515                         final int datlen = def.deflate(buf);
 516                         packDigest.update(buf, 0, datlen);
 517                         crc.update(buf, 0, datlen);
 518                         packOut.write(buf, 0, datlen);
 519                 }
 520         }
 521
 522         private void fixHeaderFooter(final byte[] origcsum, final byte[] tailcsum)
 523                         throws IOException {
 524                 final MessageDigest origDigest = Constants.newMessageDigest();
 525                 final MessageDigest tailDigest = Constants.newMessageDigest();
 526                 long origRemaining = originalEOF;
 527
 528                 packOut.seek(0);
 529                 bAvail = 0;
 530                 bOffset = 0;
 531                 fillFromFile(12);
 532
 533                 {
 534                         final int origCnt = (int) Math.min(bAvail, origRemaining);
 535                         origDigest.update(buf, 0, origCnt);
 536                         origRemaining -= origCnt;
 537                         if (origRemaining == 0)
 538                                 tailDigest.update(buf, origCnt, bAvail - origCnt);
 539                 }
 540
 541                 NB.encodeInt32(buf, 8, entryCount);
 542                 packOut.seek(0);
 543                 packOut.write(buf, 0, 12);
 544                 packOut.seek(bAvail);
 545
 546                 packDigest.reset();
 547                 packDigest.update(buf, 0, bAvail);
 548                 for (;;) {
 549                         final int n = packOut.read(buf);
 550                         if (n < 0)
 551                                 break;
 552                         if (origRemaining != 0) {
 553                                 final int origCnt = (int) Math.min(n, origRemaining);
 554                                 origDigest.update(buf, 0, origCnt);
 555                                 origRemaining -= origCnt;
 556                                 if (origRemaining == 0)
 557                                         tailDigest.update(buf, origCnt, n - origCnt);
 558                         } else
 559                                 tailDigest.update(buf, 0, n);
 560
 561                         packDigest.update(buf, 0, n);
 562                 }
 563
 564                 if (!Arrays.equals(origDigest.digest(), origcsum)
 565                                 || !Arrays.equals(tailDigest.digest(), tailcsum))
 566                         throw new IOException("Pack corrupted while writing to filesystem");
 567
 568                 packcsum = packDigest.digest();
 569                 packOut.write(packcsum);
 570         }
 571
 572         private void growEntries() {
 573                 final PackedObjectInfo[] ne;
 574
 575                 ne = new PackedObjectInfo[(int) objectCount + baseById.size()];
 576                 System.arraycopy(entries, 0, ne, 0, entryCount);
 577                 entries = ne;
 578         }
 579
 580         private void writeIdx() throws IOException {
 581                 Arrays.sort(entries, 0, entryCount);
 582                 List<PackedObjectInfo> list = Arrays.asList(entries);
 583                 if (entryCount < entries.length)
 584                         list = list.subList(0, entryCount);
 585
 586                 final FileOutputStream os = new FileOutputStream(dstIdx);
 587                 try {
 588                         final PackIndexWriter iw;
 589                         if (outputVersion <= 0)
 590                                 iw = PackIndexWriter.createOldestPossible(os, list);
 591                         else
 592                                 iw = PackIndexWriter.createVersion(os, outputVersion);
 593                         iw.write(list, packcsum);
 594                         os.getChannel().force(true);
 595                 } finally {
 596                         os.close();
 597                 }
 598         }
 599
 600         private void readPackHeader() throws IOException {
 601                 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
 602                 final int p = fillFromInput(hdrln);
 603                 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
 604                         if (buf[p + k] != Constants.PACK_SIGNATURE[k])
 605                                 throw new IOException("Not a PACK file.");
 606
 607                 final long vers = NB.decodeUInt32(buf, p + 4);
 608                 if (vers != 2 && vers != 3)
 609                         throw new IOException("Unsupported pack version " + vers + ".");
 610                 objectCount = NB.decodeUInt32(buf, p + 8);
 611                 use(hdrln);
 612         }
 613
 614         private void readPackFooter() throws IOException {
 615                 sync();
 616                 final byte[] cmpcsum = packDigest.digest();
 617                 final int c = fillFromInput(20);
 618                 packcsum = new byte[20];
 619                 System.arraycopy(buf, c, packcsum, 0, 20);
 620                 use(20);
 621                 if (packOut != null)
 622                         packOut.write(packcsum);
 623
 624                 if (!Arrays.equals(cmpcsum, packcsum))
 625                         throw new CorruptObjectException("Packfile checksum incorrect.");
 626         }
 627
 628         // Cleanup all resources associated with our input parsing.
 629         private void endInput() {
 630                 in = null;
 631                 objectData = null;
 632         }
 633
 634         // Read one entire object or delta from the input.
 635         private void indexOneObject() throws IOException {
 636                 final long pos = position();
 637
 638                 crc.reset();
 639                 int c = readFromInput();
 640                 final int typeCode = (c >> 4) & 7;
 641                 long sz = c & 15;
 642                 int shift = 4;
 643                 while ((c & 0x80) != 0) {
 644                         c = readFromInput();
 645                         sz += (c & 0x7f) << shift;
 646                         shift += 7;
 647                 }
 648
 649                 switch (typeCode) {
 650                 case Constants.OBJ_COMMIT:
 651                 case Constants.OBJ_TREE:
 652                 case Constants.OBJ_BLOB:
 653                 case Constants.OBJ_TAG:
 654                         whole(typeCode, pos, sz);
 655                         break;
 656                 case Constants.OBJ_OFS_DELTA: {
 657                         c = readFromInput();
 658                         long ofs = c & 127;
 659                         while ((c & 128) != 0) {
 660                                 ofs += 1;
 661                                 c = readFromInput();
 662                                 ofs <<= 7;
 663                                 ofs += (c & 127);
 664                         }
 665                         final Long base = new Long(pos - ofs);
 666                         ArrayList<UnresolvedDelta> r = baseByPos.get(base);
 667                         if (r == null) {
 668                                 r = new ArrayList<UnresolvedDelta>(8);
 669                                 baseByPos.put(base, r);
 670                         }
 671                         skipInflateFromInput(sz);
 672                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 673                         deltaCount++;
 674                         break;
 675                 }
 676                 case Constants.OBJ_REF_DELTA: {
 677                         c = fillFromInput(20);
 678                         crc.update(buf, c, 20);
 679                         final ObjectId base = ObjectId.fromRaw(buf, c);
 680                         use(20);
 681                         ArrayList<UnresolvedDelta> r = baseById.get(base);
 682                         if (r == null) {
 683                                 r = new ArrayList<UnresolvedDelta>(8);
 684                                 baseById.put(base, r);
 685                         }
 686                         skipInflateFromInput(sz);
 687                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 688                         deltaCount++;
 689                         break;
 690                 }
 691                 default:
 692                         throw new IOException("Unknown object type " + typeCode + ".");
 693                 }
 694         }
 695
 696         private void whole(final int type, final long pos, final long sz)
 697                         throws IOException {
 698                 final byte[] data = inflateFromInput(sz);
 699                 objectDigest.update(Constants.encodedTypeString(type));
 700                 objectDigest.update((byte) ' ');
 701                 objectDigest.update(Constants.encodeASCII(sz));
 702                 objectDigest.update((byte) 0);
 703                 objectDigest.update(data);
 704                 tempObjectId.fromRaw(objectDigest.digest(), 0);
 705
 706                 verifySafeObject(tempObjectId, type, data);
 707                 final int crc32 = (int) crc.getValue();
 708                 entries[entryCount++] = new PackedObjectInfo(pos, crc32, tempObjectId);
 709         }
 710
 711         private void verifySafeObject(final AnyObjectId id, final int type,
 712                         final byte[] data) throws IOException {
 713                 if (objCheck != null) {
 714                         try {
 715                                 objCheck.check(type, data);
 716                         } catch (CorruptObjectException e) {
 717                                 throw new IOException("Invalid "
 718                                                 + Constants.encodedTypeString(type) + " " + id.name()
 719                                                 + ":" + e.getMessage());
 720                         }
 721                 }
 722
 723                 final ObjectLoader ldr = repo.openObject(readCurs, id);
 724                 if (ldr != null) {
 725                         final byte[] existingData = ldr.getCachedBytes();
 726                         if (ldr.getType() != type || !Arrays.equals(data, existingData)) {
 727                                 throw new IOException("Collision on " + id.name());
 728                         }
 729                 }
 730         }
 731
 732         // Current position of {@link #bOffset} within the entire file.
 733         private long position() {
 734                 return bBase + bOffset;
 735         }
 736
 737         private void position(final long pos) throws IOException {
 738                 packOut.seek(pos);
 739                 bBase = pos;
 740                 bOffset = 0;
 741                 bAvail = 0;
 742         }
 743
 744         // Consume exactly one byte from the buffer and return it.
 745         private int readFromInput() throws IOException {
 746                 if (bAvail == 0)
 747                         fillFromInput(1);
 748                 bAvail--;
 749                 final int b = buf[bOffset++] & 0xff;
 750                 crc.update(b);
 751                 return b;
 752         }
 753
 754         // Consume exactly one byte from the buffer and return it.
 755         private int readFromFile() throws IOException {
 756                 if (bAvail == 0)
 757                         fillFromFile(1);
 758                 bAvail--;
 759                 final int b = buf[bOffset++] & 0xff;
 760                 crc.update(b);
 761                 return b;
 762         }
 763
 764         // Consume cnt bytes from the buffer.
 765         private void use(final int cnt) {
 766                 bOffset += cnt;
 767                 bAvail -= cnt;
 768         }
 769
 770         // Ensure at least need bytes are available in in {@link #buf}.
 771         private int fillFromInput(final int need) throws IOException {
 772                 while (bAvail < need) {
 773                         int next = bOffset + bAvail;
 774                         int free = buf.length - next;
 775                         if (free + bAvail < need) {
 776                                 sync();
 777                                 next = bAvail;
 778                                 free = buf.length - next;
 779                         }
 780                         next = in.read(buf, next, free);
 781                         if (next <= 0)
 782                                 throw new EOFException("Packfile is truncated.");
 783                         bAvail += next;
 784                 }
 785                 return bOffset;
 786         }
 787
 788         // Ensure at least need bytes are available in in {@link #buf}.
 789         private int fillFromFile(final int need) throws IOException {
 790                 if (bAvail < need) {
 791                         int next = bOffset + bAvail;
 792                         int free = buf.length - next;
 793                         if (free + bAvail < need) {
 794                                 if (bAvail > 0)
 795                                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 796                                 bOffset = 0;
 797                                 next = bAvail;
 798                                 free = buf.length - next;
 799                         }
 800                         next = packOut.read(buf, next, free);
 801                         if (next <= 0)
 802                                 throw new EOFException("Packfile is truncated.");
 803                         bAvail += next;
 804                 }
 805                 return bOffset;
 806         }
 807
 808         // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
 809         private void sync() throws IOException {
 810                 packDigest.update(buf, 0, bOffset);
 811                 if (packOut != null)
 812                         packOut.write(buf, 0, bOffset);
 813                 if (bAvail > 0)
 814                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 815                 bBase += bOffset;
 816                 bOffset = 0;
 817         }
 818
 819         private void skipInflateFromInput(long sz) throws IOException {
 820                 final Inflater inf = inflater;
 821                 try {
 822                         final byte[] dst = objectData;
 823                         int n = 0;
 824                         int p = -1;
 825                         while (!inf.finished()) {
 826                                 if (inf.needsInput()) {
 827                                         if (p >= 0) {
 828                                                 crc.update(buf, p, bAvail);
 829                                                 use(bAvail);
 830                                         }
 831                                         p = fillFromInput(1);
 832                                         inf.setInput(buf, p, bAvail);
 833                                 }
 834
 835                                 int free = dst.length - n;
 836                                 if (free < 8) {
 837                                         sz -= n;
 838                                         n = 0;
 839                                         free = dst.length;
 840                                 }
 841                                 n += inf.inflate(dst, n, free);
 842                         }
 843                         if (n != sz)
 844                                 throw new DataFormatException("wrong decompressed length");
 845                         n = bAvail - inf.getRemaining();
 846                         if (n > 0) {
 847                                 crc.update(buf, p, n);
 848                                 use(n);
 849                         }
 850                 } catch (DataFormatException dfe) {
 851                         throw corrupt(dfe);
 852                 } finally {
 853                         inf.reset();
 854                 }
 855         }
 856
 857         private byte[] inflateFromInput(final long sz) throws IOException {
 858                 final byte[] dst = new byte[(int) sz];
 859                 final Inflater inf = inflater;
 860                 try {
 861                         int n = 0;
 862                         int p = -1;
 863                         while (!inf.finished()) {
 864                                 if (inf.needsInput()) {
 865                                         if (p >= 0) {
 866                                                 crc.update(buf, p, bAvail);
 867                                                 use(bAvail);
 868                                         }
 869                                         p = fillFromInput(1);
 870                                         inf.setInput(buf, p, bAvail);
 871                                 }
 872
 873                                 n += inf.inflate(dst, n, dst.length - n);
 874                         }
 875                         if (n != sz)
 876                                 throw new DataFormatException("wrong decompressed length");
 877                         n = bAvail - inf.getRemaining();
 878                         if (n > 0) {
 879                                 crc.update(buf, p, n);
 880                                 use(n);
 881                         }
 882                         return dst;
 883                 } catch (DataFormatException dfe) {
 884                         throw corrupt(dfe);
 885                 } finally {
 886                         inf.reset();
 887                 }
 888         }
 889
 890         private byte[] inflateFromFile(final int sz) throws IOException {
 891                 final Inflater inf = inflater;
 892                 try {
 893                         final byte[] dst = new byte[sz];
 894                         int n = 0;
 895                         int p = -1;
 896                         while (!inf.finished()) {
 897                                 if (inf.needsInput()) {
 898                                         if (p >= 0) {
 899                                                 crc.update(buf, p, bAvail);
 900                                                 use(bAvail);
 901                                         }
 902                                         p = fillFromFile(1);
 903                                         inf.setInput(buf, p, bAvail);
 904                                 }
 905                                 n += inf.inflate(dst, n, sz - n);
 906                         }
 907                         n = bAvail - inf.getRemaining();
 908                         if (n > 0) {
 909                                 crc.update(buf, p, n);
 910                                 use(n);
 911                         }
 912                         return dst;
 913                 } catch (DataFormatException dfe) {
 914                         throw corrupt(dfe);
 915                 } finally {
 916                         inf.reset();
 917                 }
 918         }
 919
 920         private static CorruptObjectException corrupt(final DataFormatException dfe) {
 921                 return new CorruptObjectException("Packfile corruption detected: "
 922                                 + dfe.getMessage());
 923         }
 924
 925         private static class UnresolvedDelta {
 926                 final long position;
 927
 928                 final int crc;
 929
 930                 UnresolvedDelta(final long headerOffset, final int crc32) {
 931                         position = headerOffset;
 932                         crc = crc32;
 933                 }
 934         }
 935
 936         /**
 937          * Rename the pack to it's final name and location and open it.
 938          * <p>
 939          * If the call completes successfully the repository this IndexPack instance
 940          * was created with will have the objects in the pack available for reading
 941          * and use, without needing to scan for packs.
 942          *
 943          * @throws IOException
 944          *             The pack could not be inserted into the repository's objects
 945          *             directory. The pack no longer exists on disk, as it was
 946          *             removed prior to throwing the exception to the caller.
 947          */
 948         public void renameAndOpenPack() throws IOException {
 949                 final MessageDigest d = Constants.newMessageDigest();
 950                 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
 951                 for (int i = 0; i < entryCount; i++) {
 952                         final PackedObjectInfo oe = entries[i];
 953                         oe.copyRawTo(oeBytes, 0);
 954                         d.update(oeBytes);
 955                 }
 956
 957                 final String name = ObjectId.fromRaw(d.digest()).name();
 958                 final File packDir = new File(repo.getObjectsDirectory(), "pack");
 959                 final File finalPack = new File(packDir, "pack-" + name + ".pack");
 960                 final File finalIdx = new File(packDir, "pack-" + name + ".idx");
 961
 962                 if (finalPack.exists()) {
 963                         // If the pack is already present we should never replace it.
 964                         //
 965                         cleanupTemporaryFiles();
 966                         return;
 967                 }
 968
 969                 if (!dstPack.renameTo(finalPack)) {
 970                         cleanupTemporaryFiles();
 971                         throw new IOException("Cannot move pack to " + finalPack);
 972                 }
 973
 974                 if (!dstIdx.renameTo(finalIdx)) {
 975                         cleanupTemporaryFiles();
 976                         if (!finalPack.delete())
 977                                 finalPack.deleteOnExit();
 978                         throw new IOException("Cannot move index to " + finalIdx);
 979                 }
 980
 981                 try {
 982                         repo.openPack(finalPack, finalIdx);
 983                 } catch (IOException err) {
 984                         finalPack.delete();
 985                         finalIdx.delete();
 986                         throw err;
 987                 }
 988         }
 989
 990         private void cleanupTemporaryFiles() {
 991                 if (!dstIdx.delete())
 992                         dstIdx.deleteOnExit();
 993                 if (!dstPack.delete())
 994                         dstPack.deleteOnExit();
 995         }
 996 }