org.spearce.jgit/src/org/spearce/jgit/transport/IndexPack.java

   1 /*
   2  * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
   3  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   4  *
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  * - Redistributions of source code must retain the above copyright
  12  *   notice, this list of conditions and the following disclaimer.
  13  *
  14  * - Redistributions in binary form must reproduce the above
  15  *   copyright notice, this list of conditions and the following
  16  *   disclaimer in the documentation and/or other materials provided
  17  *   with the distribution.
  18  *
  19  * - Neither the name of the Git Development Community nor the
  20  *   names of its contributors may be used to endorse or promote
  21  *   products derived from this software without specific prior
  22  *   written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  25  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  26  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  31  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 package org.spearce.jgit.transport;
  40
  41 import java.io.EOFException;
  42 import java.io.File;
  43 import java.io.FileOutputStream;
  44 import java.io.IOException;
  45 import java.io.InputStream;
  46 import java.io.RandomAccessFile;
  47 import java.security.MessageDigest;
  48 import java.util.ArrayList;
  49 import java.util.Arrays;
  50 import java.util.HashMap;
  51 import java.util.List;
  52 import java.util.zip.CRC32;
  53 import java.util.zip.DataFormatException;
  54 import java.util.zip.Deflater;
  55 import java.util.zip.Inflater;
  56
  57 import org.spearce.jgit.errors.CorruptObjectException;
  58 import org.spearce.jgit.errors.MissingObjectException;
  59 import org.spearce.jgit.lib.AnyObjectId;
  60 import org.spearce.jgit.lib.BinaryDelta;
  61 import org.spearce.jgit.lib.Constants;
  62 import org.spearce.jgit.lib.InflaterCache;
  63 import org.spearce.jgit.lib.MutableObjectId;
  64 import org.spearce.jgit.lib.ObjectChecker;
  65 import org.spearce.jgit.lib.ObjectId;
  66 import org.spearce.jgit.lib.ObjectIdMap;
  67 import org.spearce.jgit.lib.ObjectLoader;
  68 import org.spearce.jgit.lib.PackIndexWriter;
  69 import org.spearce.jgit.lib.ProgressMonitor;
  70 import org.spearce.jgit.lib.Repository;
  71 import org.spearce.jgit.util.NB;
  72
  73 /** Indexes Git pack files for local use. */
  74 public class IndexPack {
  75         /** Progress message when reading raw data from the pack. */
  76         public static final String PROGRESS_DOWNLOAD = "Receiving objects";
  77
  78         /** Progress message when computing names of delta compressed objects. */
  79         public static final String PROGRESS_RESOLVE_DELTA = "Resolving deltas";
  80
  81         /**
  82          * Size of the internal stream buffer.
  83          * <p>
  84          * If callers are going to be supplying IndexPack a BufferedInputStream they
  85          * should use this buffer size as the size of the buffer for that
  86          * BufferedInputStream, and any other its may be wrapping. This way the
  87          * buffers will cascade efficiently and only the IndexPack buffer will be
  88          * receiving the bulk of the data stream.
  89          */
  90         public static final int BUFFER_SIZE = 8192;
  91
  92         /**
  93          * Create an index pack instance to load a new pack into a repository.
  94          * <p>
  95          * The received pack data and generated index will be saved to temporary
  96          * files within the repository's <code>objects</code> directory. To use the
  97          * data contained within them call {@link #renameAndOpenPack()} once the
  98          * indexing is complete.
  99          *
 100          * @param db
 101          *            the repository that will receive the new pack.
 102          * @param is
 103          *            stream to read the pack data from. If the stream is buffered
 104          *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
 105          * @return a new index pack instance.
 106          * @throws IOException
 107          *             a temporary file could not be created.
 108          */
 109         public static IndexPack create(final Repository db, final InputStream is)
 110                         throws IOException {
 111                 final String suffix = ".pack";
 112                 final File objdir = db.getObjectsDirectory();
 113                 final File tmp = File.createTempFile("incoming_", suffix, objdir);
 114                 final String n = tmp.getName();
 115                 final File base;
 116
 117                 base = new File(objdir, n.substring(0, n.length() - suffix.length()));
 118                 final IndexPack ip = new IndexPack(db, is, base);
 119                 ip.setIndexVersion(db.getConfig().getCore().getPackIndexVersion());
 120                 return ip;
 121         }
 122
 123         private final Repository repo;
 124
 125         private Inflater inflater;
 126
 127         private final MessageDigest objectDigest;
 128
 129         private final MutableObjectId tempObjectId;
 130
 131         private InputStream in;
 132
 133         private byte[] buf;
 134
 135         private long bBase;
 136
 137         private int bOffset;
 138
 139         private int bAvail;
 140
 141         private ObjectChecker objCheck;
 142
 143         private boolean fixThin;
 144
 145         private int outputVersion;
 146
 147         private final File dstPack;
 148
 149         private final File dstIdx;
 150
 151         private long objectCount;
 152
 153         private PackedObjectInfo[] entries;
 154
 155         private int deltaCount;
 156
 157         private int entryCount;
 158
 159         private final CRC32 crc = new CRC32();
 160
 161         private ObjectIdMap<ArrayList<UnresolvedDelta>> baseById;
 162
 163         private HashMap<Long, ArrayList<UnresolvedDelta>> baseByPos;
 164
 165         private byte[] objectData;
 166
 167         private MessageDigest packDigest;
 168
 169         private RandomAccessFile packOut;
 170
 171         private byte[] packcsum;
 172
 173         /** If {@link #fixThin} this is the last byte of the original checksum. */
 174         private long originalEOF;
 175
 176         /**
 177          * Create a new pack indexer utility.
 178          *
 179          * @param db
 180          * @param src
 181          *            stream to read the pack data from. If the stream is buffered
 182          *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
 183          * @param dstBase
 184          * @throws IOException
 185          *             the output packfile could not be created.
 186          */
 187         public IndexPack(final Repository db, final InputStream src,
 188                         final File dstBase) throws IOException {
 189                 repo = db;
 190                 in = src;
 191                 inflater = InflaterCache.get();
 192                 buf = new byte[BUFFER_SIZE];
 193                 objectData = new byte[BUFFER_SIZE];
 194                 objectDigest = Constants.newMessageDigest();
 195                 tempObjectId = new MutableObjectId();
 196                 packDigest = Constants.newMessageDigest();
 197
 198                 if (dstBase != null) {
 199                         final File dir = dstBase.getParentFile();
 200                         final String nam = dstBase.getName();
 201                         dstPack = new File(dir, nam + ".pack");
 202                         dstIdx = new File(dir, nam + ".idx");
 203                         packOut = new RandomAccessFile(dstPack, "rw");
 204                         packOut.setLength(0);
 205                 } else {
 206                         dstPack = null;
 207                         dstIdx = null;
 208                 }
 209         }
 210
 211         /**
 212          * Set the pack index file format version this instance will create.
 213          *
 214          * @param version
 215          *            the version to write. The special version 0 designates the
 216          *            oldest (most compatible) format available for the objects.
 217          * @see PackIndexWriter
 218          */
 219         public void setIndexVersion(final int version) {
 220                 outputVersion = version;
 221         }
 222
 223         /**
 224          * Configure this index pack instance to make a thin pack complete.
 225          * <p>
 226          * Thin packs are sometimes used during network transfers to allow a delta
 227          * to be sent without a base object. Such packs are not permitted on disk.
 228          * They can be fixed by copying the base object onto the end of the pack.
 229          *
 230          * @param fix
 231          *            true to enable fixing a thin pack.
 232          */
 233         public void setFixThin(final boolean fix) {
 234                 fixThin = fix;
 235         }
 236
 237         /**
 238          * Configure the checker used to validate received objects.
 239          * <p>
 240          * Usually object checking isn't necessary, as Git implementations only
 241          * create valid objects in pack files. However, additional checking may be
 242          * useful if processing data from an untrusted source.
 243          *
 244          * @param oc
 245          *            the checker instance; null to disable object checking.
 246          */
 247         public void setObjectChecker(final ObjectChecker oc) {
 248                 objCheck = oc;
 249         }
 250
 251         /**
 252          * Configure the checker used to validate received objects.
 253          * <p>
 254          * Usually object checking isn't necessary, as Git implementations only
 255          * create valid objects in pack files. However, additional checking may be
 256          * useful if processing data from an untrusted source.
 257          * <p>
 258          * This is shorthand for:
 259          *
 260          * <pre>
 261          * setObjectChecker(on ? new ObjectChecker() : null);
 262          * </pre>
 263          *
 264          * @param on
 265          *            true to enable the default checker; false to disable it.
 266          */
 267         public void setObjectChecking(final boolean on) {
 268                 setObjectChecker(on ? new ObjectChecker() : null);
 269         }
 270
 271         /**
 272          * Consume data from the input stream until the packfile is indexed.
 273          *
 274          * @param progress
 275          *            progress feedback
 276          *
 277          * @throws IOException
 278          */
 279         public void index(final ProgressMonitor progress) throws IOException {
 280                 progress.start(2 /* tasks */);
 281                 try {
 282                         try {
 283                                 readPackHeader();
 284
 285                                 entries = new PackedObjectInfo[(int) objectCount];
 286                                 baseById = new ObjectIdMap<ArrayList<UnresolvedDelta>>();
 287                                 baseByPos = new HashMap<Long, ArrayList<UnresolvedDelta>>();
 288
 289                                 progress.beginTask(PROGRESS_DOWNLOAD, (int) objectCount);
 290                                 for (int done = 0; done < objectCount; done++) {
 291                                         indexOneObject();
 292                                         progress.update(1);
 293                                         if (progress.isCancelled())
 294                                                 throw new IOException("Download cancelled");
 295                                 }
 296                                 readPackFooter();
 297                                 endInput();
 298                                 progress.endTask();
 299                                 if (deltaCount > 0) {
 300                                         if (packOut == null)
 301                                                 throw new IOException("need packOut");
 302                                         resolveDeltas(progress);
 303                                         if (entryCount < objectCount) {
 304                                                 if (!fixThin) {
 305                                                         throw new IOException("pack has "
 306                                                                         + (objectCount - entryCount)
 307                                                                         + " unresolved deltas");
 308                                                 }
 309                                                 fixThinPack(progress);
 310                                         }
 311                                 }
 312                                 if (packOut != null)
 313                                         packOut.getChannel().force(true);
 314
 315                                 packDigest = null;
 316                                 baseById = null;
 317                                 baseByPos = null;
 318
 319                                 if (dstIdx != null)
 320                                         writeIdx();
 321
 322                         } finally {
 323                                 final Inflater inf = inflater;
 324                                 inflater = null;
 325                                 InflaterCache.release(inf);
 326
 327                                 progress.endTask();
 328                                 if (packOut != null)
 329                                         packOut.close();
 330                         }
 331
 332                         if (dstPack != null)
 333                                 dstPack.setReadOnly();
 334                         if (dstIdx != null)
 335                                 dstIdx.setReadOnly();
 336                 } catch (IOException err) {
 337                         if (dstPack != null)
 338                                 dstPack.delete();
 339                         if (dstIdx != null)
 340                                 dstIdx.delete();
 341                         throw err;
 342                 }
 343         }
 344
 345         private void resolveDeltas(final ProgressMonitor progress)
 346                         throws IOException {
 347                 progress.beginTask(PROGRESS_RESOLVE_DELTA, deltaCount);
 348                 final int last = entryCount;
 349                 for (int i = 0; i < last; i++) {
 350                         final int before = entryCount;
 351                         resolveDeltas(entries[i]);
 352                         progress.update(entryCount - before);
 353                         if (progress.isCancelled())
 354                                 throw new IOException("Download cancelled during indexing");
 355                 }
 356                 progress.endTask();
 357         }
 358
 359         private void resolveDeltas(final PackedObjectInfo oe) throws IOException {
 360                 final int oldCRC = oe.getCRC();
 361                 if (baseById.containsKey(oe)
 362                                 || baseByPos.containsKey(new Long(oe.getOffset())))
 363                         resolveDeltas(oe.getOffset(), oldCRC, Constants.OBJ_BAD, null, oe);
 364         }
 365
 366         private void resolveDeltas(final long pos, final int oldCRC, int type,
 367                         byte[] data, PackedObjectInfo oe) throws IOException {
 368                 crc.reset();
 369                 position(pos);
 370                 int c = readFromFile();
 371                 final int typeCode = (c >> 4) & 7;
 372                 long sz = c & 15;
 373                 int shift = 4;
 374                 while ((c & 0x80) != 0) {
 375                         c = readFromFile();
 376                         sz += (c & 0x7f) << shift;
 377                         shift += 7;
 378                 }
 379
 380                 switch (typeCode) {
 381                 case Constants.OBJ_COMMIT:
 382                 case Constants.OBJ_TREE:
 383                 case Constants.OBJ_BLOB:
 384                 case Constants.OBJ_TAG:
 385                         type = typeCode;
 386                         data = inflateFromFile((int) sz);
 387                         break;
 388                 case Constants.OBJ_OFS_DELTA: {
 389                         c = readFromFile() & 0xff;
 390                         while ((c & 128) != 0)
 391                                 c = readFromFile() & 0xff;
 392                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 393                         break;
 394                 }
 395                 case Constants.OBJ_REF_DELTA: {
 396                         crc.update(buf, fillFromFile(20), 20);
 397                         use(20);
 398                         data = BinaryDelta.apply(data, inflateFromFile((int) sz));
 399                         break;
 400                 }
 401                 default:
 402                         throw new IOException("Unknown object type " + typeCode + ".");
 403                 }
 404
 405                 final int crc32 = (int) crc.getValue();
 406                 if (oldCRC != crc32)
 407                         throw new IOException("Corruption detected re-reading at " + pos);
 408                 if (oe == null) {
 409                         objectDigest.update(Constants.encodedTypeString(type));
 410                         objectDigest.update((byte) ' ');
 411                         objectDigest.update(Constants.encodeASCII(data.length));
 412                         objectDigest.update((byte) 0);
 413                         objectDigest.update(data);
 414                         tempObjectId.fromRaw(objectDigest.digest(), 0);
 415
 416                         verifySafeObject(tempObjectId, type, data);
 417                         oe = new PackedObjectInfo(pos, crc32, tempObjectId);
 418                         entries[entryCount++] = oe;
 419                 }
 420
 421                 resolveChildDeltas(pos, type, data, oe);
 422         }
 423
 424         private void resolveChildDeltas(final long pos, int type, byte[] data,
 425                         PackedObjectInfo oe) throws IOException {
 426                 final ArrayList<UnresolvedDelta> a = baseById.remove(oe);
 427                 final ArrayList<UnresolvedDelta> b = baseByPos.remove(new Long(pos));
 428                 int ai = 0, bi = 0;
 429                 if (a != null && b != null) {
 430                         while (ai < a.size() && bi < b.size()) {
 431                                 final UnresolvedDelta ad = a.get(ai);
 432                                 final UnresolvedDelta bd = b.get(bi);
 433                                 if (ad.position < bd.position) {
 434                                         resolveDeltas(ad.position, ad.crc, type, data, null);
 435                                         ai++;
 436                                 } else {
 437                                         resolveDeltas(bd.position, bd.crc, type, data, null);
 438                                         bi++;
 439                                 }
 440                         }
 441                 }
 442                 if (a != null)
 443                         while (ai < a.size()) {
 444                                 final UnresolvedDelta ad = a.get(ai++);
 445                                 resolveDeltas(ad.position, ad.crc, type, data, null);
 446                         }
 447                 if (b != null)
 448                         while (bi < b.size()) {
 449                                 final UnresolvedDelta bd = b.get(bi++);
 450                                 resolveDeltas(bd.position, bd.crc, type, data, null);
 451                         }
 452         }
 453
 454         private void fixThinPack(final ProgressMonitor progress) throws IOException {
 455                 growEntries();
 456
 457                 packDigest.reset();
 458                 originalEOF = packOut.length() - 20;
 459                 final Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
 460                 long end = originalEOF;
 461                 for (final ObjectId baseId : new ArrayList<ObjectId>(baseById.keySet())) {
 462                         final ObjectLoader ldr = repo.openObject(baseId);
 463                         if (ldr == null)
 464                                 continue;
 465                         final byte[] data = ldr.getBytes();
 466                         final int typeCode = ldr.getType();
 467                         final PackedObjectInfo oe;
 468
 469                         crc.reset();
 470                         packOut.seek(end);
 471                         writeWhole(def, typeCode, data);
 472                         oe = new PackedObjectInfo(end, (int) crc.getValue(), baseId);
 473                         entries[entryCount++] = oe;
 474                         end = packOut.getFilePointer();
 475
 476                         resolveChildDeltas(oe.getOffset(), typeCode, data, oe);
 477                         if (progress.isCancelled())
 478                                 throw new IOException("Download cancelled during indexing");
 479                 }
 480                 def.end();
 481
 482                 if (!baseById.isEmpty()) {
 483                         final ObjectId need = baseById.keySet().iterator().next();
 484                         throw new MissingObjectException(need, "delta base");
 485                 }
 486
 487                 fixHeaderFooter(packcsum, packDigest.digest());
 488         }
 489
 490         private void writeWhole(final Deflater def, final int typeCode,
 491                         final byte[] data) throws IOException {
 492                 int sz = data.length;
 493                 int hdrlen = 0;
 494                 buf[hdrlen++] = (byte) ((typeCode << 4) | sz & 15);
 495                 sz >>>= 4;
 496                 while (sz > 0) {
 497                         buf[hdrlen - 1] |= 0x80;
 498                         buf[hdrlen++] = (byte) (sz & 0x7f);
 499                         sz >>>= 7;
 500                 }
 501                 packDigest.update(buf, 0, hdrlen);
 502                 crc.update(buf, 0, hdrlen);
 503                 packOut.write(buf, 0, hdrlen);
 504                 def.reset();
 505                 def.setInput(data);
 506                 def.finish();
 507                 while (!def.finished()) {
 508                         final int datlen = def.deflate(buf);
 509                         packDigest.update(buf, 0, datlen);
 510                         crc.update(buf, 0, datlen);
 511                         packOut.write(buf, 0, datlen);
 512                 }
 513         }
 514
 515         private void fixHeaderFooter(final byte[] origcsum, final byte[] tailcsum)
 516                         throws IOException {
 517                 final MessageDigest origDigest = Constants.newMessageDigest();
 518                 final MessageDigest tailDigest = Constants.newMessageDigest();
 519                 long origRemaining = originalEOF;
 520
 521                 packOut.seek(0);
 522                 bAvail = 0;
 523                 bOffset = 0;
 524                 fillFromFile(12);
 525
 526                 {
 527                         final int origCnt = (int) Math.min(bAvail, origRemaining);
 528                         origDigest.update(buf, 0, origCnt);
 529                         origRemaining -= origCnt;
 530                         if (origRemaining == 0)
 531                                 tailDigest.update(buf, origCnt, bAvail - origCnt);
 532                 }
 533
 534                 NB.encodeInt32(buf, 8, entryCount);
 535                 packOut.seek(0);
 536                 packOut.write(buf, 0, 12);
 537                 packOut.seek(bAvail);
 538
 539                 packDigest.reset();
 540                 packDigest.update(buf, 0, bAvail);
 541                 for (;;) {
 542                         final int n = packOut.read(buf);
 543                         if (n < 0)
 544                                 break;
 545                         if (origRemaining != 0) {
 546                                 final int origCnt = (int) Math.min(n, origRemaining);
 547                                 origDigest.update(buf, 0, origCnt);
 548                                 origRemaining -= origCnt;
 549                                 if (origRemaining == 0)
 550                                         tailDigest.update(buf, origCnt, n - origCnt);
 551                         } else
 552                                 tailDigest.update(buf, 0, n);
 553
 554                         packDigest.update(buf, 0, n);
 555                 }
 556
 557                 if (!Arrays.equals(origDigest.digest(), origcsum)
 558                                 || !Arrays.equals(tailDigest.digest(), tailcsum))
 559                         throw new IOException("Pack corrupted while writing to filesystem");
 560
 561                 packcsum = packDigest.digest();
 562                 packOut.write(packcsum);
 563         }
 564
 565         private void growEntries() {
 566                 final PackedObjectInfo[] ne;
 567
 568                 ne = new PackedObjectInfo[(int) objectCount + baseById.size()];
 569                 System.arraycopy(entries, 0, ne, 0, entryCount);
 570                 entries = ne;
 571         }
 572
 573         private void writeIdx() throws IOException {
 574                 Arrays.sort(entries, 0, entryCount);
 575                 List<PackedObjectInfo> list = Arrays.asList(entries);
 576                 if (entryCount < entries.length)
 577                         list = list.subList(0, entryCount);
 578
 579                 final FileOutputStream os = new FileOutputStream(dstIdx);
 580                 try {
 581                         final PackIndexWriter iw;
 582                         if (outputVersion <= 0)
 583                                 iw = PackIndexWriter.createOldestPossible(os, list);
 584                         else
 585                                 iw = PackIndexWriter.createVersion(os, outputVersion);
 586                         iw.write(list, packcsum);
 587                         os.getChannel().force(true);
 588                 } finally {
 589                         os.close();
 590                 }
 591         }
 592
 593         private void readPackHeader() throws IOException {
 594                 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
 595                 final int p = fillFromInput(hdrln);
 596                 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
 597                         if (buf[p + k] != Constants.PACK_SIGNATURE[k])
 598                                 throw new IOException("Not a PACK file.");
 599
 600                 final long vers = NB.decodeUInt32(buf, p + 4);
 601                 if (vers != 2 && vers != 3)
 602                         throw new IOException("Unsupported pack version " + vers + ".");
 603                 objectCount = NB.decodeUInt32(buf, p + 8);
 604                 use(hdrln);
 605         }
 606
 607         private void readPackFooter() throws IOException {
 608                 sync();
 609                 final byte[] cmpcsum = packDigest.digest();
 610                 final int c = fillFromInput(20);
 611                 packcsum = new byte[20];
 612                 System.arraycopy(buf, c, packcsum, 0, 20);
 613                 use(20);
 614                 if (packOut != null)
 615                         packOut.write(packcsum);
 616
 617                 if (!Arrays.equals(cmpcsum, packcsum))
 618                         throw new CorruptObjectException("Packfile checksum incorrect.");
 619         }
 620
 621         // Cleanup all resources associated with our input parsing.
 622         private void endInput() {
 623                 in = null;
 624                 objectData = null;
 625         }
 626
 627         // Read one entire object or delta from the input.
 628         private void indexOneObject() throws IOException {
 629                 final long pos = position();
 630
 631                 crc.reset();
 632                 int c = readFromInput();
 633                 final int typeCode = (c >> 4) & 7;
 634                 long sz = c & 15;
 635                 int shift = 4;
 636                 while ((c & 0x80) != 0) {
 637                         c = readFromInput();
 638                         sz += (c & 0x7f) << shift;
 639                         shift += 7;
 640                 }
 641
 642                 switch (typeCode) {
 643                 case Constants.OBJ_COMMIT:
 644                 case Constants.OBJ_TREE:
 645                 case Constants.OBJ_BLOB:
 646                 case Constants.OBJ_TAG:
 647                         whole(typeCode, pos, sz);
 648                         break;
 649                 case Constants.OBJ_OFS_DELTA: {
 650                         c = readFromInput();
 651                         long ofs = c & 127;
 652                         while ((c & 128) != 0) {
 653                                 ofs += 1;
 654                                 c = readFromInput();
 655                                 ofs <<= 7;
 656                                 ofs += (c & 127);
 657                         }
 658                         final Long base = new Long(pos - ofs);
 659                         ArrayList<UnresolvedDelta> r = baseByPos.get(base);
 660                         if (r == null) {
 661                                 r = new ArrayList<UnresolvedDelta>(8);
 662                                 baseByPos.put(base, r);
 663                         }
 664                         skipInflateFromInput(sz);
 665                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 666                         deltaCount++;
 667                         break;
 668                 }
 669                 case Constants.OBJ_REF_DELTA: {
 670                         c = fillFromInput(20);
 671                         crc.update(buf, c, 20);
 672                         final ObjectId base = ObjectId.fromRaw(buf, c);
 673                         use(20);
 674                         ArrayList<UnresolvedDelta> r = baseById.get(base);
 675                         if (r == null) {
 676                                 r = new ArrayList<UnresolvedDelta>(8);
 677                                 baseById.put(base, r);
 678                         }
 679                         skipInflateFromInput(sz);
 680                         r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
 681                         deltaCount++;
 682                         break;
 683                 }
 684                 default:
 685                         throw new IOException("Unknown object type " + typeCode + ".");
 686                 }
 687         }
 688
 689         private void whole(final int type, final long pos, final long sz)
 690                         throws IOException {
 691                 final byte[] data = inflateFromInput(sz);
 692                 objectDigest.update(Constants.encodedTypeString(type));
 693                 objectDigest.update((byte) ' ');
 694                 objectDigest.update(Constants.encodeASCII(sz));
 695                 objectDigest.update((byte) 0);
 696                 objectDigest.update(data);
 697                 tempObjectId.fromRaw(objectDigest.digest(), 0);
 698
 699                 verifySafeObject(tempObjectId, type, data);
 700                 final int crc32 = (int) crc.getValue();
 701                 entries[entryCount++] = new PackedObjectInfo(pos, crc32, tempObjectId);
 702         }
 703
 704         private void verifySafeObject(final AnyObjectId id, final int type,
 705                         final byte[] data) throws IOException {
 706                 if (objCheck != null) {
 707                         try {
 708                                 objCheck.check(type, data);
 709                         } catch (CorruptObjectException e) {
 710                                 throw new IOException("Invalid "
 711                                                 + Constants.encodedTypeString(type) + " " + id.name()
 712                                                 + ":" + e.getMessage());
 713                         }
 714                 }
 715
 716                 final ObjectLoader ldr = repo.openObject(id);
 717                 if (ldr != null) {
 718                         final byte[] existingData = ldr.getCachedBytes();
 719                         if (ldr.getType() != type || !Arrays.equals(data, existingData)) {
 720                                 throw new IOException("Collision on " + id.name());
 721                         }
 722                 }
 723         }
 724
 725         // Current position of {@link #bOffset} within the entire file.
 726         private long position() {
 727                 return bBase + bOffset;
 728         }
 729
 730         private void position(final long pos) throws IOException {
 731                 packOut.seek(pos);
 732                 bBase = pos;
 733                 bOffset = 0;
 734                 bAvail = 0;
 735         }
 736
 737         // Consume exactly one byte from the buffer and return it.
 738         private int readFromInput() throws IOException {
 739                 if (bAvail == 0)
 740                         fillFromInput(1);
 741                 bAvail--;
 742                 final int b = buf[bOffset++] & 0xff;
 743                 crc.update(b);
 744                 return b;
 745         }
 746
 747         // Consume exactly one byte from the buffer and return it.
 748         private int readFromFile() throws IOException {
 749                 if (bAvail == 0)
 750                         fillFromFile(1);
 751                 bAvail--;
 752                 final int b = buf[bOffset++] & 0xff;
 753                 crc.update(b);
 754                 return b;
 755         }
 756
 757         // Consume cnt bytes from the buffer.
 758         private void use(final int cnt) {
 759                 bOffset += cnt;
 760                 bAvail -= cnt;
 761         }
 762
 763         // Ensure at least need bytes are available in in {@link #buf}.
 764         private int fillFromInput(final int need) throws IOException {
 765                 while (bAvail < need) {
 766                         int next = bOffset + bAvail;
 767                         int free = buf.length - next;
 768                         if (free + bAvail < need) {
 769                                 sync();
 770                                 next = bAvail;
 771                                 free = buf.length - next;
 772                         }
 773                         next = in.read(buf, next, free);
 774                         if (next <= 0)
 775                                 throw new EOFException("Packfile is truncated.");
 776                         bAvail += next;
 777                 }
 778                 return bOffset;
 779         }
 780
 781         // Ensure at least need bytes are available in in {@link #buf}.
 782         private int fillFromFile(final int need) throws IOException {
 783                 if (bAvail < need) {
 784                         int next = bOffset + bAvail;
 785                         int free = buf.length - next;
 786                         if (free + bAvail < need) {
 787                                 if (bAvail > 0)
 788                                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 789                                 bOffset = 0;
 790                                 next = bAvail;
 791                                 free = buf.length - next;
 792                         }
 793                         next = packOut.read(buf, next, free);
 794                         if (next <= 0)
 795                                 throw new EOFException("Packfile is truncated.");
 796                         bAvail += next;
 797                 }
 798                 return bOffset;
 799         }
 800
 801         // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
 802         private void sync() throws IOException {
 803                 packDigest.update(buf, 0, bOffset);
 804                 if (packOut != null)
 805                         packOut.write(buf, 0, bOffset);
 806                 if (bAvail > 0)
 807                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
 808                 bBase += bOffset;
 809                 bOffset = 0;
 810         }
 811
 812         private void skipInflateFromInput(long sz) throws IOException {
 813                 final Inflater inf = inflater;
 814                 try {
 815                         final byte[] dst = objectData;
 816                         int n = 0;
 817                         int p = -1;
 818                         while (!inf.finished()) {
 819                                 if (inf.needsInput()) {
 820                                         if (p >= 0) {
 821                                                 crc.update(buf, p, bAvail);
 822                                                 use(bAvail);
 823                                         }
 824                                         p = fillFromInput(1);
 825                                         inf.setInput(buf, p, bAvail);
 826                                 }
 827
 828                                 int free = dst.length - n;
 829                                 if (free < 8) {
 830                                         sz -= n;
 831                                         n = 0;
 832                                         free = dst.length;
 833                                 }
 834                                 n += inf.inflate(dst, n, free);
 835                         }
 836                         if (n != sz)
 837                                 throw new DataFormatException("wrong decompressed length");
 838                         n = bAvail - inf.getRemaining();
 839                         if (n > 0) {
 840                                 crc.update(buf, p, n);
 841                                 use(n);
 842                         }
 843                 } catch (DataFormatException dfe) {
 844                         throw corrupt(dfe);
 845                 } finally {
 846                         inf.reset();
 847                 }
 848         }
 849
 850         private byte[] inflateFromInput(final long sz) throws IOException {
 851                 final byte[] dst = new byte[(int) sz];
 852                 final Inflater inf = inflater;
 853                 try {
 854                         int n = 0;
 855                         int p = -1;
 856                         while (!inf.finished()) {
 857                                 if (inf.needsInput()) {
 858                                         if (p >= 0) {
 859                                                 crc.update(buf, p, bAvail);
 860                                                 use(bAvail);
 861                                         }
 862                                         p = fillFromInput(1);
 863                                         inf.setInput(buf, p, bAvail);
 864                                 }
 865
 866                                 n += inf.inflate(dst, n, dst.length - n);
 867                         }
 868                         if (n != sz)
 869                                 throw new DataFormatException("wrong decompressed length");
 870                         n = bAvail - inf.getRemaining();
 871                         if (n > 0) {
 872                                 crc.update(buf, p, n);
 873                                 use(n);
 874                         }
 875                         return dst;
 876                 } catch (DataFormatException dfe) {
 877                         throw corrupt(dfe);
 878                 } finally {
 879                         inf.reset();
 880                 }
 881         }
 882
 883         private byte[] inflateFromFile(final int sz) throws IOException {
 884                 final Inflater inf = inflater;
 885                 try {
 886                         final byte[] dst = new byte[sz];
 887                         int n = 0;
 888                         int p = -1;
 889                         while (!inf.finished()) {
 890                                 if (inf.needsInput()) {
 891                                         if (p >= 0) {
 892                                                 crc.update(buf, p, bAvail);
 893                                                 use(bAvail);
 894                                         }
 895                                         p = fillFromFile(1);
 896                                         inf.setInput(buf, p, bAvail);
 897                                 }
 898                                 n += inf.inflate(dst, n, sz - n);
 899                         }
 900                         n = bAvail - inf.getRemaining();
 901                         if (n > 0) {
 902                                 crc.update(buf, p, n);
 903                                 use(n);
 904                         }
 905                         return dst;
 906                 } catch (DataFormatException dfe) {
 907                         throw corrupt(dfe);
 908                 } finally {
 909                         inf.reset();
 910                 }
 911         }
 912
 913         private static CorruptObjectException corrupt(final DataFormatException dfe) {
 914                 return new CorruptObjectException("Packfile corruption detected: "
 915                                 + dfe.getMessage());
 916         }
 917
 918         private static class UnresolvedDelta {
 919                 final long position;
 920
 921                 final int crc;
 922
 923                 UnresolvedDelta(final long headerOffset, final int crc32) {
 924                         position = headerOffset;
 925                         crc = crc32;
 926                 }
 927         }
 928
 929         /**
 930          * Rename the pack to it's final name and location and open it.
 931          * <p>
 932          * If the call completes successfully the repository this IndexPack instance
 933          * was created with will have the objects in the pack available for reading
 934          * and use, without needing to scan for packs.
 935          *
 936          * @throws IOException
 937          *             The pack could not be inserted into the repository's objects
 938          *             directory. The pack no longer exists on disk, as it was
 939          *             removed prior to throwing the exception to the caller.
 940          */
 941         public void renameAndOpenPack() throws IOException {
 942                 final MessageDigest d = Constants.newMessageDigest();
 943                 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
 944                 for (int i = 0; i < entryCount; i++) {
 945                         final PackedObjectInfo oe = entries[i];
 946                         oe.copyRawTo(oeBytes, 0);
 947                         d.update(oeBytes);
 948                 }
 949
 950                 final String name = ObjectId.fromRaw(d.digest()).name();
 951                 final File packDir = new File(repo.getObjectsDirectory(), "pack");
 952                 final File finalPack = new File(packDir, "pack-" + name + ".pack");
 953                 final File finalIdx = new File(packDir, "pack-" + name + ".idx");
 954
 955                 if (finalPack.exists()) {
 956                         // If the pack is already present we should never replace it.
 957                         //
 958                         cleanupTemporaryFiles();
 959                         return;
 960                 }
 961
 962                 if (!dstPack.renameTo(finalPack)) {
 963                         cleanupTemporaryFiles();
 964                         throw new IOException("Cannot move pack to " + finalPack);
 965                 }
 966
 967                 if (!dstIdx.renameTo(finalIdx)) {
 968                         cleanupTemporaryFiles();
 969                         if (!finalPack.delete())
 970                                 finalPack.deleteOnExit();
 971                         throw new IOException("Cannot move index to " + finalIdx);
 972                 }
 973
 974                 try {
 975                         repo.openPack(finalPack, finalIdx);
 976                 } catch (IOException err) {
 977                         finalPack.delete();
 978                         finalIdx.delete();
 979                         throw err;
 980                 }
 981         }
 982
 983         private void cleanupTemporaryFiles() {
 984                 if (!dstIdx.delete())
 985                         dstIdx.deleteOnExit();
 986                 if (!dstPack.delete())
 987                         dstPack.deleteOnExit();
 988         }
 989 }