org.spearce.jgit/src/org/spearce/jgit/lib/PackFile.java

   1 /*
   2  * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
   3  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
   4  *
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  * - Redistributions of source code must retain the above copyright
  12  *   notice, this list of conditions and the following disclaimer.
  13  *
  14  * - Redistributions in binary form must reproduce the above
  15  *   copyright notice, this list of conditions and the following
  16  *   disclaimer in the documentation and/or other materials provided
  17  *   with the distribution.
  18  *
  19  * - Neither the name of the Git Development Community nor the
  20  *   names of its contributors may be used to endorse or promote
  21  *   products derived from this software without specific prior
  22  *   written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  25  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  26  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  31  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  36  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 package org.spearce.jgit.lib;
  40
  41 import java.io.EOFException;
  42 import java.io.File;
  43 import java.io.IOException;
  44 import java.io.OutputStream;
  45 import java.util.Iterator;
  46 import java.util.zip.CRC32;
  47 import java.util.zip.CheckedOutputStream;
  48 import java.util.zip.DataFormatException;
  49
  50 import org.spearce.jgit.errors.CorruptObjectException;
  51 import org.spearce.jgit.util.NB;
  52
  53 /**
  54  * A Git version 2 pack file representation. A pack file contains Git objects in
  55  * delta packed format yielding high compression of lots of object where some
  56  * objects are similar.
  57  */
  58 public class PackFile implements Iterable<PackIndex.MutableEntry> {
  59         private final WindowedFile pack;
  60
  61         private final PackIndex idx;
  62
  63         private PackReverseIndex reverseIdx;
  64
  65         /**
  66          * Construct a reader for an existing, pre-indexed packfile.
  67          *
  68          * @param parentRepo
  69          *            Git repository holding this pack file
  70          * @param idxFile
  71          *            path of the <code>.idx</code> file listing the contents.
  72          * @param packFile
  73          *            path of the <code>.pack</code> file holding the data.
  74          * @throws IOException
  75          *             the index file cannot be accessed at this time.
  76          */
  77         public PackFile(final Repository parentRepo, final File idxFile,
  78                         final File packFile) throws IOException {
  79                 pack = new WindowedFile(packFile) {
  80                         @Override
  81                         protected void onOpen() throws IOException {
  82                                 readPackHeader();
  83                         }
  84                 };
  85                 try {
  86                         idx = PackIndex.open(idxFile);
  87                 } catch (IOException ioe) {
  88                         throw ioe;
  89                 }
  90         }
  91
  92         final PackedObjectLoader resolveBase(final WindowCursor curs, final long ofs)
  93                         throws IOException {
  94                 return reader(curs, ofs);
  95         }
  96
  97         /**
  98          * Determine if an object is contained within the pack file.
  99          * <p>
 100          * For performance reasons only the index file is searched; the main pack
 101          * content is ignored entirely.
 102          * </p>
 103          *
 104          * @param id
 105          *            the object to look for. Must not be null.
 106          * @return true if the object is in this pack; false otherwise.
 107          */
 108         public boolean hasObject(final AnyObjectId id) {
 109                 return idx.hasObject(id);
 110         }
 111
 112         /**
 113          * Get an object from this pack.
 114          *
 115          * @param id
 116          *            the object to obtain from the pack. Must not be null.
 117          * @return the object loader for the requested object if it is contained in
 118          *         this pack; null if the object was not found.
 119          * @throws IOException
 120          *             the pack file or the index could not be read.
 121          */
 122         public PackedObjectLoader get(final AnyObjectId id) throws IOException {
 123                 final WindowCursor wc = new WindowCursor();
 124                 try {
 125                         return get(wc, id);
 126                 } finally {
 127                         wc.release();
 128                 }
 129         }
 130
 131         /**
 132          * Get an object from this pack.
 133          *
 134          * @param curs
 135          *            temporary working space associated with the calling thread.
 136          * @param id
 137          *            the object to obtain from the pack. Must not be null.
 138          * @return the object loader for the requested object if it is contained in
 139          *         this pack; null if the object was not found.
 140          * @throws IOException
 141          *             the pack file or the index could not be read.
 142          */
 143         public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id)
 144                         throws IOException {
 145                 final long offset = idx.findOffset(id);
 146                 if (offset == -1)
 147                         return null;
 148                 final PackedObjectLoader objReader = reader(curs, offset);
 149                 objReader.setId(id.toObjectId());
 150                 return objReader;
 151         }
 152
 153         /**
 154          * Close the resources utilized by this repository
 155          */
 156         public void close() {
 157                 UnpackedObjectCache.purge(pack);
 158                 pack.close();
 159         }
 160
 161         /**
 162          * Provide iterator over entries in associated pack index, that should also
 163          * exist in this pack file. Objects returned by such iterator are mutable
 164          * during iteration.
 165          * <p>
 166          * Iterator returns objects in SHA-1 lexicographical order.
 167          * </p>
 168          *
 169          * @return iterator over entries of associated pack index
 170          *
 171          * @see PackIndex#iterator()
 172          */
 173         public Iterator<PackIndex.MutableEntry> iterator() {
 174                 return idx.iterator();
 175         }
 176
 177         /**
 178          * Obtain the total number of objects available in this pack. This method
 179          * relies on pack index, giving number of effectively available objects.
 180          *
 181          * @return number of objects in index of this pack, likewise in this pack
 182          */
 183         long getObjectCount() {
 184                 return idx.getObjectCount();
 185         }
 186
 187         /**
 188          * Search for object id with the specified start offset in associated pack
 189          * (reverse) index.
 190          *
 191          * @param offset
 192          *            start offset of object to find
 193          * @return object id for this offset, or null if no object was found
 194          */
 195         ObjectId findObjectForOffset(final long offset) {
 196                 return getReverseIdx().findObject(offset);
 197         }
 198
 199         final UnpackedObjectCache.Entry readCache(final long position) {
 200                 return UnpackedObjectCache.get(pack, position);
 201         }
 202
 203         final void saveCache(final long position, final byte[] data, final int type) {
 204                 UnpackedObjectCache.store(pack, position, data, type);
 205         }
 206
 207         final byte[] decompress(final long position, final int totalSize,
 208                         final WindowCursor curs) throws DataFormatException, IOException {
 209                 final byte[] dstbuf = new byte[totalSize];
 210                 pack.readCompressed(position, dstbuf, curs);
 211                 return dstbuf;
 212         }
 213
 214         final void copyRawData(final PackedObjectLoader loader,
 215                         final OutputStream out, final byte buf[]) throws IOException {
 216                 final long objectOffset = loader.objectOffset;
 217                 final long dataOffset = loader.dataOffset;
 218                 final int cnt = (int) (findEndOffset(objectOffset) - dataOffset);
 219                 final WindowCursor curs = loader.curs;
 220
 221                 if (idx.hasCRC32Support()) {
 222                         final CRC32 crc = new CRC32();
 223                         int headerCnt = (int) (dataOffset - objectOffset);
 224                         while (headerCnt > 0) {
 225                                 int toRead = Math.min(headerCnt, buf.length);
 226                                 int read = pack.read(objectOffset, buf, 0, toRead, curs);
 227                                 if (read != toRead)
 228                                         throw new EOFException();
 229                                 crc.update(buf, 0, read);
 230                                 headerCnt -= toRead;
 231                         }
 232                         final CheckedOutputStream crcOut = new CheckedOutputStream(out, crc);
 233                         pack.copyToStream(dataOffset, buf, cnt, crcOut, curs);
 234                         final long computed = crc.getValue();
 235
 236                         ObjectId id;
 237                         if (loader.hasComputedId())
 238                                 id = loader.getId();
 239                         else
 240                                 id = findObjectForOffset(objectOffset);
 241                         final long expected = idx.findCRC32(id);
 242                         if (computed != expected)
 243                                 throw new CorruptObjectException(id,
 244                                                 "Possible data corruption - CRC32 of raw pack data (object offset "
 245                                                                 + objectOffset
 246                                                                 + ") mismatch CRC32 from pack index");
 247                 } else {
 248                         pack.copyToStream(dataOffset, buf, cnt, out, curs);
 249
 250                         // read to verify against Adler32 zlib checksum
 251                         loader.getCachedBytes();
 252                 }
 253         }
 254
 255         boolean supportsFastCopyRawData() {
 256                 return idx.hasCRC32Support();
 257         }
 258
 259
 260         private void readPackHeader() throws IOException {
 261                 final WindowCursor curs = new WindowCursor();
 262                 long position = 0;
 263                 final byte[] sig = new byte[Constants.PACK_SIGNATURE.length];
 264                 final byte[] intbuf = new byte[4];
 265                 final long vers;
 266
 267                 if (pack.read(position, sig, curs) != Constants.PACK_SIGNATURE.length)
 268                         throw new IOException("Not a PACK file.");
 269                 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++) {
 270                         if (sig[k] != Constants.PACK_SIGNATURE[k])
 271                                 throw new IOException("Not a PACK file.");
 272                 }
 273                 position += Constants.PACK_SIGNATURE.length;
 274
 275                 pack.readFully(position, intbuf, curs);
 276                 vers = NB.decodeUInt32(intbuf, 0);
 277                 if (vers != 2 && vers != 3)
 278                         throw new IOException("Unsupported pack version " + vers + ".");
 279                 position += 4;
 280
 281                 pack.readFully(position, intbuf, curs);
 282                 final long objectCnt = NB.decodeUInt32(intbuf, 0);
 283                 if (idx.getObjectCount() != objectCnt)
 284                         throw new IOException("Pack index"
 285                                         + " object count mismatch; expected " + objectCnt
 286                                         + " found " + idx.getObjectCount() + ": " + pack.getName());
 287         }
 288
 289         private PackedObjectLoader reader(final WindowCursor curs,
 290                         final long objOffset) throws IOException {
 291                 long pos = objOffset;
 292                 int p = 0;
 293                 final byte[] ib = curs.tempId;
 294                 pack.readFully(pos, ib, curs);
 295                 int c = ib[p++] & 0xff;
 296                 final int typeCode = (c >> 4) & 7;
 297                 long dataSize = c & 15;
 298                 int shift = 4;
 299                 while ((c & 0x80) != 0) {
 300                         c = ib[p++] & 0xff;
 301                         dataSize += (c & 0x7f) << shift;
 302                         shift += 7;
 303                 }
 304                 pos += p;
 305
 306                 switch (typeCode) {
 307                 case Constants.OBJ_COMMIT:
 308                 case Constants.OBJ_TREE:
 309                 case Constants.OBJ_BLOB:
 310                 case Constants.OBJ_TAG:
 311                         return new WholePackedObjectLoader(curs, this, pos, objOffset,
 312                                         typeCode, (int) dataSize);
 313
 314                 case Constants.OBJ_OFS_DELTA: {
 315                         pack.readFully(pos, ib, curs);
 316                         p = 0;
 317                         c = ib[p++] & 0xff;
 318                         long ofs = c & 127;
 319                         while ((c & 128) != 0) {
 320                                 ofs += 1;
 321                                 c = ib[p++] & 0xff;
 322                                 ofs <<= 7;
 323                                 ofs += (c & 127);
 324                         }
 325                         return new DeltaOfsPackedObjectLoader(curs, this, pos + p,
 326                                         objOffset, (int) dataSize, objOffset - ofs);
 327                 }
 328                 case Constants.OBJ_REF_DELTA: {
 329                         pack.readFully(pos, ib, curs);
 330                         return new DeltaRefPackedObjectLoader(curs, this, pos + ib.length,
 331                                         objOffset, (int) dataSize, ObjectId.fromRaw(ib));
 332                 }
 333                 default:
 334                         throw new IOException("Unknown object type " + typeCode + ".");
 335                 }
 336         }
 337
 338         private long findEndOffset(final long startOffset)
 339                         throws CorruptObjectException {
 340                 final long maxOffset = pack.length() - Constants.OBJECT_ID_LENGTH;
 341                 return getReverseIdx().findNextOffset(startOffset, maxOffset);
 342         }
 343
 344         private synchronized PackReverseIndex getReverseIdx() {
 345                 if (reverseIdx == null)
 346                         reverseIdx = new PackReverseIndex(idx);
 347                 return reverseIdx;
 348         }
 349 }