2 * Copyright (C) 2008-2010, Google Inc.
3 * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
4 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
5 * and other copyright owners as documented in the project's IP log.
7 * This program and the accompanying materials are made available
8 * under the terms of the Eclipse Distribution License v1.0 which
9 * accompanies this distribution, is reproduced below, and is
10 * available at http://www.eclipse.org/org/documents/edl-v10.php
12 * All rights reserved.
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
18 * - Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials provided
24 * with the distribution.
26 * - Neither the name of the Eclipse Foundation, Inc. nor the
27 * names of its contributors may be used to endorse or promote
28 * products derived from this software without specific prior
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
32 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
33 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
36 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
37 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
38 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
39 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
40 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
43 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 package org
.eclipse
.jgit
.transport
;
48 import java
.io
.EOFException
;
50 import java
.io
.FileOutputStream
;
51 import java
.io
.IOException
;
52 import java
.io
.InputStream
;
53 import java
.io
.RandomAccessFile
;
54 import java
.security
.MessageDigest
;
55 import java
.util
.ArrayList
;
56 import java
.util
.Arrays
;
57 import java
.util
.List
;
58 import java
.util
.zip
.CRC32
;
59 import java
.util
.zip
.DataFormatException
;
60 import java
.util
.zip
.Deflater
;
61 import java
.util
.zip
.Inflater
;
63 import org
.eclipse
.jgit
.errors
.CorruptObjectException
;
64 import org
.eclipse
.jgit
.errors
.MissingObjectException
;
65 import org
.eclipse
.jgit
.lib
.AnyObjectId
;
66 import org
.eclipse
.jgit
.lib
.BinaryDelta
;
67 import org
.eclipse
.jgit
.lib
.Constants
;
68 import org
.eclipse
.jgit
.lib
.InflaterCache
;
69 import org
.eclipse
.jgit
.lib
.MutableObjectId
;
70 import org
.eclipse
.jgit
.lib
.ObjectChecker
;
71 import org
.eclipse
.jgit
.lib
.ObjectDatabase
;
72 import org
.eclipse
.jgit
.lib
.ObjectId
;
73 import org
.eclipse
.jgit
.lib
.ObjectIdSubclassMap
;
74 import org
.eclipse
.jgit
.lib
.ObjectLoader
;
75 import org
.eclipse
.jgit
.lib
.PackIndexWriter
;
76 import org
.eclipse
.jgit
.lib
.PackLock
;
77 import org
.eclipse
.jgit
.lib
.ProgressMonitor
;
78 import org
.eclipse
.jgit
.lib
.Repository
;
79 import org
.eclipse
.jgit
.lib
.WindowCursor
;
80 import org
.eclipse
.jgit
.util
.NB
;
82 /** Indexes Git pack files for local use. */
83 public class IndexPack
{
84 /** Progress message when reading raw data from the pack. */
85 public static final String PROGRESS_DOWNLOAD
= "Receiving objects";
87 /** Progress message when computing names of delta compressed objects. */
88 public static final String PROGRESS_RESOLVE_DELTA
= "Resolving deltas";
91 * Size of the internal stream buffer.
93 * If callers are going to be supplying IndexPack a BufferedInputStream they
94 * should use this buffer size as the size of the buffer for that
95 * BufferedInputStream, and any other its may be wrapping. This way the
96 * buffers will cascade efficiently and only the IndexPack buffer will be
97 * receiving the bulk of the data stream.
99 public static final int BUFFER_SIZE
= 8192;
102 * Create an index pack instance to load a new pack into a repository.
104 * The received pack data and generated index will be saved to temporary
105 * files within the repository's <code>objects</code> directory. To use the
106 * data contained within them call {@link #renameAndOpenPack()} once the
107 * indexing is complete.
110 * the repository that will receive the new pack.
112 * stream to read the pack data from. If the stream is buffered
113 * use {@link #BUFFER_SIZE} as the buffer size for the stream.
114 * @return a new index pack instance.
115 * @throws IOException
116 * a temporary file could not be created.
118 public static IndexPack
create(final Repository db
, final InputStream is
)
120 final String suffix
= ".pack";
121 final File objdir
= db
.getObjectsDirectory();
122 final File tmp
= File
.createTempFile("incoming_", suffix
, objdir
);
123 final String n
= tmp
.getName();
126 base
= new File(objdir
, n
.substring(0, n
.length() - suffix
.length()));
127 final IndexPack ip
= new IndexPack(db
, is
, base
);
128 ip
.setIndexVersion(db
.getConfig().getCore().getPackIndexVersion());
132 private final Repository repo
;
135 * Object database used for loading existing objects
137 private final ObjectDatabase objectDatabase
;
139 private Inflater inflater
;
141 private final MessageDigest objectDigest
;
143 private final MutableObjectId tempObjectId
;
145 private InputStream in
;
155 private ObjectChecker objCheck
;
157 private boolean fixThin
;
159 private boolean keepEmpty
;
161 private boolean needBaseObjectIds
;
163 private int outputVersion
;
165 private final File dstPack
;
167 private final File dstIdx
;
169 private long objectCount
;
171 private PackedObjectInfo
[] entries
;
174 * Every object contained within the incoming pack.
176 * This is a subset of {@link #entries}, as thin packs can add additional
177 * objects to {@code entries} by copying already existing objects from the
178 * repository onto the end of the thin pack to make it self-contained.
180 private ObjectIdSubclassMap
<ObjectId
> newObjectIds
;
182 private int deltaCount
;
184 private int entryCount
;
186 private final CRC32 crc
= new CRC32();
188 private ObjectIdSubclassMap
<DeltaChain
> baseById
;
191 * Objects referenced by their name from deltas, that aren't in this pack.
193 * This is the set of objects that were copied onto the end of this pack to
194 * make it complete. These objects were not transmitted by the remote peer,
195 * but instead were assumed to already exist in the local repository.
197 private ObjectIdSubclassMap
<ObjectId
> baseObjectIds
;
199 private LongMap
<UnresolvedDelta
> baseByPos
;
201 private byte[] objectData
;
203 private MessageDigest packDigest
;
205 private RandomAccessFile packOut
;
207 private byte[] packcsum
;
209 /** If {@link #fixThin} this is the last byte of the original checksum. */
210 private long originalEOF
;
212 private WindowCursor readCurs
;
215 * Create a new pack indexer utility.
219 * stream to read the pack data from. If the stream is buffered
220 * use {@link #BUFFER_SIZE} as the buffer size for the stream.
222 * @throws IOException
223 * the output packfile could not be created.
225 public IndexPack(final Repository db
, final InputStream src
,
226 final File dstBase
) throws IOException
{
228 objectDatabase
= db
.getObjectDatabase().newCachedDatabase();
230 inflater
= InflaterCache
.get();
231 readCurs
= new WindowCursor();
232 buf
= new byte[BUFFER_SIZE
];
233 objectData
= new byte[BUFFER_SIZE
];
234 objectDigest
= Constants
.newMessageDigest();
235 tempObjectId
= new MutableObjectId();
236 packDigest
= Constants
.newMessageDigest();
238 if (dstBase
!= null) {
239 final File dir
= dstBase
.getParentFile();
240 final String nam
= dstBase
.getName();
241 dstPack
= new File(dir
, nam
+ ".pack");
242 dstIdx
= new File(dir
, nam
+ ".idx");
243 packOut
= new RandomAccessFile(dstPack
, "rw");
244 packOut
.setLength(0);
252 * Set the pack index file format version this instance will create.
255 * the version to write. The special version 0 designates the
256 * oldest (most compatible) format available for the objects.
257 * @see PackIndexWriter
259 public void setIndexVersion(final int version
) {
260 outputVersion
= version
;
264 * Configure this index pack instance to make a thin pack complete.
266 * Thin packs are sometimes used during network transfers to allow a delta
267 * to be sent without a base object. Such packs are not permitted on disk.
268 * They can be fixed by copying the base object onto the end of the pack.
271 * true to enable fixing a thin pack.
273 public void setFixThin(final boolean fix
) {
278 * Configure this index pack instance to keep an empty pack.
280 * By default an empty pack (a pack with no objects) is not kept, as doing
281 * so is completely pointless. With no objects in the pack there is no data
282 * stored by it, so the pack is unnecessary.
284 * @param empty true to enable keeping an empty pack.
286 public void setKeepEmpty(final boolean empty
) {
291 * Configure this index pack instance to keep track of new objects.
293 * By default an index pack doesn't save the new objects that were created
294 * when it was instantiated. Setting this flag to {@code true} allows the
295 * caller to use {@link #getNewObjectIds()} to retrieve that list.
297 * @param b {@code true} to enable keeping track of new objects.
299 public void setNeedNewObjectIds(boolean b
) {
301 newObjectIds
= new ObjectIdSubclassMap
<ObjectId
>();
306 private boolean needNewObjectIds() {
307 return newObjectIds
!= null;
311 * Configure this index pack instance to keep track of the objects assumed
314 * By default an index pack doesn't save the objects that were used as delta
315 * bases. Setting this flag to {@code true} will allow the caller to
316 * use {@link #getBaseObjectIds()} to retrieve that list.
318 * @param b {@code true} to enable keeping track of delta bases.
320 public void setNeedBaseObjectIds(boolean b
) {
321 this.needBaseObjectIds
= b
;
324 /** @return the new objects that were sent by the user */
325 public ObjectIdSubclassMap
<ObjectId
> getNewObjectIds() {
326 if (newObjectIds
!= null)
328 return new ObjectIdSubclassMap
<ObjectId
>();
331 /** @return set of objects the incoming pack assumed for delta purposes */
332 public ObjectIdSubclassMap
<ObjectId
> getBaseObjectIds() {
333 if (baseObjectIds
!= null)
334 return baseObjectIds
;
335 return new ObjectIdSubclassMap
<ObjectId
>();
339 * Configure the checker used to validate received objects.
341 * Usually object checking isn't necessary, as Git implementations only
342 * create valid objects in pack files. However, additional checking may be
343 * useful if processing data from an untrusted source.
346 * the checker instance; null to disable object checking.
348 public void setObjectChecker(final ObjectChecker oc
) {
353 * Configure the checker used to validate received objects.
355 * Usually object checking isn't necessary, as Git implementations only
356 * create valid objects in pack files. However, additional checking may be
357 * useful if processing data from an untrusted source.
359 * This is shorthand for:
362 * setObjectChecker(on ? new ObjectChecker() : null);
366 * true to enable the default checker; false to disable it.
368 public void setObjectChecking(final boolean on
) {
369 setObjectChecker(on ?
new ObjectChecker() : null);
373 * Consume data from the input stream until the packfile is indexed.
378 * @throws IOException
380 public void index(final ProgressMonitor progress
) throws IOException
{
381 progress
.start(2 /* tasks */);
386 entries
= new PackedObjectInfo
[(int) objectCount
];
387 baseById
= new ObjectIdSubclassMap
<DeltaChain
>();
388 baseByPos
= new LongMap
<UnresolvedDelta
>();
390 progress
.beginTask(PROGRESS_DOWNLOAD
, (int) objectCount
);
391 for (int done
= 0; done
< objectCount
; done
++) {
394 if (progress
.isCancelled())
395 throw new IOException("Download cancelled");
400 if (deltaCount
> 0) {
402 throw new IOException("need packOut");
403 resolveDeltas(progress
);
404 if (entryCount
< objectCount
) {
406 throw new IOException("pack has "
407 + (objectCount
- entryCount
)
408 + " unresolved deltas");
410 fixThinPack(progress
);
413 if (packOut
!= null && (keepEmpty
|| entryCount
> 0))
414 packOut
.getChannel().force(true);
420 if (dstIdx
!= null && (keepEmpty
|| entryCount
> 0))
425 InflaterCache
.release(inflater
);
428 objectDatabase
.close();
430 readCurs
= WindowCursor
.release(readCurs
);
437 if (keepEmpty
|| entryCount
> 0) {
439 dstPack
.setReadOnly();
441 dstIdx
.setReadOnly();
443 } catch (IOException err
) {
452 private void resolveDeltas(final ProgressMonitor progress
)
454 progress
.beginTask(PROGRESS_RESOLVE_DELTA
, deltaCount
);
455 final int last
= entryCount
;
456 for (int i
= 0; i
< last
; i
++) {
457 final int before
= entryCount
;
458 resolveDeltas(entries
[i
]);
459 progress
.update(entryCount
- before
);
460 if (progress
.isCancelled())
461 throw new IOException("Download cancelled during indexing");
466 private void resolveDeltas(final PackedObjectInfo oe
) throws IOException
{
467 final int oldCRC
= oe
.getCRC();
468 if (baseById
.get(oe
) != null || baseByPos
.containsKey(oe
.getOffset()))
469 resolveDeltas(oe
.getOffset(), oldCRC
, Constants
.OBJ_BAD
, null, oe
);
472 private void resolveDeltas(final long pos
, final int oldCRC
, int type
,
473 byte[] data
, PackedObjectInfo oe
) throws IOException
{
476 int c
= readFromFile();
477 final int typeCode
= (c
>> 4) & 7;
480 while ((c
& 0x80) != 0) {
482 sz
+= (c
& 0x7f) << shift
;
487 case Constants
.OBJ_COMMIT
:
488 case Constants
.OBJ_TREE
:
489 case Constants
.OBJ_BLOB
:
490 case Constants
.OBJ_TAG
:
492 data
= inflateFromFile((int) sz
);
494 case Constants
.OBJ_OFS_DELTA
: {
495 c
= readFromFile() & 0xff;
496 while ((c
& 128) != 0)
497 c
= readFromFile() & 0xff;
498 data
= BinaryDelta
.apply(data
, inflateFromFile((int) sz
));
501 case Constants
.OBJ_REF_DELTA
: {
502 crc
.update(buf
, fillFromFile(20), 20);
504 data
= BinaryDelta
.apply(data
, inflateFromFile((int) sz
));
508 throw new IOException("Unknown object type " + typeCode
+ ".");
511 final int crc32
= (int) crc
.getValue();
513 throw new IOException("Corruption detected re-reading at " + pos
);
515 objectDigest
.update(Constants
.encodedTypeString(type
));
516 objectDigest
.update((byte) ' ');
517 objectDigest
.update(Constants
.encodeASCII(data
.length
));
518 objectDigest
.update((byte) 0);
519 objectDigest
.update(data
);
520 tempObjectId
.fromRaw(objectDigest
.digest(), 0);
522 verifySafeObject(tempObjectId
, type
, data
);
523 oe
= new PackedObjectInfo(pos
, crc32
, tempObjectId
);
524 addObjectAndTrack(oe
);
527 resolveChildDeltas(pos
, type
, data
, oe
);
530 private UnresolvedDelta
removeBaseById(final AnyObjectId id
){
531 final DeltaChain d
= baseById
.get(id
);
532 return d
!= null ? d
.remove() : null;
535 private static UnresolvedDelta
reverse(UnresolvedDelta c
) {
536 UnresolvedDelta tail
= null;
538 final UnresolvedDelta n
= c
.next
;
546 private void resolveChildDeltas(final long pos
, int type
, byte[] data
,
547 PackedObjectInfo oe
) throws IOException
{
548 UnresolvedDelta a
= reverse(removeBaseById(oe
));
549 UnresolvedDelta b
= reverse(baseByPos
.remove(pos
));
550 while (a
!= null && b
!= null) {
551 if (a
.position
< b
.position
) {
552 resolveDeltas(a
.position
, a
.crc
, type
, data
, null);
555 resolveDeltas(b
.position
, b
.crc
, type
, data
, null);
559 resolveChildDeltaChain(type
, data
, a
);
560 resolveChildDeltaChain(type
, data
, b
);
563 private void resolveChildDeltaChain(final int type
, final byte[] data
,
564 UnresolvedDelta a
) throws IOException
{
566 resolveDeltas(a
.position
, a
.crc
, type
, data
, null);
571 private void fixThinPack(final ProgressMonitor progress
) throws IOException
{
574 if (needBaseObjectIds
)
575 baseObjectIds
= new ObjectIdSubclassMap
<ObjectId
>();
578 originalEOF
= packOut
.length() - 20;
579 final Deflater def
= new Deflater(Deflater
.DEFAULT_COMPRESSION
, false);
580 final List
<DeltaChain
> missing
= new ArrayList
<DeltaChain
>(64);
581 long end
= originalEOF
;
582 for (final DeltaChain baseId
: baseById
) {
583 if (baseId
.head
== null)
585 if (needBaseObjectIds
)
586 baseObjectIds
.add(baseId
);
587 final ObjectLoader ldr
= repo
.openObject(readCurs
, baseId
);
592 final byte[] data
= ldr
.getCachedBytes();
593 final int typeCode
= ldr
.getType();
594 final PackedObjectInfo oe
;
598 writeWhole(def
, typeCode
, data
);
599 oe
= new PackedObjectInfo(end
, (int) crc
.getValue(), baseId
);
600 entries
[entryCount
++] = oe
;
601 end
= packOut
.getFilePointer();
603 resolveChildDeltas(oe
.getOffset(), typeCode
, data
, oe
);
604 if (progress
.isCancelled())
605 throw new IOException("Download cancelled during indexing");
609 for (final DeltaChain base
: missing
) {
610 if (base
.head
!= null)
611 throw new MissingObjectException(base
, "delta base");
614 if (end
- originalEOF
< 20) {
615 // Ugly corner case; if what we appended on to complete deltas
616 // doesn't completely cover the SHA-1 we have to truncate off
617 // we need to shorten the file, otherwise we will include part
618 // of the old footer as object content.
619 packOut
.setLength(end
);
622 fixHeaderFooter(packcsum
, packDigest
.digest());
625 private void writeWhole(final Deflater def
, final int typeCode
,
626 final byte[] data
) throws IOException
{
627 int sz
= data
.length
;
629 buf
[hdrlen
++] = (byte) ((typeCode
<< 4) | sz
& 15);
632 buf
[hdrlen
- 1] |= 0x80;
633 buf
[hdrlen
++] = (byte) (sz
& 0x7f);
636 packDigest
.update(buf
, 0, hdrlen
);
637 crc
.update(buf
, 0, hdrlen
);
638 packOut
.write(buf
, 0, hdrlen
);
642 while (!def
.finished()) {
643 final int datlen
= def
.deflate(buf
);
644 packDigest
.update(buf
, 0, datlen
);
645 crc
.update(buf
, 0, datlen
);
646 packOut
.write(buf
, 0, datlen
);
650 private void fixHeaderFooter(final byte[] origcsum
, final byte[] tailcsum
)
652 final MessageDigest origDigest
= Constants
.newMessageDigest();
653 final MessageDigest tailDigest
= Constants
.newMessageDigest();
654 long origRemaining
= originalEOF
;
662 final int origCnt
= (int) Math
.min(bAvail
, origRemaining
);
663 origDigest
.update(buf
, 0, origCnt
);
664 origRemaining
-= origCnt
;
665 if (origRemaining
== 0)
666 tailDigest
.update(buf
, origCnt
, bAvail
- origCnt
);
669 NB
.encodeInt32(buf
, 8, entryCount
);
671 packOut
.write(buf
, 0, 12);
672 packOut
.seek(bAvail
);
675 packDigest
.update(buf
, 0, bAvail
);
677 final int n
= packOut
.read(buf
);
680 if (origRemaining
!= 0) {
681 final int origCnt
= (int) Math
.min(n
, origRemaining
);
682 origDigest
.update(buf
, 0, origCnt
);
683 origRemaining
-= origCnt
;
684 if (origRemaining
== 0)
685 tailDigest
.update(buf
, origCnt
, n
- origCnt
);
687 tailDigest
.update(buf
, 0, n
);
689 packDigest
.update(buf
, 0, n
);
692 if (!Arrays
.equals(origDigest
.digest(), origcsum
)
693 || !Arrays
.equals(tailDigest
.digest(), tailcsum
))
694 throw new IOException("Pack corrupted while writing to filesystem");
696 packcsum
= packDigest
.digest();
697 packOut
.write(packcsum
);
700 private void growEntries() {
701 final PackedObjectInfo
[] ne
;
703 ne
= new PackedObjectInfo
[(int) objectCount
+ baseById
.size()];
704 System
.arraycopy(entries
, 0, ne
, 0, entryCount
);
708 private void writeIdx() throws IOException
{
709 Arrays
.sort(entries
, 0, entryCount
);
710 List
<PackedObjectInfo
> list
= Arrays
.asList(entries
);
711 if (entryCount
< entries
.length
)
712 list
= list
.subList(0, entryCount
);
714 final FileOutputStream os
= new FileOutputStream(dstIdx
);
716 final PackIndexWriter iw
;
717 if (outputVersion
<= 0)
718 iw
= PackIndexWriter
.createOldestPossible(os
, list
);
720 iw
= PackIndexWriter
.createVersion(os
, outputVersion
);
721 iw
.write(list
, packcsum
);
722 os
.getChannel().force(true);
728 private void readPackHeader() throws IOException
{
729 final int hdrln
= Constants
.PACK_SIGNATURE
.length
+ 4 + 4;
730 final int p
= fillFromInput(hdrln
);
731 for (int k
= 0; k
< Constants
.PACK_SIGNATURE
.length
; k
++)
732 if (buf
[p
+ k
] != Constants
.PACK_SIGNATURE
[k
])
733 throw new IOException("Not a PACK file.");
735 final long vers
= NB
.decodeUInt32(buf
, p
+ 4);
736 if (vers
!= 2 && vers
!= 3)
737 throw new IOException("Unsupported pack version " + vers
+ ".");
738 objectCount
= NB
.decodeUInt32(buf
, p
+ 8);
742 private void readPackFooter() throws IOException
{
744 final byte[] cmpcsum
= packDigest
.digest();
745 final int c
= fillFromInput(20);
746 packcsum
= new byte[20];
747 System
.arraycopy(buf
, c
, packcsum
, 0, 20);
750 packOut
.write(packcsum
);
752 if (!Arrays
.equals(cmpcsum
, packcsum
))
753 throw new CorruptObjectException("Packfile checksum incorrect.");
756 // Cleanup all resources associated with our input parsing.
757 private void endInput() {
762 // Read one entire object or delta from the input.
763 private void indexOneObject() throws IOException
{
764 final long pos
= position();
767 int c
= readFromInput();
768 final int typeCode
= (c
>> 4) & 7;
771 while ((c
& 0x80) != 0) {
773 sz
+= (c
& 0x7f) << shift
;
778 case Constants
.OBJ_COMMIT
:
779 case Constants
.OBJ_TREE
:
780 case Constants
.OBJ_BLOB
:
781 case Constants
.OBJ_TAG
:
782 whole(typeCode
, pos
, sz
);
784 case Constants
.OBJ_OFS_DELTA
: {
787 while ((c
& 128) != 0) {
793 final long base
= pos
- ofs
;
794 final UnresolvedDelta n
;
795 skipInflateFromInput(sz
);
796 n
= new UnresolvedDelta(pos
, (int) crc
.getValue());
797 n
.next
= baseByPos
.put(base
, n
);
801 case Constants
.OBJ_REF_DELTA
: {
802 c
= fillFromInput(20);
803 crc
.update(buf
, c
, 20);
804 final ObjectId base
= ObjectId
.fromRaw(buf
, c
);
806 DeltaChain r
= baseById
.get(base
);
808 r
= new DeltaChain(base
);
811 skipInflateFromInput(sz
);
812 r
.add(new UnresolvedDelta(pos
, (int) crc
.getValue()));
817 throw new IOException("Unknown object type " + typeCode
+ ".");
821 private void whole(final int type
, final long pos
, final long sz
)
823 final byte[] data
= inflateFromInput(sz
);
824 objectDigest
.update(Constants
.encodedTypeString(type
));
825 objectDigest
.update((byte) ' ');
826 objectDigest
.update(Constants
.encodeASCII(sz
));
827 objectDigest
.update((byte) 0);
828 objectDigest
.update(data
);
829 tempObjectId
.fromRaw(objectDigest
.digest(), 0);
831 verifySafeObject(tempObjectId
, type
, data
);
832 final int crc32
= (int) crc
.getValue();
833 addObjectAndTrack(new PackedObjectInfo(pos
, crc32
, tempObjectId
));
836 private void verifySafeObject(final AnyObjectId id
, final int type
,
837 final byte[] data
) throws IOException
{
838 if (objCheck
!= null) {
840 objCheck
.check(type
, data
);
841 } catch (CorruptObjectException e
) {
842 throw new IOException("Invalid "
843 + Constants
.typeString(type
) + " " + id
.name()
844 + ":" + e
.getMessage());
848 final ObjectLoader ldr
= objectDatabase
.openObject(readCurs
, id
);
850 final byte[] existingData
= ldr
.getCachedBytes();
851 if (ldr
.getType() != type
|| !Arrays
.equals(data
, existingData
)) {
852 throw new IOException("Collision on " + id
.name());
857 // Current position of {@link #bOffset} within the entire file.
858 private long position() {
859 return bBase
+ bOffset
;
862 private void position(final long pos
) throws IOException
{
869 // Consume exactly one byte from the buffer and return it.
870 private int readFromInput() throws IOException
{
874 final int b
= buf
[bOffset
++] & 0xff;
879 // Consume exactly one byte from the buffer and return it.
880 private int readFromFile() throws IOException
{
884 final int b
= buf
[bOffset
++] & 0xff;
889 // Consume cnt bytes from the buffer.
890 private void use(final int cnt
) {
895 // Ensure at least need bytes are available in in {@link #buf}.
896 private int fillFromInput(final int need
) throws IOException
{
897 while (bAvail
< need
) {
898 int next
= bOffset
+ bAvail
;
899 int free
= buf
.length
- next
;
900 if (free
+ bAvail
< need
) {
903 free
= buf
.length
- next
;
905 next
= in
.read(buf
, next
, free
);
907 throw new EOFException("Packfile is truncated.");
913 // Ensure at least need bytes are available in in {@link #buf}.
914 private int fillFromFile(final int need
) throws IOException
{
916 int next
= bOffset
+ bAvail
;
917 int free
= buf
.length
- next
;
918 if (free
+ bAvail
< need
) {
920 System
.arraycopy(buf
, bOffset
, buf
, 0, bAvail
);
923 free
= buf
.length
- next
;
925 next
= packOut
.read(buf
, next
, free
);
927 throw new EOFException("Packfile is truncated.");
933 // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
934 private void sync() throws IOException
{
935 packDigest
.update(buf
, 0, bOffset
);
937 packOut
.write(buf
, 0, bOffset
);
939 System
.arraycopy(buf
, bOffset
, buf
, 0, bAvail
);
944 private void skipInflateFromInput(long sz
) throws IOException
{
945 final Inflater inf
= inflater
;
947 final byte[] dst
= objectData
;
950 while (!inf
.finished()) {
951 if (inf
.needsInput()) {
953 crc
.update(buf
, p
, bAvail
);
956 p
= fillFromInput(1);
957 inf
.setInput(buf
, p
, bAvail
);
960 int free
= dst
.length
- n
;
966 n
+= inf
.inflate(dst
, n
, free
);
969 throw new DataFormatException("wrong decompressed length");
970 n
= bAvail
- inf
.getRemaining();
972 crc
.update(buf
, p
, n
);
975 } catch (DataFormatException dfe
) {
982 private byte[] inflateFromInput(final long sz
) throws IOException
{
983 final byte[] dst
= new byte[(int) sz
];
984 final Inflater inf
= inflater
;
988 while (!inf
.finished()) {
989 if (inf
.needsInput()) {
991 crc
.update(buf
, p
, bAvail
);
994 p
= fillFromInput(1);
995 inf
.setInput(buf
, p
, bAvail
);
998 n
+= inf
.inflate(dst
, n
, dst
.length
- n
);
1001 throw new DataFormatException("wrong decompressed length");
1002 n
= bAvail
- inf
.getRemaining();
1004 crc
.update(buf
, p
, n
);
1008 } catch (DataFormatException dfe
) {
1015 private byte[] inflateFromFile(final int sz
) throws IOException
{
1016 final Inflater inf
= inflater
;
1018 final byte[] dst
= new byte[sz
];
1021 while (!inf
.finished()) {
1022 if (inf
.needsInput()) {
1024 crc
.update(buf
, p
, bAvail
);
1027 p
= fillFromFile(1);
1028 inf
.setInput(buf
, p
, bAvail
);
1030 n
+= inf
.inflate(dst
, n
, sz
- n
);
1032 n
= bAvail
- inf
.getRemaining();
1034 crc
.update(buf
, p
, n
);
1038 } catch (DataFormatException dfe
) {
1045 private static CorruptObjectException
corrupt(final DataFormatException dfe
) {
1046 return new CorruptObjectException("Packfile corruption detected: "
1047 + dfe
.getMessage());
1050 private static class DeltaChain
extends ObjectId
{
1051 UnresolvedDelta head
;
1053 DeltaChain(final AnyObjectId id
) {
1057 UnresolvedDelta
remove() {
1058 final UnresolvedDelta r
= head
;
1064 void add(final UnresolvedDelta d
) {
1070 private static class UnresolvedDelta
{
1071 final long position
;
1075 UnresolvedDelta next
;
1077 UnresolvedDelta(final long headerOffset
, final int crc32
) {
1078 position
= headerOffset
;
1084 * Rename the pack to it's final name and location and open it.
1086 * If the call completes successfully the repository this IndexPack instance
1087 * was created with will have the objects in the pack available for reading
1088 * and use, without needing to scan for packs.
1090 * @throws IOException
1091 * The pack could not be inserted into the repository's objects
1092 * directory. The pack no longer exists on disk, as it was
1093 * removed prior to throwing the exception to the caller.
1095 public void renameAndOpenPack() throws IOException
{
1096 renameAndOpenPack(null);
1100 * Rename the pack to it's final name and location and open it.
1102 * If the call completes successfully the repository this IndexPack instance
1103 * was created with will have the objects in the pack available for reading
1104 * and use, without needing to scan for packs.
1106 * @param lockMessage
1107 * message to place in the pack-*.keep file. If null, no lock
1108 * will be created, and this method returns null.
1109 * @return the pack lock object, if lockMessage is not null.
1110 * @throws IOException
1111 * The pack could not be inserted into the repository's objects
1112 * directory. The pack no longer exists on disk, as it was
1113 * removed prior to throwing the exception to the caller.
1115 public PackLock
renameAndOpenPack(final String lockMessage
)
1116 throws IOException
{
1117 if (!keepEmpty
&& entryCount
== 0) {
1118 cleanupTemporaryFiles();
1122 final MessageDigest d
= Constants
.newMessageDigest();
1123 final byte[] oeBytes
= new byte[Constants
.OBJECT_ID_LENGTH
];
1124 for (int i
= 0; i
< entryCount
; i
++) {
1125 final PackedObjectInfo oe
= entries
[i
];
1126 oe
.copyRawTo(oeBytes
, 0);
1130 final String name
= ObjectId
.fromRaw(d
.digest()).name();
1131 final File packDir
= new File(repo
.getObjectsDirectory(), "pack");
1132 final File finalPack
= new File(packDir
, "pack-" + name
+ ".pack");
1133 final File finalIdx
= new File(packDir
, "pack-" + name
+ ".idx");
1134 final PackLock keep
= new PackLock(finalPack
);
1136 if (!packDir
.exists() && !packDir
.mkdir() && !packDir
.exists()) {
1137 // The objects/pack directory isn't present, and we are unable
1138 // to create it. There is no way to move this pack in.
1140 cleanupTemporaryFiles();
1141 throw new IOException("Cannot create " + packDir
.getAbsolutePath());
1144 if (finalPack
.exists()) {
1145 // If the pack is already present we should never replace it.
1147 cleanupTemporaryFiles();
1151 if (lockMessage
!= null) {
1152 // If we have a reason to create a keep file for this pack, do
1153 // so, or fail fast and don't put the pack in place.
1156 if (!keep
.lock(lockMessage
))
1157 throw new IOException("Cannot lock pack in " + finalPack
);
1158 } catch (IOException e
) {
1159 cleanupTemporaryFiles();
1164 if (!dstPack
.renameTo(finalPack
)) {
1165 cleanupTemporaryFiles();
1167 throw new IOException("Cannot move pack to " + finalPack
);
1170 if (!dstIdx
.renameTo(finalIdx
)) {
1171 cleanupTemporaryFiles();
1173 if (!finalPack
.delete())
1174 finalPack
.deleteOnExit();
1175 throw new IOException("Cannot move index to " + finalIdx
);
1179 repo
.openPack(finalPack
, finalIdx
);
1180 } catch (IOException err
) {
1187 return lockMessage
!= null ? keep
: null;
1190 private void cleanupTemporaryFiles() {
1191 if (!dstIdx
.delete())
1192 dstIdx
.deleteOnExit();
1193 if (!dstPack
.delete())
1194 dstPack
.deleteOnExit();
1197 private void addObjectAndTrack(PackedObjectInfo oe
) {
1198 entries
[entryCount
++] = oe
;
1199 if (needNewObjectIds())
1200 newObjectIds
.add(oe
);