org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java

   1 /*
   2  * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
   3  *
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or
   7  * without modification, are permitted provided that the following
   8  * conditions are met:
   9  *
  10  * - Redistributions of source code must retain the above copyright
  11  *   notice, this list of conditions and the following disclaimer.
  12  *
  13  * - Redistributions in binary form must reproduce the above
  14  *   copyright notice, this list of conditions and the following
  15  *   disclaimer in the documentation and/or other materials provided
  16  *   with the distribution.
  17  *
  18  * - Neither the name of the Git Development Community nor the
  19  *   names of its contributors may be used to endorse or promote
  20  *   products derived from this software without specific prior
  21  *   written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  24  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  25  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  26  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  32  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  33  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  35  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 package org.spearce.jgit.lib;
  39
  40 import java.io.ByteArrayInputStream;
  41 import java.io.ByteArrayOutputStream;
  42 import java.io.File;
  43 import java.io.FileOutputStream;
  44 import java.io.IOException;
  45 import java.io.InputStream;
  46 import java.util.ArrayList;
  47 import java.util.Arrays;
  48 import java.util.Collection;
  49 import java.util.Collections;
  50 import java.util.Comparator;
  51 import java.util.Iterator;
  52 import java.util.LinkedList;
  53 import java.util.List;
  54
  55 import org.spearce.jgit.errors.MissingObjectException;
  56 import org.spearce.jgit.lib.PackIndex.MutableEntry;
  57 import org.spearce.jgit.revwalk.RevObject;
  58 import org.spearce.jgit.revwalk.RevWalk;
  59 import org.spearce.jgit.transport.IndexPack;
  60 import org.spearce.jgit.util.CountingOutputStream;
  61 import org.spearce.jgit.util.JGitTestUtil;
  62
  63 public class PackWriterTest extends RepositoryTestCase {
  64
  65         private static final List<ObjectId> EMPTY_LIST_OBJECT = Collections
  66                         .<ObjectId> emptyList();
  67
  68         private static final List<RevObject> EMPTY_LIST_REVS = Collections
  69                         .<RevObject> emptyList();
  70
  71         private PackWriter writer;
  72
  73         private ByteArrayOutputStream os;
  74
  75         private CountingOutputStream cos;
  76
  77         private File packBase;
  78
  79         private File packFile;
  80
  81         private File indexFile;
  82
  83         private PackFile pack;
  84
  85         public void setUp() throws Exception {
  86                 super.setUp();
  87                 os = new ByteArrayOutputStream();
  88                 cos = new CountingOutputStream(os);
  89                 packBase = new File(trash, "tmp_pack");
  90                 packFile = new File(trash, "tmp_pack.pack");
  91                 indexFile = new File(trash, "tmp_pack.idx");
  92                 writer = new PackWriter(db, new TextProgressMonitor());
  93         }
  94
  95         /**
  96          * Test constructor for exceptions, default settings, initialization.
  97          */
  98         public void testContructor() {
  99                 assertEquals(false, writer.isDeltaBaseAsOffset());
 100                 assertEquals(true, writer.isReuseDeltas());
 101                 assertEquals(true, writer.isReuseObjects());
 102                 assertEquals(0, writer.getObjectsNumber());
 103         }
 104
 105         /**
 106          * Change default settings and verify them.
 107          */
 108         public void testModifySettings() {
 109                 writer.setDeltaBaseAsOffset(true);
 110                 writer.setReuseDeltas(false);
 111                 writer.setReuseObjects(false);
 112
 113                 assertEquals(true, writer.isDeltaBaseAsOffset());
 114                 assertEquals(false, writer.isReuseDeltas());
 115                 assertEquals(false, writer.isReuseObjects());
 116         }
 117
 118         /**
 119          * Write empty pack by providing empty sets of interesting/uninteresting
 120          * objects and check for correct format.
 121          *
 122          * @throws IOException
 123          */
 124         public void testWriteEmptyPack1() throws IOException {
 125                 createVerifyOpenPack(EMPTY_LIST_OBJECT, EMPTY_LIST_OBJECT, false, false);
 126
 127                 assertEquals(0, writer.getObjectsNumber());
 128                 assertEquals(0, pack.getObjectCount());
 129                 assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
 130                                 .computeName().name());
 131         }
 132
 133         /**
 134          * Write empty pack by providing empty iterator of objects to write and
 135          * check for correct format.
 136          *
 137          * @throws IOException
 138          */
 139         public void testWriteEmptyPack2() throws IOException {
 140                 createVerifyOpenPack(EMPTY_LIST_REVS.iterator());
 141
 142                 assertEquals(0, writer.getObjectsNumber());
 143                 assertEquals(0, pack.getObjectCount());
 144         }
 145
 146         /**
 147          * Try to pass non-existing object as uninteresting, with non-ignoring
 148          * setting.
 149          *
 150          * @throws IOException
 151          */
 152         public void testNotIgnoreNonExistingObjects() throws IOException {
 153                 final ObjectId nonExisting = ObjectId
 154                                 .fromString("0000000000000000000000000000000000000001");
 155                 try {
 156                         createVerifyOpenPack(EMPTY_LIST_OBJECT, Collections.nCopies(1,
 157                                         nonExisting), false, false);
 158                         fail("Should have thrown MissingObjectException");
 159                 } catch (MissingObjectException x) {
 160                         // expected
 161                 }
 162         }
 163
 164         /**
 165          * Try to pass non-existing object as uninteresting, with ignoring setting.
 166          *
 167          * @throws IOException
 168          */
 169         public void testIgnoreNonExistingObjects() throws IOException {
 170                 final ObjectId nonExisting = ObjectId
 171                                 .fromString("0000000000000000000000000000000000000001");
 172                 createVerifyOpenPack(EMPTY_LIST_OBJECT, Collections.nCopies(1,
 173                                 nonExisting), false, true);
 174                 // shouldn't throw anything
 175         }
 176
 177         /**
 178          * Create pack basing on only interesting objects, then precisely verify
 179          * content. No delta reuse here.
 180          *
 181          * @throws IOException
 182          */
 183         public void testWritePack1() throws IOException {
 184                 writer.setReuseDeltas(false);
 185                 writeVerifyPack1();
 186         }
 187
 188         /**
 189          * Test writing pack without object reuse. Pack content/preparation as in
 190          * {@link #testWritePack1()}.
 191          *
 192          * @throws IOException
 193          */
 194         public void testWritePack1NoObjectReuse() throws IOException {
 195                 writer.setReuseDeltas(false);
 196                 writer.setReuseObjects(false);
 197                 writeVerifyPack1();
 198         }
 199
 200         /**
 201          * Create pack basing on both interesting and uninteresting objects, then
 202          * precisely verify content. No delta reuse here.
 203          *
 204          * @throws IOException
 205          */
 206         public void testWritePack2() throws IOException {
 207                 writeVerifyPack2(false);
 208         }
 209
 210         /**
 211          * Test pack writing with deltas reuse, delta-base first rule. Pack
 212          * content/preparation as in {@link #testWritePack2()}.
 213          *
 214          * @throws IOException
 215          */
 216         public void testWritePack2DeltasReuseRefs() throws IOException {
 217                 writeVerifyPack2(true);
 218         }
 219
 220         /**
 221          * Test pack writing with delta reuse. Delta bases referred as offsets. Pack
 222          * configuration as in {@link #testWritePack2DeltasReuseRefs()}.
 223          *
 224          * @throws IOException
 225          */
 226         public void testWritePack2DeltasReuseOffsets() throws IOException {
 227                 writer.setDeltaBaseAsOffset(true);
 228                 writeVerifyPack2(true);
 229         }
 230
 231         /**
 232          * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a
 233          * pack with CRC32 index. Pack configuration as in
 234          * {@link #testWritePack2DeltasReuseRefs()}.
 235          *
 236          * @throws IOException
 237          */
 238         public void testWritePack2DeltasCRC32Copy() throws IOException {
 239                 final File packDir = new File(db.getObjectsDirectory(), "pack");
 240                 final File crc32Pack = new File(packDir,
 241                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
 242                 final File crc32Idx = new File(packDir,
 243                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
 244                 copyFile(JGitTestUtil.getTestResourceFile(
 245                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
 246                                 crc32Idx);
 247                 db.openPack(crc32Pack, crc32Idx);
 248
 249                 writeVerifyPack2(true);
 250         }
 251
 252         /**
 253          * Create pack basing on fixed objects list, then precisely verify content.
 254          * No delta reuse here.
 255          *
 256          * @throws IOException
 257          * @throws MissingObjectException
 258          *
 259          */
 260         public void testWritePack3() throws MissingObjectException, IOException {
 261                 writer.setReuseDeltas(false);
 262                 final ObjectId forcedOrder[] = new ObjectId[] {
 263                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 264                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 265                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 266                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 267                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 268                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 269                 final RevWalk parser = new RevWalk(db);
 270                 final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
 271                 for (int i = 0; i < forcedOrder.length; i++)
 272                         forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
 273
 274                 createVerifyOpenPack(Arrays.asList(forcedOrderRevs).iterator());
 275
 276                 assertEquals(forcedOrder.length, writer.getObjectsNumber());
 277                 verifyObjectsOrder(forcedOrder);
 278                 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
 279                                 .computeName().name());
 280         }
 281
 282         /**
 283          * Another pack creation: basing on both interesting and uninteresting
 284          * objects. No delta reuse possible here, as this is a specific case when we
 285          * write only 1 commit, associated with 1 tree, 1 blob.
 286          *
 287          * @throws IOException
 288          */
 289         public void testWritePack4() throws IOException {
 290                 writeVerifyPack4(false);
 291         }
 292
 293         /**
 294          * Test thin pack writing: 1 blob delta base is on objects edge. Pack
 295          * configuration as in {@link #testWritePack4()}.
 296          *
 297          * @throws IOException
 298          */
 299         public void testWritePack4ThinPack() throws IOException {
 300                 writeVerifyPack4(true);
 301         }
 302
 303         /**
 304          * Compare sizes of packs created using {@link #testWritePack2()} and
 305          * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should
 306          * be smaller.
 307          *
 308          * @throws Exception
 309          */
 310         public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
 311                 testWritePack2();
 312                 final long sizePack2NoDeltas = cos.getCount();
 313                 tearDown();
 314                 setUp();
 315                 testWritePack2DeltasReuseRefs();
 316                 final long sizePack2DeltasRefs = cos.getCount();
 317
 318                 assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
 319         }
 320
 321         /**
 322          * Compare sizes of packs created using
 323          * {@link #testWritePack2DeltasReuseRefs()} and
 324          * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases
 325          * written as offsets should be smaller.
 326          *
 327          * @throws Exception
 328          */
 329         public void testWritePack2SizeOffsetsVsRefs() throws Exception {
 330                 testWritePack2DeltasReuseRefs();
 331                 final long sizePack2DeltasRefs = cos.getCount();
 332                 tearDown();
 333                 setUp();
 334                 testWritePack2DeltasReuseOffsets();
 335                 final long sizePack2DeltasOffsets = cos.getCount();
 336
 337                 assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
 338         }
 339
 340         /**
 341          * Compare sizes of packs created using {@link #testWritePack4()} and
 342          * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be
 343          * smaller.
 344          *
 345          * @throws Exception
 346          */
 347         public void testWritePack4SizeThinVsNoThin() throws Exception {
 348                 testWritePack4();
 349                 final long sizePack4 = cos.getCount();
 350                 tearDown();
 351                 setUp();
 352                 testWritePack4ThinPack();
 353                 final long sizePack4Thin = cos.getCount();
 354
 355                 assertTrue(sizePack4 > sizePack4Thin);
 356         }
 357
 358         public void testWriteIndex() throws Exception {
 359                 writer.setIndexVersion(2);
 360                 writeVerifyPack4(false);
 361
 362                 // Validate that IndexPack came up with the right CRC32 value.
 363                 final PackIndex idx1 = PackIndex.open(indexFile);
 364                 assertTrue(idx1 instanceof PackIndexV2);
 365                 assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId
 366                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
 367
 368                 // Validate that an index written by PackWriter is the same.
 369                 final File idx2File = new File(indexFile.getAbsolutePath() + ".2");
 370                 final FileOutputStream is = new FileOutputStream(idx2File);
 371                 try {
 372                         writer.writeIndex(is);
 373                 } finally {
 374                         is.close();
 375                 }
 376                 final PackIndex idx2 = PackIndex.open(idx2File);
 377                 assertTrue(idx2 instanceof PackIndexV2);
 378                 assertEquals(idx1.getObjectCount(), idx2.getObjectCount());
 379                 assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count());
 380
 381                 for (int i = 0; i < idx1.getObjectCount(); i++) {
 382                         final ObjectId id = idx1.getObjectId(i);
 383                         assertEquals(id, idx2.getObjectId(i));
 384                         assertEquals(idx1.findOffset(id), idx2.findOffset(id));
 385                         assertEquals(idx1.findCRC32(id), idx2.findCRC32(id));
 386                 }
 387         }
 388
 389         // TODO: testWritePackDeltasCycle()
 390         // TODO: testWritePackDeltasDepth()
 391
 392         private void writeVerifyPack1() throws IOException {
 393                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 394                 interestings.add(ObjectId
 395                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 396                 createVerifyOpenPack(interestings, EMPTY_LIST_OBJECT, false, false);
 397
 398                 final ObjectId expectedOrder[] = new ObjectId[] {
 399                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 400                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 401                                 ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
 402                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 403                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 404                                 ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
 405                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 406                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 407
 408                 assertEquals(expectedOrder.length, writer.getObjectsNumber());
 409                 verifyObjectsOrder(expectedOrder);
 410                 assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
 411                                 .computeName().name());
 412         }
 413
 414         private void writeVerifyPack2(boolean deltaReuse) throws IOException {
 415                 writer.setReuseDeltas(deltaReuse);
 416                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 417                 interestings.add(ObjectId
 418                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 419                 final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
 420                 uninterestings.add(ObjectId
 421                                 .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
 422                 createVerifyOpenPack(interestings, uninterestings, false, false);
 423
 424                 final ObjectId expectedOrder[] = new ObjectId[] {
 425                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 426                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 427                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 428                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 429                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 430                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 431                 if (deltaReuse) {
 432                         // objects order influenced (swapped) by delta-base first rule
 433                         ObjectId temp = expectedOrder[4];
 434                         expectedOrder[4] = expectedOrder[5];
 435                         expectedOrder[5] = temp;
 436                 }
 437                 assertEquals(expectedOrder.length, writer.getObjectsNumber());
 438                 verifyObjectsOrder(expectedOrder);
 439                 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
 440                                 .computeName().name());
 441         }
 442
 443         private void writeVerifyPack4(final boolean thin) throws IOException {
 444                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 445                 interestings.add(ObjectId
 446                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 447                 final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
 448                 uninterestings.add(ObjectId
 449                                 .fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
 450                 createVerifyOpenPack(interestings, uninterestings, thin, false);
 451
 452                 final ObjectId writtenObjects[] = new ObjectId[] {
 453                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 454                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 455                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
 456                 assertEquals(writtenObjects.length, writer.getObjectsNumber());
 457                 ObjectId expectedObjects[];
 458                 if (thin) {
 459                         expectedObjects = new ObjectId[4];
 460                         System.arraycopy(writtenObjects, 0, expectedObjects, 0,
 461                                         writtenObjects.length);
 462                         expectedObjects[3] = ObjectId
 463                                         .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
 464
 465                 } else {
 466                         expectedObjects = writtenObjects;
 467                 }
 468                 verifyObjectsOrder(expectedObjects);
 469                 assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
 470                                 .computeName().name());
 471         }
 472
 473         private void createVerifyOpenPack(final Collection<ObjectId> interestings,
 474                         final Collection<ObjectId> uninterestings, final boolean thin,
 475                         final boolean ignoreMissingUninteresting)
 476                         throws MissingObjectException, IOException {
 477                 writer.preparePack(interestings, uninterestings, thin,
 478                                 ignoreMissingUninteresting);
 479                 writer.writePack(cos);
 480                 verifyOpenPack(thin);
 481         }
 482
 483         private void createVerifyOpenPack(final Iterator<RevObject> objectSource)
 484                         throws MissingObjectException, IOException {
 485                 writer.preparePack(objectSource);
 486                 writer.writePack(cos);
 487                 verifyOpenPack(false);
 488         }
 489
 490         private void verifyOpenPack(final boolean thin) throws IOException {
 491                 if (thin) {
 492                         final InputStream is = new ByteArrayInputStream(os.toByteArray());
 493                         final IndexPack indexer = new IndexPack(db, is, packBase);
 494                         try {
 495                                 indexer.index(new TextProgressMonitor());
 496                                 fail("indexer should grumble about missing object");
 497                         } catch (IOException x) {
 498                                 // expected
 499                         }
 500                 }
 501                 final InputStream is = new ByteArrayInputStream(os.toByteArray());
 502                 final IndexPack indexer = new IndexPack(db, is, packBase);
 503                 indexer.setKeepEmpty(true);
 504                 indexer.setFixThin(thin);
 505                 indexer.setIndexVersion(2);
 506                 indexer.index(new TextProgressMonitor());
 507                 pack = new PackFile(indexFile, packFile);
 508         }
 509
 510         private void verifyObjectsOrder(final ObjectId objectsOrder[]) {
 511                 final List<PackIndex.MutableEntry> entries = new ArrayList<PackIndex.MutableEntry>();
 512
 513                 for (MutableEntry me : pack) {
 514                         entries.add(me.cloneEntry());
 515                 }
 516                 Collections.sort(entries, new Comparator<PackIndex.MutableEntry>() {
 517                         public int compare(MutableEntry o1, MutableEntry o2) {
 518                                 return Long.signum(o1.getOffset() - o2.getOffset());
 519                         }
 520                 });
 521
 522                 int i = 0;
 523                 for (MutableEntry me : entries) {
 524                         assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId());
 525                 }
 526         }
 527 }