org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java

   1 /*
   2  * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
   3  *
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or
   7  * without modification, are permitted provided that the following
   8  * conditions are met:
   9  *
  10  * - Redistributions of source code must retain the above copyright
  11  *   notice, this list of conditions and the following disclaimer.
  12  *
  13  * - Redistributions in binary form must reproduce the above
  14  *   copyright notice, this list of conditions and the following
  15  *   disclaimer in the documentation and/or other materials provided
  16  *   with the distribution.
  17  *
  18  * - Neither the name of the Git Development Community nor the
  19  *   names of its contributors may be used to endorse or promote
  20  *   products derived from this software without specific prior
  21  *   written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  24  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  25  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  26  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  32  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  33  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  35  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 package org.spearce.jgit.lib;
  39
  40 import java.io.ByteArrayInputStream;
  41 import java.io.ByteArrayOutputStream;
  42 import java.io.File;
  43 import java.io.IOException;
  44 import java.io.InputStream;
  45 import java.util.ArrayList;
  46 import java.util.Arrays;
  47 import java.util.Collection;
  48 import java.util.Collections;
  49 import java.util.Comparator;
  50 import java.util.Iterator;
  51 import java.util.LinkedList;
  52 import java.util.List;
  53
  54 import org.spearce.jgit.errors.MissingObjectException;
  55 import org.spearce.jgit.lib.PackIndex.MutableEntry;
  56 import org.spearce.jgit.revwalk.RevObject;
  57 import org.spearce.jgit.revwalk.RevWalk;
  58 import org.spearce.jgit.transport.IndexPack;
  59 import org.spearce.jgit.util.CountingOutputStream;
  60
  61 public class PackWriterTest extends RepositoryTestCase {
  62
  63         private static final List<ObjectId> EMPTY_LIST_OBJECT = Collections
  64                         .<ObjectId> emptyList();
  65
  66         private static final List<RevObject> EMPTY_LIST_REVS = Collections
  67                         .<RevObject> emptyList();
  68
  69         private PackWriter writer;
  70
  71         private ByteArrayOutputStream os;
  72
  73         private CountingOutputStream cos;
  74
  75         private File packBase;
  76
  77         private File packFile;
  78
  79         private File indexFile;
  80
  81         private PackFile pack;
  82
  83         public void setUp() throws Exception {
  84                 super.setUp();
  85                 os = new ByteArrayOutputStream();
  86                 cos = new CountingOutputStream(os);
  87                 packBase = new File(trash, "tmp_pack");
  88                 packFile = new File(trash, "tmp_pack.pack");
  89                 indexFile = new File(trash, "tmp_pack.idx");
  90                 writer = new PackWriter(db, cos, new TextProgressMonitor());
  91         }
  92
  93         /**
  94          * Test constructor for exceptions, default settings, initialization.
  95          */
  96         public void testContructor() {
  97                 assertEquals(false, writer.isDeltaBaseAsOffset());
  98                 assertEquals(true, writer.isReuseDeltas());
  99                 assertEquals(true, writer.isReuseObjects());
 100                 assertEquals(0, writer.getObjectsNumber());
 101         }
 102
 103         /**
 104          * Change default settings and verify them.
 105          */
 106         public void testModifySettings() {
 107                 writer.setDeltaBaseAsOffset(true);
 108                 writer.setReuseDeltas(false);
 109                 writer.setReuseObjects(false);
 110
 111                 assertEquals(true, writer.isDeltaBaseAsOffset());
 112                 assertEquals(false, writer.isReuseDeltas());
 113                 assertEquals(false, writer.isReuseObjects());
 114         }
 115
 116         /**
 117          * Write empty pack by providing empty sets of interesting/uninteresting
 118          * objects and check for correct format.
 119          *
 120          * @throws IOException
 121          */
 122         public void testWriteEmptyPack1() throws IOException {
 123                 createVerifyOpenPack(EMPTY_LIST_OBJECT, EMPTY_LIST_OBJECT, false);
 124
 125                 assertEquals(0, writer.getObjectsNumber());
 126                 assertEquals(0, pack.getObjectCount());
 127                 assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
 128                                 .computeName().toString());
 129         }
 130
 131         /**
 132          * Write empty pack by providing empty iterator of objects to write and
 133          * check for correct format.
 134          *
 135          * @throws IOException
 136          */
 137         public void testWriteEmptyPack2() throws IOException {
 138                 createVerifyOpenPack(EMPTY_LIST_REVS.iterator());
 139
 140                 assertEquals(0, writer.getObjectsNumber());
 141                 assertEquals(0, pack.getObjectCount());
 142         }
 143
 144         /**
 145          * Create pack basing on only interesting objects, then precisely verify
 146          * content. No delta reuse here.
 147          *
 148          * @throws IOException
 149          */
 150         public void testWritePack1() throws IOException {
 151                 writer.setReuseDeltas(false);
 152                 writeVerifyPack1();
 153         }
 154
 155         /**
 156          * Test writing pack without object reuse. Pack content/preparation as in
 157          * {@link #testWritePack1()}.
 158          *
 159          * @throws IOException
 160          */
 161         public void testWritePack1NoObjectReuse() throws IOException {
 162                 writer.setReuseDeltas(false);
 163                 writer.setReuseObjects(false);
 164                 writeVerifyPack1();
 165         }
 166
 167         /**
 168          * Create pack basing on both interesting and uninteresting objects, then
 169          * precisely verify content. No delta reuse here.
 170          *
 171          * @throws IOException
 172          */
 173         public void testWritePack2() throws IOException {
 174                 writeVerifyPack2(false);
 175         }
 176
 177         /**
 178          * Test pack writing with deltas reuse, delta-base first rule. Pack
 179          * content/preparation as in {@link #testWritePack2()}.
 180          *
 181          * @throws IOException
 182          */
 183         public void testWritePack2DeltasReuseRefs() throws IOException {
 184                 writeVerifyPack2(true);
 185         }
 186
 187         /**
 188          * Test pack writing with delta reuse. Delta bases referred as offsets. Pack
 189          * configuration as in {@link #testWritePack2DeltasReuseRefs()}.
 190          *
 191          * @throws IOException
 192          */
 193         public void testWritePack2DeltasReuseOffsets() throws IOException {
 194                 writer.setDeltaBaseAsOffset(true);
 195                 writeVerifyPack2(true);
 196         }
 197
 198         /**
 199          * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a
 200          * pack with CRC32 index. Pack configuration as in
 201          * {@link #testWritePack2DeltasReuseRefs()}.
 202          *
 203          * @throws IOException
 204          */
 205         public void testWritePack2DeltasCRC32Copy() throws IOException {
 206                 final File packDir = new File(db.getObjectsDirectory(), "pack");
 207                 final File crc32Pack = new File(packDir,
 208                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
 209                 final File crc32Idx = new File(packDir,
 210                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
 211                 copyFile(new File(new File("tst"),
 212                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
 213                                 crc32Idx);
 214                 db.openPack(crc32Pack, crc32Idx);
 215
 216                 writeVerifyPack2(true);
 217         }
 218
 219         /**
 220          * Create pack basing on fixed objects list, then precisely verify content.
 221          * No delta reuse here.
 222          *
 223          * @throws IOException
 224          * @throws MissingObjectException
 225          *
 226          */
 227         public void testWritePack3() throws MissingObjectException, IOException {
 228                 writer.setReuseDeltas(false);
 229                 final ObjectId forcedOrder[] = new ObjectId[] {
 230                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 231                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 232                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 233                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 234                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 235                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 236                 final RevWalk parser = new RevWalk(db);
 237                 final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
 238                 for (int i = 0; i < forcedOrder.length; i++)
 239                         forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
 240
 241                 createVerifyOpenPack(Arrays.asList(forcedOrderRevs).iterator());
 242
 243                 assertEquals(forcedOrder.length, writer.getObjectsNumber());
 244                 verifyObjectsOrder(forcedOrder);
 245                 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
 246                                 .computeName().toString());
 247         }
 248
 249         /**
 250          * Another pack creation: basing on both interesting and uninteresting
 251          * objects. No delta reuse possible here, as this is a specific case when we
 252          * write only 1 commit, associated with 1 tree, 1 blob.
 253          *
 254          * @throws IOException
 255          */
 256         public void testWritePack4() throws IOException {
 257                 writeVerifyPack4(false);
 258         }
 259
 260         /**
 261          * Test thin pack writing: 1 blob delta base is on objects edge. Pack
 262          * configuration as in {@link #testWritePack4()}.
 263          *
 264          * @throws IOException
 265          */
 266         public void testWritePack4ThinPack() throws IOException {
 267                 writeVerifyPack4(true);
 268         }
 269
 270         /**
 271          * Compare sizes of packs created using {@link #testWritePack2()} and
 272          * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should
 273          * be smaller.
 274          *
 275          * @throws Exception
 276          */
 277         public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
 278                 testWritePack2();
 279                 final int sizePack2NoDeltas = cos.getCount();
 280                 setUp();
 281                 testWritePack2DeltasReuseRefs();
 282                 final int sizePack2DeltasRefs = cos.getCount();
 283
 284                 assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
 285         }
 286
 287         /**
 288          * Compare sizes of packs created using
 289          * {@link #testWritePack2DeltasReuseRefs()} and
 290          * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases
 291          * written as offsets should be smaller.
 292          *
 293          * @throws Exception
 294          */
 295         public void testWritePack2SizeOffsetsVsRefs() throws Exception {
 296                 testWritePack2DeltasReuseRefs();
 297                 final int sizePack2DeltasRefs = cos.getCount();
 298                 setUp();
 299                 testWritePack2DeltasReuseOffsets();
 300                 final int sizePack2DeltasOffsets = cos.getCount();
 301
 302                 assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
 303         }
 304
 305         /**
 306          * Compare sizes of packs created using {@link #testWritePack4()} and
 307          * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be
 308          * smaller.
 309          *
 310          * @throws Exception
 311          */
 312         public void testWritePack4SizeThinVsNoThin() throws Exception {
 313                 testWritePack4();
 314                 final int sizePack4 = cos.getCount();
 315                 setUp();
 316                 testWritePack4ThinPack();
 317                 final int sizePack4Thin = cos.getCount();
 318
 319                 assertTrue(sizePack4 > sizePack4Thin);
 320         }
 321
 322         // TODO: testWritePackDeltasCycle()
 323         // TODO: testWritePackDeltasDepth()
 324
 325         private void writeVerifyPack1() throws IOException {
 326                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 327                 interestings.add(ObjectId
 328                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 329                 createVerifyOpenPack(interestings, EMPTY_LIST_OBJECT, false);
 330
 331                 final ObjectId expectedOrder[] = new ObjectId[] {
 332                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 333                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 334                                 ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
 335                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 336                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 337                                 ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
 338                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 339                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 340
 341                 assertEquals(expectedOrder.length, writer.getObjectsNumber());
 342                 verifyObjectsOrder(expectedOrder);
 343                 assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
 344                                 .computeName().toString());
 345         }
 346
 347         private void writeVerifyPack2(boolean deltaReuse) throws IOException {
 348                 writer.setReuseDeltas(deltaReuse);
 349                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 350                 interestings.add(ObjectId
 351                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 352                 final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
 353                 uninterestings.add(ObjectId
 354                                 .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
 355                 createVerifyOpenPack(interestings, uninterestings, false);
 356
 357                 final ObjectId expectedOrder[] = new ObjectId[] {
 358                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 359                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 360                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 361                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 362                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 363                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 364                 if (deltaReuse) {
 365                         // objects order influenced (swapped) by delta-base first rule
 366                         ObjectId temp = expectedOrder[4];
 367                         expectedOrder[4] = expectedOrder[5];
 368                         expectedOrder[5] = temp;
 369                 }
 370                 assertEquals(expectedOrder.length, writer.getObjectsNumber());
 371                 verifyObjectsOrder(expectedOrder);
 372                 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
 373                                 .computeName().toString());
 374         }
 375
 376         private void writeVerifyPack4(final boolean thin) throws IOException {
 377                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 378                 interestings.add(ObjectId
 379                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 380                 final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
 381                 uninterestings.add(ObjectId
 382                                 .fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
 383                 createVerifyOpenPack(interestings, uninterestings, thin);
 384
 385                 final ObjectId writtenObjects[] = new ObjectId[] {
 386                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 387                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 388                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
 389                 assertEquals(writtenObjects.length, writer.getObjectsNumber());
 390                 ObjectId expectedObjects[];
 391                 if (thin) {
 392                         expectedObjects = new ObjectId[4];
 393                         System.arraycopy(writtenObjects, 0, expectedObjects, 0,
 394                                         writtenObjects.length);
 395                         expectedObjects[3] = ObjectId
 396                                         .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
 397
 398                 } else {
 399                         expectedObjects = writtenObjects;
 400                 }
 401                 verifyObjectsOrder(expectedObjects);
 402                 assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
 403                                 .computeName().toString());
 404         }
 405
 406         private void createVerifyOpenPack(final Collection<ObjectId> interestings,
 407                         final Collection<ObjectId> uninterestings, final boolean thin)
 408                         throws MissingObjectException, IOException {
 409                 writer.writePack(interestings, uninterestings, thin);
 410                 verifyOpenPack(thin);
 411         }
 412
 413         private void createVerifyOpenPack(final Iterator<RevObject> objectSource)
 414                         throws MissingObjectException, IOException {
 415                 writer.writePack(objectSource);
 416                 verifyOpenPack(false);
 417         }
 418
 419         private void verifyOpenPack(final boolean thin) throws IOException {
 420                 if (thin) {
 421                         final InputStream is = new ByteArrayInputStream(os.toByteArray());
 422                         final IndexPack indexer = new IndexPack(db, is, packBase);
 423                         try {
 424                                 indexer.index(new TextProgressMonitor());
 425                                 fail("indexer should grumble about missing object");
 426                         } catch (IOException x) {
 427                                 // expected
 428                         }
 429                 }
 430                 final InputStream is = new ByteArrayInputStream(os.toByteArray());
 431                 final IndexPack indexer = new IndexPack(db, is, packBase);
 432                 indexer.setFixThin(thin);
 433                 indexer.index(new TextProgressMonitor());
 434                 pack = new PackFile(db, indexFile, packFile);
 435         }
 436
 437         private void verifyObjectsOrder(final ObjectId objectsOrder[]) {
 438                 final List<PackIndex.MutableEntry> entries = new ArrayList<PackIndex.MutableEntry>();
 439
 440                 for (MutableEntry me : pack) {
 441                         entries.add(me.cloneEntry());
 442                 }
 443                 Collections.sort(entries, new Comparator<PackIndex.MutableEntry>() {
 444                         public int compare(MutableEntry o1, MutableEntry o2) {
 445                                 return Long.signum(o1.getOffset() - o2.getOffset());
 446                         }
 447                 });
 448
 449                 int i = 0;
 450                 for (MutableEntry me : entries) {
 451                         assertEquals(objectsOrder[i++], me.copy());
 452                 }
 453         }
 454 }