3 #include "repository.h"
13 #include "pack-revindex.h"
14 #include "csum-file.h"
15 #include "tree-walk.h"
18 #include "list-objects.h"
19 #include "list-objects-filter.h"
20 #include "list-objects-filter-options.h"
21 #include "pack-objects.h"
24 #include "streaming.h"
25 #include "thread-utils.h"
26 #include "pack-bitmap.h"
27 #include "reachable.h"
28 #include "sha1-array.h"
29 #include "argv-array.h"
32 #include "object-store.h"
35 #define IN_PACK(obj) oe_in_pack(&to_pack, obj)
36 #define SIZE(obj) oe_size(&to_pack, obj)
37 #define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size)
38 #define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj)
39 #define DELTA(obj) oe_delta(&to_pack, obj)
40 #define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj)
41 #define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj)
42 #define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val)
43 #define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid)
44 #define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val)
45 #define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val)
46 #define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val)
48 static const char *pack_usage
[] = {
49 N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"),
50 N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"),
55 * Objects we are going to pack are collected in the `to_pack` structure.
56 * It contains an array (dynamically expanded) of the object data, and a map
57 * that can resolve SHA1s to their position in the array.
59 static struct packing_data to_pack
;
61 static struct pack_idx_entry
**written_list
;
62 static uint32_t nr_result
, nr_written
, nr_seen
;
63 static struct bitmap_index
*bitmap_git
;
66 static int reuse_delta
= 1, reuse_object
= 1;
67 static int keep_unreachable
, unpack_unreachable
, include_tag
;
68 static timestamp_t unpack_unreachable_expiration
;
69 static int pack_loose_unreachable
;
71 static int have_non_local_packs
;
72 static int incremental
;
73 static int ignore_packed_keep_on_disk
;
74 static int ignore_packed_keep_in_core
;
75 static int allow_ofs_delta
;
76 static struct pack_idx_option pack_idx_opts
;
77 static const char *base_name
;
78 static int progress
= 1;
79 static int window
= 10;
80 static unsigned long pack_size_limit
;
81 static int depth
= 50;
82 static int delta_search_threads
;
83 static int pack_to_stdout
;
85 static int num_preferred_base
;
86 static struct progress
*progress_state
;
88 static struct packed_git
*reuse_packfile
;
89 static uint32_t reuse_packfile_objects
;
90 static off_t reuse_packfile_offset
;
92 static int use_bitmap_index_default
= 1;
93 static int use_bitmap_index
= -1;
94 static int write_bitmap_index
;
95 static uint16_t write_bitmap_options
;
97 static int exclude_promisor_objects
;
99 static unsigned long delta_cache_size
= 0;
100 static unsigned long max_delta_cache_size
= DEFAULT_DELTA_CACHE_SIZE
;
101 static unsigned long cache_max_small_delta_size
= 1000;
103 static unsigned long window_memory_limit
= 0;
105 static struct list_objects_filter_options filter_options
;
107 enum missing_action
{
108 MA_ERROR
= 0, /* fail if any missing objects are encountered */
109 MA_ALLOW_ANY
, /* silently allow ALL missing objects */
110 MA_ALLOW_PROMISOR
, /* silently allow all missing PROMISOR objects */
112 static enum missing_action arg_missing_action
;
113 static show_object_fn fn_show_object
;
118 static uint32_t written
, written_delta
;
119 static uint32_t reused
, reused_delta
;
124 static struct commit
**indexed_commits
;
125 static unsigned int indexed_commits_nr
;
126 static unsigned int indexed_commits_alloc
;
128 static void index_commit_for_bitmap(struct commit
*commit
)
130 if (indexed_commits_nr
>= indexed_commits_alloc
) {
131 indexed_commits_alloc
= (indexed_commits_alloc
+ 32) * 2;
132 REALLOC_ARRAY(indexed_commits
, indexed_commits_alloc
);
135 indexed_commits
[indexed_commits_nr
++] = commit
;
138 static void *get_delta(struct object_entry
*entry
)
140 unsigned long size
, base_size
, delta_size
;
141 void *buf
, *base_buf
, *delta_buf
;
142 enum object_type type
;
144 buf
= read_object_file(&entry
->idx
.oid
, &type
, &size
);
146 die(_("unable to read %s"), oid_to_hex(&entry
->idx
.oid
));
147 base_buf
= read_object_file(&DELTA(entry
)->idx
.oid
, &type
,
150 die("unable to read %s",
151 oid_to_hex(&DELTA(entry
)->idx
.oid
));
152 delta_buf
= diff_delta(base_buf
, base_size
,
153 buf
, size
, &delta_size
, 0);
155 * We succesfully computed this delta once but dropped it for
156 * memory reasons. Something is very wrong if this time we
157 * recompute and create a different delta.
159 if (!delta_buf
|| delta_size
!= DELTA_SIZE(entry
))
160 BUG("delta size changed");
166 static unsigned long do_compress(void **pptr
, unsigned long size
)
170 unsigned long maxsize
;
172 git_deflate_init(&stream
, pack_compression_level
);
173 maxsize
= git_deflate_bound(&stream
, size
);
176 out
= xmalloc(maxsize
);
180 stream
.avail_in
= size
;
181 stream
.next_out
= out
;
182 stream
.avail_out
= maxsize
;
183 while (git_deflate(&stream
, Z_FINISH
) == Z_OK
)
185 git_deflate_end(&stream
);
188 return stream
.total_out
;
191 static unsigned long write_large_blob_data(struct git_istream
*st
, struct hashfile
*f
,
192 const struct object_id
*oid
)
195 unsigned char ibuf
[1024 * 16];
196 unsigned char obuf
[1024 * 16];
197 unsigned long olen
= 0;
199 git_deflate_init(&stream
, pack_compression_level
);
204 readlen
= read_istream(st
, ibuf
, sizeof(ibuf
));
206 die(_("unable to read %s"), oid_to_hex(oid
));
208 stream
.next_in
= ibuf
;
209 stream
.avail_in
= readlen
;
210 while ((stream
.avail_in
|| readlen
== 0) &&
211 (zret
== Z_OK
|| zret
== Z_BUF_ERROR
)) {
212 stream
.next_out
= obuf
;
213 stream
.avail_out
= sizeof(obuf
);
214 zret
= git_deflate(&stream
, readlen
? 0 : Z_FINISH
);
215 hashwrite(f
, obuf
, stream
.next_out
- obuf
);
216 olen
+= stream
.next_out
- obuf
;
219 die(_("deflate error (%d)"), zret
);
221 if (zret
!= Z_STREAM_END
)
222 die(_("deflate error (%d)"), zret
);
226 git_deflate_end(&stream
);
231 * we are going to reuse the existing object data as is. make
232 * sure it is not corrupt.
234 static int check_pack_inflate(struct packed_git
*p
,
235 struct pack_window
**w_curs
,
238 unsigned long expect
)
241 unsigned char fakebuf
[4096], *in
;
244 memset(&stream
, 0, sizeof(stream
));
245 git_inflate_init(&stream
);
247 in
= use_pack(p
, w_curs
, offset
, &stream
.avail_in
);
249 stream
.next_out
= fakebuf
;
250 stream
.avail_out
= sizeof(fakebuf
);
251 st
= git_inflate(&stream
, Z_FINISH
);
252 offset
+= stream
.next_in
- in
;
253 } while (st
== Z_OK
|| st
== Z_BUF_ERROR
);
254 git_inflate_end(&stream
);
255 return (st
== Z_STREAM_END
&&
256 stream
.total_out
== expect
&&
257 stream
.total_in
== len
) ? 0 : -1;
260 static void copy_pack_data(struct hashfile
*f
,
261 struct packed_git
*p
,
262 struct pack_window
**w_curs
,
270 in
= use_pack(p
, w_curs
, offset
, &avail
);
272 avail
= (unsigned long)len
;
273 hashwrite(f
, in
, avail
);
279 /* Return 0 if we will bust the pack-size limit */
280 static unsigned long write_no_reuse_object(struct hashfile
*f
, struct object_entry
*entry
,
281 unsigned long limit
, int usable_delta
)
283 unsigned long size
, datalen
;
284 unsigned char header
[MAX_PACK_OBJECT_HEADER
],
285 dheader
[MAX_PACK_OBJECT_HEADER
];
287 enum object_type type
;
289 struct git_istream
*st
= NULL
;
290 const unsigned hashsz
= the_hash_algo
->rawsz
;
293 if (oe_type(entry
) == OBJ_BLOB
&&
294 oe_size_greater_than(&to_pack
, entry
, big_file_threshold
) &&
295 (st
= open_istream(&entry
->idx
.oid
, &type
, &size
, NULL
)) != NULL
)
298 buf
= read_object_file(&entry
->idx
.oid
, &type
, &size
);
300 die(_("unable to read %s"),
301 oid_to_hex(&entry
->idx
.oid
));
304 * make sure no cached delta data remains from a
305 * previous attempt before a pack split occurred.
307 FREE_AND_NULL(entry
->delta_data
);
308 entry
->z_delta_size
= 0;
309 } else if (entry
->delta_data
) {
310 size
= DELTA_SIZE(entry
);
311 buf
= entry
->delta_data
;
312 entry
->delta_data
= NULL
;
313 type
= (allow_ofs_delta
&& DELTA(entry
)->idx
.offset
) ?
314 OBJ_OFS_DELTA
: OBJ_REF_DELTA
;
316 buf
= get_delta(entry
);
317 size
= DELTA_SIZE(entry
);
318 type
= (allow_ofs_delta
&& DELTA(entry
)->idx
.offset
) ?
319 OBJ_OFS_DELTA
: OBJ_REF_DELTA
;
322 if (st
) /* large blob case, just assume we don't compress well */
324 else if (entry
->z_delta_size
)
325 datalen
= entry
->z_delta_size
;
327 datalen
= do_compress(&buf
, size
);
330 * The object header is a byte of 'type' followed by zero or
331 * more bytes of length.
333 hdrlen
= encode_in_pack_object_header(header
, sizeof(header
),
336 if (type
== OBJ_OFS_DELTA
) {
338 * Deltas with relative base contain an additional
339 * encoding of the relative offset for the delta
340 * base from this object's position in the pack.
342 off_t ofs
= entry
->idx
.offset
- DELTA(entry
)->idx
.offset
;
343 unsigned pos
= sizeof(dheader
) - 1;
344 dheader
[pos
] = ofs
& 127;
346 dheader
[--pos
] = 128 | (--ofs
& 127);
347 if (limit
&& hdrlen
+ sizeof(dheader
) - pos
+ datalen
+ hashsz
>= limit
) {
353 hashwrite(f
, header
, hdrlen
);
354 hashwrite(f
, dheader
+ pos
, sizeof(dheader
) - pos
);
355 hdrlen
+= sizeof(dheader
) - pos
;
356 } else if (type
== OBJ_REF_DELTA
) {
358 * Deltas with a base reference contain
359 * additional bytes for the base object ID.
361 if (limit
&& hdrlen
+ hashsz
+ datalen
+ hashsz
>= limit
) {
367 hashwrite(f
, header
, hdrlen
);
368 hashwrite(f
, DELTA(entry
)->idx
.oid
.hash
, hashsz
);
371 if (limit
&& hdrlen
+ datalen
+ hashsz
>= limit
) {
377 hashwrite(f
, header
, hdrlen
);
380 datalen
= write_large_blob_data(st
, f
, &entry
->idx
.oid
);
383 hashwrite(f
, buf
, datalen
);
387 return hdrlen
+ datalen
;
390 /* Return 0 if we will bust the pack-size limit */
391 static off_t
write_reuse_object(struct hashfile
*f
, struct object_entry
*entry
,
392 unsigned long limit
, int usable_delta
)
394 struct packed_git
*p
= IN_PACK(entry
);
395 struct pack_window
*w_curs
= NULL
;
396 struct revindex_entry
*revidx
;
398 enum object_type type
= oe_type(entry
);
400 unsigned char header
[MAX_PACK_OBJECT_HEADER
],
401 dheader
[MAX_PACK_OBJECT_HEADER
];
403 const unsigned hashsz
= the_hash_algo
->rawsz
;
404 unsigned long entry_size
= SIZE(entry
);
407 type
= (allow_ofs_delta
&& DELTA(entry
)->idx
.offset
) ?
408 OBJ_OFS_DELTA
: OBJ_REF_DELTA
;
409 hdrlen
= encode_in_pack_object_header(header
, sizeof(header
),
412 offset
= entry
->in_pack_offset
;
413 revidx
= find_pack_revindex(p
, offset
);
414 datalen
= revidx
[1].offset
- offset
;
415 if (!pack_to_stdout
&& p
->index_version
> 1 &&
416 check_pack_crc(p
, &w_curs
, offset
, datalen
, revidx
->nr
)) {
417 error(_("bad packed object CRC for %s"),
418 oid_to_hex(&entry
->idx
.oid
));
420 return write_no_reuse_object(f
, entry
, limit
, usable_delta
);
423 offset
+= entry
->in_pack_header_size
;
424 datalen
-= entry
->in_pack_header_size
;
426 if (!pack_to_stdout
&& p
->index_version
== 1 &&
427 check_pack_inflate(p
, &w_curs
, offset
, datalen
, entry_size
)) {
428 error(_("corrupt packed object for %s"),
429 oid_to_hex(&entry
->idx
.oid
));
431 return write_no_reuse_object(f
, entry
, limit
, usable_delta
);
434 if (type
== OBJ_OFS_DELTA
) {
435 off_t ofs
= entry
->idx
.offset
- DELTA(entry
)->idx
.offset
;
436 unsigned pos
= sizeof(dheader
) - 1;
437 dheader
[pos
] = ofs
& 127;
439 dheader
[--pos
] = 128 | (--ofs
& 127);
440 if (limit
&& hdrlen
+ sizeof(dheader
) - pos
+ datalen
+ hashsz
>= limit
) {
444 hashwrite(f
, header
, hdrlen
);
445 hashwrite(f
, dheader
+ pos
, sizeof(dheader
) - pos
);
446 hdrlen
+= sizeof(dheader
) - pos
;
448 } else if (type
== OBJ_REF_DELTA
) {
449 if (limit
&& hdrlen
+ hashsz
+ datalen
+ hashsz
>= limit
) {
453 hashwrite(f
, header
, hdrlen
);
454 hashwrite(f
, DELTA(entry
)->idx
.oid
.hash
, hashsz
);
458 if (limit
&& hdrlen
+ datalen
+ hashsz
>= limit
) {
462 hashwrite(f
, header
, hdrlen
);
464 copy_pack_data(f
, p
, &w_curs
, offset
, datalen
);
467 return hdrlen
+ datalen
;
470 /* Return 0 if we will bust the pack-size limit */
471 static off_t
write_object(struct hashfile
*f
,
472 struct object_entry
*entry
,
477 int usable_delta
, to_reuse
;
482 /* apply size limit if limited packsize and not first object */
483 if (!pack_size_limit
|| !nr_written
)
485 else if (pack_size_limit
<= write_offset
)
487 * the earlier object did not fit the limit; avoid
488 * mistaking this with unlimited (i.e. limit = 0).
492 limit
= pack_size_limit
- write_offset
;
495 usable_delta
= 0; /* no delta */
496 else if (!pack_size_limit
)
497 usable_delta
= 1; /* unlimited packfile */
498 else if (DELTA(entry
)->idx
.offset
== (off_t
)-1)
499 usable_delta
= 0; /* base was written to another pack */
500 else if (DELTA(entry
)->idx
.offset
)
501 usable_delta
= 1; /* base already exists in this pack */
503 usable_delta
= 0; /* base could end up in another pack */
506 to_reuse
= 0; /* explicit */
507 else if (!IN_PACK(entry
))
508 to_reuse
= 0; /* can't reuse what we don't have */
509 else if (oe_type(entry
) == OBJ_REF_DELTA
||
510 oe_type(entry
) == OBJ_OFS_DELTA
)
511 /* check_object() decided it for us ... */
512 to_reuse
= usable_delta
;
513 /* ... but pack split may override that */
514 else if (oe_type(entry
) != entry
->in_pack_type
)
515 to_reuse
= 0; /* pack has delta which is unusable */
516 else if (DELTA(entry
))
517 to_reuse
= 0; /* we want to pack afresh */
519 to_reuse
= 1; /* we have it in-pack undeltified,
520 * and we do not need to deltify it.
524 len
= write_no_reuse_object(f
, entry
, limit
, usable_delta
);
526 len
= write_reuse_object(f
, entry
, limit
, usable_delta
);
534 entry
->idx
.crc32
= crc32_end(f
);
538 enum write_one_status
{
539 WRITE_ONE_SKIP
= -1, /* already written */
540 WRITE_ONE_BREAK
= 0, /* writing this will bust the limit; not written */
541 WRITE_ONE_WRITTEN
= 1, /* normal */
542 WRITE_ONE_RECURSIVE
= 2 /* already scheduled to be written */
545 static enum write_one_status
write_one(struct hashfile
*f
,
546 struct object_entry
*e
,
553 * we set offset to 1 (which is an impossible value) to mark
554 * the fact that this object is involved in "write its base
555 * first before writing a deltified object" recursion.
557 recursing
= (e
->idx
.offset
== 1);
559 warning(_("recursive delta detected for object %s"),
560 oid_to_hex(&e
->idx
.oid
));
561 return WRITE_ONE_RECURSIVE
;
562 } else if (e
->idx
.offset
|| e
->preferred_base
) {
563 /* offset is non zero if object is written already. */
564 return WRITE_ONE_SKIP
;
567 /* if we are deltified, write out base object first. */
569 e
->idx
.offset
= 1; /* now recurse */
570 switch (write_one(f
, DELTA(e
), offset
)) {
571 case WRITE_ONE_RECURSIVE
:
572 /* we cannot depend on this one */
577 case WRITE_ONE_BREAK
:
578 e
->idx
.offset
= recursing
;
579 return WRITE_ONE_BREAK
;
583 e
->idx
.offset
= *offset
;
584 size
= write_object(f
, e
, *offset
);
586 e
->idx
.offset
= recursing
;
587 return WRITE_ONE_BREAK
;
589 written_list
[nr_written
++] = &e
->idx
;
591 /* make sure off_t is sufficiently large not to wrap */
592 if (signed_add_overflows(*offset
, size
))
593 die(_("pack too large for current definition of off_t"));
595 return WRITE_ONE_WRITTEN
;
598 static int mark_tagged(const char *path
, const struct object_id
*oid
, int flag
,
601 struct object_id peeled
;
602 struct object_entry
*entry
= packlist_find(&to_pack
, oid
->hash
, NULL
);
606 if (!peel_ref(path
, &peeled
)) {
607 entry
= packlist_find(&to_pack
, peeled
.hash
, NULL
);
614 static inline void add_to_write_order(struct object_entry
**wo
,
616 struct object_entry
*e
)
624 static void add_descendants_to_write_order(struct object_entry
**wo
,
626 struct object_entry
*e
)
628 int add_to_order
= 1;
631 struct object_entry
*s
;
632 /* add this node... */
633 add_to_write_order(wo
, endp
, e
);
634 /* all its siblings... */
635 for (s
= DELTA_SIBLING(e
); s
; s
= DELTA_SIBLING(s
)) {
636 add_to_write_order(wo
, endp
, s
);
639 /* drop down a level to add left subtree nodes if possible */
640 if (DELTA_CHILD(e
)) {
645 /* our sibling might have some children, it is next */
646 if (DELTA_SIBLING(e
)) {
647 e
= DELTA_SIBLING(e
);
650 /* go back to our parent node */
652 while (e
&& !DELTA_SIBLING(e
)) {
653 /* we're on the right side of a subtree, keep
654 * going up until we can go right again */
658 /* done- we hit our original root node */
661 /* pass it off to sibling at this level */
662 e
= DELTA_SIBLING(e
);
667 static void add_family_to_write_order(struct object_entry
**wo
,
669 struct object_entry
*e
)
671 struct object_entry
*root
;
673 for (root
= e
; DELTA(root
); root
= DELTA(root
))
675 add_descendants_to_write_order(wo
, endp
, root
);
678 static struct object_entry
**compute_write_order(void)
680 unsigned int i
, wo_end
, last_untagged
;
682 struct object_entry
**wo
;
683 struct object_entry
*objects
= to_pack
.objects
;
685 for (i
= 0; i
< to_pack
.nr_objects
; i
++) {
686 objects
[i
].tagged
= 0;
687 objects
[i
].filled
= 0;
688 SET_DELTA_CHILD(&objects
[i
], NULL
);
689 SET_DELTA_SIBLING(&objects
[i
], NULL
);
693 * Fully connect delta_child/delta_sibling network.
694 * Make sure delta_sibling is sorted in the original
697 for (i
= to_pack
.nr_objects
; i
> 0;) {
698 struct object_entry
*e
= &objects
[--i
];
701 /* Mark me as the first child */
702 e
->delta_sibling_idx
= DELTA(e
)->delta_child_idx
;
703 SET_DELTA_CHILD(DELTA(e
), e
);
707 * Mark objects that are at the tip of tags.
709 for_each_tag_ref(mark_tagged
, NULL
);
712 * Give the objects in the original recency order until
713 * we see a tagged tip.
715 ALLOC_ARRAY(wo
, to_pack
.nr_objects
);
716 for (i
= wo_end
= 0; i
< to_pack
.nr_objects
; i
++) {
717 if (objects
[i
].tagged
)
719 add_to_write_order(wo
, &wo_end
, &objects
[i
]);
724 * Then fill all the tagged tips.
726 for (; i
< to_pack
.nr_objects
; i
++) {
727 if (objects
[i
].tagged
)
728 add_to_write_order(wo
, &wo_end
, &objects
[i
]);
732 * And then all remaining commits and tags.
734 for (i
= last_untagged
; i
< to_pack
.nr_objects
; i
++) {
735 if (oe_type(&objects
[i
]) != OBJ_COMMIT
&&
736 oe_type(&objects
[i
]) != OBJ_TAG
)
738 add_to_write_order(wo
, &wo_end
, &objects
[i
]);
742 * And then all the trees.
744 for (i
= last_untagged
; i
< to_pack
.nr_objects
; i
++) {
745 if (oe_type(&objects
[i
]) != OBJ_TREE
)
747 add_to_write_order(wo
, &wo_end
, &objects
[i
]);
751 * Finally all the rest in really tight order
753 for (i
= last_untagged
; i
< to_pack
.nr_objects
; i
++) {
754 if (!objects
[i
].filled
)
755 add_family_to_write_order(wo
, &wo_end
, &objects
[i
]);
758 if (wo_end
!= to_pack
.nr_objects
)
759 die(_("ordered %u objects, expected %"PRIu32
),
760 wo_end
, to_pack
.nr_objects
);
765 static off_t
write_reused_pack(struct hashfile
*f
)
767 unsigned char buffer
[8192];
768 off_t to_write
, total
;
771 if (!is_pack_valid(reuse_packfile
))
772 die(_("packfile is invalid: %s"), reuse_packfile
->pack_name
);
774 fd
= git_open(reuse_packfile
->pack_name
);
776 die_errno(_("unable to open packfile for reuse: %s"),
777 reuse_packfile
->pack_name
);
779 if (lseek(fd
, sizeof(struct pack_header
), SEEK_SET
) == -1)
780 die_errno(_("unable to seek in reused packfile"));
782 if (reuse_packfile_offset
< 0)
783 reuse_packfile_offset
= reuse_packfile
->pack_size
- the_hash_algo
->rawsz
;
785 total
= to_write
= reuse_packfile_offset
- sizeof(struct pack_header
);
788 int read_pack
= xread(fd
, buffer
, sizeof(buffer
));
791 die_errno(_("unable to read from reused packfile"));
793 if (read_pack
> to_write
)
794 read_pack
= to_write
;
796 hashwrite(f
, buffer
, read_pack
);
797 to_write
-= read_pack
;
800 * We don't know the actual number of objects written,
801 * only how many bytes written, how many bytes total, and
802 * how many objects total. So we can fake it by pretending all
803 * objects we are writing are the same size. This gives us a
804 * smooth progress meter, and at the end it matches the true
807 written
= reuse_packfile_objects
*
808 (((double)(total
- to_write
)) / total
);
809 display_progress(progress_state
, written
);
813 written
= reuse_packfile_objects
;
814 display_progress(progress_state
, written
);
815 return reuse_packfile_offset
- sizeof(struct pack_header
);
818 static const char no_split_warning
[] = N_(
819 "disabling bitmap writing, packs are split due to pack.packSizeLimit"
822 static void write_pack_file(void)
827 uint32_t nr_remaining
= nr_result
;
828 time_t last_mtime
= 0;
829 struct object_entry
**write_order
;
831 if (progress
> pack_to_stdout
)
832 progress_state
= start_progress(_("Writing objects"), nr_result
);
833 ALLOC_ARRAY(written_list
, to_pack
.nr_objects
);
834 write_order
= compute_write_order();
837 struct object_id oid
;
838 char *pack_tmp_name
= NULL
;
841 f
= hashfd_throughput(1, "<stdout>", progress_state
);
843 f
= create_tmp_packfile(&pack_tmp_name
);
845 offset
= write_pack_header(f
, nr_remaining
);
847 if (reuse_packfile
) {
849 assert(pack_to_stdout
);
851 packfile_size
= write_reused_pack(f
);
852 offset
+= packfile_size
;
856 for (; i
< to_pack
.nr_objects
; i
++) {
857 struct object_entry
*e
= write_order
[i
];
858 if (write_one(f
, e
, &offset
) == WRITE_ONE_BREAK
)
860 display_progress(progress_state
, written
);
864 * Did we write the wrong # entries in the header?
865 * If so, rewrite it like in fast-import
867 if (pack_to_stdout
) {
868 finalize_hashfile(f
, oid
.hash
, CSUM_HASH_IN_STREAM
| CSUM_CLOSE
);
869 } else if (nr_written
== nr_remaining
) {
870 finalize_hashfile(f
, oid
.hash
, CSUM_HASH_IN_STREAM
| CSUM_FSYNC
| CSUM_CLOSE
);
872 int fd
= finalize_hashfile(f
, oid
.hash
, 0);
873 fixup_pack_header_footer(fd
, oid
.hash
, pack_tmp_name
,
874 nr_written
, oid
.hash
, offset
);
876 if (write_bitmap_index
) {
877 warning(_(no_split_warning
));
878 write_bitmap_index
= 0;
882 if (!pack_to_stdout
) {
884 struct strbuf tmpname
= STRBUF_INIT
;
887 * Packs are runtime accessed in their mtime
888 * order since newer packs are more likely to contain
889 * younger objects. So if we are creating multiple
890 * packs then we should modify the mtime of later ones
891 * to preserve this property.
893 if (stat(pack_tmp_name
, &st
) < 0) {
894 warning_errno(_("failed to stat %s"), pack_tmp_name
);
895 } else if (!last_mtime
) {
896 last_mtime
= st
.st_mtime
;
899 utb
.actime
= st
.st_atime
;
900 utb
.modtime
= --last_mtime
;
901 if (utime(pack_tmp_name
, &utb
) < 0)
902 warning_errno(_("failed utime() on %s"), pack_tmp_name
);
905 strbuf_addf(&tmpname
, "%s-", base_name
);
907 if (write_bitmap_index
) {
908 bitmap_writer_set_checksum(oid
.hash
);
909 bitmap_writer_build_type_index(
910 &to_pack
, written_list
, nr_written
);
913 finish_tmp_packfile(&tmpname
, pack_tmp_name
,
914 written_list
, nr_written
,
915 &pack_idx_opts
, oid
.hash
);
917 if (write_bitmap_index
) {
918 strbuf_addf(&tmpname
, "%s.bitmap", oid_to_hex(&oid
));
920 stop_progress(&progress_state
);
922 bitmap_writer_show_progress(progress
);
923 bitmap_writer_reuse_bitmaps(&to_pack
);
924 bitmap_writer_select_commits(indexed_commits
, indexed_commits_nr
, -1);
925 bitmap_writer_build(&to_pack
);
926 bitmap_writer_finish(written_list
, nr_written
,
927 tmpname
.buf
, write_bitmap_options
);
928 write_bitmap_index
= 0;
931 strbuf_release(&tmpname
);
933 puts(oid_to_hex(&oid
));
936 /* mark written objects as written to previous pack */
937 for (j
= 0; j
< nr_written
; j
++) {
938 written_list
[j
]->offset
= (off_t
)-1;
940 nr_remaining
-= nr_written
;
941 } while (nr_remaining
&& i
< to_pack
.nr_objects
);
945 stop_progress(&progress_state
);
946 if (written
!= nr_result
)
947 die(_("wrote %"PRIu32
" objects while expecting %"PRIu32
),
951 static int no_try_delta(const char *path
)
953 static struct attr_check
*check
;
956 check
= attr_check_initl("delta", NULL
);
957 if (git_check_attr(&the_index
, path
, check
))
959 if (ATTR_FALSE(check
->items
[0].value
))
965 * When adding an object, check whether we have already added it
966 * to our packing list. If so, we can skip. However, if we are
967 * being asked to excludei t, but the previous mention was to include
968 * it, make sure to adjust its flags and tweak our numbers accordingly.
970 * As an optimization, we pass out the index position where we would have
971 * found the item, since that saves us from having to look it up again a
972 * few lines later when we want to add the new entry.
974 static int have_duplicate_entry(const struct object_id
*oid
,
978 struct object_entry
*entry
;
980 entry
= packlist_find(&to_pack
, oid
->hash
, index_pos
);
985 if (!entry
->preferred_base
)
987 entry
->preferred_base
= 1;
993 static int want_found_object(int exclude
, struct packed_git
*p
)
1001 * When asked to do --local (do not include an object that appears in a
1002 * pack we borrow from elsewhere) or --honor-pack-keep (do not include
1003 * an object that appears in a pack marked with .keep), finding a pack
1004 * that matches the criteria is sufficient for us to decide to omit it.
1005 * However, even if this pack does not satisfy the criteria, we need to
1006 * make sure no copy of this object appears in _any_ pack that makes us
1007 * to omit the object, so we need to check all the packs.
1009 * We can however first check whether these options can possible matter;
1010 * if they do not matter we know we want the object in generated pack.
1011 * Otherwise, we signal "-1" at the end to tell the caller that we do
1012 * not know either way, and it needs to check more packs.
1014 if (!ignore_packed_keep_on_disk
&&
1015 !ignore_packed_keep_in_core
&&
1016 (!local
|| !have_non_local_packs
))
1019 if (local
&& !p
->pack_local
)
1021 if (p
->pack_local
&&
1022 ((ignore_packed_keep_on_disk
&& p
->pack_keep
) ||
1023 (ignore_packed_keep_in_core
&& p
->pack_keep_in_core
)))
1026 /* we don't know yet; keep looking for more packs */
1031 * Check whether we want the object in the pack (e.g., we do not want
1032 * objects found in non-local stores if the "--local" option was used).
1034 * If the caller already knows an existing pack it wants to take the object
1035 * from, that is passed in *found_pack and *found_offset; otherwise this
1036 * function finds if there is any pack that has the object and returns the pack
1037 * and its offset in these variables.
1039 static int want_object_in_pack(const struct object_id
*oid
,
1041 struct packed_git
**found_pack
,
1042 off_t
*found_offset
)
1045 struct list_head
*pos
;
1047 if (!exclude
&& local
&& has_loose_object_nonlocal(oid
))
1051 * If we already know the pack object lives in, start checks from that
1052 * pack - in the usual case when neither --local was given nor .keep files
1053 * are present we will determine the answer right now.
1056 want
= want_found_object(exclude
, *found_pack
);
1060 list_for_each(pos
, get_packed_git_mru(the_repository
)) {
1061 struct packed_git
*p
= list_entry(pos
, struct packed_git
, mru
);
1064 if (p
== *found_pack
)
1065 offset
= *found_offset
;
1067 offset
= find_pack_entry_one(oid
->hash
, p
);
1071 if (!is_pack_valid(p
))
1073 *found_offset
= offset
;
1076 want
= want_found_object(exclude
, p
);
1077 if (!exclude
&& want
> 0)
1079 get_packed_git_mru(the_repository
));
1088 static void create_object_entry(const struct object_id
*oid
,
1089 enum object_type type
,
1094 struct packed_git
*found_pack
,
1097 struct object_entry
*entry
;
1099 entry
= packlist_alloc(&to_pack
, oid
->hash
, index_pos
);
1101 oe_set_type(entry
, type
);
1103 entry
->preferred_base
= 1;
1107 oe_set_in_pack(&to_pack
, entry
, found_pack
);
1108 entry
->in_pack_offset
= found_offset
;
1111 entry
->no_try_delta
= no_try_delta
;
1114 static const char no_closure_warning
[] = N_(
1115 "disabling bitmap writing, as some objects are not being packed"
1118 static int add_object_entry(const struct object_id
*oid
, enum object_type type
,
1119 const char *name
, int exclude
)
1121 struct packed_git
*found_pack
= NULL
;
1122 off_t found_offset
= 0;
1125 display_progress(progress_state
, ++nr_seen
);
1127 if (have_duplicate_entry(oid
, exclude
, &index_pos
))
1130 if (!want_object_in_pack(oid
, exclude
, &found_pack
, &found_offset
)) {
1131 /* The pack is missing an object, so it will not have closure */
1132 if (write_bitmap_index
) {
1133 warning(_(no_closure_warning
));
1134 write_bitmap_index
= 0;
1139 create_object_entry(oid
, type
, pack_name_hash(name
),
1140 exclude
, name
&& no_try_delta(name
),
1141 index_pos
, found_pack
, found_offset
);
1145 static int add_object_entry_from_bitmap(const struct object_id
*oid
,
1146 enum object_type type
,
1147 int flags
, uint32_t name_hash
,
1148 struct packed_git
*pack
, off_t offset
)
1152 display_progress(progress_state
, ++nr_seen
);
1154 if (have_duplicate_entry(oid
, 0, &index_pos
))
1157 if (!want_object_in_pack(oid
, 0, &pack
, &offset
))
1160 create_object_entry(oid
, type
, name_hash
, 0, 0, index_pos
, pack
, offset
);
1164 struct pbase_tree_cache
{
1165 struct object_id oid
;
1169 unsigned long tree_size
;
1172 static struct pbase_tree_cache
*(pbase_tree_cache
[256]);
1173 static int pbase_tree_cache_ix(const struct object_id
*oid
)
1175 return oid
->hash
[0] % ARRAY_SIZE(pbase_tree_cache
);
1177 static int pbase_tree_cache_ix_incr(int ix
)
1179 return (ix
+1) % ARRAY_SIZE(pbase_tree_cache
);
1182 static struct pbase_tree
{
1183 struct pbase_tree
*next
;
1184 /* This is a phony "cache" entry; we are not
1185 * going to evict it or find it through _get()
1186 * mechanism -- this is for the toplevel node that
1187 * would almost always change with any commit.
1189 struct pbase_tree_cache pcache
;
1192 static struct pbase_tree_cache
*pbase_tree_get(const struct object_id
*oid
)
1194 struct pbase_tree_cache
*ent
, *nent
;
1197 enum object_type type
;
1199 int my_ix
= pbase_tree_cache_ix(oid
);
1200 int available_ix
= -1;
1202 /* pbase-tree-cache acts as a limited hashtable.
1203 * your object will be found at your index or within a few
1204 * slots after that slot if it is cached.
1206 for (neigh
= 0; neigh
< 8; neigh
++) {
1207 ent
= pbase_tree_cache
[my_ix
];
1208 if (ent
&& !oidcmp(&ent
->oid
, oid
)) {
1212 else if (((available_ix
< 0) && (!ent
|| !ent
->ref
)) ||
1213 ((0 <= available_ix
) &&
1214 (!ent
&& pbase_tree_cache
[available_ix
])))
1215 available_ix
= my_ix
;
1218 my_ix
= pbase_tree_cache_ix_incr(my_ix
);
1221 /* Did not find one. Either we got a bogus request or
1222 * we need to read and perhaps cache.
1224 data
= read_object_file(oid
, &type
, &size
);
1227 if (type
!= OBJ_TREE
) {
1232 /* We need to either cache or return a throwaway copy */
1234 if (available_ix
< 0)
1237 ent
= pbase_tree_cache
[available_ix
];
1238 my_ix
= available_ix
;
1242 nent
= xmalloc(sizeof(*nent
));
1243 nent
->temporary
= (available_ix
< 0);
1246 /* evict and reuse */
1247 free(ent
->tree_data
);
1250 oidcpy(&nent
->oid
, oid
);
1251 nent
->tree_data
= data
;
1252 nent
->tree_size
= size
;
1254 if (!nent
->temporary
)
1255 pbase_tree_cache
[my_ix
] = nent
;
1259 static void pbase_tree_put(struct pbase_tree_cache
*cache
)
1261 if (!cache
->temporary
) {
1265 free(cache
->tree_data
);
1269 static int name_cmp_len(const char *name
)
1272 for (i
= 0; name
[i
] && name
[i
] != '\n' && name
[i
] != '/'; i
++)
1277 static void add_pbase_object(struct tree_desc
*tree
,
1280 const char *fullname
)
1282 struct name_entry entry
;
1285 while (tree_entry(tree
,&entry
)) {
1286 if (S_ISGITLINK(entry
.mode
))
1288 cmp
= tree_entry_len(&entry
) != cmplen
? 1 :
1289 memcmp(name
, entry
.path
, cmplen
);
1294 if (name
[cmplen
] != '/') {
1295 add_object_entry(entry
.oid
,
1296 object_type(entry
.mode
),
1300 if (S_ISDIR(entry
.mode
)) {
1301 struct tree_desc sub
;
1302 struct pbase_tree_cache
*tree
;
1303 const char *down
= name
+cmplen
+1;
1304 int downlen
= name_cmp_len(down
);
1306 tree
= pbase_tree_get(entry
.oid
);
1309 init_tree_desc(&sub
, tree
->tree_data
, tree
->tree_size
);
1311 add_pbase_object(&sub
, down
, downlen
, fullname
);
1312 pbase_tree_put(tree
);
1317 static unsigned *done_pbase_paths
;
1318 static int done_pbase_paths_num
;
1319 static int done_pbase_paths_alloc
;
1320 static int done_pbase_path_pos(unsigned hash
)
1323 int hi
= done_pbase_paths_num
;
1325 int mi
= lo
+ (hi
- lo
) / 2;
1326 if (done_pbase_paths
[mi
] == hash
)
1328 if (done_pbase_paths
[mi
] < hash
)
1336 static int check_pbase_path(unsigned hash
)
1338 int pos
= done_pbase_path_pos(hash
);
1342 ALLOC_GROW(done_pbase_paths
,
1343 done_pbase_paths_num
+ 1,
1344 done_pbase_paths_alloc
);
1345 done_pbase_paths_num
++;
1346 if (pos
< done_pbase_paths_num
)
1347 MOVE_ARRAY(done_pbase_paths
+ pos
+ 1, done_pbase_paths
+ pos
,
1348 done_pbase_paths_num
- pos
- 1);
1349 done_pbase_paths
[pos
] = hash
;
1353 static void add_preferred_base_object(const char *name
)
1355 struct pbase_tree
*it
;
1357 unsigned hash
= pack_name_hash(name
);
1359 if (!num_preferred_base
|| check_pbase_path(hash
))
1362 cmplen
= name_cmp_len(name
);
1363 for (it
= pbase_tree
; it
; it
= it
->next
) {
1365 add_object_entry(&it
->pcache
.oid
, OBJ_TREE
, NULL
, 1);
1368 struct tree_desc tree
;
1369 init_tree_desc(&tree
, it
->pcache
.tree_data
, it
->pcache
.tree_size
);
1370 add_pbase_object(&tree
, name
, cmplen
, name
);
1375 static void add_preferred_base(struct object_id
*oid
)
1377 struct pbase_tree
*it
;
1380 struct object_id tree_oid
;
1382 if (window
<= num_preferred_base
++)
1385 data
= read_object_with_reference(oid
, tree_type
, &size
, &tree_oid
);
1389 for (it
= pbase_tree
; it
; it
= it
->next
) {
1390 if (!oidcmp(&it
->pcache
.oid
, &tree_oid
)) {
1396 it
= xcalloc(1, sizeof(*it
));
1397 it
->next
= pbase_tree
;
1400 oidcpy(&it
->pcache
.oid
, &tree_oid
);
1401 it
->pcache
.tree_data
= data
;
1402 it
->pcache
.tree_size
= size
;
1405 static void cleanup_preferred_base(void)
1407 struct pbase_tree
*it
;
1413 struct pbase_tree
*tmp
= it
;
1415 free(tmp
->pcache
.tree_data
);
1419 for (i
= 0; i
< ARRAY_SIZE(pbase_tree_cache
); i
++) {
1420 if (!pbase_tree_cache
[i
])
1422 free(pbase_tree_cache
[i
]->tree_data
);
1423 FREE_AND_NULL(pbase_tree_cache
[i
]);
1426 FREE_AND_NULL(done_pbase_paths
);
1427 done_pbase_paths_num
= done_pbase_paths_alloc
= 0;
1430 static void check_object(struct object_entry
*entry
)
1432 unsigned long canonical_size
;
1434 if (IN_PACK(entry
)) {
1435 struct packed_git
*p
= IN_PACK(entry
);
1436 struct pack_window
*w_curs
= NULL
;
1437 const unsigned char *base_ref
= NULL
;
1438 struct object_entry
*base_entry
;
1439 unsigned long used
, used_0
;
1440 unsigned long avail
;
1442 unsigned char *buf
, c
;
1443 enum object_type type
;
1444 unsigned long in_pack_size
;
1446 buf
= use_pack(p
, &w_curs
, entry
->in_pack_offset
, &avail
);
1449 * We want in_pack_type even if we do not reuse delta
1450 * since non-delta representations could still be reused.
1452 used
= unpack_object_header_buffer(buf
, avail
,
1459 BUG("invalid type %d", type
);
1460 entry
->in_pack_type
= type
;
1463 * Determine if this is a delta and if so whether we can
1464 * reuse it or not. Otherwise let's find out as cheaply as
1465 * possible what the actual type and size for this object is.
1467 switch (entry
->in_pack_type
) {
1469 /* Not a delta hence we've already got all we need. */
1470 oe_set_type(entry
, entry
->in_pack_type
);
1471 SET_SIZE(entry
, in_pack_size
);
1472 entry
->in_pack_header_size
= used
;
1473 if (oe_type(entry
) < OBJ_COMMIT
|| oe_type(entry
) > OBJ_BLOB
)
1475 unuse_pack(&w_curs
);
1478 if (reuse_delta
&& !entry
->preferred_base
)
1479 base_ref
= use_pack(p
, &w_curs
,
1480 entry
->in_pack_offset
+ used
, NULL
);
1481 entry
->in_pack_header_size
= used
+ the_hash_algo
->rawsz
;
1484 buf
= use_pack(p
, &w_curs
,
1485 entry
->in_pack_offset
+ used
, NULL
);
1491 if (!ofs
|| MSB(ofs
, 7)) {
1492 error(_("delta base offset overflow in pack for %s"),
1493 oid_to_hex(&entry
->idx
.oid
));
1497 ofs
= (ofs
<< 7) + (c
& 127);
1499 ofs
= entry
->in_pack_offset
- ofs
;
1500 if (ofs
<= 0 || ofs
>= entry
->in_pack_offset
) {
1501 error(_("delta base offset out of bound for %s"),
1502 oid_to_hex(&entry
->idx
.oid
));
1505 if (reuse_delta
&& !entry
->preferred_base
) {
1506 struct revindex_entry
*revidx
;
1507 revidx
= find_pack_revindex(p
, ofs
);
1510 base_ref
= nth_packed_object_sha1(p
, revidx
->nr
);
1512 entry
->in_pack_header_size
= used
+ used_0
;
1517 (base_entry
= packlist_find(&to_pack
, base_ref
, NULL
)) ||
1519 bitmap_has_sha1_in_uninteresting(bitmap_git
, base_ref
)))) {
1521 * If base_ref was set above that means we wish to
1522 * reuse delta data, and either we found that object in
1523 * the list of objects we want to pack, or it's one we
1524 * know the receiver has.
1526 * Depth value does not matter - find_deltas() will
1527 * never consider reused delta as the base object to
1528 * deltify other objects against, in order to avoid
1531 oe_set_type(entry
, entry
->in_pack_type
);
1532 SET_SIZE(entry
, in_pack_size
); /* delta size */
1533 SET_DELTA_SIZE(entry
, in_pack_size
);
1536 SET_DELTA(entry
, base_entry
);
1537 entry
->delta_sibling_idx
= base_entry
->delta_child_idx
;
1538 SET_DELTA_CHILD(base_entry
, entry
);
1540 SET_DELTA_EXT(entry
, base_ref
);
1543 unuse_pack(&w_curs
);
1547 if (oe_type(entry
)) {
1551 * This must be a delta and we already know what the
1552 * final object type is. Let's extract the actual
1553 * object size from the delta header.
1555 delta_pos
= entry
->in_pack_offset
+ entry
->in_pack_header_size
;
1556 canonical_size
= get_size_from_delta(p
, &w_curs
, delta_pos
);
1557 if (canonical_size
== 0)
1559 SET_SIZE(entry
, canonical_size
);
1560 unuse_pack(&w_curs
);
1565 * No choice but to fall back to the recursive delta walk
1566 * with sha1_object_info() to find about the object type
1570 unuse_pack(&w_curs
);
1574 oid_object_info(the_repository
, &entry
->idx
.oid
, &canonical_size
));
1575 if (entry
->type_valid
) {
1576 SET_SIZE(entry
, canonical_size
);
1579 * Bad object type is checked in prepare_pack(). This is
1580 * to permit a missing preferred base object to be ignored
1581 * as a preferred base. Doing so can result in a larger
1582 * pack file, but the transfer will still take place.
1587 static int pack_offset_sort(const void *_a
, const void *_b
)
1589 const struct object_entry
*a
= *(struct object_entry
**)_a
;
1590 const struct object_entry
*b
= *(struct object_entry
**)_b
;
1591 const struct packed_git
*a_in_pack
= IN_PACK(a
);
1592 const struct packed_git
*b_in_pack
= IN_PACK(b
);
1594 /* avoid filesystem trashing with loose objects */
1595 if (!a_in_pack
&& !b_in_pack
)
1596 return oidcmp(&a
->idx
.oid
, &b
->idx
.oid
);
1598 if (a_in_pack
< b_in_pack
)
1600 if (a_in_pack
> b_in_pack
)
1602 return a
->in_pack_offset
< b
->in_pack_offset
? -1 :
1603 (a
->in_pack_offset
> b
->in_pack_offset
);
1607 * Drop an on-disk delta we were planning to reuse. Naively, this would
1608 * just involve blanking out the "delta" field, but we have to deal
1609 * with some extra book-keeping:
1611 * 1. Removing ourselves from the delta_sibling linked list.
1613 * 2. Updating our size/type to the non-delta representation. These were
1614 * either not recorded initially (size) or overwritten with the delta type
1615 * (type) when check_object() decided to reuse the delta.
1617 * 3. Resetting our delta depth, as we are now a base object.
1619 static void drop_reused_delta(struct object_entry
*entry
)
1621 unsigned *idx
= &to_pack
.objects
[entry
->delta_idx
- 1].delta_child_idx
;
1622 struct object_info oi
= OBJECT_INFO_INIT
;
1623 enum object_type type
;
1627 struct object_entry
*oe
= &to_pack
.objects
[*idx
- 1];
1630 *idx
= oe
->delta_sibling_idx
;
1632 idx
= &oe
->delta_sibling_idx
;
1634 SET_DELTA(entry
, NULL
);
1639 if (packed_object_info(the_repository
, IN_PACK(entry
), entry
->in_pack_offset
, &oi
) < 0) {
1641 * We failed to get the info from this pack for some reason;
1642 * fall back to sha1_object_info, which may find another copy.
1643 * And if that fails, the error will be recorded in oe_type(entry)
1644 * and dealt with in prepare_pack().
1647 oid_object_info(the_repository
, &entry
->idx
.oid
, &size
));
1649 oe_set_type(entry
, type
);
1651 SET_SIZE(entry
, size
);
1655 * Follow the chain of deltas from this entry onward, throwing away any links
1656 * that cause us to hit a cycle (as determined by the DFS state flags in
1659 * We also detect too-long reused chains that would violate our --depth
1662 static void break_delta_chains(struct object_entry
*entry
)
1665 * The actual depth of each object we will write is stored as an int,
1666 * as it cannot exceed our int "depth" limit. But before we break
1667 * changes based no that limit, we may potentially go as deep as the
1668 * number of objects, which is elsewhere bounded to a uint32_t.
1670 uint32_t total_depth
;
1671 struct object_entry
*cur
, *next
;
1673 for (cur
= entry
, total_depth
= 0;
1675 cur
= DELTA(cur
), total_depth
++) {
1676 if (cur
->dfs_state
== DFS_DONE
) {
1678 * We've already seen this object and know it isn't
1679 * part of a cycle. We do need to append its depth
1682 total_depth
+= cur
->depth
;
1687 * We break cycles before looping, so an ACTIVE state (or any
1688 * other cruft which made its way into the state variable)
1691 if (cur
->dfs_state
!= DFS_NONE
)
1692 BUG("confusing delta dfs state in first pass: %d",
1696 * Now we know this is the first time we've seen the object. If
1697 * it's not a delta, we're done traversing, but we'll mark it
1698 * done to save time on future traversals.
1701 cur
->dfs_state
= DFS_DONE
;
1706 * Mark ourselves as active and see if the next step causes
1707 * us to cycle to another active object. It's important to do
1708 * this _before_ we loop, because it impacts where we make the
1709 * cut, and thus how our total_depth counter works.
1710 * E.g., We may see a partial loop like:
1712 * A -> B -> C -> D -> B
1714 * Cutting B->C breaks the cycle. But now the depth of A is
1715 * only 1, and our total_depth counter is at 3. The size of the
1716 * error is always one less than the size of the cycle we
1717 * broke. Commits C and D were "lost" from A's chain.
1719 * If we instead cut D->B, then the depth of A is correct at 3.
1720 * We keep all commits in the chain that we examined.
1722 cur
->dfs_state
= DFS_ACTIVE
;
1723 if (DELTA(cur
)->dfs_state
== DFS_ACTIVE
) {
1724 drop_reused_delta(cur
);
1725 cur
->dfs_state
= DFS_DONE
;
1731 * And now that we've gone all the way to the bottom of the chain, we
1732 * need to clear the active flags and set the depth fields as
1733 * appropriate. Unlike the loop above, which can quit when it drops a
1734 * delta, we need to keep going to look for more depth cuts. So we need
1735 * an extra "next" pointer to keep going after we reset cur->delta.
1737 for (cur
= entry
; cur
; cur
= next
) {
1741 * We should have a chain of zero or more ACTIVE states down to
1742 * a final DONE. We can quit after the DONE, because either it
1743 * has no bases, or we've already handled them in a previous
1746 if (cur
->dfs_state
== DFS_DONE
)
1748 else if (cur
->dfs_state
!= DFS_ACTIVE
)
1749 BUG("confusing delta dfs state in second pass: %d",
1753 * If the total_depth is more than depth, then we need to snip
1754 * the chain into two or more smaller chains that don't exceed
1755 * the maximum depth. Most of the resulting chains will contain
1756 * (depth + 1) entries (i.e., depth deltas plus one base), and
1757 * the last chain (i.e., the one containing entry) will contain
1758 * whatever entries are left over, namely
1759 * (total_depth % (depth + 1)) of them.
1761 * Since we are iterating towards decreasing depth, we need to
1762 * decrement total_depth as we go, and we need to write to the
1763 * entry what its final depth will be after all of the
1764 * snipping. Since we're snipping into chains of length (depth
1765 * + 1) entries, the final depth of an entry will be its
1766 * original depth modulo (depth + 1). Any time we encounter an
1767 * entry whose final depth is supposed to be zero, we snip it
1768 * from its delta base, thereby making it so.
1770 cur
->depth
= (total_depth
--) % (depth
+ 1);
1772 drop_reused_delta(cur
);
1774 cur
->dfs_state
= DFS_DONE
;
1778 static void get_object_details(void)
1781 struct object_entry
**sorted_by_offset
;
1784 progress_state
= start_progress(_("Counting objects"),
1785 to_pack
.nr_objects
);
1787 sorted_by_offset
= xcalloc(to_pack
.nr_objects
, sizeof(struct object_entry
*));
1788 for (i
= 0; i
< to_pack
.nr_objects
; i
++)
1789 sorted_by_offset
[i
] = to_pack
.objects
+ i
;
1790 QSORT(sorted_by_offset
, to_pack
.nr_objects
, pack_offset_sort
);
1792 for (i
= 0; i
< to_pack
.nr_objects
; i
++) {
1793 struct object_entry
*entry
= sorted_by_offset
[i
];
1794 check_object(entry
);
1795 if (entry
->type_valid
&&
1796 oe_size_greater_than(&to_pack
, entry
, big_file_threshold
))
1797 entry
->no_try_delta
= 1;
1798 display_progress(progress_state
, i
+ 1);
1800 stop_progress(&progress_state
);
1803 * This must happen in a second pass, since we rely on the delta
1804 * information for the whole list being completed.
1806 for (i
= 0; i
< to_pack
.nr_objects
; i
++)
1807 break_delta_chains(&to_pack
.objects
[i
]);
1809 free(sorted_by_offset
);
1813 * We search for deltas in a list sorted by type, by filename hash, and then
1814 * by size, so that we see progressively smaller and smaller files.
1815 * That's because we prefer deltas to be from the bigger file
1816 * to the smaller -- deletes are potentially cheaper, but perhaps
1817 * more importantly, the bigger file is likely the more recent
1818 * one. The deepest deltas are therefore the oldest objects which are
1819 * less susceptible to be accessed often.
1821 static int type_size_sort(const void *_a
, const void *_b
)
1823 const struct object_entry
*a
= *(struct object_entry
**)_a
;
1824 const struct object_entry
*b
= *(struct object_entry
**)_b
;
1825 enum object_type a_type
= oe_type(a
);
1826 enum object_type b_type
= oe_type(b
);
1827 unsigned long a_size
= SIZE(a
);
1828 unsigned long b_size
= SIZE(b
);
1830 if (a_type
> b_type
)
1832 if (a_type
< b_type
)
1834 if (a
->hash
> b
->hash
)
1836 if (a
->hash
< b
->hash
)
1838 if (a
->preferred_base
> b
->preferred_base
)
1840 if (a
->preferred_base
< b
->preferred_base
)
1842 if (a_size
> b_size
)
1844 if (a_size
< b_size
)
1846 return a
< b
? -1 : (a
> b
); /* newest first */
1850 struct object_entry
*entry
;
1852 struct delta_index
*index
;
1856 static int delta_cacheable(unsigned long src_size
, unsigned long trg_size
,
1857 unsigned long delta_size
)
1859 if (max_delta_cache_size
&& delta_cache_size
+ delta_size
> max_delta_cache_size
)
1862 if (delta_size
< cache_max_small_delta_size
)
1865 /* cache delta, if objects are large enough compared to delta size */
1866 if ((src_size
>> 20) + (trg_size
>> 21) > (delta_size
>> 10))
1874 /* Protect access to object database */
1875 static pthread_mutex_t read_mutex
;
1876 #define read_lock() pthread_mutex_lock(&read_mutex)
1877 #define read_unlock() pthread_mutex_unlock(&read_mutex)
1879 /* Protect delta_cache_size */
1880 static pthread_mutex_t cache_mutex
;
1881 #define cache_lock() pthread_mutex_lock(&cache_mutex)
1882 #define cache_unlock() pthread_mutex_unlock(&cache_mutex)
1885 * Protect object list partitioning (e.g. struct thread_param) and
1888 static pthread_mutex_t progress_mutex
;
1889 #define progress_lock() pthread_mutex_lock(&progress_mutex)
1890 #define progress_unlock() pthread_mutex_unlock(&progress_mutex)
1893 * Access to struct object_entry is unprotected since each thread owns
1894 * a portion of the main object list. Just don't access object entries
1895 * ahead in the list because they can be stolen and would need
1896 * progress_mutex for protection.
1900 #define read_lock() (void)0
1901 #define read_unlock() (void)0
1902 #define cache_lock() (void)0
1903 #define cache_unlock() (void)0
1904 #define progress_lock() (void)0
1905 #define progress_unlock() (void)0
1910 * Return the size of the object without doing any delta
1911 * reconstruction (so non-deltas are true object sizes, but deltas
1912 * return the size of the delta data).
1914 unsigned long oe_get_size_slow(struct packing_data
*pack
,
1915 const struct object_entry
*e
)
1917 struct packed_git
*p
;
1918 struct pack_window
*w_curs
;
1920 enum object_type type
;
1921 unsigned long used
, avail
, size
;
1923 if (e
->type_
!= OBJ_OFS_DELTA
&& e
->type_
!= OBJ_REF_DELTA
) {
1925 if (oid_object_info(the_repository
, &e
->idx
.oid
, &size
) < 0)
1926 die(_("unable to get size of %s"),
1927 oid_to_hex(&e
->idx
.oid
));
1932 p
= oe_in_pack(pack
, e
);
1934 BUG("when e->type is a delta, it must belong to a pack");
1938 buf
= use_pack(p
, &w_curs
, e
->in_pack_offset
, &avail
);
1939 used
= unpack_object_header_buffer(buf
, avail
, &type
, &size
);
1941 die(_("unable to parse object header of %s"),
1942 oid_to_hex(&e
->idx
.oid
));
1944 unuse_pack(&w_curs
);
1949 static int try_delta(struct unpacked
*trg
, struct unpacked
*src
,
1950 unsigned max_depth
, unsigned long *mem_usage
)
1952 struct object_entry
*trg_entry
= trg
->entry
;
1953 struct object_entry
*src_entry
= src
->entry
;
1954 unsigned long trg_size
, src_size
, delta_size
, sizediff
, max_size
, sz
;
1956 enum object_type type
;
1959 /* Don't bother doing diffs between different types */
1960 if (oe_type(trg_entry
) != oe_type(src_entry
))
1964 * We do not bother to try a delta that we discarded on an
1965 * earlier try, but only when reusing delta data. Note that
1966 * src_entry that is marked as the preferred_base should always
1967 * be considered, as even if we produce a suboptimal delta against
1968 * it, we will still save the transfer cost, as we already know
1969 * the other side has it and we won't send src_entry at all.
1971 if (reuse_delta
&& IN_PACK(trg_entry
) &&
1972 IN_PACK(trg_entry
) == IN_PACK(src_entry
) &&
1973 !src_entry
->preferred_base
&&
1974 trg_entry
->in_pack_type
!= OBJ_REF_DELTA
&&
1975 trg_entry
->in_pack_type
!= OBJ_OFS_DELTA
)
1978 /* Let's not bust the allowed depth. */
1979 if (src
->depth
>= max_depth
)
1982 /* Now some size filtering heuristics. */
1983 trg_size
= SIZE(trg_entry
);
1984 if (!DELTA(trg_entry
)) {
1985 max_size
= trg_size
/2 - the_hash_algo
->rawsz
;
1988 max_size
= DELTA_SIZE(trg_entry
);
1989 ref_depth
= trg
->depth
;
1991 max_size
= (uint64_t)max_size
* (max_depth
- src
->depth
) /
1992 (max_depth
- ref_depth
+ 1);
1995 src_size
= SIZE(src_entry
);
1996 sizediff
= src_size
< trg_size
? trg_size
- src_size
: 0;
1997 if (sizediff
>= max_size
)
1999 if (trg_size
< src_size
/ 32)
2002 /* Load data if not already done */
2005 trg
->data
= read_object_file(&trg_entry
->idx
.oid
, &type
, &sz
);
2008 die(_("object %s cannot be read"),
2009 oid_to_hex(&trg_entry
->idx
.oid
));
2011 die(_("object %s inconsistent object length (%lu vs %lu)"),
2012 oid_to_hex(&trg_entry
->idx
.oid
), sz
,
2018 src
->data
= read_object_file(&src_entry
->idx
.oid
, &type
, &sz
);
2021 if (src_entry
->preferred_base
) {
2022 static int warned
= 0;
2024 warning(_("object %s cannot be read"),
2025 oid_to_hex(&src_entry
->idx
.oid
));
2027 * Those objects are not included in the
2028 * resulting pack. Be resilient and ignore
2029 * them if they can't be read, in case the
2030 * pack could be created nevertheless.
2034 die(_("object %s cannot be read"),
2035 oid_to_hex(&src_entry
->idx
.oid
));
2038 die(_("object %s inconsistent object length (%lu vs %lu)"),
2039 oid_to_hex(&src_entry
->idx
.oid
), sz
,
2044 src
->index
= create_delta_index(src
->data
, src_size
);
2046 static int warned
= 0;
2048 warning(_("suboptimal pack - out of memory"));
2051 *mem_usage
+= sizeof_delta_index(src
->index
);
2054 delta_buf
= create_delta(src
->index
, trg
->data
, trg_size
, &delta_size
, max_size
);
2057 if (delta_size
>= (1U << OE_DELTA_SIZE_BITS
)) {
2062 if (DELTA(trg_entry
)) {
2063 /* Prefer only shallower same-sized deltas. */
2064 if (delta_size
== DELTA_SIZE(trg_entry
) &&
2065 src
->depth
+ 1 >= trg
->depth
) {
2072 * Handle memory allocation outside of the cache
2073 * accounting lock. Compiler will optimize the strangeness
2074 * away when NO_PTHREADS is defined.
2076 free(trg_entry
->delta_data
);
2078 if (trg_entry
->delta_data
) {
2079 delta_cache_size
-= DELTA_SIZE(trg_entry
);
2080 trg_entry
->delta_data
= NULL
;
2082 if (delta_cacheable(src_size
, trg_size
, delta_size
)) {
2083 delta_cache_size
+= delta_size
;
2085 trg_entry
->delta_data
= xrealloc(delta_buf
, delta_size
);
2091 SET_DELTA(trg_entry
, src_entry
);
2092 SET_DELTA_SIZE(trg_entry
, delta_size
);
2093 trg
->depth
= src
->depth
+ 1;
2098 static unsigned int check_delta_limit(struct object_entry
*me
, unsigned int n
)
2100 struct object_entry
*child
= DELTA_CHILD(me
);
2103 unsigned int c
= check_delta_limit(child
, n
+ 1);
2106 child
= DELTA_SIBLING(child
);
2111 static unsigned long free_unpacked(struct unpacked
*n
)
2113 unsigned long freed_mem
= sizeof_delta_index(n
->index
);
2114 free_delta_index(n
->index
);
2117 freed_mem
+= SIZE(n
->entry
);
2118 FREE_AND_NULL(n
->data
);
2125 static void find_deltas(struct object_entry
**list
, unsigned *list_size
,
2126 int window
, int depth
, unsigned *processed
)
2128 uint32_t i
, idx
= 0, count
= 0;
2129 struct unpacked
*array
;
2130 unsigned long mem_usage
= 0;
2132 array
= xcalloc(window
, sizeof(struct unpacked
));
2135 struct object_entry
*entry
;
2136 struct unpacked
*n
= array
+ idx
;
2137 int j
, max_depth
, best_base
= -1;
2146 if (!entry
->preferred_base
) {
2148 display_progress(progress_state
, *processed
);
2152 mem_usage
-= free_unpacked(n
);
2155 while (window_memory_limit
&&
2156 mem_usage
> window_memory_limit
&&
2158 uint32_t tail
= (idx
+ window
- count
) % window
;
2159 mem_usage
-= free_unpacked(array
+ tail
);
2163 /* We do not compute delta to *create* objects we are not
2166 if (entry
->preferred_base
)
2170 * If the current object is at pack edge, take the depth the
2171 * objects that depend on the current object into account
2172 * otherwise they would become too deep.
2175 if (DELTA_CHILD(entry
)) {
2176 max_depth
-= check_delta_limit(entry
, 0);
2184 uint32_t other_idx
= idx
+ j
;
2186 if (other_idx
>= window
)
2187 other_idx
-= window
;
2188 m
= array
+ other_idx
;
2191 ret
= try_delta(n
, m
, max_depth
, &mem_usage
);
2195 best_base
= other_idx
;
2199 * If we decided to cache the delta data, then it is best
2200 * to compress it right away. First because we have to do
2201 * it anyway, and doing it here while we're threaded will
2202 * save a lot of time in the non threaded write phase,
2203 * as well as allow for caching more deltas within
2204 * the same cache size limit.
2206 * But only if not writing to stdout, since in that case
2207 * the network is most likely throttling writes anyway,
2208 * and therefore it is best to go to the write phase ASAP
2209 * instead, as we can afford spending more time compressing
2210 * between writes at that moment.
2212 if (entry
->delta_data
&& !pack_to_stdout
) {
2215 size
= do_compress(&entry
->delta_data
, DELTA_SIZE(entry
));
2216 if (size
< (1U << OE_Z_DELTA_BITS
)) {
2217 entry
->z_delta_size
= size
;
2219 delta_cache_size
-= DELTA_SIZE(entry
);
2220 delta_cache_size
+= entry
->z_delta_size
;
2223 FREE_AND_NULL(entry
->delta_data
);
2224 entry
->z_delta_size
= 0;
2228 /* if we made n a delta, and if n is already at max
2229 * depth, leaving it in the window is pointless. we
2230 * should evict it first.
2232 if (DELTA(entry
) && max_depth
<= n
->depth
)
2236 * Move the best delta base up in the window, after the
2237 * currently deltified object, to keep it longer. It will
2238 * be the first base object to be attempted next.
2241 struct unpacked swap
= array
[best_base
];
2242 int dist
= (window
+ idx
- best_base
) % window
;
2243 int dst
= best_base
;
2245 int src
= (dst
+ 1) % window
;
2246 array
[dst
] = array
[src
];
2254 if (count
+ 1 < window
)
2260 for (i
= 0; i
< window
; ++i
) {
2261 free_delta_index(array
[i
].index
);
2262 free(array
[i
].data
);
2269 static void try_to_free_from_threads(size_t size
)
2272 release_pack_memory(size
);
2276 static try_to_free_t old_try_to_free_routine
;
2279 * The main object list is split into smaller lists, each is handed to
2282 * The main thread waits on the condition that (at least) one of the workers
2283 * has stopped working (which is indicated in the .working member of
2284 * struct thread_params).
2286 * When a work thread has completed its work, it sets .working to 0 and
2287 * signals the main thread and waits on the condition that .data_ready
2290 * The main thread steals half of the work from the worker that has
2291 * most work left to hand it to the idle worker.
2294 struct thread_params
{
2296 struct object_entry
**list
;
2303 pthread_mutex_t mutex
;
2304 pthread_cond_t cond
;
2305 unsigned *processed
;
2308 static pthread_cond_t progress_cond
;
2311 * Mutex and conditional variable can't be statically-initialized on Windows.
2313 static void init_threaded_search(void)
2315 init_recursive_mutex(&read_mutex
);
2316 pthread_mutex_init(&cache_mutex
, NULL
);
2317 pthread_mutex_init(&progress_mutex
, NULL
);
2318 pthread_cond_init(&progress_cond
, NULL
);
2319 old_try_to_free_routine
= set_try_to_free_routine(try_to_free_from_threads
);
2322 static void cleanup_threaded_search(void)
2324 set_try_to_free_routine(old_try_to_free_routine
);
2325 pthread_cond_destroy(&progress_cond
);
2326 pthread_mutex_destroy(&read_mutex
);
2327 pthread_mutex_destroy(&cache_mutex
);
2328 pthread_mutex_destroy(&progress_mutex
);
2331 static void *threaded_find_deltas(void *arg
)
2333 struct thread_params
*me
= arg
;
2336 while (me
->remaining
) {
2339 find_deltas(me
->list
, &me
->remaining
,
2340 me
->window
, me
->depth
, me
->processed
);
2344 pthread_cond_signal(&progress_cond
);
2348 * We must not set ->data_ready before we wait on the
2349 * condition because the main thread may have set it to 1
2350 * before we get here. In order to be sure that new
2351 * work is available if we see 1 in ->data_ready, it
2352 * was initialized to 0 before this thread was spawned
2353 * and we reset it to 0 right away.
2355 pthread_mutex_lock(&me
->mutex
);
2356 while (!me
->data_ready
)
2357 pthread_cond_wait(&me
->cond
, &me
->mutex
);
2359 pthread_mutex_unlock(&me
->mutex
);
2364 /* leave ->working 1 so that this doesn't get more work assigned */
2368 static void ll_find_deltas(struct object_entry
**list
, unsigned list_size
,
2369 int window
, int depth
, unsigned *processed
)
2371 struct thread_params
*p
;
2372 int i
, ret
, active_threads
= 0;
2374 init_threaded_search();
2376 if (delta_search_threads
<= 1) {
2377 find_deltas(list
, &list_size
, window
, depth
, processed
);
2378 cleanup_threaded_search();
2381 if (progress
> pack_to_stdout
)
2382 fprintf_ln(stderr
, _("Delta compression using up to %d threads"),
2383 delta_search_threads
);
2384 p
= xcalloc(delta_search_threads
, sizeof(*p
));
2386 /* Partition the work amongst work threads. */
2387 for (i
= 0; i
< delta_search_threads
; i
++) {
2388 unsigned sub_size
= list_size
/ (delta_search_threads
- i
);
2390 /* don't use too small segments or no deltas will be found */
2391 if (sub_size
< 2*window
&& i
+1 < delta_search_threads
)
2394 p
[i
].window
= window
;
2396 p
[i
].processed
= processed
;
2398 p
[i
].data_ready
= 0;
2400 /* try to split chunks on "path" boundaries */
2401 while (sub_size
&& sub_size
< list_size
&&
2402 list
[sub_size
]->hash
&&
2403 list
[sub_size
]->hash
== list
[sub_size
-1]->hash
)
2407 p
[i
].list_size
= sub_size
;
2408 p
[i
].remaining
= sub_size
;
2411 list_size
-= sub_size
;
2414 /* Start work threads. */
2415 for (i
= 0; i
< delta_search_threads
; i
++) {
2416 if (!p
[i
].list_size
)
2418 pthread_mutex_init(&p
[i
].mutex
, NULL
);
2419 pthread_cond_init(&p
[i
].cond
, NULL
);
2420 ret
= pthread_create(&p
[i
].thread
, NULL
,
2421 threaded_find_deltas
, &p
[i
]);
2423 die(_("unable to create thread: %s"), strerror(ret
));
2428 * Now let's wait for work completion. Each time a thread is done
2429 * with its work, we steal half of the remaining work from the
2430 * thread with the largest number of unprocessed objects and give
2431 * it to that newly idle thread. This ensure good load balancing
2432 * until the remaining object list segments are simply too short
2433 * to be worth splitting anymore.
2435 while (active_threads
) {
2436 struct thread_params
*target
= NULL
;
2437 struct thread_params
*victim
= NULL
;
2438 unsigned sub_size
= 0;
2442 for (i
= 0; !target
&& i
< delta_search_threads
; i
++)
2447 pthread_cond_wait(&progress_cond
, &progress_mutex
);
2450 for (i
= 0; i
< delta_search_threads
; i
++)
2451 if (p
[i
].remaining
> 2*window
&&
2452 (!victim
|| victim
->remaining
< p
[i
].remaining
))
2455 sub_size
= victim
->remaining
/ 2;
2456 list
= victim
->list
+ victim
->list_size
- sub_size
;
2457 while (sub_size
&& list
[0]->hash
&&
2458 list
[0]->hash
== list
[-1]->hash
) {
2464 * It is possible for some "paths" to have
2465 * so many objects that no hash boundary
2466 * might be found. Let's just steal the
2467 * exact half in that case.
2469 sub_size
= victim
->remaining
/ 2;
2472 target
->list
= list
;
2473 victim
->list_size
-= sub_size
;
2474 victim
->remaining
-= sub_size
;
2476 target
->list_size
= sub_size
;
2477 target
->remaining
= sub_size
;
2478 target
->working
= 1;
2481 pthread_mutex_lock(&target
->mutex
);
2482 target
->data_ready
= 1;
2483 pthread_cond_signal(&target
->cond
);
2484 pthread_mutex_unlock(&target
->mutex
);
2487 pthread_join(target
->thread
, NULL
);
2488 pthread_cond_destroy(&target
->cond
);
2489 pthread_mutex_destroy(&target
->mutex
);
2493 cleanup_threaded_search();
2498 #define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)
2501 static void add_tag_chain(const struct object_id
*oid
)
2506 * We catch duplicates already in add_object_entry(), but we'd
2507 * prefer to do this extra check to avoid having to parse the
2508 * tag at all if we already know that it's being packed (e.g., if
2509 * it was included via bitmaps, we would not have parsed it
2512 if (packlist_find(&to_pack
, oid
->hash
, NULL
))
2515 tag
= lookup_tag(the_repository
, oid
);
2517 if (!tag
|| parse_tag(tag
) || !tag
->tagged
)
2518 die(_("unable to pack objects reachable from tag %s"),
2521 add_object_entry(&tag
->object
.oid
, OBJ_TAG
, NULL
, 0);
2523 if (tag
->tagged
->type
!= OBJ_TAG
)
2526 tag
= (struct tag
*)tag
->tagged
;
2530 static int add_ref_tag(const char *path
, const struct object_id
*oid
, int flag
, void *cb_data
)
2532 struct object_id peeled
;
2534 if (starts_with(path
, "refs/tags/") && /* is a tag? */
2535 !peel_ref(path
, &peeled
) && /* peelable? */
2536 packlist_find(&to_pack
, peeled
.hash
, NULL
)) /* object packed? */
2541 static void prepare_pack(int window
, int depth
)
2543 struct object_entry
**delta_list
;
2544 uint32_t i
, nr_deltas
;
2547 get_object_details();
2550 * If we're locally repacking then we need to be doubly careful
2551 * from now on in order to make sure no stealth corruption gets
2552 * propagated to the new pack. Clients receiving streamed packs
2553 * should validate everything they get anyway so no need to incur
2554 * the additional cost here in that case.
2556 if (!pack_to_stdout
)
2557 do_check_packed_object_crc
= 1;
2559 if (!to_pack
.nr_objects
|| !window
|| !depth
)
2562 ALLOC_ARRAY(delta_list
, to_pack
.nr_objects
);
2565 for (i
= 0; i
< to_pack
.nr_objects
; i
++) {
2566 struct object_entry
*entry
= to_pack
.objects
+ i
;
2569 /* This happens if we decided to reuse existing
2570 * delta from a pack. "reuse_delta &&" is implied.
2574 if (!entry
->type_valid
||
2575 oe_size_less_than(&to_pack
, entry
, 50))
2578 if (entry
->no_try_delta
)
2581 if (!entry
->preferred_base
) {
2583 if (oe_type(entry
) < 0)
2584 die(_("unable to get type of object %s"),
2585 oid_to_hex(&entry
->idx
.oid
));
2587 if (oe_type(entry
) < 0) {
2589 * This object is not found, but we
2590 * don't have to include it anyway.
2596 delta_list
[n
++] = entry
;
2599 if (nr_deltas
&& n
> 1) {
2600 unsigned nr_done
= 0;
2602 progress_state
= start_progress(_("Compressing objects"),
2604 QSORT(delta_list
, n
, type_size_sort
);
2605 ll_find_deltas(delta_list
, n
, window
+1, depth
, &nr_done
);
2606 stop_progress(&progress_state
);
2607 if (nr_done
!= nr_deltas
)
2608 die(_("inconsistency with delta count"));
2613 static int git_pack_config(const char *k
, const char *v
, void *cb
)
2615 if (!strcmp(k
, "pack.window")) {
2616 window
= git_config_int(k
, v
);
2619 if (!strcmp(k
, "pack.windowmemory")) {
2620 window_memory_limit
= git_config_ulong(k
, v
);
2623 if (!strcmp(k
, "pack.depth")) {
2624 depth
= git_config_int(k
, v
);
2627 if (!strcmp(k
, "pack.deltacachesize")) {
2628 max_delta_cache_size
= git_config_int(k
, v
);
2631 if (!strcmp(k
, "pack.deltacachelimit")) {
2632 cache_max_small_delta_size
= git_config_int(k
, v
);
2635 if (!strcmp(k
, "pack.writebitmaphashcache")) {
2636 if (git_config_bool(k
, v
))
2637 write_bitmap_options
|= BITMAP_OPT_HASH_CACHE
;
2639 write_bitmap_options
&= ~BITMAP_OPT_HASH_CACHE
;
2641 if (!strcmp(k
, "pack.usebitmaps")) {
2642 use_bitmap_index_default
= git_config_bool(k
, v
);
2645 if (!strcmp(k
, "pack.threads")) {
2646 delta_search_threads
= git_config_int(k
, v
);
2647 if (delta_search_threads
< 0)
2648 die(_("invalid number of threads specified (%d)"),
2649 delta_search_threads
);
2651 if (delta_search_threads
!= 1) {
2652 warning(_("no threads support, ignoring %s"), k
);
2653 delta_search_threads
= 0;
2658 if (!strcmp(k
, "pack.indexversion")) {
2659 pack_idx_opts
.version
= git_config_int(k
, v
);
2660 if (pack_idx_opts
.version
> 2)
2661 die(_("bad pack.indexversion=%"PRIu32
),
2662 pack_idx_opts
.version
);
2665 return git_default_config(k
, v
, cb
);
2668 static void read_object_list_from_stdin(void)
2670 char line
[GIT_MAX_HEXSZ
+ 1 + PATH_MAX
+ 2];
2671 struct object_id oid
;
2675 if (!fgets(line
, sizeof(line
), stdin
)) {
2679 die("BUG: fgets returned NULL, not EOF, not error!");
2685 if (line
[0] == '-') {
2686 if (get_oid_hex(line
+1, &oid
))
2687 die(_("expected edge object ID, got garbage:\n %s"),
2689 add_preferred_base(&oid
);
2692 if (parse_oid_hex(line
, &oid
, &p
))
2693 die(_("expected object ID, got garbage:\n %s"), line
);
2695 add_preferred_base_object(p
+ 1);
2696 add_object_entry(&oid
, OBJ_NONE
, p
+ 1, 0);
2700 /* Remember to update object flag allocation in object.h */
2701 #define OBJECT_ADDED (1u<<20)
2703 static void show_commit(struct commit
*commit
, void *data
)
2705 add_object_entry(&commit
->object
.oid
, OBJ_COMMIT
, NULL
, 0);
2706 commit
->object
.flags
|= OBJECT_ADDED
;
2708 if (write_bitmap_index
)
2709 index_commit_for_bitmap(commit
);
2712 static void show_object(struct object
*obj
, const char *name
, void *data
)
2714 add_preferred_base_object(name
);
2715 add_object_entry(&obj
->oid
, obj
->type
, name
, 0);
2716 obj
->flags
|= OBJECT_ADDED
;
2719 static void show_object__ma_allow_any(struct object
*obj
, const char *name
, void *data
)
2721 assert(arg_missing_action
== MA_ALLOW_ANY
);
2724 * Quietly ignore ALL missing objects. This avoids problems with
2725 * staging them now and getting an odd error later.
2727 if (!has_object_file(&obj
->oid
))
2730 show_object(obj
, name
, data
);
2733 static void show_object__ma_allow_promisor(struct object
*obj
, const char *name
, void *data
)
2735 assert(arg_missing_action
== MA_ALLOW_PROMISOR
);
2738 * Quietly ignore EXPECTED missing objects. This avoids problems with
2739 * staging them now and getting an odd error later.
2741 if (!has_object_file(&obj
->oid
) && is_promisor_object(&obj
->oid
))
2744 show_object(obj
, name
, data
);
2747 static int option_parse_missing_action(const struct option
*opt
,
2748 const char *arg
, int unset
)
2753 if (!strcmp(arg
, "error")) {
2754 arg_missing_action
= MA_ERROR
;
2755 fn_show_object
= show_object
;
2759 if (!strcmp(arg
, "allow-any")) {
2760 arg_missing_action
= MA_ALLOW_ANY
;
2761 fetch_if_missing
= 0;
2762 fn_show_object
= show_object__ma_allow_any
;
2766 if (!strcmp(arg
, "allow-promisor")) {
2767 arg_missing_action
= MA_ALLOW_PROMISOR
;
2768 fetch_if_missing
= 0;
2769 fn_show_object
= show_object__ma_allow_promisor
;
2773 die(_("invalid value for --missing"));
2777 static void show_edge(struct commit
*commit
)
2779 add_preferred_base(&commit
->object
.oid
);
2782 struct in_pack_object
{
2784 struct object
*object
;
2790 struct in_pack_object
*array
;
2793 static void mark_in_pack_object(struct object
*object
, struct packed_git
*p
, struct in_pack
*in_pack
)
2795 in_pack
->array
[in_pack
->nr
].offset
= find_pack_entry_one(object
->oid
.hash
, p
);
2796 in_pack
->array
[in_pack
->nr
].object
= object
;
2801 * Compare the objects in the offset order, in order to emulate the
2802 * "git rev-list --objects" output that produced the pack originally.
2804 static int ofscmp(const void *a_
, const void *b_
)
2806 struct in_pack_object
*a
= (struct in_pack_object
*)a_
;
2807 struct in_pack_object
*b
= (struct in_pack_object
*)b_
;
2809 if (a
->offset
< b
->offset
)
2811 else if (a
->offset
> b
->offset
)
2814 return oidcmp(&a
->object
->oid
, &b
->object
->oid
);
2817 static void add_objects_in_unpacked_packs(struct rev_info
*revs
)
2819 struct packed_git
*p
;
2820 struct in_pack in_pack
;
2823 memset(&in_pack
, 0, sizeof(in_pack
));
2825 for (p
= get_packed_git(the_repository
); p
; p
= p
->next
) {
2826 struct object_id oid
;
2829 if (!p
->pack_local
|| p
->pack_keep
|| p
->pack_keep_in_core
)
2831 if (open_pack_index(p
))
2832 die(_("cannot open pack index"));
2834 ALLOC_GROW(in_pack
.array
,
2835 in_pack
.nr
+ p
->num_objects
,
2838 for (i
= 0; i
< p
->num_objects
; i
++) {
2839 nth_packed_object_oid(&oid
, p
, i
);
2840 o
= lookup_unknown_object(oid
.hash
);
2841 if (!(o
->flags
& OBJECT_ADDED
))
2842 mark_in_pack_object(o
, p
, &in_pack
);
2843 o
->flags
|= OBJECT_ADDED
;
2848 QSORT(in_pack
.array
, in_pack
.nr
, ofscmp
);
2849 for (i
= 0; i
< in_pack
.nr
; i
++) {
2850 struct object
*o
= in_pack
.array
[i
].object
;
2851 add_object_entry(&o
->oid
, o
->type
, "", 0);
2854 free(in_pack
.array
);
2857 static int add_loose_object(const struct object_id
*oid
, const char *path
,
2860 enum object_type type
= oid_object_info(the_repository
, oid
, NULL
);
2863 warning(_("loose object at %s could not be examined"), path
);
2867 add_object_entry(oid
, type
, "", 0);
2872 * We actually don't even have to worry about reachability here.
2873 * add_object_entry will weed out duplicates, so we just add every
2874 * loose object we find.
2876 static void add_unreachable_loose_objects(void)
2878 for_each_loose_file_in_objdir(get_object_directory(),
2883 static int has_sha1_pack_kept_or_nonlocal(const struct object_id
*oid
)
2885 static struct packed_git
*last_found
= (void *)1;
2886 struct packed_git
*p
;
2888 p
= (last_found
!= (void *)1) ? last_found
:
2889 get_packed_git(the_repository
);
2892 if ((!p
->pack_local
|| p
->pack_keep
||
2893 p
->pack_keep_in_core
) &&
2894 find_pack_entry_one(oid
->hash
, p
)) {
2898 if (p
== last_found
)
2899 p
= get_packed_git(the_repository
);
2902 if (p
== last_found
)
2909 * Store a list of sha1s that are should not be discarded
2910 * because they are either written too recently, or are
2911 * reachable from another object that was.
2913 * This is filled by get_object_list.
2915 static struct oid_array recent_objects
;
2917 static int loosened_object_can_be_discarded(const struct object_id
*oid
,
2920 if (!unpack_unreachable_expiration
)
2922 if (mtime
> unpack_unreachable_expiration
)
2924 if (oid_array_lookup(&recent_objects
, oid
) >= 0)
2929 static void loosen_unused_packed_objects(struct rev_info
*revs
)
2931 struct packed_git
*p
;
2933 struct object_id oid
;
2935 for (p
= get_packed_git(the_repository
); p
; p
= p
->next
) {
2936 if (!p
->pack_local
|| p
->pack_keep
|| p
->pack_keep_in_core
)
2939 if (open_pack_index(p
))
2940 die(_("cannot open pack index"));
2942 for (i
= 0; i
< p
->num_objects
; i
++) {
2943 nth_packed_object_oid(&oid
, p
, i
);
2944 if (!packlist_find(&to_pack
, oid
.hash
, NULL
) &&
2945 !has_sha1_pack_kept_or_nonlocal(&oid
) &&
2946 !loosened_object_can_be_discarded(&oid
, p
->mtime
))
2947 if (force_object_loose(&oid
, p
->mtime
))
2948 die(_("unable to force loose object"));
2954 * This tracks any options which pack-reuse code expects to be on, or which a
2955 * reader of the pack might not understand, and which would therefore prevent
2956 * blind reuse of what we have on disk.
2958 static int pack_options_allow_reuse(void)
2960 return pack_to_stdout
&&
2962 !ignore_packed_keep_on_disk
&&
2963 !ignore_packed_keep_in_core
&&
2964 (!local
|| !have_non_local_packs
) &&
2968 static int get_object_list_from_bitmap(struct rev_info
*revs
)
2970 if (!(bitmap_git
= prepare_bitmap_walk(revs
)))
2973 if (pack_options_allow_reuse() &&
2974 !reuse_partial_packfile_from_bitmap(
2977 &reuse_packfile_objects
,
2978 &reuse_packfile_offset
)) {
2979 assert(reuse_packfile_objects
);
2980 nr_result
+= reuse_packfile_objects
;
2981 display_progress(progress_state
, nr_result
);
2984 traverse_bitmap_commit_list(bitmap_git
, &add_object_entry_from_bitmap
);
2988 static void record_recent_object(struct object
*obj
,
2992 oid_array_append(&recent_objects
, &obj
->oid
);
2995 static void record_recent_commit(struct commit
*commit
, void *data
)
2997 oid_array_append(&recent_objects
, &commit
->object
.oid
);
3000 static void get_object_list(int ac
, const char **av
)
3002 struct rev_info revs
;
3006 init_revisions(&revs
, NULL
);
3007 save_commit_buffer
= 0;
3008 setup_revisions(ac
, av
, &revs
, NULL
);
3010 /* make sure shallows are read */
3011 is_repository_shallow(the_repository
);
3013 while (fgets(line
, sizeof(line
), stdin
) != NULL
) {
3014 int len
= strlen(line
);
3015 if (len
&& line
[len
- 1] == '\n')
3020 if (!strcmp(line
, "--not")) {
3021 flags
^= UNINTERESTING
;
3022 write_bitmap_index
= 0;
3025 if (starts_with(line
, "--shallow ")) {
3026 struct object_id oid
;
3027 if (get_oid_hex(line
+ 10, &oid
))
3028 die("not an SHA-1 '%s'", line
+ 10);
3029 register_shallow(the_repository
, &oid
);
3030 use_bitmap_index
= 0;
3033 die(_("not a rev '%s'"), line
);
3035 if (handle_revision_arg(line
, &revs
, flags
, REVARG_CANNOT_BE_FILENAME
))
3036 die(_("bad revision '%s'"), line
);
3039 if (use_bitmap_index
&& !get_object_list_from_bitmap(&revs
))
3042 if (prepare_revision_walk(&revs
))
3043 die(_("revision walk setup failed"));
3044 mark_edges_uninteresting(&revs
, show_edge
);
3046 if (!fn_show_object
)
3047 fn_show_object
= show_object
;
3048 traverse_commit_list_filtered(&filter_options
, &revs
,
3049 show_commit
, fn_show_object
, NULL
,
3052 if (unpack_unreachable_expiration
) {
3053 revs
.ignore_missing_links
= 1;
3054 if (add_unseen_recent_objects_to_traversal(&revs
,
3055 unpack_unreachable_expiration
))
3056 die(_("unable to add recent objects"));
3057 if (prepare_revision_walk(&revs
))
3058 die(_("revision walk setup failed"));
3059 traverse_commit_list(&revs
, record_recent_commit
,
3060 record_recent_object
, NULL
);
3063 if (keep_unreachable
)
3064 add_objects_in_unpacked_packs(&revs
);
3065 if (pack_loose_unreachable
)
3066 add_unreachable_loose_objects();
3067 if (unpack_unreachable
)
3068 loosen_unused_packed_objects(&revs
);
3070 oid_array_clear(&recent_objects
);
3073 static void add_extra_kept_packs(const struct string_list
*names
)
3075 struct packed_git
*p
;
3080 for (p
= get_packed_git(the_repository
); p
; p
= p
->next
) {
3081 const char *name
= basename(p
->pack_name
);
3087 for (i
= 0; i
< names
->nr
; i
++)
3088 if (!fspathcmp(name
, names
->items
[i
].string
))
3091 if (i
< names
->nr
) {
3092 p
->pack_keep_in_core
= 1;
3093 ignore_packed_keep_in_core
= 1;
3099 static int option_parse_index_version(const struct option
*opt
,
3100 const char *arg
, int unset
)
3103 const char *val
= arg
;
3104 pack_idx_opts
.version
= strtoul(val
, &c
, 10);
3105 if (pack_idx_opts
.version
> 2)
3106 die(_("unsupported index version %s"), val
);
3107 if (*c
== ',' && c
[1])
3108 pack_idx_opts
.off32_limit
= strtoul(c
+1, &c
, 0);
3109 if (*c
|| pack_idx_opts
.off32_limit
& 0x80000000)
3110 die(_("bad index version '%s'"), val
);
3114 static int option_parse_unpack_unreachable(const struct option
*opt
,
3115 const char *arg
, int unset
)
3118 unpack_unreachable
= 0;
3119 unpack_unreachable_expiration
= 0;
3122 unpack_unreachable
= 1;
3124 unpack_unreachable_expiration
= approxidate(arg
);
3129 int cmd_pack_objects(int argc
, const char **argv
, const char *prefix
)
3131 int use_internal_rev_list
= 0;
3133 int all_progress_implied
= 0;
3134 struct argv_array rp
= ARGV_ARRAY_INIT
;
3135 int rev_list_unpacked
= 0, rev_list_all
= 0, rev_list_reflog
= 0;
3136 int rev_list_index
= 0;
3137 struct string_list keep_pack_list
= STRING_LIST_INIT_NODUP
;
3138 struct option pack_objects_options
[] = {
3139 OPT_SET_INT('q', "quiet", &progress
,
3140 N_("do not show progress meter"), 0),
3141 OPT_SET_INT(0, "progress", &progress
,
3142 N_("show progress meter"), 1),
3143 OPT_SET_INT(0, "all-progress", &progress
,
3144 N_("show progress meter during object writing phase"), 2),
3145 OPT_BOOL(0, "all-progress-implied",
3146 &all_progress_implied
,
3147 N_("similar to --all-progress when progress meter is shown")),
3148 { OPTION_CALLBACK
, 0, "index-version", NULL
, N_("<version>[,<offset>]"),
3149 N_("write the pack index file in the specified idx format version"),
3150 0, option_parse_index_version
},
3151 OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit
,
3152 N_("maximum size of each output pack file")),
3153 OPT_BOOL(0, "local", &local
,
3154 N_("ignore borrowed objects from alternate object store")),
3155 OPT_BOOL(0, "incremental", &incremental
,
3156 N_("ignore packed objects")),
3157 OPT_INTEGER(0, "window", &window
,
3158 N_("limit pack window by objects")),
3159 OPT_MAGNITUDE(0, "window-memory", &window_memory_limit
,
3160 N_("limit pack window by memory in addition to object limit")),
3161 OPT_INTEGER(0, "depth", &depth
,
3162 N_("maximum length of delta chain allowed in the resulting pack")),
3163 OPT_BOOL(0, "reuse-delta", &reuse_delta
,
3164 N_("reuse existing deltas")),
3165 OPT_BOOL(0, "reuse-object", &reuse_object
,
3166 N_("reuse existing objects")),
3167 OPT_BOOL(0, "delta-base-offset", &allow_ofs_delta
,
3168 N_("use OFS_DELTA objects")),
3169 OPT_INTEGER(0, "threads", &delta_search_threads
,
3170 N_("use threads when searching for best delta matches")),
3171 OPT_BOOL(0, "non-empty", &non_empty
,
3172 N_("do not create an empty pack output")),
3173 OPT_BOOL(0, "revs", &use_internal_rev_list
,
3174 N_("read revision arguments from standard input")),
3175 OPT_SET_INT_F(0, "unpacked", &rev_list_unpacked
,
3176 N_("limit the objects to those that are not yet packed"),
3177 1, PARSE_OPT_NONEG
),
3178 OPT_SET_INT_F(0, "all", &rev_list_all
,
3179 N_("include objects reachable from any reference"),
3180 1, PARSE_OPT_NONEG
),
3181 OPT_SET_INT_F(0, "reflog", &rev_list_reflog
,
3182 N_("include objects referred by reflog entries"),
3183 1, PARSE_OPT_NONEG
),
3184 OPT_SET_INT_F(0, "indexed-objects", &rev_list_index
,
3185 N_("include objects referred to by the index"),
3186 1, PARSE_OPT_NONEG
),
3187 OPT_BOOL(0, "stdout", &pack_to_stdout
,
3188 N_("output pack to stdout")),
3189 OPT_BOOL(0, "include-tag", &include_tag
,
3190 N_("include tag objects that refer to objects to be packed")),
3191 OPT_BOOL(0, "keep-unreachable", &keep_unreachable
,
3192 N_("keep unreachable objects")),
3193 OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable
,
3194 N_("pack loose unreachable objects")),
3195 { OPTION_CALLBACK
, 0, "unpack-unreachable", NULL
, N_("time"),
3196 N_("unpack unreachable objects newer than <time>"),
3197 PARSE_OPT_OPTARG
, option_parse_unpack_unreachable
},
3198 OPT_BOOL(0, "thin", &thin
,
3199 N_("create thin packs")),
3200 OPT_BOOL(0, "shallow", &shallow
,
3201 N_("create packs suitable for shallow fetches")),
3202 OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep_on_disk
,
3203 N_("ignore packs that have companion .keep file")),
3204 OPT_STRING_LIST(0, "keep-pack", &keep_pack_list
, N_("name"),
3205 N_("ignore this pack")),
3206 OPT_INTEGER(0, "compression", &pack_compression_level
,
3207 N_("pack compression level")),
3208 OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents
,
3209 N_("do not hide commits by grafts"), 0),
3210 OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index
,
3211 N_("use a bitmap index if available to speed up counting objects")),
3212 OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index
,
3213 N_("write a bitmap index together with the pack index")),
3214 OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options
),
3215 { OPTION_CALLBACK
, 0, "missing", NULL
, N_("action"),
3216 N_("handling for missing objects"), PARSE_OPT_NONEG
,
3217 option_parse_missing_action
},
3218 OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects
,
3219 N_("do not pack objects in promisor packfiles")),
3223 if (DFS_NUM_STATES
> (1 << OE_DFS_STATE_BITS
))
3224 BUG("too many dfs states, increase OE_DFS_STATE_BITS");
3226 read_replace_refs
= 0;
3228 reset_pack_idx_option(&pack_idx_opts
);
3229 git_config(git_pack_config
, NULL
);
3231 progress
= isatty(2);
3232 argc
= parse_options(argc
, argv
, prefix
, pack_objects_options
,
3236 base_name
= argv
[0];
3239 if (pack_to_stdout
!= !base_name
|| argc
)
3240 usage_with_options(pack_usage
, pack_objects_options
);
3242 if (depth
>= (1 << OE_DEPTH_BITS
)) {
3243 warning(_("delta chain depth %d is too deep, forcing %d"),
3244 depth
, (1 << OE_DEPTH_BITS
) - 1);
3245 depth
= (1 << OE_DEPTH_BITS
) - 1;
3247 if (cache_max_small_delta_size
>= (1U << OE_Z_DELTA_BITS
)) {
3248 warning(_("pack.deltaCacheLimit is too high, forcing %d"),
3249 (1U << OE_Z_DELTA_BITS
) - 1);
3250 cache_max_small_delta_size
= (1U << OE_Z_DELTA_BITS
) - 1;
3253 argv_array_push(&rp
, "pack-objects");
3255 use_internal_rev_list
= 1;
3256 argv_array_push(&rp
, shallow
3257 ? "--objects-edge-aggressive"
3258 : "--objects-edge");
3260 argv_array_push(&rp
, "--objects");
3263 use_internal_rev_list
= 1;
3264 argv_array_push(&rp
, "--all");
3266 if (rev_list_reflog
) {
3267 use_internal_rev_list
= 1;
3268 argv_array_push(&rp
, "--reflog");
3270 if (rev_list_index
) {
3271 use_internal_rev_list
= 1;
3272 argv_array_push(&rp
, "--indexed-objects");
3274 if (rev_list_unpacked
) {
3275 use_internal_rev_list
= 1;
3276 argv_array_push(&rp
, "--unpacked");
3279 if (exclude_promisor_objects
) {
3280 use_internal_rev_list
= 1;
3281 fetch_if_missing
= 0;
3282 argv_array_push(&rp
, "--exclude-promisor-objects");
3284 if (unpack_unreachable
|| keep_unreachable
|| pack_loose_unreachable
)
3285 use_internal_rev_list
= 1;
3289 if (pack_compression_level
== -1)
3290 pack_compression_level
= Z_DEFAULT_COMPRESSION
;
3291 else if (pack_compression_level
< 0 || pack_compression_level
> Z_BEST_COMPRESSION
)
3292 die(_("bad pack compression level %d"), pack_compression_level
);
3294 if (!delta_search_threads
) /* --threads=0 means autodetect */
3295 delta_search_threads
= online_cpus();
3298 if (delta_search_threads
!= 1)
3299 warning(_("no threads support, ignoring --threads"));
3301 if (!pack_to_stdout
&& !pack_size_limit
)
3302 pack_size_limit
= pack_size_limit_cfg
;
3303 if (pack_to_stdout
&& pack_size_limit
)
3304 die(_("--max-pack-size cannot be used to build a pack for transfer"));
3305 if (pack_size_limit
&& pack_size_limit
< 1024*1024) {
3306 warning(_("minimum pack size limit is 1 MiB"));
3307 pack_size_limit
= 1024*1024;
3310 if (!pack_to_stdout
&& thin
)
3311 die(_("--thin cannot be used to build an indexable pack"));
3313 if (keep_unreachable
&& unpack_unreachable
)
3314 die(_("--keep-unreachable and --unpack-unreachable are incompatible"));
3315 if (!rev_list_all
|| !rev_list_reflog
|| !rev_list_index
)
3316 unpack_unreachable_expiration
= 0;
3318 if (filter_options
.choice
) {
3319 if (!pack_to_stdout
)
3320 die(_("cannot use --filter without --stdout"));
3321 use_bitmap_index
= 0;
3325 * "soft" reasons not to use bitmaps - for on-disk repack by default we want
3327 * - to produce good pack (with bitmap index not-yet-packed objects are
3328 * packed in suboptimal order).
3330 * - to use more robust pack-generation codepath (avoiding possible
3331 * bugs in bitmap code and possible bitmap index corruption).
3333 if (!pack_to_stdout
)
3334 use_bitmap_index_default
= 0;
3336 if (use_bitmap_index
< 0)
3337 use_bitmap_index
= use_bitmap_index_default
;
3339 /* "hard" reasons not to use bitmaps; these just won't work at all */
3340 if (!use_internal_rev_list
|| (!pack_to_stdout
&& write_bitmap_index
) || is_repository_shallow(the_repository
))
3341 use_bitmap_index
= 0;
3343 if (pack_to_stdout
|| !rev_list_all
)
3344 write_bitmap_index
= 0;
3346 if (progress
&& all_progress_implied
)
3349 add_extra_kept_packs(&keep_pack_list
);
3350 if (ignore_packed_keep_on_disk
) {
3351 struct packed_git
*p
;
3352 for (p
= get_packed_git(the_repository
); p
; p
= p
->next
)
3353 if (p
->pack_local
&& p
->pack_keep
)
3355 if (!p
) /* no keep-able packs found */
3356 ignore_packed_keep_on_disk
= 0;
3360 * unlike ignore_packed_keep_on_disk above, we do not
3361 * want to unset "local" based on looking at packs, as
3362 * it also covers non-local objects
3364 struct packed_git
*p
;
3365 for (p
= get_packed_git(the_repository
); p
; p
= p
->next
) {
3366 if (!p
->pack_local
) {
3367 have_non_local_packs
= 1;
3373 prepare_packing_data(&to_pack
);
3376 progress_state
= start_progress(_("Enumerating objects"), 0);
3377 if (!use_internal_rev_list
)
3378 read_object_list_from_stdin();
3380 get_object_list(rp
.argc
, rp
.argv
);
3381 argv_array_clear(&rp
);
3383 cleanup_preferred_base();
3384 if (include_tag
&& nr_result
)
3385 for_each_ref(add_ref_tag
, NULL
);
3386 stop_progress(&progress_state
);
3388 if (non_empty
&& !nr_result
)
3391 prepare_pack(window
, depth
);
3395 _("Total %"PRIu32
" (delta %"PRIu32
"),"
3396 " reused %"PRIu32
" (delta %"PRIu32
")"),
3397 written
, written_delta
, reused
, reused_delta
);