?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
commit-graph.c: prevent overflow in `fill_commit_graph_info()`
[git/debian.git] / midx.c
blob067482a98cc42743c37c4fae2aed77110e13ee5a
1 #include "git-compat-util.h"
2 #include "abspath.h"
3 #include "alloc.h"
4 #include "config.h"
5 #include "csum-file.h"
6 #include "dir.h"
7 #include "gettext.h"
8 #include "hex.h"
9 #include "lockfile.h"
10 #include "packfile.h"
11 #include "object-file.h"
12 #include "object-store.h"
13 #include "hash-lookup.h"
14 #include "midx.h"
15 #include "progress.h"
16 #include "trace2.h"
17 #include "run-command.h"
18 #include "repository.h"
19 #include "chunk-format.h"
20 #include "pack.h"
21 #include "pack-bitmap.h"
22 #include "refs.h"
23 #include "revision.h"
24 #include "list-objects.h"
25
26 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
27 #define MIDX_VERSION 1
28 #define MIDX_BYTE_FILE_VERSION 4
29 #define MIDX_BYTE_HASH_VERSION 5
30 #define MIDX_BYTE_NUM_CHUNKS 6
31 #define MIDX_BYTE_NUM_PACKS 8
32 #define MIDX_HEADER_SIZE 12
33 #define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz)
34
35 #define MIDX_CHUNK_ALIGNMENT 4
36 #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
37 #define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
38 #define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
39 #define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
40 #define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
41 #define MIDX_CHUNKID_REVINDEX 0x52494458 /* "RIDX" */
42 #define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
43 #define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
44 #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
45 #define MIDX_LARGE_OFFSET_NEEDED 0x80000000
46
47 #define PACK_EXPIRED UINT_MAX
48
49 const unsigned char *get_midx_checksum(struct multi_pack_index *m)
50 {
51 return m->data + m->data_len - the_hash_algo->rawsz;
52 }
53
54 void get_midx_filename(struct strbuf *out, const char *object_dir)
55 {
56 strbuf_addf(out, "%s/pack/multi-pack-index", object_dir);
57 }
58
59 void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m)
60 {
61 get_midx_filename(out, m->object_dir);
62 strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m)));
63 }
64
65 static int midx_read_oid_fanout(const unsigned char *chunk_start,
66 size_t chunk_size, void *data)
67 {
68 struct multi_pack_index *m = data;
69 m->chunk_oid_fanout = (uint32_t *)chunk_start;
70
71 if (chunk_size != 4 * 256) {
72 error(_("multi-pack-index OID fanout is of the wrong size"));
73 return 1;
74 }
75 return 0;
76 }
77
78 struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local)
79 {
80 struct multi_pack_index *m = NULL;
81 int fd;
82 struct stat st;
83 size_t midx_size;
84 void *midx_map = NULL;
85 uint32_t hash_version;
86 struct strbuf midx_name = STRBUF_INIT;
87 uint32_t i;
88 const char *cur_pack_name;
89 struct chunkfile *cf = NULL;
90
91 get_midx_filename(&midx_name, object_dir);
92
93 fd = git_open(midx_name.buf);
94
95 if (fd < 0)
96 goto cleanup_fail;
97 if (fstat(fd, &st)) {
98 error_errno(_("failed to read %s"), midx_name.buf);
99 goto cleanup_fail;
100 }
101
102 midx_size = xsize_t(st.st_size);
103
104 if (midx_size < MIDX_MIN_SIZE) {
105 error(_("multi-pack-index file %s is too small"), midx_name.buf);
106 goto cleanup_fail;
107 }
108
109 strbuf_release(&midx_name);
110
111 midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
112 close(fd);
113
114 FLEX_ALLOC_STR(m, object_dir, object_dir);
115 m->data = midx_map;
116 m->data_len = midx_size;
117 m->local = local;
118
119 m->signature = get_be32(m->data);
120 if (m->signature != MIDX_SIGNATURE)
121 die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
122 m->signature, MIDX_SIGNATURE);
123
124 m->version = m->data[MIDX_BYTE_FILE_VERSION];
125 if (m->version != MIDX_VERSION)
126 die(_("multi-pack-index version %d not recognized"),
127 m->version);
128
129 hash_version = m->data[MIDX_BYTE_HASH_VERSION];
130 if (hash_version != oid_version(the_hash_algo)) {
131 error(_("multi-pack-index hash version %u does not match version %u"),
132 hash_version, oid_version(the_hash_algo));
133 goto cleanup_fail;
134 }
135 m->hash_len = the_hash_algo->rawsz;
136
137 m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
138
139 m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
140
141 cf = init_chunkfile(NULL);
142
143 if (read_table_of_contents(cf, m->data, midx_size,
144 MIDX_HEADER_SIZE, m->num_chunks))
145 goto cleanup_fail;
146
147 if (pair_chunk(cf, MIDX_CHUNKID_PACKNAMES, &m->chunk_pack_names) == CHUNK_NOT_FOUND)
148 die(_("multi-pack-index missing required pack-name chunk"));
149 if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m) == CHUNK_NOT_FOUND)
150 die(_("multi-pack-index missing required OID fanout chunk"));
151 if (pair_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, &m->chunk_oid_lookup) == CHUNK_NOT_FOUND)
152 die(_("multi-pack-index missing required OID lookup chunk"));
153 if (pair_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, &m->chunk_object_offsets) == CHUNK_NOT_FOUND)
154 die(_("multi-pack-index missing required object offsets chunk"));
155
156 pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets);
157
158 if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1))
159 pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex);
160
161 m->num_objects = ntohl(m->chunk_oid_fanout[255]);
162
163 CALLOC_ARRAY(m->pack_names, m->num_packs);
164 CALLOC_ARRAY(m->packs, m->num_packs);
165
166 cur_pack_name = (const char *)m->chunk_pack_names;
167 for (i = 0; i < m->num_packs; i++) {
168 m->pack_names[i] = cur_pack_name;
169
170 cur_pack_name += strlen(cur_pack_name) + 1;
171
172 if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
173 die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
174 m->pack_names[i - 1],
175 m->pack_names[i]);
176 }
177
178 trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs);
179 trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects);
180
181 free_chunkfile(cf);
182 return m;
183
184 cleanup_fail:
185 free(m);
186 strbuf_release(&midx_name);
187 free_chunkfile(cf);
188 if (midx_map)
189 munmap(midx_map, midx_size);
190 if (0 <= fd)
191 close(fd);
192 return NULL;
193 }
194
195 void close_midx(struct multi_pack_index *m)
196 {
197 uint32_t i;
198
199 if (!m)
200 return;
201
202 close_midx(m->next);
203
204 munmap((unsigned char *)m->data, m->data_len);
205
206 for (i = 0; i < m->num_packs; i++) {
207 if (m->packs[i])
208 m->packs[i]->multi_pack_index = 0;
209 }
210 FREE_AND_NULL(m->packs);
211 FREE_AND_NULL(m->pack_names);
212 free(m);
213 }
214
215 int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id)
216 {
217 struct strbuf pack_name = STRBUF_INIT;
218 struct packed_git *p;
219
220 if (pack_int_id >= m->num_packs)
221 die(_("bad pack-int-id: %u (%u total packs)"),
222 pack_int_id, m->num_packs);
223
224 if (m->packs[pack_int_id])
225 return 0;
226
227 strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir,
228 m->pack_names[pack_int_id]);
229
230 p = add_packed_git(pack_name.buf, pack_name.len, m->local);
231 strbuf_release(&pack_name);
232
233 if (!p)
234 return 1;
235
236 p->multi_pack_index = 1;
237 m->packs[pack_int_id] = p;
238 install_packed_git(r, p);
239 list_add_tail(&p->mru, &r->objects->packed_git_mru);
240
241 return 0;
242 }
243
244 int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result)
245 {
246 return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup,
247 the_hash_algo->rawsz, result);
248 }
249
250 struct object_id *nth_midxed_object_oid(struct object_id *oid,
251 struct multi_pack_index *m,
252 uint32_t n)
253 {
254 if (n >= m->num_objects)
255 return NULL;
256
257 oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n));
258 return oid;
259 }
260
261 off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
262 {
263 const unsigned char *offset_data;
264 uint32_t offset32;
265
266 offset_data = m->chunk_object_offsets + (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH;
267 offset32 = get_be32(offset_data + sizeof(uint32_t));
268
269 if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
270 if (sizeof(off_t) < sizeof(uint64_t))
271 die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
272
273 offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
274 return get_be64(m->chunk_large_offsets +
275 st_mult(sizeof(uint64_t), offset32));
276 }
277
278 return offset32;
279 }
280
281 uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos)
282 {
283 return get_be32(m->chunk_object_offsets +
284 (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH);
285 }
286
287 int fill_midx_entry(struct repository *r,
288 const struct object_id *oid,
289 struct pack_entry *e,
290 struct multi_pack_index *m)
291 {
292 uint32_t pos;
293 uint32_t pack_int_id;
294 struct packed_git *p;
295
296 if (!bsearch_midx(oid, m, &pos))
297 return 0;
298
299 if (pos >= m->num_objects)
300 return 0;
301
302 pack_int_id = nth_midxed_pack_int_id(m, pos);
303
304 if (prepare_midx_pack(r, m, pack_int_id))
305 return 0;
306 p = m->packs[pack_int_id];
307
308 /*
309 * We are about to tell the caller where they can locate the
310 * requested object. We better make sure the packfile is
311 * still here and can be accessed before supplying that
312 * answer, as it may have been deleted since the MIDX was
313 * loaded!
314 */
315 if (!is_pack_valid(p))
316 return 0;
317
318 if (oidset_size(&p->bad_objects) &&
319 oidset_contains(&p->bad_objects, oid))
320 return 0;
321
322 e->offset = nth_midxed_offset(m, pos);
323 e->p = p;
324
325 return 1;
326 }
327
328 /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
329 static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
330 const char *idx_name)
331 {
332 /* Skip past any initial matching prefix. */
333 while (*idx_name && *idx_name == *idx_or_pack_name) {
334 idx_name++;
335 idx_or_pack_name++;
336 }
337
338 /*
339 * If we didn't match completely, we may have matched "pack-1234." and
340 * be left with "idx" and "pack" respectively, which is also OK. We do
341 * not have to check for "idx" and "idx", because that would have been
342 * a complete match (and in that case these strcmps will be false, but
343 * we'll correctly return 0 from the final strcmp() below.
344 *
345 * Technically this matches "fooidx" and "foopack", but we'd never have
346 * such names in the first place.
347 */
348 if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
349 return 0;
350
351 /*
352 * This not only checks for a complete match, but also orders based on
353 * the first non-identical character, which means our ordering will
354 * match a raw strcmp(). That makes it OK to use this to binary search
355 * a naively-sorted list.
356 */
357 return strcmp(idx_or_pack_name, idx_name);
358 }
359
360 int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
361 {
362 uint32_t first = 0, last = m->num_packs;
363
364 while (first < last) {
365 uint32_t mid = first + (last - first) / 2;
366 const char *current;
367 int cmp;
368
369 current = m->pack_names[mid];
370 cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
371 if (!cmp)
372 return 1;
373 if (cmp > 0) {
374 first = mid + 1;
375 continue;
376 }
377 last = mid;
378 }
379
380 return 0;
381 }
382
383 int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local)
384 {
385 struct multi_pack_index *m;
386 struct multi_pack_index *m_search;
387
388 prepare_repo_settings(r);
389 if (!r->settings.core_multi_pack_index)
390 return 0;
391
392 for (m_search = r->objects->multi_pack_index; m_search; m_search = m_search->next)
393 if (!strcmp(object_dir, m_search->object_dir))
394 return 1;
395
396 m = load_multi_pack_index(object_dir, local);
397
398 if (m) {
399 struct multi_pack_index *mp = r->objects->multi_pack_index;
400 if (mp) {
401 m->next = mp->next;
402 mp->next = m;
403 } else
404 r->objects->multi_pack_index = m;
405 return 1;
406 }
407
408 return 0;
409 }
410
411 static size_t write_midx_header(struct hashfile *f,
412 unsigned char num_chunks,
413 uint32_t num_packs)
414 {
415 hashwrite_be32(f, MIDX_SIGNATURE);
416 hashwrite_u8(f, MIDX_VERSION);
417 hashwrite_u8(f, oid_version(the_hash_algo));
418 hashwrite_u8(f, num_chunks);
419 hashwrite_u8(f, 0); /* unused */
420 hashwrite_be32(f, num_packs);
421
422 return MIDX_HEADER_SIZE;
423 }
424
425 struct pack_info {
426 uint32_t orig_pack_int_id;
427 char *pack_name;
428 struct packed_git *p;
429 unsigned expired : 1;
430 };
431
432 static int pack_info_compare(const void *_a, const void *_b)
433 {
434 struct pack_info *a = (struct pack_info *)_a;
435 struct pack_info *b = (struct pack_info *)_b;
436 return strcmp(a->pack_name, b->pack_name);
437 }
438
439 static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
440 {
441 const char *pack_name = _va;
442 const struct pack_info *compar = _vb;
443
444 return cmp_idx_or_pack_name(pack_name, compar->pack_name);
445 }
446
447 struct write_midx_context {
448 struct pack_info *info;
449 size_t nr;
450 size_t alloc;
451 struct multi_pack_index *m;
452 struct progress *progress;
453 unsigned pack_paths_checked;
454
455 struct pack_midx_entry *entries;
456 size_t entries_nr;
457
458 uint32_t *pack_perm;
459 uint32_t *pack_order;
460 unsigned large_offsets_needed:1;
461 uint32_t num_large_offsets;
462
463 int preferred_pack_idx;
464
465 struct string_list *to_include;
466 };
467
468 static void add_pack_to_midx(const char *full_path, size_t full_path_len,
469 const char *file_name, void *data)
470 {
471 struct write_midx_context *ctx = data;
472
473 if (ends_with(file_name, ".idx")) {
474 display_progress(ctx->progress, ++ctx->pack_paths_checked);
475 /*
476 * Note that at most one of ctx->m and ctx->to_include are set,
477 * so we are testing midx_contains_pack() and
478 * string_list_has_string() independently (guarded by the
479 * appropriate NULL checks).
480 *
481 * We could support passing to_include while reusing an existing
482 * MIDX, but don't currently since the reuse process drags
483 * forward all packs from an existing MIDX (without checking
484 * whether or not they appear in the to_include list).
485 *
486 * If we added support for that, these next two conditional
487 * should be performed independently (likely checking
488 * to_include before the existing MIDX).
489 */
490 if (ctx->m && midx_contains_pack(ctx->m, file_name))
491 return;
492 else if (ctx->to_include &&
493 !string_list_has_string(ctx->to_include, file_name))
494 return;
495
496 ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
497
498 ctx->info[ctx->nr].p = add_packed_git(full_path,
499 full_path_len,
500 0);
501
502 if (!ctx->info[ctx->nr].p) {
503 warning(_("failed to add packfile '%s'"),
504 full_path);
505 return;
506 }
507
508 if (open_pack_index(ctx->info[ctx->nr].p)) {
509 warning(_("failed to open pack-index '%s'"),
510 full_path);
511 close_pack(ctx->info[ctx->nr].p);
512 FREE_AND_NULL(ctx->info[ctx->nr].p);
513 return;
514 }
515
516 ctx->info[ctx->nr].pack_name = xstrdup(file_name);
517 ctx->info[ctx->nr].orig_pack_int_id = ctx->nr;
518 ctx->info[ctx->nr].expired = 0;
519 ctx->nr++;
520 }
521 }
522
523 struct pack_midx_entry {
524 struct object_id oid;
525 uint32_t pack_int_id;
526 time_t pack_mtime;
527 uint64_t offset;
528 unsigned preferred : 1;
529 };
530
531 static int midx_oid_compare(const void *_a, const void *_b)
532 {
533 const struct pack_midx_entry *a = (const struct pack_midx_entry *)_a;
534 const struct pack_midx_entry *b = (const struct pack_midx_entry *)_b;
535 int cmp = oidcmp(&a->oid, &b->oid);
536
537 if (cmp)
538 return cmp;
539
540 /* Sort objects in a preferred pack first when multiple copies exist. */
541 if (a->preferred > b->preferred)
542 return -1;
543 if (a->preferred < b->preferred)
544 return 1;
545
546 if (a->pack_mtime > b->pack_mtime)
547 return -1;
548 else if (a->pack_mtime < b->pack_mtime)
549 return 1;
550
551 return a->pack_int_id - b->pack_int_id;
552 }
553
554 static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
555 struct pack_midx_entry *e,
556 uint32_t pos)
557 {
558 if (pos >= m->num_objects)
559 return 1;
560
561 nth_midxed_object_oid(&e->oid, m, pos);
562 e->pack_int_id = nth_midxed_pack_int_id(m, pos);
563 e->offset = nth_midxed_offset(m, pos);
564
565 /* consider objects in midx to be from "old" packs */
566 e->pack_mtime = 0;
567 return 0;
568 }
569
570 static void fill_pack_entry(uint32_t pack_int_id,
571 struct packed_git *p,
572 uint32_t cur_object,
573 struct pack_midx_entry *entry,
574 int preferred)
575 {
576 if (nth_packed_object_id(&entry->oid, p, cur_object) < 0)
577 die(_("failed to locate object %d in packfile"), cur_object);
578
579 entry->pack_int_id = pack_int_id;
580 entry->pack_mtime = p->mtime;
581
582 entry->offset = nth_packed_object_offset(p, cur_object);
583 entry->preferred = !!preferred;
584 }
585
586 struct midx_fanout {
587 struct pack_midx_entry *entries;
588 size_t nr, alloc;
589 };
590
591 static void midx_fanout_grow(struct midx_fanout *fanout, size_t nr)
592 {
593 if (nr < fanout->nr)
594 BUG("negative growth in midx_fanout_grow() (%"PRIuMAX" < %"PRIuMAX")",
595 (uintmax_t)nr, (uintmax_t)fanout->nr);
596 ALLOC_GROW(fanout->entries, nr, fanout->alloc);
597 }
598
599 static void midx_fanout_sort(struct midx_fanout *fanout)
600 {
601 QSORT(fanout->entries, fanout->nr, midx_oid_compare);
602 }
603
604 static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
605 struct multi_pack_index *m,
606 uint32_t cur_fanout,
607 int preferred_pack)
608 {
609 uint32_t start = 0, end;
610 uint32_t cur_object;
611
612 if (cur_fanout)
613 start = ntohl(m->chunk_oid_fanout[cur_fanout - 1]);
614 end = ntohl(m->chunk_oid_fanout[cur_fanout]);
615
616 for (cur_object = start; cur_object < end; cur_object++) {
617 if ((preferred_pack > -1) &&
618 (preferred_pack == nth_midxed_pack_int_id(m, cur_object))) {
619 /*
620 * Objects from preferred packs are added
621 * separately.
622 */
623 continue;
624 }
625
626 midx_fanout_grow(fanout, fanout->nr + 1);
627 nth_midxed_pack_midx_entry(m,
628 &fanout->entries[fanout->nr],
629 cur_object);
630 fanout->entries[fanout->nr].preferred = 0;
631 fanout->nr++;
632 }
633 }
634
635 static void midx_fanout_add_pack_fanout(struct midx_fanout *fanout,
636 struct pack_info *info,
637 uint32_t cur_pack,
638 int preferred,
639 uint32_t cur_fanout)
640 {
641 struct packed_git *pack = info[cur_pack].p;
642 uint32_t start = 0, end;
643 uint32_t cur_object;
644
645 if (cur_fanout)
646 start = get_pack_fanout(pack, cur_fanout - 1);
647 end = get_pack_fanout(pack, cur_fanout);
648
649 for (cur_object = start; cur_object < end; cur_object++) {
650 midx_fanout_grow(fanout, fanout->nr + 1);
651 fill_pack_entry(cur_pack,
652 info[cur_pack].p,
653 cur_object,
654 &fanout->entries[fanout->nr],
655 preferred);
656 fanout->nr++;
657 }
658 }
659
660 /*
661 * It is possible to artificially get into a state where there are many
662 * duplicate copies of objects. That can create high memory pressure if
663 * we are to create a list of all objects before de-duplication. To reduce
664 * this memory pressure without a significant performance drop, automatically
665 * group objects by the first byte of their object id. Use the IDX fanout
666 * tables to group the data, copy to a local array, then sort.
667 *
668 * Copy only the de-duplicated entries (selected by most-recent modified time
669 * of a packfile containing the object).
670 */
671 static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
672 struct pack_info *info,
673 uint32_t nr_packs,
674 size_t *nr_objects,
675 int preferred_pack)
676 {
677 uint32_t cur_fanout, cur_pack, cur_object;
678 size_t alloc_objects, total_objects = 0;
679 struct midx_fanout fanout = { 0 };
680 struct pack_midx_entry *deduplicated_entries = NULL;
681 uint32_t start_pack = m ? m->num_packs : 0;
682
683 for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++)
684 total_objects = st_add(total_objects,
685 info[cur_pack].p->num_objects);
686
687 /*
688 * As we de-duplicate by fanout value, we expect the fanout
689 * slices to be evenly distributed, with some noise. Hence,
690 * allocate slightly more than one 256th.
691 */
692 alloc_objects = fanout.alloc = total_objects > 3200 ? total_objects / 200 : 16;
693
694 ALLOC_ARRAY(fanout.entries, fanout.alloc);
695 ALLOC_ARRAY(deduplicated_entries, alloc_objects);
696 *nr_objects = 0;
697
698 for (cur_fanout = 0; cur_fanout < 256; cur_fanout++) {
699 fanout.nr = 0;
700
701 if (m)
702 midx_fanout_add_midx_fanout(&fanout, m, cur_fanout,
703 preferred_pack);
704
705 for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) {
706 int preferred = cur_pack == preferred_pack;
707 midx_fanout_add_pack_fanout(&fanout,
708 info, cur_pack,
709 preferred, cur_fanout);
710 }
711
712 if (-1 < preferred_pack && preferred_pack < start_pack)
713 midx_fanout_add_pack_fanout(&fanout, info,
714 preferred_pack, 1,
715 cur_fanout);
716
717 midx_fanout_sort(&fanout);
718
719 /*
720 * The batch is now sorted by OID and then mtime (descending).
721 * Take only the first duplicate.
722 */
723 for (cur_object = 0; cur_object < fanout.nr; cur_object++) {
724 if (cur_object && oideq(&fanout.entries[cur_object - 1].oid,
725 &fanout.entries[cur_object].oid))
726 continue;
727
728 ALLOC_GROW(deduplicated_entries, st_add(*nr_objects, 1),
729 alloc_objects);
730 memcpy(&deduplicated_entries[*nr_objects],
731 &fanout.entries[cur_object],
732 sizeof(struct pack_midx_entry));
733 (*nr_objects)++;
734 }
735 }
736
737 free(fanout.entries);
738 return deduplicated_entries;
739 }
740
741 static int write_midx_pack_names(struct hashfile *f, void *data)
742 {
743 struct write_midx_context *ctx = data;
744 uint32_t i;
745 unsigned char padding[MIDX_CHUNK_ALIGNMENT];
746 size_t written = 0;
747
748 for (i = 0; i < ctx->nr; i++) {
749 size_t writelen;
750
751 if (ctx->info[i].expired)
752 continue;
753
754 if (i && strcmp(ctx->info[i].pack_name, ctx->info[i - 1].pack_name) <= 0)
755 BUG("incorrect pack-file order: %s before %s",
756 ctx->info[i - 1].pack_name,
757 ctx->info[i].pack_name);
758
759 writelen = strlen(ctx->info[i].pack_name) + 1;
760 hashwrite(f, ctx->info[i].pack_name, writelen);
761 written += writelen;
762 }
763
764 /* add padding to be aligned */
765 i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
766 if (i < MIDX_CHUNK_ALIGNMENT) {
767 memset(padding, 0, sizeof(padding));
768 hashwrite(f, padding, i);
769 }
770
771 return 0;
772 }
773
774 static int write_midx_oid_fanout(struct hashfile *f,
775 void *data)
776 {
777 struct write_midx_context *ctx = data;
778 struct pack_midx_entry *list = ctx->entries;
779 struct pack_midx_entry *last = ctx->entries + ctx->entries_nr;
780 uint32_t count = 0;
781 uint32_t i;
782
783 /*
784 * Write the first-level table (the list is sorted,
785 * but we use a 256-entry lookup to be able to avoid
786 * having to do eight extra binary search iterations).
787 */
788 for (i = 0; i < 256; i++) {
789 struct pack_midx_entry *next = list;
790
791 while (next < last && next->oid.hash[0] == i) {
792 count++;
793 next++;
794 }
795
796 hashwrite_be32(f, count);
797 list = next;
798 }
799
800 return 0;
801 }
802
803 static int write_midx_oid_lookup(struct hashfile *f,
804 void *data)
805 {
806 struct write_midx_context *ctx = data;
807 unsigned char hash_len = the_hash_algo->rawsz;
808 struct pack_midx_entry *list = ctx->entries;
809 uint32_t i;
810
811 for (i = 0; i < ctx->entries_nr; i++) {
812 struct pack_midx_entry *obj = list++;
813
814 if (i < ctx->entries_nr - 1) {
815 struct pack_midx_entry *next = list;
816 if (oidcmp(&obj->oid, &next->oid) >= 0)
817 BUG("OIDs not in order: %s >= %s",
818 oid_to_hex(&obj->oid),
819 oid_to_hex(&next->oid));
820 }
821
822 hashwrite(f, obj->oid.hash, (int)hash_len);
823 }
824
825 return 0;
826 }
827
828 static int write_midx_object_offsets(struct hashfile *f,
829 void *data)
830 {
831 struct write_midx_context *ctx = data;
832 struct pack_midx_entry *list = ctx->entries;
833 uint32_t i, nr_large_offset = 0;
834
835 for (i = 0; i < ctx->entries_nr; i++) {
836 struct pack_midx_entry *obj = list++;
837
838 if (ctx->pack_perm[obj->pack_int_id] == PACK_EXPIRED)
839 BUG("object %s is in an expired pack with int-id %d",
840 oid_to_hex(&obj->oid),
841 obj->pack_int_id);
842
843 hashwrite_be32(f, ctx->pack_perm[obj->pack_int_id]);
844
845 if (ctx->large_offsets_needed && obj->offset >> 31)
846 hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++);
847 else if (!ctx->large_offsets_needed && obj->offset >> 32)
848 BUG("object %s requires a large offset (%"PRIx64") but the MIDX is not writing large offsets!",
849 oid_to_hex(&obj->oid),
850 obj->offset);
851 else
852 hashwrite_be32(f, (uint32_t)obj->offset);
853 }
854
855 return 0;
856 }
857
858 static int write_midx_large_offsets(struct hashfile *f,
859 void *data)
860 {
861 struct write_midx_context *ctx = data;
862 struct pack_midx_entry *list = ctx->entries;
863 struct pack_midx_entry *end = ctx->entries + ctx->entries_nr;
864 uint32_t nr_large_offset = ctx->num_large_offsets;
865
866 while (nr_large_offset) {
867 struct pack_midx_entry *obj;
868 uint64_t offset;
869
870 if (list >= end)
871 BUG("too many large-offset objects");
872
873 obj = list++;
874 offset = obj->offset;
875
876 if (!(offset >> 31))
877 continue;
878
879 hashwrite_be64(f, offset);
880
881 nr_large_offset--;
882 }
883
884 return 0;
885 }
886
887 static int write_midx_revindex(struct hashfile *f,
888 void *data)
889 {
890 struct write_midx_context *ctx = data;
891 uint32_t i;
892
893 for (i = 0; i < ctx->entries_nr; i++)
894 hashwrite_be32(f, ctx->pack_order[i]);
895
896 return 0;
897 }
898
899 struct midx_pack_order_data {
900 uint32_t nr;
901 uint32_t pack;
902 off_t offset;
903 };
904
905 static int midx_pack_order_cmp(const void *va, const void *vb)
906 {
907 const struct midx_pack_order_data *a = va, *b = vb;
908 if (a->pack < b->pack)
909 return -1;
910 else if (a->pack > b->pack)
911 return 1;
912 else if (a->offset < b->offset)
913 return -1;
914 else if (a->offset > b->offset)
915 return 1;
916 else
917 return 0;
918 }
919
920 static uint32_t *midx_pack_order(struct write_midx_context *ctx)
921 {
922 struct midx_pack_order_data *data;
923 uint32_t *pack_order;
924 uint32_t i;
925
926 trace2_region_enter("midx", "midx_pack_order", the_repository);
927
928 ALLOC_ARRAY(data, ctx->entries_nr);
929 for (i = 0; i < ctx->entries_nr; i++) {
930 struct pack_midx_entry *e = &ctx->entries[i];
931 data[i].nr = i;
932 data[i].pack = ctx->pack_perm[e->pack_int_id];
933 if (!e->preferred)
934 data[i].pack |= (1U << 31);
935 data[i].offset = e->offset;
936 }
937
938 QSORT(data, ctx->entries_nr, midx_pack_order_cmp);
939
940 ALLOC_ARRAY(pack_order, ctx->entries_nr);
941 for (i = 0; i < ctx->entries_nr; i++)
942 pack_order[i] = data[i].nr;
943 free(data);
944
945 trace2_region_leave("midx", "midx_pack_order", the_repository);
946
947 return pack_order;
948 }
949
950 static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
951 struct write_midx_context *ctx)
952 {
953 struct strbuf buf = STRBUF_INIT;
954 const char *tmp_file;
955
956 trace2_region_enter("midx", "write_midx_reverse_index", the_repository);
957
958 strbuf_addf(&buf, "%s-%s.rev", midx_name, hash_to_hex(midx_hash));
959
960 tmp_file = write_rev_file_order(NULL, ctx->pack_order, ctx->entries_nr,
961 midx_hash, WRITE_REV);
962
963 if (finalize_object_file(tmp_file, buf.buf))
964 die(_("cannot store reverse index file"));
965
966 strbuf_release(&buf);
967
968 trace2_region_leave("midx", "write_midx_reverse_index", the_repository);
969 }
970
971 static void clear_midx_files_ext(const char *object_dir, const char *ext,
972 unsigned char *keep_hash);
973
974 static int midx_checksum_valid(struct multi_pack_index *m)
975 {
976 return hashfile_checksum_valid(m->data, m->data_len);
977 }
978
979 static void prepare_midx_packing_data(struct packing_data *pdata,
980 struct write_midx_context *ctx)
981 {
982 uint32_t i;
983
984 trace2_region_enter("midx", "prepare_midx_packing_data", the_repository);
985
986 memset(pdata, 0, sizeof(struct packing_data));
987 prepare_packing_data(the_repository, pdata);
988
989 for (i = 0; i < ctx->entries_nr; i++) {
990 struct pack_midx_entry *from = &ctx->entries[ctx->pack_order[i]];
991 struct object_entry *to = packlist_alloc(pdata, &from->oid);
992
993 oe_set_in_pack(pdata, to,
994 ctx->info[ctx->pack_perm[from->pack_int_id]].p);
995 }
996
997 trace2_region_leave("midx", "prepare_midx_packing_data", the_repository);
998 }
999
1000 static int add_ref_to_pending(const char *refname,
1001 const struct object_id *oid,
1002 int flag, void *cb_data)
1003 {
1004 struct rev_info *revs = (struct rev_info*)cb_data;
1005 struct object_id peeled;
1006 struct object *object;
1007
1008 if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) {
1009 warning("symbolic ref is dangling: %s", refname);
1010 return 0;
1011 }
1012
1013 if (!peel_iterated_oid(oid, &peeled))
1014 oid = &peeled;
1015
1016 object = parse_object_or_die(oid, refname);
1017 if (object->type != OBJ_COMMIT)
1018 return 0;
1019
1020 add_pending_object(revs, object, "");
1021 if (bitmap_is_preferred_refname(revs->repo, refname))
1022 object->flags |= NEEDS_BITMAP;
1023 return 0;
1024 }
1025
1026 struct bitmap_commit_cb {
1027 struct commit **commits;
1028 size_t commits_nr, commits_alloc;
1029
1030 struct write_midx_context *ctx;
1031 };
1032
1033 static const struct object_id *bitmap_oid_access(size_t index,
1034 const void *_entries)
1035 {
1036 const struct pack_midx_entry *entries = _entries;
1037 return &entries[index].oid;
1038 }
1039
1040 static void bitmap_show_commit(struct commit *commit, void *_data)
1041 {
1042 struct bitmap_commit_cb *data = _data;
1043 int pos = oid_pos(&commit->object.oid, data->ctx->entries,
1044 data->ctx->entries_nr,
1045 bitmap_oid_access);
1046 if (pos < 0)
1047 return;
1048
1049 ALLOC_GROW(data->commits, data->commits_nr + 1, data->commits_alloc);
1050 data->commits[data->commits_nr++] = commit;
1051 }
1052
1053 static int read_refs_snapshot(const char *refs_snapshot,
1054 struct rev_info *revs)
1055 {
1056 struct strbuf buf = STRBUF_INIT;
1057 struct object_id oid;
1058 FILE *f = xfopen(refs_snapshot, "r");
1059
1060 while (strbuf_getline(&buf, f) != EOF) {
1061 struct object *object;
1062 int preferred = 0;
1063 char *hex = buf.buf;
1064 const char *end = NULL;
1065
1066 if (buf.len && *buf.buf == '+') {
1067 preferred = 1;
1068 hex = &buf.buf[1];
1069 }
1070
1071 if (parse_oid_hex(hex, &oid, &end) < 0)
1072 die(_("could not parse line: %s"), buf.buf);
1073 if (*end)
1074 die(_("malformed line: %s"), buf.buf);
1075
1076 object = parse_object_or_die(&oid, NULL);
1077 if (preferred)
1078 object->flags |= NEEDS_BITMAP;
1079
1080 add_pending_object(revs, object, "");
1081 }
1082
1083 fclose(f);
1084 strbuf_release(&buf);
1085 return 0;
1086 }
1087
1088 static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,
1089 const char *refs_snapshot,
1090 struct write_midx_context *ctx)
1091 {
1092 struct rev_info revs;
1093 struct bitmap_commit_cb cb = {0};
1094
1095 trace2_region_enter("midx", "find_commits_for_midx_bitmap",
1096 the_repository);
1097
1098 cb.ctx = ctx;
1099
1100 repo_init_revisions(the_repository, &revs, NULL);
1101 if (refs_snapshot) {
1102 read_refs_snapshot(refs_snapshot, &revs);
1103 } else {
1104 setup_revisions(0, NULL, &revs, NULL);
1105 for_each_ref(add_ref_to_pending, &revs);
1106 }
1107
1108 /*
1109 * Skipping promisor objects here is intentional, since it only excludes
1110 * them from the list of reachable commits that we want to select from
1111 * when computing the selection of MIDX'd commits to receive bitmaps.
1112 *
1113 * Reachability bitmaps do require that their objects be closed under
1114 * reachability, but fetching any objects missing from promisors at this
1115 * point is too late. But, if one of those objects can be reached from
1116 * an another object that is included in the bitmap, then we will
1117 * complain later that we don't have reachability closure (and fail
1118 * appropriately).
1119 */
1120 fetch_if_missing = 0;
1121 revs.exclude_promisor_objects = 1;
1122
1123 if (prepare_revision_walk(&revs))
1124 die(_("revision walk setup failed"));
1125
1126 traverse_commit_list(&revs, bitmap_show_commit, NULL, &cb);
1127 if (indexed_commits_nr_p)
1128 *indexed_commits_nr_p = cb.commits_nr;
1129
1130 release_revisions(&revs);
1131
1132 trace2_region_leave("midx", "find_commits_for_midx_bitmap",
1133 the_repository);
1134
1135 return cb.commits;
1136 }
1137
1138 static int write_midx_bitmap(const char *midx_name,
1139 const unsigned char *midx_hash,
1140 struct packing_data *pdata,
1141 struct commit **commits,
1142 uint32_t commits_nr,
1143 uint32_t *pack_order,
1144 unsigned flags)
1145 {
1146 int ret, i;
1147 uint16_t options = 0;
1148 struct pack_idx_entry **index;
1149 char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name,
1150 hash_to_hex(midx_hash));
1151
1152 trace2_region_enter("midx", "write_midx_bitmap", the_repository);
1153
1154 if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
1155 options |= BITMAP_OPT_HASH_CACHE;
1156
1157 if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
1158 options |= BITMAP_OPT_LOOKUP_TABLE;
1159
1160 /*
1161 * Build the MIDX-order index based on pdata.objects (which is already
1162 * in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
1163 * this order).
1164 */
1165 ALLOC_ARRAY(index, pdata->nr_objects);
1166 for (i = 0; i < pdata->nr_objects; i++)
1167 index[i] = &pdata->objects[i].idx;
1168
1169 bitmap_writer_show_progress(flags & MIDX_PROGRESS);
1170 bitmap_writer_build_type_index(pdata, index, pdata->nr_objects);
1171
1172 /*
1173 * bitmap_writer_finish expects objects in lex order, but pack_order
1174 * gives us exactly that. use it directly instead of re-sorting the
1175 * array.
1176 *
1177 * This changes the order of objects in 'index' between
1178 * bitmap_writer_build_type_index and bitmap_writer_finish.
1179 *
1180 * The same re-ordering takes place in the single-pack bitmap code via
1181 * write_idx_file(), which is called by finish_tmp_packfile(), which
1182 * happens between bitmap_writer_build_type_index() and
1183 * bitmap_writer_finish().
1184 */
1185 for (i = 0; i < pdata->nr_objects; i++)
1186 index[pack_order[i]] = &pdata->objects[i].idx;
1187
1188 bitmap_writer_select_commits(commits, commits_nr, -1);
1189 ret = bitmap_writer_build(pdata);
1190 if (ret < 0)
1191 goto cleanup;
1192
1193 bitmap_writer_set_checksum(midx_hash);
1194 bitmap_writer_finish(index, pdata->nr_objects, bitmap_name, options);
1195
1196 cleanup:
1197 free(index);
1198 free(bitmap_name);
1199
1200 trace2_region_leave("midx", "write_midx_bitmap", the_repository);
1201
1202 return ret;
1203 }
1204
1205 static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
1206 const char *object_dir)
1207 {
1208 struct multi_pack_index *result = NULL;
1209 struct multi_pack_index *cur;
1210 char *obj_dir_real = real_pathdup(object_dir, 1);
1211 struct strbuf cur_path_real = STRBUF_INIT;
1212
1213 /* Ensure the given object_dir is local, or a known alternate. */
1214 find_odb(r, obj_dir_real);
1215
1216 for (cur = get_multi_pack_index(r); cur; cur = cur->next) {
1217 strbuf_realpath(&cur_path_real, cur->object_dir, 1);
1218 if (!strcmp(obj_dir_real, cur_path_real.buf)) {
1219 result = cur;
1220 goto cleanup;
1221 }
1222 }
1223
1224 cleanup:
1225 free(obj_dir_real);
1226 strbuf_release(&cur_path_real);
1227 return result;
1228 }
1229
1230 static int write_midx_internal(const char *object_dir,
1231 struct string_list *packs_to_include,
1232 struct string_list *packs_to_drop,
1233 const char *preferred_pack_name,
1234 const char *refs_snapshot,
1235 unsigned flags)
1236 {
1237 struct strbuf midx_name = STRBUF_INIT;
1238 unsigned char midx_hash[GIT_MAX_RAWSZ];
1239 uint32_t i;
1240 struct hashfile *f = NULL;
1241 struct lock_file lk;
1242 struct write_midx_context ctx = { 0 };
1243 int pack_name_concat_len = 0;
1244 int dropped_packs = 0;
1245 int result = 0;
1246 struct chunkfile *cf;
1247
1248 trace2_region_enter("midx", "write_midx_internal", the_repository);
1249
1250 get_midx_filename(&midx_name, object_dir);
1251 if (safe_create_leading_directories(midx_name.buf))
1252 die_errno(_("unable to create leading directories of %s"),
1253 midx_name.buf);
1254
1255 if (!packs_to_include) {
1256 /*
1257 * Only reference an existing MIDX when not filtering which
1258 * packs to include, since all packs and objects are copied
1259 * blindly from an existing MIDX if one is present.
1260 */
1261 ctx.m = lookup_multi_pack_index(the_repository, object_dir);
1262 }
1263
1264 if (ctx.m && !midx_checksum_valid(ctx.m)) {
1265 warning(_("ignoring existing multi-pack-index; checksum mismatch"));
1266 ctx.m = NULL;
1267 }
1268
1269 ctx.nr = 0;
1270 ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
1271 ctx.info = NULL;
1272 ALLOC_ARRAY(ctx.info, ctx.alloc);
1273
1274 if (ctx.m) {
1275 for (i = 0; i < ctx.m->num_packs; i++) {
1276 ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc);
1277
1278 ctx.info[ctx.nr].orig_pack_int_id = i;
1279 ctx.info[ctx.nr].pack_name = xstrdup(ctx.m->pack_names[i]);
1280 ctx.info[ctx.nr].p = ctx.m->packs[i];
1281 ctx.info[ctx.nr].expired = 0;
1282
1283 if (flags & MIDX_WRITE_REV_INDEX) {
1284 /*
1285 * If generating a reverse index, need to have
1286 * packed_git's loaded to compare their
1287 * mtimes and object count.
1288 */
1289 if (prepare_midx_pack(the_repository, ctx.m, i)) {
1290 error(_("could not load pack"));
1291 result = 1;
1292 goto cleanup;
1293 }
1294
1295 if (open_pack_index(ctx.m->packs[i]))
1296 die(_("could not open index for %s"),
1297 ctx.m->packs[i]->pack_name);
1298 ctx.info[ctx.nr].p = ctx.m->packs[i];
1299 }
1300
1301 ctx.nr++;
1302 }
1303 }
1304
1305 ctx.pack_paths_checked = 0;
1306 if (flags & MIDX_PROGRESS)
1307 ctx.progress = start_delayed_progress(_("Adding packfiles to multi-pack-index"), 0);
1308 else
1309 ctx.progress = NULL;
1310
1311 ctx.to_include = packs_to_include;
1312
1313 for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
1314 stop_progress(&ctx.progress);
1315
1316 if ((ctx.m && ctx.nr == ctx.m->num_packs) &&
1317 !(packs_to_include || packs_to_drop)) {
1318 struct bitmap_index *bitmap_git;
1319 int bitmap_exists;
1320 int want_bitmap = flags & MIDX_WRITE_BITMAP;
1321
1322 bitmap_git = prepare_midx_bitmap_git(ctx.m);
1323 bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
1324 free_bitmap_index(bitmap_git);
1325
1326 if (bitmap_exists || !want_bitmap) {
1327 /*
1328 * The correct MIDX already exists, and so does a
1329 * corresponding bitmap (or one wasn't requested).
1330 */
1331 if (!want_bitmap)
1332 clear_midx_files_ext(object_dir, ".bitmap",
1333 NULL);
1334 goto cleanup;
1335 }
1336 }
1337
1338 if (preferred_pack_name) {
1339 ctx.preferred_pack_idx = -1;
1340
1341 for (i = 0; i < ctx.nr; i++) {
1342 if (!cmp_idx_or_pack_name(preferred_pack_name,
1343 ctx.info[i].pack_name)) {
1344 ctx.preferred_pack_idx = i;
1345 break;
1346 }
1347 }
1348
1349 if (ctx.preferred_pack_idx == -1)
1350 warning(_("unknown preferred pack: '%s'"),
1351 preferred_pack_name);
1352 } else if (ctx.nr &&
1353 (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) {
1354 struct packed_git *oldest = ctx.info[ctx.preferred_pack_idx].p;
1355 ctx.preferred_pack_idx = 0;
1356
1357 if (packs_to_drop && packs_to_drop->nr)
1358 BUG("cannot write a MIDX bitmap during expiration");
1359
1360 /*
1361 * set a preferred pack when writing a bitmap to ensure that
1362 * the pack from which the first object is selected in pseudo
1363 * pack-order has all of its objects selected from that pack
1364 * (and not another pack containing a duplicate)
1365 */
1366 for (i = 1; i < ctx.nr; i++) {
1367 struct packed_git *p = ctx.info[i].p;
1368
1369 if (!oldest->num_objects || p->mtime < oldest->mtime) {
1370 oldest = p;
1371 ctx.preferred_pack_idx = i;
1372 }
1373 }
1374
1375 if (!oldest->num_objects) {
1376 /*
1377 * If all packs are empty; unset the preferred index.
1378 * This is acceptable since there will be no duplicate
1379 * objects to resolve, so the preferred value doesn't
1380 * matter.
1381 */
1382 ctx.preferred_pack_idx = -1;
1383 }
1384 } else {
1385 /*
1386 * otherwise don't mark any pack as preferred to avoid
1387 * interfering with expiration logic below
1388 */
1389 ctx.preferred_pack_idx = -1;
1390 }
1391
1392 if (ctx.preferred_pack_idx > -1) {
1393 struct packed_git *preferred = ctx.info[ctx.preferred_pack_idx].p;
1394 if (!preferred->num_objects) {
1395 error(_("cannot select preferred pack %s with no objects"),
1396 preferred->pack_name);
1397 result = 1;
1398 goto cleanup;
1399 }
1400 }
1401
1402 ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr,
1403 ctx.preferred_pack_idx);
1404
1405 ctx.large_offsets_needed = 0;
1406 for (i = 0; i < ctx.entries_nr; i++) {
1407 if (ctx.entries[i].offset > 0x7fffffff)
1408 ctx.num_large_offsets++;
1409 if (ctx.entries[i].offset > 0xffffffff)
1410 ctx.large_offsets_needed = 1;
1411 }
1412
1413 QSORT(ctx.info, ctx.nr, pack_info_compare);
1414
1415 if (packs_to_drop && packs_to_drop->nr) {
1416 int drop_index = 0;
1417 int missing_drops = 0;
1418
1419 for (i = 0; i < ctx.nr && drop_index < packs_to_drop->nr; i++) {
1420 int cmp = strcmp(ctx.info[i].pack_name,
1421 packs_to_drop->items[drop_index].string);
1422
1423 if (!cmp) {
1424 drop_index++;
1425 ctx.info[i].expired = 1;
1426 } else if (cmp > 0) {
1427 error(_("did not see pack-file %s to drop"),
1428 packs_to_drop->items[drop_index].string);
1429 drop_index++;
1430 missing_drops++;
1431 i--;
1432 } else {
1433 ctx.info[i].expired = 0;
1434 }
1435 }
1436
1437 if (missing_drops) {
1438 result = 1;
1439 goto cleanup;
1440 }
1441 }
1442
1443 /*
1444 * pack_perm stores a permutation between pack-int-ids from the
1445 * previous multi-pack-index to the new one we are writing:
1446 *
1447 * pack_perm[old_id] = new_id
1448 */
1449 ALLOC_ARRAY(ctx.pack_perm, ctx.nr);
1450 for (i = 0; i < ctx.nr; i++) {
1451 if (ctx.info[i].expired) {
1452 dropped_packs++;
1453 ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
1454 } else {
1455 ctx.pack_perm[ctx.info[i].orig_pack_int_id] = i - dropped_packs;
1456 }
1457 }
1458
1459 for (i = 0; i < ctx.nr; i++) {
1460 if (!ctx.info[i].expired)
1461 pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
1462 }
1463
1464 /* Check that the preferred pack wasn't expired (if given). */
1465 if (preferred_pack_name) {
1466 struct pack_info *preferred = bsearch(preferred_pack_name,
1467 ctx.info, ctx.nr,
1468 sizeof(*ctx.info),
1469 idx_or_pack_name_cmp);
1470 if (preferred) {
1471 uint32_t perm = ctx.pack_perm[preferred->orig_pack_int_id];
1472 if (perm == PACK_EXPIRED)
1473 warning(_("preferred pack '%s' is expired"),
1474 preferred_pack_name);
1475 }
1476 }
1477
1478 if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
1479 pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
1480 (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
1481
1482 hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR);
1483 f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
1484
1485 if (ctx.nr - dropped_packs == 0) {
1486 error(_("no pack files to index."));
1487 result = 1;
1488 goto cleanup;
1489 }
1490
1491 if (!ctx.entries_nr) {
1492 if (flags & MIDX_WRITE_BITMAP)
1493 warning(_("refusing to write multi-pack .bitmap without any objects"));
1494 flags &= ~(MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP);
1495 }
1496
1497 cf = init_chunkfile(f);
1498
1499 add_chunk(cf, MIDX_CHUNKID_PACKNAMES, pack_name_concat_len,
1500 write_midx_pack_names);
1501 add_chunk(cf, MIDX_CHUNKID_OIDFANOUT, MIDX_CHUNK_FANOUT_SIZE,
1502 write_midx_oid_fanout);
1503 add_chunk(cf, MIDX_CHUNKID_OIDLOOKUP,
1504 st_mult(ctx.entries_nr, the_hash_algo->rawsz),
1505 write_midx_oid_lookup);
1506 add_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS,
1507 st_mult(ctx.entries_nr, MIDX_CHUNK_OFFSET_WIDTH),
1508 write_midx_object_offsets);
1509
1510 if (ctx.large_offsets_needed)
1511 add_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS,
1512 st_mult(ctx.num_large_offsets,
1513 MIDX_CHUNK_LARGE_OFFSET_WIDTH),
1514 write_midx_large_offsets);
1515
1516 if (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP)) {
1517 ctx.pack_order = midx_pack_order(&ctx);
1518 add_chunk(cf, MIDX_CHUNKID_REVINDEX,
1519 st_mult(ctx.entries_nr, sizeof(uint32_t)),
1520 write_midx_revindex);
1521 }
1522
1523 write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs);
1524 write_chunkfile(cf, &ctx);
1525
1526 finalize_hashfile(f, midx_hash, FSYNC_COMPONENT_PACK_METADATA,
1527 CSUM_FSYNC | CSUM_HASH_IN_STREAM);
1528 free_chunkfile(cf);
1529
1530 if (flags & MIDX_WRITE_REV_INDEX &&
1531 git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
1532 write_midx_reverse_index(midx_name.buf, midx_hash, &ctx);
1533
1534 if (flags & MIDX_WRITE_BITMAP) {
1535 struct packing_data pdata;
1536 struct commit **commits;
1537 uint32_t commits_nr;
1538
1539 if (!ctx.entries_nr)
1540 BUG("cannot write a bitmap without any objects");
1541
1542 prepare_midx_packing_data(&pdata, &ctx);
1543
1544 commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, &ctx);
1545
1546 /*
1547 * The previous steps translated the information from
1548 * 'entries' into information suitable for constructing
1549 * bitmaps. We no longer need that array, so clear it to
1550 * reduce memory pressure.
1551 */
1552 FREE_AND_NULL(ctx.entries);
1553 ctx.entries_nr = 0;
1554
1555 if (write_midx_bitmap(midx_name.buf, midx_hash, &pdata,
1556 commits, commits_nr, ctx.pack_order,
1557 flags) < 0) {
1558 error(_("could not write multi-pack bitmap"));
1559 result = 1;
1560 goto cleanup;
1561 }
1562 }
1563 /*
1564 * NOTE: Do not use ctx.entries beyond this point, since it might
1565 * have been freed in the previous if block.
1566 */
1567
1568 if (ctx.m)
1569 close_object_store(the_repository->objects);
1570
1571 if (commit_lock_file(&lk) < 0)
1572 die_errno(_("could not write multi-pack-index"));
1573
1574 clear_midx_files_ext(object_dir, ".bitmap", midx_hash);
1575 clear_midx_files_ext(object_dir, ".rev", midx_hash);
1576
1577 cleanup:
1578 for (i = 0; i < ctx.nr; i++) {
1579 if (ctx.info[i].p) {
1580 close_pack(ctx.info[i].p);
1581 free(ctx.info[i].p);
1582 }
1583 free(ctx.info[i].pack_name);
1584 }
1585
1586 free(ctx.info);
1587 free(ctx.entries);
1588 free(ctx.pack_perm);
1589 free(ctx.pack_order);
1590 strbuf_release(&midx_name);
1591
1592 trace2_region_leave("midx", "write_midx_internal", the_repository);
1593
1594 return result;
1595 }
1596
1597 int write_midx_file(const char *object_dir,
1598 const char *preferred_pack_name,
1599 const char *refs_snapshot,
1600 unsigned flags)
1601 {
1602 return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name,
1603 refs_snapshot, flags);
1604 }
1605
1606 int write_midx_file_only(const char *object_dir,
1607 struct string_list *packs_to_include,
1608 const char *preferred_pack_name,
1609 const char *refs_snapshot,
1610 unsigned flags)
1611 {
1612 return write_midx_internal(object_dir, packs_to_include, NULL,
1613 preferred_pack_name, refs_snapshot, flags);
1614 }
1615
1616 struct clear_midx_data {
1617 char *keep;
1618 const char *ext;
1619 };
1620
1621 static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED,
1622 const char *file_name, void *_data)
1623 {
1624 struct clear_midx_data *data = _data;
1625
1626 if (!(starts_with(file_name, "multi-pack-index-") &&
1627 ends_with(file_name, data->ext)))
1628 return;
1629 if (data->keep && !strcmp(data->keep, file_name))
1630 return;
1631
1632 if (unlink(full_path))
1633 die_errno(_("failed to remove %s"), full_path);
1634 }
1635
1636 static void clear_midx_files_ext(const char *object_dir, const char *ext,
1637 unsigned char *keep_hash)
1638 {
1639 struct clear_midx_data data;
1640 memset(&data, 0, sizeof(struct clear_midx_data));
1641
1642 if (keep_hash)
1643 data.keep = xstrfmt("multi-pack-index-%s%s",
1644 hash_to_hex(keep_hash), ext);
1645 data.ext = ext;
1646
1647 for_each_file_in_pack_dir(object_dir,
1648 clear_midx_file_ext,
1649 &data);
1650
1651 free(data.keep);
1652 }
1653
1654 void clear_midx_file(struct repository *r)
1655 {
1656 struct strbuf midx = STRBUF_INIT;
1657
1658 get_midx_filename(&midx, r->objects->odb->path);
1659
1660 if (r->objects && r->objects->multi_pack_index) {
1661 close_midx(r->objects->multi_pack_index);
1662 r->objects->multi_pack_index = NULL;
1663 }
1664
1665 if (remove_path(midx.buf))
1666 die(_("failed to clear multi-pack-index at %s"), midx.buf);
1667
1668 clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL);
1669 clear_midx_files_ext(r->objects->odb->path, ".rev", NULL);
1670
1671 strbuf_release(&midx);
1672 }
1673
1674 static int verify_midx_error;
1675
1676 __attribute__((format (printf, 1, 2)))
1677 static void midx_report(const char *fmt, ...)
1678 {
1679 va_list ap;
1680 verify_midx_error = 1;
1681 va_start(ap, fmt);
1682 vfprintf(stderr, fmt, ap);
1683 fprintf(stderr, "\n");
1684 va_end(ap);
1685 }
1686
1687 struct pair_pos_vs_id
1688 {
1689 uint32_t pos;
1690 uint32_t pack_int_id;
1691 };
1692
1693 static int compare_pair_pos_vs_id(const void *_a, const void *_b)
1694 {
1695 struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a;
1696 struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b;
1697
1698 return b->pack_int_id - a->pack_int_id;
1699 }
1700
1701 /*
1702 * Limit calls to display_progress() for performance reasons.
1703 * The interval here was arbitrarily chosen.
1704 */
1705 #define SPARSE_PROGRESS_INTERVAL (1 << 12)
1706 #define midx_display_sparse_progress(progress, n) \
1707 do { \
1708 uint64_t _n = (n); \
1709 if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \
1710 display_progress(progress, _n); \
1711 } while (0)
1712
1713 int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags)
1714 {
1715 struct pair_pos_vs_id *pairs = NULL;
1716 uint32_t i;
1717 struct progress *progress = NULL;
1718 struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
1719 verify_midx_error = 0;
1720
1721 if (!m) {
1722 int result = 0;
1723 struct stat sb;
1724 struct strbuf filename = STRBUF_INIT;
1725
1726 get_midx_filename(&filename, object_dir);
1727
1728 if (!stat(filename.buf, &sb)) {
1729 error(_("multi-pack-index file exists, but failed to parse"));
1730 result = 1;
1731 }
1732 strbuf_release(&filename);
1733 return result;
1734 }
1735
1736 if (!midx_checksum_valid(m))
1737 midx_report(_("incorrect checksum"));
1738
1739 if (flags & MIDX_PROGRESS)
1740 progress = start_delayed_progress(_("Looking for referenced packfiles"),
1741 m->num_packs);
1742 for (i = 0; i < m->num_packs; i++) {
1743 if (prepare_midx_pack(r, m, i))
1744 midx_report("failed to load pack in position %d", i);
1745
1746 display_progress(progress, i + 1);
1747 }
1748 stop_progress(&progress);
1749
1750 for (i = 0; i < 255; i++) {
1751 uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
1752 uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i + 1]);
1753
1754 if (oid_fanout1 > oid_fanout2)
1755 midx_report(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
1756 i, oid_fanout1, oid_fanout2, i + 1);
1757 }
1758
1759 if (m->num_objects == 0) {
1760 midx_report(_("the midx contains no oid"));
1761 /*
1762 * Remaining tests assume that we have objects, so we can
1763 * return here.
1764 */
1765 goto cleanup;
1766 }
1767
1768 if (flags & MIDX_PROGRESS)
1769 progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"),
1770 m->num_objects - 1);
1771 for (i = 0; i < m->num_objects - 1; i++) {
1772 struct object_id oid1, oid2;
1773
1774 nth_midxed_object_oid(&oid1, m, i);
1775 nth_midxed_object_oid(&oid2, m, i + 1);
1776
1777 if (oidcmp(&oid1, &oid2) >= 0)
1778 midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
1779 i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
1780
1781 midx_display_sparse_progress(progress, i + 1);
1782 }
1783 stop_progress(&progress);
1784
1785 /*
1786 * Create an array mapping each object to its packfile id. Sort it
1787 * to group the objects by packfile. Use this permutation to visit
1788 * each of the objects and only require 1 packfile to be open at a
1789 * time.
1790 */
1791 ALLOC_ARRAY(pairs, m->num_objects);
1792 for (i = 0; i < m->num_objects; i++) {
1793 pairs[i].pos = i;
1794 pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
1795 }
1796
1797 if (flags & MIDX_PROGRESS)
1798 progress = start_sparse_progress(_("Sorting objects by packfile"),
1799 m->num_objects);
1800 display_progress(progress, 0); /* TODO: Measure QSORT() progress */
1801 QSORT(pairs, m->num_objects, compare_pair_pos_vs_id);
1802 stop_progress(&progress);
1803
1804 if (flags & MIDX_PROGRESS)
1805 progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects);
1806 for (i = 0; i < m->num_objects; i++) {
1807 struct object_id oid;
1808 struct pack_entry e;
1809 off_t m_offset, p_offset;
1810
1811 if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
1812 m->packs[pairs[i-1].pack_int_id])
1813 {
1814 close_pack_fd(m->packs[pairs[i-1].pack_int_id]);
1815 close_pack_index(m->packs[pairs[i-1].pack_int_id]);
1816 }
1817
1818 nth_midxed_object_oid(&oid, m, pairs[i].pos);
1819
1820 if (!fill_midx_entry(r, &oid, &e, m)) {
1821 midx_report(_("failed to load pack entry for oid[%d] = %s"),
1822 pairs[i].pos, oid_to_hex(&oid));
1823 continue;
1824 }
1825
1826 if (open_pack_index(e.p)) {
1827 midx_report(_("failed to load pack-index for packfile %s"),
1828 e.p->pack_name);
1829 break;
1830 }
1831
1832 m_offset = e.offset;
1833 p_offset = find_pack_entry_one(oid.hash, e.p);
1834
1835 if (m_offset != p_offset)
1836 midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
1837 pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
1838
1839 midx_display_sparse_progress(progress, i + 1);
1840 }
1841 stop_progress(&progress);
1842
1843 cleanup:
1844 free(pairs);
1845 close_midx(m);
1846
1847 return verify_midx_error;
1848 }
1849
1850 int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags)
1851 {
1852 uint32_t i, *count, result = 0;
1853 struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
1854 struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
1855 struct progress *progress = NULL;
1856
1857 if (!m)
1858 return 0;
1859
1860 CALLOC_ARRAY(count, m->num_packs);
1861
1862 if (flags & MIDX_PROGRESS)
1863 progress = start_delayed_progress(_("Counting referenced objects"),
1864 m->num_objects);
1865 for (i = 0; i < m->num_objects; i++) {
1866 int pack_int_id = nth_midxed_pack_int_id(m, i);
1867 count[pack_int_id]++;
1868 display_progress(progress, i + 1);
1869 }
1870 stop_progress(&progress);
1871
1872 if (flags & MIDX_PROGRESS)
1873 progress = start_delayed_progress(_("Finding and deleting unreferenced packfiles"),
1874 m->num_packs);
1875 for (i = 0; i < m->num_packs; i++) {
1876 char *pack_name;
1877 display_progress(progress, i + 1);
1878
1879 if (count[i])
1880 continue;
1881
1882 if (prepare_midx_pack(r, m, i))
1883 continue;
1884
1885 if (m->packs[i]->pack_keep || m->packs[i]->is_cruft)
1886 continue;
1887
1888 pack_name = xstrdup(m->packs[i]->pack_name);
1889 close_pack(m->packs[i]);
1890
1891 string_list_insert(&packs_to_drop, m->pack_names[i]);
1892 unlink_pack_path(pack_name, 0);
1893 free(pack_name);
1894 }
1895 stop_progress(&progress);
1896
1897 free(count);
1898
1899 if (packs_to_drop.nr)
1900 result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags);
1901
1902 string_list_clear(&packs_to_drop, 0);
1903
1904 return result;
1905 }
1906
1907 struct repack_info {
1908 timestamp_t mtime;
1909 uint32_t referenced_objects;
1910 uint32_t pack_int_id;
1911 };
1912
1913 static int compare_by_mtime(const void *a_, const void *b_)
1914 {
1915 const struct repack_info *a, *b;
1916
1917 a = (const struct repack_info *)a_;
1918 b = (const struct repack_info *)b_;
1919
1920 if (a->mtime < b->mtime)
1921 return -1;
1922 if (a->mtime > b->mtime)
1923 return 1;
1924 return 0;
1925 }
1926
1927 static int fill_included_packs_all(struct repository *r,
1928 struct multi_pack_index *m,
1929 unsigned char *include_pack)
1930 {
1931 uint32_t i, count = 0;
1932 int pack_kept_objects = 0;
1933
1934 repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
1935
1936 for (i = 0; i < m->num_packs; i++) {
1937 if (prepare_midx_pack(r, m, i))
1938 continue;
1939 if (!pack_kept_objects && m->packs[i]->pack_keep)
1940 continue;
1941 if (m->packs[i]->is_cruft)
1942 continue;
1943
1944 include_pack[i] = 1;
1945 count++;
1946 }
1947
1948 return count < 2;
1949 }
1950
1951 static int fill_included_packs_batch(struct repository *r,
1952 struct multi_pack_index *m,
1953 unsigned char *include_pack,
1954 size_t batch_size)
1955 {
1956 uint32_t i, packs_to_repack;
1957 size_t total_size;
1958 struct repack_info *pack_info;
1959 int pack_kept_objects = 0;
1960
1961 CALLOC_ARRAY(pack_info, m->num_packs);
1962
1963 repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
1964
1965 for (i = 0; i < m->num_packs; i++) {
1966 pack_info[i].pack_int_id = i;
1967
1968 if (prepare_midx_pack(r, m, i))
1969 continue;
1970
1971 pack_info[i].mtime = m->packs[i]->mtime;
1972 }
1973
1974 for (i = 0; i < m->num_objects; i++) {
1975 uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
1976 pack_info[pack_int_id].referenced_objects++;
1977 }
1978
1979 QSORT(pack_info, m->num_packs, compare_by_mtime);
1980
1981 total_size = 0;
1982 packs_to_repack = 0;
1983 for (i = 0; total_size < batch_size && i < m->num_packs; i++) {
1984 int pack_int_id = pack_info[i].pack_int_id;
1985 struct packed_git *p = m->packs[pack_int_id];
1986 size_t expected_size;
1987
1988 if (!p)
1989 continue;
1990 if (!pack_kept_objects && p->pack_keep)
1991 continue;
1992 if (p->is_cruft)
1993 continue;
1994 if (open_pack_index(p) || !p->num_objects)
1995 continue;
1996
1997 expected_size = st_mult(p->pack_size,
1998 pack_info[i].referenced_objects);
1999 expected_size /= p->num_objects;
2000
2001 if (expected_size >= batch_size)
2002 continue;
2003
2004 packs_to_repack++;
2005 total_size += expected_size;
2006 include_pack[pack_int_id] = 1;
2007 }
2008
2009 free(pack_info);
2010
2011 if (packs_to_repack < 2)
2012 return 1;
2013
2014 return 0;
2015 }
2016
2017 int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags)
2018 {
2019 int result = 0;
2020 uint32_t i;
2021 unsigned char *include_pack;
2022 struct child_process cmd = CHILD_PROCESS_INIT;
2023 FILE *cmd_in;
2024 struct strbuf base_name = STRBUF_INIT;
2025 struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
2026
2027 /*
2028 * When updating the default for these configuration
2029 * variables in builtin/repack.c, these must be adjusted
2030 * to match.
2031 */
2032 int delta_base_offset = 1;
2033 int use_delta_islands = 0;
2034
2035 if (!m)
2036 return 0;
2037
2038 CALLOC_ARRAY(include_pack, m->num_packs);
2039
2040 if (batch_size) {
2041 if (fill_included_packs_batch(r, m, include_pack, batch_size))
2042 goto cleanup;
2043 } else if (fill_included_packs_all(r, m, include_pack))
2044 goto cleanup;
2045
2046 repo_config_get_bool(r, "repack.usedeltabaseoffset", &delta_base_offset);
2047 repo_config_get_bool(r, "repack.usedeltaislands", &use_delta_islands);
2048
2049 strvec_push(&cmd.args, "pack-objects");
2050
2051 strbuf_addstr(&base_name, object_dir);
2052 strbuf_addstr(&base_name, "/pack/pack");
2053 strvec_push(&cmd.args, base_name.buf);
2054
2055 if (delta_base_offset)
2056 strvec_push(&cmd.args, "--delta-base-offset");
2057 if (use_delta_islands)
2058 strvec_push(&cmd.args, "--delta-islands");
2059
2060 if (flags & MIDX_PROGRESS)
2061 strvec_push(&cmd.args, "--progress");
2062 else
2063 strvec_push(&cmd.args, "-q");
2064
2065 strbuf_release(&base_name);
2066
2067 cmd.git_cmd = 1;
2068 cmd.in = cmd.out = -1;
2069
2070 if (start_command(&cmd)) {
2071 error(_("could not start pack-objects"));
2072 result = 1;
2073 goto cleanup;
2074 }
2075
2076 cmd_in = xfdopen(cmd.in, "w");
2077
2078 for (i = 0; i < m->num_objects; i++) {
2079 struct object_id oid;
2080 uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
2081
2082 if (!include_pack[pack_int_id])
2083 continue;
2084
2085 nth_midxed_object_oid(&oid, m, i);
2086 fprintf(cmd_in, "%s\n", oid_to_hex(&oid));
2087 }
2088 fclose(cmd_in);
2089
2090 if (finish_command(&cmd)) {
2091 error(_("could not finish pack-objects"));
2092 result = 1;
2093 goto cleanup;
2094 }
2095
2096 result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags);
2097
2098 cleanup:
2099 free(include_pack);
2100 return result;
2101 }
git for Debian