?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'en/complete-sparse-checkout'
[alt-git.git] / midx.c
blob1d14661dade4a686b1b18c351c55275b2a388ee3
1 #include "git-compat-util.h"
2 #include "abspath.h"
3 #include "config.h"
4 #include "csum-file.h"
5 #include "dir.h"
6 #include "gettext.h"
7 #include "hex.h"
8 #include "lockfile.h"
9 #include "packfile.h"
10 #include "object-file.h"
11 #include "object-store-ll.h"
12 #include "hash-lookup.h"
13 #include "midx.h"
14 #include "progress.h"
15 #include "trace2.h"
16 #include "run-command.h"
17 #include "repository.h"
18 #include "chunk-format.h"
19 #include "pack.h"
20 #include "pack-bitmap.h"
21 #include "refs.h"
22 #include "revision.h"
23 #include "list-objects.h"
24
25 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
26 #define MIDX_VERSION 1
27 #define MIDX_BYTE_FILE_VERSION 4
28 #define MIDX_BYTE_HASH_VERSION 5
29 #define MIDX_BYTE_NUM_CHUNKS 6
30 #define MIDX_BYTE_NUM_PACKS 8
31 #define MIDX_HEADER_SIZE 12
32 #define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz)
33
34 #define MIDX_CHUNK_ALIGNMENT 4
35 #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
36 #define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
37 #define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
38 #define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
39 #define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
40 #define MIDX_CHUNKID_REVINDEX 0x52494458 /* "RIDX" */
41 #define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
42 #define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
43 #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
44 #define MIDX_LARGE_OFFSET_NEEDED 0x80000000
45
46 #define PACK_EXPIRED UINT_MAX
47
48 const unsigned char *get_midx_checksum(struct multi_pack_index *m)
49 {
50 return m->data + m->data_len - the_hash_algo->rawsz;
51 }
52
53 void get_midx_filename(struct strbuf *out, const char *object_dir)
54 {
55 strbuf_addf(out, "%s/pack/multi-pack-index", object_dir);
56 }
57
58 void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m)
59 {
60 get_midx_filename(out, m->object_dir);
61 strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m)));
62 }
63
64 static int midx_read_oid_fanout(const unsigned char *chunk_start,
65 size_t chunk_size, void *data)
66 {
67 int i;
68 struct multi_pack_index *m = data;
69 m->chunk_oid_fanout = (uint32_t *)chunk_start;
70
71 if (chunk_size != 4 * 256) {
72 error(_("multi-pack-index OID fanout is of the wrong size"));
73 return 1;
74 }
75 for (i = 0; i < 255; i++) {
76 uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
77 uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i+1]);
78
79 if (oid_fanout1 > oid_fanout2) {
80 error(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
81 i, oid_fanout1, oid_fanout2, i + 1);
82 return 1;
83 }
84 }
85 m->num_objects = ntohl(m->chunk_oid_fanout[255]);
86 return 0;
87 }
88
89 static int midx_read_oid_lookup(const unsigned char *chunk_start,
90 size_t chunk_size, void *data)
91 {
92 struct multi_pack_index *m = data;
93 m->chunk_oid_lookup = chunk_start;
94
95 if (chunk_size != st_mult(m->hash_len, m->num_objects)) {
96 error(_("multi-pack-index OID lookup chunk is the wrong size"));
97 return 1;
98 }
99 return 0;
100 }
101
102 static int midx_read_object_offsets(const unsigned char *chunk_start,
103 size_t chunk_size, void *data)
104 {
105 struct multi_pack_index *m = data;
106 m->chunk_object_offsets = chunk_start;
107
108 if (chunk_size != st_mult(m->num_objects, MIDX_CHUNK_OFFSET_WIDTH)) {
109 error(_("multi-pack-index object offset chunk is the wrong size"));
110 return 1;
111 }
112 return 0;
113 }
114
115 struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local)
116 {
117 struct multi_pack_index *m = NULL;
118 int fd;
119 struct stat st;
120 size_t midx_size;
121 void *midx_map = NULL;
122 uint32_t hash_version;
123 struct strbuf midx_name = STRBUF_INIT;
124 uint32_t i;
125 const char *cur_pack_name;
126 struct chunkfile *cf = NULL;
127
128 get_midx_filename(&midx_name, object_dir);
129
130 fd = git_open(midx_name.buf);
131
132 if (fd < 0)
133 goto cleanup_fail;
134 if (fstat(fd, &st)) {
135 error_errno(_("failed to read %s"), midx_name.buf);
136 goto cleanup_fail;
137 }
138
139 midx_size = xsize_t(st.st_size);
140
141 if (midx_size < MIDX_MIN_SIZE) {
142 error(_("multi-pack-index file %s is too small"), midx_name.buf);
143 goto cleanup_fail;
144 }
145
146 strbuf_release(&midx_name);
147
148 midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
149 close(fd);
150
151 FLEX_ALLOC_STR(m, object_dir, object_dir);
152 m->data = midx_map;
153 m->data_len = midx_size;
154 m->local = local;
155
156 m->signature = get_be32(m->data);
157 if (m->signature != MIDX_SIGNATURE)
158 die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
159 m->signature, MIDX_SIGNATURE);
160
161 m->version = m->data[MIDX_BYTE_FILE_VERSION];
162 if (m->version != MIDX_VERSION)
163 die(_("multi-pack-index version %d not recognized"),
164 m->version);
165
166 hash_version = m->data[MIDX_BYTE_HASH_VERSION];
167 if (hash_version != oid_version(the_hash_algo)) {
168 error(_("multi-pack-index hash version %u does not match version %u"),
169 hash_version, oid_version(the_hash_algo));
170 goto cleanup_fail;
171 }
172 m->hash_len = the_hash_algo->rawsz;
173
174 m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
175
176 m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
177
178 cf = init_chunkfile(NULL);
179
180 if (read_table_of_contents(cf, m->data, midx_size,
181 MIDX_HEADER_SIZE, m->num_chunks,
182 MIDX_CHUNK_ALIGNMENT))
183 goto cleanup_fail;
184
185 if (pair_chunk(cf, MIDX_CHUNKID_PACKNAMES, &m->chunk_pack_names, &m->chunk_pack_names_len))
186 die(_("multi-pack-index required pack-name chunk missing or corrupted"));
187 if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m))
188 die(_("multi-pack-index required OID fanout chunk missing or corrupted"));
189 if (read_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, midx_read_oid_lookup, m))
190 die(_("multi-pack-index required OID lookup chunk missing or corrupted"));
191 if (read_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, midx_read_object_offsets, m))
192 die(_("multi-pack-index required object offsets chunk missing or corrupted"));
193
194 pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets,
195 &m->chunk_large_offsets_len);
196
197 if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1))
198 pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex,
199 &m->chunk_revindex_len);
200
201 CALLOC_ARRAY(m->pack_names, m->num_packs);
202 CALLOC_ARRAY(m->packs, m->num_packs);
203
204 cur_pack_name = (const char *)m->chunk_pack_names;
205 for (i = 0; i < m->num_packs; i++) {
206 const char *end;
207 size_t avail = m->chunk_pack_names_len -
208 (cur_pack_name - (const char *)m->chunk_pack_names);
209
210 m->pack_names[i] = cur_pack_name;
211
212 end = memchr(cur_pack_name, '\0', avail);
213 if (!end)
214 die(_("multi-pack-index pack-name chunk is too short"));
215 cur_pack_name = end + 1;
216
217 if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
218 die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
219 m->pack_names[i - 1],
220 m->pack_names[i]);
221 }
222
223 trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs);
224 trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects);
225
226 free_chunkfile(cf);
227 return m;
228
229 cleanup_fail:
230 free(m);
231 strbuf_release(&midx_name);
232 free_chunkfile(cf);
233 if (midx_map)
234 munmap(midx_map, midx_size);
235 if (0 <= fd)
236 close(fd);
237 return NULL;
238 }
239
240 void close_midx(struct multi_pack_index *m)
241 {
242 uint32_t i;
243
244 if (!m)
245 return;
246
247 close_midx(m->next);
248
249 munmap((unsigned char *)m->data, m->data_len);
250
251 for (i = 0; i < m->num_packs; i++) {
252 if (m->packs[i])
253 m->packs[i]->multi_pack_index = 0;
254 }
255 FREE_AND_NULL(m->packs);
256 FREE_AND_NULL(m->pack_names);
257 free(m);
258 }
259
260 int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id)
261 {
262 struct strbuf pack_name = STRBUF_INIT;
263 struct packed_git *p;
264
265 if (pack_int_id >= m->num_packs)
266 die(_("bad pack-int-id: %u (%u total packs)"),
267 pack_int_id, m->num_packs);
268
269 if (m->packs[pack_int_id])
270 return 0;
271
272 strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir,
273 m->pack_names[pack_int_id]);
274
275 p = add_packed_git(pack_name.buf, pack_name.len, m->local);
276 strbuf_release(&pack_name);
277
278 if (!p)
279 return 1;
280
281 p->multi_pack_index = 1;
282 m->packs[pack_int_id] = p;
283 install_packed_git(r, p);
284 list_add_tail(&p->mru, &r->objects->packed_git_mru);
285
286 return 0;
287 }
288
289 int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result)
290 {
291 return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup,
292 the_hash_algo->rawsz, result);
293 }
294
295 struct object_id *nth_midxed_object_oid(struct object_id *oid,
296 struct multi_pack_index *m,
297 uint32_t n)
298 {
299 if (n >= m->num_objects)
300 return NULL;
301
302 oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n));
303 return oid;
304 }
305
306 off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
307 {
308 const unsigned char *offset_data;
309 uint32_t offset32;
310
311 offset_data = m->chunk_object_offsets + (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH;
312 offset32 = get_be32(offset_data + sizeof(uint32_t));
313
314 if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
315 if (sizeof(off_t) < sizeof(uint64_t))
316 die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
317
318 offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
319 if (offset32 >= m->chunk_large_offsets_len / sizeof(uint64_t))
320 die(_("multi-pack-index large offset out of bounds"));
321 return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32);
322 }
323
324 return offset32;
325 }
326
327 uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos)
328 {
329 return get_be32(m->chunk_object_offsets +
330 (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH);
331 }
332
333 int fill_midx_entry(struct repository *r,
334 const struct object_id *oid,
335 struct pack_entry *e,
336 struct multi_pack_index *m)
337 {
338 uint32_t pos;
339 uint32_t pack_int_id;
340 struct packed_git *p;
341
342 if (!bsearch_midx(oid, m, &pos))
343 return 0;
344
345 if (pos >= m->num_objects)
346 return 0;
347
348 pack_int_id = nth_midxed_pack_int_id(m, pos);
349
350 if (prepare_midx_pack(r, m, pack_int_id))
351 return 0;
352 p = m->packs[pack_int_id];
353
354 /*
355 * We are about to tell the caller where they can locate the
356 * requested object. We better make sure the packfile is
357 * still here and can be accessed before supplying that
358 * answer, as it may have been deleted since the MIDX was
359 * loaded!
360 */
361 if (!is_pack_valid(p))
362 return 0;
363
364 if (oidset_size(&p->bad_objects) &&
365 oidset_contains(&p->bad_objects, oid))
366 return 0;
367
368 e->offset = nth_midxed_offset(m, pos);
369 e->p = p;
370
371 return 1;
372 }
373
374 /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
375 static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
376 const char *idx_name)
377 {
378 /* Skip past any initial matching prefix. */
379 while (*idx_name && *idx_name == *idx_or_pack_name) {
380 idx_name++;
381 idx_or_pack_name++;
382 }
383
384 /*
385 * If we didn't match completely, we may have matched "pack-1234." and
386 * be left with "idx" and "pack" respectively, which is also OK. We do
387 * not have to check for "idx" and "idx", because that would have been
388 * a complete match (and in that case these strcmps will be false, but
389 * we'll correctly return 0 from the final strcmp() below.
390 *
391 * Technically this matches "fooidx" and "foopack", but we'd never have
392 * such names in the first place.
393 */
394 if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
395 return 0;
396
397 /*
398 * This not only checks for a complete match, but also orders based on
399 * the first non-identical character, which means our ordering will
400 * match a raw strcmp(). That makes it OK to use this to binary search
401 * a naively-sorted list.
402 */
403 return strcmp(idx_or_pack_name, idx_name);
404 }
405
406 int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
407 {
408 uint32_t first = 0, last = m->num_packs;
409
410 while (first < last) {
411 uint32_t mid = first + (last - first) / 2;
412 const char *current;
413 int cmp;
414
415 current = m->pack_names[mid];
416 cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
417 if (!cmp)
418 return 1;
419 if (cmp > 0) {
420 first = mid + 1;
421 continue;
422 }
423 last = mid;
424 }
425
426 return 0;
427 }
428
429 int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local)
430 {
431 struct multi_pack_index *m;
432 struct multi_pack_index *m_search;
433
434 prepare_repo_settings(r);
435 if (!r->settings.core_multi_pack_index)
436 return 0;
437
438 for (m_search = r->objects->multi_pack_index; m_search; m_search = m_search->next)
439 if (!strcmp(object_dir, m_search->object_dir))
440 return 1;
441
442 m = load_multi_pack_index(object_dir, local);
443
444 if (m) {
445 struct multi_pack_index *mp = r->objects->multi_pack_index;
446 if (mp) {
447 m->next = mp->next;
448 mp->next = m;
449 } else
450 r->objects->multi_pack_index = m;
451 return 1;
452 }
453
454 return 0;
455 }
456
457 static size_t write_midx_header(struct hashfile *f,
458 unsigned char num_chunks,
459 uint32_t num_packs)
460 {
461 hashwrite_be32(f, MIDX_SIGNATURE);
462 hashwrite_u8(f, MIDX_VERSION);
463 hashwrite_u8(f, oid_version(the_hash_algo));
464 hashwrite_u8(f, num_chunks);
465 hashwrite_u8(f, 0); /* unused */
466 hashwrite_be32(f, num_packs);
467
468 return MIDX_HEADER_SIZE;
469 }
470
471 struct pack_info {
472 uint32_t orig_pack_int_id;
473 char *pack_name;
474 struct packed_git *p;
475 unsigned expired : 1;
476 };
477
478 static int pack_info_compare(const void *_a, const void *_b)
479 {
480 struct pack_info *a = (struct pack_info *)_a;
481 struct pack_info *b = (struct pack_info *)_b;
482 return strcmp(a->pack_name, b->pack_name);
483 }
484
485 static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
486 {
487 const char *pack_name = _va;
488 const struct pack_info *compar = _vb;
489
490 return cmp_idx_or_pack_name(pack_name, compar->pack_name);
491 }
492
493 struct write_midx_context {
494 struct pack_info *info;
495 size_t nr;
496 size_t alloc;
497 struct multi_pack_index *m;
498 struct progress *progress;
499 unsigned pack_paths_checked;
500
501 struct pack_midx_entry *entries;
502 size_t entries_nr;
503
504 uint32_t *pack_perm;
505 uint32_t *pack_order;
506 unsigned large_offsets_needed:1;
507 uint32_t num_large_offsets;
508
509 int preferred_pack_idx;
510
511 struct string_list *to_include;
512 };
513
514 static void add_pack_to_midx(const char *full_path, size_t full_path_len,
515 const char *file_name, void *data)
516 {
517 struct write_midx_context *ctx = data;
518
519 if (ends_with(file_name, ".idx")) {
520 display_progress(ctx->progress, ++ctx->pack_paths_checked);
521 /*
522 * Note that at most one of ctx->m and ctx->to_include are set,
523 * so we are testing midx_contains_pack() and
524 * string_list_has_string() independently (guarded by the
525 * appropriate NULL checks).
526 *
527 * We could support passing to_include while reusing an existing
528 * MIDX, but don't currently since the reuse process drags
529 * forward all packs from an existing MIDX (without checking
530 * whether or not they appear in the to_include list).
531 *
532 * If we added support for that, these next two conditional
533 * should be performed independently (likely checking
534 * to_include before the existing MIDX).
535 */
536 if (ctx->m && midx_contains_pack(ctx->m, file_name))
537 return;
538 else if (ctx->to_include &&
539 !string_list_has_string(ctx->to_include, file_name))
540 return;
541
542 ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
543
544 ctx->info[ctx->nr].p = add_packed_git(full_path,
545 full_path_len,
546 0);
547
548 if (!ctx->info[ctx->nr].p) {
549 warning(_("failed to add packfile '%s'"),
550 full_path);
551 return;
552 }
553
554 if (open_pack_index(ctx->info[ctx->nr].p)) {
555 warning(_("failed to open pack-index '%s'"),
556 full_path);
557 close_pack(ctx->info[ctx->nr].p);
558 FREE_AND_NULL(ctx->info[ctx->nr].p);
559 return;
560 }
561
562 ctx->info[ctx->nr].pack_name = xstrdup(file_name);
563 ctx->info[ctx->nr].orig_pack_int_id = ctx->nr;
564 ctx->info[ctx->nr].expired = 0;
565 ctx->nr++;
566 }
567 }
568
569 struct pack_midx_entry {
570 struct object_id oid;
571 uint32_t pack_int_id;
572 time_t pack_mtime;
573 uint64_t offset;
574 unsigned preferred : 1;
575 };
576
577 static int midx_oid_compare(const void *_a, const void *_b)
578 {
579 const struct pack_midx_entry *a = (const struct pack_midx_entry *)_a;
580 const struct pack_midx_entry *b = (const struct pack_midx_entry *)_b;
581 int cmp = oidcmp(&a->oid, &b->oid);
582
583 if (cmp)
584 return cmp;
585
586 /* Sort objects in a preferred pack first when multiple copies exist. */
587 if (a->preferred > b->preferred)
588 return -1;
589 if (a->preferred < b->preferred)
590 return 1;
591
592 if (a->pack_mtime > b->pack_mtime)
593 return -1;
594 else if (a->pack_mtime < b->pack_mtime)
595 return 1;
596
597 return a->pack_int_id - b->pack_int_id;
598 }
599
600 static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
601 struct pack_midx_entry *e,
602 uint32_t pos)
603 {
604 if (pos >= m->num_objects)
605 return 1;
606
607 nth_midxed_object_oid(&e->oid, m, pos);
608 e->pack_int_id = nth_midxed_pack_int_id(m, pos);
609 e->offset = nth_midxed_offset(m, pos);
610
611 /* consider objects in midx to be from "old" packs */
612 e->pack_mtime = 0;
613 return 0;
614 }
615
616 static void fill_pack_entry(uint32_t pack_int_id,
617 struct packed_git *p,
618 uint32_t cur_object,
619 struct pack_midx_entry *entry,
620 int preferred)
621 {
622 if (nth_packed_object_id(&entry->oid, p, cur_object) < 0)
623 die(_("failed to locate object %d in packfile"), cur_object);
624
625 entry->pack_int_id = pack_int_id;
626 entry->pack_mtime = p->mtime;
627
628 entry->offset = nth_packed_object_offset(p, cur_object);
629 entry->preferred = !!preferred;
630 }
631
632 struct midx_fanout {
633 struct pack_midx_entry *entries;
634 size_t nr, alloc;
635 };
636
637 static void midx_fanout_grow(struct midx_fanout *fanout, size_t nr)
638 {
639 if (nr < fanout->nr)
640 BUG("negative growth in midx_fanout_grow() (%"PRIuMAX" < %"PRIuMAX")",
641 (uintmax_t)nr, (uintmax_t)fanout->nr);
642 ALLOC_GROW(fanout->entries, nr, fanout->alloc);
643 }
644
645 static void midx_fanout_sort(struct midx_fanout *fanout)
646 {
647 QSORT(fanout->entries, fanout->nr, midx_oid_compare);
648 }
649
650 static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
651 struct multi_pack_index *m,
652 uint32_t cur_fanout,
653 int preferred_pack)
654 {
655 uint32_t start = 0, end;
656 uint32_t cur_object;
657
658 if (cur_fanout)
659 start = ntohl(m->chunk_oid_fanout[cur_fanout - 1]);
660 end = ntohl(m->chunk_oid_fanout[cur_fanout]);
661
662 for (cur_object = start; cur_object < end; cur_object++) {
663 if ((preferred_pack > -1) &&
664 (preferred_pack == nth_midxed_pack_int_id(m, cur_object))) {
665 /*
666 * Objects from preferred packs are added
667 * separately.
668 */
669 continue;
670 }
671
672 midx_fanout_grow(fanout, fanout->nr + 1);
673 nth_midxed_pack_midx_entry(m,
674 &fanout->entries[fanout->nr],
675 cur_object);
676 fanout->entries[fanout->nr].preferred = 0;
677 fanout->nr++;
678 }
679 }
680
681 static void midx_fanout_add_pack_fanout(struct midx_fanout *fanout,
682 struct pack_info *info,
683 uint32_t cur_pack,
684 int preferred,
685 uint32_t cur_fanout)
686 {
687 struct packed_git *pack = info[cur_pack].p;
688 uint32_t start = 0, end;
689 uint32_t cur_object;
690
691 if (cur_fanout)
692 start = get_pack_fanout(pack, cur_fanout - 1);
693 end = get_pack_fanout(pack, cur_fanout);
694
695 for (cur_object = start; cur_object < end; cur_object++) {
696 midx_fanout_grow(fanout, fanout->nr + 1);
697 fill_pack_entry(cur_pack,
698 info[cur_pack].p,
699 cur_object,
700 &fanout->entries[fanout->nr],
701 preferred);
702 fanout->nr++;
703 }
704 }
705
706 /*
707 * It is possible to artificially get into a state where there are many
708 * duplicate copies of objects. That can create high memory pressure if
709 * we are to create a list of all objects before de-duplication. To reduce
710 * this memory pressure without a significant performance drop, automatically
711 * group objects by the first byte of their object id. Use the IDX fanout
712 * tables to group the data, copy to a local array, then sort.
713 *
714 * Copy only the de-duplicated entries (selected by most-recent modified time
715 * of a packfile containing the object).
716 */
717 static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
718 struct pack_info *info,
719 uint32_t nr_packs,
720 size_t *nr_objects,
721 int preferred_pack)
722 {
723 uint32_t cur_fanout, cur_pack, cur_object;
724 size_t alloc_objects, total_objects = 0;
725 struct midx_fanout fanout = { 0 };
726 struct pack_midx_entry *deduplicated_entries = NULL;
727 uint32_t start_pack = m ? m->num_packs : 0;
728
729 for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++)
730 total_objects = st_add(total_objects,
731 info[cur_pack].p->num_objects);
732
733 /*
734 * As we de-duplicate by fanout value, we expect the fanout
735 * slices to be evenly distributed, with some noise. Hence,
736 * allocate slightly more than one 256th.
737 */
738 alloc_objects = fanout.alloc = total_objects > 3200 ? total_objects / 200 : 16;
739
740 ALLOC_ARRAY(fanout.entries, fanout.alloc);
741 ALLOC_ARRAY(deduplicated_entries, alloc_objects);
742 *nr_objects = 0;
743
744 for (cur_fanout = 0; cur_fanout < 256; cur_fanout++) {
745 fanout.nr = 0;
746
747 if (m)
748 midx_fanout_add_midx_fanout(&fanout, m, cur_fanout,
749 preferred_pack);
750
751 for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) {
752 int preferred = cur_pack == preferred_pack;
753 midx_fanout_add_pack_fanout(&fanout,
754 info, cur_pack,
755 preferred, cur_fanout);
756 }
757
758 if (-1 < preferred_pack && preferred_pack < start_pack)
759 midx_fanout_add_pack_fanout(&fanout, info,
760 preferred_pack, 1,
761 cur_fanout);
762
763 midx_fanout_sort(&fanout);
764
765 /*
766 * The batch is now sorted by OID and then mtime (descending).
767 * Take only the first duplicate.
768 */
769 for (cur_object = 0; cur_object < fanout.nr; cur_object++) {
770 if (cur_object && oideq(&fanout.entries[cur_object - 1].oid,
771 &fanout.entries[cur_object].oid))
772 continue;
773
774 ALLOC_GROW(deduplicated_entries, st_add(*nr_objects, 1),
775 alloc_objects);
776 memcpy(&deduplicated_entries[*nr_objects],
777 &fanout.entries[cur_object],
778 sizeof(struct pack_midx_entry));
779 (*nr_objects)++;
780 }
781 }
782
783 free(fanout.entries);
784 return deduplicated_entries;
785 }
786
787 static int write_midx_pack_names(struct hashfile *f, void *data)
788 {
789 struct write_midx_context *ctx = data;
790 uint32_t i;
791 unsigned char padding[MIDX_CHUNK_ALIGNMENT];
792 size_t written = 0;
793
794 for (i = 0; i < ctx->nr; i++) {
795 size_t writelen;
796
797 if (ctx->info[i].expired)
798 continue;
799
800 if (i && strcmp(ctx->info[i].pack_name, ctx->info[i - 1].pack_name) <= 0)
801 BUG("incorrect pack-file order: %s before %s",
802 ctx->info[i - 1].pack_name,
803 ctx->info[i].pack_name);
804
805 writelen = strlen(ctx->info[i].pack_name) + 1;
806 hashwrite(f, ctx->info[i].pack_name, writelen);
807 written += writelen;
808 }
809
810 /* add padding to be aligned */
811 i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
812 if (i < MIDX_CHUNK_ALIGNMENT) {
813 memset(padding, 0, sizeof(padding));
814 hashwrite(f, padding, i);
815 }
816
817 return 0;
818 }
819
820 static int write_midx_oid_fanout(struct hashfile *f,
821 void *data)
822 {
823 struct write_midx_context *ctx = data;
824 struct pack_midx_entry *list = ctx->entries;
825 struct pack_midx_entry *last = ctx->entries + ctx->entries_nr;
826 uint32_t count = 0;
827 uint32_t i;
828
829 /*
830 * Write the first-level table (the list is sorted,
831 * but we use a 256-entry lookup to be able to avoid
832 * having to do eight extra binary search iterations).
833 */
834 for (i = 0; i < 256; i++) {
835 struct pack_midx_entry *next = list;
836
837 while (next < last && next->oid.hash[0] == i) {
838 count++;
839 next++;
840 }
841
842 hashwrite_be32(f, count);
843 list = next;
844 }
845
846 return 0;
847 }
848
849 static int write_midx_oid_lookup(struct hashfile *f,
850 void *data)
851 {
852 struct write_midx_context *ctx = data;
853 unsigned char hash_len = the_hash_algo->rawsz;
854 struct pack_midx_entry *list = ctx->entries;
855 uint32_t i;
856
857 for (i = 0; i < ctx->entries_nr; i++) {
858 struct pack_midx_entry *obj = list++;
859
860 if (i < ctx->entries_nr - 1) {
861 struct pack_midx_entry *next = list;
862 if (oidcmp(&obj->oid, &next->oid) >= 0)
863 BUG("OIDs not in order: %s >= %s",
864 oid_to_hex(&obj->oid),
865 oid_to_hex(&next->oid));
866 }
867
868 hashwrite(f, obj->oid.hash, (int)hash_len);
869 }
870
871 return 0;
872 }
873
874 static int write_midx_object_offsets(struct hashfile *f,
875 void *data)
876 {
877 struct write_midx_context *ctx = data;
878 struct pack_midx_entry *list = ctx->entries;
879 uint32_t i, nr_large_offset = 0;
880
881 for (i = 0; i < ctx->entries_nr; i++) {
882 struct pack_midx_entry *obj = list++;
883
884 if (ctx->pack_perm[obj->pack_int_id] == PACK_EXPIRED)
885 BUG("object %s is in an expired pack with int-id %d",
886 oid_to_hex(&obj->oid),
887 obj->pack_int_id);
888
889 hashwrite_be32(f, ctx->pack_perm[obj->pack_int_id]);
890
891 if (ctx->large_offsets_needed && obj->offset >> 31)
892 hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++);
893 else if (!ctx->large_offsets_needed && obj->offset >> 32)
894 BUG("object %s requires a large offset (%"PRIx64") but the MIDX is not writing large offsets!",
895 oid_to_hex(&obj->oid),
896 obj->offset);
897 else
898 hashwrite_be32(f, (uint32_t)obj->offset);
899 }
900
901 return 0;
902 }
903
904 static int write_midx_large_offsets(struct hashfile *f,
905 void *data)
906 {
907 struct write_midx_context *ctx = data;
908 struct pack_midx_entry *list = ctx->entries;
909 struct pack_midx_entry *end = ctx->entries + ctx->entries_nr;
910 uint32_t nr_large_offset = ctx->num_large_offsets;
911
912 while (nr_large_offset) {
913 struct pack_midx_entry *obj;
914 uint64_t offset;
915
916 if (list >= end)
917 BUG("too many large-offset objects");
918
919 obj = list++;
920 offset = obj->offset;
921
922 if (!(offset >> 31))
923 continue;
924
925 hashwrite_be64(f, offset);
926
927 nr_large_offset--;
928 }
929
930 return 0;
931 }
932
933 static int write_midx_revindex(struct hashfile *f,
934 void *data)
935 {
936 struct write_midx_context *ctx = data;
937 uint32_t i;
938
939 for (i = 0; i < ctx->entries_nr; i++)
940 hashwrite_be32(f, ctx->pack_order[i]);
941
942 return 0;
943 }
944
945 struct midx_pack_order_data {
946 uint32_t nr;
947 uint32_t pack;
948 off_t offset;
949 };
950
951 static int midx_pack_order_cmp(const void *va, const void *vb)
952 {
953 const struct midx_pack_order_data *a = va, *b = vb;
954 if (a->pack < b->pack)
955 return -1;
956 else if (a->pack > b->pack)
957 return 1;
958 else if (a->offset < b->offset)
959 return -1;
960 else if (a->offset > b->offset)
961 return 1;
962 else
963 return 0;
964 }
965
966 static uint32_t *midx_pack_order(struct write_midx_context *ctx)
967 {
968 struct midx_pack_order_data *data;
969 uint32_t *pack_order;
970 uint32_t i;
971
972 trace2_region_enter("midx", "midx_pack_order", the_repository);
973
974 ALLOC_ARRAY(data, ctx->entries_nr);
975 for (i = 0; i < ctx->entries_nr; i++) {
976 struct pack_midx_entry *e = &ctx->entries[i];
977 data[i].nr = i;
978 data[i].pack = ctx->pack_perm[e->pack_int_id];
979 if (!e->preferred)
980 data[i].pack |= (1U << 31);
981 data[i].offset = e->offset;
982 }
983
984 QSORT(data, ctx->entries_nr, midx_pack_order_cmp);
985
986 ALLOC_ARRAY(pack_order, ctx->entries_nr);
987 for (i = 0; i < ctx->entries_nr; i++)
988 pack_order[i] = data[i].nr;
989 free(data);
990
991 trace2_region_leave("midx", "midx_pack_order", the_repository);
992
993 return pack_order;
994 }
995
996 static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
997 struct write_midx_context *ctx)
998 {
999 struct strbuf buf = STRBUF_INIT;
1000 const char *tmp_file;
1001
1002 trace2_region_enter("midx", "write_midx_reverse_index", the_repository);
1003
1004 strbuf_addf(&buf, "%s-%s.rev", midx_name, hash_to_hex(midx_hash));
1005
1006 tmp_file = write_rev_file_order(NULL, ctx->pack_order, ctx->entries_nr,
1007 midx_hash, WRITE_REV);
1008
1009 if (finalize_object_file(tmp_file, buf.buf))
1010 die(_("cannot store reverse index file"));
1011
1012 strbuf_release(&buf);
1013
1014 trace2_region_leave("midx", "write_midx_reverse_index", the_repository);
1015 }
1016
1017 static void clear_midx_files_ext(const char *object_dir, const char *ext,
1018 unsigned char *keep_hash);
1019
1020 static int midx_checksum_valid(struct multi_pack_index *m)
1021 {
1022 return hashfile_checksum_valid(m->data, m->data_len);
1023 }
1024
1025 static void prepare_midx_packing_data(struct packing_data *pdata,
1026 struct write_midx_context *ctx)
1027 {
1028 uint32_t i;
1029
1030 trace2_region_enter("midx", "prepare_midx_packing_data", the_repository);
1031
1032 memset(pdata, 0, sizeof(struct packing_data));
1033 prepare_packing_data(the_repository, pdata);
1034
1035 for (i = 0; i < ctx->entries_nr; i++) {
1036 struct pack_midx_entry *from = &ctx->entries[ctx->pack_order[i]];
1037 struct object_entry *to = packlist_alloc(pdata, &from->oid);
1038
1039 oe_set_in_pack(pdata, to,
1040 ctx->info[ctx->pack_perm[from->pack_int_id]].p);
1041 }
1042
1043 trace2_region_leave("midx", "prepare_midx_packing_data", the_repository);
1044 }
1045
1046 static int add_ref_to_pending(const char *refname,
1047 const struct object_id *oid,
1048 int flag, void *cb_data)
1049 {
1050 struct rev_info *revs = (struct rev_info*)cb_data;
1051 struct object_id peeled;
1052 struct object *object;
1053
1054 if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) {
1055 warning("symbolic ref is dangling: %s", refname);
1056 return 0;
1057 }
1058
1059 if (!peel_iterated_oid(oid, &peeled))
1060 oid = &peeled;
1061
1062 object = parse_object_or_die(oid, refname);
1063 if (object->type != OBJ_COMMIT)
1064 return 0;
1065
1066 add_pending_object(revs, object, "");
1067 if (bitmap_is_preferred_refname(revs->repo, refname))
1068 object->flags |= NEEDS_BITMAP;
1069 return 0;
1070 }
1071
1072 struct bitmap_commit_cb {
1073 struct commit **commits;
1074 size_t commits_nr, commits_alloc;
1075
1076 struct write_midx_context *ctx;
1077 };
1078
1079 static const struct object_id *bitmap_oid_access(size_t index,
1080 const void *_entries)
1081 {
1082 const struct pack_midx_entry *entries = _entries;
1083 return &entries[index].oid;
1084 }
1085
1086 static void bitmap_show_commit(struct commit *commit, void *_data)
1087 {
1088 struct bitmap_commit_cb *data = _data;
1089 int pos = oid_pos(&commit->object.oid, data->ctx->entries,
1090 data->ctx->entries_nr,
1091 bitmap_oid_access);
1092 if (pos < 0)
1093 return;
1094
1095 ALLOC_GROW(data->commits, data->commits_nr + 1, data->commits_alloc);
1096 data->commits[data->commits_nr++] = commit;
1097 }
1098
1099 static int read_refs_snapshot(const char *refs_snapshot,
1100 struct rev_info *revs)
1101 {
1102 struct strbuf buf = STRBUF_INIT;
1103 struct object_id oid;
1104 FILE *f = xfopen(refs_snapshot, "r");
1105
1106 while (strbuf_getline(&buf, f) != EOF) {
1107 struct object *object;
1108 int preferred = 0;
1109 char *hex = buf.buf;
1110 const char *end = NULL;
1111
1112 if (buf.len && *buf.buf == '+') {
1113 preferred = 1;
1114 hex = &buf.buf[1];
1115 }
1116
1117 if (parse_oid_hex(hex, &oid, &end) < 0)
1118 die(_("could not parse line: %s"), buf.buf);
1119 if (*end)
1120 die(_("malformed line: %s"), buf.buf);
1121
1122 object = parse_object_or_die(&oid, NULL);
1123 if (preferred)
1124 object->flags |= NEEDS_BITMAP;
1125
1126 add_pending_object(revs, object, "");
1127 }
1128
1129 fclose(f);
1130 strbuf_release(&buf);
1131 return 0;
1132 }
1133
1134 static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,
1135 const char *refs_snapshot,
1136 struct write_midx_context *ctx)
1137 {
1138 struct rev_info revs;
1139 struct bitmap_commit_cb cb = {0};
1140
1141 trace2_region_enter("midx", "find_commits_for_midx_bitmap",
1142 the_repository);
1143
1144 cb.ctx = ctx;
1145
1146 repo_init_revisions(the_repository, &revs, NULL);
1147 if (refs_snapshot) {
1148 read_refs_snapshot(refs_snapshot, &revs);
1149 } else {
1150 setup_revisions(0, NULL, &revs, NULL);
1151 for_each_ref(add_ref_to_pending, &revs);
1152 }
1153
1154 /*
1155 * Skipping promisor objects here is intentional, since it only excludes
1156 * them from the list of reachable commits that we want to select from
1157 * when computing the selection of MIDX'd commits to receive bitmaps.
1158 *
1159 * Reachability bitmaps do require that their objects be closed under
1160 * reachability, but fetching any objects missing from promisors at this
1161 * point is too late. But, if one of those objects can be reached from
1162 * an another object that is included in the bitmap, then we will
1163 * complain later that we don't have reachability closure (and fail
1164 * appropriately).
1165 */
1166 fetch_if_missing = 0;
1167 revs.exclude_promisor_objects = 1;
1168
1169 if (prepare_revision_walk(&revs))
1170 die(_("revision walk setup failed"));
1171
1172 traverse_commit_list(&revs, bitmap_show_commit, NULL, &cb);
1173 if (indexed_commits_nr_p)
1174 *indexed_commits_nr_p = cb.commits_nr;
1175
1176 release_revisions(&revs);
1177
1178 trace2_region_leave("midx", "find_commits_for_midx_bitmap",
1179 the_repository);
1180
1181 return cb.commits;
1182 }
1183
1184 static int write_midx_bitmap(const char *midx_name,
1185 const unsigned char *midx_hash,
1186 struct packing_data *pdata,
1187 struct commit **commits,
1188 uint32_t commits_nr,
1189 uint32_t *pack_order,
1190 unsigned flags)
1191 {
1192 int ret, i;
1193 uint16_t options = 0;
1194 struct pack_idx_entry **index;
1195 char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name,
1196 hash_to_hex(midx_hash));
1197
1198 trace2_region_enter("midx", "write_midx_bitmap", the_repository);
1199
1200 if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
1201 options |= BITMAP_OPT_HASH_CACHE;
1202
1203 if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
1204 options |= BITMAP_OPT_LOOKUP_TABLE;
1205
1206 /*
1207 * Build the MIDX-order index based on pdata.objects (which is already
1208 * in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
1209 * this order).
1210 */
1211 ALLOC_ARRAY(index, pdata->nr_objects);
1212 for (i = 0; i < pdata->nr_objects; i++)
1213 index[i] = &pdata->objects[i].idx;
1214
1215 bitmap_writer_show_progress(flags & MIDX_PROGRESS);
1216 bitmap_writer_build_type_index(pdata, index, pdata->nr_objects);
1217
1218 /*
1219 * bitmap_writer_finish expects objects in lex order, but pack_order
1220 * gives us exactly that. use it directly instead of re-sorting the
1221 * array.
1222 *
1223 * This changes the order of objects in 'index' between
1224 * bitmap_writer_build_type_index and bitmap_writer_finish.
1225 *
1226 * The same re-ordering takes place in the single-pack bitmap code via
1227 * write_idx_file(), which is called by finish_tmp_packfile(), which
1228 * happens between bitmap_writer_build_type_index() and
1229 * bitmap_writer_finish().
1230 */
1231 for (i = 0; i < pdata->nr_objects; i++)
1232 index[pack_order[i]] = &pdata->objects[i].idx;
1233
1234 bitmap_writer_select_commits(commits, commits_nr, -1);
1235 ret = bitmap_writer_build(pdata);
1236 if (ret < 0)
1237 goto cleanup;
1238
1239 bitmap_writer_set_checksum(midx_hash);
1240 bitmap_writer_finish(index, pdata->nr_objects, bitmap_name, options);
1241
1242 cleanup:
1243 free(index);
1244 free(bitmap_name);
1245
1246 trace2_region_leave("midx", "write_midx_bitmap", the_repository);
1247
1248 return ret;
1249 }
1250
1251 static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
1252 const char *object_dir)
1253 {
1254 struct multi_pack_index *result = NULL;
1255 struct multi_pack_index *cur;
1256 char *obj_dir_real = real_pathdup(object_dir, 1);
1257 struct strbuf cur_path_real = STRBUF_INIT;
1258
1259 /* Ensure the given object_dir is local, or a known alternate. */
1260 find_odb(r, obj_dir_real);
1261
1262 for (cur = get_multi_pack_index(r); cur; cur = cur->next) {
1263 strbuf_realpath(&cur_path_real, cur->object_dir, 1);
1264 if (!strcmp(obj_dir_real, cur_path_real.buf)) {
1265 result = cur;
1266 goto cleanup;
1267 }
1268 }
1269
1270 cleanup:
1271 free(obj_dir_real);
1272 strbuf_release(&cur_path_real);
1273 return result;
1274 }
1275
1276 static int write_midx_internal(const char *object_dir,
1277 struct string_list *packs_to_include,
1278 struct string_list *packs_to_drop,
1279 const char *preferred_pack_name,
1280 const char *refs_snapshot,
1281 unsigned flags)
1282 {
1283 struct strbuf midx_name = STRBUF_INIT;
1284 unsigned char midx_hash[GIT_MAX_RAWSZ];
1285 uint32_t i;
1286 struct hashfile *f = NULL;
1287 struct lock_file lk;
1288 struct write_midx_context ctx = { 0 };
1289 int pack_name_concat_len = 0;
1290 int dropped_packs = 0;
1291 int result = 0;
1292 struct chunkfile *cf;
1293
1294 trace2_region_enter("midx", "write_midx_internal", the_repository);
1295
1296 get_midx_filename(&midx_name, object_dir);
1297 if (safe_create_leading_directories(midx_name.buf))
1298 die_errno(_("unable to create leading directories of %s"),
1299 midx_name.buf);
1300
1301 if (!packs_to_include) {
1302 /*
1303 * Only reference an existing MIDX when not filtering which
1304 * packs to include, since all packs and objects are copied
1305 * blindly from an existing MIDX if one is present.
1306 */
1307 ctx.m = lookup_multi_pack_index(the_repository, object_dir);
1308 }
1309
1310 if (ctx.m && !midx_checksum_valid(ctx.m)) {
1311 warning(_("ignoring existing multi-pack-index; checksum mismatch"));
1312 ctx.m = NULL;
1313 }
1314
1315 ctx.nr = 0;
1316 ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
1317 ctx.info = NULL;
1318 ALLOC_ARRAY(ctx.info, ctx.alloc);
1319
1320 if (ctx.m) {
1321 for (i = 0; i < ctx.m->num_packs; i++) {
1322 ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc);
1323
1324 ctx.info[ctx.nr].orig_pack_int_id = i;
1325 ctx.info[ctx.nr].pack_name = xstrdup(ctx.m->pack_names[i]);
1326 ctx.info[ctx.nr].p = ctx.m->packs[i];
1327 ctx.info[ctx.nr].expired = 0;
1328
1329 if (flags & MIDX_WRITE_REV_INDEX) {
1330 /*
1331 * If generating a reverse index, need to have
1332 * packed_git's loaded to compare their
1333 * mtimes and object count.
1334 */
1335 if (prepare_midx_pack(the_repository, ctx.m, i)) {
1336 error(_("could not load pack"));
1337 result = 1;
1338 goto cleanup;
1339 }
1340
1341 if (open_pack_index(ctx.m->packs[i]))
1342 die(_("could not open index for %s"),
1343 ctx.m->packs[i]->pack_name);
1344 ctx.info[ctx.nr].p = ctx.m->packs[i];
1345 }
1346
1347 ctx.nr++;
1348 }
1349 }
1350
1351 ctx.pack_paths_checked = 0;
1352 if (flags & MIDX_PROGRESS)
1353 ctx.progress = start_delayed_progress(_("Adding packfiles to multi-pack-index"), 0);
1354 else
1355 ctx.progress = NULL;
1356
1357 ctx.to_include = packs_to_include;
1358
1359 for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
1360 stop_progress(&ctx.progress);
1361
1362 if ((ctx.m && ctx.nr == ctx.m->num_packs) &&
1363 !(packs_to_include || packs_to_drop)) {
1364 struct bitmap_index *bitmap_git;
1365 int bitmap_exists;
1366 int want_bitmap = flags & MIDX_WRITE_BITMAP;
1367
1368 bitmap_git = prepare_midx_bitmap_git(ctx.m);
1369 bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
1370 free_bitmap_index(bitmap_git);
1371
1372 if (bitmap_exists || !want_bitmap) {
1373 /*
1374 * The correct MIDX already exists, and so does a
1375 * corresponding bitmap (or one wasn't requested).
1376 */
1377 if (!want_bitmap)
1378 clear_midx_files_ext(object_dir, ".bitmap",
1379 NULL);
1380 goto cleanup;
1381 }
1382 }
1383
1384 if (preferred_pack_name) {
1385 ctx.preferred_pack_idx = -1;
1386
1387 for (i = 0; i < ctx.nr; i++) {
1388 if (!cmp_idx_or_pack_name(preferred_pack_name,
1389 ctx.info[i].pack_name)) {
1390 ctx.preferred_pack_idx = i;
1391 break;
1392 }
1393 }
1394
1395 if (ctx.preferred_pack_idx == -1)
1396 warning(_("unknown preferred pack: '%s'"),
1397 preferred_pack_name);
1398 } else if (ctx.nr &&
1399 (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) {
1400 struct packed_git *oldest = ctx.info[ctx.preferred_pack_idx].p;
1401 ctx.preferred_pack_idx = 0;
1402
1403 if (packs_to_drop && packs_to_drop->nr)
1404 BUG("cannot write a MIDX bitmap during expiration");
1405
1406 /*
1407 * set a preferred pack when writing a bitmap to ensure that
1408 * the pack from which the first object is selected in pseudo
1409 * pack-order has all of its objects selected from that pack
1410 * (and not another pack containing a duplicate)
1411 */
1412 for (i = 1; i < ctx.nr; i++) {
1413 struct packed_git *p = ctx.info[i].p;
1414
1415 if (!oldest->num_objects || p->mtime < oldest->mtime) {
1416 oldest = p;
1417 ctx.preferred_pack_idx = i;
1418 }
1419 }
1420
1421 if (!oldest->num_objects) {
1422 /*
1423 * If all packs are empty; unset the preferred index.
1424 * This is acceptable since there will be no duplicate
1425 * objects to resolve, so the preferred value doesn't
1426 * matter.
1427 */
1428 ctx.preferred_pack_idx = -1;
1429 }
1430 } else {
1431 /*
1432 * otherwise don't mark any pack as preferred to avoid
1433 * interfering with expiration logic below
1434 */
1435 ctx.preferred_pack_idx = -1;
1436 }
1437
1438 if (ctx.preferred_pack_idx > -1) {
1439 struct packed_git *preferred = ctx.info[ctx.preferred_pack_idx].p;
1440 if (!preferred->num_objects) {
1441 error(_("cannot select preferred pack %s with no objects"),
1442 preferred->pack_name);
1443 result = 1;
1444 goto cleanup;
1445 }
1446 }
1447
1448 ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr,
1449 ctx.preferred_pack_idx);
1450
1451 ctx.large_offsets_needed = 0;
1452 for (i = 0; i < ctx.entries_nr; i++) {
1453 if (ctx.entries[i].offset > 0x7fffffff)
1454 ctx.num_large_offsets++;
1455 if (ctx.entries[i].offset > 0xffffffff)
1456 ctx.large_offsets_needed = 1;
1457 }
1458
1459 QSORT(ctx.info, ctx.nr, pack_info_compare);
1460
1461 if (packs_to_drop && packs_to_drop->nr) {
1462 int drop_index = 0;
1463 int missing_drops = 0;
1464
1465 for (i = 0; i < ctx.nr && drop_index < packs_to_drop->nr; i++) {
1466 int cmp = strcmp(ctx.info[i].pack_name,
1467 packs_to_drop->items[drop_index].string);
1468
1469 if (!cmp) {
1470 drop_index++;
1471 ctx.info[i].expired = 1;
1472 } else if (cmp > 0) {
1473 error(_("did not see pack-file %s to drop"),
1474 packs_to_drop->items[drop_index].string);
1475 drop_index++;
1476 missing_drops++;
1477 i--;
1478 } else {
1479 ctx.info[i].expired = 0;
1480 }
1481 }
1482
1483 if (missing_drops) {
1484 result = 1;
1485 goto cleanup;
1486 }
1487 }
1488
1489 /*
1490 * pack_perm stores a permutation between pack-int-ids from the
1491 * previous multi-pack-index to the new one we are writing:
1492 *
1493 * pack_perm[old_id] = new_id
1494 */
1495 ALLOC_ARRAY(ctx.pack_perm, ctx.nr);
1496 for (i = 0; i < ctx.nr; i++) {
1497 if (ctx.info[i].expired) {
1498 dropped_packs++;
1499 ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
1500 } else {
1501 ctx.pack_perm[ctx.info[i].orig_pack_int_id] = i - dropped_packs;
1502 }
1503 }
1504
1505 for (i = 0; i < ctx.nr; i++) {
1506 if (!ctx.info[i].expired)
1507 pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
1508 }
1509
1510 /* Check that the preferred pack wasn't expired (if given). */
1511 if (preferred_pack_name) {
1512 struct pack_info *preferred = bsearch(preferred_pack_name,
1513 ctx.info, ctx.nr,
1514 sizeof(*ctx.info),
1515 idx_or_pack_name_cmp);
1516 if (preferred) {
1517 uint32_t perm = ctx.pack_perm[preferred->orig_pack_int_id];
1518 if (perm == PACK_EXPIRED)
1519 warning(_("preferred pack '%s' is expired"),
1520 preferred_pack_name);
1521 }
1522 }
1523
1524 if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
1525 pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
1526 (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
1527
1528 hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR);
1529 f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
1530
1531 if (ctx.nr - dropped_packs == 0) {
1532 error(_("no pack files to index."));
1533 result = 1;
1534 goto cleanup;
1535 }
1536
1537 if (!ctx.entries_nr) {
1538 if (flags & MIDX_WRITE_BITMAP)
1539 warning(_("refusing to write multi-pack .bitmap without any objects"));
1540 flags &= ~(MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP);
1541 }
1542
1543 cf = init_chunkfile(f);
1544
1545 add_chunk(cf, MIDX_CHUNKID_PACKNAMES, pack_name_concat_len,
1546 write_midx_pack_names);
1547 add_chunk(cf, MIDX_CHUNKID_OIDFANOUT, MIDX_CHUNK_FANOUT_SIZE,
1548 write_midx_oid_fanout);
1549 add_chunk(cf, MIDX_CHUNKID_OIDLOOKUP,
1550 st_mult(ctx.entries_nr, the_hash_algo->rawsz),
1551 write_midx_oid_lookup);
1552 add_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS,
1553 st_mult(ctx.entries_nr, MIDX_CHUNK_OFFSET_WIDTH),
1554 write_midx_object_offsets);
1555
1556 if (ctx.large_offsets_needed)
1557 add_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS,
1558 st_mult(ctx.num_large_offsets,
1559 MIDX_CHUNK_LARGE_OFFSET_WIDTH),
1560 write_midx_large_offsets);
1561
1562 if (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP)) {
1563 ctx.pack_order = midx_pack_order(&ctx);
1564 add_chunk(cf, MIDX_CHUNKID_REVINDEX,
1565 st_mult(ctx.entries_nr, sizeof(uint32_t)),
1566 write_midx_revindex);
1567 }
1568
1569 write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs);
1570 write_chunkfile(cf, &ctx);
1571
1572 finalize_hashfile(f, midx_hash, FSYNC_COMPONENT_PACK_METADATA,
1573 CSUM_FSYNC | CSUM_HASH_IN_STREAM);
1574 free_chunkfile(cf);
1575
1576 if (flags & MIDX_WRITE_REV_INDEX &&
1577 git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
1578 write_midx_reverse_index(midx_name.buf, midx_hash, &ctx);
1579
1580 if (flags & MIDX_WRITE_BITMAP) {
1581 struct packing_data pdata;
1582 struct commit **commits;
1583 uint32_t commits_nr;
1584
1585 if (!ctx.entries_nr)
1586 BUG("cannot write a bitmap without any objects");
1587
1588 prepare_midx_packing_data(&pdata, &ctx);
1589
1590 commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, &ctx);
1591
1592 /*
1593 * The previous steps translated the information from
1594 * 'entries' into information suitable for constructing
1595 * bitmaps. We no longer need that array, so clear it to
1596 * reduce memory pressure.
1597 */
1598 FREE_AND_NULL(ctx.entries);
1599 ctx.entries_nr = 0;
1600
1601 if (write_midx_bitmap(midx_name.buf, midx_hash, &pdata,
1602 commits, commits_nr, ctx.pack_order,
1603 flags) < 0) {
1604 error(_("could not write multi-pack bitmap"));
1605 result = 1;
1606 goto cleanup;
1607 }
1608 }
1609 /*
1610 * NOTE: Do not use ctx.entries beyond this point, since it might
1611 * have been freed in the previous if block.
1612 */
1613
1614 if (ctx.m)
1615 close_object_store(the_repository->objects);
1616
1617 if (commit_lock_file(&lk) < 0)
1618 die_errno(_("could not write multi-pack-index"));
1619
1620 clear_midx_files_ext(object_dir, ".bitmap", midx_hash);
1621 clear_midx_files_ext(object_dir, ".rev", midx_hash);
1622
1623 cleanup:
1624 for (i = 0; i < ctx.nr; i++) {
1625 if (ctx.info[i].p) {
1626 close_pack(ctx.info[i].p);
1627 free(ctx.info[i].p);
1628 }
1629 free(ctx.info[i].pack_name);
1630 }
1631
1632 free(ctx.info);
1633 free(ctx.entries);
1634 free(ctx.pack_perm);
1635 free(ctx.pack_order);
1636 strbuf_release(&midx_name);
1637
1638 trace2_region_leave("midx", "write_midx_internal", the_repository);
1639
1640 return result;
1641 }
1642
1643 int write_midx_file(const char *object_dir,
1644 const char *preferred_pack_name,
1645 const char *refs_snapshot,
1646 unsigned flags)
1647 {
1648 return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name,
1649 refs_snapshot, flags);
1650 }
1651
1652 int write_midx_file_only(const char *object_dir,
1653 struct string_list *packs_to_include,
1654 const char *preferred_pack_name,
1655 const char *refs_snapshot,
1656 unsigned flags)
1657 {
1658 return write_midx_internal(object_dir, packs_to_include, NULL,
1659 preferred_pack_name, refs_snapshot, flags);
1660 }
1661
1662 struct clear_midx_data {
1663 char *keep;
1664 const char *ext;
1665 };
1666
1667 static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED,
1668 const char *file_name, void *_data)
1669 {
1670 struct clear_midx_data *data = _data;
1671
1672 if (!(starts_with(file_name, "multi-pack-index-") &&
1673 ends_with(file_name, data->ext)))
1674 return;
1675 if (data->keep && !strcmp(data->keep, file_name))
1676 return;
1677
1678 if (unlink(full_path))
1679 die_errno(_("failed to remove %s"), full_path);
1680 }
1681
1682 static void clear_midx_files_ext(const char *object_dir, const char *ext,
1683 unsigned char *keep_hash)
1684 {
1685 struct clear_midx_data data;
1686 memset(&data, 0, sizeof(struct clear_midx_data));
1687
1688 if (keep_hash)
1689 data.keep = xstrfmt("multi-pack-index-%s%s",
1690 hash_to_hex(keep_hash), ext);
1691 data.ext = ext;
1692
1693 for_each_file_in_pack_dir(object_dir,
1694 clear_midx_file_ext,
1695 &data);
1696
1697 free(data.keep);
1698 }
1699
1700 void clear_midx_file(struct repository *r)
1701 {
1702 struct strbuf midx = STRBUF_INIT;
1703
1704 get_midx_filename(&midx, r->objects->odb->path);
1705
1706 if (r->objects && r->objects->multi_pack_index) {
1707 close_midx(r->objects->multi_pack_index);
1708 r->objects->multi_pack_index = NULL;
1709 }
1710
1711 if (remove_path(midx.buf))
1712 die(_("failed to clear multi-pack-index at %s"), midx.buf);
1713
1714 clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL);
1715 clear_midx_files_ext(r->objects->odb->path, ".rev", NULL);
1716
1717 strbuf_release(&midx);
1718 }
1719
1720 static int verify_midx_error;
1721
1722 __attribute__((format (printf, 1, 2)))
1723 static void midx_report(const char *fmt, ...)
1724 {
1725 va_list ap;
1726 verify_midx_error = 1;
1727 va_start(ap, fmt);
1728 vfprintf(stderr, fmt, ap);
1729 fprintf(stderr, "\n");
1730 va_end(ap);
1731 }
1732
1733 struct pair_pos_vs_id
1734 {
1735 uint32_t pos;
1736 uint32_t pack_int_id;
1737 };
1738
1739 static int compare_pair_pos_vs_id(const void *_a, const void *_b)
1740 {
1741 struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a;
1742 struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b;
1743
1744 return b->pack_int_id - a->pack_int_id;
1745 }
1746
1747 /*
1748 * Limit calls to display_progress() for performance reasons.
1749 * The interval here was arbitrarily chosen.
1750 */
1751 #define SPARSE_PROGRESS_INTERVAL (1 << 12)
1752 #define midx_display_sparse_progress(progress, n) \
1753 do { \
1754 uint64_t _n = (n); \
1755 if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \
1756 display_progress(progress, _n); \
1757 } while (0)
1758
1759 int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags)
1760 {
1761 struct pair_pos_vs_id *pairs = NULL;
1762 uint32_t i;
1763 struct progress *progress = NULL;
1764 struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
1765 verify_midx_error = 0;
1766
1767 if (!m) {
1768 int result = 0;
1769 struct stat sb;
1770 struct strbuf filename = STRBUF_INIT;
1771
1772 get_midx_filename(&filename, object_dir);
1773
1774 if (!stat(filename.buf, &sb)) {
1775 error(_("multi-pack-index file exists, but failed to parse"));
1776 result = 1;
1777 }
1778 strbuf_release(&filename);
1779 return result;
1780 }
1781
1782 if (!midx_checksum_valid(m))
1783 midx_report(_("incorrect checksum"));
1784
1785 if (flags & MIDX_PROGRESS)
1786 progress = start_delayed_progress(_("Looking for referenced packfiles"),
1787 m->num_packs);
1788 for (i = 0; i < m->num_packs; i++) {
1789 if (prepare_midx_pack(r, m, i))
1790 midx_report("failed to load pack in position %d", i);
1791
1792 display_progress(progress, i + 1);
1793 }
1794 stop_progress(&progress);
1795
1796 if (m->num_objects == 0) {
1797 midx_report(_("the midx contains no oid"));
1798 /*
1799 * Remaining tests assume that we have objects, so we can
1800 * return here.
1801 */
1802 goto cleanup;
1803 }
1804
1805 if (flags & MIDX_PROGRESS)
1806 progress = start_sparse_progress(_("Verifying OID order in multi-pack-index"),
1807 m->num_objects - 1);
1808 for (i = 0; i < m->num_objects - 1; i++) {
1809 struct object_id oid1, oid2;
1810
1811 nth_midxed_object_oid(&oid1, m, i);
1812 nth_midxed_object_oid(&oid2, m, i + 1);
1813
1814 if (oidcmp(&oid1, &oid2) >= 0)
1815 midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
1816 i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
1817
1818 midx_display_sparse_progress(progress, i + 1);
1819 }
1820 stop_progress(&progress);
1821
1822 /*
1823 * Create an array mapping each object to its packfile id. Sort it
1824 * to group the objects by packfile. Use this permutation to visit
1825 * each of the objects and only require 1 packfile to be open at a
1826 * time.
1827 */
1828 ALLOC_ARRAY(pairs, m->num_objects);
1829 for (i = 0; i < m->num_objects; i++) {
1830 pairs[i].pos = i;
1831 pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
1832 }
1833
1834 if (flags & MIDX_PROGRESS)
1835 progress = start_sparse_progress(_("Sorting objects by packfile"),
1836 m->num_objects);
1837 display_progress(progress, 0); /* TODO: Measure QSORT() progress */
1838 QSORT(pairs, m->num_objects, compare_pair_pos_vs_id);
1839 stop_progress(&progress);
1840
1841 if (flags & MIDX_PROGRESS)
1842 progress = start_sparse_progress(_("Verifying object offsets"), m->num_objects);
1843 for (i = 0; i < m->num_objects; i++) {
1844 struct object_id oid;
1845 struct pack_entry e;
1846 off_t m_offset, p_offset;
1847
1848 if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
1849 m->packs[pairs[i-1].pack_int_id])
1850 {
1851 close_pack_fd(m->packs[pairs[i-1].pack_int_id]);
1852 close_pack_index(m->packs[pairs[i-1].pack_int_id]);
1853 }
1854
1855 nth_midxed_object_oid(&oid, m, pairs[i].pos);
1856
1857 if (!fill_midx_entry(r, &oid, &e, m)) {
1858 midx_report(_("failed to load pack entry for oid[%d] = %s"),
1859 pairs[i].pos, oid_to_hex(&oid));
1860 continue;
1861 }
1862
1863 if (open_pack_index(e.p)) {
1864 midx_report(_("failed to load pack-index for packfile %s"),
1865 e.p->pack_name);
1866 break;
1867 }
1868
1869 m_offset = e.offset;
1870 p_offset = find_pack_entry_one(oid.hash, e.p);
1871
1872 if (m_offset != p_offset)
1873 midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
1874 pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
1875
1876 midx_display_sparse_progress(progress, i + 1);
1877 }
1878 stop_progress(&progress);
1879
1880 cleanup:
1881 free(pairs);
1882 close_midx(m);
1883
1884 return verify_midx_error;
1885 }
1886
1887 int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags)
1888 {
1889 uint32_t i, *count, result = 0;
1890 struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
1891 struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
1892 struct progress *progress = NULL;
1893
1894 if (!m)
1895 return 0;
1896
1897 CALLOC_ARRAY(count, m->num_packs);
1898
1899 if (flags & MIDX_PROGRESS)
1900 progress = start_delayed_progress(_("Counting referenced objects"),
1901 m->num_objects);
1902 for (i = 0; i < m->num_objects; i++) {
1903 int pack_int_id = nth_midxed_pack_int_id(m, i);
1904 count[pack_int_id]++;
1905 display_progress(progress, i + 1);
1906 }
1907 stop_progress(&progress);
1908
1909 if (flags & MIDX_PROGRESS)
1910 progress = start_delayed_progress(_("Finding and deleting unreferenced packfiles"),
1911 m->num_packs);
1912 for (i = 0; i < m->num_packs; i++) {
1913 char *pack_name;
1914 display_progress(progress, i + 1);
1915
1916 if (count[i])
1917 continue;
1918
1919 if (prepare_midx_pack(r, m, i))
1920 continue;
1921
1922 if (m->packs[i]->pack_keep || m->packs[i]->is_cruft)
1923 continue;
1924
1925 pack_name = xstrdup(m->packs[i]->pack_name);
1926 close_pack(m->packs[i]);
1927
1928 string_list_insert(&packs_to_drop, m->pack_names[i]);
1929 unlink_pack_path(pack_name, 0);
1930 free(pack_name);
1931 }
1932 stop_progress(&progress);
1933
1934 free(count);
1935
1936 if (packs_to_drop.nr)
1937 result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags);
1938
1939 string_list_clear(&packs_to_drop, 0);
1940
1941 return result;
1942 }
1943
1944 struct repack_info {
1945 timestamp_t mtime;
1946 uint32_t referenced_objects;
1947 uint32_t pack_int_id;
1948 };
1949
1950 static int compare_by_mtime(const void *a_, const void *b_)
1951 {
1952 const struct repack_info *a, *b;
1953
1954 a = (const struct repack_info *)a_;
1955 b = (const struct repack_info *)b_;
1956
1957 if (a->mtime < b->mtime)
1958 return -1;
1959 if (a->mtime > b->mtime)
1960 return 1;
1961 return 0;
1962 }
1963
1964 static int fill_included_packs_all(struct repository *r,
1965 struct multi_pack_index *m,
1966 unsigned char *include_pack)
1967 {
1968 uint32_t i, count = 0;
1969 int pack_kept_objects = 0;
1970
1971 repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
1972
1973 for (i = 0; i < m->num_packs; i++) {
1974 if (prepare_midx_pack(r, m, i))
1975 continue;
1976 if (!pack_kept_objects && m->packs[i]->pack_keep)
1977 continue;
1978 if (m->packs[i]->is_cruft)
1979 continue;
1980
1981 include_pack[i] = 1;
1982 count++;
1983 }
1984
1985 return count < 2;
1986 }
1987
1988 static int fill_included_packs_batch(struct repository *r,
1989 struct multi_pack_index *m,
1990 unsigned char *include_pack,
1991 size_t batch_size)
1992 {
1993 uint32_t i, packs_to_repack;
1994 size_t total_size;
1995 struct repack_info *pack_info;
1996 int pack_kept_objects = 0;
1997
1998 CALLOC_ARRAY(pack_info, m->num_packs);
1999
2000 repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects);
2001
2002 for (i = 0; i < m->num_packs; i++) {
2003 pack_info[i].pack_int_id = i;
2004
2005 if (prepare_midx_pack(r, m, i))
2006 continue;
2007
2008 pack_info[i].mtime = m->packs[i]->mtime;
2009 }
2010
2011 for (i = 0; i < m->num_objects; i++) {
2012 uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
2013 pack_info[pack_int_id].referenced_objects++;
2014 }
2015
2016 QSORT(pack_info, m->num_packs, compare_by_mtime);
2017
2018 total_size = 0;
2019 packs_to_repack = 0;
2020 for (i = 0; total_size < batch_size && i < m->num_packs; i++) {
2021 int pack_int_id = pack_info[i].pack_int_id;
2022 struct packed_git *p = m->packs[pack_int_id];
2023 size_t expected_size;
2024
2025 if (!p)
2026 continue;
2027 if (!pack_kept_objects && p->pack_keep)
2028 continue;
2029 if (p->is_cruft)
2030 continue;
2031 if (open_pack_index(p) || !p->num_objects)
2032 continue;
2033
2034 expected_size = st_mult(p->pack_size,
2035 pack_info[i].referenced_objects);
2036 expected_size /= p->num_objects;
2037
2038 if (expected_size >= batch_size)
2039 continue;
2040
2041 packs_to_repack++;
2042 total_size += expected_size;
2043 include_pack[pack_int_id] = 1;
2044 }
2045
2046 free(pack_info);
2047
2048 if (packs_to_repack < 2)
2049 return 1;
2050
2051 return 0;
2052 }
2053
2054 int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags)
2055 {
2056 int result = 0;
2057 uint32_t i;
2058 unsigned char *include_pack;
2059 struct child_process cmd = CHILD_PROCESS_INIT;
2060 FILE *cmd_in;
2061 struct strbuf base_name = STRBUF_INIT;
2062 struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir);
2063
2064 /*
2065 * When updating the default for these configuration
2066 * variables in builtin/repack.c, these must be adjusted
2067 * to match.
2068 */
2069 int delta_base_offset = 1;
2070 int use_delta_islands = 0;
2071
2072 if (!m)
2073 return 0;
2074
2075 CALLOC_ARRAY(include_pack, m->num_packs);
2076
2077 if (batch_size) {
2078 if (fill_included_packs_batch(r, m, include_pack, batch_size))
2079 goto cleanup;
2080 } else if (fill_included_packs_all(r, m, include_pack))
2081 goto cleanup;
2082
2083 repo_config_get_bool(r, "repack.usedeltabaseoffset", &delta_base_offset);
2084 repo_config_get_bool(r, "repack.usedeltaislands", &use_delta_islands);
2085
2086 strvec_push(&cmd.args, "pack-objects");
2087
2088 strbuf_addstr(&base_name, object_dir);
2089 strbuf_addstr(&base_name, "/pack/pack");
2090 strvec_push(&cmd.args, base_name.buf);
2091
2092 if (delta_base_offset)
2093 strvec_push(&cmd.args, "--delta-base-offset");
2094 if (use_delta_islands)
2095 strvec_push(&cmd.args, "--delta-islands");
2096
2097 if (flags & MIDX_PROGRESS)
2098 strvec_push(&cmd.args, "--progress");
2099 else
2100 strvec_push(&cmd.args, "-q");
2101
2102 strbuf_release(&base_name);
2103
2104 cmd.git_cmd = 1;
2105 cmd.in = cmd.out = -1;
2106
2107 if (start_command(&cmd)) {
2108 error(_("could not start pack-objects"));
2109 result = 1;
2110 goto cleanup;
2111 }
2112
2113 cmd_in = xfdopen(cmd.in, "w");
2114
2115 for (i = 0; i < m->num_objects; i++) {
2116 struct object_id oid;
2117 uint32_t pack_int_id = nth_midxed_pack_int_id(m, i);
2118
2119 if (!include_pack[pack_int_id])
2120 continue;
2121
2122 nth_midxed_object_oid(&oid, m, i);
2123 fprintf(cmd_in, "%s\n", oid_to_hex(&oid));
2124 }
2125 fclose(cmd_in);
2126
2127 if (finish_command(&cmd)) {
2128 error(_("could not finish pack-objects"));
2129 result = 1;
2130 goto cleanup;
2131 }
2132
2133 result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags);
2134
2135 cleanup:
2136 free(include_pack);
2137 return result;
2138 }
Alternate source for your git.git needs