4 #include "git-compat-util.h"
12 #include "sha1-lookup.h"
13 #include "commit-graph.h"
14 #include "object-store.h"
17 #include "replace-object.h"
20 #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
21 #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
22 #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
23 #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
24 #define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
25 #define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
27 #define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
29 #define GRAPH_VERSION_1 0x1
30 #define GRAPH_VERSION GRAPH_VERSION_1
32 #define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
33 #define GRAPH_EDGE_LAST_MASK 0x7fffffff
34 #define GRAPH_PARENT_NONE 0x70000000
36 #define GRAPH_LAST_EDGE 0x80000000
38 #define GRAPH_HEADER_SIZE 8
39 #define GRAPH_FANOUT_SIZE (4 * 256)
40 #define GRAPH_CHUNKLOOKUP_WIDTH 12
41 #define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
42 + GRAPH_FANOUT_SIZE + the_hash_algo->rawsz)
44 char *get_commit_graph_filename(const char *obj_dir
)
46 char *filename
= xstrfmt("%s/info/commit-graph", obj_dir
);
47 char *normalized
= xmalloc(strlen(filename
) + 1);
48 normalize_path_copy(normalized
, filename
);
53 static char *get_split_graph_filename(const char *obj_dir
,
56 char *filename
= xstrfmt("%s/info/commit-graphs/graph-%s.graph",
59 char *normalized
= xmalloc(strlen(filename
) + 1);
60 normalize_path_copy(normalized
, filename
);
65 static char *get_chain_filename(const char *obj_dir
)
67 return xstrfmt("%s/info/commit-graphs/commit-graph-chain", obj_dir
);
70 static uint8_t oid_version(void)
75 static struct commit_graph
*alloc_commit_graph(void)
77 struct commit_graph
*g
= xcalloc(1, sizeof(*g
));
83 extern int read_replace_refs
;
85 static int commit_graph_compatible(struct repository
*r
)
90 if (read_replace_refs
) {
91 prepare_replace_object(r
);
92 if (hashmap_get_size(&r
->objects
->replace_map
->map
))
96 prepare_commit_graft(r
);
97 if (r
->parsed_objects
&& r
->parsed_objects
->grafts_nr
)
99 if (is_repository_shallow(r
))
105 int open_commit_graph(const char *graph_file
, int *fd
, struct stat
*st
)
107 *fd
= git_open(graph_file
);
110 if (fstat(*fd
, st
)) {
117 struct commit_graph
*load_commit_graph_one_fd_st(int fd
, struct stat
*st
)
121 struct commit_graph
*ret
;
123 graph_size
= xsize_t(st
->st_size
);
125 if (graph_size
< GRAPH_MIN_SIZE
) {
127 error(_("commit-graph file is too small"));
130 graph_map
= xmmap(NULL
, graph_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
131 ret
= parse_commit_graph(graph_map
, fd
, graph_size
);
134 munmap(graph_map
, graph_size
);
141 static int verify_commit_graph_lite(struct commit_graph
*g
)
144 * Basic validation shared between parse_commit_graph()
145 * which'll be called every time the graph is used, and the
146 * much more expensive verify_commit_graph() used by
147 * "commit-graph verify".
149 * There should only be very basic checks here to ensure that
150 * we don't e.g. segfault in fill_commit_in_graph(), but
151 * because this is a very hot codepath nothing that e.g. loops
152 * over g->num_commits, or runs a checksum on the commit-graph
155 if (!g
->chunk_oid_fanout
) {
156 error("commit-graph is missing the OID Fanout chunk");
159 if (!g
->chunk_oid_lookup
) {
160 error("commit-graph is missing the OID Lookup chunk");
163 if (!g
->chunk_commit_data
) {
164 error("commit-graph is missing the Commit Data chunk");
171 struct commit_graph
*parse_commit_graph(void *graph_map
, int fd
,
174 const unsigned char *data
, *chunk_lookup
;
176 struct commit_graph
*graph
;
177 uint64_t last_chunk_offset
;
178 uint32_t last_chunk_id
;
179 uint32_t graph_signature
;
180 unsigned char graph_version
, hash_version
;
185 if (graph_size
< GRAPH_MIN_SIZE
)
188 data
= (const unsigned char *)graph_map
;
190 graph_signature
= get_be32(data
);
191 if (graph_signature
!= GRAPH_SIGNATURE
) {
192 error(_("commit-graph signature %X does not match signature %X"),
193 graph_signature
, GRAPH_SIGNATURE
);
197 graph_version
= *(unsigned char*)(data
+ 4);
198 if (graph_version
!= GRAPH_VERSION
) {
199 error(_("commit-graph version %X does not match version %X"),
200 graph_version
, GRAPH_VERSION
);
204 hash_version
= *(unsigned char*)(data
+ 5);
205 if (hash_version
!= oid_version()) {
206 error(_("commit-graph hash version %X does not match version %X"),
207 hash_version
, oid_version());
211 graph
= alloc_commit_graph();
213 graph
->hash_len
= the_hash_algo
->rawsz
;
214 graph
->num_chunks
= *(unsigned char*)(data
+ 6);
215 graph
->graph_fd
= fd
;
216 graph
->data
= graph_map
;
217 graph
->data_len
= graph_size
;
220 last_chunk_offset
= 8;
221 chunk_lookup
= data
+ 8;
222 for (i
= 0; i
< graph
->num_chunks
; i
++) {
224 uint64_t chunk_offset
;
225 int chunk_repeated
= 0;
227 if (data
+ graph_size
- chunk_lookup
<
228 GRAPH_CHUNKLOOKUP_WIDTH
) {
229 error(_("commit-graph chunk lookup table entry missing; file may be incomplete"));
234 chunk_id
= get_be32(chunk_lookup
+ 0);
235 chunk_offset
= get_be64(chunk_lookup
+ 4);
237 chunk_lookup
+= GRAPH_CHUNKLOOKUP_WIDTH
;
239 if (chunk_offset
> graph_size
- the_hash_algo
->rawsz
) {
240 error(_("commit-graph improper chunk offset %08x%08x"), (uint32_t)(chunk_offset
>> 32),
241 (uint32_t)chunk_offset
);
247 case GRAPH_CHUNKID_OIDFANOUT
:
248 if (graph
->chunk_oid_fanout
)
251 graph
->chunk_oid_fanout
= (uint32_t*)(data
+ chunk_offset
);
254 case GRAPH_CHUNKID_OIDLOOKUP
:
255 if (graph
->chunk_oid_lookup
)
258 graph
->chunk_oid_lookup
= data
+ chunk_offset
;
261 case GRAPH_CHUNKID_DATA
:
262 if (graph
->chunk_commit_data
)
265 graph
->chunk_commit_data
= data
+ chunk_offset
;
268 case GRAPH_CHUNKID_EXTRAEDGES
:
269 if (graph
->chunk_extra_edges
)
272 graph
->chunk_extra_edges
= data
+ chunk_offset
;
275 case GRAPH_CHUNKID_BASE
:
276 if (graph
->chunk_base_graphs
)
279 graph
->chunk_base_graphs
= data
+ chunk_offset
;
282 if (chunk_repeated
) {
283 error(_("commit-graph chunk id %08x appears multiple times"), chunk_id
);
288 if (last_chunk_id
== GRAPH_CHUNKID_OIDLOOKUP
)
290 graph
->num_commits
= (chunk_offset
- last_chunk_offset
)
294 last_chunk_id
= chunk_id
;
295 last_chunk_offset
= chunk_offset
;
298 hashcpy(graph
->oid
.hash
, graph
->data
+ graph
->data_len
- graph
->hash_len
);
300 if (verify_commit_graph_lite(graph
)) {
308 static struct commit_graph
*load_commit_graph_one(const char *graph_file
)
313 struct commit_graph
*g
;
314 int open_ok
= open_commit_graph(graph_file
, &fd
, &st
);
319 g
= load_commit_graph_one_fd_st(fd
, &st
);
322 g
->filename
= xstrdup(graph_file
);
327 static struct commit_graph
*load_commit_graph_v1(struct repository
*r
, const char *obj_dir
)
329 char *graph_name
= get_commit_graph_filename(obj_dir
);
330 struct commit_graph
*g
= load_commit_graph_one(graph_name
);
334 g
->obj_dir
= obj_dir
;
339 static int add_graph_to_chain(struct commit_graph
*g
,
340 struct commit_graph
*chain
,
341 struct object_id
*oids
,
344 struct commit_graph
*cur_g
= chain
;
346 if (n
&& !g
->chunk_base_graphs
) {
347 warning(_("commit-graph has no base graphs chunk"));
355 !oideq(&oids
[n
], &cur_g
->oid
) ||
356 !hasheq(oids
[n
].hash
, g
->chunk_base_graphs
+ g
->hash_len
* n
)) {
357 warning(_("commit-graph chain does not match"));
361 cur_g
= cur_g
->base_graph
;
364 g
->base_graph
= chain
;
367 g
->num_commits_in_base
= chain
->num_commits
+ chain
->num_commits_in_base
;
372 static struct commit_graph
*load_commit_graph_chain(struct repository
*r
, const char *obj_dir
)
374 struct commit_graph
*graph_chain
= NULL
;
375 struct strbuf line
= STRBUF_INIT
;
377 struct object_id
*oids
;
378 int i
= 0, valid
= 1, count
;
379 char *chain_name
= get_chain_filename(obj_dir
);
383 fp
= fopen(chain_name
, "r");
384 stat_res
= stat(chain_name
, &st
);
389 st
.st_size
<= the_hash_algo
->hexsz
)
392 count
= st
.st_size
/ (the_hash_algo
->hexsz
+ 1);
393 oids
= xcalloc(count
, sizeof(struct object_id
));
397 for (i
= 0; i
< count
; i
++) {
398 struct object_directory
*odb
;
400 if (strbuf_getline_lf(&line
, fp
) == EOF
)
403 if (get_oid_hex(line
.buf
, &oids
[i
])) {
404 warning(_("invalid commit-graph chain: line '%s' not a hash"),
411 for (odb
= r
->objects
->odb
; odb
; odb
= odb
->next
) {
412 char *graph_name
= get_split_graph_filename(odb
->path
, line
.buf
);
413 struct commit_graph
*g
= load_commit_graph_one(graph_name
);
418 g
->obj_dir
= odb
->path
;
420 if (add_graph_to_chain(g
, graph_chain
, oids
, i
)) {
430 warning(_("unable to find all commit-graph files"));
437 strbuf_release(&line
);
442 struct commit_graph
*read_commit_graph_one(struct repository
*r
, const char *obj_dir
)
444 struct commit_graph
*g
= load_commit_graph_v1(r
, obj_dir
);
447 g
= load_commit_graph_chain(r
, obj_dir
);
452 static void prepare_commit_graph_one(struct repository
*r
, const char *obj_dir
)
455 if (r
->objects
->commit_graph
)
458 r
->objects
->commit_graph
= read_commit_graph_one(r
, obj_dir
);
462 * Return 1 if commit_graph is non-NULL, and 0 otherwise.
464 * On the first invocation, this function attemps to load the commit
465 * graph if the_repository is configured to have one.
467 static int prepare_commit_graph(struct repository
*r
)
469 struct object_directory
*odb
;
472 if (git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD
, 0))
473 die("dying as requested by the '%s' variable on commit-graph load!",
474 GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD
);
476 if (r
->objects
->commit_graph_attempted
)
477 return !!r
->objects
->commit_graph
;
478 r
->objects
->commit_graph_attempted
= 1;
480 if (!git_env_bool(GIT_TEST_COMMIT_GRAPH
, 0) &&
481 (repo_config_get_bool(r
, "core.commitgraph", &config_value
) ||
484 * This repository is not configured to use commit graphs, so
485 * do not load one. (But report commit_graph_attempted anyway
486 * so that commit graph loading is not attempted again for this
491 if (!commit_graph_compatible(r
))
495 for (odb
= r
->objects
->odb
;
496 !r
->objects
->commit_graph
&& odb
;
498 prepare_commit_graph_one(r
, odb
->path
);
499 return !!r
->objects
->commit_graph
;
502 int generation_numbers_enabled(struct repository
*r
)
504 uint32_t first_generation
;
505 struct commit_graph
*g
;
506 if (!prepare_commit_graph(r
))
509 g
= r
->objects
->commit_graph
;
514 first_generation
= get_be32(g
->chunk_commit_data
+
515 g
->hash_len
+ 8) >> 2;
517 return !!first_generation
;
520 static void close_commit_graph_one(struct commit_graph
*g
)
525 close_commit_graph_one(g
->base_graph
);
526 free_commit_graph(g
);
529 void close_commit_graph(struct raw_object_store
*o
)
531 close_commit_graph_one(o
->commit_graph
);
532 o
->commit_graph
= NULL
;
535 static int bsearch_graph(struct commit_graph
*g
, struct object_id
*oid
, uint32_t *pos
)
537 return bsearch_hash(oid
->hash
, g
->chunk_oid_fanout
,
538 g
->chunk_oid_lookup
, g
->hash_len
, pos
);
541 static void load_oid_from_graph(struct commit_graph
*g
,
543 struct object_id
*oid
)
547 while (g
&& pos
< g
->num_commits_in_base
)
551 BUG("NULL commit-graph");
553 if (pos
>= g
->num_commits
+ g
->num_commits_in_base
)
554 die(_("invalid commit position. commit-graph is likely corrupt"));
556 lex_index
= pos
- g
->num_commits_in_base
;
558 hashcpy(oid
->hash
, g
->chunk_oid_lookup
+ g
->hash_len
* lex_index
);
561 static struct commit_list
**insert_parent_or_die(struct repository
*r
,
562 struct commit_graph
*g
,
564 struct commit_list
**pptr
)
567 struct object_id oid
;
569 if (pos
>= g
->num_commits
+ g
->num_commits_in_base
)
570 die("invalid parent position %"PRIu32
, pos
);
572 load_oid_from_graph(g
, pos
, &oid
);
573 c
= lookup_commit(r
, &oid
);
575 die(_("could not find commit %s"), oid_to_hex(&oid
));
577 return &commit_list_insert(c
, pptr
)->next
;
580 static void fill_commit_graph_info(struct commit
*item
, struct commit_graph
*g
, uint32_t pos
)
582 const unsigned char *commit_data
;
585 while (pos
< g
->num_commits_in_base
)
588 lex_index
= pos
- g
->num_commits_in_base
;
589 commit_data
= g
->chunk_commit_data
+ GRAPH_DATA_WIDTH
* lex_index
;
590 item
->graph_pos
= pos
;
591 item
->generation
= get_be32(commit_data
+ g
->hash_len
+ 8) >> 2;
594 static inline void set_commit_tree(struct commit
*c
, struct tree
*t
)
599 static int fill_commit_in_graph(struct repository
*r
,
601 struct commit_graph
*g
, uint32_t pos
)
604 uint32_t *parent_data_ptr
;
605 uint64_t date_low
, date_high
;
606 struct commit_list
**pptr
;
607 const unsigned char *commit_data
;
610 while (pos
< g
->num_commits_in_base
)
613 if (pos
>= g
->num_commits
+ g
->num_commits_in_base
)
614 die(_("invalid commit position. commit-graph is likely corrupt"));
617 * Store the "full" position, but then use the
618 * "local" position for the rest of the calculation.
620 item
->graph_pos
= pos
;
621 lex_index
= pos
- g
->num_commits_in_base
;
623 commit_data
= g
->chunk_commit_data
+ (g
->hash_len
+ 16) * lex_index
;
625 item
->object
.parsed
= 1;
627 set_commit_tree(item
, NULL
);
629 date_high
= get_be32(commit_data
+ g
->hash_len
+ 8) & 0x3;
630 date_low
= get_be32(commit_data
+ g
->hash_len
+ 12);
631 item
->date
= (timestamp_t
)((date_high
<< 32) | date_low
);
633 item
->generation
= get_be32(commit_data
+ g
->hash_len
+ 8) >> 2;
635 pptr
= &item
->parents
;
637 edge_value
= get_be32(commit_data
+ g
->hash_len
);
638 if (edge_value
== GRAPH_PARENT_NONE
)
640 pptr
= insert_parent_or_die(r
, g
, edge_value
, pptr
);
642 edge_value
= get_be32(commit_data
+ g
->hash_len
+ 4);
643 if (edge_value
== GRAPH_PARENT_NONE
)
645 if (!(edge_value
& GRAPH_EXTRA_EDGES_NEEDED
)) {
646 pptr
= insert_parent_or_die(r
, g
, edge_value
, pptr
);
650 parent_data_ptr
= (uint32_t*)(g
->chunk_extra_edges
+
651 4 * (uint64_t)(edge_value
& GRAPH_EDGE_LAST_MASK
));
653 edge_value
= get_be32(parent_data_ptr
);
654 pptr
= insert_parent_or_die(r
, g
,
655 edge_value
& GRAPH_EDGE_LAST_MASK
,
658 } while (!(edge_value
& GRAPH_LAST_EDGE
));
663 static int find_commit_in_graph(struct commit
*item
, struct commit_graph
*g
, uint32_t *pos
)
665 if (item
->graph_pos
!= COMMIT_NOT_FROM_GRAPH
) {
666 *pos
= item
->graph_pos
;
669 struct commit_graph
*cur_g
= g
;
672 while (cur_g
&& !bsearch_graph(cur_g
, &(item
->object
.oid
), &lex_index
))
673 cur_g
= cur_g
->base_graph
;
676 *pos
= lex_index
+ cur_g
->num_commits_in_base
;
684 static int parse_commit_in_graph_one(struct repository
*r
,
685 struct commit_graph
*g
,
690 if (item
->object
.parsed
)
693 if (find_commit_in_graph(item
, g
, &pos
))
694 return fill_commit_in_graph(r
, item
, g
, pos
);
699 int parse_commit_in_graph(struct repository
*r
, struct commit
*item
)
701 if (!prepare_commit_graph(r
))
703 return parse_commit_in_graph_one(r
, r
->objects
->commit_graph
, item
);
706 void load_commit_graph_info(struct repository
*r
, struct commit
*item
)
709 if (!prepare_commit_graph(r
))
711 if (find_commit_in_graph(item
, r
->objects
->commit_graph
, &pos
))
712 fill_commit_graph_info(item
, r
->objects
->commit_graph
, pos
);
715 static struct tree
*load_tree_for_commit(struct repository
*r
,
716 struct commit_graph
*g
,
719 struct object_id oid
;
720 const unsigned char *commit_data
;
722 while (c
->graph_pos
< g
->num_commits_in_base
)
725 commit_data
= g
->chunk_commit_data
+
726 GRAPH_DATA_WIDTH
* (c
->graph_pos
- g
->num_commits_in_base
);
728 hashcpy(oid
.hash
, commit_data
);
729 set_commit_tree(c
, lookup_tree(r
, &oid
));
731 return c
->maybe_tree
;
734 static struct tree
*get_commit_tree_in_graph_one(struct repository
*r
,
735 struct commit_graph
*g
,
736 const struct commit
*c
)
739 return c
->maybe_tree
;
740 if (c
->graph_pos
== COMMIT_NOT_FROM_GRAPH
)
741 BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
743 return load_tree_for_commit(r
, g
, (struct commit
*)c
);
746 struct tree
*get_commit_tree_in_graph(struct repository
*r
, const struct commit
*c
)
748 return get_commit_tree_in_graph_one(r
, r
->objects
->commit_graph
, c
);
751 struct packed_commit_list
{
752 struct commit
**list
;
757 struct packed_oid_list
{
758 struct object_id
*list
;
763 struct write_commit_graph_context
{
764 struct repository
*r
;
767 struct packed_oid_list oids
;
768 struct packed_commit_list commits
;
770 unsigned long approx_nr_objects
;
771 struct progress
*progress
;
773 uint64_t progress_cnt
;
775 char *base_graph_name
;
776 int num_commit_graphs_before
;
777 int num_commit_graphs_after
;
778 char **commit_graph_filenames_before
;
779 char **commit_graph_filenames_after
;
780 char **commit_graph_hash_after
;
781 uint32_t new_num_commits_in_base
;
782 struct commit_graph
*new_base_graph
;
788 const struct split_commit_graph_opts
*split_opts
;
791 static void write_graph_chunk_fanout(struct hashfile
*f
,
792 struct write_commit_graph_context
*ctx
)
795 struct commit
**list
= ctx
->commits
.list
;
798 * Write the first-level table (the list is sorted,
799 * but we use a 256-entry lookup to be able to avoid
800 * having to do eight extra binary search iterations).
802 for (i
= 0; i
< 256; i
++) {
803 while (count
< ctx
->commits
.nr
) {
804 if ((*list
)->object
.oid
.hash
[0] != i
)
806 display_progress(ctx
->progress
, ++ctx
->progress_cnt
);
811 hashwrite_be32(f
, count
);
815 static void write_graph_chunk_oids(struct hashfile
*f
, int hash_len
,
816 struct write_commit_graph_context
*ctx
)
818 struct commit
**list
= ctx
->commits
.list
;
820 for (count
= 0; count
< ctx
->commits
.nr
; count
++, list
++) {
821 display_progress(ctx
->progress
, ++ctx
->progress_cnt
);
822 hashwrite(f
, (*list
)->object
.oid
.hash
, (int)hash_len
);
826 static const unsigned char *commit_to_sha1(size_t index
, void *table
)
828 struct commit
**commits
= table
;
829 return commits
[index
]->object
.oid
.hash
;
832 static void write_graph_chunk_data(struct hashfile
*f
, int hash_len
,
833 struct write_commit_graph_context
*ctx
)
835 struct commit
**list
= ctx
->commits
.list
;
836 struct commit
**last
= ctx
->commits
.list
+ ctx
->commits
.nr
;
837 uint32_t num_extra_edges
= 0;
839 while (list
< last
) {
840 struct commit_list
*parent
;
842 uint32_t packedDate
[2];
843 display_progress(ctx
->progress
, ++ctx
->progress_cnt
);
845 parse_commit_no_graph(*list
);
846 hashwrite(f
, get_commit_tree_oid(*list
)->hash
, hash_len
);
848 parent
= (*list
)->parents
;
851 edge_value
= GRAPH_PARENT_NONE
;
853 edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
859 edge_value
+= ctx
->new_num_commits_in_base
;
862 if (find_commit_in_graph(parent
->item
,
869 BUG("missing parent %s for commit %s",
870 oid_to_hex(&parent
->item
->object
.oid
),
871 oid_to_hex(&(*list
)->object
.oid
));
874 hashwrite_be32(f
, edge_value
);
877 parent
= parent
->next
;
880 edge_value
= GRAPH_PARENT_NONE
;
881 else if (parent
->next
)
882 edge_value
= GRAPH_EXTRA_EDGES_NEEDED
| num_extra_edges
;
884 edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
890 edge_value
+= ctx
->new_num_commits_in_base
;
893 if (find_commit_in_graph(parent
->item
,
900 BUG("missing parent %s for commit %s",
901 oid_to_hex(&parent
->item
->object
.oid
),
902 oid_to_hex(&(*list
)->object
.oid
));
905 hashwrite_be32(f
, edge_value
);
907 if (edge_value
& GRAPH_EXTRA_EDGES_NEEDED
) {
910 parent
= parent
->next
;
914 if (sizeof((*list
)->date
) > 4)
915 packedDate
[0] = htonl(((*list
)->date
>> 32) & 0x3);
919 packedDate
[0] |= htonl((*list
)->generation
<< 2);
921 packedDate
[1] = htonl((*list
)->date
);
922 hashwrite(f
, packedDate
, 8);
928 static void write_graph_chunk_extra_edges(struct hashfile
*f
,
929 struct write_commit_graph_context
*ctx
)
931 struct commit
**list
= ctx
->commits
.list
;
932 struct commit
**last
= ctx
->commits
.list
+ ctx
->commits
.nr
;
933 struct commit_list
*parent
;
935 while (list
< last
) {
938 display_progress(ctx
->progress
, ++ctx
->progress_cnt
);
940 for (parent
= (*list
)->parents
; num_parents
< 3 && parent
;
941 parent
= parent
->next
)
944 if (num_parents
<= 2) {
949 /* Since num_parents > 2, this initializer is safe. */
950 for (parent
= (*list
)->parents
->next
; parent
; parent
= parent
->next
) {
951 int edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
957 edge_value
+= ctx
->new_num_commits_in_base
;
960 if (find_commit_in_graph(parent
->item
,
967 BUG("missing parent %s for commit %s",
968 oid_to_hex(&parent
->item
->object
.oid
),
969 oid_to_hex(&(*list
)->object
.oid
));
970 else if (!parent
->next
)
971 edge_value
|= GRAPH_LAST_EDGE
;
973 hashwrite_be32(f
, edge_value
);
980 static int oid_compare(const void *_a
, const void *_b
)
982 const struct object_id
*a
= (const struct object_id
*)_a
;
983 const struct object_id
*b
= (const struct object_id
*)_b
;
987 static int add_packed_commits(const struct object_id
*oid
,
988 struct packed_git
*pack
,
992 struct write_commit_graph_context
*ctx
= (struct write_commit_graph_context
*)data
;
993 enum object_type type
;
994 off_t offset
= nth_packed_object_offset(pack
, pos
);
995 struct object_info oi
= OBJECT_INFO_INIT
;
998 display_progress(ctx
->progress
, ++ctx
->progress_done
);
1001 if (packed_object_info(ctx
->r
, pack
, offset
, &oi
) < 0)
1002 die(_("unable to get type of object %s"), oid_to_hex(oid
));
1004 if (type
!= OBJ_COMMIT
)
1007 ALLOC_GROW(ctx
->oids
.list
, ctx
->oids
.nr
+ 1, ctx
->oids
.alloc
);
1008 oidcpy(&(ctx
->oids
.list
[ctx
->oids
.nr
]), oid
);
1014 static void add_missing_parents(struct write_commit_graph_context
*ctx
, struct commit
*commit
)
1016 struct commit_list
*parent
;
1017 for (parent
= commit
->parents
; parent
; parent
= parent
->next
) {
1018 if (!(parent
->item
->object
.flags
& UNINTERESTING
)) {
1019 ALLOC_GROW(ctx
->oids
.list
, ctx
->oids
.nr
+ 1, ctx
->oids
.alloc
);
1020 oidcpy(&ctx
->oids
.list
[ctx
->oids
.nr
], &(parent
->item
->object
.oid
));
1022 parent
->item
->object
.flags
|= UNINTERESTING
;
1027 static void close_reachable(struct write_commit_graph_context
*ctx
)
1030 struct commit
*commit
;
1032 if (ctx
->report_progress
)
1033 ctx
->progress
= start_delayed_progress(
1034 _("Loading known commits in commit graph"),
1036 for (i
= 0; i
< ctx
->oids
.nr
; i
++) {
1037 display_progress(ctx
->progress
, i
+ 1);
1038 commit
= lookup_commit(ctx
->r
, &ctx
->oids
.list
[i
]);
1040 commit
->object
.flags
|= UNINTERESTING
;
1042 stop_progress(&ctx
->progress
);
1045 * As this loop runs, ctx->oids.nr may grow, but not more
1046 * than the number of missing commits in the reachable
1049 if (ctx
->report_progress
)
1050 ctx
->progress
= start_delayed_progress(
1051 _("Expanding reachable commits in commit graph"),
1053 for (i
= 0; i
< ctx
->oids
.nr
; i
++) {
1054 display_progress(ctx
->progress
, i
+ 1);
1055 commit
= lookup_commit(ctx
->r
, &ctx
->oids
.list
[i
]);
1060 if (!parse_commit(commit
) &&
1061 commit
->graph_pos
== COMMIT_NOT_FROM_GRAPH
)
1062 add_missing_parents(ctx
, commit
);
1063 } else if (!parse_commit_no_graph(commit
))
1064 add_missing_parents(ctx
, commit
);
1066 stop_progress(&ctx
->progress
);
1068 if (ctx
->report_progress
)
1069 ctx
->progress
= start_delayed_progress(
1070 _("Clearing commit marks in commit graph"),
1072 for (i
= 0; i
< ctx
->oids
.nr
; i
++) {
1073 display_progress(ctx
->progress
, i
+ 1);
1074 commit
= lookup_commit(ctx
->r
, &ctx
->oids
.list
[i
]);
1077 commit
->object
.flags
&= ~UNINTERESTING
;
1079 stop_progress(&ctx
->progress
);
1082 static void compute_generation_numbers(struct write_commit_graph_context
*ctx
)
1085 struct commit_list
*list
= NULL
;
1087 if (ctx
->report_progress
)
1088 ctx
->progress
= start_progress(
1089 _("Computing commit graph generation numbers"),
1091 for (i
= 0; i
< ctx
->commits
.nr
; i
++) {
1092 display_progress(ctx
->progress
, i
+ 1);
1093 if (ctx
->commits
.list
[i
]->generation
!= GENERATION_NUMBER_INFINITY
&&
1094 ctx
->commits
.list
[i
]->generation
!= GENERATION_NUMBER_ZERO
)
1097 commit_list_insert(ctx
->commits
.list
[i
], &list
);
1099 struct commit
*current
= list
->item
;
1100 struct commit_list
*parent
;
1101 int all_parents_computed
= 1;
1102 uint32_t max_generation
= 0;
1104 for (parent
= current
->parents
; parent
; parent
= parent
->next
) {
1105 if (parent
->item
->generation
== GENERATION_NUMBER_INFINITY
||
1106 parent
->item
->generation
== GENERATION_NUMBER_ZERO
) {
1107 all_parents_computed
= 0;
1108 commit_list_insert(parent
->item
, &list
);
1110 } else if (parent
->item
->generation
> max_generation
) {
1111 max_generation
= parent
->item
->generation
;
1115 if (all_parents_computed
) {
1116 current
->generation
= max_generation
+ 1;
1119 if (current
->generation
> GENERATION_NUMBER_MAX
)
1120 current
->generation
= GENERATION_NUMBER_MAX
;
1124 stop_progress(&ctx
->progress
);
1127 static int add_ref_to_list(const char *refname
,
1128 const struct object_id
*oid
,
1129 int flags
, void *cb_data
)
1131 struct string_list
*list
= (struct string_list
*)cb_data
;
1133 string_list_append(list
, oid_to_hex(oid
));
1137 int write_commit_graph_reachable(const char *obj_dir
, unsigned int flags
,
1138 const struct split_commit_graph_opts
*split_opts
)
1140 struct string_list list
= STRING_LIST_INIT_DUP
;
1143 for_each_ref(add_ref_to_list
, &list
);
1144 result
= write_commit_graph(obj_dir
, NULL
, &list
,
1147 string_list_clear(&list
, 0);
1151 static int fill_oids_from_packs(struct write_commit_graph_context
*ctx
,
1152 struct string_list
*pack_indexes
)
1155 struct strbuf progress_title
= STRBUF_INIT
;
1156 struct strbuf packname
= STRBUF_INIT
;
1159 strbuf_addf(&packname
, "%s/pack/", ctx
->obj_dir
);
1160 dirlen
= packname
.len
;
1161 if (ctx
->report_progress
) {
1162 strbuf_addf(&progress_title
,
1163 Q_("Finding commits for commit graph in %d pack",
1164 "Finding commits for commit graph in %d packs",
1167 ctx
->progress
= start_delayed_progress(progress_title
.buf
, 0);
1168 ctx
->progress_done
= 0;
1170 for (i
= 0; i
< pack_indexes
->nr
; i
++) {
1171 struct packed_git
*p
;
1172 strbuf_setlen(&packname
, dirlen
);
1173 strbuf_addstr(&packname
, pack_indexes
->items
[i
].string
);
1174 p
= add_packed_git(packname
.buf
, packname
.len
, 1);
1176 error(_("error adding pack %s"), packname
.buf
);
1179 if (open_pack_index(p
)) {
1180 error(_("error opening index for %s"), packname
.buf
);
1183 for_each_object_in_pack(p
, add_packed_commits
, ctx
,
1184 FOR_EACH_OBJECT_PACK_ORDER
);
1189 stop_progress(&ctx
->progress
);
1190 strbuf_release(&progress_title
);
1191 strbuf_release(&packname
);
1196 static void fill_oids_from_commit_hex(struct write_commit_graph_context
*ctx
,
1197 struct string_list
*commit_hex
)
1200 struct strbuf progress_title
= STRBUF_INIT
;
1202 if (ctx
->report_progress
) {
1203 strbuf_addf(&progress_title
,
1204 Q_("Finding commits for commit graph from %d ref",
1205 "Finding commits for commit graph from %d refs",
1208 ctx
->progress
= start_delayed_progress(
1212 for (i
= 0; i
< commit_hex
->nr
; i
++) {
1214 struct object_id oid
;
1215 struct commit
*result
;
1217 display_progress(ctx
->progress
, i
+ 1);
1218 if (commit_hex
->items
[i
].string
&&
1219 parse_oid_hex(commit_hex
->items
[i
].string
, &oid
, &end
))
1222 result
= lookup_commit_reference_gently(ctx
->r
, &oid
, 1);
1225 ALLOC_GROW(ctx
->oids
.list
, ctx
->oids
.nr
+ 1, ctx
->oids
.alloc
);
1226 oidcpy(&ctx
->oids
.list
[ctx
->oids
.nr
], &(result
->object
.oid
));
1230 stop_progress(&ctx
->progress
);
1231 strbuf_release(&progress_title
);
1234 static void fill_oids_from_all_packs(struct write_commit_graph_context
*ctx
)
1236 if (ctx
->report_progress
)
1237 ctx
->progress
= start_delayed_progress(
1238 _("Finding commits for commit graph among packed objects"),
1239 ctx
->approx_nr_objects
);
1240 for_each_packed_object(add_packed_commits
, ctx
,
1241 FOR_EACH_OBJECT_PACK_ORDER
);
1242 if (ctx
->progress_done
< ctx
->approx_nr_objects
)
1243 display_progress(ctx
->progress
, ctx
->approx_nr_objects
);
1244 stop_progress(&ctx
->progress
);
1247 static uint32_t count_distinct_commits(struct write_commit_graph_context
*ctx
)
1249 uint32_t i
, count_distinct
= 1;
1251 if (ctx
->report_progress
)
1252 ctx
->progress
= start_delayed_progress(
1253 _("Counting distinct commits in commit graph"),
1255 display_progress(ctx
->progress
, 0); /* TODO: Measure QSORT() progress */
1256 QSORT(ctx
->oids
.list
, ctx
->oids
.nr
, oid_compare
);
1258 for (i
= 1; i
< ctx
->oids
.nr
; i
++) {
1259 display_progress(ctx
->progress
, i
+ 1);
1260 if (!oideq(&ctx
->oids
.list
[i
- 1], &ctx
->oids
.list
[i
])) {
1262 struct commit
*c
= lookup_commit(ctx
->r
, &ctx
->oids
.list
[i
]);
1264 if (!c
|| c
->graph_pos
!= COMMIT_NOT_FROM_GRAPH
)
1271 stop_progress(&ctx
->progress
);
1273 return count_distinct
;
1276 static void copy_oids_to_commits(struct write_commit_graph_context
*ctx
)
1279 struct commit_list
*parent
;
1281 ctx
->num_extra_edges
= 0;
1282 if (ctx
->report_progress
)
1283 ctx
->progress
= start_delayed_progress(
1284 _("Finding extra edges in commit graph"),
1286 for (i
= 0; i
< ctx
->oids
.nr
; i
++) {
1287 int num_parents
= 0;
1288 display_progress(ctx
->progress
, i
+ 1);
1289 if (i
> 0 && oideq(&ctx
->oids
.list
[i
- 1], &ctx
->oids
.list
[i
]))
1292 ALLOC_GROW(ctx
->commits
.list
, ctx
->commits
.nr
+ 1, ctx
->commits
.alloc
);
1293 ctx
->commits
.list
[ctx
->commits
.nr
] = lookup_commit(ctx
->r
, &ctx
->oids
.list
[i
]);
1296 ctx
->commits
.list
[ctx
->commits
.nr
]->graph_pos
!= COMMIT_NOT_FROM_GRAPH
)
1299 parse_commit_no_graph(ctx
->commits
.list
[ctx
->commits
.nr
]);
1301 for (parent
= ctx
->commits
.list
[ctx
->commits
.nr
]->parents
;
1302 parent
; parent
= parent
->next
)
1305 if (num_parents
> 2)
1306 ctx
->num_extra_edges
+= num_parents
- 1;
1310 stop_progress(&ctx
->progress
);
1313 static int write_graph_chunk_base_1(struct hashfile
*f
,
1314 struct commit_graph
*g
)
1321 num
= write_graph_chunk_base_1(f
, g
->base_graph
);
1322 hashwrite(f
, g
->oid
.hash
, the_hash_algo
->rawsz
);
1326 static int write_graph_chunk_base(struct hashfile
*f
,
1327 struct write_commit_graph_context
*ctx
)
1329 int num
= write_graph_chunk_base_1(f
, ctx
->new_base_graph
);
1331 if (num
!= ctx
->num_commit_graphs_after
- 1) {
1332 error(_("failed to write correct number of base graph ids"));
1339 static int write_commit_graph_file(struct write_commit_graph_context
*ctx
)
1344 struct lock_file lk
= LOCK_INIT
;
1345 uint32_t chunk_ids
[6];
1346 uint64_t chunk_offsets
[6];
1347 const unsigned hashsz
= the_hash_algo
->rawsz
;
1348 struct strbuf progress_title
= STRBUF_INIT
;
1350 struct object_id file_hash
;
1353 struct strbuf tmp_file
= STRBUF_INIT
;
1355 strbuf_addf(&tmp_file
,
1356 "%s/info/commit-graphs/tmp_graph_XXXXXX",
1358 ctx
->graph_name
= strbuf_detach(&tmp_file
, NULL
);
1360 ctx
->graph_name
= get_commit_graph_filename(ctx
->obj_dir
);
1363 if (safe_create_leading_directories(ctx
->graph_name
)) {
1364 UNLEAK(ctx
->graph_name
);
1365 error(_("unable to create leading directories of %s"),
1371 char *lock_name
= get_chain_filename(ctx
->obj_dir
);
1373 hold_lock_file_for_update(&lk
, lock_name
, LOCK_DIE_ON_ERROR
);
1375 fd
= git_mkstemp_mode(ctx
->graph_name
, 0444);
1377 error(_("unable to create '%s'"), ctx
->graph_name
);
1381 f
= hashfd(fd
, ctx
->graph_name
);
1383 hold_lock_file_for_update(&lk
, ctx
->graph_name
, LOCK_DIE_ON_ERROR
);
1384 fd
= lk
.tempfile
->fd
;
1385 f
= hashfd(lk
.tempfile
->fd
, lk
.tempfile
->filename
.buf
);
1388 chunk_ids
[0] = GRAPH_CHUNKID_OIDFANOUT
;
1389 chunk_ids
[1] = GRAPH_CHUNKID_OIDLOOKUP
;
1390 chunk_ids
[2] = GRAPH_CHUNKID_DATA
;
1391 if (ctx
->num_extra_edges
) {
1392 chunk_ids
[num_chunks
] = GRAPH_CHUNKID_EXTRAEDGES
;
1395 if (ctx
->num_commit_graphs_after
> 1) {
1396 chunk_ids
[num_chunks
] = GRAPH_CHUNKID_BASE
;
1400 chunk_ids
[num_chunks
] = 0;
1402 chunk_offsets
[0] = 8 + (num_chunks
+ 1) * GRAPH_CHUNKLOOKUP_WIDTH
;
1403 chunk_offsets
[1] = chunk_offsets
[0] + GRAPH_FANOUT_SIZE
;
1404 chunk_offsets
[2] = chunk_offsets
[1] + hashsz
* ctx
->commits
.nr
;
1405 chunk_offsets
[3] = chunk_offsets
[2] + (hashsz
+ 16) * ctx
->commits
.nr
;
1408 if (ctx
->num_extra_edges
) {
1409 chunk_offsets
[num_chunks
+ 1] = chunk_offsets
[num_chunks
] +
1410 4 * ctx
->num_extra_edges
;
1413 if (ctx
->num_commit_graphs_after
> 1) {
1414 chunk_offsets
[num_chunks
+ 1] = chunk_offsets
[num_chunks
] +
1415 hashsz
* (ctx
->num_commit_graphs_after
- 1);
1419 hashwrite_be32(f
, GRAPH_SIGNATURE
);
1421 hashwrite_u8(f
, GRAPH_VERSION
);
1422 hashwrite_u8(f
, oid_version());
1423 hashwrite_u8(f
, num_chunks
);
1424 hashwrite_u8(f
, ctx
->num_commit_graphs_after
- 1);
1426 for (i
= 0; i
<= num_chunks
; i
++) {
1427 uint32_t chunk_write
[3];
1429 chunk_write
[0] = htonl(chunk_ids
[i
]);
1430 chunk_write
[1] = htonl(chunk_offsets
[i
] >> 32);
1431 chunk_write
[2] = htonl(chunk_offsets
[i
] & 0xffffffff);
1432 hashwrite(f
, chunk_write
, 12);
1435 if (ctx
->report_progress
) {
1436 strbuf_addf(&progress_title
,
1437 Q_("Writing out commit graph in %d pass",
1438 "Writing out commit graph in %d passes",
1441 ctx
->progress
= start_delayed_progress(
1443 num_chunks
* ctx
->commits
.nr
);
1445 write_graph_chunk_fanout(f
, ctx
);
1446 write_graph_chunk_oids(f
, hashsz
, ctx
);
1447 write_graph_chunk_data(f
, hashsz
, ctx
);
1448 if (ctx
->num_extra_edges
)
1449 write_graph_chunk_extra_edges(f
, ctx
);
1450 if (ctx
->num_commit_graphs_after
> 1 &&
1451 write_graph_chunk_base(f
, ctx
)) {
1454 stop_progress(&ctx
->progress
);
1455 strbuf_release(&progress_title
);
1457 if (ctx
->split
&& ctx
->base_graph_name
&& ctx
->num_commit_graphs_after
> 1) {
1458 char *new_base_hash
= xstrdup(oid_to_hex(&ctx
->new_base_graph
->oid
));
1459 char *new_base_name
= get_split_graph_filename(ctx
->new_base_graph
->obj_dir
, new_base_hash
);
1461 free(ctx
->commit_graph_filenames_after
[ctx
->num_commit_graphs_after
- 2]);
1462 free(ctx
->commit_graph_hash_after
[ctx
->num_commit_graphs_after
- 2]);
1463 ctx
->commit_graph_filenames_after
[ctx
->num_commit_graphs_after
- 2] = new_base_name
;
1464 ctx
->commit_graph_hash_after
[ctx
->num_commit_graphs_after
- 2] = new_base_hash
;
1467 close_commit_graph(ctx
->r
->objects
);
1468 finalize_hashfile(f
, file_hash
.hash
, CSUM_HASH_IN_STREAM
| CSUM_FSYNC
);
1471 FILE *chainf
= fdopen_lock_file(&lk
, "w");
1472 char *final_graph_name
;
1478 error(_("unable to open commit-graph chain file"));
1482 if (ctx
->base_graph_name
) {
1483 const char *dest
= ctx
->commit_graph_filenames_after
[
1484 ctx
->num_commit_graphs_after
- 2];
1486 if (strcmp(ctx
->base_graph_name
, dest
)) {
1487 result
= rename(ctx
->base_graph_name
, dest
);
1490 error(_("failed to rename base commit-graph file"));
1495 char *graph_name
= get_commit_graph_filename(ctx
->obj_dir
);
1499 ctx
->commit_graph_hash_after
[ctx
->num_commit_graphs_after
- 1] = xstrdup(oid_to_hex(&file_hash
));
1500 final_graph_name
= get_split_graph_filename(ctx
->obj_dir
,
1501 ctx
->commit_graph_hash_after
[ctx
->num_commit_graphs_after
- 1]);
1502 ctx
->commit_graph_filenames_after
[ctx
->num_commit_graphs_after
- 1] = final_graph_name
;
1504 result
= rename(ctx
->graph_name
, final_graph_name
);
1506 for (i
= 0; i
< ctx
->num_commit_graphs_after
; i
++)
1507 fprintf(lk
.tempfile
->fp
, "%s\n", ctx
->commit_graph_hash_after
[i
]);
1510 error(_("failed to rename temporary commit-graph file"));
1515 commit_lock_file(&lk
);
1520 static void split_graph_merge_strategy(struct write_commit_graph_context
*ctx
)
1522 struct commit_graph
*g
= ctx
->r
->objects
->commit_graph
;
1523 uint32_t num_commits
= ctx
->commits
.nr
;
1526 int max_commits
= 0;
1529 if (ctx
->split_opts
) {
1530 max_commits
= ctx
->split_opts
->max_commits
;
1531 size_mult
= ctx
->split_opts
->size_multiple
;
1534 g
= ctx
->r
->objects
->commit_graph
;
1535 ctx
->num_commit_graphs_after
= ctx
->num_commit_graphs_before
+ 1;
1537 while (g
&& (g
->num_commits
<= size_mult
* num_commits
||
1538 (max_commits
&& num_commits
> max_commits
))) {
1539 if (strcmp(g
->obj_dir
, ctx
->obj_dir
))
1542 num_commits
+= g
->num_commits
;
1545 ctx
->num_commit_graphs_after
--;
1548 ctx
->new_base_graph
= g
;
1550 if (ctx
->num_commit_graphs_after
== 2) {
1551 char *old_graph_name
= get_commit_graph_filename(g
->obj_dir
);
1553 if (!strcmp(g
->filename
, old_graph_name
) &&
1554 strcmp(g
->obj_dir
, ctx
->obj_dir
)) {
1555 ctx
->num_commit_graphs_after
= 1;
1556 ctx
->new_base_graph
= NULL
;
1559 free(old_graph_name
);
1562 ALLOC_ARRAY(ctx
->commit_graph_filenames_after
, ctx
->num_commit_graphs_after
);
1563 ALLOC_ARRAY(ctx
->commit_graph_hash_after
, ctx
->num_commit_graphs_after
);
1565 for (i
= 0; i
< ctx
->num_commit_graphs_after
&&
1566 i
< ctx
->num_commit_graphs_before
; i
++)
1567 ctx
->commit_graph_filenames_after
[i
] = xstrdup(ctx
->commit_graph_filenames_before
[i
]);
1569 i
= ctx
->num_commit_graphs_before
- 1;
1570 g
= ctx
->r
->objects
->commit_graph
;
1573 if (i
< ctx
->num_commit_graphs_after
)
1574 ctx
->commit_graph_hash_after
[i
] = xstrdup(oid_to_hex(&g
->oid
));
1581 static void merge_commit_graph(struct write_commit_graph_context
*ctx
,
1582 struct commit_graph
*g
)
1585 uint32_t offset
= g
->num_commits_in_base
;
1587 ALLOC_GROW(ctx
->commits
.list
, ctx
->commits
.nr
+ g
->num_commits
, ctx
->commits
.alloc
);
1589 for (i
= 0; i
< g
->num_commits
; i
++) {
1590 struct object_id oid
;
1591 struct commit
*result
;
1593 display_progress(ctx
->progress
, i
+ 1);
1595 load_oid_from_graph(g
, i
+ offset
, &oid
);
1597 /* only add commits if they still exist in the repo */
1598 result
= lookup_commit_reference_gently(ctx
->r
, &oid
, 1);
1601 ctx
->commits
.list
[ctx
->commits
.nr
] = result
;
1607 static int commit_compare(const void *_a
, const void *_b
)
1609 const struct commit
*a
= *(const struct commit
**)_a
;
1610 const struct commit
*b
= *(const struct commit
**)_b
;
1611 return oidcmp(&a
->object
.oid
, &b
->object
.oid
);
1614 static void sort_and_scan_merged_commits(struct write_commit_graph_context
*ctx
)
1616 uint32_t i
, num_parents
;
1617 struct commit_list
*parent
;
1619 if (ctx
->report_progress
)
1620 ctx
->progress
= start_delayed_progress(
1621 _("Scanning merged commits"),
1624 QSORT(ctx
->commits
.list
, ctx
->commits
.nr
, commit_compare
);
1626 ctx
->num_extra_edges
= 0;
1627 for (i
= 0; i
< ctx
->commits
.nr
; i
++) {
1628 display_progress(ctx
->progress
, i
);
1630 if (i
&& oideq(&ctx
->commits
.list
[i
- 1]->object
.oid
,
1631 &ctx
->commits
.list
[i
]->object
.oid
)) {
1632 die(_("unexpected duplicate commit id %s"),
1633 oid_to_hex(&ctx
->commits
.list
[i
]->object
.oid
));
1636 for (parent
= ctx
->commits
.list
[i
]->parents
; parent
; parent
= parent
->next
)
1639 if (num_parents
> 2)
1640 ctx
->num_extra_edges
+= num_parents
- 1;
1644 stop_progress(&ctx
->progress
);
1647 static void merge_commit_graphs(struct write_commit_graph_context
*ctx
)
1649 struct commit_graph
*g
= ctx
->r
->objects
->commit_graph
;
1650 uint32_t current_graph_number
= ctx
->num_commit_graphs_before
;
1651 struct strbuf progress_title
= STRBUF_INIT
;
1653 while (g
&& current_graph_number
>= ctx
->num_commit_graphs_after
) {
1654 current_graph_number
--;
1656 if (ctx
->report_progress
) {
1657 strbuf_addstr(&progress_title
, _("Merging commit-graph"));
1658 ctx
->progress
= start_delayed_progress(progress_title
.buf
, 0);
1661 merge_commit_graph(ctx
, g
);
1662 stop_progress(&ctx
->progress
);
1663 strbuf_release(&progress_title
);
1669 ctx
->new_base_graph
= g
;
1670 ctx
->new_num_commits_in_base
= g
->num_commits
+ g
->num_commits_in_base
;
1673 if (ctx
->new_base_graph
)
1674 ctx
->base_graph_name
= xstrdup(ctx
->new_base_graph
->filename
);
1676 sort_and_scan_merged_commits(ctx
);
1679 static void mark_commit_graphs(struct write_commit_graph_context
*ctx
)
1682 time_t now
= time(NULL
);
1684 for (i
= ctx
->num_commit_graphs_after
- 1; i
< ctx
->num_commit_graphs_before
; i
++) {
1686 struct utimbuf updated_time
;
1688 stat(ctx
->commit_graph_filenames_before
[i
], &st
);
1690 updated_time
.actime
= st
.st_atime
;
1691 updated_time
.modtime
= now
;
1692 utime(ctx
->commit_graph_filenames_before
[i
], &updated_time
);
1696 static void expire_commit_graphs(struct write_commit_graph_context
*ctx
)
1698 struct strbuf path
= STRBUF_INIT
;
1702 timestamp_t expire_time
= time(NULL
);
1704 if (ctx
->split_opts
&& ctx
->split_opts
->expire_time
)
1705 expire_time
-= ctx
->split_opts
->expire_time
;
1707 char *chain_file_name
= get_chain_filename(ctx
->obj_dir
);
1708 unlink(chain_file_name
);
1709 free(chain_file_name
);
1710 ctx
->num_commit_graphs_after
= 0;
1713 strbuf_addstr(&path
, ctx
->obj_dir
);
1714 strbuf_addstr(&path
, "/info/commit-graphs");
1715 dir
= opendir(path
.buf
);
1720 strbuf_addch(&path
, '/');
1721 dirnamelen
= path
.len
;
1722 while ((de
= readdir(dir
)) != NULL
) {
1724 uint32_t i
, found
= 0;
1726 strbuf_setlen(&path
, dirnamelen
);
1727 strbuf_addstr(&path
, de
->d_name
);
1729 stat(path
.buf
, &st
);
1731 if (st
.st_mtime
> expire_time
)
1733 if (path
.len
< 6 || strcmp(path
.buf
+ path
.len
- 6, ".graph"))
1736 for (i
= 0; i
< ctx
->num_commit_graphs_after
; i
++) {
1737 if (!strcmp(ctx
->commit_graph_filenames_after
[i
],
1749 strbuf_release(&path
);
1752 int write_commit_graph(const char *obj_dir
,
1753 struct string_list
*pack_indexes
,
1754 struct string_list
*commit_hex
,
1756 const struct split_commit_graph_opts
*split_opts
)
1758 struct write_commit_graph_context
*ctx
;
1759 uint32_t i
, count_distinct
= 0;
1763 if (!commit_graph_compatible(the_repository
))
1766 ctx
= xcalloc(1, sizeof(struct write_commit_graph_context
));
1767 ctx
->r
= the_repository
;
1769 /* normalize object dir with no trailing slash */
1770 ctx
->obj_dir
= xmallocz(strlen(obj_dir
) + 1);
1771 normalize_path_copy(ctx
->obj_dir
, obj_dir
);
1772 len
= strlen(ctx
->obj_dir
);
1773 if (len
&& ctx
->obj_dir
[len
- 1] == '/')
1774 ctx
->obj_dir
[len
- 1] = 0;
1776 ctx
->append
= flags
& COMMIT_GRAPH_APPEND
? 1 : 0;
1777 ctx
->report_progress
= flags
& COMMIT_GRAPH_PROGRESS
? 1 : 0;
1778 ctx
->split
= flags
& COMMIT_GRAPH_SPLIT
? 1 : 0;
1779 ctx
->split_opts
= split_opts
;
1782 struct commit_graph
*g
;
1783 prepare_commit_graph(ctx
->r
);
1785 g
= ctx
->r
->objects
->commit_graph
;
1788 ctx
->num_commit_graphs_before
++;
1792 if (ctx
->num_commit_graphs_before
) {
1793 ALLOC_ARRAY(ctx
->commit_graph_filenames_before
, ctx
->num_commit_graphs_before
);
1794 i
= ctx
->num_commit_graphs_before
;
1795 g
= ctx
->r
->objects
->commit_graph
;
1798 ctx
->commit_graph_filenames_before
[--i
] = xstrdup(g
->filename
);
1804 ctx
->approx_nr_objects
= approximate_object_count();
1805 ctx
->oids
.alloc
= ctx
->approx_nr_objects
/ 32;
1807 if (ctx
->split
&& split_opts
&& ctx
->oids
.alloc
> split_opts
->max_commits
)
1808 ctx
->oids
.alloc
= split_opts
->max_commits
;
1811 prepare_commit_graph_one(ctx
->r
, ctx
->obj_dir
);
1812 if (ctx
->r
->objects
->commit_graph
)
1813 ctx
->oids
.alloc
+= ctx
->r
->objects
->commit_graph
->num_commits
;
1816 if (ctx
->oids
.alloc
< 1024)
1817 ctx
->oids
.alloc
= 1024;
1818 ALLOC_ARRAY(ctx
->oids
.list
, ctx
->oids
.alloc
);
1820 if (ctx
->append
&& ctx
->r
->objects
->commit_graph
) {
1821 struct commit_graph
*g
= ctx
->r
->objects
->commit_graph
;
1822 for (i
= 0; i
< g
->num_commits
; i
++) {
1823 const unsigned char *hash
= g
->chunk_oid_lookup
+ g
->hash_len
* i
;
1824 hashcpy(ctx
->oids
.list
[ctx
->oids
.nr
++].hash
, hash
);
1829 if ((res
= fill_oids_from_packs(ctx
, pack_indexes
)))
1834 fill_oids_from_commit_hex(ctx
, commit_hex
);
1836 if (!pack_indexes
&& !commit_hex
)
1837 fill_oids_from_all_packs(ctx
);
1839 close_reachable(ctx
);
1841 count_distinct
= count_distinct_commits(ctx
);
1843 if (count_distinct
>= GRAPH_EDGE_LAST_MASK
) {
1844 error(_("the commit graph format cannot write %d commits"), count_distinct
);
1849 ctx
->commits
.alloc
= count_distinct
;
1850 ALLOC_ARRAY(ctx
->commits
.list
, ctx
->commits
.alloc
);
1852 copy_oids_to_commits(ctx
);
1854 if (ctx
->commits
.nr
>= GRAPH_EDGE_LAST_MASK
) {
1855 error(_("too many commits to write graph"));
1860 if (!ctx
->commits
.nr
)
1864 split_graph_merge_strategy(ctx
);
1866 merge_commit_graphs(ctx
);
1868 ctx
->num_commit_graphs_after
= 1;
1870 compute_generation_numbers(ctx
);
1872 res
= write_commit_graph_file(ctx
);
1875 mark_commit_graphs(ctx
);
1877 expire_commit_graphs(ctx
);
1880 free(ctx
->graph_name
);
1881 free(ctx
->commits
.list
);
1882 free(ctx
->oids
.list
);
1885 if (ctx
->commit_graph_filenames_after
) {
1886 for (i
= 0; i
< ctx
->num_commit_graphs_after
; i
++) {
1887 free(ctx
->commit_graph_filenames_after
[i
]);
1888 free(ctx
->commit_graph_hash_after
[i
]);
1891 for (i
= 0; i
< ctx
->num_commit_graphs_before
; i
++)
1892 free(ctx
->commit_graph_filenames_before
[i
]);
1894 free(ctx
->commit_graph_filenames_after
);
1895 free(ctx
->commit_graph_filenames_before
);
1896 free(ctx
->commit_graph_hash_after
);
1904 #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
1905 static int verify_commit_graph_error
;
1907 static void graph_report(const char *fmt
, ...)
1911 verify_commit_graph_error
= 1;
1913 vfprintf(stderr
, fmt
, ap
);
1914 fprintf(stderr
, "\n");
1918 #define GENERATION_ZERO_EXISTS 1
1919 #define GENERATION_NUMBER_EXISTS 2
1921 int verify_commit_graph(struct repository
*r
, struct commit_graph
*g
, int flags
)
1923 uint32_t i
, cur_fanout_pos
= 0;
1924 struct object_id prev_oid
, cur_oid
, checksum
;
1925 int generation_zero
= 0;
1928 struct progress
*progress
= NULL
;
1929 int local_error
= 0;
1932 graph_report("no commit-graph file loaded");
1936 verify_commit_graph_error
= verify_commit_graph_lite(g
);
1937 if (verify_commit_graph_error
)
1938 return verify_commit_graph_error
;
1940 devnull
= open("/dev/null", O_WRONLY
);
1941 f
= hashfd(devnull
, NULL
);
1942 hashwrite(f
, g
->data
, g
->data_len
- g
->hash_len
);
1943 finalize_hashfile(f
, checksum
.hash
, CSUM_CLOSE
);
1944 if (!hasheq(checksum
.hash
, g
->data
+ g
->data_len
- g
->hash_len
)) {
1945 graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
1946 verify_commit_graph_error
= VERIFY_COMMIT_GRAPH_ERROR_HASH
;
1949 for (i
= 0; i
< g
->num_commits
; i
++) {
1950 struct commit
*graph_commit
;
1952 hashcpy(cur_oid
.hash
, g
->chunk_oid_lookup
+ g
->hash_len
* i
);
1954 if (i
&& oidcmp(&prev_oid
, &cur_oid
) >= 0)
1955 graph_report(_("commit-graph has incorrect OID order: %s then %s"),
1956 oid_to_hex(&prev_oid
),
1957 oid_to_hex(&cur_oid
));
1959 oidcpy(&prev_oid
, &cur_oid
);
1961 while (cur_oid
.hash
[0] > cur_fanout_pos
) {
1962 uint32_t fanout_value
= get_be32(g
->chunk_oid_fanout
+ cur_fanout_pos
);
1964 if (i
!= fanout_value
)
1965 graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
1966 cur_fanout_pos
, fanout_value
, i
);
1970 graph_commit
= lookup_commit(r
, &cur_oid
);
1971 if (!parse_commit_in_graph_one(r
, g
, graph_commit
))
1972 graph_report(_("failed to parse commit %s from commit-graph"),
1973 oid_to_hex(&cur_oid
));
1976 while (cur_fanout_pos
< 256) {
1977 uint32_t fanout_value
= get_be32(g
->chunk_oid_fanout
+ cur_fanout_pos
);
1979 if (g
->num_commits
!= fanout_value
)
1980 graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
1981 cur_fanout_pos
, fanout_value
, i
);
1986 if (verify_commit_graph_error
& ~VERIFY_COMMIT_GRAPH_ERROR_HASH
)
1987 return verify_commit_graph_error
;
1989 progress
= start_progress(_("Verifying commits in commit graph"),
1991 for (i
= 0; i
< g
->num_commits
; i
++) {
1992 struct commit
*graph_commit
, *odb_commit
;
1993 struct commit_list
*graph_parents
, *odb_parents
;
1994 uint32_t max_generation
= 0;
1996 display_progress(progress
, i
+ 1);
1997 hashcpy(cur_oid
.hash
, g
->chunk_oid_lookup
+ g
->hash_len
* i
);
1999 graph_commit
= lookup_commit(r
, &cur_oid
);
2000 odb_commit
= (struct commit
*)create_object(r
, &cur_oid
, alloc_commit_node(r
));
2001 if (parse_commit_internal(odb_commit
, 0, 0)) {
2002 graph_report(_("failed to parse commit %s from object database for commit-graph"),
2003 oid_to_hex(&cur_oid
));
2007 if (!oideq(&get_commit_tree_in_graph_one(r
, g
, graph_commit
)->object
.oid
,
2008 get_commit_tree_oid(odb_commit
)))
2009 graph_report(_("root tree OID for commit %s in commit-graph is %s != %s"),
2010 oid_to_hex(&cur_oid
),
2011 oid_to_hex(get_commit_tree_oid(graph_commit
)),
2012 oid_to_hex(get_commit_tree_oid(odb_commit
)));
2014 graph_parents
= graph_commit
->parents
;
2015 odb_parents
= odb_commit
->parents
;
2017 while (graph_parents
) {
2018 if (odb_parents
== NULL
) {
2019 graph_report(_("commit-graph parent list for commit %s is too long"),
2020 oid_to_hex(&cur_oid
));
2024 /* parse parent in case it is in a base graph */
2025 parse_commit_in_graph_one(r
, g
, graph_parents
->item
);
2027 if (!oideq(&graph_parents
->item
->object
.oid
, &odb_parents
->item
->object
.oid
))
2028 graph_report(_("commit-graph parent for %s is %s != %s"),
2029 oid_to_hex(&cur_oid
),
2030 oid_to_hex(&graph_parents
->item
->object
.oid
),
2031 oid_to_hex(&odb_parents
->item
->object
.oid
));
2033 if (graph_parents
->item
->generation
> max_generation
)
2034 max_generation
= graph_parents
->item
->generation
;
2036 graph_parents
= graph_parents
->next
;
2037 odb_parents
= odb_parents
->next
;
2040 if (odb_parents
!= NULL
)
2041 graph_report(_("commit-graph parent list for commit %s terminates early"),
2042 oid_to_hex(&cur_oid
));
2044 if (!graph_commit
->generation
) {
2045 if (generation_zero
== GENERATION_NUMBER_EXISTS
)
2046 graph_report(_("commit-graph has generation number zero for commit %s, but non-zero elsewhere"),
2047 oid_to_hex(&cur_oid
));
2048 generation_zero
= GENERATION_ZERO_EXISTS
;
2049 } else if (generation_zero
== GENERATION_ZERO_EXISTS
)
2050 graph_report(_("commit-graph has non-zero generation number for commit %s, but zero elsewhere"),
2051 oid_to_hex(&cur_oid
));
2053 if (generation_zero
== GENERATION_ZERO_EXISTS
)
2057 * If one of our parents has generation GENERATION_NUMBER_MAX, then
2058 * our generation is also GENERATION_NUMBER_MAX. Decrement to avoid
2059 * extra logic in the following condition.
2061 if (max_generation
== GENERATION_NUMBER_MAX
)
2064 if (graph_commit
->generation
!= max_generation
+ 1)
2065 graph_report(_("commit-graph generation for commit %s is %u != %u"),
2066 oid_to_hex(&cur_oid
),
2067 graph_commit
->generation
,
2068 max_generation
+ 1);
2070 if (graph_commit
->date
!= odb_commit
->date
)
2071 graph_report(_("commit date for commit %s in commit-graph is %"PRItime
" != %"PRItime
),
2072 oid_to_hex(&cur_oid
),
2076 stop_progress(&progress
);
2078 local_error
= verify_commit_graph_error
;
2080 if (!(flags
& COMMIT_GRAPH_VERIFY_SHALLOW
) && g
->base_graph
)
2081 local_error
|= verify_commit_graph(r
, g
->base_graph
, flags
);
2086 void free_commit_graph(struct commit_graph
*g
)
2090 if (g
->graph_fd
>= 0) {
2091 munmap((void *)g
->data
, g
->data_len
);