3 #include "git-compat-util.h"
10 #include "sha1-lookup.h"
11 #include "commit-graph.h"
13 #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
14 #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
15 #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
16 #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
17 #define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
19 #define GRAPH_DATA_WIDTH 36
21 #define GRAPH_VERSION_1 0x1
22 #define GRAPH_VERSION GRAPH_VERSION_1
24 #define GRAPH_OID_VERSION_SHA1 1
25 #define GRAPH_OID_LEN_SHA1 GIT_SHA1_RAWSZ
26 #define GRAPH_OID_VERSION GRAPH_OID_VERSION_SHA1
27 #define GRAPH_OID_LEN GRAPH_OID_LEN_SHA1
29 #define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
30 #define GRAPH_PARENT_MISSING 0x7fffffff
31 #define GRAPH_EDGE_LAST_MASK 0x7fffffff
32 #define GRAPH_PARENT_NONE 0x70000000
34 #define GRAPH_LAST_EDGE 0x80000000
36 #define GRAPH_FANOUT_SIZE (4 * 256)
37 #define GRAPH_CHUNKLOOKUP_WIDTH 12
38 #define GRAPH_MIN_SIZE (5 * GRAPH_CHUNKLOOKUP_WIDTH + GRAPH_FANOUT_SIZE + \
41 char *get_commit_graph_filename(const char *obj_dir
)
43 return xstrfmt("%s/info/commit-graph", obj_dir
);
46 static struct commit_graph
*alloc_commit_graph(void)
48 struct commit_graph
*g
= xcalloc(1, sizeof(*g
));
54 struct commit_graph
*load_commit_graph_one(const char *graph_file
)
57 const unsigned char *data
, *chunk_lookup
;
61 struct commit_graph
*graph
;
62 int fd
= git_open(graph_file
);
63 uint64_t last_chunk_offset
;
64 uint32_t last_chunk_id
;
65 uint32_t graph_signature
;
66 unsigned char graph_version
, hash_version
;
74 graph_size
= xsize_t(st
.st_size
);
76 if (graph_size
< GRAPH_MIN_SIZE
) {
78 die("graph file %s is too small", graph_file
);
80 graph_map
= xmmap(NULL
, graph_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
81 data
= (const unsigned char *)graph_map
;
83 graph_signature
= get_be32(data
);
84 if (graph_signature
!= GRAPH_SIGNATURE
) {
85 error("graph signature %X does not match signature %X",
86 graph_signature
, GRAPH_SIGNATURE
);
90 graph_version
= *(unsigned char*)(data
+ 4);
91 if (graph_version
!= GRAPH_VERSION
) {
92 error("graph version %X does not match version %X",
93 graph_version
, GRAPH_VERSION
);
97 hash_version
= *(unsigned char*)(data
+ 5);
98 if (hash_version
!= GRAPH_OID_VERSION
) {
99 error("hash version %X does not match version %X",
100 hash_version
, GRAPH_OID_VERSION
);
104 graph
= alloc_commit_graph();
106 graph
->hash_len
= GRAPH_OID_LEN
;
107 graph
->num_chunks
= *(unsigned char*)(data
+ 6);
108 graph
->graph_fd
= fd
;
109 graph
->data
= graph_map
;
110 graph
->data_len
= graph_size
;
113 last_chunk_offset
= 8;
114 chunk_lookup
= data
+ 8;
115 for (i
= 0; i
< graph
->num_chunks
; i
++) {
116 uint32_t chunk_id
= get_be32(chunk_lookup
+ 0);
117 uint64_t chunk_offset
= get_be64(chunk_lookup
+ 4);
118 int chunk_repeated
= 0;
120 chunk_lookup
+= GRAPH_CHUNKLOOKUP_WIDTH
;
122 if (chunk_offset
> graph_size
- GIT_MAX_RAWSZ
) {
123 error("improper chunk offset %08x%08x", (uint32_t)(chunk_offset
>> 32),
124 (uint32_t)chunk_offset
);
129 case GRAPH_CHUNKID_OIDFANOUT
:
130 if (graph
->chunk_oid_fanout
)
133 graph
->chunk_oid_fanout
= (uint32_t*)(data
+ chunk_offset
);
136 case GRAPH_CHUNKID_OIDLOOKUP
:
137 if (graph
->chunk_oid_lookup
)
140 graph
->chunk_oid_lookup
= data
+ chunk_offset
;
143 case GRAPH_CHUNKID_DATA
:
144 if (graph
->chunk_commit_data
)
147 graph
->chunk_commit_data
= data
+ chunk_offset
;
150 case GRAPH_CHUNKID_LARGEEDGES
:
151 if (graph
->chunk_large_edges
)
154 graph
->chunk_large_edges
= data
+ chunk_offset
;
158 if (chunk_repeated
) {
159 error("chunk id %08x appears multiple times", chunk_id
);
163 if (last_chunk_id
== GRAPH_CHUNKID_OIDLOOKUP
)
165 graph
->num_commits
= (chunk_offset
- last_chunk_offset
)
169 last_chunk_id
= chunk_id
;
170 last_chunk_offset
= chunk_offset
;
176 munmap(graph_map
, graph_size
);
182 static struct commit_graph
*commit_graph
= NULL
;
184 static void prepare_commit_graph_one(const char *obj_dir
)
191 graph_name
= get_commit_graph_filename(obj_dir
);
192 commit_graph
= load_commit_graph_one(graph_name
);
194 FREE_AND_NULL(graph_name
);
197 static int prepare_commit_graph_run_once
= 0;
198 static void prepare_commit_graph(void)
200 struct alternate_object_database
*alt
;
203 if (prepare_commit_graph_run_once
)
205 prepare_commit_graph_run_once
= 1;
207 obj_dir
= get_object_directory();
208 prepare_commit_graph_one(obj_dir
);
210 for (alt
= alt_odb_list
; !commit_graph
&& alt
; alt
= alt
->next
)
211 prepare_commit_graph_one(alt
->path
);
214 static void close_commit_graph(void)
219 if (commit_graph
->graph_fd
>= 0) {
220 munmap((void *)commit_graph
->data
, commit_graph
->data_len
);
221 commit_graph
->data
= NULL
;
222 close(commit_graph
->graph_fd
);
225 FREE_AND_NULL(commit_graph
);
228 static int bsearch_graph(struct commit_graph
*g
, struct object_id
*oid
, uint32_t *pos
)
230 return bsearch_hash(oid
->hash
, g
->chunk_oid_fanout
,
231 g
->chunk_oid_lookup
, g
->hash_len
, pos
);
234 static struct commit_list
**insert_parent_or_die(struct commit_graph
*g
,
236 struct commit_list
**pptr
)
239 struct object_id oid
;
240 hashcpy(oid
.hash
, g
->chunk_oid_lookup
+ g
->hash_len
* pos
);
241 c
= lookup_commit(&oid
);
243 die("could not find commit %s", oid_to_hex(&oid
));
245 return &commit_list_insert(c
, pptr
)->next
;
248 static int fill_commit_in_graph(struct commit
*item
, struct commit_graph
*g
, uint32_t pos
)
251 uint32_t *parent_data_ptr
;
252 uint64_t date_low
, date_high
;
253 struct commit_list
**pptr
;
254 const unsigned char *commit_data
= g
->chunk_commit_data
+ (g
->hash_len
+ 16) * pos
;
256 item
->object
.parsed
= 1;
257 item
->graph_pos
= pos
;
259 item
->maybe_tree
= NULL
;
261 date_high
= get_be32(commit_data
+ g
->hash_len
+ 8) & 0x3;
262 date_low
= get_be32(commit_data
+ g
->hash_len
+ 12);
263 item
->date
= (timestamp_t
)((date_high
<< 32) | date_low
);
265 pptr
= &item
->parents
;
267 edge_value
= get_be32(commit_data
+ g
->hash_len
);
268 if (edge_value
== GRAPH_PARENT_NONE
)
270 pptr
= insert_parent_or_die(g
, edge_value
, pptr
);
272 edge_value
= get_be32(commit_data
+ g
->hash_len
+ 4);
273 if (edge_value
== GRAPH_PARENT_NONE
)
275 if (!(edge_value
& GRAPH_OCTOPUS_EDGES_NEEDED
)) {
276 pptr
= insert_parent_or_die(g
, edge_value
, pptr
);
280 parent_data_ptr
= (uint32_t*)(g
->chunk_large_edges
+
281 4 * (uint64_t)(edge_value
& GRAPH_EDGE_LAST_MASK
));
283 edge_value
= get_be32(parent_data_ptr
);
284 pptr
= insert_parent_or_die(g
,
285 edge_value
& GRAPH_EDGE_LAST_MASK
,
288 } while (!(edge_value
& GRAPH_LAST_EDGE
));
293 int parse_commit_in_graph(struct commit
*item
)
295 if (!core_commit_graph
)
297 if (item
->object
.parsed
)
300 prepare_commit_graph();
304 if (item
->graph_pos
!= COMMIT_NOT_FROM_GRAPH
) {
305 pos
= item
->graph_pos
;
308 found
= bsearch_graph(commit_graph
, &(item
->object
.oid
), &pos
);
312 return fill_commit_in_graph(item
, commit_graph
, pos
);
318 static struct tree
*load_tree_for_commit(struct commit_graph
*g
, struct commit
*c
)
320 struct object_id oid
;
321 const unsigned char *commit_data
= g
->chunk_commit_data
+
322 GRAPH_DATA_WIDTH
* (c
->graph_pos
);
324 hashcpy(oid
.hash
, commit_data
);
325 c
->maybe_tree
= lookup_tree(&oid
);
327 return c
->maybe_tree
;
330 struct tree
*get_commit_tree_in_graph(const struct commit
*c
)
333 return c
->maybe_tree
;
334 if (c
->graph_pos
== COMMIT_NOT_FROM_GRAPH
)
335 BUG("get_commit_tree_in_graph called from non-commit-graph commit");
337 return load_tree_for_commit(commit_graph
, (struct commit
*)c
);
340 static void write_graph_chunk_fanout(struct hashfile
*f
,
341 struct commit
**commits
,
345 struct commit
**list
= commits
;
348 * Write the first-level table (the list is sorted,
349 * but we use a 256-entry lookup to be able to avoid
350 * having to do eight extra binary search iterations).
352 for (i
= 0; i
< 256; i
++) {
353 while (count
< nr_commits
) {
354 if ((*list
)->object
.oid
.hash
[0] != i
)
360 hashwrite_be32(f
, count
);
364 static void write_graph_chunk_oids(struct hashfile
*f
, int hash_len
,
365 struct commit
**commits
, int nr_commits
)
367 struct commit
**list
= commits
;
369 for (count
= 0; count
< nr_commits
; count
++, list
++)
370 hashwrite(f
, (*list
)->object
.oid
.hash
, (int)hash_len
);
373 static const unsigned char *commit_to_sha1(size_t index
, void *table
)
375 struct commit
**commits
= table
;
376 return commits
[index
]->object
.oid
.hash
;
379 static void write_graph_chunk_data(struct hashfile
*f
, int hash_len
,
380 struct commit
**commits
, int nr_commits
)
382 struct commit
**list
= commits
;
383 struct commit
**last
= commits
+ nr_commits
;
384 uint32_t num_extra_edges
= 0;
386 while (list
< last
) {
387 struct commit_list
*parent
;
389 uint32_t packedDate
[2];
392 hashwrite(f
, get_commit_tree_oid(*list
)->hash
, hash_len
);
394 parent
= (*list
)->parents
;
397 edge_value
= GRAPH_PARENT_NONE
;
399 edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
405 edge_value
= GRAPH_PARENT_MISSING
;
408 hashwrite_be32(f
, edge_value
);
411 parent
= parent
->next
;
414 edge_value
= GRAPH_PARENT_NONE
;
415 else if (parent
->next
)
416 edge_value
= GRAPH_OCTOPUS_EDGES_NEEDED
| num_extra_edges
;
418 edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
423 edge_value
= GRAPH_PARENT_MISSING
;
426 hashwrite_be32(f
, edge_value
);
428 if (edge_value
& GRAPH_OCTOPUS_EDGES_NEEDED
) {
431 parent
= parent
->next
;
435 if (sizeof((*list
)->date
) > 4)
436 packedDate
[0] = htonl(((*list
)->date
>> 32) & 0x3);
440 packedDate
[1] = htonl((*list
)->date
);
441 hashwrite(f
, packedDate
, 8);
447 static void write_graph_chunk_large_edges(struct hashfile
*f
,
448 struct commit
**commits
,
451 struct commit
**list
= commits
;
452 struct commit
**last
= commits
+ nr_commits
;
453 struct commit_list
*parent
;
455 while (list
< last
) {
457 for (parent
= (*list
)->parents
; num_parents
< 3 && parent
;
458 parent
= parent
->next
)
461 if (num_parents
<= 2) {
466 /* Since num_parents > 2, this initializer is safe. */
467 for (parent
= (*list
)->parents
->next
; parent
; parent
= parent
->next
) {
468 int edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
474 edge_value
= GRAPH_PARENT_MISSING
;
475 else if (!parent
->next
)
476 edge_value
|= GRAPH_LAST_EDGE
;
478 hashwrite_be32(f
, edge_value
);
485 static int commit_compare(const void *_a
, const void *_b
)
487 const struct object_id
*a
= (const struct object_id
*)_a
;
488 const struct object_id
*b
= (const struct object_id
*)_b
;
492 struct packed_commit_list
{
493 struct commit
**list
;
498 struct packed_oid_list
{
499 struct object_id
*list
;
504 static int add_packed_commits(const struct object_id
*oid
,
505 struct packed_git
*pack
,
509 struct packed_oid_list
*list
= (struct packed_oid_list
*)data
;
510 enum object_type type
;
511 off_t offset
= nth_packed_object_offset(pack
, pos
);
512 struct object_info oi
= OBJECT_INFO_INIT
;
515 if (packed_object_info(pack
, offset
, &oi
) < 0)
516 die("unable to get type of object %s", oid_to_hex(oid
));
518 if (type
!= OBJ_COMMIT
)
521 ALLOC_GROW(list
->list
, list
->nr
+ 1, list
->alloc
);
522 oidcpy(&(list
->list
[list
->nr
]), oid
);
528 static void add_missing_parents(struct packed_oid_list
*oids
, struct commit
*commit
)
530 struct commit_list
*parent
;
531 for (parent
= commit
->parents
; parent
; parent
= parent
->next
) {
532 if (!(parent
->item
->object
.flags
& UNINTERESTING
)) {
533 ALLOC_GROW(oids
->list
, oids
->nr
+ 1, oids
->alloc
);
534 oidcpy(&oids
->list
[oids
->nr
], &(parent
->item
->object
.oid
));
536 parent
->item
->object
.flags
|= UNINTERESTING
;
541 static void close_reachable(struct packed_oid_list
*oids
)
544 struct commit
*commit
;
546 for (i
= 0; i
< oids
->nr
; i
++) {
547 commit
= lookup_commit(&oids
->list
[i
]);
549 commit
->object
.flags
|= UNINTERESTING
;
553 * As this loop runs, oids->nr may grow, but not more
554 * than the number of missing commits in the reachable
557 for (i
= 0; i
< oids
->nr
; i
++) {
558 commit
= lookup_commit(&oids
->list
[i
]);
560 if (commit
&& !parse_commit(commit
))
561 add_missing_parents(oids
, commit
);
564 for (i
= 0; i
< oids
->nr
; i
++) {
565 commit
= lookup_commit(&oids
->list
[i
]);
568 commit
->object
.flags
&= ~UNINTERESTING
;
572 void write_commit_graph(const char *obj_dir
,
573 const char **pack_indexes
,
575 const char **commit_hex
,
579 struct packed_oid_list oids
;
580 struct packed_commit_list commits
;
582 uint32_t i
, count_distinct
= 0;
585 struct lock_file lk
= LOCK_INIT
;
586 uint32_t chunk_ids
[5];
587 uint64_t chunk_offsets
[5];
590 struct commit_list
*parent
;
593 oids
.alloc
= approximate_object_count() / 4;
596 prepare_commit_graph_one(obj_dir
);
598 oids
.alloc
+= commit_graph
->num_commits
;
601 if (oids
.alloc
< 1024)
603 ALLOC_ARRAY(oids
.list
, oids
.alloc
);
605 if (append
&& commit_graph
) {
606 for (i
= 0; i
< commit_graph
->num_commits
; i
++) {
607 const unsigned char *hash
= commit_graph
->chunk_oid_lookup
+
608 commit_graph
->hash_len
* i
;
609 hashcpy(oids
.list
[oids
.nr
++].hash
, hash
);
614 struct strbuf packname
= STRBUF_INIT
;
616 strbuf_addf(&packname
, "%s/pack/", obj_dir
);
617 dirlen
= packname
.len
;
618 for (i
= 0; i
< nr_packs
; i
++) {
619 struct packed_git
*p
;
620 strbuf_setlen(&packname
, dirlen
);
621 strbuf_addstr(&packname
, pack_indexes
[i
]);
622 p
= add_packed_git(packname
.buf
, packname
.len
, 1);
624 die("error adding pack %s", packname
.buf
);
625 if (open_pack_index(p
))
626 die("error opening index for %s", packname
.buf
);
627 for_each_object_in_pack(p
, add_packed_commits
, &oids
);
630 strbuf_release(&packname
);
634 for (i
= 0; i
< nr_commits
; i
++) {
636 struct object_id oid
;
637 struct commit
*result
;
639 if (commit_hex
[i
] && parse_oid_hex(commit_hex
[i
], &oid
, &end
))
642 result
= lookup_commit_reference_gently(&oid
, 1);
645 ALLOC_GROW(oids
.list
, oids
.nr
+ 1, oids
.alloc
);
646 oidcpy(&oids
.list
[oids
.nr
], &(result
->object
.oid
));
652 if (!pack_indexes
&& !commit_hex
)
653 for_each_packed_object(add_packed_commits
, &oids
, 0);
655 close_reachable(&oids
);
657 QSORT(oids
.list
, oids
.nr
, commit_compare
);
660 for (i
= 1; i
< oids
.nr
; i
++) {
661 if (oidcmp(&oids
.list
[i
-1], &oids
.list
[i
]))
665 if (count_distinct
>= GRAPH_PARENT_MISSING
)
666 die(_("the commit graph format cannot write %d commits"), count_distinct
);
669 commits
.alloc
= count_distinct
;
670 ALLOC_ARRAY(commits
.list
, commits
.alloc
);
673 for (i
= 0; i
< oids
.nr
; i
++) {
675 if (i
> 0 && !oidcmp(&oids
.list
[i
-1], &oids
.list
[i
]))
678 commits
.list
[commits
.nr
] = lookup_commit(&oids
.list
[i
]);
679 parse_commit(commits
.list
[commits
.nr
]);
681 for (parent
= commits
.list
[commits
.nr
]->parents
;
682 parent
; parent
= parent
->next
)
686 num_extra_edges
+= num_parents
- 1;
690 num_chunks
= num_extra_edges
? 4 : 3;
692 if (commits
.nr
>= GRAPH_PARENT_MISSING
)
693 die(_("too many commits to write graph"));
695 graph_name
= get_commit_graph_filename(obj_dir
);
696 fd
= hold_lock_file_for_update(&lk
, graph_name
, 0);
699 struct strbuf folder
= STRBUF_INIT
;
700 strbuf_addstr(&folder
, graph_name
);
701 strbuf_setlen(&folder
, strrchr(folder
.buf
, '/') - folder
.buf
);
703 if (mkdir(folder
.buf
, 0777) < 0)
704 die_errno(_("cannot mkdir %s"), folder
.buf
);
705 strbuf_release(&folder
);
707 fd
= hold_lock_file_for_update(&lk
, graph_name
, LOCK_DIE_ON_ERROR
);
710 die_errno("unable to create '%s'", graph_name
);
713 f
= hashfd(lk
.tempfile
->fd
, lk
.tempfile
->filename
.buf
);
715 hashwrite_be32(f
, GRAPH_SIGNATURE
);
717 hashwrite_u8(f
, GRAPH_VERSION
);
718 hashwrite_u8(f
, GRAPH_OID_VERSION
);
719 hashwrite_u8(f
, num_chunks
);
720 hashwrite_u8(f
, 0); /* unused padding byte */
722 chunk_ids
[0] = GRAPH_CHUNKID_OIDFANOUT
;
723 chunk_ids
[1] = GRAPH_CHUNKID_OIDLOOKUP
;
724 chunk_ids
[2] = GRAPH_CHUNKID_DATA
;
726 chunk_ids
[3] = GRAPH_CHUNKID_LARGEEDGES
;
731 chunk_offsets
[0] = 8 + (num_chunks
+ 1) * GRAPH_CHUNKLOOKUP_WIDTH
;
732 chunk_offsets
[1] = chunk_offsets
[0] + GRAPH_FANOUT_SIZE
;
733 chunk_offsets
[2] = chunk_offsets
[1] + GRAPH_OID_LEN
* commits
.nr
;
734 chunk_offsets
[3] = chunk_offsets
[2] + (GRAPH_OID_LEN
+ 16) * commits
.nr
;
735 chunk_offsets
[4] = chunk_offsets
[3] + 4 * num_extra_edges
;
737 for (i
= 0; i
<= num_chunks
; i
++) {
738 uint32_t chunk_write
[3];
740 chunk_write
[0] = htonl(chunk_ids
[i
]);
741 chunk_write
[1] = htonl(chunk_offsets
[i
] >> 32);
742 chunk_write
[2] = htonl(chunk_offsets
[i
] & 0xffffffff);
743 hashwrite(f
, chunk_write
, 12);
746 write_graph_chunk_fanout(f
, commits
.list
, commits
.nr
);
747 write_graph_chunk_oids(f
, GRAPH_OID_LEN
, commits
.list
, commits
.nr
);
748 write_graph_chunk_data(f
, GRAPH_OID_LEN
, commits
.list
, commits
.nr
);
749 write_graph_chunk_large_edges(f
, commits
.list
, commits
.nr
);
751 close_commit_graph();
752 finalize_hashfile(f
, NULL
, CSUM_HASH_IN_STREAM
| CSUM_FSYNC
);
753 commit_lock_file(&lk
);