3 #include "git-compat-util.h"
10 #include "sha1-lookup.h"
11 #include "commit-graph.h"
12 #include "object-store.h"
14 #define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
15 #define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
16 #define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
17 #define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
18 #define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
20 #define GRAPH_DATA_WIDTH 36
22 #define GRAPH_VERSION_1 0x1
23 #define GRAPH_VERSION GRAPH_VERSION_1
25 #define GRAPH_OID_VERSION_SHA1 1
26 #define GRAPH_OID_LEN_SHA1 GIT_SHA1_RAWSZ
27 #define GRAPH_OID_VERSION GRAPH_OID_VERSION_SHA1
28 #define GRAPH_OID_LEN GRAPH_OID_LEN_SHA1
30 #define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
31 #define GRAPH_PARENT_MISSING 0x7fffffff
32 #define GRAPH_EDGE_LAST_MASK 0x7fffffff
33 #define GRAPH_PARENT_NONE 0x70000000
35 #define GRAPH_LAST_EDGE 0x80000000
37 #define GRAPH_FANOUT_SIZE (4 * 256)
38 #define GRAPH_CHUNKLOOKUP_WIDTH 12
39 #define GRAPH_MIN_SIZE (5 * GRAPH_CHUNKLOOKUP_WIDTH + GRAPH_FANOUT_SIZE + \
42 char *get_commit_graph_filename(const char *obj_dir
)
44 return xstrfmt("%s/info/commit-graph", obj_dir
);
47 static struct commit_graph
*alloc_commit_graph(void)
49 struct commit_graph
*g
= xcalloc(1, sizeof(*g
));
55 struct commit_graph
*load_commit_graph_one(const char *graph_file
)
58 const unsigned char *data
, *chunk_lookup
;
62 struct commit_graph
*graph
;
63 int fd
= git_open(graph_file
);
64 uint64_t last_chunk_offset
;
65 uint32_t last_chunk_id
;
66 uint32_t graph_signature
;
67 unsigned char graph_version
, hash_version
;
75 graph_size
= xsize_t(st
.st_size
);
77 if (graph_size
< GRAPH_MIN_SIZE
) {
79 die("graph file %s is too small", graph_file
);
81 graph_map
= xmmap(NULL
, graph_size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
82 data
= (const unsigned char *)graph_map
;
84 graph_signature
= get_be32(data
);
85 if (graph_signature
!= GRAPH_SIGNATURE
) {
86 error("graph signature %X does not match signature %X",
87 graph_signature
, GRAPH_SIGNATURE
);
91 graph_version
= *(unsigned char*)(data
+ 4);
92 if (graph_version
!= GRAPH_VERSION
) {
93 error("graph version %X does not match version %X",
94 graph_version
, GRAPH_VERSION
);
98 hash_version
= *(unsigned char*)(data
+ 5);
99 if (hash_version
!= GRAPH_OID_VERSION
) {
100 error("hash version %X does not match version %X",
101 hash_version
, GRAPH_OID_VERSION
);
105 graph
= alloc_commit_graph();
107 graph
->hash_len
= GRAPH_OID_LEN
;
108 graph
->num_chunks
= *(unsigned char*)(data
+ 6);
109 graph
->graph_fd
= fd
;
110 graph
->data
= graph_map
;
111 graph
->data_len
= graph_size
;
114 last_chunk_offset
= 8;
115 chunk_lookup
= data
+ 8;
116 for (i
= 0; i
< graph
->num_chunks
; i
++) {
117 uint32_t chunk_id
= get_be32(chunk_lookup
+ 0);
118 uint64_t chunk_offset
= get_be64(chunk_lookup
+ 4);
119 int chunk_repeated
= 0;
121 chunk_lookup
+= GRAPH_CHUNKLOOKUP_WIDTH
;
123 if (chunk_offset
> graph_size
- GIT_MAX_RAWSZ
) {
124 error("improper chunk offset %08x%08x", (uint32_t)(chunk_offset
>> 32),
125 (uint32_t)chunk_offset
);
130 case GRAPH_CHUNKID_OIDFANOUT
:
131 if (graph
->chunk_oid_fanout
)
134 graph
->chunk_oid_fanout
= (uint32_t*)(data
+ chunk_offset
);
137 case GRAPH_CHUNKID_OIDLOOKUP
:
138 if (graph
->chunk_oid_lookup
)
141 graph
->chunk_oid_lookup
= data
+ chunk_offset
;
144 case GRAPH_CHUNKID_DATA
:
145 if (graph
->chunk_commit_data
)
148 graph
->chunk_commit_data
= data
+ chunk_offset
;
151 case GRAPH_CHUNKID_LARGEEDGES
:
152 if (graph
->chunk_large_edges
)
155 graph
->chunk_large_edges
= data
+ chunk_offset
;
159 if (chunk_repeated
) {
160 error("chunk id %08x appears multiple times", chunk_id
);
164 if (last_chunk_id
== GRAPH_CHUNKID_OIDLOOKUP
)
166 graph
->num_commits
= (chunk_offset
- last_chunk_offset
)
170 last_chunk_id
= chunk_id
;
171 last_chunk_offset
= chunk_offset
;
177 munmap(graph_map
, graph_size
);
183 static struct commit_graph
*commit_graph
= NULL
;
185 static void prepare_commit_graph_one(const char *obj_dir
)
192 graph_name
= get_commit_graph_filename(obj_dir
);
193 commit_graph
= load_commit_graph_one(graph_name
);
195 FREE_AND_NULL(graph_name
);
198 static int prepare_commit_graph_run_once
= 0;
199 static void prepare_commit_graph(void)
201 struct alternate_object_database
*alt
;
204 if (prepare_commit_graph_run_once
)
206 prepare_commit_graph_run_once
= 1;
208 obj_dir
= get_object_directory();
209 prepare_commit_graph_one(obj_dir
);
210 prepare_alt_odb(the_repository
);
211 for (alt
= the_repository
->objects
->alt_odb_list
;
212 !commit_graph
&& alt
;
214 prepare_commit_graph_one(alt
->path
);
217 static void close_commit_graph(void)
222 if (commit_graph
->graph_fd
>= 0) {
223 munmap((void *)commit_graph
->data
, commit_graph
->data_len
);
224 commit_graph
->data
= NULL
;
225 close(commit_graph
->graph_fd
);
228 FREE_AND_NULL(commit_graph
);
231 static int bsearch_graph(struct commit_graph
*g
, struct object_id
*oid
, uint32_t *pos
)
233 return bsearch_hash(oid
->hash
, g
->chunk_oid_fanout
,
234 g
->chunk_oid_lookup
, g
->hash_len
, pos
);
237 static struct commit_list
**insert_parent_or_die(struct commit_graph
*g
,
239 struct commit_list
**pptr
)
242 struct object_id oid
;
243 hashcpy(oid
.hash
, g
->chunk_oid_lookup
+ g
->hash_len
* pos
);
244 c
= lookup_commit(&oid
);
246 die("could not find commit %s", oid_to_hex(&oid
));
248 return &commit_list_insert(c
, pptr
)->next
;
251 static int fill_commit_in_graph(struct commit
*item
, struct commit_graph
*g
, uint32_t pos
)
253 struct object_id oid
;
255 uint32_t *parent_data_ptr
;
256 uint64_t date_low
, date_high
;
257 struct commit_list
**pptr
;
258 const unsigned char *commit_data
= g
->chunk_commit_data
+ (g
->hash_len
+ 16) * pos
;
260 item
->object
.parsed
= 1;
261 item
->graph_pos
= pos
;
263 hashcpy(oid
.hash
, commit_data
);
264 item
->tree
= lookup_tree(&oid
);
266 date_high
= get_be32(commit_data
+ g
->hash_len
+ 8) & 0x3;
267 date_low
= get_be32(commit_data
+ g
->hash_len
+ 12);
268 item
->date
= (timestamp_t
)((date_high
<< 32) | date_low
);
270 pptr
= &item
->parents
;
272 edge_value
= get_be32(commit_data
+ g
->hash_len
);
273 if (edge_value
== GRAPH_PARENT_NONE
)
275 pptr
= insert_parent_or_die(g
, edge_value
, pptr
);
277 edge_value
= get_be32(commit_data
+ g
->hash_len
+ 4);
278 if (edge_value
== GRAPH_PARENT_NONE
)
280 if (!(edge_value
& GRAPH_OCTOPUS_EDGES_NEEDED
)) {
281 pptr
= insert_parent_or_die(g
, edge_value
, pptr
);
285 parent_data_ptr
= (uint32_t*)(g
->chunk_large_edges
+
286 4 * (uint64_t)(edge_value
& GRAPH_EDGE_LAST_MASK
));
288 edge_value
= get_be32(parent_data_ptr
);
289 pptr
= insert_parent_or_die(g
,
290 edge_value
& GRAPH_EDGE_LAST_MASK
,
293 } while (!(edge_value
& GRAPH_LAST_EDGE
));
298 int parse_commit_in_graph(struct commit
*item
)
300 if (!core_commit_graph
)
302 if (item
->object
.parsed
)
305 prepare_commit_graph();
309 if (item
->graph_pos
!= COMMIT_NOT_FROM_GRAPH
) {
310 pos
= item
->graph_pos
;
313 found
= bsearch_graph(commit_graph
, &(item
->object
.oid
), &pos
);
317 return fill_commit_in_graph(item
, commit_graph
, pos
);
323 static void write_graph_chunk_fanout(struct hashfile
*f
,
324 struct commit
**commits
,
328 struct commit
**list
= commits
;
331 * Write the first-level table (the list is sorted,
332 * but we use a 256-entry lookup to be able to avoid
333 * having to do eight extra binary search iterations).
335 for (i
= 0; i
< 256; i
++) {
336 while (count
< nr_commits
) {
337 if ((*list
)->object
.oid
.hash
[0] != i
)
343 hashwrite_be32(f
, count
);
347 static void write_graph_chunk_oids(struct hashfile
*f
, int hash_len
,
348 struct commit
**commits
, int nr_commits
)
350 struct commit
**list
= commits
;
352 for (count
= 0; count
< nr_commits
; count
++, list
++)
353 hashwrite(f
, (*list
)->object
.oid
.hash
, (int)hash_len
);
356 static const unsigned char *commit_to_sha1(size_t index
, void *table
)
358 struct commit
**commits
= table
;
359 return commits
[index
]->object
.oid
.hash
;
362 static void write_graph_chunk_data(struct hashfile
*f
, int hash_len
,
363 struct commit
**commits
, int nr_commits
)
365 struct commit
**list
= commits
;
366 struct commit
**last
= commits
+ nr_commits
;
367 uint32_t num_extra_edges
= 0;
369 while (list
< last
) {
370 struct commit_list
*parent
;
372 uint32_t packedDate
[2];
375 hashwrite(f
, (*list
)->tree
->object
.oid
.hash
, hash_len
);
377 parent
= (*list
)->parents
;
380 edge_value
= GRAPH_PARENT_NONE
;
382 edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
388 edge_value
= GRAPH_PARENT_MISSING
;
391 hashwrite_be32(f
, edge_value
);
394 parent
= parent
->next
;
397 edge_value
= GRAPH_PARENT_NONE
;
398 else if (parent
->next
)
399 edge_value
= GRAPH_OCTOPUS_EDGES_NEEDED
| num_extra_edges
;
401 edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
406 edge_value
= GRAPH_PARENT_MISSING
;
409 hashwrite_be32(f
, edge_value
);
411 if (edge_value
& GRAPH_OCTOPUS_EDGES_NEEDED
) {
414 parent
= parent
->next
;
418 if (sizeof((*list
)->date
) > 4)
419 packedDate
[0] = htonl(((*list
)->date
>> 32) & 0x3);
423 packedDate
[1] = htonl((*list
)->date
);
424 hashwrite(f
, packedDate
, 8);
430 static void write_graph_chunk_large_edges(struct hashfile
*f
,
431 struct commit
**commits
,
434 struct commit
**list
= commits
;
435 struct commit
**last
= commits
+ nr_commits
;
436 struct commit_list
*parent
;
438 while (list
< last
) {
440 for (parent
= (*list
)->parents
; num_parents
< 3 && parent
;
441 parent
= parent
->next
)
444 if (num_parents
<= 2) {
449 /* Since num_parents > 2, this initializer is safe. */
450 for (parent
= (*list
)->parents
->next
; parent
; parent
= parent
->next
) {
451 int edge_value
= sha1_pos(parent
->item
->object
.oid
.hash
,
457 edge_value
= GRAPH_PARENT_MISSING
;
458 else if (!parent
->next
)
459 edge_value
|= GRAPH_LAST_EDGE
;
461 hashwrite_be32(f
, edge_value
);
468 static int commit_compare(const void *_a
, const void *_b
)
470 const struct object_id
*a
= (const struct object_id
*)_a
;
471 const struct object_id
*b
= (const struct object_id
*)_b
;
475 struct packed_commit_list
{
476 struct commit
**list
;
481 struct packed_oid_list
{
482 struct object_id
*list
;
487 static int add_packed_commits(const struct object_id
*oid
,
488 struct packed_git
*pack
,
492 struct packed_oid_list
*list
= (struct packed_oid_list
*)data
;
493 enum object_type type
;
494 off_t offset
= nth_packed_object_offset(pack
, pos
);
495 struct object_info oi
= OBJECT_INFO_INIT
;
498 if (packed_object_info(pack
, offset
, &oi
) < 0)
499 die("unable to get type of object %s", oid_to_hex(oid
));
501 if (type
!= OBJ_COMMIT
)
504 ALLOC_GROW(list
->list
, list
->nr
+ 1, list
->alloc
);
505 oidcpy(&(list
->list
[list
->nr
]), oid
);
511 static void add_missing_parents(struct packed_oid_list
*oids
, struct commit
*commit
)
513 struct commit_list
*parent
;
514 for (parent
= commit
->parents
; parent
; parent
= parent
->next
) {
515 if (!(parent
->item
->object
.flags
& UNINTERESTING
)) {
516 ALLOC_GROW(oids
->list
, oids
->nr
+ 1, oids
->alloc
);
517 oidcpy(&oids
->list
[oids
->nr
], &(parent
->item
->object
.oid
));
519 parent
->item
->object
.flags
|= UNINTERESTING
;
524 static void close_reachable(struct packed_oid_list
*oids
)
527 struct commit
*commit
;
529 for (i
= 0; i
< oids
->nr
; i
++) {
530 commit
= lookup_commit(&oids
->list
[i
]);
532 commit
->object
.flags
|= UNINTERESTING
;
536 * As this loop runs, oids->nr may grow, but not more
537 * than the number of missing commits in the reachable
540 for (i
= 0; i
< oids
->nr
; i
++) {
541 commit
= lookup_commit(&oids
->list
[i
]);
543 if (commit
&& !parse_commit(commit
))
544 add_missing_parents(oids
, commit
);
547 for (i
= 0; i
< oids
->nr
; i
++) {
548 commit
= lookup_commit(&oids
->list
[i
]);
551 commit
->object
.flags
&= ~UNINTERESTING
;
555 void write_commit_graph(const char *obj_dir
,
556 const char **pack_indexes
,
558 const char **commit_hex
,
562 struct packed_oid_list oids
;
563 struct packed_commit_list commits
;
565 uint32_t i
, count_distinct
= 0;
568 struct lock_file lk
= LOCK_INIT
;
569 uint32_t chunk_ids
[5];
570 uint64_t chunk_offsets
[5];
573 struct commit_list
*parent
;
576 oids
.alloc
= approximate_object_count() / 4;
579 prepare_commit_graph_one(obj_dir
);
581 oids
.alloc
+= commit_graph
->num_commits
;
584 if (oids
.alloc
< 1024)
586 ALLOC_ARRAY(oids
.list
, oids
.alloc
);
588 if (append
&& commit_graph
) {
589 for (i
= 0; i
< commit_graph
->num_commits
; i
++) {
590 const unsigned char *hash
= commit_graph
->chunk_oid_lookup
+
591 commit_graph
->hash_len
* i
;
592 hashcpy(oids
.list
[oids
.nr
++].hash
, hash
);
597 struct strbuf packname
= STRBUF_INIT
;
599 strbuf_addf(&packname
, "%s/pack/", obj_dir
);
600 dirlen
= packname
.len
;
601 for (i
= 0; i
< nr_packs
; i
++) {
602 struct packed_git
*p
;
603 strbuf_setlen(&packname
, dirlen
);
604 strbuf_addstr(&packname
, pack_indexes
[i
]);
605 p
= add_packed_git(packname
.buf
, packname
.len
, 1);
607 die("error adding pack %s", packname
.buf
);
608 if (open_pack_index(p
))
609 die("error opening index for %s", packname
.buf
);
610 for_each_object_in_pack(p
, add_packed_commits
, &oids
);
613 strbuf_release(&packname
);
617 for (i
= 0; i
< nr_commits
; i
++) {
619 struct object_id oid
;
620 struct commit
*result
;
622 if (commit_hex
[i
] && parse_oid_hex(commit_hex
[i
], &oid
, &end
))
625 result
= lookup_commit_reference_gently(&oid
, 1);
628 ALLOC_GROW(oids
.list
, oids
.nr
+ 1, oids
.alloc
);
629 oidcpy(&oids
.list
[oids
.nr
], &(result
->object
.oid
));
635 if (!pack_indexes
&& !commit_hex
)
636 for_each_packed_object(add_packed_commits
, &oids
, 0);
638 close_reachable(&oids
);
640 QSORT(oids
.list
, oids
.nr
, commit_compare
);
643 for (i
= 1; i
< oids
.nr
; i
++) {
644 if (oidcmp(&oids
.list
[i
-1], &oids
.list
[i
]))
648 if (count_distinct
>= GRAPH_PARENT_MISSING
)
649 die(_("the commit graph format cannot write %d commits"), count_distinct
);
652 commits
.alloc
= count_distinct
;
653 ALLOC_ARRAY(commits
.list
, commits
.alloc
);
656 for (i
= 0; i
< oids
.nr
; i
++) {
658 if (i
> 0 && !oidcmp(&oids
.list
[i
-1], &oids
.list
[i
]))
661 commits
.list
[commits
.nr
] = lookup_commit(&oids
.list
[i
]);
662 parse_commit(commits
.list
[commits
.nr
]);
664 for (parent
= commits
.list
[commits
.nr
]->parents
;
665 parent
; parent
= parent
->next
)
669 num_extra_edges
+= num_parents
- 1;
673 num_chunks
= num_extra_edges
? 4 : 3;
675 if (commits
.nr
>= GRAPH_PARENT_MISSING
)
676 die(_("too many commits to write graph"));
678 graph_name
= get_commit_graph_filename(obj_dir
);
679 fd
= hold_lock_file_for_update(&lk
, graph_name
, 0);
682 struct strbuf folder
= STRBUF_INIT
;
683 strbuf_addstr(&folder
, graph_name
);
684 strbuf_setlen(&folder
, strrchr(folder
.buf
, '/') - folder
.buf
);
686 if (mkdir(folder
.buf
, 0777) < 0)
687 die_errno(_("cannot mkdir %s"), folder
.buf
);
688 strbuf_release(&folder
);
690 fd
= hold_lock_file_for_update(&lk
, graph_name
, LOCK_DIE_ON_ERROR
);
693 die_errno("unable to create '%s'", graph_name
);
696 f
= hashfd(lk
.tempfile
->fd
, lk
.tempfile
->filename
.buf
);
698 hashwrite_be32(f
, GRAPH_SIGNATURE
);
700 hashwrite_u8(f
, GRAPH_VERSION
);
701 hashwrite_u8(f
, GRAPH_OID_VERSION
);
702 hashwrite_u8(f
, num_chunks
);
703 hashwrite_u8(f
, 0); /* unused padding byte */
705 chunk_ids
[0] = GRAPH_CHUNKID_OIDFANOUT
;
706 chunk_ids
[1] = GRAPH_CHUNKID_OIDLOOKUP
;
707 chunk_ids
[2] = GRAPH_CHUNKID_DATA
;
709 chunk_ids
[3] = GRAPH_CHUNKID_LARGEEDGES
;
714 chunk_offsets
[0] = 8 + (num_chunks
+ 1) * GRAPH_CHUNKLOOKUP_WIDTH
;
715 chunk_offsets
[1] = chunk_offsets
[0] + GRAPH_FANOUT_SIZE
;
716 chunk_offsets
[2] = chunk_offsets
[1] + GRAPH_OID_LEN
* commits
.nr
;
717 chunk_offsets
[3] = chunk_offsets
[2] + (GRAPH_OID_LEN
+ 16) * commits
.nr
;
718 chunk_offsets
[4] = chunk_offsets
[3] + 4 * num_extra_edges
;
720 for (i
= 0; i
<= num_chunks
; i
++) {
721 uint32_t chunk_write
[3];
723 chunk_write
[0] = htonl(chunk_ids
[i
]);
724 chunk_write
[1] = htonl(chunk_offsets
[i
] >> 32);
725 chunk_write
[2] = htonl(chunk_offsets
[i
] & 0xffffffff);
726 hashwrite(f
, chunk_write
, 12);
729 write_graph_chunk_fanout(f
, commits
.list
, commits
.nr
);
730 write_graph_chunk_oids(f
, GRAPH_OID_LEN
, commits
.list
, commits
.nr
);
731 write_graph_chunk_data(f
, GRAPH_OID_LEN
, commits
.list
, commits
.nr
);
732 write_graph_chunk_large_edges(f
, commits
.list
, commits
.nr
);
734 close_commit_graph();
735 finalize_hashfile(f
, NULL
, CSUM_HASH_IN_STREAM
| CSUM_FSYNC
);
736 commit_lock_file(&lk
);