2 #include "environment.h"
8 #include "chunk-format.h"
9 #include "pack-mtimes.h"
11 #include "pack-objects.h"
13 void reset_pack_idx_option(struct pack_idx_option
*opts
)
15 memset(opts
, 0, sizeof(*opts
));
17 opts
->off32_limit
= 0x7fffffff;
20 static int sha1_compare(const void *_a
, const void *_b
)
22 struct pack_idx_entry
*a
= *(struct pack_idx_entry
**)_a
;
23 struct pack_idx_entry
*b
= *(struct pack_idx_entry
**)_b
;
24 return oidcmp(&a
->oid
, &b
->oid
);
27 static int cmp_uint32(const void *a_
, const void *b_
)
29 uint32_t a
= *((uint32_t *)a_
);
30 uint32_t b
= *((uint32_t *)b_
);
32 return (a
< b
) ? -1 : (a
!= b
);
35 static int need_large_offset(off_t offset
, const struct pack_idx_option
*opts
)
39 if ((offset
>> 31) || (opts
->off32_limit
< offset
))
41 if (!opts
->anomaly_nr
)
44 return !!bsearch(&ofsval
, opts
->anomaly
, opts
->anomaly_nr
,
45 sizeof(ofsval
), cmp_uint32
);
49 * The *sha1 contains the pack content SHA1 hash.
50 * The objects array passed in will be sorted by SHA1 on exit.
52 const char *write_idx_file(const char *index_name
, struct pack_idx_entry
**objects
,
53 int nr_objects
, const struct pack_idx_option
*opts
,
54 const unsigned char *sha1
)
57 struct pack_idx_entry
**sorted_by_sha
, **list
, **last
;
58 off_t last_obj_offset
= 0;
60 uint32_t index_version
;
63 sorted_by_sha
= objects
;
65 last
= sorted_by_sha
+ nr_objects
;
66 for (i
= 0; i
< nr_objects
; ++i
) {
67 if (objects
[i
]->offset
> last_obj_offset
)
68 last_obj_offset
= objects
[i
]->offset
;
70 QSORT(sorted_by_sha
, nr_objects
, sha1_compare
);
73 sorted_by_sha
= list
= last
= NULL
;
75 if (opts
->flags
& WRITE_IDX_VERIFY
) {
77 f
= hashfd_check(index_name
);
80 struct strbuf tmp_file
= STRBUF_INIT
;
81 fd
= odb_mkstemp(&tmp_file
, "pack/tmp_idx_XXXXXX");
82 index_name
= strbuf_detach(&tmp_file
, NULL
);
85 fd
= xopen(index_name
, O_CREAT
|O_EXCL
|O_WRONLY
, 0600);
87 f
= hashfd(fd
, index_name
);
90 /* if last object's offset is >= 2^31 we should use index V2 */
91 index_version
= need_large_offset(last_obj_offset
, opts
) ? 2 : opts
->version
;
93 /* index versions 2 and above need a header */
94 if (index_version
>= 2) {
95 struct pack_idx_header hdr
;
96 hdr
.idx_signature
= htonl(PACK_IDX_SIGNATURE
);
97 hdr
.idx_version
= htonl(index_version
);
98 hashwrite(f
, &hdr
, sizeof(hdr
));
102 * Write the first-level table (the list is sorted,
103 * but we use a 256-entry lookup to be able to avoid
104 * having to do eight extra binary search iterations).
106 for (i
= 0; i
< 256; i
++) {
107 struct pack_idx_entry
**next
= list
;
108 while (next
< last
) {
109 struct pack_idx_entry
*obj
= *next
;
110 if (obj
->oid
.hash
[0] != i
)
114 hashwrite_be32(f
, next
- sorted_by_sha
);
119 * Write the actual SHA1 entries..
121 list
= sorted_by_sha
;
122 for (i
= 0; i
< nr_objects
; i
++) {
123 struct pack_idx_entry
*obj
= *list
++;
124 if (index_version
< 2)
125 hashwrite_be32(f
, obj
->offset
);
126 hashwrite(f
, obj
->oid
.hash
, the_hash_algo
->rawsz
);
127 if ((opts
->flags
& WRITE_IDX_STRICT
) &&
128 (i
&& oideq(&list
[-2]->oid
, &obj
->oid
)))
129 die("The same object %s appears twice in the pack",
130 oid_to_hex(&obj
->oid
));
133 if (index_version
>= 2) {
134 unsigned int nr_large_offset
= 0;
136 /* write the crc32 table */
137 list
= sorted_by_sha
;
138 for (i
= 0; i
< nr_objects
; i
++) {
139 struct pack_idx_entry
*obj
= *list
++;
140 hashwrite_be32(f
, obj
->crc32
);
143 /* write the 32-bit offset table */
144 list
= sorted_by_sha
;
145 for (i
= 0; i
< nr_objects
; i
++) {
146 struct pack_idx_entry
*obj
= *list
++;
149 offset
= (need_large_offset(obj
->offset
, opts
)
150 ? (0x80000000 | nr_large_offset
++)
152 hashwrite_be32(f
, offset
);
155 /* write the large offset table */
156 list
= sorted_by_sha
;
157 while (nr_large_offset
) {
158 struct pack_idx_entry
*obj
= *list
++;
159 uint64_t offset
= obj
->offset
;
161 if (!need_large_offset(offset
, opts
))
163 hashwrite_be64(f
, offset
);
168 hashwrite(f
, sha1
, the_hash_algo
->rawsz
);
169 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
170 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
|
171 ((opts
->flags
& WRITE_IDX_VERIFY
) ? 0 : CSUM_FSYNC
));
175 static int pack_order_cmp(const void *va
, const void *vb
, void *ctx
)
177 struct pack_idx_entry
**objects
= ctx
;
179 off_t oa
= objects
[*(uint32_t*)va
]->offset
;
180 off_t ob
= objects
[*(uint32_t*)vb
]->offset
;
189 static void write_rev_header(struct hashfile
*f
)
191 hashwrite_be32(f
, RIDX_SIGNATURE
);
192 hashwrite_be32(f
, RIDX_VERSION
);
193 hashwrite_be32(f
, oid_version(the_hash_algo
));
196 static void write_rev_index_positions(struct hashfile
*f
,
197 uint32_t *pack_order
,
201 for (i
= 0; i
< nr_objects
; i
++)
202 hashwrite_be32(f
, pack_order
[i
]);
205 static void write_rev_trailer(struct hashfile
*f
, const unsigned char *hash
)
207 hashwrite(f
, hash
, the_hash_algo
->rawsz
);
210 const char *write_rev_file(const char *rev_name
,
211 struct pack_idx_entry
**objects
,
213 const unsigned char *hash
,
216 uint32_t *pack_order
;
220 if (!(flags
& WRITE_REV
) && !(flags
& WRITE_REV_VERIFY
))
223 ALLOC_ARRAY(pack_order
, nr_objects
);
224 for (i
= 0; i
< nr_objects
; i
++)
226 QSORT_S(pack_order
, nr_objects
, pack_order_cmp
, objects
);
228 ret
= write_rev_file_order(rev_name
, pack_order
, nr_objects
, hash
,
236 const char *write_rev_file_order(const char *rev_name
,
237 uint32_t *pack_order
,
239 const unsigned char *hash
,
245 if ((flags
& WRITE_REV
) && (flags
& WRITE_REV_VERIFY
))
246 die(_("cannot both write and verify reverse index"));
248 if (flags
& WRITE_REV
) {
250 struct strbuf tmp_file
= STRBUF_INIT
;
251 fd
= odb_mkstemp(&tmp_file
, "pack/tmp_rev_XXXXXX");
252 rev_name
= strbuf_detach(&tmp_file
, NULL
);
255 fd
= xopen(rev_name
, O_CREAT
|O_EXCL
|O_WRONLY
, 0600);
257 f
= hashfd(fd
, rev_name
);
258 } else if (flags
& WRITE_REV_VERIFY
) {
260 if (stat(rev_name
, &statbuf
)) {
261 if (errno
== ENOENT
) {
262 /* .rev files are optional */
265 die_errno(_("could not stat: %s"), rev_name
);
267 f
= hashfd_check(rev_name
);
273 write_rev_index_positions(f
, pack_order
, nr_objects
);
274 write_rev_trailer(f
, hash
);
276 if (rev_name
&& adjust_shared_perm(rev_name
) < 0)
277 die(_("failed to make %s readable"), rev_name
);
279 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
280 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
|
281 ((flags
& WRITE_IDX_VERIFY
) ? 0 : CSUM_FSYNC
));
286 static void write_mtimes_header(struct hashfile
*f
)
288 hashwrite_be32(f
, MTIMES_SIGNATURE
);
289 hashwrite_be32(f
, MTIMES_VERSION
);
290 hashwrite_be32(f
, oid_version(the_hash_algo
));
294 * Writes the object mtimes of "objects" for use in a .mtimes file.
295 * Note that objects must be in lexicographic (index) order, which is
296 * the expected ordering of these values in the .mtimes file.
298 static void write_mtimes_objects(struct hashfile
*f
,
299 struct packing_data
*to_pack
,
300 struct pack_idx_entry
**objects
,
304 for (i
= 0; i
< nr_objects
; i
++) {
305 struct object_entry
*e
= (struct object_entry
*)objects
[i
];
306 hashwrite_be32(f
, oe_cruft_mtime(to_pack
, e
));
310 static void write_mtimes_trailer(struct hashfile
*f
, const unsigned char *hash
)
312 hashwrite(f
, hash
, the_hash_algo
->rawsz
);
315 static const char *write_mtimes_file(struct packing_data
*to_pack
,
316 struct pack_idx_entry
**objects
,
318 const unsigned char *hash
)
320 struct strbuf tmp_file
= STRBUF_INIT
;
321 const char *mtimes_name
;
326 BUG("cannot call write_mtimes_file with NULL packing_data");
328 fd
= odb_mkstemp(&tmp_file
, "pack/tmp_mtimes_XXXXXX");
329 mtimes_name
= strbuf_detach(&tmp_file
, NULL
);
330 f
= hashfd(fd
, mtimes_name
);
332 write_mtimes_header(f
);
333 write_mtimes_objects(f
, to_pack
, objects
, nr_objects
);
334 write_mtimes_trailer(f
, hash
);
336 if (adjust_shared_perm(mtimes_name
) < 0)
337 die(_("failed to make %s readable"), mtimes_name
);
339 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
340 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
| CSUM_FSYNC
);
345 off_t
write_pack_header(struct hashfile
*f
, uint32_t nr_entries
)
347 struct pack_header hdr
;
349 hdr
.hdr_signature
= htonl(PACK_SIGNATURE
);
350 hdr
.hdr_version
= htonl(PACK_VERSION
);
351 hdr
.hdr_entries
= htonl(nr_entries
);
352 hashwrite(f
, &hdr
, sizeof(hdr
));
357 * Update pack header with object_count and compute new SHA1 for pack data
358 * associated to pack_fd, and write that SHA1 at the end. That new SHA1
359 * is also returned in new_pack_sha1.
361 * If partial_pack_sha1 is non null, then the SHA1 of the existing pack
362 * (without the header update) is computed and validated against the
363 * one provided in partial_pack_sha1. The validation is performed at
364 * partial_pack_offset bytes in the pack file. The SHA1 of the remaining
365 * data (i.e. from partial_pack_offset to the end) is then computed and
366 * returned in partial_pack_sha1.
368 * Note that new_pack_sha1 is updated last, so both new_pack_sha1 and
369 * partial_pack_sha1 can refer to the same buffer if the caller is not
370 * interested in the resulting SHA1 of pack data above partial_pack_offset.
372 void fixup_pack_header_footer(int pack_fd
,
373 unsigned char *new_pack_hash
,
374 const char *pack_name
,
375 uint32_t object_count
,
376 unsigned char *partial_pack_hash
,
377 off_t partial_pack_offset
)
379 int aligned_sz
, buf_sz
= 8 * 1024;
380 git_hash_ctx old_hash_ctx
, new_hash_ctx
;
381 struct pack_header hdr
;
385 the_hash_algo
->init_fn(&old_hash_ctx
);
386 the_hash_algo
->init_fn(&new_hash_ctx
);
388 if (lseek(pack_fd
, 0, SEEK_SET
) != 0)
389 die_errno("Failed seeking to start of '%s'", pack_name
);
390 read_result
= read_in_full(pack_fd
, &hdr
, sizeof(hdr
));
392 die_errno("Unable to reread header of '%s'", pack_name
);
393 else if (read_result
!= sizeof(hdr
))
394 die_errno("Unexpected short read for header of '%s'",
396 if (lseek(pack_fd
, 0, SEEK_SET
) != 0)
397 die_errno("Failed seeking to start of '%s'", pack_name
);
398 the_hash_algo
->update_fn(&old_hash_ctx
, &hdr
, sizeof(hdr
));
399 hdr
.hdr_entries
= htonl(object_count
);
400 the_hash_algo
->update_fn(&new_hash_ctx
, &hdr
, sizeof(hdr
));
401 write_or_die(pack_fd
, &hdr
, sizeof(hdr
));
402 partial_pack_offset
-= sizeof(hdr
);
404 buf
= xmalloc(buf_sz
);
405 aligned_sz
= buf_sz
- sizeof(hdr
);
408 m
= (partial_pack_hash
&& partial_pack_offset
< aligned_sz
) ?
409 partial_pack_offset
: aligned_sz
;
410 n
= xread(pack_fd
, buf
, m
);
414 die_errno("Failed to checksum '%s'", pack_name
);
415 the_hash_algo
->update_fn(&new_hash_ctx
, buf
, n
);
421 if (!partial_pack_hash
)
424 the_hash_algo
->update_fn(&old_hash_ctx
, buf
, n
);
425 partial_pack_offset
-= n
;
426 if (partial_pack_offset
== 0) {
427 unsigned char hash
[GIT_MAX_RAWSZ
];
428 the_hash_algo
->final_fn(hash
, &old_hash_ctx
);
429 if (!hasheq(hash
, partial_pack_hash
))
430 die("Unexpected checksum for %s "
431 "(disk corruption?)", pack_name
);
433 * Now let's compute the SHA1 of the remainder of the
434 * pack, which also means making partial_pack_offset
435 * big enough not to matter anymore.
437 the_hash_algo
->init_fn(&old_hash_ctx
);
438 partial_pack_offset
= ~partial_pack_offset
;
439 partial_pack_offset
-= MSB(partial_pack_offset
, 1);
444 if (partial_pack_hash
)
445 the_hash_algo
->final_fn(partial_pack_hash
, &old_hash_ctx
);
446 the_hash_algo
->final_fn(new_pack_hash
, &new_hash_ctx
);
447 write_or_die(pack_fd
, new_pack_hash
, the_hash_algo
->rawsz
);
448 fsync_component_or_die(FSYNC_COMPONENT_PACK
, pack_fd
, pack_name
);
451 char *index_pack_lockfile(int ip_out
, int *is_well_formed
)
453 char packname
[GIT_MAX_HEXSZ
+ 6];
454 const int len
= the_hash_algo
->hexsz
+ 6;
457 * The first thing we expect from index-pack's output
458 * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where
459 * %40s is the newly created pack SHA1 name. In the "keep"
460 * case, we need it to remove the corresponding .keep file
461 * later on. If we don't get that then tough luck with it.
463 if (read_in_full(ip_out
, packname
, len
) == len
&& packname
[len
-1] == '\n') {
469 if (skip_prefix(packname
, "keep\t", &name
))
470 return xstrfmt("%s/pack/pack-%s.keep",
471 get_object_directory(), name
);
480 * The per-object header is a pretty dense thing, which is
481 * - first byte: low four bits are "size", then three bits of "type",
482 * and the high bit is "size continues".
483 * - each byte afterwards: low seven bits are size continuation,
484 * with the high bit being "size continues"
486 int encode_in_pack_object_header(unsigned char *hdr
, int hdr_len
,
487 enum object_type type
, uintmax_t size
)
492 if (type
< OBJ_COMMIT
|| type
> OBJ_REF_DELTA
)
493 die("bad type %d", type
);
495 c
= (type
<< 4) | (size
& 15);
499 die("object size is too enormous to format");
509 struct hashfile
*create_tmp_packfile(char **pack_tmp_name
)
511 struct strbuf tmpname
= STRBUF_INIT
;
514 fd
= odb_mkstemp(&tmpname
, "pack/tmp_pack_XXXXXX");
515 *pack_tmp_name
= strbuf_detach(&tmpname
, NULL
);
516 return hashfd(fd
, *pack_tmp_name
);
519 static void rename_tmp_packfile(struct strbuf
*name_prefix
, const char *source
,
522 size_t name_prefix_len
= name_prefix
->len
;
524 strbuf_addstr(name_prefix
, ext
);
525 if (rename(source
, name_prefix
->buf
))
526 die_errno("unable to rename temporary file to '%s'",
528 strbuf_setlen(name_prefix
, name_prefix_len
);
531 void rename_tmp_packfile_idx(struct strbuf
*name_buffer
,
534 rename_tmp_packfile(name_buffer
, *idx_tmp_name
, "idx");
537 void stage_tmp_packfiles(struct strbuf
*name_buffer
,
538 const char *pack_tmp_name
,
539 struct pack_idx_entry
**written_list
,
541 struct packing_data
*to_pack
,
542 struct pack_idx_option
*pack_idx_opts
,
543 unsigned char hash
[],
546 const char *rev_tmp_name
= NULL
;
547 const char *mtimes_tmp_name
= NULL
;
549 if (adjust_shared_perm(pack_tmp_name
))
550 die_errno("unable to make temporary pack file readable");
552 *idx_tmp_name
= (char *)write_idx_file(NULL
, written_list
, nr_written
,
553 pack_idx_opts
, hash
);
554 if (adjust_shared_perm(*idx_tmp_name
))
555 die_errno("unable to make temporary index file readable");
557 rev_tmp_name
= write_rev_file(NULL
, written_list
, nr_written
, hash
,
558 pack_idx_opts
->flags
);
560 if (pack_idx_opts
->flags
& WRITE_MTIMES
) {
561 mtimes_tmp_name
= write_mtimes_file(to_pack
, written_list
,
566 rename_tmp_packfile(name_buffer
, pack_tmp_name
, "pack");
568 rename_tmp_packfile(name_buffer
, rev_tmp_name
, "rev");
570 rename_tmp_packfile(name_buffer
, mtimes_tmp_name
, "mtimes");
572 free((char *)rev_tmp_name
);
575 void write_promisor_file(const char *promisor_name
, struct ref
**sought
, int nr_sought
)
578 FILE *output
= xfopen(promisor_name
, "w");
580 for (i
= 0; i
< nr_sought
; i
++)
581 fprintf(output
, "%s %s\n", oid_to_hex(&sought
[i
]->old_oid
),
584 err
= ferror(output
);
585 err
|= fclose(output
);
587 die(_("could not write '%s' promisor file"), promisor_name
);