2 #include "bulk-checkin.h"
4 #include "environment.h"
8 #include "object-store-ll.h"
14 #include "replace-object.h"
18 #include "tree-walk.h"
23 static int dry_run
, quiet
, recover
, has_errors
, strict
;
24 static const char unpack_usage
[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
26 /* We always read in 4kB chunks. */
27 static unsigned char buffer
[4096];
28 static unsigned int offset
, len
;
29 static off_t consumed_bytes
;
30 static off_t max_input_size
;
31 static git_hash_ctx ctx
;
32 static struct fsck_options fsck_options
= FSCK_OPTIONS_STRICT
;
33 static struct progress
*progress
;
36 * When running under --strict mode, objects whose reachability are
37 * suspect are kept in core without getting written in the object
45 static struct decoration obj_decorate
;
47 static struct obj_buffer
*lookup_object_buffer(struct object
*base
)
49 return lookup_decoration(&obj_decorate
, base
);
52 static void add_object_buffer(struct object
*object
, char *buffer
, unsigned long size
)
54 struct obj_buffer
*obj
;
58 if (add_decoration(&obj_decorate
, object
, obj
))
59 die("object %s tried to add buffer twice!", oid_to_hex(&object
->oid
));
63 * Make sure at least "min" bytes are available in the buffer, and
64 * return the pointer to the buffer.
66 static void *fill(int min
)
69 return buffer
+ offset
;
70 if (min
> sizeof(buffer
))
71 die("cannot fill %d bytes", min
);
73 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
74 memmove(buffer
, buffer
+ offset
, len
);
78 ssize_t ret
= xread(0, buffer
+ len
, sizeof(buffer
) - len
);
82 die_errno("read error on input");
89 static void use(int bytes
)
92 die("used more bytes than were available");
96 /* make sure off_t is sufficiently large not to wrap */
97 if (signed_add_overflows(consumed_bytes
, bytes
))
98 die("pack too large for current definition of off_t");
99 consumed_bytes
+= bytes
;
100 if (max_input_size
&& consumed_bytes
> max_input_size
)
101 die(_("pack exceeds maximum allowed size"));
102 display_throughput(progress
, consumed_bytes
);
106 * Decompress zstream from the standard input into a newly
107 * allocated buffer of specified size and return the buffer.
108 * The caller is responsible to free the returned buffer.
110 * But for dry_run mode, "get_data()" is only used to check the
111 * integrity of data, and the returned buffer is not used at all.
112 * Therefore, in dry_run mode, "get_data()" will release the small
113 * allocated buffer which is reused to hold temporary zstream output
114 * and return NULL instead of returning garbage data.
116 static void *get_data(unsigned long size
)
119 unsigned long bufsize
= dry_run
&& size
> 8192 ? 8192 : size
;
120 void *buf
= xmallocz(bufsize
);
122 memset(&stream
, 0, sizeof(stream
));
124 stream
.next_out
= buf
;
125 stream
.avail_out
= bufsize
;
126 stream
.next_in
= fill(1);
127 stream
.avail_in
= len
;
128 git_inflate_init(&stream
);
131 int ret
= git_inflate(&stream
, 0);
132 use(len
- stream
.avail_in
);
133 if (stream
.total_out
== size
&& ret
== Z_STREAM_END
)
136 error("inflate returned %d", ret
);
143 stream
.next_in
= fill(1);
144 stream
.avail_in
= len
;
146 /* reuse the buffer in dry_run mode */
147 stream
.next_out
= buf
;
148 stream
.avail_out
= bufsize
> size
- stream
.total_out
?
149 size
- stream
.total_out
:
153 git_inflate_end(&stream
);
160 struct object_id base_oid
;
165 struct delta_info
*next
;
168 static struct delta_info
*delta_list
;
170 static void add_delta_to_list(unsigned nr
, const struct object_id
*base_oid
,
172 void *delta
, unsigned long size
)
174 struct delta_info
*info
= xmalloc(sizeof(*info
));
176 oidcpy(&info
->base_oid
, base_oid
);
177 info
->base_offset
= base_offset
;
181 info
->next
= delta_list
;
187 struct object_id oid
;
191 /* Remember to update object flag allocation in object.h */
192 #define FLAG_OPEN (1u<<20)
193 #define FLAG_WRITTEN (1u<<21)
195 static struct obj_info
*obj_list
;
196 static unsigned nr_objects
;
199 * Called only from check_object() after it verified this object
202 static void write_cached_object(struct object
*obj
, struct obj_buffer
*obj_buf
)
204 struct object_id oid
;
206 if (write_object_file(obj_buf
->buffer
, obj_buf
->size
,
207 obj
->type
, &oid
) < 0)
208 die("failed to write object %s", oid_to_hex(&obj
->oid
));
209 obj
->flags
|= FLAG_WRITTEN
;
213 * At the very end of the processing, write_rest() scans the objects
214 * that have reachability requirements and calls this function.
215 * Verify its reachability and validity recursively and write it out.
217 static int check_object(struct object
*obj
, enum object_type type
,
219 struct fsck_options
*options UNUSED
)
221 struct obj_buffer
*obj_buf
;
226 if (obj
->flags
& FLAG_WRITTEN
)
229 if (type
!= OBJ_ANY
&& obj
->type
!= type
)
230 die("object type mismatch");
232 if (!(obj
->flags
& FLAG_OPEN
)) {
234 int type
= oid_object_info(the_repository
, &obj
->oid
, &size
);
235 if (type
!= obj
->type
|| type
<= 0)
236 die("object of unexpected type");
237 obj
->flags
|= FLAG_WRITTEN
;
241 obj_buf
= lookup_object_buffer(obj
);
243 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj
->oid
));
244 if (fsck_object(obj
, obj_buf
->buffer
, obj_buf
->size
, &fsck_options
))
245 die("fsck error in packed object");
246 fsck_options
.walk
= check_object
;
247 if (fsck_walk(obj
, NULL
, &fsck_options
))
248 die("Error on reachable objects of %s", oid_to_hex(&obj
->oid
));
249 write_cached_object(obj
, obj_buf
);
253 static void write_rest(void)
256 for (i
= 0; i
< nr_objects
; i
++) {
258 check_object(obj_list
[i
].obj
, OBJ_ANY
, NULL
, NULL
);
262 static void added_object(unsigned nr
, enum object_type type
,
263 void *data
, unsigned long size
);
266 * Write out nr-th object from the list, now we know the contents
267 * of it. Under --strict, this buffers structured objects in-core,
268 * to be checked at the end.
270 static void write_object(unsigned nr
, enum object_type type
,
271 void *buf
, unsigned long size
)
274 if (write_object_file(buf
, size
, type
,
275 &obj_list
[nr
].oid
) < 0)
276 die("failed to write object");
277 added_object(nr
, type
, buf
, size
);
279 obj_list
[nr
].obj
= NULL
;
280 } else if (type
== OBJ_BLOB
) {
282 if (write_object_file(buf
, size
, type
,
283 &obj_list
[nr
].oid
) < 0)
284 die("failed to write object");
285 added_object(nr
, type
, buf
, size
);
288 blob
= lookup_blob(the_repository
, &obj_list
[nr
].oid
);
290 blob
->object
.flags
|= FLAG_WRITTEN
;
292 die("invalid blob object");
293 obj_list
[nr
].obj
= NULL
;
297 hash_object_file(the_hash_algo
, buf
, size
, type
,
299 added_object(nr
, type
, buf
, size
);
300 obj
= parse_object_buffer(the_repository
, &obj_list
[nr
].oid
,
304 die("invalid %s", type_name(type
));
305 add_object_buffer(obj
, buf
, size
);
306 obj
->flags
|= FLAG_OPEN
;
307 obj_list
[nr
].obj
= obj
;
311 static void resolve_delta(unsigned nr
, enum object_type type
,
312 void *base
, unsigned long base_size
,
313 void *delta
, unsigned long delta_size
)
316 unsigned long result_size
;
318 result
= patch_delta(base
, base_size
,
322 die("failed to apply delta");
324 write_object(nr
, type
, result
, result_size
);
328 * We now know the contents of an object (which is nr-th in the pack);
329 * resolve all the deltified objects that are based on it.
331 static void added_object(unsigned nr
, enum object_type type
,
332 void *data
, unsigned long size
)
334 struct delta_info
**p
= &delta_list
;
335 struct delta_info
*info
;
337 while ((info
= *p
) != NULL
) {
338 if (oideq(&info
->base_oid
, &obj_list
[nr
].oid
) ||
339 info
->base_offset
== obj_list
[nr
].offset
) {
342 resolve_delta(info
->nr
, type
, data
, size
,
343 info
->delta
, info
->size
);
351 static void unpack_non_delta_entry(enum object_type type
, unsigned long size
,
354 void *buf
= get_data(size
);
357 write_object(nr
, type
, buf
, size
);
360 struct input_zstream_data
{
361 git_zstream
*zstream
;
362 unsigned char buf
[8192];
366 static const void *feed_input_zstream(struct input_stream
*in_stream
,
367 unsigned long *readlen
)
369 struct input_zstream_data
*data
= in_stream
->data
;
370 git_zstream
*zstream
= data
->zstream
;
373 if (in_stream
->is_finished
) {
378 zstream
->next_out
= data
->buf
;
379 zstream
->avail_out
= sizeof(data
->buf
);
380 zstream
->next_in
= in
;
381 zstream
->avail_in
= len
;
383 data
->status
= git_inflate(zstream
, 0);
385 in_stream
->is_finished
= data
->status
!= Z_OK
;
386 use(len
- zstream
->avail_in
);
387 *readlen
= sizeof(data
->buf
) - zstream
->avail_out
;
392 static void stream_blob(unsigned long size
, unsigned nr
)
394 git_zstream zstream
= { 0 };
395 struct input_zstream_data data
= { 0 };
396 struct input_stream in_stream
= {
397 .read
= feed_input_zstream
,
400 struct obj_info
*info
= &obj_list
[nr
];
402 data
.zstream
= &zstream
;
403 git_inflate_init(&zstream
);
405 if (stream_loose_object(&in_stream
, size
, &info
->oid
))
406 die(_("failed to write object in stream"));
408 if (data
.status
!= Z_STREAM_END
)
409 die(_("inflate returned (%d)"), data
.status
);
410 git_inflate_end(&zstream
);
413 struct blob
*blob
= lookup_blob(the_repository
, &info
->oid
);
416 die(_("invalid blob object from stream"));
417 blob
->object
.flags
|= FLAG_WRITTEN
;
422 static int resolve_against_held(unsigned nr
, const struct object_id
*base
,
423 void *delta_data
, unsigned long delta_size
)
426 struct obj_buffer
*obj_buffer
;
427 obj
= lookup_object(the_repository
, base
);
430 obj_buffer
= lookup_object_buffer(obj
);
433 resolve_delta(nr
, obj
->type
, obj_buffer
->buffer
,
434 obj_buffer
->size
, delta_data
, delta_size
);
438 static void unpack_delta_entry(enum object_type type
, unsigned long delta_size
,
441 void *delta_data
, *base
;
442 unsigned long base_size
;
443 struct object_id base_oid
;
445 if (type
== OBJ_REF_DELTA
) {
446 oidread(&base_oid
, fill(the_hash_algo
->rawsz
));
447 use(the_hash_algo
->rawsz
);
448 delta_data
= get_data(delta_size
);
451 if (repo_has_object_file(the_repository
, &base_oid
))
452 ; /* Ok we have this one */
453 else if (resolve_against_held(nr
, &base_oid
,
454 delta_data
, delta_size
))
455 return; /* we are done */
457 /* cannot resolve yet --- queue it */
458 oidclr(&obj_list
[nr
].oid
);
459 add_delta_to_list(nr
, &base_oid
, 0, delta_data
, delta_size
);
463 unsigned base_found
= 0;
464 unsigned char *pack
, c
;
466 unsigned lo
, mid
, hi
;
471 base_offset
= c
& 127;
474 if (!base_offset
|| MSB(base_offset
, 7))
475 die("offset value overflow for delta base object");
479 base_offset
= (base_offset
<< 7) + (c
& 127);
481 base_offset
= obj_list
[nr
].offset
- base_offset
;
482 if (base_offset
<= 0 || base_offset
>= obj_list
[nr
].offset
)
483 die("offset value out of bound for delta base object");
485 delta_data
= get_data(delta_size
);
491 mid
= lo
+ (hi
- lo
) / 2;
492 if (base_offset
< obj_list
[mid
].offset
) {
494 } else if (base_offset
> obj_list
[mid
].offset
) {
497 oidcpy(&base_oid
, &obj_list
[mid
].oid
);
498 base_found
= !is_null_oid(&base_oid
);
504 * The delta base object is itself a delta that
505 * has not been resolved yet.
507 oidclr(&obj_list
[nr
].oid
);
508 add_delta_to_list(nr
, null_oid(), base_offset
,
509 delta_data
, delta_size
);
514 if (resolve_against_held(nr
, &base_oid
, delta_data
, delta_size
))
517 base
= repo_read_object_file(the_repository
, &base_oid
, &type
,
520 error("failed to read delta-pack base object %s",
521 oid_to_hex(&base_oid
));
527 resolve_delta(nr
, type
, base
, base_size
, delta_data
, delta_size
);
531 static void unpack_one(unsigned nr
)
535 unsigned long size
, c
;
536 enum object_type type
;
538 obj_list
[nr
].offset
= consumed_bytes
;
550 size
+= (c
& 0x7f) << shift
;
556 if (!dry_run
&& size
> big_file_threshold
) {
557 stream_blob(size
, nr
);
564 unpack_non_delta_entry(type
, size
, nr
);
568 unpack_delta_entry(type
, size
, nr
);
571 error("bad object type %d", type
);
579 static void unpack_all(void)
582 struct pack_header
*hdr
= fill(sizeof(struct pack_header
));
584 nr_objects
= ntohl(hdr
->hdr_entries
);
586 if (ntohl(hdr
->hdr_signature
) != PACK_SIGNATURE
)
587 die("bad pack file");
588 if (!pack_version_ok(hdr
->hdr_version
))
589 die("unknown pack file version %"PRIu32
,
590 ntohl(hdr
->hdr_version
));
591 use(sizeof(struct pack_header
));
594 progress
= start_progress(_("Unpacking objects"), nr_objects
);
595 CALLOC_ARRAY(obj_list
, nr_objects
);
596 begin_odb_transaction();
597 for (i
= 0; i
< nr_objects
; i
++) {
599 display_progress(progress
, i
+ 1);
601 end_odb_transaction();
602 stop_progress(&progress
);
605 die("unresolved deltas left after unpacking");
608 int cmd_unpack_objects(int argc
, const char **argv
, const char *prefix UNUSED
)
611 struct object_id oid
;
613 disable_replace_refs();
615 git_config(git_default_config
, NULL
);
619 for (i
= 1 ; i
< argc
; i
++) {
620 const char *arg
= argv
[i
];
623 if (!strcmp(arg
, "-n")) {
627 if (!strcmp(arg
, "-q")) {
631 if (!strcmp(arg
, "-r")) {
635 if (!strcmp(arg
, "--strict")) {
639 if (skip_prefix(arg
, "--strict=", &arg
)) {
641 fsck_set_msg_types(&fsck_options
, arg
);
644 if (starts_with(arg
, "--pack_header=")) {
645 struct pack_header
*hdr
;
648 hdr
= (struct pack_header
*)buffer
;
649 hdr
->hdr_signature
= htonl(PACK_SIGNATURE
);
650 hdr
->hdr_version
= htonl(strtoul(arg
+ 14, &c
, 10));
653 hdr
->hdr_entries
= htonl(strtoul(c
+ 1, &c
, 10));
659 if (skip_prefix(arg
, "--max-input-size=", &arg
)) {
660 max_input_size
= strtoumax(arg
, NULL
, 10);
666 /* We don't take any non-flag arguments now.. Maybe some day */
669 the_hash_algo
->init_fn(&ctx
);
671 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
672 the_hash_algo
->final_oid_fn(&oid
, &ctx
);
675 if (fsck_finish(&fsck_options
))
676 die(_("fsck error in pack objects"));
678 if (!hasheq(fill(the_hash_algo
->rawsz
), oid
.hash
))
679 die("final sha1 did not match");
680 use(the_hash_algo
->rawsz
);
682 /* Write the last part of the buffer to stdout */
684 int ret
= xwrite(1, buffer
+ offset
, len
);