4 #include "object-store.h"
12 #include "tree-walk.h"
17 static int dry_run
, quiet
, recover
, has_errors
, strict
;
18 static const char unpack_usage
[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
20 /* We always read in 4kB chunks. */
21 static unsigned char buffer
[4096];
22 static unsigned int offset
, len
;
23 static off_t consumed_bytes
;
24 static off_t max_input_size
;
25 static git_hash_ctx ctx
;
26 static struct fsck_options fsck_options
= FSCK_OPTIONS_STRICT
;
29 * When running under --strict mode, objects whose reachability are
30 * suspect are kept in core without getting written in the object
38 static struct decoration obj_decorate
;
40 static struct obj_buffer
*lookup_object_buffer(struct object
*base
)
42 return lookup_decoration(&obj_decorate
, base
);
45 static void add_object_buffer(struct object
*object
, char *buffer
, unsigned long size
)
47 struct obj_buffer
*obj
;
48 obj
= xcalloc(1, sizeof(struct obj_buffer
));
51 if (add_decoration(&obj_decorate
, object
, obj
))
52 die("object %s tried to add buffer twice!", oid_to_hex(&object
->oid
));
56 * Make sure at least "min" bytes are available in the buffer, and
57 * return the pointer to the buffer.
59 static void *fill(int min
)
62 return buffer
+ offset
;
63 if (min
> sizeof(buffer
))
64 die("cannot fill %d bytes", min
);
66 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
67 memmove(buffer
, buffer
+ offset
, len
);
71 ssize_t ret
= xread(0, buffer
+ len
, sizeof(buffer
) - len
);
75 die_errno("read error on input");
82 static void use(int bytes
)
85 die("used more bytes than were available");
89 /* make sure off_t is sufficiently large not to wrap */
90 if (signed_add_overflows(consumed_bytes
, bytes
))
91 die("pack too large for current definition of off_t");
92 consumed_bytes
+= bytes
;
93 if (max_input_size
&& consumed_bytes
> max_input_size
)
94 die(_("pack exceeds maximum allowed size"));
97 static void *get_data(unsigned long size
)
100 void *buf
= xmallocz(size
);
102 memset(&stream
, 0, sizeof(stream
));
104 stream
.next_out
= buf
;
105 stream
.avail_out
= size
;
106 stream
.next_in
= fill(1);
107 stream
.avail_in
= len
;
108 git_inflate_init(&stream
);
111 int ret
= git_inflate(&stream
, 0);
112 use(len
- stream
.avail_in
);
113 if (stream
.total_out
== size
&& ret
== Z_STREAM_END
)
116 error("inflate returned %d", ret
);
123 stream
.next_in
= fill(1);
124 stream
.avail_in
= len
;
126 git_inflate_end(&stream
);
131 struct object_id base_oid
;
136 struct delta_info
*next
;
139 static struct delta_info
*delta_list
;
141 static void add_delta_to_list(unsigned nr
, const struct object_id
*base_oid
,
143 void *delta
, unsigned long size
)
145 struct delta_info
*info
= xmalloc(sizeof(*info
));
147 oidcpy(&info
->base_oid
, base_oid
);
148 info
->base_offset
= base_offset
;
152 info
->next
= delta_list
;
158 struct object_id oid
;
162 /* Remember to update object flag allocation in object.h */
163 #define FLAG_OPEN (1u<<20)
164 #define FLAG_WRITTEN (1u<<21)
166 static struct obj_info
*obj_list
;
167 static unsigned nr_objects
;
170 * Called only from check_object() after it verified this object
173 static void write_cached_object(struct object
*obj
, struct obj_buffer
*obj_buf
)
175 struct object_id oid
;
177 if (write_object_file(obj_buf
->buffer
, obj_buf
->size
,
178 type_name(obj
->type
), &oid
) < 0)
179 die("failed to write object %s", oid_to_hex(&obj
->oid
));
180 obj
->flags
|= FLAG_WRITTEN
;
184 * At the very end of the processing, write_rest() scans the objects
185 * that have reachability requirements and calls this function.
186 * Verify its reachability and validity recursively and write it out.
188 static int check_object(struct object
*obj
, int type
, void *data
, struct fsck_options
*options
)
190 struct obj_buffer
*obj_buf
;
195 if (obj
->flags
& FLAG_WRITTEN
)
198 if (type
!= OBJ_ANY
&& obj
->type
!= type
)
199 die("object type mismatch");
201 if (!(obj
->flags
& FLAG_OPEN
)) {
203 int type
= oid_object_info(the_repository
, &obj
->oid
, &size
);
204 if (type
!= obj
->type
|| type
<= 0)
205 die("object of unexpected type");
206 obj
->flags
|= FLAG_WRITTEN
;
210 obj_buf
= lookup_object_buffer(obj
);
212 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj
->oid
));
213 if (fsck_object(obj
, obj_buf
->buffer
, obj_buf
->size
, &fsck_options
))
214 die("fsck error in packed object");
215 fsck_options
.walk
= check_object
;
216 if (fsck_walk(obj
, NULL
, &fsck_options
))
217 die("Error on reachable objects of %s", oid_to_hex(&obj
->oid
));
218 write_cached_object(obj
, obj_buf
);
222 static void write_rest(void)
225 for (i
= 0; i
< nr_objects
; i
++) {
227 check_object(obj_list
[i
].obj
, OBJ_ANY
, NULL
, NULL
);
231 static void added_object(unsigned nr
, enum object_type type
,
232 void *data
, unsigned long size
);
235 * Write out nr-th object from the list, now we know the contents
236 * of it. Under --strict, this buffers structured objects in-core,
237 * to be checked at the end.
239 static void write_object(unsigned nr
, enum object_type type
,
240 void *buf
, unsigned long size
)
243 if (write_object_file(buf
, size
, type_name(type
),
244 &obj_list
[nr
].oid
) < 0)
245 die("failed to write object");
246 added_object(nr
, type
, buf
, size
);
248 obj_list
[nr
].obj
= NULL
;
249 } else if (type
== OBJ_BLOB
) {
251 if (write_object_file(buf
, size
, type_name(type
),
252 &obj_list
[nr
].oid
) < 0)
253 die("failed to write object");
254 added_object(nr
, type
, buf
, size
);
257 blob
= lookup_blob(&obj_list
[nr
].oid
);
259 blob
->object
.flags
|= FLAG_WRITTEN
;
261 die("invalid blob object");
262 obj_list
[nr
].obj
= NULL
;
266 hash_object_file(buf
, size
, type_name(type
), &obj_list
[nr
].oid
);
267 added_object(nr
, type
, buf
, size
);
268 obj
= parse_object_buffer(&obj_list
[nr
].oid
, type
, size
, buf
,
271 die("invalid %s", type_name(type
));
272 add_object_buffer(obj
, buf
, size
);
273 obj
->flags
|= FLAG_OPEN
;
274 obj_list
[nr
].obj
= obj
;
278 static void resolve_delta(unsigned nr
, enum object_type type
,
279 void *base
, unsigned long base_size
,
280 void *delta
, unsigned long delta_size
)
283 unsigned long result_size
;
285 result
= patch_delta(base
, base_size
,
289 die("failed to apply delta");
291 write_object(nr
, type
, result
, result_size
);
295 * We now know the contents of an object (which is nr-th in the pack);
296 * resolve all the deltified objects that are based on it.
298 static void added_object(unsigned nr
, enum object_type type
,
299 void *data
, unsigned long size
)
301 struct delta_info
**p
= &delta_list
;
302 struct delta_info
*info
;
304 while ((info
= *p
) != NULL
) {
305 if (!oidcmp(&info
->base_oid
, &obj_list
[nr
].oid
) ||
306 info
->base_offset
== obj_list
[nr
].offset
) {
309 resolve_delta(info
->nr
, type
, data
, size
,
310 info
->delta
, info
->size
);
318 static void unpack_non_delta_entry(enum object_type type
, unsigned long size
,
321 void *buf
= get_data(size
);
324 write_object(nr
, type
, buf
, size
);
329 static int resolve_against_held(unsigned nr
, const struct object_id
*base
,
330 void *delta_data
, unsigned long delta_size
)
333 struct obj_buffer
*obj_buffer
;
334 obj
= lookup_object(base
->hash
);
337 obj_buffer
= lookup_object_buffer(obj
);
340 resolve_delta(nr
, obj
->type
, obj_buffer
->buffer
,
341 obj_buffer
->size
, delta_data
, delta_size
);
345 static void unpack_delta_entry(enum object_type type
, unsigned long delta_size
,
348 void *delta_data
, *base
;
349 unsigned long base_size
;
350 struct object_id base_oid
;
352 if (type
== OBJ_REF_DELTA
) {
353 hashcpy(base_oid
.hash
, fill(the_hash_algo
->rawsz
));
354 use(the_hash_algo
->rawsz
);
355 delta_data
= get_data(delta_size
);
356 if (dry_run
|| !delta_data
) {
360 if (has_object_file(&base_oid
))
361 ; /* Ok we have this one */
362 else if (resolve_against_held(nr
, &base_oid
,
363 delta_data
, delta_size
))
364 return; /* we are done */
366 /* cannot resolve yet --- queue it */
367 oidclr(&obj_list
[nr
].oid
);
368 add_delta_to_list(nr
, &base_oid
, 0, delta_data
, delta_size
);
372 unsigned base_found
= 0;
373 unsigned char *pack
, c
;
375 unsigned lo
, mid
, hi
;
380 base_offset
= c
& 127;
383 if (!base_offset
|| MSB(base_offset
, 7))
384 die("offset value overflow for delta base object");
388 base_offset
= (base_offset
<< 7) + (c
& 127);
390 base_offset
= obj_list
[nr
].offset
- base_offset
;
391 if (base_offset
<= 0 || base_offset
>= obj_list
[nr
].offset
)
392 die("offset value out of bound for delta base object");
394 delta_data
= get_data(delta_size
);
395 if (dry_run
|| !delta_data
) {
402 mid
= lo
+ (hi
- lo
) / 2;
403 if (base_offset
< obj_list
[mid
].offset
) {
405 } else if (base_offset
> obj_list
[mid
].offset
) {
408 oidcpy(&base_oid
, &obj_list
[mid
].oid
);
409 base_found
= !is_null_oid(&base_oid
);
415 * The delta base object is itself a delta that
416 * has not been resolved yet.
418 oidclr(&obj_list
[nr
].oid
);
419 add_delta_to_list(nr
, &null_oid
, base_offset
, delta_data
, delta_size
);
424 if (resolve_against_held(nr
, &base_oid
, delta_data
, delta_size
))
427 base
= read_object_file(&base_oid
, &type
, &base_size
);
429 error("failed to read delta-pack base object %s",
430 oid_to_hex(&base_oid
));
436 resolve_delta(nr
, type
, base
, base_size
, delta_data
, delta_size
);
440 static void unpack_one(unsigned nr
)
444 unsigned long size
, c
;
445 enum object_type type
;
447 obj_list
[nr
].offset
= consumed_bytes
;
459 size
+= (c
& 0x7f) << shift
;
468 unpack_non_delta_entry(type
, size
, nr
);
472 unpack_delta_entry(type
, size
, nr
);
475 error("bad object type %d", type
);
483 static void unpack_all(void)
486 struct progress
*progress
= NULL
;
487 struct pack_header
*hdr
= fill(sizeof(struct pack_header
));
489 nr_objects
= ntohl(hdr
->hdr_entries
);
491 if (ntohl(hdr
->hdr_signature
) != PACK_SIGNATURE
)
492 die("bad pack file");
493 if (!pack_version_ok(hdr
->hdr_version
))
494 die("unknown pack file version %"PRIu32
,
495 ntohl(hdr
->hdr_version
));
496 use(sizeof(struct pack_header
));
499 progress
= start_progress(_("Unpacking objects"), nr_objects
);
500 obj_list
= xcalloc(nr_objects
, sizeof(*obj_list
));
501 for (i
= 0; i
< nr_objects
; i
++) {
503 display_progress(progress
, i
+ 1);
505 stop_progress(&progress
);
508 die("unresolved deltas left after unpacking");
511 int cmd_unpack_objects(int argc
, const char **argv
, const char *prefix
)
514 struct object_id oid
;
516 check_replace_refs
= 0;
518 git_config(git_default_config
, NULL
);
522 for (i
= 1 ; i
< argc
; i
++) {
523 const char *arg
= argv
[i
];
526 if (!strcmp(arg
, "-n")) {
530 if (!strcmp(arg
, "-q")) {
534 if (!strcmp(arg
, "-r")) {
538 if (!strcmp(arg
, "--strict")) {
542 if (skip_prefix(arg
, "--strict=", &arg
)) {
544 fsck_set_msg_types(&fsck_options
, arg
);
547 if (starts_with(arg
, "--pack_header=")) {
548 struct pack_header
*hdr
;
551 hdr
= (struct pack_header
*)buffer
;
552 hdr
->hdr_signature
= htonl(PACK_SIGNATURE
);
553 hdr
->hdr_version
= htonl(strtoul(arg
+ 14, &c
, 10));
556 hdr
->hdr_entries
= htonl(strtoul(c
+ 1, &c
, 10));
562 if (skip_prefix(arg
, "--max-input-size=", &arg
)) {
563 max_input_size
= strtoumax(arg
, NULL
, 10);
569 /* We don't take any non-flag arguments now.. Maybe some day */
572 the_hash_algo
->init_fn(&ctx
);
574 the_hash_algo
->update_fn(&ctx
, buffer
, offset
);
575 the_hash_algo
->final_fn(oid
.hash
, &ctx
);
578 if (fsck_finish(&fsck_options
))
579 die(_("fsck error in pack objects"));
581 if (hashcmp(fill(the_hash_algo
->rawsz
), oid
.hash
))
582 die("final sha1 did not match");
583 use(the_hash_algo
->rawsz
);
585 /* Write the last part of the buffer to stdout */
587 int ret
= xwrite(1, buffer
+ offset
, len
);