git doc: direct bug reporters to mailing list archive
[git.git] / builtin / unpack-objects.c
blob6e81ca8ca28a67ce3d7bc0d62556fd41eda2ba52
1 #include "builtin.h"
2 #include "cache.h"
3 #include "config.h"
4 #include "object.h"
5 #include "delta.h"
6 #include "pack.h"
7 #include "blob.h"
8 #include "commit.h"
9 #include "tag.h"
10 #include "tree.h"
11 #include "tree-walk.h"
12 #include "progress.h"
13 #include "decorate.h"
14 #include "fsck.h"
16 static int dry_run, quiet, recover, has_errors, strict;
17 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
19 /* We always read in 4kB chunks. */
20 static unsigned char buffer[4096];
21 static unsigned int offset, len;
22 static off_t consumed_bytes;
23 static off_t max_input_size;
24 static git_hash_ctx ctx;
25 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
28 * When running under --strict mode, objects whose reachability are
29 * suspect are kept in core without getting written in the object
30 * store.
32 struct obj_buffer {
33 char *buffer;
34 unsigned long size;
37 static struct decoration obj_decorate;
39 static struct obj_buffer *lookup_object_buffer(struct object *base)
41 return lookup_decoration(&obj_decorate, base);
44 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
46 struct obj_buffer *obj;
47 obj = xcalloc(1, sizeof(struct obj_buffer));
48 obj->buffer = buffer;
49 obj->size = size;
50 if (add_decoration(&obj_decorate, object, obj))
51 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
55 * Make sure at least "min" bytes are available in the buffer, and
56 * return the pointer to the buffer.
58 static void *fill(int min)
60 if (min <= len)
61 return buffer + offset;
62 if (min > sizeof(buffer))
63 die("cannot fill %d bytes", min);
64 if (offset) {
65 the_hash_algo->update_fn(&ctx, buffer, offset);
66 memmove(buffer, buffer + offset, len);
67 offset = 0;
69 do {
70 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
71 if (ret <= 0) {
72 if (!ret)
73 die("early EOF");
74 die_errno("read error on input");
76 len += ret;
77 } while (len < min);
78 return buffer;
81 static void use(int bytes)
83 if (bytes > len)
84 die("used more bytes than were available");
85 len -= bytes;
86 offset += bytes;
88 /* make sure off_t is sufficiently large not to wrap */
89 if (signed_add_overflows(consumed_bytes, bytes))
90 die("pack too large for current definition of off_t");
91 consumed_bytes += bytes;
92 if (max_input_size && consumed_bytes > max_input_size)
93 die(_("pack exceeds maximum allowed size"));
96 static void *get_data(unsigned long size)
98 git_zstream stream;
99 void *buf = xmallocz(size);
101 memset(&stream, 0, sizeof(stream));
103 stream.next_out = buf;
104 stream.avail_out = size;
105 stream.next_in = fill(1);
106 stream.avail_in = len;
107 git_inflate_init(&stream);
109 for (;;) {
110 int ret = git_inflate(&stream, 0);
111 use(len - stream.avail_in);
112 if (stream.total_out == size && ret == Z_STREAM_END)
113 break;
114 if (ret != Z_OK) {
115 error("inflate returned %d", ret);
116 FREE_AND_NULL(buf);
117 if (!recover)
118 exit(1);
119 has_errors = 1;
120 break;
122 stream.next_in = fill(1);
123 stream.avail_in = len;
125 git_inflate_end(&stream);
126 return buf;
129 struct delta_info {
130 struct object_id base_oid;
131 unsigned nr;
132 off_t base_offset;
133 unsigned long size;
134 void *delta;
135 struct delta_info *next;
138 static struct delta_info *delta_list;
140 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
141 off_t base_offset,
142 void *delta, unsigned long size)
144 struct delta_info *info = xmalloc(sizeof(*info));
146 oidcpy(&info->base_oid, base_oid);
147 info->base_offset = base_offset;
148 info->size = size;
149 info->delta = delta;
150 info->nr = nr;
151 info->next = delta_list;
152 delta_list = info;
155 struct obj_info {
156 off_t offset;
157 struct object_id oid;
158 struct object *obj;
161 /* Remember to update object flag allocation in object.h */
162 #define FLAG_OPEN (1u<<20)
163 #define FLAG_WRITTEN (1u<<21)
165 static struct obj_info *obj_list;
166 static unsigned nr_objects;
169 * Called only from check_object() after it verified this object
170 * is Ok.
172 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
174 struct object_id oid;
176 if (write_object_file(obj_buf->buffer, obj_buf->size,
177 type_name(obj->type), &oid) < 0)
178 die("failed to write object %s", oid_to_hex(&obj->oid));
179 obj->flags |= FLAG_WRITTEN;
183 * At the very end of the processing, write_rest() scans the objects
184 * that have reachability requirements and calls this function.
185 * Verify its reachability and validity recursively and write it out.
187 static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
189 struct obj_buffer *obj_buf;
191 if (!obj)
192 return 1;
194 if (obj->flags & FLAG_WRITTEN)
195 return 0;
197 if (type != OBJ_ANY && obj->type != type)
198 die("object type mismatch");
200 if (!(obj->flags & FLAG_OPEN)) {
201 unsigned long size;
202 int type = oid_object_info(the_repository, &obj->oid, &size);
203 if (type != obj->type || type <= 0)
204 die("object of unexpected type");
205 obj->flags |= FLAG_WRITTEN;
206 return 0;
209 obj_buf = lookup_object_buffer(obj);
210 if (!obj_buf)
211 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
212 if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
213 die("fsck error in packed object");
214 fsck_options.walk = check_object;
215 if (fsck_walk(obj, NULL, &fsck_options))
216 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
217 write_cached_object(obj, obj_buf);
218 return 0;
221 static void write_rest(void)
223 unsigned i;
224 for (i = 0; i < nr_objects; i++) {
225 if (obj_list[i].obj)
226 check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
230 static void added_object(unsigned nr, enum object_type type,
231 void *data, unsigned long size);
234 * Write out nr-th object from the list, now we know the contents
235 * of it. Under --strict, this buffers structured objects in-core,
236 * to be checked at the end.
238 static void write_object(unsigned nr, enum object_type type,
239 void *buf, unsigned long size)
241 if (!strict) {
242 if (write_object_file(buf, size, type_name(type),
243 &obj_list[nr].oid) < 0)
244 die("failed to write object");
245 added_object(nr, type, buf, size);
246 free(buf);
247 obj_list[nr].obj = NULL;
248 } else if (type == OBJ_BLOB) {
249 struct blob *blob;
250 if (write_object_file(buf, size, type_name(type),
251 &obj_list[nr].oid) < 0)
252 die("failed to write object");
253 added_object(nr, type, buf, size);
254 free(buf);
256 blob = lookup_blob(&obj_list[nr].oid);
257 if (blob)
258 blob->object.flags |= FLAG_WRITTEN;
259 else
260 die("invalid blob object");
261 obj_list[nr].obj = NULL;
262 } else {
263 struct object *obj;
264 int eaten;
265 hash_object_file(buf, size, type_name(type), &obj_list[nr].oid);
266 added_object(nr, type, buf, size);
267 obj = parse_object_buffer(&obj_list[nr].oid, type, size, buf,
268 &eaten);
269 if (!obj)
270 die("invalid %s", type_name(type));
271 add_object_buffer(obj, buf, size);
272 obj->flags |= FLAG_OPEN;
273 obj_list[nr].obj = obj;
277 static void resolve_delta(unsigned nr, enum object_type type,
278 void *base, unsigned long base_size,
279 void *delta, unsigned long delta_size)
281 void *result;
282 unsigned long result_size;
284 result = patch_delta(base, base_size,
285 delta, delta_size,
286 &result_size);
287 if (!result)
288 die("failed to apply delta");
289 free(delta);
290 write_object(nr, type, result, result_size);
294 * We now know the contents of an object (which is nr-th in the pack);
295 * resolve all the deltified objects that are based on it.
297 static void added_object(unsigned nr, enum object_type type,
298 void *data, unsigned long size)
300 struct delta_info **p = &delta_list;
301 struct delta_info *info;
303 while ((info = *p) != NULL) {
304 if (!oidcmp(&info->base_oid, &obj_list[nr].oid) ||
305 info->base_offset == obj_list[nr].offset) {
306 *p = info->next;
307 p = &delta_list;
308 resolve_delta(info->nr, type, data, size,
309 info->delta, info->size);
310 free(info);
311 continue;
313 p = &info->next;
317 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
318 unsigned nr)
320 void *buf = get_data(size);
322 if (!dry_run && buf)
323 write_object(nr, type, buf, size);
324 else
325 free(buf);
328 static int resolve_against_held(unsigned nr, const struct object_id *base,
329 void *delta_data, unsigned long delta_size)
331 struct object *obj;
332 struct obj_buffer *obj_buffer;
333 obj = lookup_object(base->hash);
334 if (!obj)
335 return 0;
336 obj_buffer = lookup_object_buffer(obj);
337 if (!obj_buffer)
338 return 0;
339 resolve_delta(nr, obj->type, obj_buffer->buffer,
340 obj_buffer->size, delta_data, delta_size);
341 return 1;
344 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
345 unsigned nr)
347 void *delta_data, *base;
348 unsigned long base_size;
349 struct object_id base_oid;
351 if (type == OBJ_REF_DELTA) {
352 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
353 use(the_hash_algo->rawsz);
354 delta_data = get_data(delta_size);
355 if (dry_run || !delta_data) {
356 free(delta_data);
357 return;
359 if (has_object_file(&base_oid))
360 ; /* Ok we have this one */
361 else if (resolve_against_held(nr, &base_oid,
362 delta_data, delta_size))
363 return; /* we are done */
364 else {
365 /* cannot resolve yet --- queue it */
366 oidclr(&obj_list[nr].oid);
367 add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
368 return;
370 } else {
371 unsigned base_found = 0;
372 unsigned char *pack, c;
373 off_t base_offset;
374 unsigned lo, mid, hi;
376 pack = fill(1);
377 c = *pack;
378 use(1);
379 base_offset = c & 127;
380 while (c & 128) {
381 base_offset += 1;
382 if (!base_offset || MSB(base_offset, 7))
383 die("offset value overflow for delta base object");
384 pack = fill(1);
385 c = *pack;
386 use(1);
387 base_offset = (base_offset << 7) + (c & 127);
389 base_offset = obj_list[nr].offset - base_offset;
390 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
391 die("offset value out of bound for delta base object");
393 delta_data = get_data(delta_size);
394 if (dry_run || !delta_data) {
395 free(delta_data);
396 return;
398 lo = 0;
399 hi = nr;
400 while (lo < hi) {
401 mid = lo + (hi - lo) / 2;
402 if (base_offset < obj_list[mid].offset) {
403 hi = mid;
404 } else if (base_offset > obj_list[mid].offset) {
405 lo = mid + 1;
406 } else {
407 oidcpy(&base_oid, &obj_list[mid].oid);
408 base_found = !is_null_oid(&base_oid);
409 break;
412 if (!base_found) {
414 * The delta base object is itself a delta that
415 * has not been resolved yet.
417 oidclr(&obj_list[nr].oid);
418 add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
419 return;
423 if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
424 return;
426 base = read_object_file(&base_oid, &type, &base_size);
427 if (!base) {
428 error("failed to read delta-pack base object %s",
429 oid_to_hex(&base_oid));
430 if (!recover)
431 exit(1);
432 has_errors = 1;
433 return;
435 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
436 free(base);
439 static void unpack_one(unsigned nr)
441 unsigned shift;
442 unsigned char *pack;
443 unsigned long size, c;
444 enum object_type type;
446 obj_list[nr].offset = consumed_bytes;
448 pack = fill(1);
449 c = *pack;
450 use(1);
451 type = (c >> 4) & 7;
452 size = (c & 15);
453 shift = 4;
454 while (c & 0x80) {
455 pack = fill(1);
456 c = *pack;
457 use(1);
458 size += (c & 0x7f) << shift;
459 shift += 7;
462 switch (type) {
463 case OBJ_COMMIT:
464 case OBJ_TREE:
465 case OBJ_BLOB:
466 case OBJ_TAG:
467 unpack_non_delta_entry(type, size, nr);
468 return;
469 case OBJ_REF_DELTA:
470 case OBJ_OFS_DELTA:
471 unpack_delta_entry(type, size, nr);
472 return;
473 default:
474 error("bad object type %d", type);
475 has_errors = 1;
476 if (recover)
477 return;
478 exit(1);
482 static void unpack_all(void)
484 int i;
485 struct progress *progress = NULL;
486 struct pack_header *hdr = fill(sizeof(struct pack_header));
488 nr_objects = ntohl(hdr->hdr_entries);
490 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
491 die("bad pack file");
492 if (!pack_version_ok(hdr->hdr_version))
493 die("unknown pack file version %"PRIu32,
494 ntohl(hdr->hdr_version));
495 use(sizeof(struct pack_header));
497 if (!quiet)
498 progress = start_progress(_("Unpacking objects"), nr_objects);
499 obj_list = xcalloc(nr_objects, sizeof(*obj_list));
500 for (i = 0; i < nr_objects; i++) {
501 unpack_one(i);
502 display_progress(progress, i + 1);
504 stop_progress(&progress);
506 if (delta_list)
507 die("unresolved deltas left after unpacking");
510 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
512 int i;
513 struct object_id oid;
515 check_replace_refs = 0;
517 git_config(git_default_config, NULL);
519 quiet = !isatty(2);
521 for (i = 1 ; i < argc; i++) {
522 const char *arg = argv[i];
524 if (*arg == '-') {
525 if (!strcmp(arg, "-n")) {
526 dry_run = 1;
527 continue;
529 if (!strcmp(arg, "-q")) {
530 quiet = 1;
531 continue;
533 if (!strcmp(arg, "-r")) {
534 recover = 1;
535 continue;
537 if (!strcmp(arg, "--strict")) {
538 strict = 1;
539 continue;
541 if (skip_prefix(arg, "--strict=", &arg)) {
542 strict = 1;
543 fsck_set_msg_types(&fsck_options, arg);
544 continue;
546 if (starts_with(arg, "--pack_header=")) {
547 struct pack_header *hdr;
548 char *c;
550 hdr = (struct pack_header *)buffer;
551 hdr->hdr_signature = htonl(PACK_SIGNATURE);
552 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
553 if (*c != ',')
554 die("bad %s", arg);
555 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
556 if (*c)
557 die("bad %s", arg);
558 len = sizeof(*hdr);
559 continue;
561 if (skip_prefix(arg, "--max-input-size=", &arg)) {
562 max_input_size = strtoumax(arg, NULL, 10);
563 continue;
565 usage(unpack_usage);
568 /* We don't take any non-flag arguments now.. Maybe some day */
569 usage(unpack_usage);
571 the_hash_algo->init_fn(&ctx);
572 unpack_all();
573 the_hash_algo->update_fn(&ctx, buffer, offset);
574 the_hash_algo->final_fn(oid.hash, &ctx);
575 if (strict) {
576 write_rest();
577 if (fsck_finish(&fsck_options))
578 die(_("fsck error in pack objects"));
580 if (hashcmp(fill(the_hash_algo->rawsz), oid.hash))
581 die("final sha1 did not match");
582 use(the_hash_algo->rawsz);
584 /* Write the last part of the buffer to stdout */
585 while (len) {
586 int ret = xwrite(1, buffer + offset, len);
587 if (ret <= 0)
588 break;
589 len -= ret;
590 offset += ret;
593 /* All done */
594 return has_errors;