The seventh batch for 2.26
[git.git] / builtin / unpack-objects.c
blobdd4a75e030d219dfb3e5e0eeb59683c14f02fdf0
1 #include "builtin.h"
2 #include "cache.h"
3 #include "config.h"
4 #include "object-store.h"
5 #include "object.h"
6 #include "delta.h"
7 #include "pack.h"
8 #include "blob.h"
9 #include "commit.h"
10 #include "tag.h"
11 #include "tree.h"
12 #include "tree-walk.h"
13 #include "progress.h"
14 #include "decorate.h"
15 #include "fsck.h"
17 static int dry_run, quiet, recover, has_errors, strict;
18 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
20 /* We always read in 4kB chunks. */
21 static unsigned char buffer[4096];
22 static unsigned int offset, len;
23 static off_t consumed_bytes;
24 static off_t max_input_size;
25 static git_hash_ctx ctx;
26 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
27 static struct progress *progress;
30 * When running under --strict mode, objects whose reachability are
31 * suspect are kept in core without getting written in the object
32 * store.
34 struct obj_buffer {
35 char *buffer;
36 unsigned long size;
39 static struct decoration obj_decorate;
41 static struct obj_buffer *lookup_object_buffer(struct object *base)
43 return lookup_decoration(&obj_decorate, base);
46 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
48 struct obj_buffer *obj;
49 obj = xcalloc(1, sizeof(struct obj_buffer));
50 obj->buffer = buffer;
51 obj->size = size;
52 if (add_decoration(&obj_decorate, object, obj))
53 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
57 * Make sure at least "min" bytes are available in the buffer, and
58 * return the pointer to the buffer.
60 static void *fill(int min)
62 if (min <= len)
63 return buffer + offset;
64 if (min > sizeof(buffer))
65 die("cannot fill %d bytes", min);
66 if (offset) {
67 the_hash_algo->update_fn(&ctx, buffer, offset);
68 memmove(buffer, buffer + offset, len);
69 offset = 0;
71 do {
72 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
73 if (ret <= 0) {
74 if (!ret)
75 die("early EOF");
76 die_errno("read error on input");
78 len += ret;
79 } while (len < min);
80 return buffer;
83 static void use(int bytes)
85 if (bytes > len)
86 die("used more bytes than were available");
87 len -= bytes;
88 offset += bytes;
90 /* make sure off_t is sufficiently large not to wrap */
91 if (signed_add_overflows(consumed_bytes, bytes))
92 die("pack too large for current definition of off_t");
93 consumed_bytes += bytes;
94 if (max_input_size && consumed_bytes > max_input_size)
95 die(_("pack exceeds maximum allowed size"));
96 display_throughput(progress, consumed_bytes);
99 static void *get_data(unsigned long size)
101 git_zstream stream;
102 void *buf = xmallocz(size);
104 memset(&stream, 0, sizeof(stream));
106 stream.next_out = buf;
107 stream.avail_out = size;
108 stream.next_in = fill(1);
109 stream.avail_in = len;
110 git_inflate_init(&stream);
112 for (;;) {
113 int ret = git_inflate(&stream, 0);
114 use(len - stream.avail_in);
115 if (stream.total_out == size && ret == Z_STREAM_END)
116 break;
117 if (ret != Z_OK) {
118 error("inflate returned %d", ret);
119 FREE_AND_NULL(buf);
120 if (!recover)
121 exit(1);
122 has_errors = 1;
123 break;
125 stream.next_in = fill(1);
126 stream.avail_in = len;
128 git_inflate_end(&stream);
129 return buf;
132 struct delta_info {
133 struct object_id base_oid;
134 unsigned nr;
135 off_t base_offset;
136 unsigned long size;
137 void *delta;
138 struct delta_info *next;
141 static struct delta_info *delta_list;
143 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
144 off_t base_offset,
145 void *delta, unsigned long size)
147 struct delta_info *info = xmalloc(sizeof(*info));
149 oidcpy(&info->base_oid, base_oid);
150 info->base_offset = base_offset;
151 info->size = size;
152 info->delta = delta;
153 info->nr = nr;
154 info->next = delta_list;
155 delta_list = info;
158 struct obj_info {
159 off_t offset;
160 struct object_id oid;
161 struct object *obj;
164 /* Remember to update object flag allocation in object.h */
165 #define FLAG_OPEN (1u<<20)
166 #define FLAG_WRITTEN (1u<<21)
168 static struct obj_info *obj_list;
169 static unsigned nr_objects;
172 * Called only from check_object() after it verified this object
173 * is Ok.
175 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
177 struct object_id oid;
179 if (write_object_file(obj_buf->buffer, obj_buf->size,
180 type_name(obj->type), &oid) < 0)
181 die("failed to write object %s", oid_to_hex(&obj->oid));
182 obj->flags |= FLAG_WRITTEN;
186 * At the very end of the processing, write_rest() scans the objects
187 * that have reachability requirements and calls this function.
188 * Verify its reachability and validity recursively and write it out.
190 static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
192 struct obj_buffer *obj_buf;
194 if (!obj)
195 return 1;
197 if (obj->flags & FLAG_WRITTEN)
198 return 0;
200 if (type != OBJ_ANY && obj->type != type)
201 die("object type mismatch");
203 if (!(obj->flags & FLAG_OPEN)) {
204 unsigned long size;
205 int type = oid_object_info(the_repository, &obj->oid, &size);
206 if (type != obj->type || type <= 0)
207 die("object of unexpected type");
208 obj->flags |= FLAG_WRITTEN;
209 return 0;
212 obj_buf = lookup_object_buffer(obj);
213 if (!obj_buf)
214 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
215 if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
216 die("fsck error in packed object");
217 fsck_options.walk = check_object;
218 if (fsck_walk(obj, NULL, &fsck_options))
219 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
220 write_cached_object(obj, obj_buf);
221 return 0;
224 static void write_rest(void)
226 unsigned i;
227 for (i = 0; i < nr_objects; i++) {
228 if (obj_list[i].obj)
229 check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
233 static void added_object(unsigned nr, enum object_type type,
234 void *data, unsigned long size);
237 * Write out nr-th object from the list, now we know the contents
238 * of it. Under --strict, this buffers structured objects in-core,
239 * to be checked at the end.
241 static void write_object(unsigned nr, enum object_type type,
242 void *buf, unsigned long size)
244 if (!strict) {
245 if (write_object_file(buf, size, type_name(type),
246 &obj_list[nr].oid) < 0)
247 die("failed to write object");
248 added_object(nr, type, buf, size);
249 free(buf);
250 obj_list[nr].obj = NULL;
251 } else if (type == OBJ_BLOB) {
252 struct blob *blob;
253 if (write_object_file(buf, size, type_name(type),
254 &obj_list[nr].oid) < 0)
255 die("failed to write object");
256 added_object(nr, type, buf, size);
257 free(buf);
259 blob = lookup_blob(the_repository, &obj_list[nr].oid);
260 if (blob)
261 blob->object.flags |= FLAG_WRITTEN;
262 else
263 die("invalid blob object");
264 obj_list[nr].obj = NULL;
265 } else {
266 struct object *obj;
267 int eaten;
268 hash_object_file(the_hash_algo, buf, size, type_name(type),
269 &obj_list[nr].oid);
270 added_object(nr, type, buf, size);
271 obj = parse_object_buffer(the_repository, &obj_list[nr].oid,
272 type, size, buf,
273 &eaten);
274 if (!obj)
275 die("invalid %s", type_name(type));
276 add_object_buffer(obj, buf, size);
277 obj->flags |= FLAG_OPEN;
278 obj_list[nr].obj = obj;
282 static void resolve_delta(unsigned nr, enum object_type type,
283 void *base, unsigned long base_size,
284 void *delta, unsigned long delta_size)
286 void *result;
287 unsigned long result_size;
289 result = patch_delta(base, base_size,
290 delta, delta_size,
291 &result_size);
292 if (!result)
293 die("failed to apply delta");
294 free(delta);
295 write_object(nr, type, result, result_size);
299 * We now know the contents of an object (which is nr-th in the pack);
300 * resolve all the deltified objects that are based on it.
302 static void added_object(unsigned nr, enum object_type type,
303 void *data, unsigned long size)
305 struct delta_info **p = &delta_list;
306 struct delta_info *info;
308 while ((info = *p) != NULL) {
309 if (oideq(&info->base_oid, &obj_list[nr].oid) ||
310 info->base_offset == obj_list[nr].offset) {
311 *p = info->next;
312 p = &delta_list;
313 resolve_delta(info->nr, type, data, size,
314 info->delta, info->size);
315 free(info);
316 continue;
318 p = &info->next;
322 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
323 unsigned nr)
325 void *buf = get_data(size);
327 if (!dry_run && buf)
328 write_object(nr, type, buf, size);
329 else
330 free(buf);
333 static int resolve_against_held(unsigned nr, const struct object_id *base,
334 void *delta_data, unsigned long delta_size)
336 struct object *obj;
337 struct obj_buffer *obj_buffer;
338 obj = lookup_object(the_repository, base);
339 if (!obj)
340 return 0;
341 obj_buffer = lookup_object_buffer(obj);
342 if (!obj_buffer)
343 return 0;
344 resolve_delta(nr, obj->type, obj_buffer->buffer,
345 obj_buffer->size, delta_data, delta_size);
346 return 1;
349 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
350 unsigned nr)
352 void *delta_data, *base;
353 unsigned long base_size;
354 struct object_id base_oid;
356 if (type == OBJ_REF_DELTA) {
357 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
358 use(the_hash_algo->rawsz);
359 delta_data = get_data(delta_size);
360 if (dry_run || !delta_data) {
361 free(delta_data);
362 return;
364 if (has_object_file(&base_oid))
365 ; /* Ok we have this one */
366 else if (resolve_against_held(nr, &base_oid,
367 delta_data, delta_size))
368 return; /* we are done */
369 else {
370 /* cannot resolve yet --- queue it */
371 oidclr(&obj_list[nr].oid);
372 add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
373 return;
375 } else {
376 unsigned base_found = 0;
377 unsigned char *pack, c;
378 off_t base_offset;
379 unsigned lo, mid, hi;
381 pack = fill(1);
382 c = *pack;
383 use(1);
384 base_offset = c & 127;
385 while (c & 128) {
386 base_offset += 1;
387 if (!base_offset || MSB(base_offset, 7))
388 die("offset value overflow for delta base object");
389 pack = fill(1);
390 c = *pack;
391 use(1);
392 base_offset = (base_offset << 7) + (c & 127);
394 base_offset = obj_list[nr].offset - base_offset;
395 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
396 die("offset value out of bound for delta base object");
398 delta_data = get_data(delta_size);
399 if (dry_run || !delta_data) {
400 free(delta_data);
401 return;
403 lo = 0;
404 hi = nr;
405 while (lo < hi) {
406 mid = lo + (hi - lo) / 2;
407 if (base_offset < obj_list[mid].offset) {
408 hi = mid;
409 } else if (base_offset > obj_list[mid].offset) {
410 lo = mid + 1;
411 } else {
412 oidcpy(&base_oid, &obj_list[mid].oid);
413 base_found = !is_null_oid(&base_oid);
414 break;
417 if (!base_found) {
419 * The delta base object is itself a delta that
420 * has not been resolved yet.
422 oidclr(&obj_list[nr].oid);
423 add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
424 return;
428 if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
429 return;
431 base = read_object_file(&base_oid, &type, &base_size);
432 if (!base) {
433 error("failed to read delta-pack base object %s",
434 oid_to_hex(&base_oid));
435 if (!recover)
436 exit(1);
437 has_errors = 1;
438 return;
440 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
441 free(base);
444 static void unpack_one(unsigned nr)
446 unsigned shift;
447 unsigned char *pack;
448 unsigned long size, c;
449 enum object_type type;
451 obj_list[nr].offset = consumed_bytes;
453 pack = fill(1);
454 c = *pack;
455 use(1);
456 type = (c >> 4) & 7;
457 size = (c & 15);
458 shift = 4;
459 while (c & 0x80) {
460 pack = fill(1);
461 c = *pack;
462 use(1);
463 size += (c & 0x7f) << shift;
464 shift += 7;
467 switch (type) {
468 case OBJ_COMMIT:
469 case OBJ_TREE:
470 case OBJ_BLOB:
471 case OBJ_TAG:
472 unpack_non_delta_entry(type, size, nr);
473 return;
474 case OBJ_REF_DELTA:
475 case OBJ_OFS_DELTA:
476 unpack_delta_entry(type, size, nr);
477 return;
478 default:
479 error("bad object type %d", type);
480 has_errors = 1;
481 if (recover)
482 return;
483 exit(1);
487 static void unpack_all(void)
489 int i;
490 struct pack_header *hdr = fill(sizeof(struct pack_header));
492 nr_objects = ntohl(hdr->hdr_entries);
494 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
495 die("bad pack file");
496 if (!pack_version_ok(hdr->hdr_version))
497 die("unknown pack file version %"PRIu32,
498 ntohl(hdr->hdr_version));
499 use(sizeof(struct pack_header));
501 if (!quiet)
502 progress = start_progress(_("Unpacking objects"), nr_objects);
503 obj_list = xcalloc(nr_objects, sizeof(*obj_list));
504 for (i = 0; i < nr_objects; i++) {
505 unpack_one(i);
506 display_progress(progress, i + 1);
508 stop_progress(&progress);
510 if (delta_list)
511 die("unresolved deltas left after unpacking");
514 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
516 int i;
517 struct object_id oid;
519 read_replace_refs = 0;
521 git_config(git_default_config, NULL);
523 quiet = !isatty(2);
525 for (i = 1 ; i < argc; i++) {
526 const char *arg = argv[i];
528 if (*arg == '-') {
529 if (!strcmp(arg, "-n")) {
530 dry_run = 1;
531 continue;
533 if (!strcmp(arg, "-q")) {
534 quiet = 1;
535 continue;
537 if (!strcmp(arg, "-r")) {
538 recover = 1;
539 continue;
541 if (!strcmp(arg, "--strict")) {
542 strict = 1;
543 continue;
545 if (skip_prefix(arg, "--strict=", &arg)) {
546 strict = 1;
547 fsck_set_msg_types(&fsck_options, arg);
548 continue;
550 if (starts_with(arg, "--pack_header=")) {
551 struct pack_header *hdr;
552 char *c;
554 hdr = (struct pack_header *)buffer;
555 hdr->hdr_signature = htonl(PACK_SIGNATURE);
556 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
557 if (*c != ',')
558 die("bad %s", arg);
559 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
560 if (*c)
561 die("bad %s", arg);
562 len = sizeof(*hdr);
563 continue;
565 if (skip_prefix(arg, "--max-input-size=", &arg)) {
566 max_input_size = strtoumax(arg, NULL, 10);
567 continue;
569 usage(unpack_usage);
572 /* We don't take any non-flag arguments now.. Maybe some day */
573 usage(unpack_usage);
575 the_hash_algo->init_fn(&ctx);
576 unpack_all();
577 the_hash_algo->update_fn(&ctx, buffer, offset);
578 the_hash_algo->final_fn(oid.hash, &ctx);
579 if (strict) {
580 write_rest();
581 if (fsck_finish(&fsck_options))
582 die(_("fsck error in pack objects"));
584 if (!hasheq(fill(the_hash_algo->rawsz), oid.hash))
585 die("final sha1 did not match");
586 use(the_hash_algo->rawsz);
588 /* Write the last part of the buffer to stdout */
589 while (len) {
590 int ret = xwrite(1, buffer + offset, len);
591 if (ret <= 0)
592 break;
593 len -= ret;
594 offset += ret;
597 /* All done */
598 return has_errors;