usage: report vsnprintf(3) failure
[git.git] / reftable / reader.c
blob2663b039381d1c3290193e8afe5ad3d9a13df9fc
1 /*
2 Copyright 2020 Google LLC
4 Use of this source code is governed by a BSD-style
5 license that can be found in the LICENSE file or at
6 https://developers.google.com/open-source/licenses/bsd
7 */
9 #include "reader.h"
11 #include "system.h"
12 #include "block.h"
13 #include "constants.h"
14 #include "generic.h"
15 #include "iter.h"
16 #include "record.h"
17 #include "reftable-error.h"
18 #include "reftable-generic.h"
20 uint64_t block_source_size(struct reftable_block_source *source)
22 return source->ops->size(source->arg);
25 int block_source_read_block(struct reftable_block_source *source,
26 struct reftable_block *dest, uint64_t off,
27 uint32_t size)
29 int result = source->ops->read_block(source->arg, dest, off, size);
30 dest->source = *source;
31 return result;
34 void block_source_close(struct reftable_block_source *source)
36 if (!source->ops) {
37 return;
40 source->ops->close(source->arg);
41 source->ops = NULL;
44 static struct reftable_reader_offsets *
45 reader_offsets_for(struct reftable_reader *r, uint8_t typ)
47 switch (typ) {
48 case BLOCK_TYPE_REF:
49 return &r->ref_offsets;
50 case BLOCK_TYPE_LOG:
51 return &r->log_offsets;
52 case BLOCK_TYPE_OBJ:
53 return &r->obj_offsets;
55 abort();
58 static int reader_get_block(struct reftable_reader *r,
59 struct reftable_block *dest, uint64_t off,
60 uint32_t sz)
62 if (off >= r->size)
63 return 0;
65 if (off + sz > r->size) {
66 sz = r->size - off;
69 return block_source_read_block(&r->source, dest, off, sz);
72 uint32_t reftable_reader_hash_id(struct reftable_reader *r)
74 return r->hash_id;
77 const char *reader_name(struct reftable_reader *r)
79 return r->name;
82 static int parse_footer(struct reftable_reader *r, uint8_t *footer,
83 uint8_t *header)
85 uint8_t *f = footer;
86 uint8_t first_block_typ;
87 int err = 0;
88 uint32_t computed_crc;
89 uint32_t file_crc;
91 if (memcmp(f, "REFT", 4)) {
92 err = REFTABLE_FORMAT_ERROR;
93 goto done;
95 f += 4;
97 if (memcmp(footer, header, header_size(r->version))) {
98 err = REFTABLE_FORMAT_ERROR;
99 goto done;
102 f++;
103 r->block_size = get_be24(f);
105 f += 3;
106 r->min_update_index = get_be64(f);
107 f += 8;
108 r->max_update_index = get_be64(f);
109 f += 8;
111 if (r->version == 1) {
112 r->hash_id = GIT_SHA1_FORMAT_ID;
113 } else {
114 r->hash_id = get_be32(f);
115 switch (r->hash_id) {
116 case GIT_SHA1_FORMAT_ID:
117 break;
118 case GIT_SHA256_FORMAT_ID:
119 break;
120 default:
121 err = REFTABLE_FORMAT_ERROR;
122 goto done;
124 f += 4;
127 r->ref_offsets.index_offset = get_be64(f);
128 f += 8;
130 r->obj_offsets.offset = get_be64(f);
131 f += 8;
133 r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1);
134 r->obj_offsets.offset >>= 5;
136 r->obj_offsets.index_offset = get_be64(f);
137 f += 8;
138 r->log_offsets.offset = get_be64(f);
139 f += 8;
140 r->log_offsets.index_offset = get_be64(f);
141 f += 8;
143 computed_crc = crc32(0, footer, f - footer);
144 file_crc = get_be32(f);
145 f += 4;
146 if (computed_crc != file_crc) {
147 err = REFTABLE_FORMAT_ERROR;
148 goto done;
151 first_block_typ = header[header_size(r->version)];
152 r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF);
153 r->ref_offsets.offset = 0;
154 r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG ||
155 r->log_offsets.offset > 0);
156 r->obj_offsets.is_present = r->obj_offsets.offset > 0;
157 if (r->obj_offsets.is_present && !r->object_id_len) {
158 err = REFTABLE_FORMAT_ERROR;
159 goto done;
162 err = 0;
163 done:
164 return err;
167 int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
168 const char *name)
170 struct reftable_block footer = { NULL };
171 struct reftable_block header = { NULL };
172 int err = 0;
173 uint64_t file_size = block_source_size(source);
175 /* Need +1 to read type of first block. */
176 uint32_t read_size = header_size(2) + 1; /* read v2 because it's larger. */
177 memset(r, 0, sizeof(struct reftable_reader));
179 if (read_size > file_size) {
180 err = REFTABLE_FORMAT_ERROR;
181 goto done;
184 err = block_source_read_block(source, &header, 0, read_size);
185 if (err != read_size) {
186 err = REFTABLE_IO_ERROR;
187 goto done;
190 if (memcmp(header.data, "REFT", 4)) {
191 err = REFTABLE_FORMAT_ERROR;
192 goto done;
194 r->version = header.data[4];
195 if (r->version != 1 && r->version != 2) {
196 err = REFTABLE_FORMAT_ERROR;
197 goto done;
200 r->size = file_size - footer_size(r->version);
201 r->source = *source;
202 r->name = xstrdup(name);
203 r->hash_id = 0;
205 err = block_source_read_block(source, &footer, r->size,
206 footer_size(r->version));
207 if (err != footer_size(r->version)) {
208 err = REFTABLE_IO_ERROR;
209 goto done;
212 err = parse_footer(r, footer.data, header.data);
213 done:
214 reftable_block_done(&footer);
215 reftable_block_done(&header);
216 return err;
219 struct table_iter {
220 struct reftable_reader *r;
221 uint8_t typ;
222 uint64_t block_off;
223 struct block_iter bi;
224 int is_finished;
226 #define TABLE_ITER_INIT { \
227 .bi = BLOCK_ITER_INIT \
230 static void table_iter_copy_from(struct table_iter *dest,
231 struct table_iter *src)
233 dest->r = src->r;
234 dest->typ = src->typ;
235 dest->block_off = src->block_off;
236 dest->is_finished = src->is_finished;
237 block_iter_copy_from(&dest->bi, &src->bi);
240 static int table_iter_next_in_block(struct table_iter *ti,
241 struct reftable_record *rec)
243 int res = block_iter_next(&ti->bi, rec);
244 if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) {
245 rec->u.ref.update_index += ti->r->min_update_index;
248 return res;
251 static void table_iter_block_done(struct table_iter *ti)
253 if (!ti->bi.br) {
254 return;
256 reftable_block_done(&ti->bi.br->block);
257 FREE_AND_NULL(ti->bi.br);
259 ti->bi.last_key.len = 0;
260 ti->bi.next_off = 0;
263 static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off,
264 int version)
266 int32_t result = 0;
268 if (off == 0) {
269 data += header_size(version);
272 *typ = data[0];
273 if (reftable_is_block_type(*typ)) {
274 result = get_be24(data + 1);
276 return result;
279 int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
280 uint64_t next_off, uint8_t want_typ)
282 int32_t guess_block_size = r->block_size ? r->block_size :
283 DEFAULT_BLOCK_SIZE;
284 struct reftable_block block = { NULL };
285 uint8_t block_typ = 0;
286 int err = 0;
287 uint32_t header_off = next_off ? 0 : header_size(r->version);
288 int32_t block_size = 0;
290 if (next_off >= r->size)
291 return 1;
293 err = reader_get_block(r, &block, next_off, guess_block_size);
294 if (err < 0)
295 goto done;
297 block_size = extract_block_size(block.data, &block_typ, next_off,
298 r->version);
299 if (block_size < 0) {
300 err = block_size;
301 goto done;
303 if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) {
304 err = 1;
305 goto done;
308 if (block_size > guess_block_size) {
309 reftable_block_done(&block);
310 err = reader_get_block(r, &block, next_off, block_size);
311 if (err < 0) {
312 goto done;
316 err = block_reader_init(br, &block, header_off, r->block_size,
317 hash_size(r->hash_id));
318 done:
319 reftable_block_done(&block);
321 return err;
324 static int table_iter_next_block(struct table_iter *dest,
325 struct table_iter *src)
327 uint64_t next_block_off = src->block_off + src->bi.br->full_block_size;
328 struct block_reader br = { 0 };
329 int err = 0;
331 dest->r = src->r;
332 dest->typ = src->typ;
333 dest->block_off = next_block_off;
335 err = reader_init_block_reader(src->r, &br, next_block_off, src->typ);
336 if (err > 0) {
337 dest->is_finished = 1;
338 return 1;
340 if (err != 0)
341 return err;
342 else {
343 struct block_reader *brp =
344 reftable_malloc(sizeof(struct block_reader));
345 *brp = br;
347 dest->is_finished = 0;
348 block_reader_start(brp, &dest->bi);
350 return 0;
353 static int table_iter_next(struct table_iter *ti, struct reftable_record *rec)
355 if (reftable_record_type(rec) != ti->typ)
356 return REFTABLE_API_ERROR;
358 while (1) {
359 struct table_iter next = TABLE_ITER_INIT;
360 int err = 0;
361 if (ti->is_finished) {
362 return 1;
365 err = table_iter_next_in_block(ti, rec);
366 if (err <= 0) {
367 return err;
370 err = table_iter_next_block(&next, ti);
371 if (err != 0) {
372 ti->is_finished = 1;
374 table_iter_block_done(ti);
375 if (err != 0) {
376 return err;
378 table_iter_copy_from(ti, &next);
379 block_iter_close(&next.bi);
383 static int table_iter_next_void(void *ti, struct reftable_record *rec)
385 return table_iter_next(ti, rec);
388 static void table_iter_close(void *p)
390 struct table_iter *ti = p;
391 table_iter_block_done(ti);
392 block_iter_close(&ti->bi);
395 static struct reftable_iterator_vtable table_iter_vtable = {
396 .next = &table_iter_next_void,
397 .close = &table_iter_close,
400 static void iterator_from_table_iter(struct reftable_iterator *it,
401 struct table_iter *ti)
403 assert(!it->ops);
404 it->iter_arg = ti;
405 it->ops = &table_iter_vtable;
408 static int reader_table_iter_at(struct reftable_reader *r,
409 struct table_iter *ti, uint64_t off,
410 uint8_t typ)
412 struct block_reader br = { 0 };
413 struct block_reader *brp = NULL;
415 int err = reader_init_block_reader(r, &br, off, typ);
416 if (err != 0)
417 return err;
419 brp = reftable_malloc(sizeof(struct block_reader));
420 *brp = br;
421 ti->r = r;
422 ti->typ = block_reader_type(brp);
423 ti->block_off = off;
424 block_reader_start(brp, &ti->bi);
425 return 0;
428 static int reader_start(struct reftable_reader *r, struct table_iter *ti,
429 uint8_t typ, int index)
431 struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
432 uint64_t off = offs->offset;
433 if (index) {
434 off = offs->index_offset;
435 if (off == 0) {
436 return 1;
438 typ = BLOCK_TYPE_INDEX;
441 return reader_table_iter_at(r, ti, off, typ);
444 static int reader_seek_linear(struct table_iter *ti,
445 struct reftable_record *want)
447 struct strbuf want_key = STRBUF_INIT;
448 struct strbuf got_key = STRBUF_INIT;
449 struct table_iter next = TABLE_ITER_INIT;
450 struct reftable_record rec;
451 int err = -1;
453 reftable_record_init(&rec, reftable_record_type(want));
454 reftable_record_key(want, &want_key);
456 while (1) {
457 err = table_iter_next_block(&next, ti);
458 if (err < 0)
459 goto done;
461 if (err > 0) {
462 break;
465 err = block_reader_first_key(next.bi.br, &got_key);
466 if (err < 0)
467 goto done;
469 if (strbuf_cmp(&got_key, &want_key) > 0) {
470 table_iter_block_done(&next);
471 break;
474 table_iter_block_done(ti);
475 table_iter_copy_from(ti, &next);
478 err = block_iter_seek(&ti->bi, &want_key);
479 if (err < 0)
480 goto done;
481 err = 0;
483 done:
484 block_iter_close(&next.bi);
485 reftable_record_release(&rec);
486 strbuf_release(&want_key);
487 strbuf_release(&got_key);
488 return err;
491 static int reader_seek_indexed(struct reftable_reader *r,
492 struct reftable_iterator *it,
493 struct reftable_record *rec)
495 struct reftable_record want_index = {
496 .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = STRBUF_INIT }
498 struct reftable_record index_result = {
499 .type = BLOCK_TYPE_INDEX,
500 .u.idx = { .last_key = STRBUF_INIT },
502 struct table_iter index_iter = TABLE_ITER_INIT;
503 struct table_iter next = TABLE_ITER_INIT;
504 int err = 0;
506 reftable_record_key(rec, &want_index.u.idx.last_key);
507 err = reader_start(r, &index_iter, reftable_record_type(rec), 1);
508 if (err < 0)
509 goto done;
512 * The index may consist of multiple levels, where each level may have
513 * multiple index blocks. We start by doing a linear search in the
514 * highest layer that identifies the relevant index block as well as
515 * the record inside that block that corresponds to our wanted key.
517 err = reader_seek_linear(&index_iter, &want_index);
518 if (err < 0)
519 goto done;
522 * Traverse down the levels until we find a non-index entry.
524 while (1) {
526 * In case we seek a record that does not exist the index iter
527 * will tell us that the iterator is over. This works because
528 * the last index entry of the current level will contain the
529 * last key it knows about. So in case our seeked key is larger
530 * than the last indexed key we know that it won't exist.
532 * There is one subtlety in the layout of the index section
533 * that makes this work as expected: the highest-level index is
534 * at end of the section and will point backwards and thus we
535 * start reading from the end of the index section, not the
536 * beginning.
538 * If that wasn't the case and the order was reversed then the
539 * linear seek would seek into the lower levels and traverse
540 * all levels of the index only to find out that the key does
541 * not exist.
543 err = table_iter_next(&index_iter, &index_result);
544 table_iter_block_done(&index_iter);
545 if (err != 0)
546 goto done;
548 err = reader_table_iter_at(r, &next, index_result.u.idx.offset,
550 if (err != 0)
551 goto done;
553 err = block_iter_seek(&next.bi, &want_index.u.idx.last_key);
554 if (err < 0)
555 goto done;
557 if (next.typ == reftable_record_type(rec)) {
558 err = 0;
559 break;
562 if (next.typ != BLOCK_TYPE_INDEX) {
563 err = REFTABLE_FORMAT_ERROR;
564 break;
567 table_iter_copy_from(&index_iter, &next);
570 if (err == 0) {
571 struct table_iter empty = TABLE_ITER_INIT;
572 struct table_iter *malloced = reftable_calloc(1, sizeof(*malloced));
573 *malloced = empty;
574 table_iter_copy_from(malloced, &next);
575 iterator_from_table_iter(it, malloced);
577 done:
578 block_iter_close(&next.bi);
579 table_iter_close(&index_iter);
580 reftable_record_release(&want_index);
581 reftable_record_release(&index_result);
582 return err;
585 static int reader_seek_internal(struct reftable_reader *r,
586 struct reftable_iterator *it,
587 struct reftable_record *rec)
589 struct reftable_reader_offsets *offs =
590 reader_offsets_for(r, reftable_record_type(rec));
591 uint64_t idx = offs->index_offset;
592 struct table_iter ti = TABLE_ITER_INIT;
593 int err = 0;
594 if (idx > 0)
595 return reader_seek_indexed(r, it, rec);
597 err = reader_start(r, &ti, reftable_record_type(rec), 0);
598 if (err < 0)
599 return err;
600 err = reader_seek_linear(&ti, rec);
601 if (err < 0)
602 return err;
603 else {
604 struct table_iter *p =
605 reftable_malloc(sizeof(struct table_iter));
606 *p = ti;
607 iterator_from_table_iter(it, p);
610 return 0;
613 static int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
614 struct reftable_record *rec)
616 uint8_t typ = reftable_record_type(rec);
618 struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
619 if (!offs->is_present) {
620 iterator_set_empty(it);
621 return 0;
624 return reader_seek_internal(r, it, rec);
627 int reftable_reader_seek_ref(struct reftable_reader *r,
628 struct reftable_iterator *it, const char *name)
630 struct reftable_record rec = {
631 .type = BLOCK_TYPE_REF,
632 .u.ref = {
633 .refname = (char *)name,
636 return reader_seek(r, it, &rec);
639 int reftable_reader_seek_log_at(struct reftable_reader *r,
640 struct reftable_iterator *it, const char *name,
641 uint64_t update_index)
643 struct reftable_record rec = { .type = BLOCK_TYPE_LOG,
644 .u.log = {
645 .refname = (char *)name,
646 .update_index = update_index,
647 } };
648 return reader_seek(r, it, &rec);
651 int reftable_reader_seek_log(struct reftable_reader *r,
652 struct reftable_iterator *it, const char *name)
654 uint64_t max = ~((uint64_t)0);
655 return reftable_reader_seek_log_at(r, it, name, max);
658 void reader_close(struct reftable_reader *r)
660 block_source_close(&r->source);
661 FREE_AND_NULL(r->name);
664 int reftable_new_reader(struct reftable_reader **p,
665 struct reftable_block_source *src, char const *name)
667 struct reftable_reader *rd = reftable_calloc(1, sizeof(*rd));
668 int err = init_reader(rd, src, name);
669 if (err == 0) {
670 *p = rd;
671 } else {
672 block_source_close(src);
673 reftable_free(rd);
675 return err;
678 void reftable_reader_free(struct reftable_reader *r)
680 if (!r)
681 return;
682 reader_close(r);
683 reftable_free(r);
686 static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
687 struct reftable_iterator *it,
688 uint8_t *oid)
690 struct reftable_record want = {
691 .type = BLOCK_TYPE_OBJ,
692 .u.obj = {
693 .hash_prefix = oid,
694 .hash_prefix_len = r->object_id_len,
697 struct reftable_iterator oit = { NULL };
698 struct reftable_record got = {
699 .type = BLOCK_TYPE_OBJ,
700 .u.obj = { 0 },
702 int err = 0;
703 struct indexed_table_ref_iter *itr = NULL;
705 /* Look through the reverse index. */
706 err = reader_seek(r, &oit, &want);
707 if (err != 0)
708 goto done;
710 /* read out the reftable_obj_record */
711 err = iterator_next(&oit, &got);
712 if (err < 0)
713 goto done;
715 if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix,
716 r->object_id_len)) {
717 /* didn't find it; return empty iterator */
718 iterator_set_empty(it);
719 err = 0;
720 goto done;
723 err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id),
724 got.u.obj.offsets,
725 got.u.obj.offset_len);
726 if (err < 0)
727 goto done;
728 got.u.obj.offsets = NULL;
729 iterator_from_indexed_table_ref_iter(it, itr);
731 done:
732 reftable_iterator_destroy(&oit);
733 reftable_record_release(&got);
734 return err;
737 static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
738 struct reftable_iterator *it,
739 uint8_t *oid)
741 struct table_iter ti_empty = TABLE_ITER_INIT;
742 struct table_iter *ti = reftable_calloc(1, sizeof(*ti));
743 struct filtering_ref_iterator *filter = NULL;
744 struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT;
745 int oid_len = hash_size(r->hash_id);
746 int err;
748 *ti = ti_empty;
749 err = reader_start(r, ti, BLOCK_TYPE_REF, 0);
750 if (err < 0) {
751 reftable_free(ti);
752 return err;
755 filter = reftable_malloc(sizeof(struct filtering_ref_iterator));
756 *filter = empty;
758 strbuf_add(&filter->oid, oid, oid_len);
759 reftable_table_from_reader(&filter->tab, r);
760 filter->double_check = 0;
761 iterator_from_table_iter(&filter->it, ti);
763 iterator_from_filtering_ref_iterator(it, filter);
764 return 0;
767 int reftable_reader_refs_for(struct reftable_reader *r,
768 struct reftable_iterator *it, uint8_t *oid)
770 if (r->obj_offsets.is_present)
771 return reftable_reader_refs_for_indexed(r, it, oid);
772 return reftable_reader_refs_for_unindexed(r, it, oid);
775 uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
777 return r->max_update_index;
780 uint64_t reftable_reader_min_update_index(struct reftable_reader *r)
782 return r->min_update_index;
785 /* generic table interface. */
787 static int reftable_reader_seek_void(void *tab, struct reftable_iterator *it,
788 struct reftable_record *rec)
790 return reader_seek(tab, it, rec);
793 static uint32_t reftable_reader_hash_id_void(void *tab)
795 return reftable_reader_hash_id(tab);
798 static uint64_t reftable_reader_min_update_index_void(void *tab)
800 return reftable_reader_min_update_index(tab);
803 static uint64_t reftable_reader_max_update_index_void(void *tab)
805 return reftable_reader_max_update_index(tab);
808 static struct reftable_table_vtable reader_vtable = {
809 .seek_record = reftable_reader_seek_void,
810 .hash_id = reftable_reader_hash_id_void,
811 .min_update_index = reftable_reader_min_update_index_void,
812 .max_update_index = reftable_reader_max_update_index_void,
815 void reftable_table_from_reader(struct reftable_table *tab,
816 struct reftable_reader *reader)
818 assert(!tab->ops);
819 tab->ops = &reader_vtable;
820 tab->table_arg = reader;
824 int reftable_reader_print_file(const char *tablename)
826 struct reftable_block_source src = { NULL };
827 int err = reftable_block_source_from_file(&src, tablename);
828 struct reftable_reader *r = NULL;
829 struct reftable_table tab = { NULL };
830 if (err < 0)
831 goto done;
833 err = reftable_new_reader(&r, &src, tablename);
834 if (err < 0)
835 goto done;
837 reftable_table_from_reader(&tab, r);
838 err = reftable_table_print(&tab);
839 done:
840 reftable_reader_free(r);
841 return err;