reftable/readwrite_test.c

   1 /*
   2 Copyright 2020 Google LLC
   3
   4 Use of this source code is governed by a BSD-style
   5 license that can be found in the LICENSE file or at
   6 https://developers.google.com/open-source/licenses/bsd
   7 */
   8
   9 #include "system.h"
  10
  11 #include "basics.h"
  12 #include "block.h"
  13 #include "blocksource.h"
  14 #include "constants.h"
  15 #include "reader.h"
  16 #include "record.h"
  17 #include "test_framework.h"
  18 #include "reftable-tests.h"
  19 #include "reftable-writer.h"
  20
  21 static const int update_index = 5;
  22
  23 static void test_buffer(void)
  24 {
  25         struct strbuf buf = STRBUF_INIT;
  26         struct reftable_block_source source = { NULL };
  27         struct reftable_block out = { NULL };
  28         int n;
  29         uint8_t in[] = "hello";
  30         strbuf_add(&buf, in, sizeof(in));
  31         block_source_from_strbuf(&source, &buf);
  32         EXPECT(block_source_size(&source) == 6);
  33         n = block_source_read_block(&source, &out, 0, sizeof(in));
  34         EXPECT(n == sizeof(in));
  35         EXPECT(!memcmp(in, out.data, n));
  36         reftable_block_done(&out);
  37
  38         n = block_source_read_block(&source, &out, 1, 2);
  39         EXPECT(n == 2);
  40         EXPECT(!memcmp(out.data, "el", 2));
  41
  42         reftable_block_done(&out);
  43         block_source_close(&source);
  44         strbuf_release(&buf);
  45 }
  46
  47 static void write_table(char ***names, struct strbuf *buf, int N,
  48                         int block_size, uint32_t hash_id)
  49 {
  50         struct reftable_write_options opts = {
  51                 .block_size = block_size,
  52                 .hash_id = hash_id,
  53         };
  54         struct reftable_writer *w =
  55                 reftable_new_writer(&strbuf_add_void, buf, &opts);
  56         struct reftable_ref_record ref = { NULL };
  57         int i = 0, n;
  58         struct reftable_log_record log = { NULL };
  59         const struct reftable_stats *stats = NULL;
  60         *names = reftable_calloc(sizeof(char *) * (N + 1));
  61         reftable_writer_set_limits(w, update_index, update_index);
  62         for (i = 0; i < N; i++) {
  63                 uint8_t hash[GIT_SHA256_RAWSZ] = { 0 };
  64                 char name[100];
  65                 int n;
  66
  67                 set_test_hash(hash, i);
  68
  69                 snprintf(name, sizeof(name), "refs/heads/branch%02d", i);
  70
  71                 ref.refname = name;
  72                 ref.update_index = update_index;
  73                 ref.value_type = REFTABLE_REF_VAL1;
  74                 ref.value.val1 = hash;
  75                 (*names)[i] = xstrdup(name);
  76
  77                 n = reftable_writer_add_ref(w, &ref);
  78                 EXPECT(n == 0);
  79         }
  80
  81         for (i = 0; i < N; i++) {
  82                 uint8_t hash[GIT_SHA256_RAWSZ] = { 0 };
  83                 char name[100];
  84                 int n;
  85
  86                 set_test_hash(hash, i);
  87
  88                 snprintf(name, sizeof(name), "refs/heads/branch%02d", i);
  89
  90                 log.refname = name;
  91                 log.update_index = update_index;
  92                 log.value_type = REFTABLE_LOG_UPDATE;
  93                 log.value.update.new_hash = hash;
  94                 log.value.update.message = "message";
  95
  96                 n = reftable_writer_add_log(w, &log);
  97                 EXPECT(n == 0);
  98         }
  99
 100         n = reftable_writer_close(w);
 101         EXPECT(n == 0);
 102
 103         stats = writer_stats(w);
 104         for (i = 0; i < stats->ref_stats.blocks; i++) {
 105                 int off = i * opts.block_size;
 106                 if (off == 0) {
 107                         off = header_size(
 108                                 (hash_id == GIT_SHA256_FORMAT_ID) ? 2 : 1);
 109                 }
 110                 EXPECT(buf->buf[off] == 'r');
 111         }
 112
 113         EXPECT(stats->log_stats.blocks > 0);
 114         reftable_writer_free(w);
 115 }
 116
 117 static void test_log_buffer_size(void)
 118 {
 119         struct strbuf buf = STRBUF_INIT;
 120         struct reftable_write_options opts = {
 121                 .block_size = 4096,
 122         };
 123         int err;
 124         int i;
 125         struct reftable_log_record
 126                 log = { .refname = "refs/heads/master",
 127                         .update_index = 0xa,
 128                         .value_type = REFTABLE_LOG_UPDATE,
 129                         .value = { .update = {
 130                                            .name = "Han-Wen Nienhuys",
 131                                            .email = "hanwen@google.com",
 132                                            .tz_offset = 100,
 133                                            .time = 0x5e430672,
 134                                            .message = "commit: 9\n",
 135                                    } } };
 136         struct reftable_writer *w =
 137                 reftable_new_writer(&strbuf_add_void, &buf, &opts);
 138
 139         /* This tests buffer extension for log compression. Must use a random
 140            hash, to ensure that the compressed part is larger than the original.
 141         */
 142         uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ];
 143         for (i = 0; i < GIT_SHA1_RAWSZ; i++) {
 144                 hash1[i] = (uint8_t)(rand() % 256);
 145                 hash2[i] = (uint8_t)(rand() % 256);
 146         }
 147         log.value.update.old_hash = hash1;
 148         log.value.update.new_hash = hash2;
 149         reftable_writer_set_limits(w, update_index, update_index);
 150         err = reftable_writer_add_log(w, &log);
 151         EXPECT_ERR(err);
 152         err = reftable_writer_close(w);
 153         EXPECT_ERR(err);
 154         reftable_writer_free(w);
 155         strbuf_release(&buf);
 156 }
 157
 158 static void test_log_write_read(void)
 159 {
 160         int N = 2;
 161         char **names = reftable_calloc(sizeof(char *) * (N + 1));
 162         int err;
 163         struct reftable_write_options opts = {
 164                 .block_size = 256,
 165         };
 166         struct reftable_ref_record ref = { NULL };
 167         int i = 0;
 168         struct reftable_log_record log = { NULL };
 169         int n;
 170         struct reftable_iterator it = { NULL };
 171         struct reftable_reader rd = { NULL };
 172         struct reftable_block_source source = { NULL };
 173         struct strbuf buf = STRBUF_INIT;
 174         struct reftable_writer *w =
 175                 reftable_new_writer(&strbuf_add_void, &buf, &opts);
 176         const struct reftable_stats *stats = NULL;
 177         reftable_writer_set_limits(w, 0, N);
 178         for (i = 0; i < N; i++) {
 179                 char name[256];
 180                 struct reftable_ref_record ref = { NULL };
 181                 snprintf(name, sizeof(name), "b%02d%0*d", i, 130, 7);
 182                 names[i] = xstrdup(name);
 183                 ref.refname = name;
 184                 ref.update_index = i;
 185
 186                 err = reftable_writer_add_ref(w, &ref);
 187                 EXPECT_ERR(err);
 188         }
 189         for (i = 0; i < N; i++) {
 190                 uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ];
 191                 struct reftable_log_record log = { NULL };
 192                 set_test_hash(hash1, i);
 193                 set_test_hash(hash2, i + 1);
 194
 195                 log.refname = names[i];
 196                 log.update_index = i;
 197                 log.value_type = REFTABLE_LOG_UPDATE;
 198                 log.value.update.old_hash = hash1;
 199                 log.value.update.new_hash = hash2;
 200
 201                 err = reftable_writer_add_log(w, &log);
 202                 EXPECT_ERR(err);
 203         }
 204
 205         n = reftable_writer_close(w);
 206         EXPECT(n == 0);
 207
 208         stats = writer_stats(w);
 209         EXPECT(stats->log_stats.blocks > 0);
 210         reftable_writer_free(w);
 211         w = NULL;
 212
 213         block_source_from_strbuf(&source, &buf);
 214
 215         err = init_reader(&rd, &source, "file.log");
 216         EXPECT_ERR(err);
 217
 218         err = reftable_reader_seek_ref(&rd, &it, names[N - 1]);
 219         EXPECT_ERR(err);
 220
 221         err = reftable_iterator_next_ref(&it, &ref);
 222         EXPECT_ERR(err);
 223
 224         /* end of iteration. */
 225         err = reftable_iterator_next_ref(&it, &ref);
 226         EXPECT(0 < err);
 227
 228         reftable_iterator_destroy(&it);
 229         reftable_ref_record_release(&ref);
 230
 231         err = reftable_reader_seek_log(&rd, &it, "");
 232         EXPECT_ERR(err);
 233
 234         i = 0;
 235         while (1) {
 236                 int err = reftable_iterator_next_log(&it, &log);
 237                 if (err > 0) {
 238                         break;
 239                 }
 240
 241                 EXPECT_ERR(err);
 242                 EXPECT_STREQ(names[i], log.refname);
 243                 EXPECT(i == log.update_index);
 244                 i++;
 245                 reftable_log_record_release(&log);
 246         }
 247
 248         EXPECT(i == N);
 249         reftable_iterator_destroy(&it);
 250
 251         /* cleanup. */
 252         strbuf_release(&buf);
 253         free_names(names);
 254         reader_close(&rd);
 255 }
 256
 257 static void test_table_read_write_sequential(void)
 258 {
 259         char **names;
 260         struct strbuf buf = STRBUF_INIT;
 261         int N = 50;
 262         struct reftable_iterator it = { NULL };
 263         struct reftable_block_source source = { NULL };
 264         struct reftable_reader rd = { NULL };
 265         int err = 0;
 266         int j = 0;
 267
 268         write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID);
 269
 270         block_source_from_strbuf(&source, &buf);
 271
 272         err = init_reader(&rd, &source, "file.ref");
 273         EXPECT_ERR(err);
 274
 275         err = reftable_reader_seek_ref(&rd, &it, "");
 276         EXPECT_ERR(err);
 277
 278         while (1) {
 279                 struct reftable_ref_record ref = { NULL };
 280                 int r = reftable_iterator_next_ref(&it, &ref);
 281                 EXPECT(r >= 0);
 282                 if (r > 0) {
 283                         break;
 284                 }
 285                 EXPECT(0 == strcmp(names[j], ref.refname));
 286                 EXPECT(update_index == ref.update_index);
 287
 288                 j++;
 289                 reftable_ref_record_release(&ref);
 290         }
 291         EXPECT(j == N);
 292         reftable_iterator_destroy(&it);
 293         strbuf_release(&buf);
 294         free_names(names);
 295
 296         reader_close(&rd);
 297 }
 298
 299 static void test_table_write_small_table(void)
 300 {
 301         char **names;
 302         struct strbuf buf = STRBUF_INIT;
 303         int N = 1;
 304         write_table(&names, &buf, N, 4096, GIT_SHA1_FORMAT_ID);
 305         EXPECT(buf.len < 200);
 306         strbuf_release(&buf);
 307         free_names(names);
 308 }
 309
 310 static void test_table_read_api(void)
 311 {
 312         char **names;
 313         struct strbuf buf = STRBUF_INIT;
 314         int N = 50;
 315         struct reftable_reader rd = { NULL };
 316         struct reftable_block_source source = { NULL };
 317         int err;
 318         int i;
 319         struct reftable_log_record log = { NULL };
 320         struct reftable_iterator it = { NULL };
 321
 322         write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID);
 323
 324         block_source_from_strbuf(&source, &buf);
 325
 326         err = init_reader(&rd, &source, "file.ref");
 327         EXPECT_ERR(err);
 328
 329         err = reftable_reader_seek_ref(&rd, &it, names[0]);
 330         EXPECT_ERR(err);
 331
 332         err = reftable_iterator_next_log(&it, &log);
 333         EXPECT(err == REFTABLE_API_ERROR);
 334
 335         strbuf_release(&buf);
 336         for (i = 0; i < N; i++) {
 337                 reftable_free(names[i]);
 338         }
 339         reftable_iterator_destroy(&it);
 340         reftable_free(names);
 341         reader_close(&rd);
 342         strbuf_release(&buf);
 343 }
 344
 345 static void test_table_read_write_seek(int index, int hash_id)
 346 {
 347         char **names;
 348         struct strbuf buf = STRBUF_INIT;
 349         int N = 50;
 350         struct reftable_reader rd = { NULL };
 351         struct reftable_block_source source = { NULL };
 352         int err;
 353         int i = 0;
 354
 355         struct reftable_iterator it = { NULL };
 356         struct strbuf pastLast = STRBUF_INIT;
 357         struct reftable_ref_record ref = { NULL };
 358
 359         write_table(&names, &buf, N, 256, hash_id);
 360
 361         block_source_from_strbuf(&source, &buf);
 362
 363         err = init_reader(&rd, &source, "file.ref");
 364         EXPECT_ERR(err);
 365         EXPECT(hash_id == reftable_reader_hash_id(&rd));
 366
 367         if (!index) {
 368                 rd.ref_offsets.index_offset = 0;
 369         } else {
 370                 EXPECT(rd.ref_offsets.index_offset > 0);
 371         }
 372
 373         for (i = 1; i < N; i++) {
 374                 int err = reftable_reader_seek_ref(&rd, &it, names[i]);
 375                 EXPECT_ERR(err);
 376                 err = reftable_iterator_next_ref(&it, &ref);
 377                 EXPECT_ERR(err);
 378                 EXPECT(0 == strcmp(names[i], ref.refname));
 379                 EXPECT(REFTABLE_REF_VAL1 == ref.value_type);
 380                 EXPECT(i == ref.value.val1[0]);
 381
 382                 reftable_ref_record_release(&ref);
 383                 reftable_iterator_destroy(&it);
 384         }
 385
 386         strbuf_addstr(&pastLast, names[N - 1]);
 387         strbuf_addstr(&pastLast, "/");
 388
 389         err = reftable_reader_seek_ref(&rd, &it, pastLast.buf);
 390         if (err == 0) {
 391                 struct reftable_ref_record ref = { NULL };
 392                 int err = reftable_iterator_next_ref(&it, &ref);
 393                 EXPECT(err > 0);
 394         } else {
 395                 EXPECT(err > 0);
 396         }
 397
 398         strbuf_release(&pastLast);
 399         reftable_iterator_destroy(&it);
 400
 401         strbuf_release(&buf);
 402         for (i = 0; i < N; i++) {
 403                 reftable_free(names[i]);
 404         }
 405         reftable_free(names);
 406         reader_close(&rd);
 407 }
 408
 409 static void test_table_read_write_seek_linear(void)
 410 {
 411         test_table_read_write_seek(0, GIT_SHA1_FORMAT_ID);
 412 }
 413
 414 static void test_table_read_write_seek_linear_sha256(void)
 415 {
 416         test_table_read_write_seek(0, GIT_SHA256_FORMAT_ID);
 417 }
 418
 419 static void test_table_read_write_seek_index(void)
 420 {
 421         test_table_read_write_seek(1, GIT_SHA1_FORMAT_ID);
 422 }
 423
 424 static void test_table_refs_for(int indexed)
 425 {
 426         int N = 50;
 427         char **want_names = reftable_calloc(sizeof(char *) * (N + 1));
 428         int want_names_len = 0;
 429         uint8_t want_hash[GIT_SHA1_RAWSZ];
 430
 431         struct reftable_write_options opts = {
 432                 .block_size = 256,
 433         };
 434         struct reftable_ref_record ref = { NULL };
 435         int i = 0;
 436         int n;
 437         int err;
 438         struct reftable_reader rd;
 439         struct reftable_block_source source = { NULL };
 440
 441         struct strbuf buf = STRBUF_INIT;
 442         struct reftable_writer *w =
 443                 reftable_new_writer(&strbuf_add_void, &buf, &opts);
 444
 445         struct reftable_iterator it = { NULL };
 446         int j;
 447
 448         set_test_hash(want_hash, 4);
 449
 450         for (i = 0; i < N; i++) {
 451                 uint8_t hash[GIT_SHA1_RAWSZ];
 452                 char fill[51] = { 0 };
 453                 char name[100];
 454                 uint8_t hash1[GIT_SHA1_RAWSZ];
 455                 uint8_t hash2[GIT_SHA1_RAWSZ];
 456                 struct reftable_ref_record ref = { NULL };
 457
 458                 memset(hash, i, sizeof(hash));
 459                 memset(fill, 'x', 50);
 460                 /* Put the variable part in the start */
 461                 snprintf(name, sizeof(name), "br%02d%s", i, fill);
 462                 name[40] = 0;
 463                 ref.refname = name;
 464
 465                 set_test_hash(hash1, i / 4);
 466                 set_test_hash(hash2, 3 + i / 4);
 467                 ref.value_type = REFTABLE_REF_VAL2;
 468                 ref.value.val2.value = hash1;
 469                 ref.value.val2.target_value = hash2;
 470
 471                 /* 80 bytes / entry, so 3 entries per block. Yields 17
 472                  */
 473                 /* blocks. */
 474                 n = reftable_writer_add_ref(w, &ref);
 475                 EXPECT(n == 0);
 476
 477                 if (!memcmp(hash1, want_hash, GIT_SHA1_RAWSZ) ||
 478                     !memcmp(hash2, want_hash, GIT_SHA1_RAWSZ)) {
 479                         want_names[want_names_len++] = xstrdup(name);
 480                 }
 481         }
 482
 483         n = reftable_writer_close(w);
 484         EXPECT(n == 0);
 485
 486         reftable_writer_free(w);
 487         w = NULL;
 488
 489         block_source_from_strbuf(&source, &buf);
 490
 491         err = init_reader(&rd, &source, "file.ref");
 492         EXPECT_ERR(err);
 493         if (!indexed) {
 494                 rd.obj_offsets.is_present = 0;
 495         }
 496
 497         err = reftable_reader_seek_ref(&rd, &it, "");
 498         EXPECT_ERR(err);
 499         reftable_iterator_destroy(&it);
 500
 501         err = reftable_reader_refs_for(&rd, &it, want_hash);
 502         EXPECT_ERR(err);
 503
 504         j = 0;
 505         while (1) {
 506                 int err = reftable_iterator_next_ref(&it, &ref);
 507                 EXPECT(err >= 0);
 508                 if (err > 0) {
 509                         break;
 510                 }
 511
 512                 EXPECT(j < want_names_len);
 513                 EXPECT(0 == strcmp(ref.refname, want_names[j]));
 514                 j++;
 515                 reftable_ref_record_release(&ref);
 516         }
 517         EXPECT(j == want_names_len);
 518
 519         strbuf_release(&buf);
 520         free_names(want_names);
 521         reftable_iterator_destroy(&it);
 522         reader_close(&rd);
 523 }
 524
 525 static void test_table_refs_for_no_index(void)
 526 {
 527         test_table_refs_for(0);
 528 }
 529
 530 static void test_table_refs_for_obj_index(void)
 531 {
 532         test_table_refs_for(1);
 533 }
 534
 535 static void test_write_empty_table(void)
 536 {
 537         struct reftable_write_options opts = { 0 };
 538         struct strbuf buf = STRBUF_INIT;
 539         struct reftable_writer *w =
 540                 reftable_new_writer(&strbuf_add_void, &buf, &opts);
 541         struct reftable_block_source source = { NULL };
 542         struct reftable_reader *rd = NULL;
 543         struct reftable_ref_record rec = { NULL };
 544         struct reftable_iterator it = { NULL };
 545         int err;
 546
 547         reftable_writer_set_limits(w, 1, 1);
 548
 549         err = reftable_writer_close(w);
 550         EXPECT(err == REFTABLE_EMPTY_TABLE_ERROR);
 551         reftable_writer_free(w);
 552
 553         EXPECT(buf.len == header_size(1) + footer_size(1));
 554
 555         block_source_from_strbuf(&source, &buf);
 556
 557         err = reftable_new_reader(&rd, &source, "filename");
 558         EXPECT_ERR(err);
 559
 560         err = reftable_reader_seek_ref(rd, &it, "");
 561         EXPECT_ERR(err);
 562
 563         err = reftable_iterator_next_ref(&it, &rec);
 564         EXPECT(err > 0);
 565
 566         reftable_iterator_destroy(&it);
 567         reftable_reader_free(rd);
 568         strbuf_release(&buf);
 569 }
 570
 571 static void test_write_key_order(void)
 572 {
 573         struct reftable_write_options opts = { 0 };
 574         struct strbuf buf = STRBUF_INIT;
 575         struct reftable_writer *w =
 576                 reftable_new_writer(&strbuf_add_void, &buf, &opts);
 577         struct reftable_ref_record refs[2] = {
 578                 {
 579                         .refname = "b",
 580                         .update_index = 1,
 581                         .value_type = REFTABLE_REF_SYMREF,
 582                         .value = {
 583                                 .symref = "target",
 584                         },
 585                 }, {
 586                         .refname = "a",
 587                         .update_index = 1,
 588                         .value_type = REFTABLE_REF_SYMREF,
 589                         .value = {
 590                                 .symref = "target",
 591                         },
 592                 }
 593         };
 594         int err;
 595
 596         reftable_writer_set_limits(w, 1, 1);
 597         err = reftable_writer_add_ref(w, &refs[0]);
 598         EXPECT_ERR(err);
 599         err = reftable_writer_add_ref(w, &refs[1]);
 600         printf("%d\n", err);
 601         EXPECT(err == REFTABLE_API_ERROR);
 602         reftable_writer_close(w);
 603         reftable_writer_free(w);
 604         strbuf_release(&buf);
 605 }
 606
 607 static void test_corrupt_table_empty(void)
 608 {
 609         struct strbuf buf = STRBUF_INIT;
 610         struct reftable_block_source source = { NULL };
 611         struct reftable_reader rd = { NULL };
 612         int err;
 613
 614         block_source_from_strbuf(&source, &buf);
 615         err = init_reader(&rd, &source, "file.log");
 616         EXPECT(err == REFTABLE_FORMAT_ERROR);
 617 }
 618
 619 static void test_corrupt_table(void)
 620 {
 621         uint8_t zeros[1024] = { 0 };
 622         struct strbuf buf = STRBUF_INIT;
 623         struct reftable_block_source source = { NULL };
 624         struct reftable_reader rd = { NULL };
 625         int err;
 626         strbuf_add(&buf, zeros, sizeof(zeros));
 627
 628         block_source_from_strbuf(&source, &buf);
 629         err = init_reader(&rd, &source, "file.log");
 630         EXPECT(err == REFTABLE_FORMAT_ERROR);
 631         strbuf_release(&buf);
 632 }
 633
 634 int readwrite_test_main(int argc, const char *argv[])
 635 {
 636         RUN_TEST(test_corrupt_table);
 637         RUN_TEST(test_corrupt_table_empty);
 638         RUN_TEST(test_log_write_read);
 639         RUN_TEST(test_write_key_order);
 640         RUN_TEST(test_table_read_write_seek_linear_sha256);
 641         RUN_TEST(test_log_buffer_size);
 642         RUN_TEST(test_table_write_small_table);
 643         RUN_TEST(test_buffer);
 644         RUN_TEST(test_table_read_api);
 645         RUN_TEST(test_table_read_write_sequential);
 646         RUN_TEST(test_table_read_write_seek_linear);
 647         RUN_TEST(test_table_read_write_seek_index);
 648         RUN_TEST(test_table_refs_for_no_index);
 649         RUN_TEST(test_table_refs_for_obj_index);
 650         RUN_TEST(test_write_empty_table);
 651         return 0;
 652 }