src/test/test_consdiffmgr.c

   1 /* Copyright (c) 2017-2021, The Tor Project, Inc. */
   2 /* See LICENSE for licensing information */
   3
   4 #define CONSDIFFMGR_PRIVATE
   5
   6 #include "core/or/or.h"
   7 #include "app/config/config.h"
   8 #include "feature/dircache/conscache.h"
   9 #include "feature/dircommon/consdiff.h"
  10 #include "feature/dircache/consdiffmgr.h"
  11 #include "core/mainloop/cpuworker.h"
  12 #include "lib/crypt_ops/crypto_rand.h"
  13 #include "feature/nodelist/networkstatus.h"
  14 #include "feature/dirparse/ns_parse.h"
  15 #include "lib/evloop/workqueue.h"
  16 #include "lib/compress/compress.h"
  17 #include "lib/encoding/confline.h"
  18
  19 #include "feature/nodelist/networkstatus_st.h"
  20
  21 #include "test/test.h"
  22 #include "test/log_test_helpers.h"
  23
  24 #define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
  25
  26 static char *
  27 consensus_diff_apply_(const char *c, const char *d)
  28 {
  29   size_t c_len = strlen(c);
  30   size_t d_len = strlen(d);
  31   // We use memdup here to ensure that the input is NOT nul-terminated.
  32   // This makes it likelier for us to spot bugs.
  33   char *c_tmp = tor_memdup(c, c_len);
  34   char *d_tmp = tor_memdup(d, d_len);
  35   char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len);
  36   tor_free(c_tmp);
  37   tor_free(d_tmp);
  38   return result;
  39 }
  40
  41 // ============================== Setup/teardown the consdiffmgr
  42 // These functions get run before/after each test in this module
  43
  44 static void *
  45 consdiffmgr_test_setup(const struct testcase_t *arg)
  46 {
  47   (void)arg;
  48   char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
  49   tor_free(get_options_mutable()->CacheDirectory);
  50   get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
  51   check_private_dir(ddir_fname, CPD_CREATE, NULL);
  52
  53   consdiff_cfg_t consdiff_cfg = { 300 };
  54   consdiffmgr_configure(&consdiff_cfg);
  55   return (void *)1; // must return something non-null.
  56 }
  57 static int
  58 consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
  59 {
  60   (void)arg;
  61   (void)ignore;
  62   consdiffmgr_free_all();
  63   return 1;
  64 }
  65 static struct testcase_setup_t setup_diffmgr = {
  66   consdiffmgr_test_setup,
  67   consdiffmgr_test_teardown
  68 };
  69
  70 // ============================== NS faking functions
  71 // These functions are for making quick fake consensus objects and
  72 // strings that are just good enough for consdiff and consdiffmgr.
  73
  74 static networkstatus_t *
  75 fake_ns_new(consensus_flavor_t flav, time_t valid_after)
  76 {
  77   networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
  78   ns->type = NS_TYPE_CONSENSUS;
  79   ns->flavor = flav;
  80   ns->valid_after = valid_after;
  81   return ns;
  82 }
  83
  84 static char *
  85 fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
  86 {
  87   const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
  88   char valid_after_string[ISO_TIME_LEN+1];
  89
  90   format_iso_time(valid_after_string, valid_after);
  91   char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
  92   char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
  93
  94   char *consensus;
  95   tor_asprintf(&consensus,
  96                "network-status-version 3%s\n"
  97                "vote-status consensus\n"
  98                "valid-after %s\n"
  99                "r name ccccccccccccccccc etc\nsample\n"
 100                "r name eeeeeeeeeeeeeeeee etc\nbar\n"
 101                "%s\n"
 102                "directory-signature hello-there\n"
 103                "directory-signature %s\n",
 104                flavor_string,
 105                valid_after_string,
 106                random_stuff,
 107                random_stuff2);
 108   tor_free(random_stuff);
 109   tor_free(random_stuff2);
 110   return consensus;
 111 }
 112
 113 // ============================== Cpuworker mocking code
 114 // These mocking functions and types capture the cpuworker calls
 115 // so we can inspect them and run them in the main thread.
 116 static smartlist_t *fake_cpuworker_queue = NULL;
 117 typedef struct fake_work_queue_ent_t {
 118   enum workqueue_reply_t (*fn)(void *, void *);
 119   void (*reply_fn)(void *);
 120   void *arg;
 121 } fake_work_queue_ent_t;
 122 static struct workqueue_entry_t *
 123 mock_cpuworker_queue_work(workqueue_priority_t prio,
 124                           enum workqueue_reply_t (*fn)(void *, void *),
 125                           void (*reply_fn)(void *),
 126                           void *arg)
 127 {
 128   (void) prio;
 129
 130   if (! fake_cpuworker_queue)
 131     fake_cpuworker_queue = smartlist_new();
 132
 133   fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
 134   ent->fn = fn;
 135   ent->reply_fn = reply_fn;
 136   ent->arg = arg;
 137   smartlist_add(fake_cpuworker_queue, ent);
 138   return (struct workqueue_entry_t *)ent;
 139 }
 140 static int
 141 mock_cpuworker_run_work(void)
 142 {
 143   if (! fake_cpuworker_queue)
 144     return 0;
 145   SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
 146       enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
 147       if (r != WQ_RPL_REPLY)
 148         return -1;
 149   });
 150   return 0;
 151 }
 152 static void
 153 mock_cpuworker_handle_replies(void)
 154 {
 155   if (! fake_cpuworker_queue)
 156     return;
 157   SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
 158       ent->reply_fn(ent->arg);
 159       tor_free(ent);
 160   });
 161   smartlist_free(fake_cpuworker_queue);
 162   fake_cpuworker_queue = NULL;
 163 }
 164
 165 // ==============================  Other helpers
 166
 167 static consdiff_status_t
 168 lookup_diff_from(consensus_cache_entry_t **out,
 169                  consensus_flavor_t flav,
 170                  const char *str1)
 171 {
 172   uint8_t digest[DIGEST256_LEN];
 173   if (router_get_networkstatus_v3_sha3_as_signed(digest,
 174                                                  str1, strlen(str1))<0) {
 175     TT_FAIL(("Unable to compute sha3-as-signed"));
 176     return CONSDIFF_NOT_FOUND;
 177   }
 178   return consdiffmgr_find_diff_from(out, flav,
 179                                     DIGEST_SHA3_256, digest, sizeof(digest),
 180                                     NO_METHOD);
 181 }
 182
 183 static int
 184 lookup_apply_and_verify_diff(consensus_flavor_t flav,
 185                              const char *str1,
 186                              const char *str2)
 187 {
 188   consensus_cache_entry_t *ent = NULL;
 189   consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
 190   if (ent == NULL || status != CONSDIFF_AVAILABLE) {
 191     return -1;
 192   }
 193
 194   consensus_cache_entry_incref(ent);
 195   size_t size;
 196   const char *diff_string = NULL;
 197   char *diff_owned = NULL;
 198   int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent);
 199   consensus_cache_entry_decref(ent);
 200   if (diff_string == NULL || r < 0)
 201     return -1;
 202
 203   char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size);
 204   tor_free(diff_owned);
 205   if (applied == NULL)
 206     return -1;
 207
 208   int match = !strcmp(applied, str2);
 209   tor_free(applied);
 210   return match ? 0 : -1;
 211 }
 212
 213 static void
 214 cdm_reload(void)
 215 {
 216   consdiffmgr_free_all();
 217   cdm_cache_get();
 218   consdiffmgr_rescan();
 219 }
 220
 221 // ==============================  Beginning of tests
 222
 223 #if 0
 224 static int got_failure = 0;
 225 static void
 226 got_assertion_failure(void)
 227 {
 228   ++got_failure;
 229 }
 230
 231 /* XXXX This test won't work, because there is currently no way to actually
 232  * XXXX capture a real assertion failure. */
 233 static void
 234 test_consdiffmgr_init_failure(void *arg)
 235 {
 236   (void)arg;
 237   // Capture assertions and bugs.
 238
 239   /* As in ...test_setup, but do not create the datadir. The missing directory
 240    * will cause a failure. */
 241   char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
 242   tor_free(get_options_mutable()->CacheDirectory);
 243   get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
 244
 245   consdiff_cfg_t consdiff_cfg = { 7200, 300 };
 246
 247   tor_set_failed_assertion_callback(got_assertion_failure);
 248   tor_capture_bugs_(1);
 249   consdiffmgr_configure(&consdiff_cfg); // This should fail.
 250   tt_int_op(got_failure, OP_EQ, 1);
 251   const smartlist_t *bugs = tor_get_captured_bug_log_();
 252   tt_int_op(smartlist_len(bugs), OP_EQ, 1);
 253
 254  done:
 255   tor_end_capture_bugs_();
 256 }
 257 #endif /* 0 */
 258
 259 static void
 260 test_consdiffmgr_sha3_helper(void *arg)
 261 {
 262   (void) arg;
 263   consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
 264   config_line_t *lines = NULL;
 265   char *mem_op_hex_tmp = NULL;
 266   config_line_prepend(&lines, "good-sha",
 267                       "F00DF00DF00DF00DF00DF00DF00DF00D"
 268                       "F00DF00DF00DF00DF00DF00DF00DF00D");
 269   config_line_prepend(&lines, "short-sha",
 270                       "F00DF00DF00DF00DF00DF00DF00DF00D"
 271                       "F00DF00DF00DF00DF00DF00DF00DF0");
 272   config_line_prepend(&lines, "long-sha",
 273                       "F00DF00DF00DF00DF00DF00DF00DF00D"
 274                       "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
 275   config_line_prepend(&lines, "not-sha",
 276                       "F00DF00DF00DF00DF00DF00DF00DF00D"
 277                       "F00DF00DF00DF00DF00DF00DF00DXXXX");
 278   consensus_cache_entry_t *ent =
 279     consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
 280
 281   uint8_t buf[DIGEST256_LEN];
 282   tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
 283   tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
 284   test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
 285                       "F00DF00DF00DF00DF00DF00DF00DF00D");
 286
 287   tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
 288   tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
 289   tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
 290   tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
 291
 292  done:
 293   consensus_cache_entry_decref(ent);
 294   config_free_lines(lines);
 295   tor_free(mem_op_hex_tmp);
 296 }
 297
 298 static void
 299 test_consdiffmgr_add(void *arg)
 300 {
 301   (void) arg;
 302   time_t now = approx_time();
 303
 304   const char *body = NULL;
 305   char *body_owned = NULL;
 306
 307   consensus_cache_entry_t *ent = NULL;
 308   networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
 309   const char *dummy = "foo";
 310   int r = consdiffmgr_add_consensus(dummy, ns_tmp);
 311   tt_int_op(r, OP_EQ, 0);
 312
 313   /* If we add it again, it won't work */
 314   setup_capture_of_logs(LOG_INFO);
 315   dummy = "bar";
 316   r = consdiffmgr_add_consensus(dummy, ns_tmp);
 317   tt_int_op(r, OP_EQ, -1);
 318   expect_single_log_msg_containing("We already have a copy of that "
 319                                    "consensus");
 320   mock_clean_saved_logs();
 321
 322   /* But it will work fine if the flavor is different */
 323   dummy = "baz";
 324   ns_tmp->flavor = FLAV_MICRODESC;
 325   r = consdiffmgr_add_consensus(dummy, ns_tmp);
 326   tt_int_op(r, OP_EQ, 0);
 327
 328   /* And it will work fine if the time is different */
 329   dummy = "quux";
 330   ns_tmp->flavor = FLAV_NS;
 331   ns_tmp->valid_after = now - 60;
 332   r = consdiffmgr_add_consensus(dummy, ns_tmp);
 333   tt_int_op(r, OP_EQ, 0);
 334
 335   /* If we add one a long long time ago, it will fail. */
 336   dummy = "xyzzy";
 337   ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
 338   r = consdiffmgr_add_consensus(dummy, ns_tmp);
 339   tt_int_op(r, OP_EQ, -1);
 340   expect_log_msg_containing("it's too old.");
 341
 342   /* Try looking up a consensuses. */
 343   ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
 344   tt_assert(ent);
 345   consensus_cache_entry_incref(ent);
 346   size_t s;
 347   r = uncompress_or_set_ptr(&body, &s, &body_owned, ent);
 348   tt_int_op(r, OP_EQ, 0);
 349   tt_int_op(s, OP_EQ, 4);
 350   tt_mem_op(body, OP_EQ, "quux", 4);
 351
 352   /* Try looking up another entry, but fail */
 353   tt_ptr_op(cdm_cache_lookup_consensus(FLAV_MICRODESC, now - 60), OP_EQ, NULL);
 354   tt_ptr_op(cdm_cache_lookup_consensus(FLAV_NS, now - 61), OP_EQ, NULL);
 355
 356  done:
 357   networkstatus_vote_free(ns_tmp);
 358   teardown_capture_of_logs();
 359   consensus_cache_entry_decref(ent);
 360   tor_free(body_owned);
 361 }
 362
 363 static void
 364 test_consdiffmgr_make_diffs(void *arg)
 365 {
 366   (void)arg;
 367   networkstatus_t *ns = NULL;
 368   char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
 369   char *applied = NULL, *diff_text = NULL;
 370   time_t now = approx_time();
 371   int r;
 372   consensus_cache_entry_t *diff = NULL;
 373   uint8_t md_ns_sha3[DIGEST256_LEN];
 374   consdiff_status_t diff_status;
 375
 376   MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
 377
 378   // Try rescan with no consensuses: shouldn't crash or queue work.
 379   consdiffmgr_rescan();
 380   tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
 381
 382   // Make two consensuses, 1 hour sec ago.
 383   ns = fake_ns_new(FLAV_NS, now-3600);
 384   ns_body = fake_ns_body_new(FLAV_NS, now-3600);
 385   r = consdiffmgr_add_consensus(ns_body, ns);
 386   networkstatus_vote_free(ns);
 387   tor_free(ns_body);
 388   tt_int_op(r, OP_EQ, 0);
 389
 390   ns = fake_ns_new(FLAV_MICRODESC, now-3600);
 391   md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
 392   r = consdiffmgr_add_consensus(md_ns_body, ns);
 393   router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body,
 394                                              strlen(md_ns_body));
 395   networkstatus_vote_free(ns);
 396   tt_int_op(r, OP_EQ, 0);
 397
 398   // No diffs will be generated.
 399   consdiffmgr_rescan();
 400   tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
 401
 402   // Add a MD consensus from 45 minutes ago. This should cause one diff
 403   // worth of work to get queued.
 404   ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
 405   md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
 406   r = consdiffmgr_add_consensus(md_ns_body_2, ns);
 407   networkstatus_vote_free(ns);
 408   tt_int_op(r, OP_EQ, 0);
 409
 410   consdiffmgr_rescan();
 411   tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
 412   tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
 413   diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
 414                                            DIGEST_SHA3_256,
 415                                            md_ns_sha3, DIGEST256_LEN,
 416                                            NO_METHOD);
 417   tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
 418
 419   // Now run that process and get the diff.
 420   r = mock_cpuworker_run_work();
 421   tt_int_op(r, OP_EQ, 0);
 422   mock_cpuworker_handle_replies();
 423
 424   // At this point we should be able to get that diff.
 425   diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
 426                                            DIGEST_SHA3_256,
 427                                            md_ns_sha3, DIGEST256_LEN,
 428                                            NO_METHOD);
 429   tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
 430   tt_assert(diff);
 431
 432   /* Make sure applying the diff actually works */
 433   const uint8_t *diff_body;
 434   size_t diff_size;
 435   r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
 436   tt_int_op(r, OP_EQ, 0);
 437   diff_text = tor_memdup_nulterm(diff_body, diff_size);
 438   applied = consensus_diff_apply_(md_ns_body, diff_text);
 439   tt_assert(applied);
 440   tt_str_op(applied, OP_EQ, md_ns_body_2);
 441
 442   /* Rescan again: no more work to do. */
 443   consdiffmgr_rescan();
 444   tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
 445
 446  done:
 447   tor_free(md_ns_body);
 448   tor_free(md_ns_body_2);
 449   tor_free(diff_text);
 450   tor_free(applied);
 451 }
 452
 453 static void
 454 test_consdiffmgr_diff_rules(void *arg)
 455 {
 456   (void)arg;
 457 #define N 6
 458   char *md_body[N], *ns_body[N];
 459   networkstatus_t *md_ns[N], *ns_ns[N];
 460   int i;
 461
 462   MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
 463
 464   /* Create a bunch of consensus things at 15-second intervals. */
 465   time_t start = approx_time() - 120;
 466   for (i = 0; i < N; ++i) {
 467     time_t when = start + i * 15;
 468     md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
 469     ns_body[i] = fake_ns_body_new(FLAV_NS, when);
 470     md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
 471     ns_ns[i] = fake_ns_new(FLAV_NS, when);
 472   }
 473
 474   /* For the MD consensuses: add 4 of them, and make sure that
 475    * diffs are created to one consensus (the most recent) only. */
 476   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
 477   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
 478   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
 479   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
 480   consdiffmgr_rescan();
 481   tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
 482   tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
 483   tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
 484   mock_cpuworker_handle_replies();
 485   tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
 486
 487   /* For the NS consensuses: add 3, generate, and add one older one and
 488    * make sure that older one is the only one whose diff is generated */
 489   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
 490   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
 491   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
 492   consdiffmgr_rescan();
 493   tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
 494   tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
 495   tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
 496   mock_cpuworker_handle_replies();
 497
 498   /* At this point, we should actually have working diffs! */
 499   tt_int_op(0, OP_EQ,
 500        lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
 501   tt_int_op(0, OP_EQ,
 502        lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
 503
 504   tt_int_op(0, OP_EQ,
 505        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
 506   tt_int_op(0, OP_EQ,
 507        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
 508   tt_int_op(0, OP_EQ,
 509        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
 510
 511   /* Self-to-self diff won't be present */
 512   consensus_cache_entry_t *ent;
 513   tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
 514        lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
 515   /* No diff from 2 has been added yet */
 516   tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
 517        lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
 518   /* No diff arriving at old things. */
 519   tt_int_op(-1, OP_EQ,
 520        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
 521   /* No backwards diff */
 522   tt_int_op(-1, OP_EQ,
 523        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
 524
 525   /* Now, an update: add number 2 and make sure it's the only one whose diff
 526    * is regenerated. */
 527   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
 528   consdiffmgr_rescan();
 529   tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
 530   tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
 531   tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
 532   mock_cpuworker_handle_replies();
 533
 534   tt_int_op(0, OP_EQ,
 535             lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
 536
 537   /* Finally: reload, and make sure that the information is still indexed */
 538   cdm_reload();
 539
 540   tt_int_op(0, OP_EQ,
 541        lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
 542   tt_int_op(0, OP_EQ,
 543        lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
 544   tt_int_op(0, OP_EQ,
 545        lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
 546
 547   tt_int_op(0, OP_EQ,
 548        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
 549   tt_int_op(0, OP_EQ,
 550        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
 551   tt_int_op(0, OP_EQ,
 552        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
 553
 554  done:
 555   for (i = 0; i < N; ++i) {
 556     tor_free(md_body[i]);
 557     tor_free(ns_body[i]);
 558     networkstatus_vote_free(md_ns[i]);
 559     networkstatus_vote_free(ns_ns[i]);
 560   }
 561   UNMOCK(cpuworker_queue_work);
 562 #undef N
 563 }
 564
 565 static void
 566 test_consdiffmgr_diff_failure(void *arg)
 567 {
 568   (void)arg;
 569   MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
 570
 571   /* We're going to make sure that if we have a bogus request where
 572    * we can't actually compute a diff, the world must not end. */
 573   networkstatus_t *ns1 = NULL;
 574   networkstatus_t *ns2 = NULL;
 575   int r;
 576
 577   ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
 578   ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
 579   r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
 580   tt_int_op(r, OP_EQ, 0);
 581   // We refuse to compute a diff to or from a line holding only a single dot.
 582   // We can add it here, though.
 583   r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
 584   tt_int_op(r, OP_EQ, 0);
 585
 586   consdiffmgr_rescan();
 587   tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
 588   setup_capture_of_logs(LOG_WARN);
 589   tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
 590   tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
 591   expect_single_log_msg_containing("one of the lines to be added is \".\".");
 592   mock_clean_saved_logs();
 593   mock_cpuworker_handle_replies();
 594   expect_single_log_msg_containing("Worker was unable to compute consensus "
 595                                    "diff from ");
 596
 597   /* Make sure the diff is not present */
 598   consensus_cache_entry_t *ent;
 599   tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
 600             lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
 601
 602  done:
 603   teardown_capture_of_logs();
 604   UNMOCK(cpuworker_queue_work);
 605   networkstatus_vote_free(ns1);
 606   networkstatus_vote_free(ns2);
 607 }
 608
 609 static void
 610 test_consdiffmgr_diff_pending(void *arg)
 611 {
 612 #define N 3
 613   (void)arg;
 614   char *md_body[N];
 615   networkstatus_t *md_ns[N];
 616   time_t start = approx_time() - 120;
 617   int i;
 618   for (i = 0; i < N; ++i) {
 619     time_t when = start + i * 30;
 620     md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
 621     md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
 622   }
 623
 624   MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
 625
 626   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
 627   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
 628   /* Make a diff */
 629   consdiffmgr_rescan();
 630   tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
 631
 632   /* Look it up.  Is it pending? */
 633   consensus_cache_entry_t *ent = NULL;
 634   consdiff_status_t diff_status;
 635   diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
 636   tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
 637   tt_ptr_op(ent, OP_EQ, NULL);
 638
 639   /* Add another old consensus.  only one new diff should launch! */
 640   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
 641   consdiffmgr_rescan();
 642   tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
 643
 644   tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
 645   mock_cpuworker_handle_replies();
 646
 647   tt_int_op(0, OP_EQ,
 648        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
 649   tt_int_op(0, OP_EQ,
 650        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
 651
 652  done:
 653   UNMOCK(cpuworker_queue_work);
 654   for (i = 0; i < N; ++i) {
 655     tor_free(md_body[i]);
 656     networkstatus_vote_free(md_ns[i]);
 657   }
 658 #undef N
 659 }
 660
 661 static void
 662 test_consdiffmgr_cleanup_old(void *arg)
 663 {
 664   (void)arg;
 665   config_line_t *labels = NULL;
 666   consensus_cache_entry_t *ent = NULL;
 667   consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
 668
 669   /* This item will be will be cleanable because it has a valid-after
 670    * time far in the past. */
 671   config_line_prepend(&labels, "document-type", "confribble-blarg");
 672   config_line_prepend(&labels, "consensus-valid-after",
 673                       "1980-10-10T10:10:10");
 674   ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
 675   tt_assert(ent);
 676   consensus_cache_entry_decref(ent);
 677
 678   setup_capture_of_logs(LOG_DEBUG);
 679   tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
 680   expect_log_msg_containing("Deleting entry because its consensus-valid-"
 681                             "after value (1980-10-10T10:10:10) was too old");
 682
 683  done:
 684   teardown_capture_of_logs();
 685   config_free_lines(labels);
 686 }
 687
 688 static void
 689 test_consdiffmgr_cleanup_bad_valid_after(void *arg)
 690 {
 691   /* This will seem cleanable, but isn't, because its valid-after time is
 692    * malformed. */
 693
 694   (void)arg;
 695   config_line_t *labels = NULL;
 696   consensus_cache_entry_t *ent = NULL;
 697   consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
 698
 699   config_line_prepend(&labels, "document-type", "consensus");
 700   config_line_prepend(&labels, "consensus-valid-after",
 701                       "whan that aprille with his shoures soote"); // (~1385?)
 702   ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
 703   tt_assert(ent);
 704   consensus_cache_entry_decref(ent);
 705
 706   setup_capture_of_logs(LOG_DEBUG);
 707   tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
 708   expect_log_msg_containing("Ignoring entry because its consensus-valid-"
 709                             "after value (\"whan that aprille with his "
 710                             "shoures soote\") was unparseable");
 711
 712  done:
 713   teardown_capture_of_logs();
 714   config_free_lines(labels);
 715 }
 716
 717 static void
 718 test_consdiffmgr_cleanup_no_valid_after(void *arg)
 719 {
 720   (void)arg;
 721   config_line_t *labels = NULL;
 722   consensus_cache_entry_t *ent = NULL;
 723   consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
 724
 725   /* This item will be will be uncleanable because it has no recognized
 726    * valid-after. */
 727   config_line_prepend(&labels, "document-type", "consensus");
 728   config_line_prepend(&labels, "confrooble-voolid-oofter",
 729                       "2010-10-10T09:08:07");
 730   ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
 731   tt_assert(ent);
 732   consensus_cache_entry_decref(ent);
 733
 734   setup_capture_of_logs(LOG_DEBUG);
 735   tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
 736   expect_log_msg_containing("Ignoring entry because it had no consensus-"
 737                             "valid-after label");
 738
 739  done:
 740   teardown_capture_of_logs();
 741   config_free_lines(labels);
 742 }
 743
 744 static void
 745 test_consdiffmgr_cleanup_old_diffs(void *arg)
 746 {
 747   (void)arg;
 748 #define N 4
 749   char *md_body[N];
 750   networkstatus_t *md_ns[N];
 751   int i;
 752   consensus_cache_entry_t *hold_ent = NULL, *ent;
 753
 754   /* Make sure that the cleanup function removes diffs to the not-most-recent
 755    * consensus. */
 756
 757   MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
 758
 759   /* Create a bunch of consensus things at 15-second intervals. */
 760   time_t start = approx_time() - 120;
 761   for (i = 0; i < N; ++i) {
 762     time_t when = start + i * 15;
 763     md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
 764     md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
 765   }
 766
 767   /* add the first 3. */
 768   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
 769   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
 770   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
 771   /* Make diffs. */
 772   consdiffmgr_rescan();
 773   tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
 774   tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
 775   tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
 776   mock_cpuworker_handle_replies();
 777   tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
 778
 779   /* Nothing is deletable now */
 780   tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
 781   tt_int_op(0, OP_EQ,
 782        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
 783   tt_int_op(0, OP_EQ,
 784        lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
 785
 786   tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
 787             lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
 788   consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
 789
 790   /* Now add an even-more-recent consensus; this should make all previous
 791    * diffs deletable, and make delete */
 792   tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
 793   tt_int_op(2 * n_diff_compression_methods() +
 794             (n_consensus_compression_methods() - 1) , OP_EQ,
 795             consdiffmgr_cleanup());
 796
 797   tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
 798             lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
 799   /* This one is marked deletable but still in the hashtable */
 800   tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
 801             lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
 802   tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
 803             lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
 804
 805   /* Everything should be valid at this point */
 806   tt_int_op(0, OP_EQ, consdiffmgr_validate());
 807
 808   /* And if we recan NOW, we'll purge the hashtable of the entries,
 809    * and launch attempts to generate new ones */
 810   consdiffmgr_rescan();
 811   tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
 812             lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
 813   tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
 814             lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
 815   tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
 816             lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
 817
 818   /* We're still holding on to this, though, so we can still map it! */
 819   const uint8_t *t1 = NULL;
 820   size_t s;
 821   int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
 822   tt_int_op(r, OP_EQ, 0);
 823   tt_assert(t1);
 824
 825  done:
 826   for (i = 0; i < N; ++i) {
 827     tor_free(md_body[i]);
 828     networkstatus_vote_free(md_ns[i]);
 829   }
 830   consensus_cache_entry_decref(hold_ent);
 831   UNMOCK(cpuworker_queue_work);
 832 #undef N
 833 }
 834
 835 static void
 836 test_consdiffmgr_validate(void *arg)
 837 {
 838   (void)arg;
 839   config_line_t *lines = NULL;
 840   consensus_cache_entry_t *ent = NULL;
 841   consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
 842   smartlist_t *vals = smartlist_new();
 843
 844   /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
 845    * one with a wrong sha3, and one with no sha3. */
 846   config_line_prepend(&lines, "id", "wrong sha3");
 847   config_line_prepend(&lines, "sha3-digest",
 848                       "F00DF00DF00DF00DF00DF00DF00DF00D"
 849                       "F00DF00DF00DF00DF00DF00DF00DF00D");
 850   ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
 851   consensus_cache_entry_decref(ent);
 852   config_free_lines(lines);
 853   lines = NULL;
 854
 855   config_line_prepend(&lines, "id", "bad sha3");
 856   config_line_prepend(&lines, "sha3-digest",
 857                       "now is the winter of our dicotheque");
 858   ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
 859   consensus_cache_entry_decref(ent);
 860   config_free_lines(lines);
 861   lines = NULL;
 862
 863   config_line_prepend(&lines, "id", "no sha3");
 864   ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
 865   consensus_cache_entry_decref(ent);
 866   config_free_lines(lines);
 867   lines = NULL;
 868
 869   config_line_prepend(&lines, "id", "good sha3");
 870   config_line_prepend(&lines, "sha3-digest",
 871                       "8d8b1998616cd6b4c4055da8d38728dc"
 872                       "93c758d4131a53c7d81aa6337dee1c05");
 873   ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
 874   consensus_cache_entry_decref(ent);
 875   config_free_lines(lines);
 876   lines = NULL;
 877
 878   cdm_reload();
 879   cache = cdm_cache_get();
 880   tt_int_op(1, OP_EQ, consdiffmgr_validate());
 881
 882   consensus_cache_find_all(vals, cache, "id", "good sha3");
 883   tt_int_op(smartlist_len(vals), OP_EQ, 1);
 884   smartlist_clear(vals);
 885
 886   consensus_cache_find_all(vals, cache, "id", "no sha3");
 887   tt_int_op(smartlist_len(vals), OP_EQ, 1);
 888   smartlist_clear(vals);
 889
 890   consensus_cache_find_all(vals, cache, "id", "wrong sha3");
 891   tt_int_op(smartlist_len(vals), OP_EQ, 0);
 892   consensus_cache_find_all(vals, cache, "id", "bad sha3");
 893   tt_int_op(smartlist_len(vals), OP_EQ, 0);
 894
 895  done:
 896   smartlist_free(vals);
 897 }
 898
 899 #define TEST(name)                                      \
 900   { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
 901
 902 struct testcase_t consdiffmgr_tests[] = {
 903 #if 0
 904   { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
 905 #endif
 906   TEST(sha3_helper),
 907   TEST(add),
 908   TEST(make_diffs),
 909   TEST(diff_rules),
 910   TEST(diff_failure),
 911   TEST(diff_pending),
 912   TEST(cleanup_old),
 913   TEST(cleanup_bad_valid_after),
 914   TEST(cleanup_no_valid_after),
 915   TEST(cleanup_old_diffs),
 916   TEST(validate),
 917
 918   // XXXX Test: non-cacheing cases of replyfn().
 919
 920   END_OF_TESTCASES
 921 };