src/test/test_stats.c

   1 /* Copyright (c) 2001-2004, Roger Dingledine.
   2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   3  * Copyright (c) 2007-2021, The Tor Project, Inc. */
   4 /* See LICENSE for licensing information */
   5
   6 /**
   7  * \file test_stats.c
   8  * \brief Unit tests for the statistics (reputation history) module.
   9  **/
  10
  11 #include "orconfig.h"
  12 #include "lib/crypt_ops/crypto_rand.h"
  13 #include "app/config/or_state_st.h"
  14 #include "test/rng_test_helpers.h"
  15 #include "feature/hs/hs_cache.h"
  16 #include "test/hs_test_helpers.h"
  17
  18 #include <stdio.h>
  19
  20 #ifdef _WIN32
  21 /* For mkdir() */
  22 #include <direct.h>
  23 #else
  24 #include <dirent.h>
  25 #endif /* defined(_WIN32) */
  26
  27 #include <math.h>
  28
  29 /* These macros pull in declarations for some functions and structures that
  30  * are typically file-private. */
  31 #define CIRCUITSTATS_PRIVATE
  32 #define CIRCUITLIST_PRIVATE
  33 #define MAINLOOP_PRIVATE
  34 #define STATEFILE_PRIVATE
  35 #define BWHIST_PRIVATE
  36 #define REPHIST_PRIVATE
  37 #define ROUTER_PRIVATE
  38
  39 #include "core/or/or.h"
  40 #include "lib/err/backtrace.h"
  41 #include "lib/buf/buffers.h"
  42 #include "core/or/circuitstats.h"
  43 #include "app/config/config.h"
  44 #include "test/test.h"
  45 #include "core/mainloop/mainloop.h"
  46 #include "lib/memarea/memarea.h"
  47 #include "feature/stats/connstats.h"
  48 #include "feature/stats/rephist.h"
  49 #include "app/config/statefile.h"
  50 #include "feature/stats/bwhist.h"
  51 #include "feature/stats/bw_array_st.h"
  52 #include "feature/relay/router.h"
  53
  54 /** Run unit tests for some stats code. */
  55 static void
  56 test_stats(void *arg)
  57 {
  58   time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
  59   char *s = NULL;
  60   int i;
  61
  62   /* Start with testing exit port statistics; we shouldn't collect exit
  63    * stats without initializing them. */
  64   (void)arg;
  65   rep_hist_note_exit_stream_opened(80);
  66   rep_hist_note_exit_bytes(80, 100, 10000);
  67   s = rep_hist_format_exit_stats(now + 86400);
  68   tt_ptr_op(s, OP_EQ, NULL);
  69
  70   /* Initialize stats, note some streams and bytes, and generate history
  71    * string. */
  72   rep_hist_exit_stats_init(now);
  73   rep_hist_note_exit_stream_opened(80);
  74   rep_hist_note_exit_bytes(80, 100, 10000);
  75   rep_hist_note_exit_stream_opened(443);
  76   rep_hist_note_exit_bytes(443, 100, 10000);
  77   rep_hist_note_exit_bytes(443, 100, 10000);
  78   s = rep_hist_format_exit_stats(now + 86400);
  79   tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
  80              "exit-kibibytes-written 80=1,443=1,other=0\n"
  81              "exit-kibibytes-read 80=10,443=20,other=0\n"
  82              "exit-streams-opened 80=4,443=4,other=0\n",OP_EQ, s);
  83   tor_free(s);
  84
  85   /* Add a few bytes on 10 more ports and ensure that only the top 10
  86    * ports are contained in the history string. */
  87   for (i = 50; i < 60; i++) {
  88     rep_hist_note_exit_bytes(i, i, i);
  89     rep_hist_note_exit_stream_opened(i);
  90   }
  91   s = rep_hist_format_exit_stats(now + 86400);
  92   tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
  93              "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
  94              "59=1,80=1,443=1,other=1\n"
  95              "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
  96              "59=1,80=10,443=20,other=1\n"
  97              "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4,"
  98              "59=4,80=4,443=4,other=4\n",OP_EQ, s);
  99   tor_free(s);
 100
 101   /* Stop collecting stats, add some bytes, and ensure we don't generate
 102    * a history string. */
 103   rep_hist_exit_stats_term();
 104   rep_hist_note_exit_bytes(80, 100, 10000);
 105   s = rep_hist_format_exit_stats(now + 86400);
 106   tt_ptr_op(s, OP_EQ, NULL);
 107
 108   /* Re-start stats, add some bytes, reset stats, and see what history we
 109    * get when observing no streams or bytes at all. */
 110   rep_hist_exit_stats_init(now);
 111   rep_hist_note_exit_stream_opened(80);
 112   rep_hist_note_exit_bytes(80, 100, 10000);
 113   rep_hist_reset_exit_stats(now);
 114   s = rep_hist_format_exit_stats(now + 86400);
 115   tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
 116              "exit-kibibytes-written other=0\n"
 117              "exit-kibibytes-read other=0\n"
 118              "exit-streams-opened other=0\n",OP_EQ, s);
 119   tor_free(s);
 120
 121   /* Continue with testing connection statistics; we shouldn't collect
 122    * conn stats without initializing them. */
 123   conn_stats_note_or_conn_bytes(1, 20, 400, now, false);
 124   s = conn_stats_format(now + 86400);
 125   tt_ptr_op(s, OP_EQ, NULL);
 126
 127   /* Initialize stats, note bytes, and generate history string. */
 128   conn_stats_init(now);
 129   conn_stats_note_or_conn_bytes(1, 30000, 400000, now, false);
 130   conn_stats_note_or_conn_bytes(1, 30000, 400000, now + 5, false);
 131   conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 10, true);
 132   conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 15, true);
 133   s = conn_stats_format(now + 86400);
 134   tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n"
 135             "ipv6-conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",
 136             OP_EQ, s);
 137   tor_free(s);
 138
 139   /* Stop collecting stats, add some bytes, and ensure we don't generate
 140    * a history string. */
 141   conn_stats_terminate();
 142   conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 15, true);
 143   s = conn_stats_format(now + 86400);
 144   tt_ptr_op(s, OP_EQ, NULL);
 145
 146   /* Re-start stats, add some bytes, reset stats, and see what history we
 147    * get when observing no bytes at all. */
 148   conn_stats_init(now);
 149   conn_stats_note_or_conn_bytes(1, 30000, 400000, now, false);
 150   conn_stats_note_or_conn_bytes(1, 30000, 400000, now + 5, false);
 151   conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 10, true);
 152   conn_stats_note_or_conn_bytes(2, 400000, 30000, now + 15, true);
 153   conn_stats_reset(now);
 154   s = conn_stats_format(now + 86400);
 155   tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n"
 156             "ipv6-conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",
 157             OP_EQ, s);
 158   tor_free(s);
 159
 160   /* Continue with testing buffer statistics; we shouldn't collect buffer
 161    * stats without initializing them. */
 162   rep_hist_add_buffer_stats(2.0, 2.0, 20);
 163   s = rep_hist_format_buffer_stats(now + 86400);
 164   tt_ptr_op(s, OP_EQ, NULL);
 165
 166   /* Initialize stats, add statistics for a single circuit, and generate
 167    * the history string. */
 168   rep_hist_buffer_stats_init(now);
 169   rep_hist_add_buffer_stats(2.0, 2.0, 20);
 170   s = rep_hist_format_buffer_stats(now + 86400);
 171   tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
 172              "cell-processed-cells 20,0,0,0,0,0,0,0,0,0\n"
 173              "cell-queued-cells 2.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
 174                                "0.00,0.00\n"
 175              "cell-time-in-queue 2,0,0,0,0,0,0,0,0,0\n"
 176              "cell-circuits-per-decile 1\n",OP_EQ, s);
 177   tor_free(s);
 178
 179   /* Add nineteen more circuit statistics to the one that's already in the
 180    * history to see that the math works correctly. */
 181   for (i = 21; i < 30; i++)
 182     rep_hist_add_buffer_stats(2.0, 2.0, i);
 183   for (i = 20; i < 30; i++)
 184     rep_hist_add_buffer_stats(3.5, 3.5, i);
 185   s = rep_hist_format_buffer_stats(now + 86400);
 186   tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
 187              "cell-processed-cells 29,28,27,26,25,24,23,22,21,20\n"
 188              "cell-queued-cells 2.75,2.75,2.75,2.75,2.75,2.75,2.75,2.75,"
 189                                "2.75,2.75\n"
 190              "cell-time-in-queue 3,3,3,3,3,3,3,3,3,3\n"
 191              "cell-circuits-per-decile 2\n",OP_EQ, s);
 192   tor_free(s);
 193
 194   /* Stop collecting stats, add statistics for one circuit, and ensure we
 195    * don't generate a history string. */
 196   rep_hist_buffer_stats_term();
 197   rep_hist_add_buffer_stats(2.0, 2.0, 20);
 198   s = rep_hist_format_buffer_stats(now + 86400);
 199   tt_ptr_op(s, OP_EQ, NULL);
 200
 201   /* Re-start stats, add statistics for one circuit, reset stats, and make
 202    * sure that the history has all zeros. */
 203   rep_hist_buffer_stats_init(now);
 204   rep_hist_add_buffer_stats(2.0, 2.0, 20);
 205   rep_hist_reset_buffer_stats(now);
 206   s = rep_hist_format_buffer_stats(now + 86400);
 207   tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
 208              "cell-processed-cells 0,0,0,0,0,0,0,0,0,0\n"
 209              "cell-queued-cells 0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
 210                                "0.00,0.00\n"
 211              "cell-time-in-queue 0,0,0,0,0,0,0,0,0,0\n"
 212              "cell-circuits-per-decile 0\n",OP_EQ, s);
 213
 214  done:
 215   tor_free(s);
 216 }
 217
 218 /** Run unit tests the mtbf stats code. */
 219 static void
 220 test_rephist_mtbf(void *arg)
 221 {
 222   (void)arg;
 223
 224   time_t now = 1572500000; /* 2010-10-31 05:33:20 UTC */
 225   time_t far_future = MAX(now, time(NULL)) + 365*24*60*60;
 226   int r;
 227
 228   /* Make a temporary datadir for these tests */
 229   char *ddir_fname = tor_strdup(get_fname_rnd("datadir_mtbf"));
 230   tor_free(get_options_mutable()->DataDirectory);
 231   get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
 232   check_private_dir(ddir_fname, CPD_CREATE, NULL);
 233
 234   rep_history_clean(far_future);
 235
 236   /* No data */
 237
 238   r = rep_hist_load_mtbf_data(now);
 239   tt_int_op(r, OP_EQ, -1);
 240   rep_history_clean(far_future);
 241
 242   /* Blank data */
 243
 244   r = rep_hist_record_mtbf_data(now, 0);
 245   tt_int_op(r, OP_EQ, 0);
 246   r = rep_hist_load_mtbf_data(now);
 247   tt_int_op(r, OP_EQ, 0);
 248   rep_history_clean(far_future);
 249
 250   r = rep_hist_record_mtbf_data(now, 1);
 251   tt_int_op(r, OP_EQ, 0);
 252   r = rep_hist_load_mtbf_data(now);
 253   tt_int_op(r, OP_EQ, 0);
 254   rep_history_clean(far_future);
 255
 256  done:
 257   rep_history_clean(far_future);
 258   tor_free(ddir_fname);
 259 }
 260
 261 static void
 262 test_commit_max(void *arg)
 263 {
 264   (void) arg;
 265   bw_array_t *b = bw_array_new();
 266   time_t now = b->cur_obs_time;
 267
 268   commit_max(b);
 269   tt_int_op(b->next_period, OP_EQ, now + 2*86400);
 270
 271   b->total_in_period = 100;
 272   b->max_total = 10;
 273   commit_max(b);
 274   tor_assert(b->total_in_period == 0);
 275   tor_assert(b->max_total == 0);
 276   tt_int_op(b->totals[1], OP_EQ, 100);
 277   tt_int_op(b->maxima[1], OP_EQ, 10);
 278   tt_int_op(b->next_period, OP_EQ, now + 3*86400);
 279
 280   commit_max(b);
 281   tt_int_op(b->next_period, OP_EQ, now + 4*86400);
 282
 283   commit_max(b);
 284   tt_int_op(b->next_period, OP_EQ, now + 5*86400);
 285
 286   b->total_in_period = 100;
 287   b->max_total = 10;
 288   commit_max(b);
 289   tor_assert(!b->next_max_idx);
 290   tt_int_op(b->cur_obs_idx, OP_EQ, 0);
 291   tt_int_op(b->totals[4], OP_EQ, 100);
 292   tt_int_op(b->maxima[4], OP_EQ, 10);
 293   tt_int_op(b->next_period, OP_EQ, now + 6*86400);
 294  done:
 295   bw_array_free(b);
 296 }
 297
 298 #define test_obs(b, idx, time, tot, max) STMT_BEGIN \
 299     tt_int_op(b->cur_obs_idx, OP_EQ, idx); \
 300     tt_int_op(b->cur_obs_time, OP_EQ, time); \
 301     tt_int_op(b->total_obs, OP_EQ, tot); \
 302     tt_int_op(b->max_total, OP_EQ, max); \
 303   STMT_END;
 304
 305 static void
 306 test_advance_obs(void *arg)
 307 {
 308   (void) arg;
 309   int iter, tot = 0;
 310   bw_array_t *b = bw_array_new();
 311   time_t now = b->cur_obs_time;
 312
 313   for (iter = 0; iter < 10; ++iter) {
 314     b->obs[b->cur_obs_idx] += 10;
 315     tot += 10;
 316     advance_obs(b);
 317     if (iter == 9) {
 318       /* The current value under cur_obs_idx was zeroed in last iterN. */
 319       test_obs(b, 0, now+iter+1, tot - 10, tot);
 320       break;
 321     }
 322     test_obs(b, iter+1, now+iter+1, tot, tot);
 323   }
 324
 325   b->total_in_period = 100;
 326   b->cur_obs_time = now + NUM_SECS_BW_SUM_INTERVAL - 1;
 327   advance_obs(b);
 328   test_obs(b, 1, now+NUM_SECS_BW_SUM_INTERVAL, 80, 0);
 329   tt_int_op(b->maxima[0], OP_EQ, 100);
 330   tt_int_op(b->totals[0], OP_EQ, 100);
 331   tt_int_op(b->num_maxes_set, OP_EQ, 1);
 332  done:
 333   bw_array_free(b);
 334 }
 335
 336 #define test_add_obs_(b, now, checknow, bw, tot) STMT_BEGIN \
 337     tot += bw; \
 338     add_obs(b, now, bw); \
 339     tt_int_op(b->cur_obs_time, OP_EQ, checknow); \
 340     tt_int_op(b->obs[b->cur_obs_idx], OP_EQ, bw); \
 341     tt_int_op(b->total_in_period, OP_EQ, tot); \
 342   STMT_END;
 343
 344 static void
 345 test_add_obs(void *arg)
 346 {
 347   (void) arg;
 348   bw_array_t *b = bw_array_new();
 349   time_t now = b->cur_obs_time;
 350   uint64_t bw = 0, tot = 0;
 351   /* Requests for the past should not be entertained. */
 352   test_add_obs_(b, now-1, now, bw, tot);
 353   /* Test the expected functionalities for random values. */
 354   now += 53;
 355   bw = 97;
 356   test_add_obs_(b, now, now, bw, tot);
 357
 358   now += 60*60;
 359   bw = 90;
 360   test_add_obs_(b, now, now, bw, tot);
 361
 362   now += 24*60*60;
 363   bw = 100;
 364   tot = 0;
 365   test_add_obs_(b, now, now, bw, tot);
 366  done:
 367   bw_array_free(b);
 368 }
 369
 370 static or_options_t mock_options;
 371
 372 static const or_options_t *
 373 mock_get_options(void)
 374 {
 375   return &mock_options;
 376 }
 377
 378 #define MAX_HIST_VALUE_LEN 21*NUM_TOTALS
 379
 380 #define set_test_case(b, max, idx, a1, a2, a3, a4, a5) STMT_BEGIN \
 381     b->num_maxes_set = max; \
 382     b->next_max_idx = idx; \
 383     b->totals[0] = a1; \
 384     b->totals[1] = a2; \
 385     b->totals[2] = a3; \
 386     b->totals[3] = a4; \
 387     b->totals[4] = a5; \
 388   STMT_END;
 389
 390 #define test_fill_bw(b, buf, rv, str, checkrv) STMT_BEGIN \
 391     buf = tor_malloc_zero(MAX_HIST_VALUE_LEN); \
 392     rv = bwhist_fill_bandwidth_history(buf, MAX_HIST_VALUE_LEN, b); \
 393     tt_str_op(buf, OP_EQ, str); \
 394     tt_int_op(rv, OP_EQ, checkrv); \
 395     tor_free(buf); \
 396   STMT_END;
 397
 398 static void
 399 test_fill_bandwidth_history(void *arg)
 400 {
 401   (void) arg;
 402   bw_array_t *b = bw_array_new();
 403   char *buf;
 404   size_t rv;
 405   /* Remember bandwidth is rounded down to the nearest 1K. */
 406   /* Day 1. */
 407   set_test_case(b, 0, 0, 0, 0, 0, 0, 0);
 408   buf = tor_malloc_zero(MAX_HIST_VALUE_LEN);
 409   rv = bwhist_fill_bandwidth_history(buf, MAX_HIST_VALUE_LEN, b);
 410   tt_int_op(rv, OP_EQ, 0);
 411   tor_free(buf);
 412   /* Day 2. */
 413   set_test_case(b, 1, 1, 1000, 0, 0, 0, 0);
 414   test_fill_bw(b, buf, rv, "0", 1);
 415   /* Day 3. */
 416   set_test_case(b, 2, 2, 1000, 1500, 0, 0, 0);
 417   test_fill_bw(b, buf, rv, "0,1024", 6);
 418   /* Day 4. */
 419   set_test_case(b, 3, 3, 1000, 1500, 3500, 0, 0);
 420   test_fill_bw(b, buf, rv, "0,1024,3072", 11);
 421   /* Day 5. */
 422   set_test_case(b, 4, 4, 1000, 1500, 3500, 8000, 0);
 423   test_fill_bw(b, buf, rv, "0,1024,3072,7168", 16);
 424   /* Day 6. */
 425   set_test_case(b, 5, 0, 1000, 1500, 3500, 8000, 6000);
 426   test_fill_bw(b, buf, rv, "0,1024,3072,7168,5120", 21);
 427   /* Day 7. */
 428   /* Remember oldest entry first. */
 429   set_test_case(b, 5, 1, 10000, 1500, 3500, 8000, 6000);
 430   test_fill_bw(b, buf, rv, "1024,3072,7168,5120,9216", 24);
 431   /* Mocking get_options to manipulate RelayBandwidthRate. */
 432   MOCK(get_options, mock_get_options);
 433   /* Limits bandwidth to 1 KBps. */
 434   /* Cutoff is set to 88473600. */
 435   mock_options.RelayBandwidthRate = 1024;
 436   set_test_case(b, 5, 2, 88573600, 88473600, 10000, 8000, 6000);
 437   test_fill_bw(b, buf, rv, "9216,7168,5120,88473600,88473600", 32);
 438  done:
 439   UNMOCK(get_options);
 440   bw_array_free(b);
 441 }
 442
 443 #define set_test_bw_lines(r, w, dr, dw, when) STMT_BEGIN \
 444     bwhist_note_bytes_read(r, when, false); \
 445     bwhist_note_bytes_written(w, when, false); \
 446     bwhist_note_dir_bytes_read(dr, when); \
 447     bwhist_note_dir_bytes_written(dw, when); \
 448   STMT_END;
 449
 450 #define test_get_bw_lines(str, checkstr) STMT_BEGIN \
 451     str = bwhist_get_bandwidth_lines(); \
 452     tt_str_op(str, OP_EQ, checkstr); \
 453     tor_free(str); \
 454   STMT_END;
 455
 456 static void
 457 test_get_bandwidth_lines(void *arg)
 458 {
 459   (void) arg;
 460   char *str = NULL, *checkstr = NULL;
 461   char t[ISO_TIME_LEN+1];
 462   int len = (67+MAX_HIST_VALUE_LEN)*4;
 463   checkstr = tor_malloc_zero(len);
 464   time_t now = time(NULL);
 465   bwhist_init();
 466
 467   /* Day 1. */
 468   now += 86400;
 469   set_test_bw_lines(5000, 5500, 3000, 3500, now - 6*60*60);
 470   /* Day 2. */
 471   now += 86400;
 472   set_test_bw_lines(50000, 55000, 30000, 35000, now - 6*60*60);
 473   /* Day 3. */
 474   now += 86400;
 475   set_test_bw_lines(25000, 27500, 15000, 17500, now - 6*60*60);
 476   /* Day 4. */
 477   now += 86400;
 478   set_test_bw_lines(90000, 76000, 60000, 45000, now - 6*60*60);
 479   /* Day 5. */
 480   now += 86400;
 481   set_test_bw_lines(500, 55000, 30000, 35000, now - 6*60*60);
 482   set_test_bw_lines(0, 0, 0, 0, now);
 483   format_iso_time(t, now);
 484   tor_snprintf(checkstr, len, "write-history %s (86400 s) "
 485                     "5120,54272,26624,75776,54272\n"
 486                     "read-history %s (86400 s) "
 487                     "4096,49152,24576,89088,0\n"
 488                     "dirreq-write-history %s (86400 s) "
 489                     "3072,34816,17408,44032,34816\n"
 490                     "dirreq-read-history %s (86400 s) "
 491                     "2048,29696,14336,59392,29696\n",
 492                     t, t, t, t);
 493   test_get_bw_lines(str, checkstr);
 494
 495  done:
 496   tor_free(str);
 497   tor_free(checkstr);
 498   bwhist_free_all();
 499 }
 500
 501 static bool
 502 mock_should_collect_v3_stats(void)
 503 {
 504   return true;
 505 }
 506
 507 /* Test v3 metrics */
 508 static void
 509 test_rephist_v3_onions(void *arg)
 510 {
 511   int ret;
 512
 513   char *stats_string = NULL;
 514   char *desc1_str = NULL;
 515   ed25519_keypair_t signing_kp1;
 516   hs_descriptor_t *desc1 = NULL;
 517
 518   const hs_v3_stats_t *hs_v3_stats = NULL;
 519
 520   (void) arg;
 521
 522   MOCK(should_collect_v3_stats, mock_should_collect_v3_stats);
 523
 524   get_options_mutable()->HiddenServiceStatistics = 1;
 525
 526   /* Initialize the subsystems */
 527   hs_cache_init();
 528   rep_hist_hs_stats_init(0);
 529
 530   /* Change time to 03-01-2002 23:36 UTC */
 531   update_approx_time(1010101010);
 532
 533   /* HS stats should be zero here */
 534   hs_v3_stats = rep_hist_get_hs_v3_stats();
 535   tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period),
 536             OP_EQ, 0);
 537
 538   /* Generate a valid descriptor */
 539   ret = ed25519_keypair_generate(&signing_kp1, 0);
 540   tt_int_op(ret, OP_EQ, 0);
 541   desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 42);
 542   tt_assert(desc1);
 543   ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str);
 544   tt_int_op(ret, OP_EQ, 0);
 545
 546   /* Store descriptor and check that stats got updated */
 547   ret = hs_cache_store_as_dir(desc1_str);
 548   tt_int_op(ret, OP_EQ, 0);
 549   hs_v3_stats = rep_hist_get_hs_v3_stats();
 550   tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period),
 551             OP_EQ, 1);
 552
 553   /* cleanup */
 554   hs_descriptor_free(desc1);
 555   tor_free(desc1_str);
 556
 557   /* Generate another valid descriptor */
 558   ret = ed25519_keypair_generate(&signing_kp1, 0);
 559   tt_int_op(ret, OP_EQ, 0);
 560   desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 42);
 561   tt_assert(desc1);
 562   ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str);
 563   tt_int_op(ret, OP_EQ, 0);
 564
 565   /* Store descriptor and check that stats are updated */
 566   ret = hs_cache_store_as_dir(desc1_str);
 567   tt_int_op(ret, OP_EQ, 0);
 568   hs_v3_stats = rep_hist_get_hs_v3_stats();
 569   tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period),
 570             OP_EQ, 2);
 571
 572   /* Check that storing the same descriptor twice does not work */
 573   ret = hs_cache_store_as_dir(desc1_str);
 574   tt_int_op(ret, OP_EQ, -1);
 575
 576   /* cleanup */
 577   hs_descriptor_free(desc1);
 578   tor_free(desc1_str);
 579
 580   /* Create a descriptor with the same identity key but diff rev counter and
 581      same blinded key */
 582   desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 43);
 583   tt_assert(desc1);
 584   ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str);
 585   tt_int_op(ret, OP_EQ, 0);
 586
 587   /* Store descriptor and check that stats are updated */
 588   ret = hs_cache_store_as_dir(desc1_str);
 589   tt_int_op(ret, OP_EQ, 0);
 590   tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period),
 591             OP_EQ, 2);
 592
 593   /* cleanup */
 594   hs_descriptor_free(desc1);
 595   tor_free(desc1_str);
 596
 597   /* Now let's skip to four days forward so that the blinded key rolls
 598      forward */
 599   update_approx_time(approx_time() + 345600);
 600
 601   /* Now create a descriptor with the same identity key but diff rev counter
 602      and different blinded key */
 603   desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 44);
 604   tt_assert(desc1);
 605   ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str);
 606   tt_int_op(ret, OP_EQ, 0);
 607
 608   /* Store descriptor and check that stats are updated */
 609   ret = hs_cache_store_as_dir(desc1_str);
 610   tt_int_op(ret, OP_EQ, 0);
 611   tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period),
 612             OP_EQ, 3);
 613
 614   /* cleanup */
 615   hs_descriptor_free(desc1);
 616   tor_free(desc1_str);
 617
 618   /* Because of differential privacy we can't actually check the stat value,
 619      but let's just check that it's formatted correctly. */
 620   stats_string = rep_hist_format_hs_stats(approx_time(), true);
 621   tt_assert(strstr(stats_string, "hidserv-dir-v3-onions-seen"));
 622
 623  done:
 624   UNMOCK(should_collect_v3_stats);
 625   tor_free(stats_string);
 626 }
 627
 628 static void
 629 test_load_stats_file(void *arg)
 630 {
 631   int ret;
 632   char *content = NULL, *read_file_content = NULL, *fname = NULL;
 633
 634   (void) arg;
 635
 636   /* Load conn-stats. */
 637   fname = get_datadir_fname("conn-stats");
 638   tt_assert(fname);
 639   read_file_content = tor_strdup(
 640     "conn-bi-direct 2020-12-13 15:48:53 (86400 s) 12,34,56,78\n"
 641     "ipv6-conn-bi-direct 2020-12-14 15:48:53 (86400 s) 21,43,65,87\n");
 642   write_str_to_file(fname, read_file_content, 0);
 643   ret = load_stats_file("conn-stats", "conn-bi-direct", 1607874000, &content);
 644   tt_int_op(ret, OP_EQ, 1);
 645   tt_str_op(read_file_content, OP_EQ, content);
 646
 647   /* Load hidserv-stats. */
 648   tor_free(fname);
 649   fname = get_datadir_fname("hidserv-stats");
 650   tt_assert(fname);
 651   tor_free(read_file_content);
 652   read_file_content = tor_strdup(
 653     "hidserv-stats-end 2020-12-13 15:48:53 (86400 s)\n"
 654     "hidserv-rend-relayed-cells 48754891 delta_f=2048 epsilon=0.30 "
 655       "bin_size=1024\n"
 656     "hidserv-dir-onions-seen 53 delta_f=8 epsilon=0.30 bin_size=8\n");
 657   write_str_to_file(fname, read_file_content, 0);
 658   tor_free(content);
 659   ret = load_stats_file("hidserv-stats", "hidserv-stats-end", 1607874000,
 660                         &content);
 661   tt_int_op(ret, OP_EQ, 1);
 662   tt_str_op(read_file_content, OP_EQ, content);
 663
 664   /* Load dirreq-stats. */
 665   tor_free(fname);
 666   fname = get_datadir_fname("dirreq-stats");
 667   tt_assert(fname);
 668   tor_free(read_file_content);
 669   read_file_content = tor_strdup(
 670     "dirreq-stats-end 2020-12-13 15:48:53 (86400 s)\n"
 671     "dirreq-v3-ips ru=1728,us=1144,de=696,ir=432,gb=328,fr=304,in=296,ua=232\n"
 672     "dirreq-v3-reqs ru=3616,us=3576,de=1896,fr=800,gb=632,ir=616\n"
 673     "dirreq-v3-resp ok=18472,not-enough-sigs=0,unavailable=0,not-found=0,"
 674       "not-modified=3136,busy=0\n"
 675     "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
 676     "dirreq-v3-tunneled-dl complete=18124,timeout=348,running=4,min=257,"
 677       "d1=133653,d2=221050,q1=261242,d3=300622,d4=399758,md=539051,d6=721322,"
 678       "d7=959866,q3=1103363,d8=1302035,d9=2046125,max=113404000\n");
 679   write_str_to_file(fname, read_file_content, 0);
 680   tor_free(content);
 681   ret = load_stats_file("dirreq-stats", "dirreq-stats-end", 1607874000,
 682                         &content);
 683   tt_int_op(ret, OP_EQ, 1);
 684   tt_str_op(read_file_content, OP_EQ, content);
 685
 686   /* Attempt to load future-stats file not starting with timestamp tag. */
 687   tor_free(fname);
 688   fname = get_datadir_fname("future-stats");
 689   tt_assert(fname);
 690   tor_free(read_file_content);
 691   read_file_content = tor_strdup(
 692     "future-stuff-at-file-start\n"
 693     "future-stats 2020-12-13 15:48:53 (86400 s)\n");
 694   write_str_to_file(fname, read_file_content, 0);
 695   tor_free(content);
 696   ret = load_stats_file("future-stats", "future-stats", 1607874000, &content);
 697   tt_int_op(ret, OP_EQ, 1);
 698   tt_str_op(read_file_content, OP_EQ, content);
 699
 700  done:
 701   tor_free(fname);
 702   tor_free(read_file_content);
 703   tor_free(content);
 704 }
 705
 706 /** Test the overload stats logic. */
 707 static void
 708 test_overload_stats(void *arg)
 709 {
 710   time_t current_time = 1010101010;
 711   char *stats_str = NULL;
 712   (void) arg;
 713
 714   /* Change time to 03-01-2002 23:36 UTC */
 715   /* This should make the extrainfo timestamp be "2002-01-03 23:00:00" */
 716   update_approx_time(current_time);
 717
 718   /* With an empty rephist we shouldn't get anything back */
 719   stats_str = rep_hist_get_overload_stats_lines();
 720   tt_assert(!stats_str);
 721
 722   /* Note a DNS overload */
 723   rep_hist_note_overload(OVERLOAD_GENERAL);
 724
 725   /* Move the time forward one hour */
 726   current_time += 3600;
 727   update_approx_time(current_time);
 728
 729   /* Now check the string */
 730   stats_str = rep_hist_get_overload_stats_lines();
 731   tt_str_op("overload-general 1 2002-01-03 23:00:00\n", OP_EQ, stats_str);
 732   tor_free(stats_str);
 733
 734   /* Move the time forward 72 hours: see that the line has disappeared. */
 735   current_time += 3600*72;
 736   update_approx_time(current_time);
 737
 738   stats_str = rep_hist_get_overload_stats_lines();
 739   tt_assert(!stats_str);
 740
 741   /* Now the time should be 2002-01-07 00:00:00 */
 742
 743   /* Note a DNS overload */
 744   rep_hist_note_overload(OVERLOAD_GENERAL);
 745
 746   stats_str = rep_hist_get_overload_stats_lines();
 747   tt_str_op("overload-general 1 2002-01-07 00:00:00\n", OP_EQ, stats_str);
 748   tor_free(stats_str);
 749
 750   /* Also note an fd exhaustion event */
 751   rep_hist_note_overload(OVERLOAD_FD_EXHAUSTED);
 752
 753   stats_str = rep_hist_get_overload_stats_lines();
 754   tt_str_op("overload-general 1 2002-01-07 00:00:00\n"
 755             "overload-fd-exhausted 1 2002-01-07 00:00:00\n", OP_EQ, stats_str);
 756   tor_free(stats_str);
 757
 758   /* Move the time forward. Register DNS overload. See that the time changed */
 759   current_time += 3600*2;
 760   update_approx_time(current_time);
 761
 762   rep_hist_note_overload(OVERLOAD_GENERAL);
 763
 764   stats_str = rep_hist_get_overload_stats_lines();
 765   tt_str_op("overload-general 1 2002-01-07 02:00:00\n"
 766             "overload-fd-exhausted 1 2002-01-07 00:00:00\n", OP_EQ, stats_str);
 767   tor_free(stats_str);
 768
 769   /* Move the time forward. Register a bandwidth ratelimit event. See that the
 770      string is added */
 771   current_time += 3600*2;
 772   update_approx_time(current_time);
 773
 774   /* Register the rate limit event */
 775   rep_hist_note_overload(OVERLOAD_READ);
 776   /* Also set some rate limiting values that should be reflected on the log */
 777   get_options_mutable()->BandwidthRate = 1000;
 778   get_options_mutable()->BandwidthBurst = 2000;
 779
 780   stats_str = rep_hist_get_overload_stats_lines();
 781   tt_str_op("overload-general 1 2002-01-07 02:00:00\n"
 782             "overload-ratelimits 1 2002-01-07 04:00:00 1000 2000 1 0\n"
 783             "overload-fd-exhausted 1 2002-01-07 00:00:00\n", OP_EQ, stats_str);
 784   tor_free(stats_str);
 785
 786   /* Move the time forward 24 hours: no rate limit line anymore. */
 787   current_time += 3600*24;
 788   update_approx_time(current_time);
 789
 790   stats_str = rep_hist_get_overload_stats_lines();
 791   tt_str_op("overload-general 1 2002-01-07 02:00:00\n"
 792             "overload-fd-exhausted 1 2002-01-07 00:00:00\n", OP_EQ, stats_str);
 793   tor_free(stats_str);
 794
 795   /* Move the time forward 44 hours: no fd exhausted line anymore. */
 796   current_time += 3600*44;
 797   update_approx_time(current_time);
 798
 799   stats_str = rep_hist_get_overload_stats_lines();
 800   tt_str_op("overload-general 1 2002-01-07 02:00:00\n", OP_EQ, stats_str);
 801   tor_free(stats_str);
 802
 803   /* Move the time forward 2 hours: there is nothing left. */
 804   current_time += 3600*2;
 805   update_approx_time(current_time);
 806
 807   stats_str = rep_hist_get_overload_stats_lines();
 808   tt_assert(!stats_str);
 809
 810   /* Now test the rate-limit rate-limiter ;) */
 811   for (int i = 0; i < 10; i++) {
 812     rep_hist_note_overload(OVERLOAD_READ);
 813   }
 814   /* We already have an event registered from the previous tests. We just
 815    * registered ten more overload events, but only one should have been counted
 816    * because of the rate limiter */
 817   stats_str = rep_hist_get_overload_stats_lines();
 818   tt_str_op("overload-ratelimits 1 2002-01-10 02:00:00 1000 2000 2 0\n",
 819             OP_EQ, stats_str);
 820   tor_free(stats_str);
 821
 822   /* Increment time by 59 secs and try again. No additional events should
 823      register */
 824   current_time += 59;
 825   update_approx_time(current_time);
 826
 827   for (int i = 0; i < 10; i++) {
 828     rep_hist_note_overload(OVERLOAD_READ);
 829   }
 830   stats_str = rep_hist_get_overload_stats_lines();
 831   tt_str_op("overload-ratelimits 1 2002-01-10 02:00:00 1000 2000 2 0\n",
 832             OP_EQ, stats_str);
 833   tor_free(stats_str);
 834
 835   /* Now increment time by 2 secs -- taking it after the minute rate limiting
 836      and see that events will register again */
 837   current_time += 2;
 838   update_approx_time(current_time);
 839
 840   for (int i = 0; i < 10; i++) {
 841     rep_hist_note_overload(OVERLOAD_READ);
 842     rep_hist_note_overload(OVERLOAD_WRITE);
 843   }
 844   stats_str = rep_hist_get_overload_stats_lines();
 845   tt_str_op("overload-ratelimits 1 2002-01-10 02:00:00 1000 2000 3 1\n",
 846             OP_EQ, stats_str);
 847   tor_free(stats_str);
 848
 849  done:
 850   tor_free(stats_str);
 851 }
 852
 853 #define ENT(name)                                                       \
 854   { #name, test_ ## name , 0, NULL, NULL }
 855 #define FORK(name)                                                      \
 856   { #name, test_ ## name , TT_FORK, NULL, NULL }
 857
 858 struct testcase_t stats_tests[] = {
 859   FORK(stats),
 860   ENT(rephist_mtbf),
 861   FORK(commit_max),
 862   FORK(advance_obs),
 863   FORK(add_obs),
 864   FORK(fill_bandwidth_history),
 865   FORK(get_bandwidth_lines),
 866   FORK(rephist_v3_onions),
 867   FORK(load_stats_file),
 868   FORK(overload_stats),
 869
 870   END_OF_TESTCASES
 871 };