src/test/test_oos.c

   1 /* Copyright (c) 2016-2020, The Tor Project, Inc. */
   2 /* See LICENSE for licensing information */
   3
   4 /* Unit tests for OOS handler */
   5
   6 #define CONNECTION_PRIVATE
   7
   8 #include "core/or/or.h"
   9 #include "app/config/config.h"
  10 #include "core/mainloop/connection.h"
  11 #include "core/or/connection_or.h"
  12 #include "feature/dircommon/directory.h"
  13 #include "core/mainloop/mainloop.h"
  14 #include "test/test.h"
  15
  16 #include "feature/dircommon/dir_connection_st.h"
  17 #include "core/or/or_connection_st.h"
  18
  19 static or_options_t mock_options;
  20
  21 static void
  22 reset_options_mock(void)
  23 {
  24   memset(&mock_options, 0, sizeof(or_options_t));
  25 }
  26
  27 static const or_options_t *
  28 mock_get_options(void)
  29 {
  30   return &mock_options;
  31 }
  32
  33 static int moribund_calls = 0;
  34 static int moribund_conns = 0;
  35
  36 static int
  37 mock_connection_count_moribund(void)
  38 {
  39   ++moribund_calls;
  40
  41   return moribund_conns;
  42 }
  43
  44 /*
  45  * For unit test purposes it's sufficient to tell that
  46  * kill_conn_list_for_oos() was called with an approximately
  47  * sane argument; it's just the thing we returned from the
  48  * mock for pick_oos_victims().
  49  */
  50
  51 static int kill_conn_list_calls = 0;
  52 static int kill_conn_list_killed = 0;
  53
  54 static void
  55 kill_conn_list_mock(smartlist_t *conns)
  56 {
  57   ++kill_conn_list_calls;
  58
  59   tt_ptr_op(conns, OP_NE, NULL);
  60
  61   kill_conn_list_killed += smartlist_len(conns);
  62
  63  done:
  64   return;
  65 }
  66
  67 static int pick_oos_mock_calls = 0;
  68 static int pick_oos_mock_fail = 0;
  69 static int pick_oos_mock_last_n = 0;
  70
  71 static smartlist_t *
  72 pick_oos_victims_mock(int n)
  73 {
  74   smartlist_t *l = NULL;
  75   int i;
  76
  77   ++pick_oos_mock_calls;
  78
  79   tt_int_op(n, OP_GT, 0);
  80
  81   if (!pick_oos_mock_fail) {
  82     /*
  83      * connection_check_oos() just passes the list onto
  84      * kill_conn_list_for_oos(); we don't need to simulate
  85      * its content for this mock, just its existence, but
  86      * we do need to check the parameter.
  87      */
  88     l = smartlist_new();
  89     for (i = 0; i < n; ++i) smartlist_add(l, NULL);
  90   } else {
  91     l = NULL;
  92   }
  93
  94   pick_oos_mock_last_n = n;
  95
  96  done:
  97   return l;
  98 }
  99
 100 /** Unit test for the logic in connection_check_oos(), which is concerned
 101  * with comparing thresholds and connection counts to decide if an OOS has
 102  * occurred and if so, how many connections to try to kill, and then using
 103  * pick_oos_victims() and kill_conn_list_for_oos() to carry out its grim
 104  * duty.
 105  */
 106 static void
 107 test_oos_connection_check_oos(void *arg)
 108 {
 109   (void)arg;
 110
 111   /* Set up mocks */
 112   reset_options_mock();
 113   /* OOS handling is only sensitive to these fields */
 114   mock_options.ConnLimit = 32;
 115   mock_options.ConnLimit_ = 64;
 116   mock_options.ConnLimit_high_thresh = 60;
 117   mock_options.ConnLimit_low_thresh = 50;
 118   MOCK(get_options, mock_get_options);
 119   moribund_calls = 0;
 120   moribund_conns = 0;
 121   MOCK(connection_count_moribund, mock_connection_count_moribund);
 122   kill_conn_list_calls = 0;
 123   kill_conn_list_killed = 0;
 124   MOCK(kill_conn_list_for_oos, kill_conn_list_mock);
 125   pick_oos_mock_calls = 0;
 126   pick_oos_mock_fail = 0;
 127   MOCK(pick_oos_victims, pick_oos_victims_mock);
 128
 129   /* No OOS case */
 130   connection_check_oos(50, 0);
 131   tt_int_op(moribund_calls, OP_EQ, 0);
 132   tt_int_op(pick_oos_mock_calls, OP_EQ, 0);
 133   tt_int_op(kill_conn_list_calls, OP_EQ, 0);
 134
 135   /* OOS from socket count, nothing moribund */
 136   connection_check_oos(62, 0);
 137   tt_int_op(moribund_calls, OP_EQ, 1);
 138   tt_int_op(pick_oos_mock_calls, OP_EQ, 1);
 139   /* 12 == 62 - ConnLimit_low_thresh */
 140   tt_int_op(pick_oos_mock_last_n, OP_EQ, 12);
 141   tt_int_op(kill_conn_list_calls, OP_EQ, 1);
 142   tt_int_op(kill_conn_list_killed, OP_EQ, 12);
 143
 144   /* OOS from socket count, some are moribund */
 145   kill_conn_list_killed = 0;
 146   moribund_conns = 5;
 147   connection_check_oos(62, 0);
 148   tt_int_op(moribund_calls, OP_EQ, 2);
 149   tt_int_op(pick_oos_mock_calls, OP_EQ, 2);
 150   /* 7 == 62 - ConnLimit_low_thresh - moribund_conns */
 151   tt_int_op(pick_oos_mock_last_n, OP_EQ, 7);
 152   tt_int_op(kill_conn_list_calls, OP_EQ, 2);
 153   tt_int_op(kill_conn_list_killed, OP_EQ, 7);
 154
 155   /* OOS from socket count, but pick fails */
 156   kill_conn_list_killed = 0;
 157   moribund_conns = 0;
 158   pick_oos_mock_fail = 1;
 159   connection_check_oos(62, 0);
 160   tt_int_op(moribund_calls, OP_EQ, 3);
 161   tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
 162   tt_int_op(kill_conn_list_calls, OP_EQ, 2);
 163   tt_int_op(kill_conn_list_killed, OP_EQ, 0);
 164   pick_oos_mock_fail = 0;
 165
 166   /*
 167    * OOS from socket count with so many moribund conns
 168    * we have none to kill.
 169    */
 170   kill_conn_list_killed = 0;
 171   moribund_conns = 15;
 172   connection_check_oos(62, 0);
 173   tt_int_op(moribund_calls, OP_EQ, 4);
 174   tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
 175   tt_int_op(kill_conn_list_calls, OP_EQ, 2);
 176
 177   /*
 178    * OOS from socket exhaustion; OOS handler will try to
 179    * kill 1/10 (5) of the connections.
 180    */
 181   kill_conn_list_killed = 0;
 182   moribund_conns = 0;
 183   connection_check_oos(50, 1);
 184   tt_int_op(moribund_calls, OP_EQ, 5);
 185   tt_int_op(pick_oos_mock_calls, OP_EQ, 4);
 186   tt_int_op(kill_conn_list_calls, OP_EQ, 3);
 187   tt_int_op(kill_conn_list_killed, OP_EQ, 5);
 188
 189   /* OOS from socket exhaustion with moribund conns */
 190   kill_conn_list_killed = 0;
 191   moribund_conns = 2;
 192   connection_check_oos(50, 1);
 193   tt_int_op(moribund_calls, OP_EQ, 6);
 194   tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
 195   tt_int_op(kill_conn_list_calls, OP_EQ, 4);
 196   tt_int_op(kill_conn_list_killed, OP_EQ, 3);
 197
 198   /* OOS from socket exhaustion with many moribund conns */
 199   kill_conn_list_killed = 0;
 200   moribund_conns = 7;
 201   connection_check_oos(50, 1);
 202   tt_int_op(moribund_calls, OP_EQ, 7);
 203   tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
 204   tt_int_op(kill_conn_list_calls, OP_EQ, 4);
 205
 206   /* OOS with both socket exhaustion and above-threshold */
 207   kill_conn_list_killed = 0;
 208   moribund_conns = 0;
 209   connection_check_oos(62, 1);
 210   tt_int_op(moribund_calls, OP_EQ, 8);
 211   tt_int_op(pick_oos_mock_calls, OP_EQ, 6);
 212   tt_int_op(kill_conn_list_calls, OP_EQ, 5);
 213   tt_int_op(kill_conn_list_killed, OP_EQ, 12);
 214
 215   /*
 216    * OOS with both socket exhaustion and above-threshold with some
 217    * moribund conns
 218    */
 219   kill_conn_list_killed = 0;
 220   moribund_conns = 5;
 221   connection_check_oos(62, 1);
 222   tt_int_op(moribund_calls, OP_EQ, 9);
 223   tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
 224   tt_int_op(kill_conn_list_calls, OP_EQ, 6);
 225   tt_int_op(kill_conn_list_killed, OP_EQ, 7);
 226
 227   /*
 228    * OOS with both socket exhaustion and above-threshold with many
 229    * moribund conns
 230    */
 231   kill_conn_list_killed = 0;
 232   moribund_conns = 15;
 233   connection_check_oos(62, 1);
 234   tt_int_op(moribund_calls, OP_EQ, 10);
 235   tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
 236   tt_int_op(kill_conn_list_calls, OP_EQ, 6);
 237
 238  done:
 239
 240   UNMOCK(pick_oos_victims);
 241   UNMOCK(kill_conn_list_for_oos);
 242   UNMOCK(connection_count_moribund);
 243   UNMOCK(get_options);
 244
 245   return;
 246 }
 247
 248 static int cfe_calls = 0;
 249
 250 static void
 251 close_for_error_mock(or_connection_t *orconn, int flush)
 252 {
 253   (void)flush;
 254
 255   tt_ptr_op(orconn, OP_NE, NULL);
 256   ++cfe_calls;
 257
 258  done:
 259   return;
 260 }
 261
 262 static int mark_calls = 0;
 263
 264 static void
 265 mark_for_close_oos_mock(connection_t *conn,
 266                         int line, const char *file)
 267 {
 268   (void)line;
 269   (void)file;
 270
 271   tt_ptr_op(conn, OP_NE, NULL);
 272   ++mark_calls;
 273
 274  done:
 275   return;
 276 }
 277
 278 static void
 279 test_oos_kill_conn_list(void *arg)
 280 {
 281   connection_t *c1, *c2;
 282   or_connection_t *or_c1 = NULL;
 283   dir_connection_t *dir_c2 = NULL;
 284   smartlist_t *l = NULL;
 285   (void)arg;
 286
 287   /* Set up mocks */
 288   mark_calls = 0;
 289   MOCK(connection_mark_for_close_internal_, mark_for_close_oos_mock);
 290   cfe_calls = 0;
 291   MOCK(connection_or_close_for_error, close_for_error_mock);
 292
 293   /* Make fake conns */
 294   or_c1 = tor_malloc_zero(sizeof(*or_c1));
 295   or_c1->base_.magic = OR_CONNECTION_MAGIC;
 296   or_c1->base_.type = CONN_TYPE_OR;
 297   c1 = TO_CONN(or_c1);
 298   dir_c2 = tor_malloc_zero(sizeof(*dir_c2));
 299   dir_c2->base_.magic = DIR_CONNECTION_MAGIC;
 300   dir_c2->base_.type = CONN_TYPE_DIR;
 301   dir_c2->base_.state = DIR_CONN_STATE_MIN_;
 302   dir_c2->base_.purpose = DIR_PURPOSE_MIN_;
 303   c2 = TO_CONN(dir_c2);
 304
 305   tt_ptr_op(c1, OP_NE, NULL);
 306   tt_ptr_op(c2, OP_NE, NULL);
 307
 308   /* Make list */
 309   l = smartlist_new();
 310   smartlist_add(l, c1);
 311   smartlist_add(l, c2);
 312
 313   /* Run kill_conn_list_for_oos() */
 314   kill_conn_list_for_oos(l);
 315
 316   /* Check call counters */
 317   tt_int_op(mark_calls, OP_EQ, 1);
 318   tt_int_op(cfe_calls, OP_EQ, 1);
 319
 320  done:
 321
 322   UNMOCK(connection_or_close_for_error);
 323   UNMOCK(connection_mark_for_close_internal_);
 324
 325   if (l) smartlist_free(l);
 326   tor_free(or_c1);
 327   tor_free(dir_c2);
 328
 329   return;
 330 }
 331
 332 static smartlist_t *conns_for_mock = NULL;
 333
 334 static smartlist_t *
 335 get_conns_mock(void)
 336 {
 337   return conns_for_mock;
 338 }
 339
 340 /*
 341  * For this mock, we pretend all conns have either zero or one circuits,
 342  * depending on if this appears on the list of things to say have a circuit.
 343  */
 344
 345 static smartlist_t *conns_with_circs = NULL;
 346
 347 static int
 348 get_num_circuits_mock(or_connection_t *conn)
 349 {
 350   int circs = 0;
 351
 352   tt_ptr_op(conn, OP_NE, NULL);
 353
 354   if (conns_with_circs &&
 355       smartlist_contains(conns_with_circs, TO_CONN(conn))) {
 356     circs = 1;
 357   }
 358
 359  done:
 360   return circs;
 361 }
 362
 363 static void
 364 test_oos_pick_oos_victims(void *arg)
 365 {
 366   (void)arg;
 367   or_connection_t *ortmp;
 368   dir_connection_t *dirtmp;
 369   smartlist_t *picked;
 370
 371   /* Set up mocks */
 372   conns_for_mock = smartlist_new();
 373   MOCK(get_connection_array, get_conns_mock);
 374   conns_with_circs = smartlist_new();
 375   MOCK(connection_or_get_num_circuits, get_num_circuits_mock);
 376
 377   /* Make some fake connections */
 378   ortmp = tor_malloc_zero(sizeof(*ortmp));
 379   ortmp->base_.magic = OR_CONNECTION_MAGIC;
 380   ortmp->base_.type = CONN_TYPE_OR;
 381   smartlist_add(conns_for_mock, TO_CONN(ortmp));
 382   /* We'll pretend this one has a circuit too */
 383   smartlist_add(conns_with_circs, TO_CONN(ortmp));
 384   /* Next one */
 385   ortmp = tor_malloc_zero(sizeof(*ortmp));
 386   ortmp->base_.magic = OR_CONNECTION_MAGIC;
 387   ortmp->base_.type = CONN_TYPE_OR;
 388   smartlist_add(conns_for_mock, TO_CONN(ortmp));
 389   /* Next one is moribund */
 390   ortmp = tor_malloc_zero(sizeof(*ortmp));
 391   ortmp->base_.magic = OR_CONNECTION_MAGIC;
 392   ortmp->base_.type = CONN_TYPE_OR;
 393   ortmp->base_.marked_for_close = 1;
 394   smartlist_add(conns_for_mock, TO_CONN(ortmp));
 395   /* Last one isn't an orconn */
 396   dirtmp = tor_malloc_zero(sizeof(*dirtmp));
 397   dirtmp->base_.magic = DIR_CONNECTION_MAGIC;
 398   dirtmp->base_.type = CONN_TYPE_DIR;
 399   smartlist_add(conns_for_mock, TO_CONN(dirtmp));
 400
 401   /* Try picking one */
 402   picked = pick_oos_victims(1);
 403   /* It should be the one with circuits */
 404   tt_ptr_op(picked, OP_NE, NULL);
 405   tt_int_op(smartlist_len(picked), OP_EQ, 1);
 406   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
 407   smartlist_free(picked);
 408
 409   /* Try picking none */
 410   picked = pick_oos_victims(0);
 411   /* We should get an empty list */
 412   tt_ptr_op(picked, OP_NE, NULL);
 413   tt_int_op(smartlist_len(picked), OP_EQ, 0);
 414   smartlist_free(picked);
 415
 416   /* Try picking two */
 417   picked = pick_oos_victims(2);
 418   /* We should get both active orconns */
 419   tt_ptr_op(picked, OP_NE, NULL);
 420   tt_int_op(smartlist_len(picked), OP_EQ, 2);
 421   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
 422   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
 423   smartlist_free(picked);
 424
 425   /* Try picking three - only two are eligible */
 426   picked = pick_oos_victims(3);
 427   tt_int_op(smartlist_len(picked), OP_EQ, 2);
 428   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
 429   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
 430   smartlist_free(picked);
 431
 432  done:
 433
 434   /* Free leftover stuff */
 435   if (conns_with_circs) {
 436     smartlist_free(conns_with_circs);
 437     conns_with_circs = NULL;
 438   }
 439
 440   UNMOCK(connection_or_get_num_circuits);
 441
 442   if (conns_for_mock) {
 443     SMARTLIST_FOREACH(conns_for_mock, connection_t *, c, tor_free(c));
 444     smartlist_free(conns_for_mock);
 445     conns_for_mock = NULL;
 446   }
 447
 448   UNMOCK(get_connection_array);
 449
 450   return;
 451 }
 452
 453 struct testcase_t oos_tests[] = {
 454   { "connection_check_oos", test_oos_connection_check_oos,
 455     TT_FORK, NULL, NULL },
 456   { "kill_conn_list", test_oos_kill_conn_list, TT_FORK, NULL, NULL },
 457   { "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL },
 458   END_OF_TESTCASES
 459 };