1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // The tests in this file attempt to verify the following through simulation:
6 // a) That a server experiencing overload will actually benefit from the
7 // anti-DDoS throttling logic, i.e. that its traffic spike will subside
8 // and be distributed over a longer period of time;
9 // b) That "well-behaved" clients of a server under DDoS attack actually
10 // benefit from the anti-DDoS throttling logic; and
11 // c) That the approximate increase in "perceived downtime" introduced by
12 // anti-DDoS throttling for various different actual downtimes is what
13 // we expect it to be.
19 #include "base/environment.h"
20 #include "base/memory/scoped_vector.h"
21 #include "base/rand_util.h"
22 #include "base/time.h"
23 #include "net/url_request/url_request_throttler_manager.h"
24 #include "net/url_request/url_request_throttler_test_support.h"
25 #include "testing/gtest/include/gtest/gtest.h"
27 using base::TimeDelta
;
28 using base::TimeTicks
;
33 // Set this variable in your environment if you want to see verbose results
34 // of the simulation tests.
35 const char kShowSimulationVariableName
[] = "SHOW_SIMULATION_RESULTS";
37 // Prints output only if a given environment variable is set. We use this
38 // to not print any output for human evaluation when the test is run without
40 void VerboseOut(const char* format
, ...) {
41 static bool have_checked_environment
= false;
42 static bool should_print
= false;
43 if (!have_checked_environment
) {
44 have_checked_environment
= true;
45 scoped_ptr
<base::Environment
> env(base::Environment::Create());
46 if (env
->HasVar(kShowSimulationVariableName
))
52 va_start(arglist
, format
);
53 vprintf(format
, arglist
);
58 // A simple two-phase discrete time simulation. Actors are added in the order
59 // they should take action at every tick of the clock. Ticks of the clock
61 // - Phase 1 advances every actor's time to a new absolute time.
62 // - Phase 2 asks each actor to perform their action.
63 class DiscreteTimeSimulation
{
68 virtual void AdvanceTime(const TimeTicks
& absolute_time
) = 0;
69 virtual void PerformAction() = 0;
72 DiscreteTimeSimulation() {}
74 // Adds an |actor| to the simulation. The client of the simulation maintains
75 // ownership of |actor| and must ensure its lifetime exceeds that of the
76 // simulation. Actors should be added in the order you wish for them to
77 // act at each tick of the simulation.
78 void AddActor(Actor
* actor
) {
79 actors_
.push_back(actor
);
82 // Runs the simulation for, pretending |time_between_ticks| passes from one
83 // tick to the next. The start time will be the current real time. The
84 // simulation will stop when the simulated duration is equal to or greater
85 // than |maximum_simulated_duration|.
86 void RunSimulation(const TimeDelta
& maximum_simulated_duration
,
87 const TimeDelta
& time_between_ticks
) {
88 TimeTicks start_time
= TimeTicks();
89 TimeTicks now
= start_time
;
90 while ((now
- start_time
) <= maximum_simulated_duration
) {
91 for (std::vector
<Actor
*>::iterator it
= actors_
.begin();
94 (*it
)->AdvanceTime(now
);
97 for (std::vector
<Actor
*>::iterator it
= actors_
.begin();
100 (*it
)->PerformAction();
103 now
+= time_between_ticks
;
108 std::vector
<Actor
*> actors_
;
110 DISALLOW_COPY_AND_ASSIGN(DiscreteTimeSimulation
);
113 // Represents a web server in a simulation of a server under attack by
114 // a lot of clients. Must be added to the simulation's list of actors
115 // after all |Requester| objects.
116 class Server
: public DiscreteTimeSimulation::Actor
{
118 Server(int max_queries_per_tick
,
119 double request_drop_ratio
)
120 : max_queries_per_tick_(max_queries_per_tick
),
121 request_drop_ratio_(request_drop_ratio
),
122 num_overloaded_ticks_remaining_(0),
123 num_current_tick_queries_(0),
124 num_overloaded_ticks_(0),
125 max_experienced_queries_per_tick_(0) {
128 void SetDowntime(const TimeTicks
& start_time
, const TimeDelta
& duration
) {
129 start_downtime_
= start_time
;
130 end_downtime_
= start_time
+ duration
;
133 virtual void AdvanceTime(const TimeTicks
& absolute_time
) OVERRIDE
{
134 now_
= absolute_time
;
137 virtual void PerformAction() OVERRIDE
{
138 // We are inserted at the end of the actor's list, so all Requester
139 // instances have already done their bit.
140 if (num_current_tick_queries_
> max_experienced_queries_per_tick_
)
141 max_experienced_queries_per_tick_
= num_current_tick_queries_
;
143 if (num_current_tick_queries_
> max_queries_per_tick_
) {
144 // We pretend the server fails for the next several ticks after it
146 num_overloaded_ticks_remaining_
= 5;
147 ++num_overloaded_ticks_
;
148 } else if (num_overloaded_ticks_remaining_
> 0) {
149 --num_overloaded_ticks_remaining_
;
152 requests_per_tick_
.push_back(num_current_tick_queries_
);
153 num_current_tick_queries_
= 0;
156 // This is called by Requester. It returns the response code from
158 int HandleRequest() {
159 ++num_current_tick_queries_
;
160 if (!start_downtime_
.is_null() &&
161 start_downtime_
< now_
&& now_
< end_downtime_
) {
162 // TODO(joi): For the simulation measuring the increase in perceived
163 // downtime, it might be interesting to count separately the
164 // queries seen by the server (assuming a front-end reverse proxy
165 // is what actually serves up the 503s in this case) so that we could
166 // visualize the traffic spike seen by the server when it comes up,
167 // which would in many situations be ameliorated by the anti-DDoS
172 if ((num_overloaded_ticks_remaining_
> 0 ||
173 num_current_tick_queries_
> max_queries_per_tick_
) &&
174 base::RandDouble() < request_drop_ratio_
) {
181 int num_overloaded_ticks() const {
182 return num_overloaded_ticks_
;
185 int max_experienced_queries_per_tick() const {
186 return max_experienced_queries_per_tick_
;
189 std::string
VisualizeASCII(int terminal_width
) {
190 // Account for | characters we place at left of graph.
193 VerboseOut("Overloaded for %d of %d ticks.\n",
194 num_overloaded_ticks_
, requests_per_tick_
.size());
195 VerboseOut("Got maximum of %d requests in a tick.\n\n",
196 max_experienced_queries_per_tick_
);
198 VerboseOut("Traffic graph:\n\n");
200 // Printing the graph like this is a bit overkill, but was very useful
201 // while developing the various simulations to see if they were testing
202 // the corner cases we want to simulate.
204 // Find the smallest number of whole ticks we need to group into a
205 // column that will let all ticks fit into the column width we have.
206 int num_ticks
= requests_per_tick_
.size();
207 double ticks_per_column_exact
=
208 static_cast<double>(num_ticks
) / static_cast<double>(terminal_width
);
209 int ticks_per_column
= std::ceil(ticks_per_column_exact
);
210 DCHECK_GE(ticks_per_column
* terminal_width
, num_ticks
);
212 // Sum up the column values.
213 int num_columns
= num_ticks
/ ticks_per_column
;
214 if (num_ticks
% ticks_per_column
)
216 DCHECK_LE(num_columns
, terminal_width
);
217 scoped_array
<int> columns(new int[num_columns
]);
218 for (int tx
= 0; tx
< num_ticks
; ++tx
) {
219 int cx
= tx
/ ticks_per_column
;
220 if (tx
% ticks_per_column
== 0)
222 columns
[cx
] += requests_per_tick_
[tx
];
225 // Find the lowest integer divisor that will let the column values
226 // be represented in a graph of maximum height 50.
228 for (int cx
= 0; cx
< num_columns
; ++cx
)
229 max_value
= std::max(max_value
, columns
[cx
]);
230 const int kNumRows
= 50;
231 double row_divisor_exact
= max_value
/ static_cast<double>(kNumRows
);
232 int row_divisor
= std::ceil(row_divisor_exact
);
233 DCHECK_GE(row_divisor
* kNumRows
, max_value
);
235 // To show the overload line, we calculate the appropriate value.
236 int overload_value
= max_queries_per_tick_
* ticks_per_column
;
238 // When num_ticks is not a whole multiple of ticks_per_column, the last
239 // column includes fewer ticks than the others. In this case, don't
240 // print it so that we don't show an inconsistent value.
241 int num_printed_columns
= num_columns
;
242 if (num_ticks
% ticks_per_column
)
243 --num_printed_columns
;
245 // This is a top-to-bottom traversal of rows, left-to-right per row.
247 for (int rx
= 0; rx
< kNumRows
; ++rx
) {
248 int range_min
= (kNumRows
- rx
) * row_divisor
;
249 int range_max
= range_min
+ row_divisor
;
251 range_min
= -1; // Make 0 values fit in the bottom range.
253 for (int cx
= 0; cx
< num_printed_columns
; ++cx
) {
255 // Show the overload line.
256 if (range_min
< overload_value
&& overload_value
<= range_max
)
259 // Preferentially, show the graph line.
260 if (range_min
< columns
[cx
] && columns
[cx
] <= range_max
)
263 output
.append(1, block
);
268 output
.append(num_printed_columns
, '=');
275 TimeTicks start_downtime_
; // Can be 0 to say "no downtime".
276 TimeTicks end_downtime_
;
277 const int max_queries_per_tick_
;
278 const double request_drop_ratio_
; // Ratio of requests to 503 when failing.
279 int num_overloaded_ticks_remaining_
;
280 int num_current_tick_queries_
;
281 int num_overloaded_ticks_
;
282 int max_experienced_queries_per_tick_
;
283 std::vector
<int> requests_per_tick_
;
285 DISALLOW_COPY_AND_ASSIGN(Server
);
288 class TestingURLRequestThrottlerManager
: public URLRequestThrottlerManager
{
290 TestingURLRequestThrottlerManager() : URLRequestThrottlerManager() {
294 // Mock throttler entry used by Requester class.
295 class MockURLRequestThrottlerEntry
: public URLRequestThrottlerEntry
{
297 explicit MockURLRequestThrottlerEntry(
298 URLRequestThrottlerManager
* manager
)
299 : URLRequestThrottlerEntry(manager
, ""),
300 mock_backoff_entry_(&backoff_policy_
) {
302 virtual ~MockURLRequestThrottlerEntry() {}
304 virtual const BackoffEntry
* GetBackoffEntry() const OVERRIDE
{
305 return &mock_backoff_entry_
;
308 virtual BackoffEntry
* GetBackoffEntry() OVERRIDE
{
309 return &mock_backoff_entry_
;
312 virtual TimeTicks
ImplGetTimeNow() const OVERRIDE
{
316 void SetFakeNow(const TimeTicks
& fake_time
) {
317 fake_now_
= fake_time
;
318 mock_backoff_entry_
.set_fake_now(fake_time
);
321 TimeTicks
fake_now() const {
327 MockBackoffEntry mock_backoff_entry_
;
330 // Registry of results for a class of |Requester| objects (e.g. attackers vs.
332 class RequesterResults
{
335 : num_attempts_(0), num_successful_(0), num_failed_(0), num_blocked_(0) {
353 int num_attempts() const { return num_attempts_
; }
354 int num_successful() const { return num_successful_
; }
355 int num_failed() const { return num_failed_
; }
356 int num_blocked() const { return num_blocked_
; }
358 double GetBlockedRatio() {
359 DCHECK(num_attempts_
);
360 return static_cast<double>(num_blocked_
) /
361 static_cast<double>(num_attempts_
);
364 double GetSuccessRatio() {
365 DCHECK(num_attempts_
);
366 return static_cast<double>(num_successful_
) /
367 static_cast<double>(num_attempts_
);
370 void PrintResults(const char* class_description
) {
371 if (num_attempts_
== 0) {
372 VerboseOut("No data for %s\n", class_description
);
376 VerboseOut("Requester results for %s\n", class_description
);
377 VerboseOut(" %d attempts\n", num_attempts_
);
378 VerboseOut(" %d successes\n", num_successful_
);
379 VerboseOut(" %d 5xx responses\n", num_failed_
);
380 VerboseOut(" %d requests blocked\n", num_blocked_
);
381 VerboseOut(" %.2f success ratio\n", GetSuccessRatio());
382 VerboseOut(" %.2f blocked ratio\n", GetBlockedRatio());
393 // Represents an Requester in a simulated DDoS situation, that periodically
394 // requests a specific resource.
395 class Requester
: public DiscreteTimeSimulation::Actor
{
397 Requester(MockURLRequestThrottlerEntry
* throttler_entry
,
398 const TimeDelta
& time_between_requests
,
400 RequesterResults
* results
)
401 : throttler_entry_(throttler_entry
),
402 time_between_requests_(time_between_requests
),
403 last_attempt_was_failure_(false),
409 void AdvanceTime(const TimeTicks
& absolute_time
) OVERRIDE
{
410 if (time_of_last_success_
.is_null())
411 time_of_last_success_
= absolute_time
;
413 throttler_entry_
->SetFakeNow(absolute_time
);
416 void PerformAction() OVERRIDE
{
417 TimeDelta effective_delay
= time_between_requests_
;
418 TimeDelta current_jitter
= TimeDelta::FromMilliseconds(
419 request_jitter_
.InMilliseconds() * base::RandDouble());
420 if (base::RandInt(0, 1)) {
421 effective_delay
-= current_jitter
;
423 effective_delay
+= current_jitter
;
426 if (throttler_entry_
->fake_now() - time_of_last_attempt_
>
428 if (!throttler_entry_
->IsDuringExponentialBackoff()) {
429 int status_code
= server_
->HandleRequest();
430 MockURLRequestThrottlerHeaderAdapter
response_headers(status_code
);
431 throttler_entry_
->UpdateWithResponse("", &response_headers
);
433 if (status_code
== 200) {
435 results_
->AddSuccess();
437 if (last_attempt_was_failure_
) {
438 last_downtime_duration_
=
439 throttler_entry_
->fake_now() - time_of_last_success_
;
442 time_of_last_success_
= throttler_entry_
->fake_now();
443 last_attempt_was_failure_
= false;
446 results_
->AddFailure();
447 last_attempt_was_failure_
= true;
451 results_
->AddBlocked();
452 last_attempt_was_failure_
= true;
455 time_of_last_attempt_
= throttler_entry_
->fake_now();
459 // Adds a delay until the first request, equal to a uniformly distributed
460 // value between now and now + max_delay.
461 void SetStartupJitter(const TimeDelta
& max_delay
) {
462 int delay_ms
= base::RandInt(0, max_delay
.InMilliseconds());
463 time_of_last_attempt_
= TimeTicks() +
464 TimeDelta::FromMilliseconds(delay_ms
) - time_between_requests_
;
467 void SetRequestJitter(const TimeDelta
& request_jitter
) {
468 request_jitter_
= request_jitter
;
471 TimeDelta
last_downtime_duration() const { return last_downtime_duration_
; }
474 scoped_refptr
<MockURLRequestThrottlerEntry
> throttler_entry_
;
475 const TimeDelta time_between_requests_
;
476 TimeDelta request_jitter_
;
477 TimeTicks time_of_last_attempt_
;
478 TimeTicks time_of_last_success_
;
479 bool last_attempt_was_failure_
;
480 TimeDelta last_downtime_duration_
;
481 Server
* const server_
;
482 RequesterResults
* const results_
; // May be NULL.
484 DISALLOW_COPY_AND_ASSIGN(Requester
);
487 void SimulateAttack(Server
* server
,
488 RequesterResults
* attacker_results
,
489 RequesterResults
* client_results
,
490 bool enable_throttling
) {
491 const size_t kNumAttackers
= 50;
492 const size_t kNumClients
= 50;
493 DiscreteTimeSimulation simulation
;
494 TestingURLRequestThrottlerManager manager
;
495 ScopedVector
<Requester
> requesters
;
496 for (size_t i
= 0; i
< kNumAttackers
; ++i
) {
497 // Use a tiny time_between_requests so the attackers will ping the
498 // server at every tick of the simulation.
499 scoped_refptr
<MockURLRequestThrottlerEntry
> throttler_entry(
500 new MockURLRequestThrottlerEntry(&manager
));
501 if (!enable_throttling
)
502 throttler_entry
->DisableBackoffThrottling();
504 Requester
* attacker
= new Requester(throttler_entry
.get(),
505 TimeDelta::FromMilliseconds(1),
508 attacker
->SetStartupJitter(TimeDelta::FromSeconds(120));
509 requesters
.push_back(attacker
);
510 simulation
.AddActor(attacker
);
512 for (size_t i
= 0; i
< kNumClients
; ++i
) {
513 // Normal clients only make requests every 2 minutes, plus/minus 1 minute.
514 scoped_refptr
<MockURLRequestThrottlerEntry
> throttler_entry(
515 new MockURLRequestThrottlerEntry(&manager
));
516 if (!enable_throttling
)
517 throttler_entry
->DisableBackoffThrottling();
519 Requester
* client
= new Requester(throttler_entry
.get(),
520 TimeDelta::FromMinutes(2),
523 client
->SetStartupJitter(TimeDelta::FromSeconds(120));
524 client
->SetRequestJitter(TimeDelta::FromMinutes(1));
525 requesters
.push_back(client
);
526 simulation
.AddActor(client
);
528 simulation
.AddActor(server
);
530 simulation
.RunSimulation(TimeDelta::FromMinutes(6),
531 TimeDelta::FromSeconds(1));
534 TEST(URLRequestThrottlerSimulation
, HelpsInAttack
) {
535 Server
unprotected_server(30, 1.0);
536 RequesterResults unprotected_attacker_results
;
537 RequesterResults unprotected_client_results
;
538 Server
protected_server(30, 1.0);
539 RequesterResults protected_attacker_results
;
540 RequesterResults protected_client_results
;
541 SimulateAttack(&unprotected_server
,
542 &unprotected_attacker_results
,
543 &unprotected_client_results
,
545 SimulateAttack(&protected_server
,
546 &protected_attacker_results
,
547 &protected_client_results
,
550 // These assert that the DDoS protection actually benefits the
551 // server. Manual inspection of the traffic graphs will show this
552 // even more clearly.
553 EXPECT_GT(unprotected_server
.num_overloaded_ticks(),
554 protected_server
.num_overloaded_ticks());
555 EXPECT_GT(unprotected_server
.max_experienced_queries_per_tick(),
556 protected_server
.max_experienced_queries_per_tick());
558 // These assert that the DDoS protection actually benefits non-malicious
559 // (and non-degenerate/accidentally DDoSing) users.
560 EXPECT_LT(protected_client_results
.GetBlockedRatio(),
561 protected_attacker_results
.GetBlockedRatio());
562 EXPECT_GT(protected_client_results
.GetSuccessRatio(),
563 unprotected_client_results
.GetSuccessRatio());
565 // The rest is just for optional manual evaluation of the results;
566 // in particular the traffic pattern is interesting.
568 VerboseOut("\nUnprotected server's results:\n\n");
569 VerboseOut(unprotected_server
.VisualizeASCII(132).c_str());
571 VerboseOut("Protected server's results:\n\n");
572 VerboseOut(protected_server
.VisualizeASCII(132).c_str());
575 unprotected_attacker_results
.PrintResults(
576 "attackers attacking unprotected server.");
577 unprotected_client_results
.PrintResults(
578 "normal clients making requests to unprotected server.");
579 protected_attacker_results
.PrintResults(
580 "attackers attacking protected server.");
581 protected_client_results
.PrintResults(
582 "normal clients making requests to protected server.");
585 // Returns the downtime perceived by the client, as a ratio of the
587 double SimulateDowntime(const TimeDelta
& duration
,
588 const TimeDelta
& average_client_interval
,
589 bool enable_throttling
) {
590 TimeDelta time_between_ticks
= duration
/ 200;
591 TimeTicks start_downtime
= TimeTicks() + (duration
/ 2);
593 // A server that never rejects requests, but will go down for maintenance.
594 Server
server(std::numeric_limits
<int>::max(), 1.0);
595 server
.SetDowntime(start_downtime
, duration
);
597 TestingURLRequestThrottlerManager manager
;
598 scoped_refptr
<MockURLRequestThrottlerEntry
> throttler_entry(
599 new MockURLRequestThrottlerEntry(&manager
));
600 if (!enable_throttling
)
601 throttler_entry
->DisableBackoffThrottling();
604 throttler_entry
.get(), average_client_interval
, &server
, NULL
);
605 requester
.SetStartupJitter(duration
/ 3);
606 requester
.SetRequestJitter(average_client_interval
);
608 DiscreteTimeSimulation simulation
;
609 simulation
.AddActor(&requester
);
610 simulation
.AddActor(&server
);
612 simulation
.RunSimulation(duration
* 2, time_between_ticks
);
614 return static_cast<double>(
615 requester
.last_downtime_duration().InMilliseconds()) /
616 static_cast<double>(duration
.InMilliseconds());
619 TEST(URLRequestThrottlerSimulation
, PerceivedDowntimeRatio
) {
621 // Expected interval that we expect the ratio of downtime when anti-DDoS
622 // is enabled and downtime when anti-DDoS is not enabled to fall within.
624 // The expected interval depends on two things: The exponential back-off
625 // policy encoded in URLRequestThrottlerEntry, and the test or set of
626 // tests that the Stats object is tracking (e.g. a test where the client
627 // retries very rapidly on a very long downtime will tend to increase the
630 // To determine an appropriate new interval when parameters have changed,
631 // run the test a few times (you may have to Ctrl-C out of it after a few
632 // seconds) and choose an interval that the test converges quickly and
633 // reliably to. Then set the new interval, and run the test e.g. 20 times
634 // in succession to make sure it never takes an obscenely long time to
635 // converge to this interval.
636 double expected_min_increase
;
637 double expected_max_increase
;
640 double total_ratio_unprotected
;
641 double total_ratio_protected
;
643 bool DidConverge(double* increase_ratio_out
) {
644 double unprotected_ratio
= total_ratio_unprotected
/ num_runs
;
645 double protected_ratio
= total_ratio_protected
/ num_runs
;
646 double increase_ratio
= protected_ratio
/ unprotected_ratio
;
647 if (increase_ratio_out
)
648 *increase_ratio_out
= increase_ratio
;
649 return expected_min_increase
<= increase_ratio
&&
650 increase_ratio
<= expected_max_increase
;
653 void ReportTrialResult(double increase_ratio
) {
655 " Perceived downtime with throttling is %.4f times without.\n",
657 VerboseOut(" Test result after %d trials.\n", num_runs
);
661 Stats global_stats
= { 1.08, 1.15 };
665 TimeDelta average_client_interval
;
668 void PrintTrialDescription() {
669 double duration_minutes
=
670 static_cast<double>(duration
.InSeconds()) / 60.0;
671 double interval_minutes
=
672 static_cast<double>(average_client_interval
.InSeconds()) / 60.0;
673 VerboseOut("Trial with %.2f min downtime, avg. interval %.2f min.\n",
674 duration_minutes
, interval_minutes
);
678 // We don't set or check expected ratio intervals on individual
679 // experiments as this might make the test too fragile, but we
680 // print them out at the end for manual evaluation (we want to be
681 // able to make claims about the expected ratios depending on the
682 // type of behavior of the client and the downtime, e.g. the difference
683 // in behavior between a client making requests every few minutes vs.
684 // one that makes a request every 15 seconds).
686 { TimeDelta::FromSeconds(10), TimeDelta::FromSeconds(3) },
687 { TimeDelta::FromSeconds(30), TimeDelta::FromSeconds(7) },
688 { TimeDelta::FromMinutes(5), TimeDelta::FromSeconds(30) },
689 { TimeDelta::FromMinutes(10), TimeDelta::FromSeconds(20) },
690 { TimeDelta::FromMinutes(20), TimeDelta::FromSeconds(15) },
691 { TimeDelta::FromMinutes(20), TimeDelta::FromSeconds(50) },
692 { TimeDelta::FromMinutes(30), TimeDelta::FromMinutes(2) },
693 { TimeDelta::FromMinutes(30), TimeDelta::FromMinutes(5) },
694 { TimeDelta::FromMinutes(40), TimeDelta::FromMinutes(7) },
695 { TimeDelta::FromMinutes(40), TimeDelta::FromMinutes(2) },
696 { TimeDelta::FromMinutes(40), TimeDelta::FromSeconds(15) },
697 { TimeDelta::FromMinutes(60), TimeDelta::FromMinutes(7) },
698 { TimeDelta::FromMinutes(60), TimeDelta::FromMinutes(2) },
699 { TimeDelta::FromMinutes(60), TimeDelta::FromSeconds(15) },
700 { TimeDelta::FromMinutes(80), TimeDelta::FromMinutes(20) },
701 { TimeDelta::FromMinutes(80), TimeDelta::FromMinutes(3) },
702 { TimeDelta::FromMinutes(80), TimeDelta::FromSeconds(15) },
705 { TimeDelta::FromMinutes(45), TimeDelta::FromMilliseconds(500) },
708 // If things don't converge by the time we've done 100K trials, then
709 // clearly one or more of the expected intervals are wrong.
710 while (global_stats
.num_runs
< 100000) {
711 for (size_t i
= 0; i
< ARRAYSIZE_UNSAFE(trials
); ++i
) {
712 ++global_stats
.num_runs
;
713 ++trials
[i
].stats
.num_runs
;
714 double ratio_unprotected
= SimulateDowntime(
715 trials
[i
].duration
, trials
[i
].average_client_interval
, false);
716 double ratio_protected
= SimulateDowntime(
717 trials
[i
].duration
, trials
[i
].average_client_interval
, true);
718 global_stats
.total_ratio_unprotected
+= ratio_unprotected
;
719 global_stats
.total_ratio_protected
+= ratio_protected
;
720 trials
[i
].stats
.total_ratio_unprotected
+= ratio_unprotected
;
721 trials
[i
].stats
.total_ratio_protected
+= ratio_protected
;
724 double increase_ratio
;
725 if (global_stats
.DidConverge(&increase_ratio
))
728 if (global_stats
.num_runs
> 200) {
729 VerboseOut("Test has not yet converged on expected interval.\n");
730 global_stats
.ReportTrialResult(increase_ratio
);
734 double average_increase_ratio
;
735 EXPECT_TRUE(global_stats
.DidConverge(&average_increase_ratio
));
737 // Print individual trial results for optional manual evaluation.
738 double max_increase_ratio
= 0.0;
739 for (size_t i
= 0; i
< ARRAYSIZE_UNSAFE(trials
); ++i
) {
740 double increase_ratio
;
741 trials
[i
].stats
.DidConverge(&increase_ratio
);
742 max_increase_ratio
= std::max(max_increase_ratio
, increase_ratio
);
743 trials
[i
].PrintTrialDescription();
744 trials
[i
].stats
.ReportTrialResult(increase_ratio
);
747 VerboseOut("Average increase ratio was %.4f\n", average_increase_ratio
);
748 VerboseOut("Maximum increase ratio was %.4f\n", max_increase_ratio
);