Move StatisticsRecorder out of histogram.cc/h for further refactoring.
[chromium-blink-merge.git] / net / url_request / url_request_throttler_unittest.cc
bloba0827798cb0edc88b8012c03d2265956c14b4c41
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_throttler_manager.h"
7 #include "base/memory/scoped_ptr.h"
8 #include "base/metrics/histogram.h"
9 #include "base/metrics/statistics_recorder.h"
10 #include "base/pickle.h"
11 #include "base/stringprintf.h"
12 #include "base/string_number_conversions.h"
13 #include "base/time.h"
14 #include "net/base/load_flags.h"
15 #include "net/base/test_completion_callback.h"
16 #include "net/url_request/url_request_context.h"
17 #include "net/url_request/url_request_test_util.h"
18 #include "net/url_request/url_request_throttler_header_interface.h"
19 #include "net/url_request/url_request_throttler_test_support.h"
20 #include "testing/gtest/include/gtest/gtest.h"
22 using base::TimeDelta;
23 using base::TimeTicks;
25 namespace net {
27 namespace {
29 using base::Histogram;
30 using base::StatisticsRecorder;
32 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
33 public:
34 explicit MockURLRequestThrottlerEntry(
35 net::URLRequestThrottlerManager* manager)
36 : net::URLRequestThrottlerEntry(manager, ""),
37 mock_backoff_entry_(&backoff_policy_) {
38 InitPolicy();
40 MockURLRequestThrottlerEntry(
41 net::URLRequestThrottlerManager* manager,
42 const TimeTicks& exponential_backoff_release_time,
43 const TimeTicks& sliding_window_release_time,
44 const TimeTicks& fake_now)
45 : net::URLRequestThrottlerEntry(manager, ""),
46 fake_time_now_(fake_now),
47 mock_backoff_entry_(&backoff_policy_) {
48 InitPolicy();
50 mock_backoff_entry_.set_fake_now(fake_now);
51 set_exponential_backoff_release_time(exponential_backoff_release_time);
52 set_sliding_window_release_time(sliding_window_release_time);
55 void InitPolicy() {
56 // Some tests become flaky if we have jitter.
57 backoff_policy_.jitter_factor = 0.0;
59 // This lets us avoid having to make multiple failures initially (this
60 // logic is already tested in the BackoffEntry unit tests).
61 backoff_policy_.num_errors_to_ignore = 0;
64 const BackoffEntry* GetBackoffEntry() const {
65 return &mock_backoff_entry_;
68 BackoffEntry* GetBackoffEntry() {
69 return &mock_backoff_entry_;
72 static bool ExplicitUserRequest(int load_flags) {
73 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags);
76 void ResetToBlank(const TimeTicks& time_now) {
77 fake_time_now_ = time_now;
78 mock_backoff_entry_.set_fake_now(time_now);
80 GetBackoffEntry()->Reset();
81 GetBackoffEntry()->SetCustomReleaseTime(time_now);
82 set_sliding_window_release_time(time_now);
85 // Overridden for tests.
86 virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; }
88 void set_exponential_backoff_release_time(
89 const base::TimeTicks& release_time) {
90 GetBackoffEntry()->SetCustomReleaseTime(release_time);
93 base::TimeTicks sliding_window_release_time() const {
94 return URLRequestThrottlerEntry::sliding_window_release_time();
97 void set_sliding_window_release_time(
98 const base::TimeTicks& release_time) {
99 URLRequestThrottlerEntry::set_sliding_window_release_time(
100 release_time);
103 TimeTicks fake_time_now_;
104 MockBackoffEntry mock_backoff_entry_;
106 protected:
107 virtual ~MockURLRequestThrottlerEntry() {}
110 class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
111 public:
112 MockURLRequestThrottlerManager() : create_entry_index_(0) {}
114 // Method to process the URL using URLRequestThrottlerManager protected
115 // method.
116 std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
118 // Method to use the garbage collecting method of URLRequestThrottlerManager.
119 void DoGarbageCollectEntries() { GarbageCollectEntries(); }
121 // Returns the number of entries in the map.
122 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
124 void CreateEntry(bool is_outdated) {
125 TimeTicks time = TimeTicks::Now();
126 if (is_outdated) {
127 time -= TimeDelta::FromMilliseconds(
128 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
130 std::string fake_url_string("http://www.fakeurl.com/");
131 fake_url_string.append(base::IntToString(create_entry_index_++));
132 GURL fake_url(fake_url_string);
133 OverrideEntryForTests(
134 fake_url,
135 new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
136 TimeTicks::Now()));
139 private:
140 int create_entry_index_;
143 struct TimeAndBool {
144 TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
145 time = time_value;
146 result = expected;
147 line = line_num;
149 TimeTicks time;
150 bool result;
151 int line;
154 struct GurlAndString {
155 GurlAndString(const GURL& url_value,
156 const std::string& expected,
157 int line_num) {
158 url = url_value;
159 result = expected;
160 line = line_num;
162 GURL url;
163 std::string result;
164 int line;
167 } // namespace
169 class URLRequestThrottlerEntryTest : public testing::Test {
170 protected:
171 URLRequestThrottlerEntryTest() : request_(GURL(), NULL, &context_) {
174 virtual void SetUp();
176 // After calling this function, histogram snapshots in |samples_| contain
177 // only the delta caused by the test case currently running.
178 void CalculateHistogramDeltas();
180 TimeTicks now_;
181 MockURLRequestThrottlerManager manager_; // Dummy object, not used.
182 scoped_refptr<MockURLRequestThrottlerEntry> entry_;
184 std::map<std::string, Histogram::SampleSet> original_samples_;
185 std::map<std::string, Histogram::SampleSet> samples_;
187 TestURLRequestContext context_;
188 TestURLRequest request_;
191 // List of all histograms we care about in these unit tests.
192 const char* kHistogramNames[] = {
193 "Throttling.FailureCountAtSuccess",
194 "Throttling.PerceivedDowntime",
195 "Throttling.RequestThrottled",
196 "Throttling.SiteOptedOut",
199 void URLRequestThrottlerEntryTest::SetUp() {
200 request_.set_load_flags(0);
202 now_ = TimeTicks::Now();
203 entry_ = new MockURLRequestThrottlerEntry(&manager_);
204 entry_->ResetToBlank(now_);
206 for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
207 // Must retrieve original samples for each histogram for comparison
208 // as other tests may affect them.
209 const char* name = kHistogramNames[i];
210 Histogram::SampleSet& original = original_samples_[name];
211 Histogram* histogram;
212 if (StatisticsRecorder::FindHistogram(name, &histogram)) {
213 histogram->SnapshotSample(&original);
218 void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
219 for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
220 const char* name = kHistogramNames[i];
221 Histogram::SampleSet& original = original_samples_[name];
222 Histogram::SampleSet& sample = samples_[name];
224 Histogram* histogram;
225 if (StatisticsRecorder::FindHistogram(name, &histogram)) {
226 ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag, histogram->flags());
228 histogram->SnapshotSample(&sample);
229 // Ensure |original| size is same as |sample|, then subtract original
230 // values.
231 original.Resize(*histogram);
232 sample.Subtract(original);
236 // Ensure we don't accidentally use the originals in our tests.
237 original_samples_.clear();
240 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
241 return out << time.ToInternalValue();
244 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
245 entry_->set_exponential_backoff_release_time(
246 entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
247 EXPECT_TRUE(entry_->ShouldRejectRequest(request_));
249 // Also end-to-end test the load flags exceptions.
250 request_.set_load_flags(LOAD_MAYBE_USER_GESTURE);
251 EXPECT_FALSE(entry_->ShouldRejectRequest(request_));
253 CalculateHistogramDeltas();
254 ASSERT_EQ(1, samples_["Throttling.RequestThrottled"].counts(0));
255 ASSERT_EQ(1, samples_["Throttling.RequestThrottled"].counts(1));
258 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
259 entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
260 EXPECT_FALSE(entry_->ShouldRejectRequest(request_));
261 entry_->set_exponential_backoff_release_time(
262 entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
263 EXPECT_FALSE(entry_->ShouldRejectRequest(request_));
265 CalculateHistogramDeltas();
266 ASSERT_EQ(2, samples_["Throttling.RequestThrottled"].counts(0));
267 ASSERT_EQ(0, samples_["Throttling.RequestThrottled"].counts(1));
270 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
271 MockURLRequestThrottlerHeaderAdapter failure_response(503);
272 entry_->UpdateWithResponse("", &failure_response);
273 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
274 << "A failure should increase the release_time";
277 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
278 MockURLRequestThrottlerHeaderAdapter success_response(200);
279 entry_->UpdateWithResponse("", &success_response);
280 EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
281 << "A success should not add any delay";
284 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
285 MockURLRequestThrottlerHeaderAdapter failure_response(503);
286 MockURLRequestThrottlerHeaderAdapter success_response(200);
287 entry_->UpdateWithResponse("", &success_response);
288 entry_->UpdateWithResponse("", &failure_response);
289 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
290 << "This scenario should add delay";
291 entry_->UpdateWithResponse("", &success_response);
294 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
295 TimeDelta lifetime = TimeDelta::FromMilliseconds(
296 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
297 const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
299 TimeAndBool test_values[] = {
300 TimeAndBool(now_, false, __LINE__),
301 TimeAndBool(now_ - kFiveMs, false, __LINE__),
302 TimeAndBool(now_ + kFiveMs, false, __LINE__),
303 TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
304 TimeAndBool(now_ - lifetime, true, __LINE__),
305 TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
307 for (unsigned int i = 0; i < arraysize(test_values); ++i) {
308 entry_->set_exponential_backoff_release_time(test_values[i].time);
309 EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
310 "Test case #" << i << " line " << test_values[i].line << " failed";
314 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
315 for (int i = 0; i < 30; ++i) {
316 MockURLRequestThrottlerHeaderAdapter response_adapter(503);
317 entry_->UpdateWithResponse("", &response_adapter);
320 TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
321 EXPECT_EQ(delay.InMilliseconds(),
322 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
325 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
326 MockURLRequestThrottlerHeaderAdapter response_adapter(503);
327 for (int i = 0; i < 5; ++i)
328 entry_->UpdateWithResponse("", &response_adapter);
330 TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
332 // Inform the entry that a response body was malformed, which is supposed to
333 // increase the back-off time. Note that we also submit a successful
334 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
335 // is what happens in practice (if a body is received, then a non-500
336 // response must also have been received).
337 entry_->ReceivedContentWasMalformed(200);
338 MockURLRequestThrottlerHeaderAdapter success_adapter(200);
339 entry_->UpdateWithResponse("", &success_adapter);
340 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
343 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
344 int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
345 int sliding_window =
346 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
348 TimeTicks time_1 = entry_->fake_time_now_ +
349 TimeDelta::FromMilliseconds(sliding_window / 3);
350 TimeTicks time_2 = entry_->fake_time_now_ +
351 TimeDelta::FromMilliseconds(2 * sliding_window / 3);
352 TimeTicks time_3 = entry_->fake_time_now_ +
353 TimeDelta::FromMilliseconds(sliding_window);
354 TimeTicks time_4 = entry_->fake_time_now_ +
355 TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
357 entry_->set_exponential_backoff_release_time(time_1);
359 for (int i = 0; i < max_send / 2; ++i) {
360 EXPECT_EQ(2 * sliding_window / 3,
361 entry_->ReserveSendingTimeForNextRequest(time_2));
363 EXPECT_EQ(time_2, entry_->sliding_window_release_time());
365 entry_->fake_time_now_ = time_3;
367 for (int i = 0; i < (max_send + 1) / 2; ++i)
368 EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
370 EXPECT_EQ(time_4, entry_->sliding_window_release_time());
373 TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) {
374 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
375 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
376 LOAD_MAYBE_USER_GESTURE));
377 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
378 ~LOAD_MAYBE_USER_GESTURE));
381 class URLRequestThrottlerManagerTest : public testing::Test {
382 protected:
383 URLRequestThrottlerManagerTest()
384 : request_(GURL(), NULL, &context_) {
387 virtual void SetUp() {
388 request_.set_load_flags(0);
391 // context_ must be declared before request_.
392 TestURLRequestContext context_;
393 TestURLRequest request_;
396 TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) {
397 MockURLRequestThrottlerManager manager;
398 GurlAndString test_values[] = {
399 GurlAndString(GURL("http://www.example.com"),
400 std::string("http://www.example.com/"),
401 __LINE__),
402 GurlAndString(GURL("http://www.Example.com"),
403 std::string("http://www.example.com/"),
404 __LINE__),
405 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
406 std::string("http://www.ex4mple.com/pr4c71c41"),
407 __LINE__),
408 GurlAndString(GURL("http://www.example.com/0/token/false"),
409 std::string("http://www.example.com/0/token/false"),
410 __LINE__),
411 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
412 std::string("http://www.example.com/index.php"),
413 __LINE__),
414 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
415 std::string("http://www.example.com/index.php"),
416 __LINE__),
417 GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
418 std::string("http://www.example.com/index.php"),
419 __LINE__),
420 GurlAndString(GURL("http://www.example.com:1234/"),
421 std::string("http://www.example.com:1234/"),
422 __LINE__)};
424 for (unsigned int i = 0; i < arraysize(test_values); ++i) {
425 std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
426 EXPECT_EQ(temp, test_values[i].result) <<
427 "Test case #" << i << " line " << test_values[i].line << " failed";
431 TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) {
432 MockURLRequestThrottlerManager manager;
434 manager.CreateEntry(true); // true = Entry is outdated.
435 manager.CreateEntry(true);
436 manager.CreateEntry(true);
437 manager.DoGarbageCollectEntries();
438 EXPECT_EQ(0, manager.GetNumberOfEntries());
440 manager.CreateEntry(false);
441 manager.CreateEntry(false);
442 manager.CreateEntry(false);
443 manager.CreateEntry(true);
444 manager.DoGarbageCollectEntries();
445 EXPECT_EQ(3, manager.GetNumberOfEntries());
448 TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) {
449 MockURLRequestThrottlerManager manager;
451 manager.RegisterRequestUrl(GURL("http://www.example.com/"));
452 manager.RegisterRequestUrl(GURL("http://www.google.com/"));
453 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
454 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
455 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
457 EXPECT_EQ(3, manager.GetNumberOfEntries());
460 void ExpectEntryAllowsAllOnErrorIfOptedOut(
461 net::URLRequestThrottlerEntryInterface* entry,
462 bool opted_out,
463 const URLRequest& request) {
464 EXPECT_FALSE(entry->ShouldRejectRequest(request));
465 MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
466 for (int i = 0; i < 10; ++i) {
467 // Host doesn't really matter in this scenario so we skip it.
468 entry->UpdateWithResponse("", &failure_adapter);
470 EXPECT_NE(opted_out, entry->ShouldRejectRequest(request));
472 if (opted_out) {
473 // We're not mocking out GetTimeNow() in this scenario
474 // so add a 100 ms buffer to avoid flakiness (that should always
475 // give enough time to get from the TimeTicks::Now() call here
476 // to the TimeTicks::Now() call in the entry class).
477 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
478 entry->GetExponentialBackoffReleaseTime());
479 } else {
480 // As above, add 100 ms.
481 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
482 entry->GetExponentialBackoffReleaseTime());
486 TEST_F(URLRequestThrottlerManagerTest, OptOutHeader) {
487 MockURLRequestThrottlerManager manager;
488 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry =
489 manager.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
491 // Fake a response with the opt-out header.
492 MockURLRequestThrottlerHeaderAdapter response_adapter(
494 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue,
495 200);
496 entry->UpdateWithResponse("www.google.com", &response_adapter);
498 // Ensure that the same entry on error always allows everything.
499 ExpectEntryAllowsAllOnErrorIfOptedOut(entry, true, request_);
501 // Ensure that a freshly created entry (for a different URL on an
502 // already opted-out host) also gets "always allow" behavior.
503 scoped_refptr<net::URLRequestThrottlerEntryInterface> other_entry =
504 manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
505 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry, true, request_);
507 // Fake a response with the opt-out header incorrectly specified.
508 scoped_refptr<net::URLRequestThrottlerEntryInterface> no_opt_out_entry =
509 manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
510 MockURLRequestThrottlerHeaderAdapter wrong_adapter("", "yesplease", 200);
511 no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter);
512 ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry, false, request_);
514 // A localhost entry should always be opted out.
515 scoped_refptr<net::URLRequestThrottlerEntryInterface> localhost_entry =
516 manager.RegisterRequestUrl(GURL("http://localhost/hello"));
517 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry, true, request_);
520 TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) {
521 for (int i = 0; i < 3; ++i) {
522 MockURLRequestThrottlerManager manager;
523 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_before =
524 manager.RegisterRequestUrl(GURL("http://www.example.com/"));
525 MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
526 for (int j = 0; j < 10; ++j) {
527 // Host doesn't really matter in this scenario so we skip it.
528 entry_before->UpdateWithResponse("", &failure_adapter);
530 EXPECT_TRUE(entry_before->ShouldRejectRequest(request_));
532 switch (i) {
533 case 0:
534 manager.OnIPAddressChanged();
535 break;
536 case 1:
537 manager.OnConnectionTypeChanged(
538 net::NetworkChangeNotifier::CONNECTION_UNKNOWN);
539 break;
540 case 2:
541 manager.OnConnectionTypeChanged(
542 net::NetworkChangeNotifier::CONNECTION_NONE);
543 break;
544 default:
545 FAIL();
548 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_after =
549 manager.RegisterRequestUrl(GURL("http://www.example.com/"));
550 EXPECT_FALSE(entry_after->ShouldRejectRequest(request_));
554 } // namespace net