Revert 162240 - See also:
[chromium-blink-merge.git] / net / url_request / url_request_throttler_unittest.cc
blob221d0c8788a1d1f0df7524dfd9e1dac45d976bfb
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_throttler_manager.h"
7 #include "base/memory/scoped_ptr.h"
8 #include "base/metrics/histogram.h"
9 #include "base/metrics/histogram_samples.h"
10 #include "base/metrics/statistics_recorder.h"
11 #include "base/pickle.h"
12 #include "base/stl_util.h"
13 #include "base/stringprintf.h"
14 #include "base/string_number_conversions.h"
15 #include "base/time.h"
16 #include "net/base/load_flags.h"
17 #include "net/base/test_completion_callback.h"
18 #include "net/url_request/url_request_context.h"
19 #include "net/url_request/url_request_test_util.h"
20 #include "net/url_request/url_request_throttler_header_interface.h"
21 #include "net/url_request/url_request_throttler_test_support.h"
22 #include "testing/gtest/include/gtest/gtest.h"
24 using base::TimeDelta;
25 using base::TimeTicks;
27 namespace net {
29 namespace {
31 using base::Histogram;
32 using base::HistogramSamples;
33 using base::StatisticsRecorder;
35 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
36 public:
37 explicit MockURLRequestThrottlerEntry(
38 net::URLRequestThrottlerManager* manager)
39 : net::URLRequestThrottlerEntry(manager, ""),
40 mock_backoff_entry_(&backoff_policy_) {
41 InitPolicy();
43 MockURLRequestThrottlerEntry(
44 net::URLRequestThrottlerManager* manager,
45 const TimeTicks& exponential_backoff_release_time,
46 const TimeTicks& sliding_window_release_time,
47 const TimeTicks& fake_now)
48 : net::URLRequestThrottlerEntry(manager, ""),
49 fake_time_now_(fake_now),
50 mock_backoff_entry_(&backoff_policy_) {
51 InitPolicy();
53 mock_backoff_entry_.set_fake_now(fake_now);
54 set_exponential_backoff_release_time(exponential_backoff_release_time);
55 set_sliding_window_release_time(sliding_window_release_time);
58 void InitPolicy() {
59 // Some tests become flaky if we have jitter.
60 backoff_policy_.jitter_factor = 0.0;
62 // This lets us avoid having to make multiple failures initially (this
63 // logic is already tested in the BackoffEntry unit tests).
64 backoff_policy_.num_errors_to_ignore = 0;
67 const BackoffEntry* GetBackoffEntry() const {
68 return &mock_backoff_entry_;
71 BackoffEntry* GetBackoffEntry() {
72 return &mock_backoff_entry_;
75 static bool ExplicitUserRequest(int load_flags) {
76 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags);
79 void ResetToBlank(const TimeTicks& time_now) {
80 fake_time_now_ = time_now;
81 mock_backoff_entry_.set_fake_now(time_now);
83 GetBackoffEntry()->Reset();
84 GetBackoffEntry()->SetCustomReleaseTime(time_now);
85 set_sliding_window_release_time(time_now);
88 // Overridden for tests.
89 virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; }
91 void set_exponential_backoff_release_time(
92 const base::TimeTicks& release_time) {
93 GetBackoffEntry()->SetCustomReleaseTime(release_time);
96 base::TimeTicks sliding_window_release_time() const {
97 return URLRequestThrottlerEntry::sliding_window_release_time();
100 void set_sliding_window_release_time(
101 const base::TimeTicks& release_time) {
102 URLRequestThrottlerEntry::set_sliding_window_release_time(
103 release_time);
106 TimeTicks fake_time_now_;
107 MockBackoffEntry mock_backoff_entry_;
109 protected:
110 virtual ~MockURLRequestThrottlerEntry() {}
113 class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
114 public:
115 MockURLRequestThrottlerManager() : create_entry_index_(0) {}
117 // Method to process the URL using URLRequestThrottlerManager protected
118 // method.
119 std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
121 // Method to use the garbage collecting method of URLRequestThrottlerManager.
122 void DoGarbageCollectEntries() { GarbageCollectEntries(); }
124 // Returns the number of entries in the map.
125 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
127 void CreateEntry(bool is_outdated) {
128 TimeTicks time = TimeTicks::Now();
129 if (is_outdated) {
130 time -= TimeDelta::FromMilliseconds(
131 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
133 std::string fake_url_string("http://www.fakeurl.com/");
134 fake_url_string.append(base::IntToString(create_entry_index_++));
135 GURL fake_url(fake_url_string);
136 OverrideEntryForTests(
137 fake_url,
138 new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
139 TimeTicks::Now()));
142 private:
143 int create_entry_index_;
146 struct TimeAndBool {
147 TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
148 time = time_value;
149 result = expected;
150 line = line_num;
152 TimeTicks time;
153 bool result;
154 int line;
157 struct GurlAndString {
158 GurlAndString(const GURL& url_value,
159 const std::string& expected,
160 int line_num) {
161 url = url_value;
162 result = expected;
163 line = line_num;
165 GURL url;
166 std::string result;
167 int line;
170 } // namespace
172 class URLRequestThrottlerEntryTest : public testing::Test {
173 protected:
174 URLRequestThrottlerEntryTest() : request_(GURL(), NULL, &context_) {
177 virtual void SetUp();
178 virtual void TearDown();
180 // After calling this function, histogram snapshots in |samples_| contain
181 // only the delta caused by the test case currently running.
182 void CalculateHistogramDeltas();
184 TimeTicks now_;
185 MockURLRequestThrottlerManager manager_; // Dummy object, not used.
186 scoped_refptr<MockURLRequestThrottlerEntry> entry_;
188 std::map<std::string, HistogramSamples*> original_samples_;
189 std::map<std::string, HistogramSamples*> samples_;
191 TestURLRequestContext context_;
192 TestURLRequest request_;
195 // List of all histograms we care about in these unit tests.
196 const char* kHistogramNames[] = {
197 "Throttling.FailureCountAtSuccess",
198 "Throttling.PerceivedDowntime",
199 "Throttling.RequestThrottled",
200 "Throttling.SiteOptedOut",
203 void URLRequestThrottlerEntryTest::SetUp() {
204 request_.set_load_flags(0);
206 now_ = TimeTicks::Now();
207 entry_ = new MockURLRequestThrottlerEntry(&manager_);
208 entry_->ResetToBlank(now_);
210 for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
211 // Must retrieve original samples for each histogram for comparison
212 // as other tests may affect them.
213 const char* name = kHistogramNames[i];
214 Histogram* histogram = StatisticsRecorder::FindHistogram(name);
215 if (histogram) {
216 original_samples_[name] = histogram->SnapshotSamples().release();
217 } else {
218 original_samples_[name] = NULL;
223 void URLRequestThrottlerEntryTest::TearDown() {
224 STLDeleteValues(&original_samples_);
225 STLDeleteValues(&samples_);
228 void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
229 for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
230 const char* name = kHistogramNames[i];
231 HistogramSamples* original = original_samples_[name];
233 Histogram* histogram = StatisticsRecorder::FindHistogram(name);
234 if (histogram) {
235 ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag, histogram->flags());
237 scoped_ptr<HistogramSamples> samples(histogram->SnapshotSamples());
238 if (original)
239 samples->Subtract(*original);
240 samples_[name] = samples.release();
244 // Ensure we don't accidentally use the originals in our tests.
245 STLDeleteValues(&original_samples_);
246 original_samples_.clear();
249 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
250 return out << time.ToInternalValue();
253 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
254 entry_->set_exponential_backoff_release_time(
255 entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
256 EXPECT_TRUE(entry_->ShouldRejectRequest(request_));
258 // Also end-to-end test the load flags exceptions.
259 request_.set_load_flags(LOAD_MAYBE_USER_GESTURE);
260 EXPECT_FALSE(entry_->ShouldRejectRequest(request_));
262 CalculateHistogramDeltas();
263 ASSERT_EQ(1, samples_["Throttling.RequestThrottled"]->GetCount(0));
264 ASSERT_EQ(1, samples_["Throttling.RequestThrottled"]->GetCount(1));
267 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
268 entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
269 EXPECT_FALSE(entry_->ShouldRejectRequest(request_));
270 entry_->set_exponential_backoff_release_time(
271 entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
272 EXPECT_FALSE(entry_->ShouldRejectRequest(request_));
274 CalculateHistogramDeltas();
275 ASSERT_EQ(2, samples_["Throttling.RequestThrottled"]->GetCount(0));
276 ASSERT_EQ(0, samples_["Throttling.RequestThrottled"]->GetCount(1));
279 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
280 MockURLRequestThrottlerHeaderAdapter failure_response(503);
281 entry_->UpdateWithResponse("", &failure_response);
282 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
283 << "A failure should increase the release_time";
286 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
287 MockURLRequestThrottlerHeaderAdapter success_response(200);
288 entry_->UpdateWithResponse("", &success_response);
289 EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
290 << "A success should not add any delay";
293 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
294 MockURLRequestThrottlerHeaderAdapter failure_response(503);
295 MockURLRequestThrottlerHeaderAdapter success_response(200);
296 entry_->UpdateWithResponse("", &success_response);
297 entry_->UpdateWithResponse("", &failure_response);
298 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
299 << "This scenario should add delay";
300 entry_->UpdateWithResponse("", &success_response);
303 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
304 TimeDelta lifetime = TimeDelta::FromMilliseconds(
305 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
306 const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
308 TimeAndBool test_values[] = {
309 TimeAndBool(now_, false, __LINE__),
310 TimeAndBool(now_ - kFiveMs, false, __LINE__),
311 TimeAndBool(now_ + kFiveMs, false, __LINE__),
312 TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
313 TimeAndBool(now_ - lifetime, true, __LINE__),
314 TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
316 for (unsigned int i = 0; i < arraysize(test_values); ++i) {
317 entry_->set_exponential_backoff_release_time(test_values[i].time);
318 EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
319 "Test case #" << i << " line " << test_values[i].line << " failed";
323 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
324 for (int i = 0; i < 30; ++i) {
325 MockURLRequestThrottlerHeaderAdapter response_adapter(503);
326 entry_->UpdateWithResponse("", &response_adapter);
329 TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
330 EXPECT_EQ(delay.InMilliseconds(),
331 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
334 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
335 MockURLRequestThrottlerHeaderAdapter response_adapter(503);
336 for (int i = 0; i < 5; ++i)
337 entry_->UpdateWithResponse("", &response_adapter);
339 TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
341 // Inform the entry that a response body was malformed, which is supposed to
342 // increase the back-off time. Note that we also submit a successful
343 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
344 // is what happens in practice (if a body is received, then a non-500
345 // response must also have been received).
346 entry_->ReceivedContentWasMalformed(200);
347 MockURLRequestThrottlerHeaderAdapter success_adapter(200);
348 entry_->UpdateWithResponse("", &success_adapter);
349 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
352 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
353 int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
354 int sliding_window =
355 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
357 TimeTicks time_1 = entry_->fake_time_now_ +
358 TimeDelta::FromMilliseconds(sliding_window / 3);
359 TimeTicks time_2 = entry_->fake_time_now_ +
360 TimeDelta::FromMilliseconds(2 * sliding_window / 3);
361 TimeTicks time_3 = entry_->fake_time_now_ +
362 TimeDelta::FromMilliseconds(sliding_window);
363 TimeTicks time_4 = entry_->fake_time_now_ +
364 TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
366 entry_->set_exponential_backoff_release_time(time_1);
368 for (int i = 0; i < max_send / 2; ++i) {
369 EXPECT_EQ(2 * sliding_window / 3,
370 entry_->ReserveSendingTimeForNextRequest(time_2));
372 EXPECT_EQ(time_2, entry_->sliding_window_release_time());
374 entry_->fake_time_now_ = time_3;
376 for (int i = 0; i < (max_send + 1) / 2; ++i)
377 EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
379 EXPECT_EQ(time_4, entry_->sliding_window_release_time());
382 TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) {
383 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
384 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
385 LOAD_MAYBE_USER_GESTURE));
386 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
387 ~LOAD_MAYBE_USER_GESTURE));
390 class URLRequestThrottlerManagerTest : public testing::Test {
391 protected:
392 URLRequestThrottlerManagerTest()
393 : request_(GURL(), NULL, &context_) {
396 virtual void SetUp() {
397 request_.set_load_flags(0);
400 // context_ must be declared before request_.
401 TestURLRequestContext context_;
402 TestURLRequest request_;
405 TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) {
406 MockURLRequestThrottlerManager manager;
407 GurlAndString test_values[] = {
408 GurlAndString(GURL("http://www.example.com"),
409 std::string("http://www.example.com/"),
410 __LINE__),
411 GurlAndString(GURL("http://www.Example.com"),
412 std::string("http://www.example.com/"),
413 __LINE__),
414 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
415 std::string("http://www.ex4mple.com/pr4c71c41"),
416 __LINE__),
417 GurlAndString(GURL("http://www.example.com/0/token/false"),
418 std::string("http://www.example.com/0/token/false"),
419 __LINE__),
420 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
421 std::string("http://www.example.com/index.php"),
422 __LINE__),
423 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
424 std::string("http://www.example.com/index.php"),
425 __LINE__),
426 GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
427 std::string("http://www.example.com/index.php"),
428 __LINE__),
429 GurlAndString(GURL("http://www.example.com:1234/"),
430 std::string("http://www.example.com:1234/"),
431 __LINE__)};
433 for (unsigned int i = 0; i < arraysize(test_values); ++i) {
434 std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
435 EXPECT_EQ(temp, test_values[i].result) <<
436 "Test case #" << i << " line " << test_values[i].line << " failed";
440 TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) {
441 MockURLRequestThrottlerManager manager;
443 manager.CreateEntry(true); // true = Entry is outdated.
444 manager.CreateEntry(true);
445 manager.CreateEntry(true);
446 manager.DoGarbageCollectEntries();
447 EXPECT_EQ(0, manager.GetNumberOfEntries());
449 manager.CreateEntry(false);
450 manager.CreateEntry(false);
451 manager.CreateEntry(false);
452 manager.CreateEntry(true);
453 manager.DoGarbageCollectEntries();
454 EXPECT_EQ(3, manager.GetNumberOfEntries());
457 TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) {
458 MockURLRequestThrottlerManager manager;
460 manager.RegisterRequestUrl(GURL("http://www.example.com/"));
461 manager.RegisterRequestUrl(GURL("http://www.google.com/"));
462 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
463 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
464 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
466 EXPECT_EQ(3, manager.GetNumberOfEntries());
469 void ExpectEntryAllowsAllOnErrorIfOptedOut(
470 net::URLRequestThrottlerEntryInterface* entry,
471 bool opted_out,
472 const URLRequest& request) {
473 EXPECT_FALSE(entry->ShouldRejectRequest(request));
474 MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
475 for (int i = 0; i < 10; ++i) {
476 // Host doesn't really matter in this scenario so we skip it.
477 entry->UpdateWithResponse("", &failure_adapter);
479 EXPECT_NE(opted_out, entry->ShouldRejectRequest(request));
481 if (opted_out) {
482 // We're not mocking out GetTimeNow() in this scenario
483 // so add a 100 ms buffer to avoid flakiness (that should always
484 // give enough time to get from the TimeTicks::Now() call here
485 // to the TimeTicks::Now() call in the entry class).
486 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
487 entry->GetExponentialBackoffReleaseTime());
488 } else {
489 // As above, add 100 ms.
490 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
491 entry->GetExponentialBackoffReleaseTime());
495 TEST_F(URLRequestThrottlerManagerTest, OptOutHeader) {
496 MockURLRequestThrottlerManager manager;
497 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry =
498 manager.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
500 // Fake a response with the opt-out header.
501 MockURLRequestThrottlerHeaderAdapter response_adapter(
503 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue,
504 200);
505 entry->UpdateWithResponse("www.google.com", &response_adapter);
507 // Ensure that the same entry on error always allows everything.
508 ExpectEntryAllowsAllOnErrorIfOptedOut(entry, true, request_);
510 // Ensure that a freshly created entry (for a different URL on an
511 // already opted-out host) also gets "always allow" behavior.
512 scoped_refptr<net::URLRequestThrottlerEntryInterface> other_entry =
513 manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
514 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry, true, request_);
516 // Fake a response with the opt-out header incorrectly specified.
517 scoped_refptr<net::URLRequestThrottlerEntryInterface> no_opt_out_entry =
518 manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
519 MockURLRequestThrottlerHeaderAdapter wrong_adapter("", "yesplease", 200);
520 no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter);
521 ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry, false, request_);
523 // A localhost entry should always be opted out.
524 scoped_refptr<net::URLRequestThrottlerEntryInterface> localhost_entry =
525 manager.RegisterRequestUrl(GURL("http://localhost/hello"));
526 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry, true, request_);
529 TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) {
530 for (int i = 0; i < 3; ++i) {
531 MockURLRequestThrottlerManager manager;
532 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_before =
533 manager.RegisterRequestUrl(GURL("http://www.example.com/"));
534 MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
535 for (int j = 0; j < 10; ++j) {
536 // Host doesn't really matter in this scenario so we skip it.
537 entry_before->UpdateWithResponse("", &failure_adapter);
539 EXPECT_TRUE(entry_before->ShouldRejectRequest(request_));
541 switch (i) {
542 case 0:
543 manager.OnIPAddressChanged();
544 break;
545 case 1:
546 manager.OnConnectionTypeChanged(
547 net::NetworkChangeNotifier::CONNECTION_UNKNOWN);
548 break;
549 case 2:
550 manager.OnConnectionTypeChanged(
551 net::NetworkChangeNotifier::CONNECTION_NONE);
552 break;
553 default:
554 FAIL();
557 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_after =
558 manager.RegisterRequestUrl(GURL("http://www.example.com/"));
559 EXPECT_FALSE(entry_after->ShouldRejectRequest(request_));
563 } // namespace net