1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_throttler_manager.h"
7 #include "base/memory/scoped_ptr.h"
8 #include "base/metrics/histogram.h"
9 #include "base/metrics/histogram_samples.h"
10 #include "base/metrics/statistics_recorder.h"
11 #include "base/pickle.h"
12 #include "base/stl_util.h"
13 #include "base/stringprintf.h"
14 #include "base/string_number_conversions.h"
15 #include "base/time.h"
16 #include "net/base/load_flags.h"
17 #include "net/base/test_completion_callback.h"
18 #include "net/url_request/url_request_context.h"
19 #include "net/url_request/url_request_test_util.h"
20 #include "net/url_request/url_request_throttler_header_interface.h"
21 #include "net/url_request/url_request_throttler_test_support.h"
22 #include "testing/gtest/include/gtest/gtest.h"
24 using base::TimeDelta
;
25 using base::TimeTicks
;
31 using base::Histogram
;
32 using base::HistogramSamples
;
33 using base::StatisticsRecorder
;
35 class MockURLRequestThrottlerEntry
: public URLRequestThrottlerEntry
{
37 explicit MockURLRequestThrottlerEntry(
38 net::URLRequestThrottlerManager
* manager
)
39 : net::URLRequestThrottlerEntry(manager
, ""),
40 mock_backoff_entry_(&backoff_policy_
) {
43 MockURLRequestThrottlerEntry(
44 net::URLRequestThrottlerManager
* manager
,
45 const TimeTicks
& exponential_backoff_release_time
,
46 const TimeTicks
& sliding_window_release_time
,
47 const TimeTicks
& fake_now
)
48 : net::URLRequestThrottlerEntry(manager
, ""),
49 fake_time_now_(fake_now
),
50 mock_backoff_entry_(&backoff_policy_
) {
53 mock_backoff_entry_
.set_fake_now(fake_now
);
54 set_exponential_backoff_release_time(exponential_backoff_release_time
);
55 set_sliding_window_release_time(sliding_window_release_time
);
59 // Some tests become flaky if we have jitter.
60 backoff_policy_
.jitter_factor
= 0.0;
62 // This lets us avoid having to make multiple failures initially (this
63 // logic is already tested in the BackoffEntry unit tests).
64 backoff_policy_
.num_errors_to_ignore
= 0;
67 const BackoffEntry
* GetBackoffEntry() const {
68 return &mock_backoff_entry_
;
71 BackoffEntry
* GetBackoffEntry() {
72 return &mock_backoff_entry_
;
75 static bool ExplicitUserRequest(int load_flags
) {
76 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags
);
79 void ResetToBlank(const TimeTicks
& time_now
) {
80 fake_time_now_
= time_now
;
81 mock_backoff_entry_
.set_fake_now(time_now
);
83 GetBackoffEntry()->Reset();
84 GetBackoffEntry()->SetCustomReleaseTime(time_now
);
85 set_sliding_window_release_time(time_now
);
88 // Overridden for tests.
89 virtual TimeTicks
ImplGetTimeNow() const OVERRIDE
{ return fake_time_now_
; }
91 void set_exponential_backoff_release_time(
92 const base::TimeTicks
& release_time
) {
93 GetBackoffEntry()->SetCustomReleaseTime(release_time
);
96 base::TimeTicks
sliding_window_release_time() const {
97 return URLRequestThrottlerEntry::sliding_window_release_time();
100 void set_sliding_window_release_time(
101 const base::TimeTicks
& release_time
) {
102 URLRequestThrottlerEntry::set_sliding_window_release_time(
106 TimeTicks fake_time_now_
;
107 MockBackoffEntry mock_backoff_entry_
;
110 virtual ~MockURLRequestThrottlerEntry() {}
113 class MockURLRequestThrottlerManager
: public URLRequestThrottlerManager
{
115 MockURLRequestThrottlerManager() : create_entry_index_(0) {}
117 // Method to process the URL using URLRequestThrottlerManager protected
119 std::string
DoGetUrlIdFromUrl(const GURL
& url
) { return GetIdFromUrl(url
); }
121 // Method to use the garbage collecting method of URLRequestThrottlerManager.
122 void DoGarbageCollectEntries() { GarbageCollectEntries(); }
124 // Returns the number of entries in the map.
125 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
127 void CreateEntry(bool is_outdated
) {
128 TimeTicks time
= TimeTicks::Now();
130 time
-= TimeDelta::FromMilliseconds(
131 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
+ 1000);
133 std::string
fake_url_string("http://www.fakeurl.com/");
134 fake_url_string
.append(base::IntToString(create_entry_index_
++));
135 GURL
fake_url(fake_url_string
);
136 OverrideEntryForTests(
138 new MockURLRequestThrottlerEntry(this, time
, TimeTicks::Now(),
143 int create_entry_index_
;
147 TimeAndBool(const TimeTicks
& time_value
, bool expected
, int line_num
) {
157 struct GurlAndString
{
158 GurlAndString(const GURL
& url_value
,
159 const std::string
& expected
,
172 class URLRequestThrottlerEntryTest
: public testing::Test
{
174 URLRequestThrottlerEntryTest() : request_(GURL(), NULL
, &context_
) {
177 virtual void SetUp();
178 virtual void TearDown();
180 // After calling this function, histogram snapshots in |samples_| contain
181 // only the delta caused by the test case currently running.
182 void CalculateHistogramDeltas();
185 MockURLRequestThrottlerManager manager_
; // Dummy object, not used.
186 scoped_refptr
<MockURLRequestThrottlerEntry
> entry_
;
188 std::map
<std::string
, HistogramSamples
*> original_samples_
;
189 std::map
<std::string
, HistogramSamples
*> samples_
;
191 TestURLRequestContext context_
;
192 TestURLRequest request_
;
195 // List of all histograms we care about in these unit tests.
196 const char* kHistogramNames
[] = {
197 "Throttling.FailureCountAtSuccess",
198 "Throttling.PerceivedDowntime",
199 "Throttling.RequestThrottled",
200 "Throttling.SiteOptedOut",
203 void URLRequestThrottlerEntryTest::SetUp() {
204 request_
.set_load_flags(0);
206 now_
= TimeTicks::Now();
207 entry_
= new MockURLRequestThrottlerEntry(&manager_
);
208 entry_
->ResetToBlank(now_
);
210 for (size_t i
= 0; i
< arraysize(kHistogramNames
); ++i
) {
211 // Must retrieve original samples for each histogram for comparison
212 // as other tests may affect them.
213 const char* name
= kHistogramNames
[i
];
214 Histogram
* histogram
= StatisticsRecorder::FindHistogram(name
);
216 original_samples_
[name
] = histogram
->SnapshotSamples().release();
218 original_samples_
[name
] = NULL
;
223 void URLRequestThrottlerEntryTest::TearDown() {
224 STLDeleteValues(&original_samples_
);
225 STLDeleteValues(&samples_
);
228 void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
229 for (size_t i
= 0; i
< arraysize(kHistogramNames
); ++i
) {
230 const char* name
= kHistogramNames
[i
];
231 HistogramSamples
* original
= original_samples_
[name
];
233 Histogram
* histogram
= StatisticsRecorder::FindHistogram(name
);
235 ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag
, histogram
->flags());
237 scoped_ptr
<HistogramSamples
> samples(histogram
->SnapshotSamples());
239 samples
->Subtract(*original
);
240 samples_
[name
] = samples
.release();
244 // Ensure we don't accidentally use the originals in our tests.
245 STLDeleteValues(&original_samples_
);
246 original_samples_
.clear();
249 std::ostream
& operator<<(std::ostream
& out
, const base::TimeTicks
& time
) {
250 return out
<< time
.ToInternalValue();
253 TEST_F(URLRequestThrottlerEntryTest
, InterfaceDuringExponentialBackoff
) {
254 entry_
->set_exponential_backoff_release_time(
255 entry_
->fake_time_now_
+ TimeDelta::FromMilliseconds(1));
256 EXPECT_TRUE(entry_
->ShouldRejectRequest(request_
));
258 // Also end-to-end test the load flags exceptions.
259 request_
.set_load_flags(LOAD_MAYBE_USER_GESTURE
);
260 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
262 CalculateHistogramDeltas();
263 ASSERT_EQ(1, samples_
["Throttling.RequestThrottled"]->GetCount(0));
264 ASSERT_EQ(1, samples_
["Throttling.RequestThrottled"]->GetCount(1));
267 TEST_F(URLRequestThrottlerEntryTest
, InterfaceNotDuringExponentialBackoff
) {
268 entry_
->set_exponential_backoff_release_time(entry_
->fake_time_now_
);
269 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
270 entry_
->set_exponential_backoff_release_time(
271 entry_
->fake_time_now_
- TimeDelta::FromMilliseconds(1));
272 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
274 CalculateHistogramDeltas();
275 ASSERT_EQ(2, samples_
["Throttling.RequestThrottled"]->GetCount(0));
276 ASSERT_EQ(0, samples_
["Throttling.RequestThrottled"]->GetCount(1));
279 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateFailure
) {
280 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
281 entry_
->UpdateWithResponse("", &failure_response
);
282 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
283 << "A failure should increase the release_time";
286 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccess
) {
287 MockURLRequestThrottlerHeaderAdapter
success_response(200);
288 entry_
->UpdateWithResponse("", &success_response
);
289 EXPECT_EQ(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
290 << "A success should not add any delay";
293 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccessThenFailure
) {
294 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
295 MockURLRequestThrottlerHeaderAdapter
success_response(200);
296 entry_
->UpdateWithResponse("", &success_response
);
297 entry_
->UpdateWithResponse("", &failure_response
);
298 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
299 << "This scenario should add delay";
300 entry_
->UpdateWithResponse("", &success_response
);
303 TEST_F(URLRequestThrottlerEntryTest
, IsEntryReallyOutdated
) {
304 TimeDelta lifetime
= TimeDelta::FromMilliseconds(
305 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
);
306 const TimeDelta kFiveMs
= TimeDelta::FromMilliseconds(5);
308 TimeAndBool test_values
[] = {
309 TimeAndBool(now_
, false, __LINE__
),
310 TimeAndBool(now_
- kFiveMs
, false, __LINE__
),
311 TimeAndBool(now_
+ kFiveMs
, false, __LINE__
),
312 TimeAndBool(now_
- (lifetime
- kFiveMs
), false, __LINE__
),
313 TimeAndBool(now_
- lifetime
, true, __LINE__
),
314 TimeAndBool(now_
- (lifetime
+ kFiveMs
), true, __LINE__
)};
316 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
317 entry_
->set_exponential_backoff_release_time(test_values
[i
].time
);
318 EXPECT_EQ(entry_
->IsEntryOutdated(), test_values
[i
].result
) <<
319 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
323 TEST_F(URLRequestThrottlerEntryTest
, MaxAllowedBackoff
) {
324 for (int i
= 0; i
< 30; ++i
) {
325 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
326 entry_
->UpdateWithResponse("", &response_adapter
);
329 TimeDelta delay
= entry_
->GetExponentialBackoffReleaseTime() - now_
;
330 EXPECT_EQ(delay
.InMilliseconds(),
331 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs
);
334 TEST_F(URLRequestThrottlerEntryTest
, MalformedContent
) {
335 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
336 for (int i
= 0; i
< 5; ++i
)
337 entry_
->UpdateWithResponse("", &response_adapter
);
339 TimeTicks release_after_failures
= entry_
->GetExponentialBackoffReleaseTime();
341 // Inform the entry that a response body was malformed, which is supposed to
342 // increase the back-off time. Note that we also submit a successful
343 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
344 // is what happens in practice (if a body is received, then a non-500
345 // response must also have been received).
346 entry_
->ReceivedContentWasMalformed(200);
347 MockURLRequestThrottlerHeaderAdapter
success_adapter(200);
348 entry_
->UpdateWithResponse("", &success_adapter
);
349 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), release_after_failures
);
352 TEST_F(URLRequestThrottlerEntryTest
, SlidingWindow
) {
353 int max_send
= URLRequestThrottlerEntry::kDefaultMaxSendThreshold
;
355 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs
;
357 TimeTicks time_1
= entry_
->fake_time_now_
+
358 TimeDelta::FromMilliseconds(sliding_window
/ 3);
359 TimeTicks time_2
= entry_
->fake_time_now_
+
360 TimeDelta::FromMilliseconds(2 * sliding_window
/ 3);
361 TimeTicks time_3
= entry_
->fake_time_now_
+
362 TimeDelta::FromMilliseconds(sliding_window
);
363 TimeTicks time_4
= entry_
->fake_time_now_
+
364 TimeDelta::FromMilliseconds(sliding_window
+ 2 * sliding_window
/ 3);
366 entry_
->set_exponential_backoff_release_time(time_1
);
368 for (int i
= 0; i
< max_send
/ 2; ++i
) {
369 EXPECT_EQ(2 * sliding_window
/ 3,
370 entry_
->ReserveSendingTimeForNextRequest(time_2
));
372 EXPECT_EQ(time_2
, entry_
->sliding_window_release_time());
374 entry_
->fake_time_now_
= time_3
;
376 for (int i
= 0; i
< (max_send
+ 1) / 2; ++i
)
377 EXPECT_EQ(0, entry_
->ReserveSendingTimeForNextRequest(TimeTicks()));
379 EXPECT_EQ(time_4
, entry_
->sliding_window_release_time());
382 TEST_F(URLRequestThrottlerEntryTest
, ExplicitUserRequest
) {
383 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
384 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
385 LOAD_MAYBE_USER_GESTURE
));
386 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
387 ~LOAD_MAYBE_USER_GESTURE
));
390 class URLRequestThrottlerManagerTest
: public testing::Test
{
392 URLRequestThrottlerManagerTest()
393 : request_(GURL(), NULL
, &context_
) {
396 virtual void SetUp() {
397 request_
.set_load_flags(0);
400 // context_ must be declared before request_.
401 TestURLRequestContext context_
;
402 TestURLRequest request_
;
405 TEST_F(URLRequestThrottlerManagerTest
, IsUrlStandardised
) {
406 MockURLRequestThrottlerManager manager
;
407 GurlAndString test_values
[] = {
408 GurlAndString(GURL("http://www.example.com"),
409 std::string("http://www.example.com/"),
411 GurlAndString(GURL("http://www.Example.com"),
412 std::string("http://www.example.com/"),
414 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
415 std::string("http://www.ex4mple.com/pr4c71c41"),
417 GurlAndString(GURL("http://www.example.com/0/token/false"),
418 std::string("http://www.example.com/0/token/false"),
420 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
421 std::string("http://www.example.com/index.php"),
423 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
424 std::string("http://www.example.com/index.php"),
426 GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
427 std::string("http://www.example.com/index.php"),
429 GurlAndString(GURL("http://www.example.com:1234/"),
430 std::string("http://www.example.com:1234/"),
433 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
434 std::string temp
= manager
.DoGetUrlIdFromUrl(test_values
[i
].url
);
435 EXPECT_EQ(temp
, test_values
[i
].result
) <<
436 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
440 TEST_F(URLRequestThrottlerManagerTest
, AreEntriesBeingCollected
) {
441 MockURLRequestThrottlerManager manager
;
443 manager
.CreateEntry(true); // true = Entry is outdated.
444 manager
.CreateEntry(true);
445 manager
.CreateEntry(true);
446 manager
.DoGarbageCollectEntries();
447 EXPECT_EQ(0, manager
.GetNumberOfEntries());
449 manager
.CreateEntry(false);
450 manager
.CreateEntry(false);
451 manager
.CreateEntry(false);
452 manager
.CreateEntry(true);
453 manager
.DoGarbageCollectEntries();
454 EXPECT_EQ(3, manager
.GetNumberOfEntries());
457 TEST_F(URLRequestThrottlerManagerTest
, IsHostBeingRegistered
) {
458 MockURLRequestThrottlerManager manager
;
460 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
461 manager
.RegisterRequestUrl(GURL("http://www.google.com/"));
462 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
463 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
464 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
466 EXPECT_EQ(3, manager
.GetNumberOfEntries());
469 void ExpectEntryAllowsAllOnErrorIfOptedOut(
470 net::URLRequestThrottlerEntryInterface
* entry
,
472 const URLRequest
& request
) {
473 EXPECT_FALSE(entry
->ShouldRejectRequest(request
));
474 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
475 for (int i
= 0; i
< 10; ++i
) {
476 // Host doesn't really matter in this scenario so we skip it.
477 entry
->UpdateWithResponse("", &failure_adapter
);
479 EXPECT_NE(opted_out
, entry
->ShouldRejectRequest(request
));
482 // We're not mocking out GetTimeNow() in this scenario
483 // so add a 100 ms buffer to avoid flakiness (that should always
484 // give enough time to get from the TimeTicks::Now() call here
485 // to the TimeTicks::Now() call in the entry class).
486 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
487 entry
->GetExponentialBackoffReleaseTime());
489 // As above, add 100 ms.
490 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
491 entry
->GetExponentialBackoffReleaseTime());
495 TEST_F(URLRequestThrottlerManagerTest
, OptOutHeader
) {
496 MockURLRequestThrottlerManager manager
;
497 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry
=
498 manager
.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
500 // Fake a response with the opt-out header.
501 MockURLRequestThrottlerHeaderAdapter
response_adapter(
503 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue
,
505 entry
->UpdateWithResponse("www.google.com", &response_adapter
);
507 // Ensure that the same entry on error always allows everything.
508 ExpectEntryAllowsAllOnErrorIfOptedOut(entry
, true, request_
);
510 // Ensure that a freshly created entry (for a different URL on an
511 // already opted-out host) also gets "always allow" behavior.
512 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> other_entry
=
513 manager
.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
514 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry
, true, request_
);
516 // Fake a response with the opt-out header incorrectly specified.
517 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> no_opt_out_entry
=
518 manager
.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
519 MockURLRequestThrottlerHeaderAdapter
wrong_adapter("", "yesplease", 200);
520 no_opt_out_entry
->UpdateWithResponse("www.nike.com", &wrong_adapter
);
521 ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry
, false, request_
);
523 // A localhost entry should always be opted out.
524 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> localhost_entry
=
525 manager
.RegisterRequestUrl(GURL("http://localhost/hello"));
526 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry
, true, request_
);
529 TEST_F(URLRequestThrottlerManagerTest
, ClearOnNetworkChange
) {
530 for (int i
= 0; i
< 3; ++i
) {
531 MockURLRequestThrottlerManager manager
;
532 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry_before
=
533 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
534 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
535 for (int j
= 0; j
< 10; ++j
) {
536 // Host doesn't really matter in this scenario so we skip it.
537 entry_before
->UpdateWithResponse("", &failure_adapter
);
539 EXPECT_TRUE(entry_before
->ShouldRejectRequest(request_
));
543 manager
.OnIPAddressChanged();
546 manager
.OnConnectionTypeChanged(
547 net::NetworkChangeNotifier::CONNECTION_UNKNOWN
);
550 manager
.OnConnectionTypeChanged(
551 net::NetworkChangeNotifier::CONNECTION_NONE
);
557 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry_after
=
558 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
559 EXPECT_FALSE(entry_after
->ShouldRejectRequest(request_
));