1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_throttler_manager.h"
7 #include "base/memory/scoped_ptr.h"
8 #include "base/metrics/histogram.h"
9 #include "base/metrics/statistics_recorder.h"
10 #include "base/pickle.h"
11 #include "base/stringprintf.h"
12 #include "base/string_number_conversions.h"
13 #include "base/time.h"
14 #include "net/base/load_flags.h"
15 #include "net/base/test_completion_callback.h"
16 #include "net/url_request/url_request_context.h"
17 #include "net/url_request/url_request_test_util.h"
18 #include "net/url_request/url_request_throttler_header_interface.h"
19 #include "net/url_request/url_request_throttler_test_support.h"
20 #include "testing/gtest/include/gtest/gtest.h"
22 using base::TimeDelta
;
23 using base::TimeTicks
;
29 using base::Histogram
;
30 using base::StatisticsRecorder
;
32 class MockURLRequestThrottlerEntry
: public URLRequestThrottlerEntry
{
34 explicit MockURLRequestThrottlerEntry(
35 net::URLRequestThrottlerManager
* manager
)
36 : net::URLRequestThrottlerEntry(manager
, ""),
37 mock_backoff_entry_(&backoff_policy_
) {
40 MockURLRequestThrottlerEntry(
41 net::URLRequestThrottlerManager
* manager
,
42 const TimeTicks
& exponential_backoff_release_time
,
43 const TimeTicks
& sliding_window_release_time
,
44 const TimeTicks
& fake_now
)
45 : net::URLRequestThrottlerEntry(manager
, ""),
46 fake_time_now_(fake_now
),
47 mock_backoff_entry_(&backoff_policy_
) {
50 mock_backoff_entry_
.set_fake_now(fake_now
);
51 set_exponential_backoff_release_time(exponential_backoff_release_time
);
52 set_sliding_window_release_time(sliding_window_release_time
);
56 // Some tests become flaky if we have jitter.
57 backoff_policy_
.jitter_factor
= 0.0;
59 // This lets us avoid having to make multiple failures initially (this
60 // logic is already tested in the BackoffEntry unit tests).
61 backoff_policy_
.num_errors_to_ignore
= 0;
64 const BackoffEntry
* GetBackoffEntry() const {
65 return &mock_backoff_entry_
;
68 BackoffEntry
* GetBackoffEntry() {
69 return &mock_backoff_entry_
;
72 static bool ExplicitUserRequest(int load_flags
) {
73 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags
);
76 void ResetToBlank(const TimeTicks
& time_now
) {
77 fake_time_now_
= time_now
;
78 mock_backoff_entry_
.set_fake_now(time_now
);
80 GetBackoffEntry()->Reset();
81 GetBackoffEntry()->SetCustomReleaseTime(time_now
);
82 set_sliding_window_release_time(time_now
);
85 // Overridden for tests.
86 virtual TimeTicks
ImplGetTimeNow() const OVERRIDE
{ return fake_time_now_
; }
88 void set_exponential_backoff_release_time(
89 const base::TimeTicks
& release_time
) {
90 GetBackoffEntry()->SetCustomReleaseTime(release_time
);
93 base::TimeTicks
sliding_window_release_time() const {
94 return URLRequestThrottlerEntry::sliding_window_release_time();
97 void set_sliding_window_release_time(
98 const base::TimeTicks
& release_time
) {
99 URLRequestThrottlerEntry::set_sliding_window_release_time(
103 TimeTicks fake_time_now_
;
104 MockBackoffEntry mock_backoff_entry_
;
107 virtual ~MockURLRequestThrottlerEntry() {}
110 class MockURLRequestThrottlerManager
: public URLRequestThrottlerManager
{
112 MockURLRequestThrottlerManager() : create_entry_index_(0) {}
114 // Method to process the URL using URLRequestThrottlerManager protected
116 std::string
DoGetUrlIdFromUrl(const GURL
& url
) { return GetIdFromUrl(url
); }
118 // Method to use the garbage collecting method of URLRequestThrottlerManager.
119 void DoGarbageCollectEntries() { GarbageCollectEntries(); }
121 // Returns the number of entries in the map.
122 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
124 void CreateEntry(bool is_outdated
) {
125 TimeTicks time
= TimeTicks::Now();
127 time
-= TimeDelta::FromMilliseconds(
128 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
+ 1000);
130 std::string
fake_url_string("http://www.fakeurl.com/");
131 fake_url_string
.append(base::IntToString(create_entry_index_
++));
132 GURL
fake_url(fake_url_string
);
133 OverrideEntryForTests(
135 new MockURLRequestThrottlerEntry(this, time
, TimeTicks::Now(),
140 int create_entry_index_
;
144 TimeAndBool(const TimeTicks
& time_value
, bool expected
, int line_num
) {
154 struct GurlAndString
{
155 GurlAndString(const GURL
& url_value
,
156 const std::string
& expected
,
169 class URLRequestThrottlerEntryTest
: public testing::Test
{
171 URLRequestThrottlerEntryTest() : request_(GURL(), NULL
, &context_
) {
174 virtual void SetUp();
176 // After calling this function, histogram snapshots in |samples_| contain
177 // only the delta caused by the test case currently running.
178 void CalculateHistogramDeltas();
181 MockURLRequestThrottlerManager manager_
; // Dummy object, not used.
182 scoped_refptr
<MockURLRequestThrottlerEntry
> entry_
;
184 std::map
<std::string
, Histogram::SampleSet
> original_samples_
;
185 std::map
<std::string
, Histogram::SampleSet
> samples_
;
187 TestURLRequestContext context_
;
188 TestURLRequest request_
;
191 // List of all histograms we care about in these unit tests.
192 const char* kHistogramNames
[] = {
193 "Throttling.FailureCountAtSuccess",
194 "Throttling.PerceivedDowntime",
195 "Throttling.RequestThrottled",
196 "Throttling.SiteOptedOut",
199 void URLRequestThrottlerEntryTest::SetUp() {
200 request_
.set_load_flags(0);
202 now_
= TimeTicks::Now();
203 entry_
= new MockURLRequestThrottlerEntry(&manager_
);
204 entry_
->ResetToBlank(now_
);
206 for (size_t i
= 0; i
< arraysize(kHistogramNames
); ++i
) {
207 // Must retrieve original samples for each histogram for comparison
208 // as other tests may affect them.
209 const char* name
= kHistogramNames
[i
];
210 Histogram::SampleSet
& original
= original_samples_
[name
];
211 Histogram
* histogram
;
212 if (StatisticsRecorder::FindHistogram(name
, &histogram
)) {
213 histogram
->SnapshotSample(&original
);
218 void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
219 for (size_t i
= 0; i
< arraysize(kHistogramNames
); ++i
) {
220 const char* name
= kHistogramNames
[i
];
221 Histogram::SampleSet
& original
= original_samples_
[name
];
222 Histogram::SampleSet
& sample
= samples_
[name
];
224 Histogram
* histogram
;
225 if (StatisticsRecorder::FindHistogram(name
, &histogram
)) {
226 ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag
, histogram
->flags());
228 histogram
->SnapshotSample(&sample
);
229 // Ensure |original| size is same as |sample|, then subtract original
231 original
.Resize(*histogram
);
232 sample
.Subtract(original
);
236 // Ensure we don't accidentally use the originals in our tests.
237 original_samples_
.clear();
240 std::ostream
& operator<<(std::ostream
& out
, const base::TimeTicks
& time
) {
241 return out
<< time
.ToInternalValue();
244 TEST_F(URLRequestThrottlerEntryTest
, InterfaceDuringExponentialBackoff
) {
245 entry_
->set_exponential_backoff_release_time(
246 entry_
->fake_time_now_
+ TimeDelta::FromMilliseconds(1));
247 EXPECT_TRUE(entry_
->ShouldRejectRequest(request_
));
249 // Also end-to-end test the load flags exceptions.
250 request_
.set_load_flags(LOAD_MAYBE_USER_GESTURE
);
251 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
253 CalculateHistogramDeltas();
254 ASSERT_EQ(1, samples_
["Throttling.RequestThrottled"].counts(0));
255 ASSERT_EQ(1, samples_
["Throttling.RequestThrottled"].counts(1));
258 TEST_F(URLRequestThrottlerEntryTest
, InterfaceNotDuringExponentialBackoff
) {
259 entry_
->set_exponential_backoff_release_time(entry_
->fake_time_now_
);
260 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
261 entry_
->set_exponential_backoff_release_time(
262 entry_
->fake_time_now_
- TimeDelta::FromMilliseconds(1));
263 EXPECT_FALSE(entry_
->ShouldRejectRequest(request_
));
265 CalculateHistogramDeltas();
266 ASSERT_EQ(2, samples_
["Throttling.RequestThrottled"].counts(0));
267 ASSERT_EQ(0, samples_
["Throttling.RequestThrottled"].counts(1));
270 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateFailure
) {
271 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
272 entry_
->UpdateWithResponse("", &failure_response
);
273 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
274 << "A failure should increase the release_time";
277 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccess
) {
278 MockURLRequestThrottlerHeaderAdapter
success_response(200);
279 entry_
->UpdateWithResponse("", &success_response
);
280 EXPECT_EQ(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
281 << "A success should not add any delay";
284 TEST_F(URLRequestThrottlerEntryTest
, InterfaceUpdateSuccessThenFailure
) {
285 MockURLRequestThrottlerHeaderAdapter
failure_response(503);
286 MockURLRequestThrottlerHeaderAdapter
success_response(200);
287 entry_
->UpdateWithResponse("", &success_response
);
288 entry_
->UpdateWithResponse("", &failure_response
);
289 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), entry_
->fake_time_now_
)
290 << "This scenario should add delay";
291 entry_
->UpdateWithResponse("", &success_response
);
294 TEST_F(URLRequestThrottlerEntryTest
, IsEntryReallyOutdated
) {
295 TimeDelta lifetime
= TimeDelta::FromMilliseconds(
296 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs
);
297 const TimeDelta kFiveMs
= TimeDelta::FromMilliseconds(5);
299 TimeAndBool test_values
[] = {
300 TimeAndBool(now_
, false, __LINE__
),
301 TimeAndBool(now_
- kFiveMs
, false, __LINE__
),
302 TimeAndBool(now_
+ kFiveMs
, false, __LINE__
),
303 TimeAndBool(now_
- (lifetime
- kFiveMs
), false, __LINE__
),
304 TimeAndBool(now_
- lifetime
, true, __LINE__
),
305 TimeAndBool(now_
- (lifetime
+ kFiveMs
), true, __LINE__
)};
307 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
308 entry_
->set_exponential_backoff_release_time(test_values
[i
].time
);
309 EXPECT_EQ(entry_
->IsEntryOutdated(), test_values
[i
].result
) <<
310 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
314 TEST_F(URLRequestThrottlerEntryTest
, MaxAllowedBackoff
) {
315 for (int i
= 0; i
< 30; ++i
) {
316 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
317 entry_
->UpdateWithResponse("", &response_adapter
);
320 TimeDelta delay
= entry_
->GetExponentialBackoffReleaseTime() - now_
;
321 EXPECT_EQ(delay
.InMilliseconds(),
322 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs
);
325 TEST_F(URLRequestThrottlerEntryTest
, MalformedContent
) {
326 MockURLRequestThrottlerHeaderAdapter
response_adapter(503);
327 for (int i
= 0; i
< 5; ++i
)
328 entry_
->UpdateWithResponse("", &response_adapter
);
330 TimeTicks release_after_failures
= entry_
->GetExponentialBackoffReleaseTime();
332 // Inform the entry that a response body was malformed, which is supposed to
333 // increase the back-off time. Note that we also submit a successful
334 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
335 // is what happens in practice (if a body is received, then a non-500
336 // response must also have been received).
337 entry_
->ReceivedContentWasMalformed(200);
338 MockURLRequestThrottlerHeaderAdapter
success_adapter(200);
339 entry_
->UpdateWithResponse("", &success_adapter
);
340 EXPECT_GT(entry_
->GetExponentialBackoffReleaseTime(), release_after_failures
);
343 TEST_F(URLRequestThrottlerEntryTest
, SlidingWindow
) {
344 int max_send
= URLRequestThrottlerEntry::kDefaultMaxSendThreshold
;
346 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs
;
348 TimeTicks time_1
= entry_
->fake_time_now_
+
349 TimeDelta::FromMilliseconds(sliding_window
/ 3);
350 TimeTicks time_2
= entry_
->fake_time_now_
+
351 TimeDelta::FromMilliseconds(2 * sliding_window
/ 3);
352 TimeTicks time_3
= entry_
->fake_time_now_
+
353 TimeDelta::FromMilliseconds(sliding_window
);
354 TimeTicks time_4
= entry_
->fake_time_now_
+
355 TimeDelta::FromMilliseconds(sliding_window
+ 2 * sliding_window
/ 3);
357 entry_
->set_exponential_backoff_release_time(time_1
);
359 for (int i
= 0; i
< max_send
/ 2; ++i
) {
360 EXPECT_EQ(2 * sliding_window
/ 3,
361 entry_
->ReserveSendingTimeForNextRequest(time_2
));
363 EXPECT_EQ(time_2
, entry_
->sliding_window_release_time());
365 entry_
->fake_time_now_
= time_3
;
367 for (int i
= 0; i
< (max_send
+ 1) / 2; ++i
)
368 EXPECT_EQ(0, entry_
->ReserveSendingTimeForNextRequest(TimeTicks()));
370 EXPECT_EQ(time_4
, entry_
->sliding_window_release_time());
373 TEST_F(URLRequestThrottlerEntryTest
, ExplicitUserRequest
) {
374 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0));
375 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
376 LOAD_MAYBE_USER_GESTURE
));
377 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(
378 ~LOAD_MAYBE_USER_GESTURE
));
381 class URLRequestThrottlerManagerTest
: public testing::Test
{
383 URLRequestThrottlerManagerTest()
384 : request_(GURL(), NULL
, &context_
) {
387 virtual void SetUp() {
388 request_
.set_load_flags(0);
391 // context_ must be declared before request_.
392 TestURLRequestContext context_
;
393 TestURLRequest request_
;
396 TEST_F(URLRequestThrottlerManagerTest
, IsUrlStandardised
) {
397 MockURLRequestThrottlerManager manager
;
398 GurlAndString test_values
[] = {
399 GurlAndString(GURL("http://www.example.com"),
400 std::string("http://www.example.com/"),
402 GurlAndString(GURL("http://www.Example.com"),
403 std::string("http://www.example.com/"),
405 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
406 std::string("http://www.ex4mple.com/pr4c71c41"),
408 GurlAndString(GURL("http://www.example.com/0/token/false"),
409 std::string("http://www.example.com/0/token/false"),
411 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
412 std::string("http://www.example.com/index.php"),
414 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
415 std::string("http://www.example.com/index.php"),
417 GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
418 std::string("http://www.example.com/index.php"),
420 GurlAndString(GURL("http://www.example.com:1234/"),
421 std::string("http://www.example.com:1234/"),
424 for (unsigned int i
= 0; i
< arraysize(test_values
); ++i
) {
425 std::string temp
= manager
.DoGetUrlIdFromUrl(test_values
[i
].url
);
426 EXPECT_EQ(temp
, test_values
[i
].result
) <<
427 "Test case #" << i
<< " line " << test_values
[i
].line
<< " failed";
431 TEST_F(URLRequestThrottlerManagerTest
, AreEntriesBeingCollected
) {
432 MockURLRequestThrottlerManager manager
;
434 manager
.CreateEntry(true); // true = Entry is outdated.
435 manager
.CreateEntry(true);
436 manager
.CreateEntry(true);
437 manager
.DoGarbageCollectEntries();
438 EXPECT_EQ(0, manager
.GetNumberOfEntries());
440 manager
.CreateEntry(false);
441 manager
.CreateEntry(false);
442 manager
.CreateEntry(false);
443 manager
.CreateEntry(true);
444 manager
.DoGarbageCollectEntries();
445 EXPECT_EQ(3, manager
.GetNumberOfEntries());
448 TEST_F(URLRequestThrottlerManagerTest
, IsHostBeingRegistered
) {
449 MockURLRequestThrottlerManager manager
;
451 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
452 manager
.RegisterRequestUrl(GURL("http://www.google.com/"));
453 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
454 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
455 manager
.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
457 EXPECT_EQ(3, manager
.GetNumberOfEntries());
460 void ExpectEntryAllowsAllOnErrorIfOptedOut(
461 net::URLRequestThrottlerEntryInterface
* entry
,
463 const URLRequest
& request
) {
464 EXPECT_FALSE(entry
->ShouldRejectRequest(request
));
465 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
466 for (int i
= 0; i
< 10; ++i
) {
467 // Host doesn't really matter in this scenario so we skip it.
468 entry
->UpdateWithResponse("", &failure_adapter
);
470 EXPECT_NE(opted_out
, entry
->ShouldRejectRequest(request
));
473 // We're not mocking out GetTimeNow() in this scenario
474 // so add a 100 ms buffer to avoid flakiness (that should always
475 // give enough time to get from the TimeTicks::Now() call here
476 // to the TimeTicks::Now() call in the entry class).
477 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
478 entry
->GetExponentialBackoffReleaseTime());
480 // As above, add 100 ms.
481 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
482 entry
->GetExponentialBackoffReleaseTime());
486 TEST_F(URLRequestThrottlerManagerTest
, OptOutHeader
) {
487 MockURLRequestThrottlerManager manager
;
488 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry
=
489 manager
.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
491 // Fake a response with the opt-out header.
492 MockURLRequestThrottlerHeaderAdapter
response_adapter(
494 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue
,
496 entry
->UpdateWithResponse("www.google.com", &response_adapter
);
498 // Ensure that the same entry on error always allows everything.
499 ExpectEntryAllowsAllOnErrorIfOptedOut(entry
, true, request_
);
501 // Ensure that a freshly created entry (for a different URL on an
502 // already opted-out host) also gets "always allow" behavior.
503 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> other_entry
=
504 manager
.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
505 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry
, true, request_
);
507 // Fake a response with the opt-out header incorrectly specified.
508 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> no_opt_out_entry
=
509 manager
.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
510 MockURLRequestThrottlerHeaderAdapter
wrong_adapter("", "yesplease", 200);
511 no_opt_out_entry
->UpdateWithResponse("www.nike.com", &wrong_adapter
);
512 ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry
, false, request_
);
514 // A localhost entry should always be opted out.
515 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> localhost_entry
=
516 manager
.RegisterRequestUrl(GURL("http://localhost/hello"));
517 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry
, true, request_
);
520 TEST_F(URLRequestThrottlerManagerTest
, ClearOnNetworkChange
) {
521 for (int i
= 0; i
< 3; ++i
) {
522 MockURLRequestThrottlerManager manager
;
523 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry_before
=
524 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
525 MockURLRequestThrottlerHeaderAdapter
failure_adapter(503);
526 for (int j
= 0; j
< 10; ++j
) {
527 // Host doesn't really matter in this scenario so we skip it.
528 entry_before
->UpdateWithResponse("", &failure_adapter
);
530 EXPECT_TRUE(entry_before
->ShouldRejectRequest(request_
));
534 manager
.OnIPAddressChanged();
537 manager
.OnConnectionTypeChanged(
538 net::NetworkChangeNotifier::CONNECTION_UNKNOWN
);
541 manager
.OnConnectionTypeChanged(
542 net::NetworkChangeNotifier::CONNECTION_NONE
);
548 scoped_refptr
<net::URLRequestThrottlerEntryInterface
> entry_after
=
549 manager
.RegisterRequestUrl(GURL("http://www.example.com/"));
550 EXPECT_FALSE(entry_after
->ShouldRejectRequest(request_
));