components/suggestions/suggestions_service.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/suggestions/suggestions_service.h"
   6
   7 #include <string>
   8
   9 #include "base/location.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/metrics/sparse_histogram.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_util.h"
  16 #include "base/thread_task_runner_handle.h"
  17 #include "base/time/time.h"
  18 #include "components/pref_registry/pref_registry_syncable.h"
  19 #include "components/suggestions/blacklist_store.h"
  20 #include "components/suggestions/suggestions_store.h"
  21 #include "components/variations/net/variations_http_header_provider.h"
  22 #include "net/base/escape.h"
  23 #include "net/base/load_flags.h"
  24 #include "net/base/net_errors.h"
  25 #include "net/base/url_util.h"
  26 #include "net/http/http_response_headers.h"
  27 #include "net/http/http_status_code.h"
  28 #include "net/http/http_util.h"
  29 #include "net/url_request/url_fetcher.h"
  30 #include "net/url_request/url_request_status.h"
  31 #include "url/gurl.h"
  32
  33 using base::CancelableClosure;
  34 using base::TimeDelta;
  35 using base::TimeTicks;
  36
  37 namespace suggestions {
  38
  39 namespace {
  40
  41 // Used to UMA log the state of the last response from the server.
  42 enum SuggestionsResponseState {
  43   RESPONSE_EMPTY,
  44   RESPONSE_INVALID,
  45   RESPONSE_VALID,
  46   RESPONSE_STATE_SIZE
  47 };
  48
  49 // Will log the supplied response |state|.
  50 void LogResponseState(SuggestionsResponseState state) {
  51   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
  52                             RESPONSE_STATE_SIZE);
  53 }
  54
  55 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
  56                               const GURL& candidate_url) {
  57   return GURL(blacklist_url_prefix +
  58               net::EscapeQueryParamValue(candidate_url.spec(), true));
  59 }
  60
  61 // Runs each callback in |requestors| on |suggestions|, then deallocates
  62 // |requestors|.
  63 void DispatchRequestsAndClear(
  64     const SuggestionsProfile& suggestions,
  65     std::vector<SuggestionsService::ResponseCallback>* requestors) {
  66   std::vector<SuggestionsService::ResponseCallback> temp_requestors;
  67   temp_requestors.swap(*requestors);
  68   std::vector<SuggestionsService::ResponseCallback>::iterator it;
  69   for (it = temp_requestors.begin(); it != temp_requestors.end(); ++it) {
  70     if (!it->is_null()) it->Run(suggestions);
  71   }
  72 }
  73
  74 // Default delay used when scheduling a request.
  75 const int kDefaultSchedulingDelaySec = 1;
  76
  77 // Multiplier on the delay used when re-scheduling a failed request.
  78 const int kSchedulingBackoffMultiplier = 2;
  79
  80 // Maximum valid delay for scheduling a request. Candidate delays larger than
  81 // this are rejected. This means the maximum backoff is at least 5 / 2 minutes.
  82 const int kSchedulingMaxDelaySec = 5 * 60;
  83
  84 }  // namespace
  85
  86 // TODO(mathp): Put this in TemplateURL.
  87 const char kSuggestionsURL[] = "https://www.google.com/chromesuggestions?t=2";
  88 const char kSuggestionsBlacklistURLPrefix[] =
  89     "https://www.google.com/chromesuggestions/blacklist?t=2&url=";
  90 const char kSuggestionsBlacklistURLParam[] = "url";
  91
  92 // The default expiry timeout is 72 hours.
  93 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
  94
  95 SuggestionsService::SuggestionsService(
  96     net::URLRequestContextGetter* url_request_context,
  97     scoped_ptr<SuggestionsStore> suggestions_store,
  98     scoped_ptr<ImageManager> thumbnail_manager,
  99     scoped_ptr<BlacklistStore> blacklist_store)
 100     : url_request_context_(url_request_context),
 101       suggestions_store_(suggestions_store.Pass()),
 102       thumbnail_manager_(thumbnail_manager.Pass()),
 103       blacklist_store_(blacklist_store.Pass()),
 104       scheduling_delay_(TimeDelta::FromSeconds(kDefaultSchedulingDelaySec)),
 105       suggestions_url_(kSuggestionsURL),
 106       blacklist_url_prefix_(kSuggestionsBlacklistURLPrefix),
 107       weak_ptr_factory_(this) {}
 108
 109 SuggestionsService::~SuggestionsService() {}
 110
 111 void SuggestionsService::FetchSuggestionsData(
 112     SyncState sync_state,
 113     SuggestionsService::ResponseCallback callback) {
 114   DCHECK(thread_checker_.CalledOnValidThread());
 115   waiting_requestors_.push_back(callback);
 116   if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
 117     // Cancel any ongoing request, to stop interacting with the server.
 118     pending_request_.reset(NULL);
 119     suggestions_store_->ClearSuggestions();
 120     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
 121   } else if (sync_state == INITIALIZED_ENABLED_HISTORY ||
 122              sync_state == NOT_INITIALIZED_ENABLED) {
 123     // Sync is enabled. Serve previously cached suggestions if available, else
 124     // an empty set of suggestions.
 125     ServeFromCache();
 126
 127     // Issue a network request to refresh the suggestions in the cache.
 128     IssueRequestIfNoneOngoing(suggestions_url_);
 129   } else {
 130     NOTREACHED();
 131   }
 132 }
 133
 134 void SuggestionsService::GetPageThumbnail(
 135     const GURL& url,
 136     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
 137   thumbnail_manager_->GetImageForURL(url, callback);
 138 }
 139
 140 void SuggestionsService::BlacklistURL(
 141     const GURL& candidate_url,
 142     const SuggestionsService::ResponseCallback& callback,
 143     const base::Closure& fail_callback) {
 144   DCHECK(thread_checker_.CalledOnValidThread());
 145
 146   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
 147     fail_callback.Run();
 148     return;
 149   }
 150
 151   waiting_requestors_.push_back(callback);
 152   ServeFromCache();
 153   // Blacklist uploads are scheduled on any request completion, so only schedule
 154   // an upload if there is no ongoing request.
 155   if (!pending_request_.get()) {
 156     ScheduleBlacklistUpload();
 157   }
 158 }
 159
 160 void SuggestionsService::UndoBlacklistURL(
 161     const GURL& url,
 162     const SuggestionsService::ResponseCallback& callback,
 163     const base::Closure& fail_callback) {
 164   DCHECK(thread_checker_.CalledOnValidThread());
 165   TimeDelta time_delta;
 166   if (blacklist_store_->GetTimeUntilURLReadyForUpload(url, &time_delta) &&
 167       time_delta > TimeDelta::FromSeconds(0) &&
 168       blacklist_store_->RemoveUrl(url)) {
 169     // The URL was not yet candidate for upload to the server and could be
 170     // removed from the blacklist.
 171     waiting_requestors_.push_back(callback);
 172     ServeFromCache();
 173     return;
 174   }
 175   fail_callback.Run();
 176 }
 177
 178 // static
 179 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
 180                                            GURL* url) {
 181   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
 182                                               kSuggestionsBlacklistURLPrefix,
 183                                               true);
 184   if (!is_blacklist_request) return false;
 185
 186   // Extract the blacklisted URL from the blacklist request.
 187   std::string blacklisted;
 188   if (!net::GetValueForKeyInQuery(
 189           request.GetOriginalURL(),
 190           kSuggestionsBlacklistURLParam,
 191           &blacklisted)) {
 192     return false;
 193   }
 194
 195   GURL blacklisted_url(blacklisted);
 196   blacklisted_url.Swap(url);
 197   return true;
 198 }
 199
 200 // static
 201 void SuggestionsService::RegisterProfilePrefs(
 202     user_prefs::PrefRegistrySyncable* registry) {
 203   SuggestionsStore::RegisterProfilePrefs(registry);
 204   BlacklistStore::RegisterProfilePrefs(registry);
 205 }
 206
 207 void SuggestionsService::SetDefaultExpiryTimestamp(
 208     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
 209   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
 210     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
 211     // Do not set expiry if the server has already provided a more specific
 212     // expiry time for this suggestion.
 213     if (!suggestion->has_expiry_ts()) {
 214       suggestion->set_expiry_ts(default_timestamp_usec);
 215     }
 216   }
 217 }
 218
 219 void SuggestionsService::IssueRequestIfNoneOngoing(const GURL& url) {
 220   // If there is an ongoing request, let it complete.
 221   if (pending_request_.get()) {
 222     return;
 223   }
 224   pending_request_ = CreateSuggestionsRequest(url);
 225   pending_request_->Start();
 226   last_request_started_time_ = TimeTicks::Now();
 227 }
 228
 229 scoped_ptr<net::URLFetcher> SuggestionsService::CreateSuggestionsRequest(
 230     const GURL& url) {
 231   scoped_ptr<net::URLFetcher> request =
 232       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
 233   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
 234   request->SetRequestContext(url_request_context_);
 235   // Add Chrome experiment state to the request headers.
 236   net::HttpRequestHeaders headers;
 237   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 238       request->GetOriginalURL(), false, false, &headers);
 239   request->SetExtraRequestHeaders(headers.ToString());
 240   return request;
 241 }
 242
 243 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
 244   DCHECK(thread_checker_.CalledOnValidThread());
 245   DCHECK_EQ(pending_request_.get(), source);
 246
 247   // The fetcher will be deleted when the request is handled.
 248   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
 249
 250   const net::URLRequestStatus& request_status = request->GetStatus();
 251   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
 252     // This represents network errors (i.e. the server did not provide a
 253     // response).
 254     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
 255                                 -request_status.error());
 256     DVLOG(1) << "Suggestions server request failed with error: "
 257              << request_status.error() << ": "
 258              << net::ErrorToString(request_status.error());
 259     UpdateBlacklistDelay(false);
 260     ScheduleBlacklistUpload();
 261     return;
 262   }
 263
 264   const int response_code = request->GetResponseCode();
 265   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
 266   if (response_code != net::HTTP_OK) {
 267     // A non-200 response code means that server has no (longer) suggestions for
 268     // this user. Aggressively clear the cache.
 269     suggestions_store_->ClearSuggestions();
 270     UpdateBlacklistDelay(false);
 271     ScheduleBlacklistUpload();
 272     return;
 273   }
 274
 275   const TimeDelta latency = TimeTicks::Now() - last_request_started_time_;
 276   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
 277
 278   // Handle a successful blacklisting.
 279   GURL blacklisted_url;
 280   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
 281     blacklist_store_->RemoveUrl(blacklisted_url);
 282   }
 283
 284   std::string suggestions_data;
 285   bool success = request->GetResponseAsString(&suggestions_data);
 286   DCHECK(success);
 287
 288   // Parse the received suggestions and update the cache, or take proper action
 289   // in the case of invalid response.
 290   SuggestionsProfile suggestions;
 291   if (suggestions_data.empty()) {
 292     LogResponseState(RESPONSE_EMPTY);
 293     suggestions_store_->ClearSuggestions();
 294   } else if (suggestions.ParseFromString(suggestions_data)) {
 295     LogResponseState(RESPONSE_VALID);
 296     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
 297         .ToInternalValue();
 298     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
 299     suggestions_store_->StoreSuggestions(suggestions);
 300   } else {
 301     LogResponseState(RESPONSE_INVALID);
 302   }
 303
 304   UpdateBlacklistDelay(true);
 305   ScheduleBlacklistUpload();
 306 }
 307
 308 void SuggestionsService::Shutdown() {
 309   // Cancel pending request, then serve existing requestors from cache.
 310   pending_request_.reset(NULL);
 311   ServeFromCache();
 312 }
 313
 314 void SuggestionsService::ServeFromCache() {
 315   SuggestionsProfile suggestions;
 316   // In case of empty cache or error, |suggestions| stays empty.
 317   suggestions_store_->LoadSuggestions(&suggestions);
 318   thumbnail_manager_->Initialize(suggestions);
 319   FilterAndServe(&suggestions);
 320 }
 321
 322 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
 323   blacklist_store_->FilterSuggestions(suggestions);
 324   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
 325 }
 326
 327 void SuggestionsService::ScheduleBlacklistUpload() {
 328   DCHECK(thread_checker_.CalledOnValidThread());
 329   TimeDelta time_delta;
 330   if (blacklist_store_->GetTimeUntilReadyForUpload(&time_delta)) {
 331     // Blacklist cache is not empty: schedule.
 332     base::Closure blacklist_cb =
 333         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
 334                    weak_ptr_factory_.GetWeakPtr());
 335     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 336         FROM_HERE, blacklist_cb, time_delta + scheduling_delay_);
 337   }
 338 }
 339
 340 void SuggestionsService::UploadOneFromBlacklist() {
 341   DCHECK(thread_checker_.CalledOnValidThread());
 342
 343   GURL blacklist_url;
 344   if (blacklist_store_->GetCandidateForUpload(&blacklist_url)) {
 345     // Issue a blacklisting request. Even if this request ends up not being sent
 346     // because of an ongoing request, a blacklist request is later scheduled.
 347     IssueRequestIfNoneOngoing(
 348         BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
 349     return;
 350   }
 351
 352   // Even though there's no candidate for upload, the blacklist might not be
 353   // empty.
 354   ScheduleBlacklistUpload();
 355 }
 356
 357 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
 358   DCHECK(thread_checker_.CalledOnValidThread());
 359
 360   if (last_request_successful) {
 361     scheduling_delay_ = TimeDelta::FromSeconds(kDefaultSchedulingDelaySec);
 362   } else {
 363     TimeDelta candidate_delay =
 364         scheduling_delay_ * kSchedulingBackoffMultiplier;
 365     if (candidate_delay < TimeDelta::FromSeconds(kSchedulingMaxDelaySec))
 366       scheduling_delay_ = candidate_delay;
 367   }
 368 }
 369
 370 }  // namespace suggestions