components/omnibox/base_search_provider.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/omnibox/base_search_provider.h"
   6
   7 #include "base/i18n/case_conversion.h"
   8 #include "base/strings/string_util.h"
   9 #include "base/strings/utf_string_conversions.h"
  10 #include "components/metrics/proto/omnibox_event.pb.h"
  11 #include "components/metrics/proto/omnibox_input_type.pb.h"
  12 #include "components/omnibox/autocomplete_provider_client.h"
  13 #include "components/omnibox/autocomplete_provider_listener.h"
  14 #include "components/omnibox/omnibox_field_trial.h"
  15 #include "components/omnibox/suggestion_answer.h"
  16 #include "components/search_engines/template_url.h"
  17 #include "components/search_engines/template_url_prepopulate_data.h"
  18 #include "components/search_engines/template_url_service.h"
  19 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  20 #include "net/url_request/url_fetcher.h"
  21 #include "net/url_request/url_fetcher_delegate.h"
  22 #include "url/gurl.h"
  23
  24 using metrics::OmniboxEventProto;
  25
  26 // SuggestionDeletionHandler -------------------------------------------------
  27
  28 // This class handles making requests to the server in order to delete
  29 // personalized suggestions.
  30 class SuggestionDeletionHandler : public net::URLFetcherDelegate {
  31  public:
  32   typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
  33       DeletionCompletedCallback;
  34
  35   SuggestionDeletionHandler(
  36       const std::string& deletion_url,
  37       net::URLRequestContextGetter* request_context,
  38       const DeletionCompletedCallback& callback);
  39
  40   ~SuggestionDeletionHandler() override;
  41
  42  private:
  43   // net::URLFetcherDelegate:
  44   void OnURLFetchComplete(const net::URLFetcher* source) override;
  45
  46   scoped_ptr<net::URLFetcher> deletion_fetcher_;
  47   DeletionCompletedCallback callback_;
  48
  49   DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
  50 };
  51
  52 SuggestionDeletionHandler::SuggestionDeletionHandler(
  53     const std::string& deletion_url,
  54     net::URLRequestContextGetter* request_context,
  55     const DeletionCompletedCallback& callback) : callback_(callback) {
  56   GURL url(deletion_url);
  57   DCHECK(url.is_valid());
  58
  59   deletion_fetcher_ =
  60       net::URLFetcher::Create(BaseSearchProvider::kDeletionURLFetcherID, url,
  61                               net::URLFetcher::GET, this);
  62   deletion_fetcher_->SetRequestContext(request_context);
  63   deletion_fetcher_->Start();
  64 }
  65
  66 SuggestionDeletionHandler::~SuggestionDeletionHandler() {
  67 }
  68
  69 void SuggestionDeletionHandler::OnURLFetchComplete(
  70     const net::URLFetcher* source) {
  71   DCHECK(source == deletion_fetcher_.get());
  72   callback_.Run(
  73       source->GetStatus().is_success() && (source->GetResponseCode() == 200),
  74       this);
  75 }
  76
  77 // BaseSearchProvider ---------------------------------------------------------
  78
  79 // static
  80 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
  81 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
  82 const int BaseSearchProvider::kDeletionURLFetcherID = 3;
  83
  84 BaseSearchProvider::BaseSearchProvider(AutocompleteProvider::Type type,
  85                                        AutocompleteProviderClient* client)
  86     : AutocompleteProvider(type),
  87       client_(client),
  88       field_trial_triggered_(false),
  89       field_trial_triggered_in_session_(false) {
  90 }
  91
  92 // static
  93 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
  94   return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
  95 }
  96
  97 // static
  98 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
  99     const base::string16& suggestion,
 100     AutocompleteMatchType::Type type,
 101     bool from_keyword_provider,
 102     const TemplateURL* template_url,
 103     const SearchTermsData& search_terms_data) {
 104   // These calls use a number of default values.  For instance, they assume
 105   // that if this match is from a keyword provider, then the user is in keyword
 106   // mode.  They also assume the caller knows what it's doing and we set
 107   // this match to look as if it was received/created synchronously.
 108   SearchSuggestionParser::SuggestResult suggest_result(
 109       suggestion, type, suggestion, base::string16(), base::string16(),
 110       base::string16(), base::string16(), nullptr, std::string(),
 111       std::string(), from_keyword_provider, 0, false, false, base::string16());
 112   suggest_result.set_received_after_last_keystroke(false);
 113   return CreateSearchSuggestion(
 114       NULL, AutocompleteInput(), from_keyword_provider, suggest_result,
 115       template_url, search_terms_data, 0, false);
 116 }
 117
 118 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
 119   DCHECK(match.deletable);
 120   if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
 121     deletion_handlers_.push_back(new SuggestionDeletionHandler(
 122         match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
 123         client_->GetRequestContext(),
 124         base::Bind(&BaseSearchProvider::OnDeletionComplete,
 125                    base::Unretained(this))));
 126   }
 127
 128   TemplateURL* template_url =
 129       match.GetTemplateURL(client_->GetTemplateURLService(), false);
 130   // This may be NULL if the template corresponding to the keyword has been
 131   // deleted or there is no keyword set.
 132   if (template_url != NULL) {
 133     client_->DeleteMatchingURLsForKeywordFromHistory(template_url->id(),
 134                                                      match.contents);
 135   }
 136
 137   // Immediately update the list of matches to show the match was deleted,
 138   // regardless of whether the server request actually succeeds.
 139   DeleteMatchFromMatches(match);
 140 }
 141
 142 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 143   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
 144   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
 145   new_entry.set_provider(AsOmniboxEventProviderType());
 146   new_entry.set_provider_done(done_);
 147   std::vector<uint32> field_trial_hashes;
 148   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
 149   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
 150     if (field_trial_triggered_)
 151       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
 152     if (field_trial_triggered_in_session_) {
 153       new_entry.mutable_field_trial_triggered_in_session()->Add(
 154           field_trial_hashes[i]);
 155     }
 156   }
 157 }
 158
 159 // static
 160 const char BaseSearchProvider::kRelevanceFromServerKey[] =
 161     "relevance_from_server";
 162 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
 163 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
 164 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
 165 const char BaseSearchProvider::kTrue[] = "true";
 166 const char BaseSearchProvider::kFalse[] = "false";
 167
 168 BaseSearchProvider::~BaseSearchProvider() {}
 169
 170 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
 171                                         AutocompleteMatch* match) {
 172   if (deletion_url.empty())
 173     return;
 174
 175   TemplateURLService* template_url_service = client_->GetTemplateURLService();
 176   if (!template_url_service)
 177     return;
 178   GURL url =
 179       template_url_service->GetDefaultSearchProvider()->GenerateSearchURL(
 180           template_url_service->search_terms_data());
 181   url = url.GetOrigin().Resolve(deletion_url);
 182   if (url.is_valid()) {
 183     match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
 184         url.spec());
 185     match->deletable = true;
 186   }
 187 }
 188
 189 // static
 190 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 191     AutocompleteProvider* autocomplete_provider,
 192     const AutocompleteInput& input,
 193     const bool in_keyword_mode,
 194     const SearchSuggestionParser::SuggestResult& suggestion,
 195     const TemplateURL* template_url,
 196     const SearchTermsData& search_terms_data,
 197     int accepted_suggestion,
 198     bool append_extra_query_params) {
 199   AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
 200                           suggestion.type());
 201
 202   if (!template_url)
 203     return match;
 204   match.keyword = template_url->keyword();
 205   match.contents = suggestion.match_contents();
 206   match.contents_class = suggestion.match_contents_class();
 207   match.answer_contents = suggestion.answer_contents();
 208   match.answer_type = suggestion.answer_type();
 209   match.answer = SuggestionAnswer::copy(suggestion.answer());
 210   if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_TAIL) {
 211     match.RecordAdditionalInfo(
 212         kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
 213     match.RecordAdditionalInfo(
 214         kACMatchPropertyContentsPrefix,
 215         base::UTF16ToUTF8(suggestion.match_contents_prefix()));
 216     match.RecordAdditionalInfo(
 217         kACMatchPropertyContentsStartIndex,
 218         static_cast<int>(
 219             suggestion.suggestion().length() - match.contents.length()));
 220   }
 221
 222   if (!suggestion.annotation().empty()) {
 223     match.description = suggestion.annotation();
 224     AutocompleteMatch::AddLastClassificationIfNecessary(
 225         &match.description_class, 0, ACMatchClassification::NONE);
 226   }
 227
 228   // suggestion.match_contents() should have already been collapsed.
 229   match.allowed_to_be_default_match =
 230       (!in_keyword_mode || suggestion.from_keyword_provider()) &&
 231       (base::CollapseWhitespace(input.text(), false) ==
 232        suggestion.match_contents());
 233
 234   // When the user forced a query, we need to make sure all the fill_into_edit
 235   // values preserve that property.  Otherwise, if the user starts editing a
 236   // suggestion, non-Search results will suddenly appear.
 237   if (input.type() == metrics::OmniboxInputType::FORCED_QUERY)
 238     match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
 239   if (suggestion.from_keyword_provider())
 240     match.fill_into_edit.append(match.keyword + base::char16(' '));
 241   // We only allow inlinable navsuggestions that were received before the
 242   // last keystroke because we don't want asynchronous inline autocompletions.
 243   if (!input.prevent_inline_autocomplete() &&
 244       !suggestion.received_after_last_keystroke() &&
 245       (!in_keyword_mode || suggestion.from_keyword_provider()) &&
 246       base::StartsWith(
 247           base::i18n::ToLower(suggestion.suggestion()),
 248           base::i18n::ToLower(input.text()), base::CompareCase::SENSITIVE)) {
 249     match.inline_autocompletion =
 250         suggestion.suggestion().substr(input.text().length());
 251     match.allowed_to_be_default_match = true;
 252   }
 253   match.fill_into_edit.append(suggestion.suggestion());
 254
 255   const TemplateURLRef& search_url = template_url->url_ref();
 256   DCHECK(search_url.SupportsReplacement(search_terms_data));
 257   match.search_terms_args.reset(
 258       new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
 259   match.search_terms_args->original_query = input.text();
 260   match.search_terms_args->accepted_suggestion = accepted_suggestion;
 261   match.search_terms_args->enable_omnibox_start_margin = true;
 262   match.search_terms_args->suggest_query_params =
 263       suggestion.suggest_query_params();
 264   match.search_terms_args->append_extra_query_params =
 265       append_extra_query_params;
 266   // This is the destination URL sans assisted query stats.  This must be set
 267   // so the AutocompleteController can properly de-dupe; the controller will
 268   // eventually overwrite it before it reaches the user.
 269   match.destination_url =
 270       GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(),
 271                                          search_terms_data));
 272
 273   // Search results don't look like URLs.
 274   match.transition = suggestion.from_keyword_provider() ?
 275       ui::PAGE_TRANSITION_KEYWORD : ui::PAGE_TRANSITION_GENERATED;
 276
 277   return match;
 278 }
 279
 280 // static
 281 bool BaseSearchProvider::ZeroSuggestEnabled(
 282     const GURL& suggest_url,
 283     const TemplateURL* template_url,
 284     OmniboxEventProto::PageClassification page_classification,
 285     const SearchTermsData& search_terms_data,
 286     const AutocompleteProviderClient* client) {
 287   if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
 288     return false;
 289
 290   // Make sure we are sending the suggest request through a cryptographically
 291   // secure channel to prevent exposing the current page URL or personalized
 292   // results without encryption.
 293   if (!suggest_url.SchemeIsCryptographic())
 294     return false;
 295
 296   // Don't show zero suggest on the NTP.
 297   // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
 298   // under the conditions described in crbug.com/305366.
 299   if ((page_classification ==
 300        OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
 301       (page_classification ==
 302        OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
 303     return false;
 304
 305   // Don't run if in incognito mode.
 306   if (client->IsOffTheRecord())
 307     return false;
 308
 309   // Don't run if we can't get preferences or search suggest is not enabled.
 310   if (!client->SearchSuggestEnabled())
 311     return false;
 312
 313   // Only make the request if we know that the provider supports zero suggest
 314   // (currently only the prepopulated Google provider).
 315   if (template_url == NULL ||
 316       !template_url->SupportsReplacement(search_terms_data) ||
 317       TemplateURLPrepopulateData::GetEngineType(
 318           *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE)
 319     return false;
 320
 321   return true;
 322 }
 323
 324 // static
 325 bool BaseSearchProvider::CanSendURL(
 326     const GURL& current_page_url,
 327     const GURL& suggest_url,
 328     const TemplateURL* template_url,
 329     OmniboxEventProto::PageClassification page_classification,
 330     const SearchTermsData& search_terms_data,
 331     AutocompleteProviderClient* client) {
 332   if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
 333                           search_terms_data, client))
 334     return false;
 335
 336   if (!current_page_url.is_valid())
 337     return false;
 338
 339   // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
 340   // provider.
 341   if ((current_page_url.scheme() != url::kHttpScheme) &&
 342       ((current_page_url.scheme() != url::kHttpsScheme) ||
 343        !net::registry_controlled_domains::SameDomainOrHost(
 344            current_page_url, suggest_url,
 345            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
 346     return false;
 347
 348   if (!client->TabSyncEnabledAndUnencrypted())
 349     return false;
 350
 351   return true;
 352 }
 353
 354 void BaseSearchProvider::AddMatchToMap(
 355     const SearchSuggestionParser::SuggestResult& result,
 356     const std::string& metadata,
 357     int accepted_suggestion,
 358     bool mark_as_deletable,
 359     bool in_keyword_mode,
 360     MatchMap* map) {
 361   AutocompleteMatch match = CreateSearchSuggestion(
 362       this, GetInput(result.from_keyword_provider()), in_keyword_mode, result,
 363       GetTemplateURL(result.from_keyword_provider()),
 364       client_->GetTemplateURLService()->search_terms_data(),
 365       accepted_suggestion, ShouldAppendExtraParams(result));
 366   if (!match.destination_url.is_valid())
 367     return;
 368   match.search_terms_args->bookmark_bar_pinned =
 369       client_->BookmarkBarIsVisible();
 370   match.RecordAdditionalInfo(kRelevanceFromServerKey,
 371                              result.relevance_from_server() ? kTrue : kFalse);
 372   match.RecordAdditionalInfo(kShouldPrefetchKey,
 373                              result.should_prefetch() ? kTrue : kFalse);
 374   SetDeletionURL(result.deletion_url(), &match);
 375   if (mark_as_deletable)
 376     match.deletable = true;
 377   // Metadata is needed only for prefetching queries.
 378   if (result.should_prefetch())
 379     match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 380
 381   // Try to add |match| to |map|.  If a match for this suggestion is
 382   // already in |map|, replace it if |match| is more relevant.
 383   // NOTE: Keep this ToLower() call in sync with url_database.cc.
 384   MatchKey match_key(
 385       std::make_pair(base::i18n::ToLower(result.suggestion()),
 386                      match.search_terms_args->suggest_query_params));
 387   const std::pair<MatchMap::iterator, bool> i(
 388        map->insert(std::make_pair(match_key, match)));
 389
 390   bool should_prefetch = result.should_prefetch();
 391   if (!i.second) {
 392     // NOTE: We purposefully do a direct relevance comparison here instead of
 393     // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
 394     // added first" rather than "items alphabetically first" when the scores
 395     // are equal. The only case this matters is when a user has results with
 396     // the same score that differ only by capitalization; because the history
 397     // system returns results sorted by recency, this means we'll pick the most
 398     // recent such result even if the precision of our relevance score is too
 399     // low to distinguish the two.
 400     if (match.relevance > i.first->second.relevance) {
 401       match.duplicate_matches.insert(match.duplicate_matches.end(),
 402                                      i.first->second.duplicate_matches.begin(),
 403                                      i.first->second.duplicate_matches.end());
 404       i.first->second.duplicate_matches.clear();
 405       match.duplicate_matches.push_back(i.first->second);
 406       i.first->second = match;
 407     } else {
 408       i.first->second.duplicate_matches.push_back(match);
 409       if (match.keyword == i.first->second.keyword) {
 410         // Old and new matches are from the same search provider. It is okay to
 411         // record one match's prefetch data onto a different match (for the same
 412         // query string) for the following reasons:
 413         // 1. Because the suggest server only sends down a query string from
 414         // which we construct a URL, rather than sending a full URL, and because
 415         // we construct URLs from query strings in the same way every time, the
 416         // URLs for the two matches will be the same. Therefore, we won't end up
 417         // prefetching something the server didn't intend.
 418         // 2. Presumably the server sets the prefetch bit on a match it things
 419         // is sufficiently relevant that the user is likely to choose it.
 420         // Surely setting the prefetch bit on a match of even higher relevance
 421         // won't violate this assumption.
 422         should_prefetch |= ShouldPrefetch(i.first->second);
 423         i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
 424                                              should_prefetch ? kTrue : kFalse);
 425         if (should_prefetch)
 426           i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 427       }
 428     }
 429     // Copy over answer data from lower-ranking item, if necessary.
 430     // This depends on the lower-ranking item always being added last - see
 431     // use of push_back above.
 432     AutocompleteMatch& more_relevant_match = i.first->second;
 433     const AutocompleteMatch& less_relevant_match =
 434         more_relevant_match.duplicate_matches.back();
 435     if (less_relevant_match.answer && !more_relevant_match.answer) {
 436       more_relevant_match.answer_type = less_relevant_match.answer_type;
 437       more_relevant_match.answer_contents = less_relevant_match.answer_contents;
 438       more_relevant_match.answer =
 439           SuggestionAnswer::copy(less_relevant_match.answer.get());
 440     }
 441   }
 442 }
 443
 444 bool BaseSearchProvider::ParseSuggestResults(
 445     const base::Value& root_val,
 446     int default_result_relevance,
 447     bool is_keyword_result,
 448     SearchSuggestionParser::Results* results) {
 449   if (!SearchSuggestionParser::ParseSuggestResults(
 450           root_val, GetInput(is_keyword_result), client_->GetSchemeClassifier(),
 451           default_result_relevance, client_->GetAcceptLanguages(),
 452           is_keyword_result, results))
 453     return false;
 454
 455   for (const GURL& url : results->answers_image_urls)
 456     client_->PrefetchImage(url);
 457
 458   field_trial_triggered_ |= results->field_trial_triggered;
 459   field_trial_triggered_in_session_ |= results->field_trial_triggered;
 460   return true;
 461 }
 462
 463 void BaseSearchProvider::DeleteMatchFromMatches(
 464     const AutocompleteMatch& match) {
 465   for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
 466     // Find the desired match to delete by checking the type and contents.
 467     // We can't check the destination URL, because the autocomplete controller
 468     // may have reformulated that. Not that while checking for matching
 469     // contents works for personalized suggestions, if more match types gain
 470     // deletion support, this algorithm may need to be re-examined.
 471     if (i->contents == match.contents && i->type == match.type) {
 472       matches_.erase(i);
 473       break;
 474     }
 475   }
 476 }
 477
 478 void BaseSearchProvider::OnDeletionComplete(
 479     bool success, SuggestionDeletionHandler* handler) {
 480   RecordDeletionResult(success);
 481   SuggestionDeletionHandlers::iterator it = std::find(
 482       deletion_handlers_.begin(), deletion_handlers_.end(), handler);
 483   DCHECK(it != deletion_handlers_.end());
 484   deletion_handlers_.erase(it);
 485 }