1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This file contains the Search autocomplete provider. This provider is
6 // responsible for all autocomplete entries that start with "Search <engine>
7 // for ...", including searching for the current input string, search
8 // history, and search suggestions. An instance of it gets created and
9 // managed by the autocomplete controller.
11 #ifndef CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_
12 #define CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_
18 #include "base/basictypes.h"
19 #include "base/compiler_specific.h"
20 #include "base/memory/scoped_ptr.h"
21 #include "base/time/time.h"
22 #include "base/timer/timer.h"
23 #include "chrome/browser/autocomplete/autocomplete_input.h"
24 #include "chrome/browser/autocomplete/autocomplete_match.h"
25 #include "chrome/browser/autocomplete/autocomplete_provider.h"
26 #include "chrome/browser/history/history_types.h"
27 #include "chrome/browser/search_engines/template_url.h"
28 #include "net/url_request/url_fetcher_delegate.h"
31 class SearchProviderTest
;
32 class TemplateURLService
;
42 // Autocomplete provider for searches and suggestions from a search engine.
44 // After construction, the autocomplete controller repeatedly calls Start()
45 // with some user input, each time expecting to receive a small set of the best
46 // matches (either synchronously or asynchronously).
48 // Initially the provider creates a match that searches for the current input
49 // text. It also starts a task to query the Suggest servers. When that data
50 // comes back, the provider creates and returns matches for the best
52 class SearchProvider
: public AutocompleteProvider
,
53 public net::URLFetcherDelegate
{
55 // ID used in creating URLFetcher for default provider's suggest results.
56 static const int kDefaultProviderURLFetcherID
;
58 // ID used in creating URLFetcher for keyword provider's suggest results.
59 static const int kKeywordProviderURLFetcherID
;
61 SearchProvider(AutocompleteProviderListener
* listener
, Profile
* profile
);
63 // Returns an AutocompleteMatch with the given |autocomplete_provider|,
64 // |relevance|, and |type|, which represents a search via |template_url| for
65 // |query_string|. If |template_url| is NULL, returns a match with an invalid
68 // |input_text| is the original user input, which may differ from
69 // |query_string|; e.g. the user typed "foo" and got a search suggestion of
70 // "food", which we're now marking up. This is used to highlight portions of
71 // the match contents to distinguish locally-typed text from suggested text.
73 // |input| and |is_keyword| are necessary for various other details, like
74 // whether we should allow inline autocompletion and what the transition type
75 // should be. |accepted_suggestion| and |omnibox_start_margin| are used along
76 // with |input_text| to generate Assisted Query Stats.
77 // |append_extra_query_params| should be set if |template_url| is the default
78 // search engine, so the destination URL will contain any
79 // command-line-specified query params.
80 static AutocompleteMatch
CreateSearchSuggestion(
81 AutocompleteProvider
* autocomplete_provider
,
82 const AutocompleteInput
& input
,
83 const string16
& input_text
,
85 AutocompleteMatch::Type type
,
87 const string16
& match_contents
,
88 const string16
& annotation
,
89 const TemplateURL
* template_url
,
90 const string16
& query_string
,
91 const std::string
& suggest_query_params
,
92 int accepted_suggestion
,
93 int omnibox_start_margin
,
94 bool append_extra_query_params
);
96 // Returns whether the SearchProvider previously flagged |match| as a query
97 // that should be prefetched.
98 static bool ShouldPrefetch(const AutocompleteMatch
& match
);
100 // Extracts the suggest response metadata which SearchProvider previously
101 // stored for |match|.
102 static std::string
GetSuggestMetadata(const AutocompleteMatch
& match
);
104 // AutocompleteProvider:
105 virtual void AddProviderInfo(ProvidersInfo
* provider_info
) const OVERRIDE
;
106 virtual void ResetSession() OVERRIDE
;
108 bool field_trial_triggered_in_session() const {
109 return field_trial_triggered_in_session_
;
112 // This URL may be sent with suggest requests; see comments on CanSendURL().
113 void set_current_page_url(const GURL
& current_page_url
) {
114 current_page_url_
= current_page_url
;
118 // TODO(hfung): Remove ZeroSuggestProvider as a friend class after
119 // refactoring common code to a new base class.
120 friend class SearchProviderTest
;
121 friend class ZeroSuggestProvider
;
122 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest
, CanSendURL
);
123 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest
, NavigationInline
);
124 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest
, NavigationInlineDomainClassify
);
125 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest
, NavigationInlineSchemeSubstring
);
126 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest
, RemoveStaleResultsTest
);
127 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest
, SuggestRelevanceExperiment
);
128 FRIEND_TEST_ALL_PREFIXES(AutocompleteProviderTest
, GetDestinationURL
);
129 FRIEND_TEST_ALL_PREFIXES(InstantExtendedPrefetchTest
, ClearPrefetchedResults
);
130 FRIEND_TEST_ALL_PREFIXES(InstantExtendedPrefetchTest
, SetPrefetchQuery
);
132 // Manages the providers (TemplateURLs) used by SearchProvider. Two providers
134 // . The default provider. This corresponds to the user's default search
135 // engine. This is always used, except for the rare case of no default
137 // . The keyword provider. This is used if the user has typed in a keyword.
140 explicit Providers(TemplateURLService
* template_url_service
);
142 // Returns true if the specified providers match the two providers cached
144 bool equal(const string16
& default_provider
,
145 const string16
& keyword_provider
) const {
146 return (default_provider
== default_provider_
) &&
147 (keyword_provider
== keyword_provider_
);
150 // Resets the cached providers.
151 void set(const string16
& default_provider
,
152 const string16
& keyword_provider
) {
153 default_provider_
= default_provider
;
154 keyword_provider_
= keyword_provider
;
157 TemplateURLService
* template_url_service() { return template_url_service_
; }
158 const string16
& default_provider() const { return default_provider_
; }
159 const string16
& keyword_provider() const { return keyword_provider_
; }
161 // NOTE: These may return NULL even if the provider members are nonempty!
162 const TemplateURL
* GetDefaultProviderURL() const;
163 const TemplateURL
* GetKeywordProviderURL() const;
165 // Returns true if there is a valid keyword provider.
166 bool has_keyword_provider() const { return !keyword_provider_
.empty(); }
169 TemplateURLService
* template_url_service_
;
171 // Cached across the life of a query so we behave consistently even if the
172 // user changes their default while the query is running.
173 string16 default_provider_
;
174 string16 keyword_provider_
;
176 DISALLOW_COPY_AND_ASSIGN(Providers
);
179 // The Result classes are intermediate representations of AutocompleteMatches,
180 // simply containing relevance-ranked search and navigation suggestions.
181 // They may be cached to provide some synchronous matches while requests for
182 // new suggestions from updated input are in flight.
183 // TODO(msw) Extend these classes to generate their corresponding matches and
184 // other requisite data, in order to consolidate and simplify the
185 // highly fragmented SearchProvider logic for each Result type.
188 Result(bool from_keyword_provider
,
190 bool relevance_from_server
);
193 bool from_keyword_provider() const { return from_keyword_provider_
; }
195 int relevance() const { return relevance_
; }
196 void set_relevance(int relevance
) { relevance_
= relevance
; }
198 bool relevance_from_server() const { return relevance_from_server_
; }
199 void set_relevance_from_server(bool relevance_from_server
) {
200 relevance_from_server_
= relevance_from_server
;
203 // Returns if this result is inlineable against the current input |input|.
204 // Non-inlineable results are stale.
205 virtual bool IsInlineable(const string16
& input
) const = 0;
207 // Returns the default relevance value for this result (which may
208 // be left over from a previous omnibox input) given the current
209 // input and whether the current input caused a keyword provider
211 virtual int CalculateRelevance(const AutocompleteInput
& input
,
212 bool keyword_provider_requested
) const = 0;
215 // True if the result came from the keyword provider.
216 bool from_keyword_provider_
;
218 // The relevance score.
222 // Whether this result's relevance score was fully or partly calculated
223 // based on server information, and thus is assumed to be more accurate.
224 // This is ultimately used in
225 // SearchProvider::ConvertResultsToAutocompleteMatches(), see comments
227 bool relevance_from_server_
;
230 class SuggestResult
: public Result
{
232 SuggestResult(const string16
& suggestion
,
233 const string16
& match_contents
,
234 const string16
& annotation
,
235 const std::string
& suggest_query_params
,
236 bool from_keyword_provider
,
238 bool relevance_from_server
,
239 bool should_prefetch
);
240 virtual ~SuggestResult();
242 const string16
& suggestion() const { return suggestion_
; }
243 const string16
& match_contents() const { return match_contents_
; }
244 const string16
& annotation() const { return annotation_
; }
245 const std::string
& suggest_query_params() const {
246 return suggest_query_params_
;
248 bool should_prefetch() const { return should_prefetch_
; }
251 virtual bool IsInlineable(const string16
& input
) const OVERRIDE
;
252 virtual int CalculateRelevance(
253 const AutocompleteInput
& input
,
254 bool keyword_provider_requested
) const OVERRIDE
;
257 // The search terms to be used for this suggestion.
258 string16 suggestion_
;
260 // The contents to be displayed in the autocomplete match.
261 string16 match_contents_
;
263 // Optional annotation for the |match_contents_| for disambiguation.
264 // This may be displayed in the autocomplete match contents, but is defined
265 // separately to facilitate different formatting.
266 string16 annotation_
;
268 // Optional additional parameters to be added to the search URL.
269 std::string suggest_query_params_
;
271 // Should this result be prefetched?
272 bool should_prefetch_
;
275 class NavigationResult
: public Result
{
277 // |provider| is necessary to use StringForURLDisplay() in order to
278 // compute |formatted_url_|.
279 NavigationResult(const AutocompleteProvider
& provider
,
281 const string16
& description
,
282 bool from_keyword_provider
,
284 bool relevance_from_server
);
285 virtual ~NavigationResult();
287 const GURL
& url() const { return url_
; }
288 const string16
& description() const { return description_
; }
289 const string16
& formatted_url() const { return formatted_url_
; }
292 virtual bool IsInlineable(const string16
& input
) const OVERRIDE
;
293 virtual int CalculateRelevance(
294 const AutocompleteInput
& input
,
295 bool keyword_provider_requested
) const OVERRIDE
;
298 // The suggested url for navigation.
301 // The properly formatted ("fixed up") URL string with equivalent meaning
302 // to the one in |url_|.
303 string16 formatted_url_
;
305 // The suggested navigational result description; generally the site name.
306 string16 description_
;
309 class CompareScoredResults
;
311 typedef std::vector
<SuggestResult
> SuggestResults
;
312 typedef std::vector
<NavigationResult
> NavigationResults
;
313 typedef std::vector
<history::KeywordSearchTermVisit
> HistoryResults
;
314 typedef std::pair
<string16
, std::string
> MatchKey
;
315 typedef std::map
<MatchKey
, AutocompleteMatch
> MatchMap
;
317 // A simple structure bundling most of the information (including
318 // both SuggestResults and NavigationResults) returned by a call to
319 // the suggest server.
321 // This has to be declared after the typedefs since it relies on some of them.
326 // Clears |suggest_results| and |navigation_results| and resets
327 // |verbatim_relevance| to -1 (implies unset).
330 // Returns whether any of the results (including verbatim) have
331 // server-provided scores.
332 bool HasServerProvidedScores() const;
334 // Query suggestions sorted by relevance score.
335 SuggestResults suggest_results
;
337 // Navigational suggestions sorted by relevance score.
338 NavigationResults navigation_results
;
340 // The server supplied verbatim relevance scores. Negative values
341 // indicate that there is no suggested score; a value of 0
342 // suppresses the verbatim result.
343 int verbatim_relevance
;
345 // The JSON metadata associated with this server response.
346 std::string metadata
;
349 DISALLOW_COPY_AND_ASSIGN(Results
);
352 virtual ~SearchProvider();
354 // Removes non-inlineable results until either the top result can inline
355 // autocomplete the current input or verbatim outscores the top result.
356 static void RemoveStaleResults(const string16
& input
,
357 int verbatim_relevance
,
358 SuggestResults
* suggest_results
,
359 NavigationResults
* navigation_results
);
361 // Calculates the relevance score for the keyword verbatim result (if the
362 // input matches one of the profile's keyword).
363 static int CalculateRelevanceForKeywordVerbatim(AutocompleteInput::Type type
,
364 bool prefer_keyword
);
366 // AutocompleteProvider:
367 virtual void Start(const AutocompleteInput
& input
,
368 bool minimal_changes
) OVERRIDE
;
369 virtual void Stop(bool clear_cached_results
) OVERRIDE
;
371 // net::URLFetcherDelegate:
372 virtual void OnURLFetchComplete(const net::URLFetcher
* source
) OVERRIDE
;
374 // Called when timer_ expires.
377 // Runs the history query, if necessary. The history query is synchronous.
378 // This does not update |done_|.
379 void DoHistoryQuery(bool minimal_changes
);
381 // Determines whether an asynchronous subcomponent query should run for the
382 // current input. If so, starts it if necessary; otherwise stops it.
383 // NOTE: This function does not update |done_|. Callers must do so.
384 void StartOrStopSuggestQuery(bool minimal_changes
);
386 // Returns true when the current query can be sent to the Suggest service.
387 // This will be false e.g. when Suggest is disabled, the query contains
388 // potentially private data, etc.
389 bool IsQuerySuitableForSuggest() const;
391 // Stops the suggest query.
392 // NOTE: This does not update |done_|. Callers must do so.
395 // Clears the current results.
396 void ClearAllResults();
398 // Removes stale results for both default and keyword providers. See comments
399 // on RemoveStaleResults().
400 void RemoveAllStaleResults();
402 // Apply calculated relevance scores to the current results.
403 void ApplyCalculatedRelevance();
404 void ApplyCalculatedSuggestRelevance(SuggestResults
* list
);
405 void ApplyCalculatedNavigationRelevance(NavigationResults
* list
);
407 // Starts a new URLFetcher requesting suggest results from |template_url|;
408 // callers own the returned URLFetcher, which is NULL for invalid providers.
409 net::URLFetcher
* CreateSuggestFetcher(int id
,
410 const TemplateURL
* template_url
,
411 const AutocompleteInput
& input
);
413 // Parses results from the suggest server and updates the appropriate suggest
414 // and navigation result lists, depending on whether |is_keyword| is true.
415 // Returns whether the appropriate result list members were updated.
416 bool ParseSuggestResults(base::Value
* root_val
, bool is_keyword
);
418 // Converts the parsed results to a set of AutocompleteMatches, |matches_|.
419 void ConvertResultsToAutocompleteMatches();
421 // Returns an iterator to the first match in |matches_| which might
422 // be chosen as default. If
423 // |autocomplete_result_will_reorder_for_default_match| is false,
424 // this simply means the first match; otherwise, it means the first
425 // match for which the |allowed_to_be_default_match| member is true.
426 ACMatches::const_iterator
FindTopMatch(
427 bool autocomplete_result_will_reorder_for_default_match
) const;
429 // Checks if suggested relevances violate certain expected constraints.
430 // See UpdateMatches() for the use and explanation of these constraints.
431 bool IsTopMatchNavigationInKeywordMode(
432 bool autocomplete_result_will_reorder_for_default_match
) const;
433 bool IsTopMatchScoreTooLow(
434 bool autocomplete_result_will_reorder_for_default_match
) const;
435 bool IsTopMatchSearchWithURLInput(
436 bool autocomplete_result_will_reorder_for_default_match
) const;
437 bool HasValidDefaultMatch(
438 bool autocomplete_result_will_reorder_for_default_match
) const;
440 // Updates |matches_| from the latest results; applies calculated relevances
441 // if suggested relevances cause undesriable behavior. Updates |done_|.
442 void UpdateMatches();
444 // Converts an appropriate number of navigation results in
445 // |navigation_results| to matches and adds them to |matches|.
446 void AddNavigationResultsToMatches(
447 const NavigationResults
& navigation_results
,
450 // Adds a match for each result in |results| to |map|. |is_keyword| indicates
451 // whether the results correspond to the keyword provider or default provider.
452 void AddHistoryResultsToMap(const HistoryResults
& results
,
454 int did_not_accept_suggestion
,
457 // Calculates relevance scores for all |results|.
458 SuggestResults
ScoreHistoryResults(const HistoryResults
& results
,
459 bool base_prevent_inline_autocomplete
,
460 bool input_multiple_words
,
461 const string16
& input_text
,
464 // Adds matches for |results| to |map|.
465 void AddSuggestResultsToMap(const SuggestResults
& results
,
466 const std::string
& metadata
,
469 // Gets the relevance score for the verbatim result. This value may be
470 // provided by the suggest server or calculated locally; if
471 // |relevance_from_server| is non-NULL, it will be set to indicate which of
473 int GetVerbatimRelevance(bool* relevance_from_server
) const;
475 // Calculates the relevance score for the verbatim result from the
476 // default search engine. This version takes into account context:
477 // i.e., whether the user has entered a keyword-based search or not.
478 int CalculateRelevanceForVerbatim() const;
480 // Calculates the relevance score for the verbatim result from the default
481 // search engine *ignoring* whether the input is a keyword-based search
482 // or not. This function should only be used to determine the minimum
483 // relevance score that the best result from this provider should have.
484 // For normal use, prefer the above function.
485 int CalculateRelevanceForVerbatimIgnoringKeywordModeState() const;
487 // Gets the relevance score for the keyword verbatim result.
488 // |relevance_from_server| is handled as in GetVerbatimRelevance().
489 // TODO(mpearson): Refactor so this duplication isn't necesary or
490 // restructure so one static function takes all the parameters it needs
491 // (rather than looking at internal state).
492 int GetKeywordVerbatimRelevance(bool* relevance_from_server
) const;
494 // |time| is the time at which this query was last seen. |is_keyword|
495 // indicates whether the results correspond to the keyword provider or default
496 // provider. |use_aggressive_method| says whether this function can use a
497 // method that gives high scores (1200+) rather than one that gives lower
498 // scores. When using the aggressive method, scores may exceed 1300
499 // unless |prevent_search_history_inlining| is set.
500 int CalculateRelevanceForHistory(const base::Time
& time
,
502 bool use_aggressive_method
,
503 bool prevent_search_history_inlining
) const;
505 // Creates an AutocompleteMatch for "Search <engine> for |query_string|" with
506 // the supplied relevance. Adds this match to |map|; if such a match already
507 // exists, whichever one has lower relevance is eliminated.
508 void AddMatchToMap(const string16
& input_text
,
510 bool relevance_from_server
,
511 bool should_prefetch
,
512 const std::string
& metadata
,
513 AutocompleteMatch::Type type
,
515 const string16
& match_contents
,
516 const string16
& annotation
,
517 const string16
& query_string
,
518 int accepted_suggestion
,
519 const std::string
& suggest_query_params
,
522 // Returns an AutocompleteMatch for a navigational suggestion.
523 AutocompleteMatch
NavigationToMatch(const NavigationResult
& navigation
);
525 // Resets the scores of all |keyword_navigation_results_| matches to
526 // be below that of the top keyword query match (the verbatim match
527 // as expressed by |keyword_verbatim_relevance_| or keyword query
528 // suggestions stored in |keyword_suggest_results_|). If there
529 // are no keyword suggestions and keyword verbatim is suppressed,
530 // then drops the suggested relevance scores for the navsuggestions
531 // and drops the request to suppress verbatim, thereby introducing the
532 // keyword verbatim match which will naturally outscore the navsuggestions.
533 void DemoteKeywordNavigationMatchesPastTopQuery();
535 // Updates the value of |done_| from the internal state.
538 // Returns whether we can send the URL of the current page in any suggest
539 // requests. Doing this requires that all the following hold:
540 // * The user has suggest enabled in their settings and is not in incognito
541 // mode. (Incognito disables suggest entirely.)
542 // * The current URL is HTTP, or HTTPS with the same domain as the suggest
543 // server. Non-HTTP[S] URLs (e.g. FTP/file URLs) may contain sensitive
544 // information. HTTPS URLs may also contain sensitive information, but if
545 // they're on the same domain as the suggest server, then the relevant
546 // entity could have already seen/logged this data.
547 // * The suggest request is sent over HTTPS. This avoids leaking the current
548 // page URL in world-readable network traffic.
549 // * The user's suggest provider is Google. We might want to allow other
550 // providers to see this data someday, but for now this has only been
551 // implemented for Google. Also see next bullet.
552 // * The user is OK in principle with sending URLs of current pages to their
553 // provider. Today, there is no explicit setting that controls this, but if
554 // the user has tab sync enabled and tab sync is unencrypted, then they're
555 // already sending this data to Google for sync purposes. Thus we use this
556 // setting as a proxy for "it's OK to send such data". In the future,
557 // especially if we want to support suggest providers other than Google, we
558 // may change this to be a standalone setting or part of some explicit
560 static bool CanSendURL(
561 const GURL
& current_page_url
,
562 const GURL
& suggest_url
,
563 const TemplateURL
* template_url
,
564 AutocompleteInput::PageClassification page_classification
,
567 // The amount of time to wait before sending a new suggest request after the
568 // previous one. Non-const because some unittests modify this value.
569 static int kMinimumTimeBetweenSuggestQueriesMs
;
571 // The following keys are used to record additional information on matches.
573 // We annotate our AutocompleteMatches with whether their relevance scores
574 // were server-provided using this key in the |additional_info| field.
575 static const char kRelevanceFromServerKey
[];
577 // Indicates whether the server said a match should be prefetched.
578 static const char kShouldPrefetchKey
[];
580 // Used to store metadata from the server response, which is needed for
582 static const char kSuggestMetadataKey
[];
584 // These are the values for the above keys.
585 static const char kTrue
[];
586 static const char kFalse
[];
588 // Maintains the TemplateURLs used.
589 Providers providers_
;
592 AutocompleteInput input_
;
594 // Input when searching against the keyword provider.
595 AutocompleteInput keyword_input_
;
597 // Searches in the user's history that begin with the input text.
598 HistoryResults keyword_history_results_
;
599 HistoryResults default_history_results_
;
601 // Number of suggest results that haven't yet arrived. If greater than 0 it
602 // indicates one of the URLFetchers is still running.
603 int suggest_results_pending_
;
605 // A timer to start a query to the suggest server after the user has stopped
606 // typing for long enough.
607 base::OneShotTimer
<SearchProvider
> timer_
;
609 // The time at which we sent a query to the suggest server.
610 base::TimeTicks time_suggest_request_sent_
;
612 // Fetchers used to retrieve results for the keyword and default providers.
613 scoped_ptr
<net::URLFetcher
> keyword_fetcher_
;
614 scoped_ptr
<net::URLFetcher
> default_fetcher_
;
616 // Results from the default and keyword search providers.
617 Results default_results_
;
618 Results keyword_results_
;
620 // Whether a field trial, if any, has triggered in the most recent
621 // autocomplete query. This field is set to false in Start() and may be set
622 // to true if either the default provider or keyword provider has completed
623 // and their corresponding suggest response contained
624 // '"google:fieldtrialtriggered":true'.
625 // If the autocomplete query has not returned, this field is set to false.
626 bool field_trial_triggered_
;
628 // Same as above except that it is maintained across the current Omnibox
630 bool field_trial_triggered_in_session_
;
632 // If true, search history query suggestions will score low enough that
633 // they will not be inlined.
634 bool prevent_search_history_inlining_
;
636 GURL current_page_url_
;
638 DISALLOW_COPY_AND_ASSIGN(SearchProvider
);
641 #endif // CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_