components/search_engines/template_url.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/search_engines/template_url.h"
   6
   7 #include <string>
   8 #include <vector>
   9
  10 #include "base/basictypes.h"
  11 #include "base/command_line.h"
  12 #include "base/format_macros.h"
  13 #include "base/i18n/icu_string_conversions.h"
  14 #include "base/i18n/rtl.h"
  15 #include "base/logging.h"
  16 #include "base/metrics/field_trial.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_number_conversions.h"
  19 #include "base/strings/string_split.h"
  20 #include "base/strings/string_util.h"
  21 #include "base/strings/stringprintf.h"
  22 #include "base/strings/utf_string_conversions.h"
  23 #include "components/google/core/browser/google_util.h"
  24 #include "components/metrics/proto/omnibox_input_type.pb.h"
  25 #include "components/search_engines/search_engines_switches.h"
  26 #include "components/search_engines/search_terms_data.h"
  27 #include "google_apis/google_api_keys.h"
  28 #include "net/base/escape.h"
  29 #include "net/base/mime_util.h"
  30 #include "net/base/net_util.h"
  31 #include "url/gurl.h"
  32
  33 namespace {
  34
  35 // The TemplateURLRef has any number of terms that need to be replaced. Each of
  36 // the terms is enclosed in braces. If the character preceeding the final
  37 // brace is a ?, it indicates the term is optional and can be replaced with
  38 // an empty string.
  39 const char kStartParameter = '{';
  40 const char kEndParameter = '}';
  41 const char kOptional = '?';
  42
  43 // Known parameters found in the URL.
  44 const char kSearchTermsParameter[] = "searchTerms";
  45 const char kSearchTermsParameterFull[] = "{searchTerms}";
  46 const char kSearchTermsParameterFullEscaped[] = "%7BsearchTerms%7D";
  47 const char kCountParameter[] = "count";
  48 const char kStartIndexParameter[] = "startIndex";
  49 const char kStartPageParameter[] = "startPage";
  50 const char kLanguageParameter[] = "language";
  51 const char kInputEncodingParameter[] = "inputEncoding";
  52 const char kOutputEncodingParameter[] = "outputEncoding";
  53
  54 const char kGoogleAssistedQueryStatsParameter[] = "google:assistedQueryStats";
  55
  56 // Host/Domain Google searches are relative to.
  57 const char kGoogleBaseURLParameter[] = "google:baseURL";
  58 const char kGoogleBaseURLParameterFull[] = "{google:baseURL}";
  59
  60 // Like google:baseURL, but for the Search Suggest capability.
  61 const char kGoogleBaseSuggestURLParameter[] = "google:baseSuggestURL";
  62 const char kGoogleBaseSuggestURLParameterFull[] = "{google:baseSuggestURL}";
  63 const char kGoogleBookmarkBarPinnedParameter[] = "google:bookmarkBarPinned";
  64 const char kGoogleContextualSearchContextData[] =
  65     "google:contextualSearchContextData";
  66 const char kGoogleContextualSearchVersion[] = "google:contextualSearchVersion";
  67 const char kGoogleCurrentPageUrlParameter[] = "google:currentPageUrl";
  68 const char kGoogleCursorPositionParameter[] = "google:cursorPosition";
  69 const char kGoogleForceInstantResultsParameter[] = "google:forceInstantResults";
  70 const char kGoogleImageSearchSource[] = "google:imageSearchSource";
  71 const char kGoogleImageThumbnailParameter[] = "google:imageThumbnail";
  72 const char kGoogleImageOriginalWidth[] = "google:imageOriginalWidth";
  73 const char kGoogleImageOriginalHeight[] = "google:imageOriginalHeight";
  74 const char kGoogleImageURLParameter[] = "google:imageURL";
  75 const char kGoogleInputTypeParameter[] = "google:inputType";
  76 const char kGoogleInstantExtendedEnabledParameter[] =
  77     "google:instantExtendedEnabledParameter";
  78 const char kGoogleInstantExtendedEnabledKey[] =
  79     "google:instantExtendedEnabledKey";
  80 const char kGoogleInstantExtendedEnabledKeyFull[] =
  81     "{google:instantExtendedEnabledKey}";
  82 const char kGoogleNTPIsThemedParameter[] = "google:ntpIsThemedParameter";
  83 const char kGoogleOmniboxStartMarginParameter[] =
  84     "google:omniboxStartMarginParameter";
  85 const char kGoogleOriginalQueryForSuggestionParameter[] =
  86     "google:originalQueryForSuggestion";
  87 const char kGooglePageClassificationParameter[] = "google:pageClassification";
  88 const char kGooglePrefetchQuery[] = "google:prefetchQuery";
  89 const char kGoogleRLZParameter[] = "google:RLZ";
  90 const char kGoogleSearchClient[] = "google:searchClient";
  91 const char kGoogleSearchFieldtrialParameter[] =
  92     "google:searchFieldtrialParameter";
  93 const char kGoogleSearchVersion[] = "google:searchVersion";
  94 const char kGoogleSessionToken[] = "google:sessionToken";
  95 const char kGoogleSourceIdParameter[] = "google:sourceId";
  96 const char kGoogleSuggestAPIKeyParameter[] = "google:suggestAPIKeyParameter";
  97 const char kGoogleSuggestClient[] = "google:suggestClient";
  98 const char kGoogleSuggestRequestId[] = "google:suggestRid";
  99
 100 // Same as kSearchTermsParameter, with no escaping.
 101 const char kGoogleUnescapedSearchTermsParameter[] =
 102     "google:unescapedSearchTerms";
 103 const char kGoogleUnescapedSearchTermsParameterFull[] =
 104     "{google:unescapedSearchTerms}";
 105
 106 // Display value for kSearchTermsParameter.
 107 const char kDisplaySearchTerms[] = "%s";
 108
 109 // Display value for kGoogleUnescapedSearchTermsParameter.
 110 const char kDisplayUnescapedSearchTerms[] = "%S";
 111
 112 // Used if the count parameter is not optional. Indicates we want 10 search
 113 // results.
 114 const char kDefaultCount[] = "10";
 115
 116 // Used if the parameter kOutputEncodingParameter is required.
 117 const char kOutputEncodingType[] = "UTF-8";
 118
 119 // Attempts to encode |terms| and |original_query| in |encoding| and escape
 120 // them.  |terms| may be escaped as path or query depending on |is_in_query|;
 121 // |original_query| is always escaped as query.  Returns whether the encoding
 122 // process succeeded.
 123 bool TryEncoding(const base::string16& terms,
 124                  const base::string16& original_query,
 125                  const char* encoding,
 126                  bool is_in_query,
 127                  base::string16* escaped_terms,
 128                  base::string16* escaped_original_query) {
 129   DCHECK(escaped_terms);
 130   DCHECK(escaped_original_query);
 131   std::string encoded_terms;
 132   if (!base::UTF16ToCodepage(terms, encoding,
 133       base::OnStringConversionError::SKIP, &encoded_terms))
 134     return false;
 135   *escaped_terms = base::UTF8ToUTF16(is_in_query ?
 136       net::EscapeQueryParamValue(encoded_terms, true) :
 137       net::EscapePath(encoded_terms));
 138   if (original_query.empty())
 139     return true;
 140   std::string encoded_original_query;
 141   if (!base::UTF16ToCodepage(original_query, encoding,
 142       base::OnStringConversionError::SKIP, &encoded_original_query))
 143     return false;
 144   *escaped_original_query = base::UTF8ToUTF16(
 145       net::EscapeQueryParamValue(encoded_original_query, true));
 146   return true;
 147 }
 148
 149 // Extract query key and host given a list of parameters coming from the URL
 150 // query or ref.
 151 std::string FindSearchTermsKey(const std::string& params) {
 152   if (params.empty())
 153     return std::string();
 154   url::Component query, key, value;
 155   query.len = static_cast<int>(params.size());
 156   while (url::ExtractQueryKeyValue(params.c_str(), &query, &key, &value)) {
 157     if (key.is_nonempty() && value.is_nonempty()) {
 158       std::string value_string = params.substr(value.begin, value.len);
 159       if (value_string.find(kSearchTermsParameterFull, 0) !=
 160           std::string::npos ||
 161           value_string.find(kGoogleUnescapedSearchTermsParameterFull, 0) !=
 162           std::string::npos) {
 163         return params.substr(key.begin, key.len);
 164       }
 165     }
 166   }
 167   return std::string();
 168 }
 169
 170 // Extract the position of the search terms' parameter in the URL path.
 171 bool FindSearchTermsInPath(const std::string& path,
 172                            url::Component* parameter_position) {
 173   DCHECK(parameter_position);
 174   parameter_position->reset();
 175   const size_t begin = path.find(kSearchTermsParameterFullEscaped);
 176   if (begin == std::string::npos)
 177     return false;
 178   parameter_position->begin = begin;
 179   parameter_position->len = arraysize(kSearchTermsParameterFullEscaped) - 1;
 180   return true;
 181 }
 182
 183 bool IsTemplateParameterString(const std::string& param) {
 184   return (param.length() > 2) && (*(param.begin()) == kStartParameter) &&
 185       (*(param.rbegin()) == kEndParameter);
 186 }
 187
 188 }  // namespace
 189
 190
 191 // TemplateURLRef::SearchTermsArgs --------------------------------------------
 192
 193 TemplateURLRef::SearchTermsArgs::SearchTermsArgs(
 194     const base::string16& search_terms)
 195     : search_terms(search_terms),
 196       input_type(metrics::OmniboxInputType::INVALID),
 197       accepted_suggestion(NO_SUGGESTIONS_AVAILABLE),
 198       cursor_position(base::string16::npos),
 199       enable_omnibox_start_margin(false),
 200       page_classification(metrics::OmniboxEventProto::INVALID_SPEC),
 201       bookmark_bar_pinned(false),
 202       append_extra_query_params(false),
 203       force_instant_results(false),
 204       from_app_list(false),
 205       contextual_search_params(ContextualSearchParams()) {
 206 }
 207
 208 TemplateURLRef::SearchTermsArgs::~SearchTermsArgs() {
 209 }
 210
 211 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 212     ContextualSearchParams()
 213     : version(-1),
 214       start(base::string16::npos),
 215       end(base::string16::npos),
 216       resolve(true) {
 217 }
 218
 219 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 220     ContextualSearchParams(
 221         const int version,
 222         const std::string& selection,
 223         const std::string& base_page_url,
 224         const bool resolve)
 225     : version(version),
 226       start(base::string16::npos),
 227       end(base::string16::npos),
 228       selection(selection),
 229       base_page_url(base_page_url),
 230       resolve(resolve) {
 231 }
 232
 233 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 234     ContextualSearchParams(
 235         const int version,
 236         const size_t start,
 237         const size_t end,
 238         const std::string& selection,
 239         const std::string& content,
 240         const std::string& base_page_url,
 241         const std::string& encoding,
 242         const bool resolve)
 243     : version(version),
 244       start(start),
 245       end(end),
 246       selection(selection),
 247       content(content),
 248       base_page_url(base_page_url),
 249       encoding(encoding),
 250       resolve(resolve) {
 251 }
 252
 253 TemplateURLRef::SearchTermsArgs::ContextualSearchParams::
 254     ~ContextualSearchParams() {
 255 }
 256
 257 // TemplateURLRef -------------------------------------------------------------
 258
 259 TemplateURLRef::TemplateURLRef(TemplateURL* owner, Type type)
 260     : owner_(owner),
 261       type_(type),
 262       index_in_owner_(0),
 263       parsed_(false),
 264       valid_(false),
 265       supports_replacements_(false),
 266       search_term_position_in_path_(std::string::npos),
 267       search_term_key_location_(url::Parsed::QUERY),
 268       prepopulated_(false) {
 269   DCHECK(owner_);
 270   DCHECK_NE(INDEXED, type_);
 271 }
 272
 273 TemplateURLRef::TemplateURLRef(TemplateURL* owner, size_t index_in_owner)
 274     : owner_(owner),
 275       type_(INDEXED),
 276       index_in_owner_(index_in_owner),
 277       parsed_(false),
 278       valid_(false),
 279       supports_replacements_(false),
 280       search_term_position_in_path_(std::string::npos),
 281       search_term_key_location_(url::Parsed::QUERY),
 282       prepopulated_(false) {
 283   DCHECK(owner_);
 284   DCHECK_LT(index_in_owner_, owner_->URLCount());
 285 }
 286
 287 TemplateURLRef::~TemplateURLRef() {
 288 }
 289
 290 std::string TemplateURLRef::GetURL() const {
 291   switch (type_) {
 292     case SEARCH:            return owner_->url();
 293     case SUGGEST:           return owner_->suggestions_url();
 294     case INSTANT:           return owner_->instant_url();
 295     case IMAGE:             return owner_->image_url();
 296     case NEW_TAB:           return owner_->new_tab_url();
 297     case CONTEXTUAL_SEARCH: return owner_->contextual_search_url();
 298     case INDEXED:           return owner_->GetURL(index_in_owner_);
 299     default:       NOTREACHED(); return std::string();  // NOLINT
 300   }
 301 }
 302
 303 std::string TemplateURLRef::GetPostParamsString() const {
 304   switch (type_) {
 305     case INDEXED:
 306     case SEARCH:            return owner_->search_url_post_params();
 307     case SUGGEST:           return owner_->suggestions_url_post_params();
 308     case INSTANT:           return owner_->instant_url_post_params();
 309     case NEW_TAB:           return std::string();
 310     case CONTEXTUAL_SEARCH: return std::string();
 311     case IMAGE:             return owner_->image_url_post_params();
 312     default:      NOTREACHED(); return std::string();  // NOLINT
 313   }
 314 }
 315
 316 bool TemplateURLRef::UsesPOSTMethod(
 317     const SearchTermsData& search_terms_data) const {
 318   ParseIfNecessary(search_terms_data);
 319   return !post_params_.empty();
 320 }
 321
 322 bool TemplateURLRef::EncodeFormData(const PostParams& post_params,
 323                                     PostContent* post_content) const {
 324   if (post_params.empty())
 325     return true;
 326   if (!post_content)
 327     return false;
 328
 329   const char kUploadDataMIMEType[] = "multipart/form-data; boundary=";
 330   const char kMultipartBoundary[] = "----+*+----%016" PRIx64 "----+*+----";
 331   // Each name/value pair is stored in a body part which is preceded by a
 332   // boundary delimiter line. Uses random number generator here to create
 333   // a unique boundary delimiter for form data encoding.
 334   std::string boundary = base::StringPrintf(kMultipartBoundary,
 335                                             base::RandUint64());
 336   // Sets the content MIME type.
 337   post_content->first = kUploadDataMIMEType;
 338   post_content->first += boundary;
 339   // Encodes the post parameters.
 340   std::string* post_data = &post_content->second;
 341   post_data->clear();
 342   for (const auto& param : post_params) {
 343     DCHECK(!param.name.empty());
 344     net::AddMultipartValueForUpload(param.name, param.value, boundary,
 345                                     param.content_type, post_data);
 346   }
 347   net::AddMultipartFinalDelimiterForUpload(boundary, post_data);
 348   return true;
 349 }
 350
 351 bool TemplateURLRef::SupportsReplacement(
 352     const SearchTermsData& search_terms_data) const {
 353   ParseIfNecessary(search_terms_data);
 354   return valid_ && supports_replacements_;
 355 }
 356
 357 std::string TemplateURLRef::ReplaceSearchTerms(
 358     const SearchTermsArgs& search_terms_args,
 359     const SearchTermsData& search_terms_data,
 360     PostContent* post_content) const {
 361   ParseIfNecessary(search_terms_data);
 362   if (!valid_)
 363     return std::string();
 364
 365   std::string url(HandleReplacements(search_terms_args, search_terms_data,
 366                                      post_content));
 367
 368   GURL gurl(url);
 369   if (!gurl.is_valid())
 370     return url;
 371
 372   std::vector<std::string> query_params;
 373   if (search_terms_args.append_extra_query_params) {
 374     std::string extra_params(
 375         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 376             switches::kExtraSearchQueryParams));
 377     if (!extra_params.empty())
 378       query_params.push_back(extra_params);
 379   }
 380   if (!search_terms_args.suggest_query_params.empty())
 381     query_params.push_back(search_terms_args.suggest_query_params);
 382   if (!gurl.query().empty())
 383     query_params.push_back(gurl.query());
 384
 385   if (query_params.empty())
 386     return url;
 387
 388   GURL::Replacements replacements;
 389   std::string query_str = JoinString(query_params, "&");
 390   replacements.SetQueryStr(query_str);
 391   return gurl.ReplaceComponents(replacements).possibly_invalid_spec();
 392 }
 393
 394 bool TemplateURLRef::IsValid(const SearchTermsData& search_terms_data) const {
 395   ParseIfNecessary(search_terms_data);
 396   return valid_;
 397 }
 398
 399 base::string16 TemplateURLRef::DisplayURL(
 400     const SearchTermsData& search_terms_data) const {
 401   ParseIfNecessary(search_terms_data);
 402   std::string result(GetURL());
 403   if (valid_ && !replacements_.empty()) {
 404     ReplaceSubstringsAfterOffset(&result, 0,
 405                                  kSearchTermsParameterFull,
 406                                  kDisplaySearchTerms);
 407     ReplaceSubstringsAfterOffset(&result, 0,
 408                                  kGoogleUnescapedSearchTermsParameterFull,
 409                                  kDisplayUnescapedSearchTerms);
 410   }
 411   return base::UTF8ToUTF16(result);
 412 }
 413
 414 // static
 415 std::string TemplateURLRef::DisplayURLToURLRef(
 416     const base::string16& display_url) {
 417   std::string result = base::UTF16ToUTF8(display_url);
 418   ReplaceSubstringsAfterOffset(&result, 0,
 419                                kDisplaySearchTerms,
 420                                kSearchTermsParameterFull);
 421   ReplaceSubstringsAfterOffset(&result, 0,
 422                                kDisplayUnescapedSearchTerms,
 423                                kGoogleUnescapedSearchTermsParameterFull);
 424   return result;
 425 }
 426
 427 const std::string& TemplateURLRef::GetHost(
 428     const SearchTermsData& search_terms_data) const {
 429   ParseIfNecessary(search_terms_data);
 430   return host_;
 431 }
 432
 433 const std::string& TemplateURLRef::GetPath(
 434     const SearchTermsData& search_terms_data) const {
 435   ParseIfNecessary(search_terms_data);
 436   return path_;
 437 }
 438
 439 const std::string& TemplateURLRef::GetSearchTermKey(
 440     const SearchTermsData& search_terms_data) const {
 441   ParseIfNecessary(search_terms_data);
 442   return search_term_key_;
 443 }
 444
 445 size_t TemplateURLRef::GetSearchTermPositionInPath(
 446     const SearchTermsData& search_terms_data) const {
 447   ParseIfNecessary(search_terms_data);
 448   return search_term_position_in_path_;
 449 }
 450
 451 url::Parsed::ComponentType TemplateURLRef::GetSearchTermKeyLocation(
 452     const SearchTermsData& search_terms_data) const {
 453   ParseIfNecessary(search_terms_data);
 454   return search_term_key_location_;
 455 }
 456
 457 base::string16 TemplateURLRef::SearchTermToString16(
 458     const std::string& term) const {
 459   const std::vector<std::string>& encodings = owner_->input_encodings();
 460   base::string16 result;
 461
 462   std::string unescaped = net::UnescapeURLComponent(
 463       term,
 464       net::UnescapeRule::REPLACE_PLUS_WITH_SPACE |
 465       net::UnescapeRule::URL_SPECIAL_CHARS);
 466   for (size_t i = 0; i < encodings.size(); ++i) {
 467     if (base::CodepageToUTF16(unescaped, encodings[i].c_str(),
 468                               base::OnStringConversionError::FAIL, &result))
 469       return result;
 470   }
 471
 472   // Always fall back on UTF-8 if it works.
 473   if (base::CodepageToUTF16(unescaped, base::kCodepageUTF8,
 474                             base::OnStringConversionError::FAIL, &result))
 475     return result;
 476
 477   // When nothing worked, just use the escaped text. We have no idea what the
 478   // encoding is. We need to substitute spaces for pluses ourselves since we're
 479   // not sending it through an unescaper.
 480   result = base::UTF8ToUTF16(term);
 481   std::replace(result.begin(), result.end(), '+', ' ');
 482   return result;
 483 }
 484
 485 bool TemplateURLRef::HasGoogleBaseURLs(
 486     const SearchTermsData& search_terms_data) const {
 487   ParseIfNecessary(search_terms_data);
 488   for (size_t i = 0; i < replacements_.size(); ++i) {
 489     if ((replacements_[i].type == GOOGLE_BASE_URL) ||
 490         (replacements_[i].type == GOOGLE_BASE_SUGGEST_URL))
 491       return true;
 492   }
 493   return false;
 494 }
 495
 496 bool TemplateURLRef::ExtractSearchTermsFromURL(
 497     const GURL& url,
 498     base::string16* search_terms,
 499     const SearchTermsData& search_terms_data,
 500     url::Parsed::ComponentType* search_terms_component,
 501     url::Component* search_terms_position) const {
 502   DCHECK(search_terms);
 503   search_terms->clear();
 504
 505   ParseIfNecessary(search_terms_data);
 506
 507   // We need a search term in the template URL to extract something.
 508   if (search_term_key_.empty() &&
 509       (search_term_key_location_ != url::Parsed::PATH))
 510     return false;
 511
 512   // Fill-in the replacements. We don't care about search terms in the pattern,
 513   // so we use the empty string.
 514   // Currently we assume the search term only shows in URL, not in post params.
 515   GURL pattern(ReplaceSearchTerms(SearchTermsArgs(base::string16()),
 516                                   search_terms_data, NULL));
 517   // Host, path and port must match.
 518   if ((url.port() != pattern.port()) ||
 519       (url.host() != host_) ||
 520       ((url.path() != path_) &&
 521           (search_term_key_location_ != url::Parsed::PATH))) {
 522     return false;
 523   }
 524
 525   std::string source;
 526   url::Component position;
 527   net::UnescapeRule::Type unescape_rules =
 528       net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS;
 529
 530   if (search_term_key_location_ == url::Parsed::PATH) {
 531     source = url.path();
 532
 533     // Characters in the path before and after search terms must match.
 534     if (source.length() < path_.length())
 535       return false;
 536     position.begin = search_term_position_in_path_;
 537     position.len = source.length() - path_.length();
 538     if (source.substr(0, position.begin) + source.substr(position.end()) !=
 539         path_)
 540       return false;
 541   } else {
 542     DCHECK(search_term_key_location_ == url::Parsed::QUERY ||
 543            search_term_key_location_ == url::Parsed::REF);
 544     source = (search_term_key_location_ == url::Parsed::QUERY) ?
 545         url.query() : url.ref();
 546
 547     url::Component query, key, value;
 548     query.len = static_cast<int>(source.size());
 549     bool key_found = false;
 550     while (url::ExtractQueryKeyValue(source.c_str(), &query, &key, &value)) {
 551       if (key.is_nonempty()) {
 552         if (source.substr(key.begin, key.len) == search_term_key_) {
 553           // Fail if search term key is found twice.
 554           if (key_found)
 555             return false;
 556           key_found = true;
 557           position = value;
 558         }
 559       }
 560     }
 561     if (!key_found)
 562       return false;
 563     unescape_rules |= net::UnescapeRule::REPLACE_PLUS_WITH_SPACE;
 564   }
 565
 566   // Extract the search term.
 567   *search_terms = net::UnescapeAndDecodeUTF8URLComponent(
 568       source.substr(position.begin, position.len), unescape_rules);
 569   if (search_terms_component)
 570     *search_terms_component = search_term_key_location_;
 571   if (search_terms_position)
 572     *search_terms_position = position;
 573   return true;
 574 }
 575
 576 void TemplateURLRef::InvalidateCachedValues() const {
 577   supports_replacements_ = valid_ = parsed_ = false;
 578   host_.clear();
 579   path_.clear();
 580   search_term_key_.clear();
 581   search_term_position_in_path_ = std::string::npos;
 582   search_term_key_location_ = url::Parsed::QUERY;
 583   replacements_.clear();
 584   post_params_.clear();
 585 }
 586
 587 bool TemplateURLRef::ParseParameter(size_t start,
 588                                     size_t end,
 589                                     std::string* url,
 590                                     Replacements* replacements) const {
 591   DCHECK(start != std::string::npos &&
 592          end != std::string::npos && end > start);
 593   size_t length = end - start - 1;
 594   bool optional = false;
 595   if ((*url)[end - 1] == kOptional) {
 596     optional = true;
 597     length--;
 598   }
 599   std::string parameter(url->substr(start + 1, length));
 600   std::string full_parameter(url->substr(start, end - start + 1));
 601   // Remove the parameter from the string.  For parameters who replacement is
 602   // constant and already known, just replace them directly.  For other cases,
 603   // like parameters whose values may change over time, use |replacements|.
 604   url->erase(start, end - start + 1);
 605   if (parameter == kSearchTermsParameter) {
 606     replacements->push_back(Replacement(SEARCH_TERMS, start));
 607   } else if (parameter == kCountParameter) {
 608     if (!optional)
 609       url->insert(start, kDefaultCount);
 610   } else if (parameter == kGoogleAssistedQueryStatsParameter) {
 611     replacements->push_back(Replacement(GOOGLE_ASSISTED_QUERY_STATS, start));
 612   } else if (parameter == kGoogleBaseURLParameter) {
 613     replacements->push_back(Replacement(GOOGLE_BASE_URL, start));
 614   } else if (parameter == kGoogleBaseSuggestURLParameter) {
 615     replacements->push_back(Replacement(GOOGLE_BASE_SUGGEST_URL, start));
 616   } else if (parameter == kGoogleBookmarkBarPinnedParameter) {
 617     replacements->push_back(Replacement(GOOGLE_BOOKMARK_BAR_PINNED, start));
 618   } else if (parameter == kGoogleCurrentPageUrlParameter) {
 619     replacements->push_back(Replacement(GOOGLE_CURRENT_PAGE_URL, start));
 620   } else if (parameter == kGoogleCursorPositionParameter) {
 621     replacements->push_back(Replacement(GOOGLE_CURSOR_POSITION, start));
 622   } else if (parameter == kGoogleForceInstantResultsParameter) {
 623     replacements->push_back(Replacement(GOOGLE_FORCE_INSTANT_RESULTS, start));
 624   } else if (parameter == kGoogleImageOriginalHeight) {
 625     replacements->push_back(
 626         Replacement(TemplateURLRef::GOOGLE_IMAGE_ORIGINAL_HEIGHT, start));
 627   } else if (parameter == kGoogleImageOriginalWidth) {
 628     replacements->push_back(
 629         Replacement(TemplateURLRef::GOOGLE_IMAGE_ORIGINAL_WIDTH, start));
 630   } else if (parameter == kGoogleImageSearchSource) {
 631     replacements->push_back(
 632         Replacement(TemplateURLRef::GOOGLE_IMAGE_SEARCH_SOURCE, start));
 633   } else if (parameter == kGoogleImageThumbnailParameter) {
 634     replacements->push_back(
 635         Replacement(TemplateURLRef::GOOGLE_IMAGE_THUMBNAIL, start));
 636   } else if (parameter == kGoogleImageURLParameter) {
 637     replacements->push_back(Replacement(TemplateURLRef::GOOGLE_IMAGE_URL,
 638                                         start));
 639   } else if (parameter == kGoogleInputTypeParameter) {
 640     replacements->push_back(Replacement(TemplateURLRef::GOOGLE_INPUT_TYPE,
 641                                         start));
 642   } else if (parameter == kGoogleInstantExtendedEnabledParameter) {
 643     replacements->push_back(Replacement(GOOGLE_INSTANT_EXTENDED_ENABLED,
 644                                         start));
 645   } else if (parameter == kGoogleInstantExtendedEnabledKey) {
 646     url->insert(start, google_util::kInstantExtendedAPIParam);
 647   } else if (parameter == kGoogleNTPIsThemedParameter) {
 648     replacements->push_back(Replacement(GOOGLE_NTP_IS_THEMED, start));
 649   } else if (parameter == kGoogleOmniboxStartMarginParameter) {
 650     replacements->push_back(Replacement(GOOGLE_OMNIBOX_START_MARGIN, start));
 651   } else if (parameter == kGoogleContextualSearchVersion) {
 652     replacements->push_back(
 653         Replacement(GOOGLE_CONTEXTUAL_SEARCH_VERSION, start));
 654   } else if (parameter == kGoogleContextualSearchContextData) {
 655     replacements->push_back(
 656         Replacement(GOOGLE_CONTEXTUAL_SEARCH_CONTEXT_DATA, start));
 657   } else if (parameter == kGoogleOriginalQueryForSuggestionParameter) {
 658     replacements->push_back(Replacement(GOOGLE_ORIGINAL_QUERY_FOR_SUGGESTION,
 659                                         start));
 660   } else if (parameter == kGooglePageClassificationParameter) {
 661     replacements->push_back(Replacement(GOOGLE_PAGE_CLASSIFICATION, start));
 662   } else if (parameter == kGooglePrefetchQuery) {
 663     replacements->push_back(Replacement(GOOGLE_PREFETCH_QUERY, start));
 664   } else if (parameter == kGoogleRLZParameter) {
 665     replacements->push_back(Replacement(GOOGLE_RLZ, start));
 666   } else if (parameter == kGoogleSearchClient) {
 667     replacements->push_back(Replacement(GOOGLE_SEARCH_CLIENT, start));
 668   } else if (parameter == kGoogleSearchFieldtrialParameter) {
 669     replacements->push_back(Replacement(GOOGLE_SEARCH_FIELDTRIAL_GROUP, start));
 670   } else if (parameter == kGoogleSearchVersion) {
 671     replacements->push_back(Replacement(GOOGLE_SEARCH_VERSION, start));
 672   } else if (parameter == kGoogleSessionToken) {
 673     replacements->push_back(Replacement(GOOGLE_SESSION_TOKEN, start));
 674   } else if (parameter == kGoogleSourceIdParameter) {
 675 #if defined(OS_ANDROID)
 676     url->insert(start, "sourceid=chrome-mobile&");
 677 #else
 678     url->insert(start, "sourceid=chrome&");
 679 #endif
 680   } else if (parameter == kGoogleSuggestAPIKeyParameter) {
 681     url->insert(start,
 682                 net::EscapeQueryParamValue(google_apis::GetAPIKey(), false));
 683   } else if (parameter == kGoogleSuggestClient) {
 684     replacements->push_back(Replacement(GOOGLE_SUGGEST_CLIENT, start));
 685   } else if (parameter == kGoogleSuggestRequestId) {
 686     replacements->push_back(Replacement(GOOGLE_SUGGEST_REQUEST_ID, start));
 687   } else if (parameter == kGoogleUnescapedSearchTermsParameter) {
 688     replacements->push_back(Replacement(GOOGLE_UNESCAPED_SEARCH_TERMS, start));
 689   } else if (parameter == kInputEncodingParameter) {
 690     replacements->push_back(Replacement(ENCODING, start));
 691   } else if (parameter == kLanguageParameter) {
 692     replacements->push_back(Replacement(LANGUAGE, start));
 693   } else if (parameter == kOutputEncodingParameter) {
 694     if (!optional)
 695       url->insert(start, kOutputEncodingType);
 696   } else if ((parameter == kStartIndexParameter) ||
 697              (parameter == kStartPageParameter)) {
 698     // We don't support these.
 699     if (!optional)
 700       url->insert(start, "1");
 701   } else if (!prepopulated_) {
 702     // If it's a prepopulated URL, we know that it's safe to remove unknown
 703     // parameters, so just ignore this and return true below. Otherwise it could
 704     // be some garbage but can also be a javascript block. Put it back.
 705     url->insert(start, full_parameter);
 706     return false;
 707   }
 708   return true;
 709 }
 710
 711 std::string TemplateURLRef::ParseURL(const std::string& url,
 712                                      Replacements* replacements,
 713                                      PostParams* post_params,
 714                                      bool* valid) const {
 715   *valid = false;
 716   std::string parsed_url = url;
 717   for (size_t last = 0; last != std::string::npos; ) {
 718     last = parsed_url.find(kStartParameter, last);
 719     if (last != std::string::npos) {
 720       size_t template_end = parsed_url.find(kEndParameter, last);
 721       if (template_end != std::string::npos) {
 722         // Since we allow Javascript in the URL, {} pairs could be nested. Match
 723         // only leaf pairs with supported parameters.
 724         size_t next_template_start = parsed_url.find(kStartParameter, last + 1);
 725         if (next_template_start == std::string::npos ||
 726             next_template_start > template_end) {
 727           // If successful, ParseParameter erases from the string as such no
 728           // need to update |last|. If failed, move |last| to the end of pair.
 729           if (!ParseParameter(last, template_end, &parsed_url, replacements)) {
 730             // |template_end| + 1 may be beyond the end of the string.
 731             last = template_end;
 732           }
 733         } else {
 734           last = next_template_start;
 735         }
 736       } else {
 737         // Open brace without a closing brace, return.
 738         return std::string();
 739       }
 740     }
 741   }
 742
 743   // Handles the post parameters.
 744   const std::string& post_params_string = GetPostParamsString();
 745   if (!post_params_string.empty()) {
 746     typedef std::vector<std::string> Strings;
 747     Strings param_list;
 748     base::SplitString(post_params_string, ',', &param_list);
 749
 750     for (Strings::const_iterator iterator = param_list.begin();
 751          iterator != param_list.end(); ++iterator) {
 752       Strings parts;
 753       // The '=' delimiter is required and the name must be not empty.
 754       base::SplitString(*iterator, '=', &parts);
 755       if ((parts.size() != 2U) || parts[0].empty())
 756         return std::string();
 757
 758       std::string& value = parts[1];
 759       size_t replacements_size = replacements->size();
 760       if (IsTemplateParameterString(value))
 761         ParseParameter(0, value.length() - 1, &value, replacements);
 762       PostParam param = { parts[0], value };
 763       post_params->push_back(param);
 764       // If there was a replacement added, points its index to last added
 765       // PostParam.
 766       if (replacements->size() > replacements_size) {
 767         DCHECK_EQ(replacements_size + 1, replacements->size());
 768         Replacement* r = &replacements->back();
 769         r->is_post_param = true;
 770         r->index = post_params->size() - 1;
 771       }
 772     }
 773     DCHECK(!post_params->empty());
 774   }
 775
 776   *valid = true;
 777   return parsed_url;
 778 }
 779
 780 void TemplateURLRef::ParseIfNecessary(
 781     const SearchTermsData& search_terms_data) const {
 782   if (!parsed_) {
 783     InvalidateCachedValues();
 784     parsed_ = true;
 785     parsed_url_ = ParseURL(GetURL(), &replacements_, &post_params_, &valid_);
 786     supports_replacements_ = false;
 787     if (valid_) {
 788       bool has_only_one_search_term = false;
 789       for (Replacements::const_iterator i = replacements_.begin();
 790            i != replacements_.end(); ++i) {
 791         if ((i->type == SEARCH_TERMS) ||
 792             (i->type == GOOGLE_UNESCAPED_SEARCH_TERMS)) {
 793           if (has_only_one_search_term) {
 794             has_only_one_search_term = false;
 795             break;
 796           }
 797           has_only_one_search_term = true;
 798           supports_replacements_ = true;
 799         }
 800       }
 801       // Only parse the host/key if there is one search term. Technically there
 802       // could be more than one term, but it's uncommon; so we punt.
 803       if (has_only_one_search_term)
 804         ParseHostAndSearchTermKey(search_terms_data);
 805     }
 806   }
 807 }
 808
 809 void TemplateURLRef::ParseHostAndSearchTermKey(
 810     const SearchTermsData& search_terms_data) const {
 811   std::string url_string(GetURL());
 812   ReplaceSubstringsAfterOffset(&url_string, 0,
 813                                kGoogleBaseURLParameterFull,
 814                                search_terms_data.GoogleBaseURLValue());
 815   ReplaceSubstringsAfterOffset(&url_string, 0,
 816                                kGoogleBaseSuggestURLParameterFull,
 817                                search_terms_data.GoogleBaseSuggestURLValue());
 818
 819   search_term_key_.clear();
 820   search_term_position_in_path_ = std::string::npos;
 821   host_.clear();
 822   path_.clear();
 823   search_term_key_location_ = url::Parsed::QUERY;
 824
 825   GURL url(url_string);
 826   if (!url.is_valid())
 827     return;
 828
 829   std::string query_key = FindSearchTermsKey(url.query());
 830   std::string ref_key = FindSearchTermsKey(url.ref());
 831   url::Component parameter_position;
 832   const bool in_query = !query_key.empty();
 833   const bool in_ref = !ref_key.empty();
 834   const bool in_path = FindSearchTermsInPath(url.path(), &parameter_position);
 835   if (in_query ? (in_ref || in_path) : (in_ref == in_path))
 836     return;  // No key or multiple keys found.  We only handle having one key.
 837
 838   host_ = url.host();
 839   path_ = url.path();
 840   if (in_query) {
 841     search_term_key_ = query_key;
 842     search_term_key_location_ = url::Parsed::QUERY;
 843   } else if (in_ref) {
 844     search_term_key_ = ref_key;
 845     search_term_key_location_ = url::Parsed::REF;
 846   } else {
 847     DCHECK(in_path);
 848     DCHECK_GE(parameter_position.begin, 1);  // Path must start with '/'.
 849     search_term_key_location_ = url::Parsed::PATH;
 850     search_term_position_in_path_ = parameter_position.begin;
 851     // Remove the "{searchTerms}" itself from |path_|.
 852     path_.erase(parameter_position.begin, parameter_position.len);
 853   }
 854 }
 855
 856 void TemplateURLRef::HandleReplacement(const std::string& name,
 857                                        const std::string& value,
 858                                        const Replacement& replacement,
 859                                        std::string* url) const {
 860   size_t pos = replacement.index;
 861   if (replacement.is_post_param) {
 862     DCHECK_LT(pos, post_params_.size());
 863     DCHECK(!post_params_[pos].name.empty());
 864     post_params_[pos].value = value;
 865   } else {
 866     url->insert(pos, name.empty() ? value : (name + "=" + value + "&"));
 867   }
 868 }
 869
 870 std::string TemplateURLRef::HandleReplacements(
 871     const SearchTermsArgs& search_terms_args,
 872     const SearchTermsData& search_terms_data,
 873     PostContent* post_content) const {
 874   if (replacements_.empty()) {
 875     if (!post_params_.empty())
 876       EncodeFormData(post_params_, post_content);
 877     return parsed_url_;
 878   }
 879
 880   // Determine if the search terms are in the query or before. We're escaping
 881   // space as '+' in the former case and as '%20' in the latter case.
 882   bool is_in_query = true;
 883   for (Replacements::iterator i = replacements_.begin();
 884        i != replacements_.end(); ++i) {
 885     if (i->type == SEARCH_TERMS) {
 886       base::string16::size_type query_start = parsed_url_.find('?');
 887       is_in_query = query_start != base::string16::npos &&
 888           (static_cast<base::string16::size_type>(i->index) > query_start);
 889       break;
 890     }
 891   }
 892
 893   std::string input_encoding;
 894   base::string16 encoded_terms;
 895   base::string16 encoded_original_query;
 896   owner_->EncodeSearchTerms(search_terms_args, is_in_query, &input_encoding,
 897                             &encoded_terms, &encoded_original_query);
 898
 899   std::string url = parsed_url_;
 900
 901   // replacements_ is ordered in ascending order, as such we need to iterate
 902   // from the back.
 903   for (Replacements::reverse_iterator i = replacements_.rbegin();
 904        i != replacements_.rend(); ++i) {
 905     switch (i->type) {
 906       case ENCODING:
 907         HandleReplacement(std::string(), input_encoding, *i, &url);
 908         break;
 909
 910       case GOOGLE_ASSISTED_QUERY_STATS:
 911         DCHECK(!i->is_post_param);
 912         if (!search_terms_args.assisted_query_stats.empty()) {
 913           // Get the base URL without substituting AQS to avoid infinite
 914           // recursion.  We need the URL to find out if it meets all
 915           // AQS requirements (e.g. HTTPS protocol check).
 916           // See TemplateURLRef::SearchTermsArgs for more details.
 917           SearchTermsArgs search_terms_args_without_aqs(search_terms_args);
 918           search_terms_args_without_aqs.assisted_query_stats.clear();
 919           GURL base_url(ReplaceSearchTerms(
 920               search_terms_args_without_aqs, search_terms_data, NULL));
 921           if (base_url.SchemeIs(url::kHttpsScheme)) {
 922             HandleReplacement(
 923                 "aqs", search_terms_args.assisted_query_stats, *i, &url);
 924           }
 925         }
 926         break;
 927
 928       case GOOGLE_BASE_URL:
 929         DCHECK(!i->is_post_param);
 930         HandleReplacement(
 931             std::string(), search_terms_data.GoogleBaseURLValue(), *i, &url);
 932         break;
 933
 934       case GOOGLE_BASE_SUGGEST_URL:
 935         DCHECK(!i->is_post_param);
 936         HandleReplacement(
 937             std::string(), search_terms_data.GoogleBaseSuggestURLValue(), *i,
 938             &url);
 939         break;
 940
 941       case GOOGLE_BOOKMARK_BAR_PINNED:
 942         if (search_terms_data.IsShowingSearchTermsOnSearchResultsPages()) {
 943           // Log whether the bookmark bar is pinned when the user is seeing
 944           // InstantExtended on the SRP.
 945           DCHECK(!i->is_post_param);
 946           HandleReplacement(
 947               "bmbp", search_terms_args.bookmark_bar_pinned ? "1" : "0", *i,
 948               &url);
 949         }
 950         break;
 951
 952       case GOOGLE_CURRENT_PAGE_URL:
 953         DCHECK(!i->is_post_param);
 954         if (!search_terms_args.current_page_url.empty()) {
 955           const std::string& escaped_current_page_url =
 956               net::EscapeQueryParamValue(search_terms_args.current_page_url,
 957                                          true);
 958           HandleReplacement("url", escaped_current_page_url, *i, &url);
 959         }
 960         break;
 961
 962       case GOOGLE_CURSOR_POSITION:
 963         DCHECK(!i->is_post_param);
 964         if (search_terms_args.cursor_position != base::string16::npos)
 965           HandleReplacement(
 966               "cp",
 967               base::StringPrintf("%" PRIuS, search_terms_args.cursor_position),
 968               *i,
 969               &url);
 970         break;
 971
 972       case GOOGLE_FORCE_INSTANT_RESULTS:
 973         DCHECK(!i->is_post_param);
 974         HandleReplacement(std::string(),
 975                           search_terms_data.ForceInstantResultsParam(
 976                               search_terms_args.force_instant_results),
 977                           *i,
 978                           &url);
 979         break;
 980
 981       case GOOGLE_INPUT_TYPE:
 982         DCHECK(!i->is_post_param);
 983         HandleReplacement(
 984             "oit", base::IntToString(search_terms_args.input_type), *i, &url);
 985         break;
 986
 987       case GOOGLE_INSTANT_EXTENDED_ENABLED:
 988         DCHECK(!i->is_post_param);
 989         HandleReplacement(std::string(),
 990                           search_terms_data.InstantExtendedEnabledParam(
 991                               type_ == SEARCH),
 992                           *i,
 993                           &url);
 994         break;
 995
 996       case GOOGLE_NTP_IS_THEMED:
 997         DCHECK(!i->is_post_param);
 998         HandleReplacement(
 999             std::string(), search_terms_data.NTPIsThemedParam(), *i, &url);
1000         break;
1001
1002       case GOOGLE_OMNIBOX_START_MARGIN:
1003         DCHECK(!i->is_post_param);
1004         if (search_terms_args.enable_omnibox_start_margin) {
1005           int omnibox_start_margin = search_terms_data.OmniboxStartMargin();
1006           if (omnibox_start_margin >= 0) {
1007             HandleReplacement("es_sm", base::IntToString(omnibox_start_margin),
1008                               *i, &url);
1009           }
1010         }
1011         break;
1012
1013       case GOOGLE_CONTEXTUAL_SEARCH_VERSION:
1014         if (search_terms_args.contextual_search_params.version >= 0) {
1015           HandleReplacement(
1016               "ctxs",
1017               base::IntToString(
1018                   search_terms_args.contextual_search_params.version),
1019               *i,
1020               &url);
1021         }
1022         break;
1023
1024       case GOOGLE_CONTEXTUAL_SEARCH_CONTEXT_DATA: {
1025         DCHECK(!i->is_post_param);
1026         std::string context_data;
1027
1028         const SearchTermsArgs::ContextualSearchParams& params =
1029             search_terms_args.contextual_search_params;
1030
1031         if (params.start != std::string::npos) {
1032           context_data.append("ctxs_start=" + base::IntToString(
1033               params.start) + "&");
1034         }
1035
1036         if (params.end != std::string::npos) {
1037           context_data.append("ctxs_end=" + base::IntToString(
1038               params.end) + "&");
1039         }
1040
1041         if (!params.selection.empty())
1042           context_data.append("q=" + params.selection + "&");
1043
1044         if (!params.content.empty())
1045           context_data.append("ctxs_content=" + params.content + "&");
1046
1047         if (!params.base_page_url.empty())
1048           context_data.append("ctxsl_url=" + params.base_page_url + "&");
1049
1050         if (!params.encoding.empty()) {
1051           context_data.append("ctxs_encoding=" + params.encoding + "&");
1052         }
1053
1054         context_data.append(
1055             params.resolve ? "ctxsl_resolve=1" : "ctxsl_resolve=0");
1056
1057         HandleReplacement(std::string(), context_data, *i, &url);
1058         break;
1059       }
1060
1061       case GOOGLE_ORIGINAL_QUERY_FOR_SUGGESTION:
1062         DCHECK(!i->is_post_param);
1063         if (search_terms_args.accepted_suggestion >= 0 ||
1064             !search_terms_args.assisted_query_stats.empty()) {
1065           HandleReplacement(
1066               "oq", base::UTF16ToUTF8(encoded_original_query), *i, &url);
1067         }
1068         break;
1069
1070       case GOOGLE_PAGE_CLASSIFICATION:
1071         if (search_terms_args.page_classification !=
1072             metrics::OmniboxEventProto::INVALID_SPEC) {
1073           HandleReplacement(
1074               "pgcl", base::IntToString(search_terms_args.page_classification),
1075               *i, &url);
1076         }
1077         break;
1078
1079       case GOOGLE_PREFETCH_QUERY: {
1080         const std::string& query = search_terms_args.prefetch_query;
1081         const std::string& type = search_terms_args.prefetch_query_type;
1082         if (!query.empty() && !type.empty()) {
1083           HandleReplacement(
1084               std::string(), "pfq=" + query + "&qha=" + type + "&", *i, &url);
1085         }
1086         break;
1087       }
1088
1089       case GOOGLE_RLZ: {
1090         DCHECK(!i->is_post_param);
1091         // On platforms that don't have RLZ, we still want this branch
1092         // to happen so that we replace the RLZ template with the
1093         // empty string.  (If we don't handle this case, we hit a
1094         // NOTREACHED below.)
1095         base::string16 rlz_string = search_terms_data.GetRlzParameterValue(
1096             search_terms_args.from_app_list);
1097         if (!rlz_string.empty()) {
1098           HandleReplacement("rlz", base::UTF16ToUTF8(rlz_string), *i, &url);
1099         }
1100         break;
1101       }
1102
1103       case GOOGLE_SEARCH_CLIENT: {
1104         DCHECK(!i->is_post_param);
1105         std::string client = search_terms_data.GetSearchClient();
1106         if (!client.empty())
1107           HandleReplacement("client", client, *i, &url);
1108         break;
1109       }
1110
1111       case GOOGLE_SEARCH_FIELDTRIAL_GROUP:
1112         // We are not currently running any fieldtrials that modulate the search
1113         // url.  If we do, then we'd have some conditional insert such as:
1114         // url.insert(i->index, used_www ? "gcx=w&" : "gcx=c&");
1115         break;
1116
1117       case GOOGLE_SEARCH_VERSION:
1118         if (search_terms_data.EnableAnswersInSuggest())
1119           HandleReplacement("gs_rn", "42", *i, &url);
1120         break;
1121
1122       case GOOGLE_SESSION_TOKEN: {
1123         std::string token = search_terms_args.session_token;
1124         if (!token.empty())
1125           HandleReplacement("psi", token, *i, &url);
1126         break;
1127       }
1128
1129       case GOOGLE_SUGGEST_CLIENT:
1130         HandleReplacement(
1131             std::string(), search_terms_data.GetSuggestClient(), *i, &url);
1132         break;
1133
1134       case GOOGLE_SUGGEST_REQUEST_ID:
1135         HandleReplacement(
1136             std::string(), search_terms_data.GetSuggestRequestIdentifier(), *i,
1137             &url);
1138         break;
1139
1140       case GOOGLE_UNESCAPED_SEARCH_TERMS: {
1141         std::string unescaped_terms;
1142         base::UTF16ToCodepage(search_terms_args.search_terms,
1143                               input_encoding.c_str(),
1144                               base::OnStringConversionError::SKIP,
1145                               &unescaped_terms);
1146         HandleReplacement(std::string(), unescaped_terms, *i, &url);
1147         break;
1148       }
1149
1150       case LANGUAGE:
1151         HandleReplacement(
1152             std::string(), search_terms_data.GetApplicationLocale(), *i, &url);
1153         break;
1154
1155       case SEARCH_TERMS:
1156         HandleReplacement(
1157             std::string(), base::UTF16ToUTF8(encoded_terms), *i, &url);
1158         break;
1159
1160       case GOOGLE_IMAGE_THUMBNAIL:
1161         HandleReplacement(
1162             std::string(), search_terms_args.image_thumbnail_content, *i, &url);
1163         post_params_[i->index].content_type = "image/jpeg";
1164         break;
1165
1166       case GOOGLE_IMAGE_URL:
1167         if (search_terms_args.image_url.is_valid()) {
1168           HandleReplacement(
1169               std::string(), search_terms_args.image_url.spec(), *i, &url);
1170         }
1171         break;
1172
1173       case GOOGLE_IMAGE_ORIGINAL_WIDTH:
1174         if (!search_terms_args.image_original_size.IsEmpty()) {
1175           HandleReplacement(
1176               std::string(),
1177               base::IntToString(search_terms_args.image_original_size.width()),
1178               *i, &url);
1179         }
1180         break;
1181
1182       case GOOGLE_IMAGE_ORIGINAL_HEIGHT:
1183         if (!search_terms_args.image_original_size.IsEmpty()) {
1184           HandleReplacement(
1185               std::string(),
1186               base::IntToString(search_terms_args.image_original_size.height()),
1187               *i, &url);
1188         }
1189         break;
1190
1191       case GOOGLE_IMAGE_SEARCH_SOURCE:
1192         HandleReplacement(
1193             std::string(), search_terms_data.GoogleImageSearchSource(), *i,
1194             &url);
1195         break;
1196
1197       default:
1198         NOTREACHED();
1199         break;
1200     }
1201   }
1202
1203   if (!post_params_.empty())
1204     EncodeFormData(post_params_, post_content);
1205
1206   return url;
1207 }
1208
1209
1210 // TemplateURL ----------------------------------------------------------------
1211
1212 TemplateURL::AssociatedExtensionInfo::AssociatedExtensionInfo(
1213     Type type,
1214     const std::string& extension_id)
1215     : type(type),
1216       extension_id(extension_id),
1217       wants_to_be_default_engine(false) {
1218   DCHECK_NE(NORMAL, type);
1219 }
1220
1221 TemplateURL::AssociatedExtensionInfo::~AssociatedExtensionInfo() {
1222 }
1223
1224 TemplateURL::TemplateURL(const TemplateURLData& data)
1225     : data_(data),
1226       url_ref_(this, TemplateURLRef::SEARCH),
1227       suggestions_url_ref_(this,
1228                            TemplateURLRef::SUGGEST),
1229       instant_url_ref_(this,
1230                        TemplateURLRef::INSTANT),
1231       image_url_ref_(this, TemplateURLRef::IMAGE),
1232       new_tab_url_ref_(this, TemplateURLRef::NEW_TAB),
1233       contextual_search_url_ref_(this, TemplateURLRef::CONTEXTUAL_SEARCH) {
1234   SetPrepopulateId(data_.prepopulate_id);
1235
1236   if (data_.search_terms_replacement_key ==
1237       kGoogleInstantExtendedEnabledKeyFull) {
1238     data_.search_terms_replacement_key = google_util::kInstantExtendedAPIParam;
1239   }
1240 }
1241
1242 TemplateURL::~TemplateURL() {
1243 }
1244
1245 // static
1246 base::string16 TemplateURL::GenerateKeyword(const GURL& url) {
1247   DCHECK(url.is_valid());
1248   // Strip "www." off the front of the keyword; otherwise the keyword won't work
1249   // properly.  See http://code.google.com/p/chromium/issues/detail?id=6984 .
1250   // Special case: if the host was exactly "www." (not sure this can happen but
1251   // perhaps with some weird intranet and custom DNS server?), ensure we at
1252   // least don't return the empty string.
1253   base::string16 keyword(net::StripWWWFromHost(url));
1254   return keyword.empty() ? base::ASCIIToUTF16("www") : keyword;
1255 }
1256
1257 // static
1258 GURL TemplateURL::GenerateFaviconURL(const GURL& url) {
1259   DCHECK(url.is_valid());
1260   GURL::Replacements rep;
1261
1262   const char favicon_path[] = "/favicon.ico";
1263   int favicon_path_len = arraysize(favicon_path) - 1;
1264
1265   rep.SetPath(favicon_path, url::Component(0, favicon_path_len));
1266   rep.ClearUsername();
1267   rep.ClearPassword();
1268   rep.ClearQuery();
1269   rep.ClearRef();
1270   return url.ReplaceComponents(rep);
1271 }
1272
1273 // static
1274 bool TemplateURL::MatchesData(const TemplateURL* t_url,
1275                               const TemplateURLData* data,
1276                               const SearchTermsData& search_terms_data) {
1277   if (!t_url || !data)
1278     return !t_url && !data;
1279
1280   return (t_url->short_name() == data->short_name) &&
1281       t_url->HasSameKeywordAs(*data, search_terms_data) &&
1282       (t_url->url() == data->url()) &&
1283       (t_url->suggestions_url() == data->suggestions_url) &&
1284       (t_url->instant_url() == data->instant_url) &&
1285       (t_url->image_url() == data->image_url) &&
1286       (t_url->new_tab_url() == data->new_tab_url) &&
1287       (t_url->search_url_post_params() == data->search_url_post_params) &&
1288       (t_url->suggestions_url_post_params() ==
1289           data->suggestions_url_post_params) &&
1290       (t_url->instant_url_post_params() == data->instant_url_post_params) &&
1291       (t_url->image_url_post_params() == data->image_url_post_params) &&
1292       (t_url->favicon_url() == data->favicon_url) &&
1293       (t_url->safe_for_autoreplace() == data->safe_for_autoreplace) &&
1294       (t_url->show_in_default_list() == data->show_in_default_list) &&
1295       (t_url->input_encodings() == data->input_encodings) &&
1296       (t_url->alternate_urls() == data->alternate_urls) &&
1297       (t_url->search_terms_replacement_key() ==
1298           data->search_terms_replacement_key);
1299 }
1300
1301 base::string16 TemplateURL::AdjustedShortNameForLocaleDirection() const {
1302   base::string16 bidi_safe_short_name = data_.short_name;
1303   base::i18n::AdjustStringForLocaleDirection(&bidi_safe_short_name);
1304   return bidi_safe_short_name;
1305 }
1306
1307 bool TemplateURL::ShowInDefaultList(
1308     const SearchTermsData& search_terms_data) const {
1309   return data_.show_in_default_list &&
1310       url_ref_.SupportsReplacement(search_terms_data);
1311 }
1312
1313 bool TemplateURL::SupportsReplacement(
1314     const SearchTermsData& search_terms_data) const {
1315   return url_ref_.SupportsReplacement(search_terms_data);
1316 }
1317
1318 bool TemplateURL::HasGoogleBaseURLs(
1319     const SearchTermsData& search_terms_data) const {
1320   return url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1321       suggestions_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1322       instant_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1323       image_url_ref_.HasGoogleBaseURLs(search_terms_data) ||
1324       new_tab_url_ref_.HasGoogleBaseURLs(search_terms_data);
1325 }
1326
1327 bool TemplateURL::IsGoogleSearchURLWithReplaceableKeyword(
1328     const SearchTermsData& search_terms_data) const {
1329   return (GetType() == NORMAL) &&
1330       url_ref_.HasGoogleBaseURLs(search_terms_data) &&
1331       google_util::IsGoogleHostname(base::UTF16ToUTF8(data_.keyword()),
1332                                     google_util::DISALLOW_SUBDOMAIN);
1333 }
1334
1335 bool TemplateURL::HasSameKeywordAs(
1336     const TemplateURLData& other,
1337     const SearchTermsData& search_terms_data) const {
1338   return (data_.keyword() == other.keyword()) ||
1339       (IsGoogleSearchURLWithReplaceableKeyword(search_terms_data) &&
1340        TemplateURL(other).IsGoogleSearchURLWithReplaceableKeyword(
1341            search_terms_data));
1342 }
1343
1344 TemplateURL::Type TemplateURL::GetType() const {
1345   return extension_info_ ? extension_info_->type : NORMAL;
1346 }
1347
1348 std::string TemplateURL::GetExtensionId() const {
1349   DCHECK(extension_info_);
1350   return extension_info_->extension_id;
1351 }
1352
1353 size_t TemplateURL::URLCount() const {
1354   // Add 1 for the regular search URL.
1355   return data_.alternate_urls.size() + 1;
1356 }
1357
1358 const std::string& TemplateURL::GetURL(size_t index) const {
1359   DCHECK_LT(index, URLCount());
1360
1361   return (index < data_.alternate_urls.size()) ?
1362       data_.alternate_urls[index] : url();
1363 }
1364
1365 bool TemplateURL::ExtractSearchTermsFromURL(
1366     const GURL& url,
1367     const SearchTermsData& search_terms_data,
1368     base::string16* search_terms) {
1369   return FindSearchTermsInURL(url, search_terms_data, search_terms, NULL, NULL);
1370 }
1371
1372 bool TemplateURL::IsSearchURL(
1373     const GURL& url,
1374     const SearchTermsData& search_terms_data) {
1375   base::string16 search_terms;
1376   return ExtractSearchTermsFromURL(url, search_terms_data, &search_terms) &&
1377       !search_terms.empty();
1378 }
1379
1380 bool TemplateURL::HasSearchTermsReplacementKey(const GURL& url) const {
1381   // Look for the key both in the query and the ref.
1382   std::string params[] = {url.query(), url.ref()};
1383
1384   for (int i = 0; i < 2; ++i) {
1385     url::Component query, key, value;
1386     query.len = static_cast<int>(params[i].size());
1387     while (url::ExtractQueryKeyValue(params[i].c_str(), &query, &key, &value)) {
1388       if (key.is_nonempty() &&
1389           params[i].substr(key.begin, key.len) ==
1390               search_terms_replacement_key()) {
1391         return true;
1392       }
1393     }
1394   }
1395   return false;
1396 }
1397
1398 bool TemplateURL::ReplaceSearchTermsInURL(
1399     const GURL& url,
1400     const TemplateURLRef::SearchTermsArgs& search_terms_args,
1401     const SearchTermsData& search_terms_data,
1402     GURL* result) {
1403   // TODO(beaudoin): Use AQS from |search_terms_args| too.
1404   url::Parsed::ComponentType search_term_component;
1405   url::Component search_terms_position;
1406   base::string16 search_terms;
1407   if (!FindSearchTermsInURL(url, search_terms_data, &search_terms,
1408                             &search_term_component, &search_terms_position)) {
1409     return false;
1410   }
1411   DCHECK(search_terms_position.is_nonempty());
1412
1413   // Query and ref are encoded in the same way.
1414   const bool is_in_query = (search_term_component != url::Parsed::PATH);
1415
1416   std::string input_encoding;
1417   base::string16 encoded_terms;
1418   base::string16 encoded_original_query;
1419   EncodeSearchTerms(search_terms_args, is_in_query, &input_encoding,
1420                     &encoded_terms, &encoded_original_query);
1421
1422   std::string old_params;
1423   if (search_term_component == url::Parsed::QUERY) {
1424     old_params = url.query();
1425   } else if (search_term_component == url::Parsed::REF) {
1426     old_params = url.ref();
1427   } else {
1428     DCHECK_EQ(search_term_component, url::Parsed::PATH);
1429     old_params = url.path();
1430   }
1431
1432   std::string new_params(old_params, 0, search_terms_position.begin);
1433   new_params += base::UTF16ToUTF8(search_terms_args.search_terms);
1434   new_params += old_params.substr(search_terms_position.end());
1435   GURL::Replacements replacements;
1436
1437   if (search_term_component == url::Parsed::QUERY) {
1438     replacements.SetQueryStr(new_params);
1439   } else if (search_term_component == url::Parsed::REF) {
1440     replacements.SetRefStr(new_params);
1441   } else {
1442     DCHECK_EQ(search_term_component, url::Parsed::PATH);
1443     replacements.SetPathStr(new_params);
1444   }
1445
1446   *result = url.ReplaceComponents(replacements);
1447   return true;
1448 }
1449
1450 void TemplateURL::EncodeSearchTerms(
1451     const TemplateURLRef::SearchTermsArgs& search_terms_args,
1452     bool is_in_query,
1453     std::string* input_encoding,
1454     base::string16* encoded_terms,
1455     base::string16* encoded_original_query) const {
1456
1457   std::vector<std::string> encodings(input_encodings());
1458   if (std::find(encodings.begin(), encodings.end(), "UTF-8") == encodings.end())
1459     encodings.push_back("UTF-8");
1460   for (std::vector<std::string>::const_iterator i(encodings.begin());
1461        i != encodings.end(); ++i) {
1462     if (TryEncoding(search_terms_args.search_terms,
1463                     search_terms_args.original_query, i->c_str(),
1464                     is_in_query, encoded_terms, encoded_original_query)) {
1465       *input_encoding = *i;
1466       return;
1467     }
1468   }
1469   NOTREACHED();
1470 }
1471
1472 GURL TemplateURL::GenerateSearchURL(
1473     const SearchTermsData& search_terms_data) const {
1474   if (!url_ref_.IsValid(search_terms_data))
1475     return GURL();
1476
1477   if (!url_ref_.SupportsReplacement(search_terms_data))
1478     return GURL(url());
1479
1480   // Use something obscure for the search terms argument so that in the rare
1481   // case the term replaces the URL it's unlikely another keyword would have the
1482   // same url.
1483   // TODO(jnd): Add additional parameters to get post data when the search URL
1484   // has post parameters.
1485   return GURL(url_ref_.ReplaceSearchTerms(
1486       TemplateURLRef::SearchTermsArgs(
1487           base::ASCIIToUTF16("blah.blah.blah.blah.blah")),
1488       search_terms_data, NULL));
1489 }
1490
1491 void TemplateURL::CopyFrom(const TemplateURL& other) {
1492   if (this == &other)
1493     return;
1494
1495   data_ = other.data_;
1496   url_ref_.InvalidateCachedValues();
1497   suggestions_url_ref_.InvalidateCachedValues();
1498   instant_url_ref_.InvalidateCachedValues();
1499   SetPrepopulateId(other.data_.prepopulate_id);
1500 }
1501
1502 void TemplateURL::SetURL(const std::string& url) {
1503   data_.SetURL(url);
1504   url_ref_.InvalidateCachedValues();
1505 }
1506
1507 void TemplateURL::SetPrepopulateId(int id) {
1508   data_.prepopulate_id = id;
1509   const bool prepopulated = id > 0;
1510   url_ref_.prepopulated_ = prepopulated;
1511   suggestions_url_ref_.prepopulated_ = prepopulated;
1512   instant_url_ref_.prepopulated_ = prepopulated;
1513 }
1514
1515 void TemplateURL::ResetKeywordIfNecessary(
1516     const SearchTermsData& search_terms_data,
1517     bool force) {
1518   if (IsGoogleSearchURLWithReplaceableKeyword(search_terms_data) || force) {
1519     DCHECK(GetType() != OMNIBOX_API_EXTENSION);
1520     GURL url(GenerateSearchURL(search_terms_data));
1521     if (url.is_valid())
1522       data_.SetKeyword(GenerateKeyword(url));
1523   }
1524 }
1525
1526 bool TemplateURL::FindSearchTermsInURL(
1527     const GURL& url,
1528     const SearchTermsData& search_terms_data,
1529     base::string16* search_terms,
1530     url::Parsed::ComponentType* search_term_component,
1531     url::Component* search_terms_position) {
1532   DCHECK(search_terms);
1533   search_terms->clear();
1534
1535   // Try to match with every pattern.
1536   for (size_t i = 0; i < URLCount(); ++i) {
1537     TemplateURLRef ref(this, i);
1538     if (ref.ExtractSearchTermsFromURL(url, search_terms, search_terms_data,
1539         search_term_component, search_terms_position)) {
1540       // If ExtractSearchTermsFromURL() returns true and |search_terms| is empty
1541       // it means the pattern matched but no search terms were present. In this
1542       // case we fail immediately without looking for matches in subsequent
1543       // patterns. This means that given patterns
1544       //    [ "http://foo/#q={searchTerms}", "http://foo/?q={searchTerms}" ],
1545       // calling ExtractSearchTermsFromURL() on "http://foo/?q=bar#q=' would
1546       // return false. This is important for at least Google, where such URLs
1547       // are invalid.
1548       return !search_terms->empty();
1549     }
1550   }
1551   return false;
1552 }