chrome/browser/template_url_model.cc

   1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/template_url_model.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/logging.h"
  10 #include "base/string_util.h"
  11 #include "chrome/app/locales/locale_settings.h"
  12 #include "chrome/browser/browser_process.h"
  13 #include "chrome/browser/google_url_tracker.h"
  14 #include "chrome/browser/history/history.h"
  15 #include "chrome/browser/profile.h"
  16 #include "chrome/browser/rlz/rlz.h"
  17 #include "chrome/browser/template_url.h"
  18 #include "chrome/browser/template_url_prepopulate_data.h"
  19 #include "chrome/common/l10n_util.h"
  20 #include "chrome/common/pref_names.h"
  21 #include "chrome/common/pref_service.h"
  22 #include "chrome/common/stl_util-inl.h"
  23 #include "googleurl/src/gurl.h"
  24 #include "googleurl/src/url_parse.h"
  25 #include "net/base/net_util.h"
  26 #include "unicode/rbbi.h"
  27 #include "unicode/uchar.h"
  28
  29 // String in the URL that is replaced by the search term.
  30 static const wchar_t kSearchTermParameter[] = L"{searchTerms}";
  31
  32 // String in Initializer that is replaced with kSearchTermParameter.
  33 static const wchar_t kTemplateParameter[](L"%s");
  34
  35 // Term used when generating a search url. Use something obscure so that on
  36 // the rare case the term replaces the URL it's unlikely another keyword would
  37 // have the same url.
  38 static const wchar_t kReplacementTerm[] = L"blah.blah.blah.blah.blah";
  39
  40 class TemplateURLModel::LessWithPrefix {
  41  public:
  42   // We want to find the set of keywords that begin with a prefix.  The STL
  43   // algorithms will return the set of elements that are "equal to" the
  44   // prefix, where "equal(x, y)" means "!(cmp(x, y) || cmp(y, x))".  When
  45   // cmp() is the typical std::less<>, this results in lexicographic equality;
  46   // we need to extend this to mark a prefix as "not less than" a keyword it
  47   // begins, which will cause the desired elements to be considered "equal to"
  48   // the prefix.  Note: this is still a strict weak ordering, as required by
  49   // equal_range() (though I will not prove that here).
  50   //
  51   // Unfortunately the calling convention is not "prefix and element" but
  52   // rather "two elements", so we pass the prefix as a fake "element" which has
  53   // a NULL KeywordDataElement pointer.
  54   bool operator()(const KeywordToTemplateMap::value_type& elem1,
  55                   const KeywordToTemplateMap::value_type& elem2) const {
  56     return (elem1.second == NULL) ?
  57         (elem2.first.compare(0, elem1.first.length(), elem1.first) > 0) :
  58         (elem1.first < elem2.first);
  59   }
  60 };
  61
  62 TemplateURLModel::TemplateURLModel(Profile* profile)
  63     : profile_(profile),
  64       loaded_(false),
  65       load_handle_(0),
  66       default_search_provider_(NULL),
  67       next_id_(1) {
  68   DCHECK(profile_);
  69   Init(NULL, 0);
  70 }
  71
  72 TemplateURLModel::TemplateURLModel(const Initializer* initializers,
  73                                    const int count)
  74     : profile_(NULL),
  75       loaded_(true),
  76       load_handle_(0),
  77       service_(NULL),
  78       default_search_provider_(NULL),
  79       next_id_(1) {
  80   Init(initializers, count);
  81 }
  82
  83 TemplateURLModel::~TemplateURLModel() {
  84   if (load_handle_) {
  85     DCHECK(service_.get());
  86     service_->CancelRequest(load_handle_);
  87   }
  88
  89   STLDeleteElements(&template_urls_);
  90
  91   NotificationService* ns = NotificationService::current();
  92   if (profile_) {
  93     ns->RemoveObserver(this, NOTIFY_HISTORY_URL_VISITED,
  94                        Source<Profile>(profile_->GetOriginalProfile()));
  95   }
  96   ns->RemoveObserver(this, NOTIFY_GOOGLE_URL_UPDATED,
  97                      NotificationService::AllSources());
  98 }
  99
 100 void TemplateURLModel::Init(const Initializer* initializers,
 101                             int num_initializers) {
 102   // Register for notifications.
 103   NotificationService* ns = NotificationService::current();
 104   if (profile_) {
 105     // TODO(sky): bug 1166191. The keywords should be moved into the history
 106     // db, which will mean we no longer need this notification and the history
 107     // backend can handle automatically adding the search terms as the user
 108     // navigates.
 109     ns->AddObserver(this, NOTIFY_HISTORY_URL_VISITED,
 110                     Source<Profile>(profile_->GetOriginalProfile()));
 111   }
 112   ns->AddObserver(this, NOTIFY_GOOGLE_URL_UPDATED,
 113                   NotificationService::AllSources());
 114
 115   // Add specific initializers, if any.
 116   for (int i(0); i < num_initializers; ++i) {
 117     DCHECK(initializers[i].keyword);
 118     DCHECK(initializers[i].url);
 119     DCHECK(initializers[i].content);
 120
 121     size_t template_position =
 122         std::wstring(initializers[i].url).find(kTemplateParameter);
 123     DCHECK(template_position != std::wstring::npos);
 124     std::wstring osd_url(initializers[i].url);
 125     osd_url.replace(template_position, arraysize(kTemplateParameter) - 1,
 126                     kSearchTermParameter);
 127
 128     // TemplateURLModel ends up owning the TemplateURL, don't try and free it.
 129     TemplateURL* template_url = new TemplateURL();
 130     template_url->set_keyword(initializers[i].keyword);
 131     template_url->set_short_name(initializers[i].content);
 132     template_url->SetURL(osd_url, 0, 0);
 133     Add(template_url);
 134   }
 135
 136   // Request a server check for the correct Google URL if Google is the default
 137   // search engine.
 138   const TemplateURL* default_provider = GetDefaultSearchProvider();
 139   if (default_provider) {
 140     const TemplateURLRef* default_provider_ref = default_provider->url();
 141     if (default_provider_ref && default_provider_ref->HasGoogleBaseURLs())
 142       GoogleURLTracker::RequestServerCheck();
 143   }
 144 }
 145
 146 // static
 147 std::wstring TemplateURLModel::GenerateKeyword(const GURL& url,
 148                                                bool autodetected) {
 149   // Don't autogenerate keywords for referrers that are the result of a form
 150   // submission (TODO: right now we approximate this by checking for the URL
 151   // having a query, but we should replace this with a call to WebCore to see if
 152   // the originating page was actually a form submission), anything other than
 153   // http, or referrers with a path.
 154   //
 155   // If we relax the path constraint, we need to be sure to sanitize the path
 156   // elements and update AutocompletePopup to look for keywords using the path.
 157   // See http://b/issue?id=863583.
 158   if (!url.is_valid() ||
 159       (autodetected && (url.has_query() || (url.scheme() != "http") ||
 160                         ((url.path() != "") && (url.path() != "/")))))
 161     return std::wstring();
 162
 163   // Strip "www." off the front of the keyword; otherwise the keyword won't work
 164   // properly.  See http://b/issue?id=1205573.
 165   return net::StripWWW(UTF8ToWide(url.host()));
 166 }
 167
 168 // static
 169 std::wstring TemplateURLModel::CleanUserInputKeyword(
 170     const std::wstring& keyword) {
 171   // Remove the scheme.
 172   std::wstring result(l10n_util::ToLower(keyword));
 173   url_parse::Component scheme_component;
 174   if (url_parse::ExtractScheme(keyword.c_str(),
 175                                static_cast<int>(keyword.length()),
 176                                &scheme_component)) {
 177     // Include trailing ':'.
 178     result.erase(0, scheme_component.end() + 1);
 179     // Many schemes usually have "//" after them, so strip it too.
 180     const std::wstring after_scheme(L"//");
 181     if (result.compare(0, after_scheme.length(), after_scheme) == 0)
 182       result.erase(0, after_scheme.length());
 183   }
 184
 185   // Remove leading "www.".
 186   result = net::StripWWW(result);
 187
 188   // Remove trailing "/".
 189   return (result.length() > 0 && result[result.length() - 1] == L'/') ?
 190       result.substr(0, result.length() - 1) : result;
 191 }
 192
 193 // static
 194 GURL TemplateURLModel::GenerateSearchURL(const TemplateURL* t_url) {
 195   DCHECK(t_url);
 196   const TemplateURLRef* search_ref = t_url->url();
 197   if (!search_ref || !search_ref->IsValid())
 198     return GURL();
 199
 200   if (!search_ref->SupportsReplacement())
 201     return GURL(search_ref->url());
 202
 203   return GURL(search_ref->ReplaceSearchTerms(*t_url, kReplacementTerm,
 204       TemplateURLRef::NO_SUGGESTIONS_AVAILABLE, std::wstring()));
 205 }
 206
 207 bool TemplateURLModel::CanReplaceKeyword(
 208     const std::wstring& keyword,
 209     const std::wstring& url,
 210     const TemplateURL** template_url_to_replace) {
 211   DCHECK(!keyword.empty()); // This should only be called for non-empty
 212                             // keywords. If we need to support empty kewords
 213                             // the code needs to change slightly.
 214   const TemplateURL* existing_url = GetTemplateURLForKeyword(keyword);
 215   if (existing_url) {
 216     // We already have a TemplateURL for this keyword. Only allow it to be
 217     // replaced if the TemplateURL can be replaced.
 218     if (template_url_to_replace)
 219       *template_url_to_replace = existing_url;
 220     return CanReplace(existing_url);
 221   }
 222
 223   // We don't have a TemplateURL with keyword. Only allow a new one if there
 224   // isn't a TemplateURL for the specified host, or there is one but it can
 225   // be replaced. We do this to ensure that if the user assigns a different
 226   // keyword to a generated TemplateURL, we won't regenerate another keyword for
 227   // the same host.
 228   GURL gurl(url);
 229   if (gurl.is_valid() && !gurl.host().empty())
 230     return CanReplaceKeywordForHost(gurl.host(), template_url_to_replace);
 231   return true;
 232 }
 233
 234 void TemplateURLModel::FindMatchingKeywords(
 235     const std::wstring& prefix,
 236     bool support_replacement_only,
 237     std::vector<std::wstring>* matches) const {
 238   // Sanity check args.
 239   if (prefix.empty())
 240     return;
 241   DCHECK(matches != NULL);
 242   DCHECK(matches->empty());  // The code for exact matches assumes this.
 243
 244   // Find matching keyword range.  Searches the element map for keywords
 245   // beginning with |prefix| and stores the endpoints of the resulting set in
 246   // |match_range|.
 247   const std::pair<KeywordToTemplateMap::const_iterator,
 248                   KeywordToTemplateMap::const_iterator> match_range(
 249       std::equal_range(
 250           keyword_to_template_map_.begin(), keyword_to_template_map_.end(),
 251           KeywordToTemplateMap::value_type(prefix, NULL), LessWithPrefix()));
 252
 253   // Return vector of matching keywords.
 254   for (KeywordToTemplateMap::const_iterator i(match_range.first);
 255        i != match_range.second; ++i) {
 256     DCHECK(i->second->url());
 257     if (!support_replacement_only || i->second->url()->SupportsReplacement())
 258       matches->push_back(i->first);
 259   }
 260 }
 261
 262 const TemplateURL* TemplateURLModel::GetTemplateURLForKeyword(
 263                                      const std::wstring& keyword) const {
 264   KeywordToTemplateMap::const_iterator elem(
 265       keyword_to_template_map_.find(keyword));
 266   return (elem == keyword_to_template_map_.end()) ? NULL : elem->second;
 267 }
 268
 269 const TemplateURL* TemplateURLModel::GetTemplateURLForHost(
 270     const std::string& host) const {
 271   HostToURLsMap::const_iterator iter = host_to_urls_map_.find(host);
 272   if (iter == host_to_urls_map_.end() || iter->second.empty())
 273     return NULL;
 274   return *(iter->second.begin());  // Return the 1st element.
 275 }
 276
 277 void TemplateURLModel::Add(TemplateURL* template_url) {
 278   DCHECK(template_url);
 279   DCHECK(template_url->id() == 0);
 280   DCHECK(find(template_urls_.begin(), template_urls_.end(), template_url) ==
 281          template_urls_.end());
 282   template_url->set_id(++next_id_);
 283   template_urls_.push_back(template_url);
 284   AddToMaps(template_url);
 285
 286   if (service_.get())
 287     service_->AddKeyword(*template_url);
 288
 289   if (loaded_) {
 290     FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 291                       OnTemplateURLModelChanged());
 292   }
 293 }
 294
 295 void TemplateURLModel::AddToMaps(const TemplateURL* template_url) {
 296   if (!template_url->keyword().empty())
 297     keyword_to_template_map_[template_url->keyword()] = template_url;
 298
 299   const GURL url(GenerateSearchURL(template_url));
 300   if (url.is_valid() && url.has_host())
 301     host_to_urls_map_[url.host()].insert(template_url);
 302 }
 303
 304 void TemplateURLModel::Remove(const TemplateURL* template_url) {
 305   TemplateURLVector::iterator i = find(template_urls_.begin(),
 306                                        template_urls_.end(),
 307                                        template_url);
 308   if (i == template_urls_.end())
 309     return;
 310
 311   if (template_url == default_search_provider_) {
 312     // Should never delete the default search provider.
 313     NOTREACHED();
 314     return;
 315   }
 316
 317   RemoveFromMaps(template_url);
 318
 319   // Remove it from the vector containing all TemplateURLs.
 320   template_urls_.erase(i);
 321
 322   if (loaded_) {
 323     FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 324                       OnTemplateURLModelChanged());
 325   }
 326
 327   if (service_.get())
 328     service_->RemoveKeyword(*template_url);
 329
 330   if (profile_) {
 331     HistoryService* history =
 332         profile_->GetHistoryService(Profile::EXPLICIT_ACCESS);
 333     if (history)
 334       history->DeleteAllSearchTermsForKeyword(template_url->id());
 335   }
 336
 337   // We own the TemplateURL and need to delete it.
 338   delete template_url;
 339 }
 340
 341 void TemplateURLModel::Replace(const TemplateURL* existing_turl,
 342                                TemplateURL* new_turl) {
 343   DCHECK(existing_turl && new_turl);
 344
 345   TemplateURLVector::iterator i = find(template_urls_.begin(),
 346                                        template_urls_.end(),
 347                                        existing_turl);
 348   DCHECK(i != template_urls_.end());
 349   RemoveFromMaps(existing_turl);
 350   template_urls_.erase(i);
 351
 352   new_turl->set_id(existing_turl->id());
 353
 354   template_urls_.push_back(new_turl);
 355   AddToMaps(new_turl);
 356
 357   if (service_.get())
 358     service_->UpdateKeyword(*new_turl);
 359
 360   if (default_search_provider_ == existing_turl)
 361     SetDefaultSearchProvider(new_turl);
 362
 363   if (loaded_) {
 364     FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 365                       OnTemplateURLModelChanged());
 366   }
 367
 368   delete existing_turl;
 369 }
 370
 371 void TemplateURLModel::RemoveAutoGeneratedBetween(Time created_after,
 372                                                   Time created_before) {
 373   for (size_t i = 0; i < template_urls_.size();) {
 374     if (template_urls_[i]->date_created() >= created_after &&
 375         (created_before.is_null() ||
 376          template_urls_[i]->date_created() < created_before) &&
 377         CanReplace(template_urls_[i])) {
 378       Remove(template_urls_[i]);
 379     } else {
 380       ++i;
 381     }
 382   }
 383 }
 384
 385 void TemplateURLModel::RemoveAutoGeneratedSince(Time created_after) {
 386   RemoveAutoGeneratedBetween(created_after, Time());
 387 }
 388
 389 void TemplateURLModel::SetKeywordSearchTermsForURL(const TemplateURL* t_url,
 390                                                    const GURL& url,
 391                                                    const std::wstring& term) {
 392   HistoryService* history = profile_  ?
 393       profile_->GetHistoryService(Profile::EXPLICIT_ACCESS) : NULL;
 394   if (!history)
 395     return;
 396   history->SetKeywordSearchTermsForURL(url, t_url->id(), term);
 397 }
 398
 399 void TemplateURLModel::RemoveFromMaps(const TemplateURL* template_url) {
 400   if (!template_url->keyword().empty()) {
 401     keyword_to_template_map_.erase(template_url->keyword());
 402   }
 403
 404   const GURL url(GenerateSearchURL(template_url));
 405   if (url.is_valid() && url.has_host()) {
 406     const std::string host(url.host());
 407     DCHECK(host_to_urls_map_.find(host) != host_to_urls_map_.end());
 408     TemplateURLSet& urls = host_to_urls_map_[host];
 409     DCHECK(urls.find(template_url) != urls.end());
 410     urls.erase(urls.find(template_url));
 411     if (urls.empty())
 412       host_to_urls_map_.erase(host_to_urls_map_.find(host));
 413   }
 414 }
 415
 416 void TemplateURLModel::RemoveFromMapsByPointer(
 417     const TemplateURL* template_url) {
 418   DCHECK(template_url);
 419   for (KeywordToTemplateMap::iterator i = keyword_to_template_map_.begin();
 420        i != keyword_to_template_map_.end(); ++i) {
 421     if (i->second == template_url) {
 422       keyword_to_template_map_.erase(i);
 423       // A given TemplateURL only occurs once in the map. As soon as we find the
 424       // entry, stop.
 425       break;
 426     }
 427   }
 428
 429   for (HostToURLsMap::iterator i = host_to_urls_map_.begin();
 430        i != host_to_urls_map_.end(); ++i) {
 431     TemplateURLSet::iterator url_set_iterator = i->second.find(template_url);
 432     if (url_set_iterator != i->second.end()) {
 433       i->second.erase(url_set_iterator);
 434       if (i->second.empty())
 435         host_to_urls_map_.erase(i);
 436       // A given TemplateURL only occurs once in the map. As soon as we find the
 437       // entry, stop.
 438       return;
 439     }
 440   }
 441 }
 442
 443 void TemplateURLModel::SetTemplateURLs(
 444       const std::vector<const TemplateURL*>& urls) {
 445   DCHECK(template_urls_.empty()); // This should only be called on load,
 446                                   // when we have no TemplateURLs.
 447
 448   // Add mappings for the new items.
 449   for (TemplateURLVector::const_iterator i = urls.begin(); i != urls.end();
 450        ++i) {
 451     next_id_ = std::max(next_id_, (*i)->id());
 452     AddToMaps(*i);
 453   }
 454
 455   template_urls_ = urls;
 456 }
 457
 458 std::vector<const TemplateURL*> TemplateURLModel::GetTemplateURLs() const {
 459   return template_urls_;
 460 }
 461
 462 void TemplateURLModel::IncrementUsageCount(const TemplateURL* url) {
 463   DCHECK(url && find(template_urls_.begin(), template_urls_.end(), url) !=
 464          template_urls_.end());
 465   const_cast<TemplateURL*>(url)->set_usage_count(url->usage_count() + 1);
 466   if (service_.get())
 467     service_.get()->UpdateKeyword(*url);
 468 }
 469
 470 void TemplateURLModel::ResetTemplateURL(const TemplateURL* url,
 471                                         const std::wstring& title,
 472                                         const std::wstring& keyword,
 473                                         const std::wstring& search_url) {
 474   DCHECK(url && find(template_urls_.begin(), template_urls_.end(), url) !=
 475          template_urls_.end());
 476   RemoveFromMaps(url);
 477   TemplateURL* modifiable_url = const_cast<TemplateURL*>(url);
 478   modifiable_url->set_short_name(title);
 479   modifiable_url->set_keyword(keyword);
 480   if ((modifiable_url->url() && search_url.empty()) ||
 481       (!modifiable_url->url() && !search_url.empty()) ||
 482       (modifiable_url->url() && modifiable_url->url()->url() != search_url)) {
 483     // The urls have changed, reset the favicon url.
 484     modifiable_url->SetFavIconURL(GURL());
 485     modifiable_url->SetURL(search_url, 0, 0);
 486   }
 487   modifiable_url->set_safe_for_autoreplace(false);
 488   AddToMaps(url);
 489   if (service_.get())
 490     service_.get()->UpdateKeyword(*url);
 491
 492   FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 493                     OnTemplateURLModelChanged());
 494 }
 495
 496 void TemplateURLModel::SetDefaultSearchProvider(const TemplateURL* url) {
 497   if (default_search_provider_ == url)
 498     return;
 499
 500   DCHECK(!url || find(template_urls_.begin(), template_urls_.end(), url) !=
 501          template_urls_.end());
 502   default_search_provider_ = url;
 503
 504   if (url) {
 505     TemplateURL* modifiable_url = const_cast<TemplateURL*>(url);
 506     // Don't mark the url as edited, otherwise we won't be able to rev the
 507     // templateurls we ship with.
 508     modifiable_url->set_show_in_default_list(true);
 509     if (service_.get())
 510       service_.get()->UpdateKeyword(*url);
 511
 512     const TemplateURLRef* url_ref = url->url();
 513     if (url_ref && url_ref->HasGoogleBaseURLs()) {
 514       GoogleURLTracker::RequestServerCheck();
 515       RLZTracker::RecordProductEvent(RLZTracker::CHROME,
 516                                      RLZTracker::CHROME_OMNIBOX,
 517                                      RLZTracker::SET_TO_GOOGLE);
 518     }
 519   }
 520
 521   SaveDefaultSearchProviderToPrefs(url);
 522
 523   if (service_.get())
 524     service_->SetDefaultSearchProvider(url);
 525
 526   if (loaded_) {
 527     FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 528                       OnTemplateURLModelChanged());
 529   }
 530 }
 531
 532 const TemplateURL* TemplateURLModel::GetDefaultSearchProvider() {
 533   if (loaded_)
 534     return default_search_provider_;
 535
 536   if (!prefs_default_search_provider_.get()) {
 537     TemplateURL* default_from_prefs;
 538     if (LoadDefaultSearchProviderFromPrefs(&default_from_prefs)) {
 539       prefs_default_search_provider_.reset(default_from_prefs);
 540     } else {
 541       std::vector<TemplateURL*> loaded_urls;
 542       size_t default_search_index;
 543       TemplateURLPrepopulateData::GetPrepopulatedEngines(GetPrefs(),
 544                                                          &loaded_urls,
 545                                                          &default_search_index);
 546       if (default_search_index < loaded_urls.size()) {
 547         prefs_default_search_provider_.reset(loaded_urls[default_search_index]);
 548         loaded_urls.erase(loaded_urls.begin() + default_search_index);
 549       }
 550       STLDeleteElements(&loaded_urls);
 551     }
 552   }
 553
 554   return prefs_default_search_provider_.get();
 555 }
 556
 557 void TemplateURLModel::AddObserver(TemplateURLModelObserver* observer) {
 558   model_observers_.AddObserver(observer);
 559 }
 560
 561 void TemplateURLModel::RemoveObserver(TemplateURLModelObserver* observer) {
 562   model_observers_.RemoveObserver(observer);
 563 }
 564
 565 void TemplateURLModel::Load() {
 566   if (loaded_ || load_handle_)
 567     return;
 568
 569   if (!service_.get())
 570     service_ = profile_->GetWebDataService(Profile::EXPLICIT_ACCESS);
 571
 572   if (service_.get()) {
 573     load_handle_ = service_->GetKeywords(this);
 574   } else {
 575     loaded_ = true;
 576     NotifyLoaded();
 577   }
 578 }
 579
 580 void TemplateURLModel::OnWebDataServiceRequestDone(
 581                        WebDataService::Handle h,
 582                        const WDTypedResult* result) {
 583   // Reset the load_handle so that we don't try and cancel the load in
 584   // the destructor.
 585   load_handle_ = 0;
 586
 587   if (!result) {
 588     // Results are null if the database went away.
 589     loaded_ = true;
 590     NotifyLoaded();
 591     return;
 592   }
 593
 594   DCHECK(result->GetType() == KEYWORDS_RESULT);
 595
 596   WDKeywordsResult keyword_result = reinterpret_cast<
 597       const WDResult<WDKeywordsResult>*>(result)->GetValue();
 598
 599   // prefs_default_search_provider_ is only needed before we've finished
 600   // loading. Now that we've loaded we can nuke it.
 601   prefs_default_search_provider_.reset();
 602
 603   // Compiler won't convert std::vector<TemplateURL*> to
 604   // std::vector<const TemplateURL*>.
 605   std::vector<const TemplateURL*> template_urls =
 606       *reinterpret_cast<std::vector<const TemplateURL*>* >(
 607           &keyword_result.keywords);
 608   const int resource_keyword_version =
 609       TemplateURLPrepopulateData::GetDataVersion();
 610   if (keyword_result.builtin_keyword_version != resource_keyword_version) {
 611     // There should never be duplicate TemplateURLs. We had a bug such that
 612     // duplicate TemplateURLs existed for one locale. As such we invoke
 613     // RemoveDuplicatePrepopulateIDs to nuke the duplicates.
 614     RemoveDuplicatePrepopulateIDs(&template_urls);
 615   }
 616   SetTemplateURLs(template_urls);
 617
 618   if (keyword_result.default_search_provider_id) {
 619     // See if we can find the default search provider.
 620     for (TemplateURLVector::iterator i = template_urls_.begin();
 621          i != template_urls_.end(); ++i) {
 622       if ((*i)->id() == keyword_result.default_search_provider_id) {
 623         default_search_provider_ = *i;
 624         break;
 625       }
 626     }
 627   }
 628
 629   if (keyword_result.builtin_keyword_version != resource_keyword_version) {
 630     MergeEnginesFromPrepopulateData();
 631     service_->SetBuiltinKeywordVersion(resource_keyword_version);
 632   }
 633
 634   // Always save the default search provider to prefs. That way we don't have to
 635   // worry about it being out of sync.
 636   if (default_search_provider_)
 637     SaveDefaultSearchProviderToPrefs(default_search_provider_);
 638
 639   // Delete any hosts that were deleted before we finished loading.
 640   for (std::vector<std::wstring>::iterator i = hosts_to_delete_.begin();
 641        i != hosts_to_delete_.end(); ++i) {
 642     DeleteGeneratedKeywordsMatchingHost(*i);
 643   }
 644   hosts_to_delete_.clear();
 645
 646   // Index any visits that occurred before we finished loading.
 647   for (size_t i = 0; i < visits_to_add_.size(); ++i)
 648     UpdateKeywordSearchTermsForURL(visits_to_add_[i]);
 649   visits_to_add_.clear();
 650
 651   loaded_ = true;
 652
 653   FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 654                     OnTemplateURLModelChanged());
 655
 656   NotifyLoaded();
 657 }
 658
 659 void TemplateURLModel::RemoveDuplicatePrepopulateIDs(
 660     std::vector<const TemplateURL*>* urls) {
 661   std::set<int> ids;
 662   for (std::vector<const TemplateURL*>::iterator i = urls->begin();
 663        i != urls->end(); ) {
 664     int prepopulate_id = (*i)->prepopulate_id();
 665     if (prepopulate_id) {
 666       if (ids.find(prepopulate_id) != ids.end()) {
 667         if (service_.get())
 668           service_->RemoveKeyword(**i);
 669         delete *i;
 670         i = urls->erase(i);
 671       } else {
 672         ids.insert(prepopulate_id);
 673         ++i;
 674       }
 675     } else {
 676       ++i;
 677     }
 678   }
 679 }
 680
 681 void TemplateURLModel::Observe(NotificationType type,
 682                                const NotificationSource& source,
 683                                const NotificationDetails& details) {
 684   if (type == NOTIFY_HISTORY_URL_VISITED) {
 685     Details<history::URLVisitedDetails> visit_details(details);
 686
 687     if (!loaded())
 688       visits_to_add_.push_back(visit_details->row);
 689     else
 690       UpdateKeywordSearchTermsForURL(visit_details->row);
 691   } else if (type == NOTIFY_GOOGLE_URL_UPDATED) {
 692     if (loaded_)
 693       GoogleBaseURLChanged();
 694   } else {
 695     NOTREACHED();
 696   }
 697 }
 698
 699 void TemplateURLModel::DeleteGeneratedKeywordsMatchingHost(
 700     const std::wstring& host) {
 701   const std::wstring host_slash = host + L"/";
 702   // Iterate backwards as we may end up removing multiple entries.
 703   for (int i = static_cast<int>(template_urls_.size()) - 1; i >= 0; --i) {
 704     if (CanReplace(template_urls_[i]) &&
 705         (template_urls_[i]->keyword() == host ||
 706          template_urls_[i]->keyword().compare(0, host_slash.length(),
 707                                               host_slash) == 0)) {
 708       Remove(template_urls_[i]);
 709     }
 710   }
 711 }
 712
 713 void TemplateURLModel::NotifyLoaded() {
 714   NotificationService::current()->
 715       Notify(TEMPLATE_URL_MODEL_LOADED, Source<TemplateURLModel>(this),
 716              NotificationService::NoDetails());
 717 }
 718
 719 void TemplateURLModel::MergeEnginesFromPrepopulateData() {
 720   // Build a map from prepopulate id to TemplateURL of existing urls.
 721   std::map<int, const TemplateURL*> id_to_turl;
 722   for (size_t i = 0; i < template_urls_.size(); ++i) {
 723     if (template_urls_[i]->prepopulate_id() > 0)
 724       id_to_turl[template_urls_[i]->prepopulate_id()] = template_urls_[i];
 725   }
 726
 727   std::vector<TemplateURL*> loaded_urls;
 728   size_t default_search_index;
 729   TemplateURLPrepopulateData::GetPrepopulatedEngines(GetPrefs(),
 730                                                      &loaded_urls,
 731                                                      &default_search_index);
 732
 733   for (size_t i = 0; i < loaded_urls.size(); ++i) {
 734     scoped_ptr<TemplateURL> t_url(loaded_urls[i]);
 735
 736     if (!t_url->prepopulate_id()) {
 737       // Prepopulate engines need an id.
 738       NOTREACHED();
 739       continue;
 740     }
 741
 742     const TemplateURL* existing_url = id_to_turl[t_url->prepopulate_id()];
 743     if (existing_url) {
 744       if (!existing_url->safe_for_autoreplace()) {
 745         // User edited the entry, preserve the keyword and description.
 746         loaded_urls[i]->set_safe_for_autoreplace(false);
 747         loaded_urls[i]->set_keyword(existing_url->keyword());
 748         loaded_urls[i]->set_autogenerate_keyword(
 749             existing_url->autogenerate_keyword());
 750         loaded_urls[i]->set_short_name(existing_url->short_name());
 751       }
 752       Replace(existing_url, loaded_urls[i]);
 753       id_to_turl[t_url->prepopulate_id()] = loaded_urls[i];
 754     } else {
 755       Add(loaded_urls[i]);
 756     }
 757     if (i == default_search_index && !default_search_provider_)
 758       SetDefaultSearchProvider(loaded_urls[i]);
 759
 760     t_url.release();
 761   }
 762 }
 763
 764 void TemplateURLModel::SaveDefaultSearchProviderToPrefs(
 765     const TemplateURL* t_url) {
 766   PrefService* prefs = GetPrefs();
 767   if (!prefs)
 768     return;
 769
 770   RegisterPrefs(prefs);
 771
 772   const std::wstring search_url =
 773       (t_url && t_url->url()) ? t_url->url()->url() : std::wstring();
 774   prefs->SetString(prefs::kDefaultSearchProviderSearchURL, search_url);
 775
 776   const std::wstring suggest_url =
 777       (t_url && t_url->suggestions_url()) ? t_url->suggestions_url()->url() :
 778                                             std::wstring();
 779   prefs->SetString(prefs::kDefaultSearchProviderSuggestURL, suggest_url);
 780
 781   const std::wstring name =
 782       t_url ? t_url->short_name() : std::wstring();
 783   prefs->SetString(prefs::kDefaultSearchProviderName, name);
 784
 785   const std::wstring id_string =
 786       t_url ? Int64ToWString(t_url->id()) : std::wstring();
 787   prefs->SetString(prefs::kDefaultSearchProviderID, id_string);
 788
 789   prefs->ScheduleSavePersistentPrefs(g_browser_process->file_thread());
 790 }
 791
 792 bool TemplateURLModel::LoadDefaultSearchProviderFromPrefs(
 793     TemplateURL** default_provider) {
 794   PrefService* prefs = GetPrefs();
 795   if (!prefs || !prefs->HasPrefPath(prefs::kDefaultSearchProviderSearchURL) ||
 796       !prefs->HasPrefPath(prefs::kDefaultSearchProviderSuggestURL) ||
 797       !prefs->HasPrefPath(prefs::kDefaultSearchProviderName) ||
 798       !prefs->HasPrefPath(prefs::kDefaultSearchProviderID)) {
 799     return false;
 800   }
 801   RegisterPrefs(prefs);
 802
 803   std::wstring suggest_url =
 804       prefs->GetString(prefs::kDefaultSearchProviderSuggestURL);
 805   std::wstring search_url =
 806       prefs->GetString(prefs::kDefaultSearchProviderSearchURL);
 807
 808   if (suggest_url.empty() && search_url.empty()) {
 809     // The user doesn't want a default search provider.
 810     *default_provider = NULL;
 811     return true;
 812   }
 813
 814   std::wstring name = prefs->GetString(prefs::kDefaultSearchProviderName);
 815
 816   std::wstring id_string = prefs->GetString(prefs::kDefaultSearchProviderID);
 817
 818   *default_provider = new TemplateURL();
 819   (*default_provider)->set_short_name(name);
 820   (*default_provider)->SetURL(search_url, 0, 0);
 821   (*default_provider)->SetSuggestionsURL(suggest_url, 0, 0);
 822   if (!id_string.empty())
 823     (*default_provider)->set_id(StringToInt64(id_string));
 824   return true;
 825 }
 826
 827 void TemplateURLModel::RegisterPrefs(PrefService* prefs) {
 828   if (prefs->IsPrefRegistered(prefs::kDefaultSearchProviderName))
 829     return;
 830   prefs->RegisterStringPref(
 831       prefs::kDefaultSearchProviderName, std::wstring());
 832   prefs->RegisterStringPref(
 833       prefs::kDefaultSearchProviderID, std::wstring());
 834   prefs->RegisterStringPref(
 835       prefs::kDefaultSearchProviderSuggestURL, std::wstring());
 836   prefs->RegisterStringPref(
 837       prefs::kDefaultSearchProviderSearchURL, std::wstring());
 838 }
 839
 840 bool TemplateURLModel::CanReplaceKeywordForHost(
 841     const std::string& host,
 842     const TemplateURL** to_replace) {
 843   const HostToURLsMap::iterator matching_urls = host_to_urls_map_.find(host);
 844   const bool have_matching_urls = (matching_urls != host_to_urls_map_.end());
 845   if (have_matching_urls) {
 846     TemplateURLSet& urls = matching_urls->second;
 847     for (TemplateURLSet::iterator i = urls.begin(); i != urls.end(); ++i) {
 848       const TemplateURL* url = *i;
 849       if (CanReplace(url)) {
 850         if (to_replace)
 851           *to_replace = url;
 852         return true;
 853       }
 854     }
 855   }
 856
 857   if (to_replace)
 858     *to_replace = NULL;
 859   return !have_matching_urls;
 860 }
 861
 862 bool TemplateURLModel::CanReplace(const TemplateURL* t_url) {
 863   return (t_url != default_search_provider_ && !t_url->show_in_default_list() &&
 864           t_url->safe_for_autoreplace());
 865 }
 866
 867 PrefService* TemplateURLModel::GetPrefs() {
 868   return profile_ ? profile_->GetPrefs() : NULL;
 869 }
 870
 871 void TemplateURLModel::UpdateKeywordSearchTermsForURL(
 872     const history::URLRow& row) {
 873   if (!row.url().is_valid() ||
 874       !row.url().parsed_for_possibly_invalid_spec().query.is_nonempty()) {
 875     return;
 876   }
 877
 878   HostToURLsMap::const_iterator t_urls_for_host_iterator =
 879       host_to_urls_map_.find(row.url().host());
 880   if (t_urls_for_host_iterator == host_to_urls_map_.end() ||
 881       t_urls_for_host_iterator->second.empty()) {
 882     return;
 883   }
 884
 885   const TemplateURLSet& urls_for_host = t_urls_for_host_iterator->second;
 886   QueryTerms query_terms;
 887   bool built_terms = false;  // Most URLs won't match a TemplateURLs host;
 888                              // so we lazily build the query_terms.
 889   const std::string path = row.url().path();
 890
 891   for (TemplateURLSet::const_iterator i = urls_for_host.begin();
 892        i != urls_for_host.end(); ++i) {
 893     const TemplateURLRef* search_ref = (*i)->url();
 894
 895     // Count the URL against a TemplateURL if the host and path of the
 896     // visited URL match that of the TemplateURL as well as the search term's
 897     // key of the TemplateURL occurring in the visited url.
 898     //
 899     // NOTE: Even though we're iterating over TemplateURLs indexed by the host
 900     // of the URL we still need to call GetHost on the search_ref. In
 901     // particular, GetHost returns an empty string if search_ref doesn't support
 902     // replacement or isn't valid for use in keyword search terms.
 903
 904     if (search_ref && search_ref->GetHost() == row.url().host() &&
 905         search_ref->GetPath() == path) {
 906       if (!built_terms && !BuildQueryTerms(row.url(), &query_terms)) {
 907         // No query terms. No need to continue with the rest of the
 908         // TemplateURLs.
 909         return;
 910       }
 911       built_terms = true;
 912
 913       QueryTerms::iterator terms_iterator =
 914           query_terms.find(search_ref->GetSearchTermKey());
 915       if (terms_iterator != query_terms.end() &&
 916           !terms_iterator->second.empty()) {
 917         SetKeywordSearchTermsForURL(
 918             *i, row.url(), search_ref->SearchTermToWide(*(*i),
 919             terms_iterator->second));
 920       }
 921     }
 922   }
 923 }
 924
 925 // static
 926 bool TemplateURLModel::BuildQueryTerms(const GURL& url,
 927                                        QueryTerms* query_terms) {
 928   url_parse::Component query = url.parsed_for_possibly_invalid_spec().query;
 929   url_parse::Component key, value;
 930   size_t valid_term_count = 0;
 931   while (url_parse::ExtractQueryKeyValue(url.spec().c_str(), &query, &key,
 932                                          &value)) {
 933     if (key.is_nonempty() && value.is_nonempty()) {
 934       std::string key_string = url.spec().substr(key.begin, key.len);
 935       std::string value_string = url.spec().substr(value.begin, value.len);
 936       QueryTerms::iterator query_terms_iterator =
 937           query_terms->find(key_string);
 938       if (query_terms_iterator != query_terms->end()) {
 939         if (!query_terms_iterator->second.empty() &&
 940             query_terms_iterator->second != value_string) {
 941           // The term occurs in multiple places with different values. Treat
 942           // this as if the term doesn't occur by setting the value to an empty
 943           // string.
 944           (*query_terms)[key_string] = std::string();
 945           DCHECK (valid_term_count > 0);
 946           valid_term_count--;
 947         }
 948       } else {
 949         valid_term_count++;
 950         (*query_terms)[key_string] = value_string;
 951       }
 952     }
 953   }
 954   return (valid_term_count > 0);
 955 }
 956
 957 void TemplateURLModel::GoogleBaseURLChanged() {
 958   bool something_changed = false;
 959   for (size_t i = 0; i < template_urls_.size(); ++i) {
 960     const TemplateURL* t_url = template_urls_[i];
 961     if ((t_url->url() && t_url->url()->HasGoogleBaseURLs()) ||
 962         (t_url->suggestions_url() &&
 963          t_url->suggestions_url()->HasGoogleBaseURLs())) {
 964       RemoveFromMapsByPointer(t_url);
 965       t_url->InvalidateCachedValues();
 966       AddToMaps(t_url);
 967       something_changed = true;
 968     }
 969   }
 970
 971   if (something_changed && loaded_) {
 972     FOR_EACH_OBSERVER(TemplateURLModelObserver, model_observers_,
 973                       OnTemplateURLModelChanged());
 974   }
 975 }
 976