chrome/browser/search_engines/template_url_parser.cc

   1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/search_engines/template_url_parser.h"
   6
   7 #include <algorithm>
   8 #include <map>
   9 #include <vector>
  10
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_ptr.h"
  13 #include "base/string_number_conversions.h"
  14 #include "base/string_util.h"
  15 #include "base/utf_string_conversions.h"
  16 #include "chrome/browser/search_engines/template_url.h"
  17 #include "chrome/common/url_constants.h"
  18 #include "googleurl/src/gurl.h"
  19 #include "libxml/parser.h"
  20 #include "libxml/xmlwriter.h"
  21
  22 namespace {
  23
  24 //
  25 // NOTE: libxml uses the UTF-8 encoding. As 0-127 of UTF-8 corresponds
  26 // to that of char, the following names are all in terms of char. This avoids
  27 // having to convert to wide, then do comparisons
  28
  29 // Defines for element names of the OSD document:
  30 static const char kURLElement[] = "Url";
  31 static const char kParamElement[] = "Param";
  32 static const char kShortNameElement[] = "ShortName";
  33 static const char kDescriptionElement[] = "Description";
  34 static const char kImageElement[] = "Image";
  35 static const char kOpenSearchDescriptionElement[] = "OpenSearchDescription";
  36 static const char kFirefoxSearchDescriptionElement[] = "SearchPlugin";
  37 static const char kLanguageElement[] = "Language";
  38 static const char kInputEncodingElement[] = "InputEncoding";
  39
  40 // Various XML attributes used.
  41 static const char kURLTypeAttribute[] = "type";
  42 static const char kURLTemplateAttribute[] = "template";
  43 static const char kImageTypeAttribute[] = "type";
  44 static const char kImageWidthAttribute[] = "width";
  45 static const char kImageHeightAttribute[] = "height";
  46 static const char kURLIndexOffsetAttribute[] = "indexOffset";
  47 static const char kURLPageOffsetAttribute[] = "pageOffset";
  48 static const char kParamNameAttribute[] = "name";
  49 static const char kParamValueAttribute[] = "value";
  50 static const char kParamMethodAttribute[] = "method";
  51
  52 // Mime type for search results.
  53 static const char kHTMLType[] = "text/html";
  54
  55 // Mime type for as you type suggestions.
  56 static const char kSuggestionType[] = "application/x-suggestions+json";
  57
  58 // Namespace identifier.
  59 static const char kOSDNS[] = "xmlns";
  60
  61 // The namespace for documents we understand.
  62 static const char kNameSpace[] = "http://a9.com/-/spec/opensearch/1.1/";
  63
  64 // Removes the namespace from the specified |name|, ex: os:Url -> Url.
  65 static void PruneNamespace(std::string* name) {
  66   size_t index = name->find_first_of(":");
  67   if (index != std::string::npos)
  68     name->erase(0, index + 1);
  69 }
  70
  71 //
  72 // To minimize memory overhead while parsing, a SAX style parser is used.
  73 // ParsingContext is used to maintain the state we're in the document
  74 // while parsing.
  75 class ParsingContext {
  76  public:
  77   // Enum of the known element types.
  78   enum ElementType {
  79     UNKNOWN,
  80     OPEN_SEARCH_DESCRIPTION,
  81     URL,
  82     PARAM,
  83     SHORT_NAME,
  84     DESCRIPTION,
  85     IMAGE,
  86     LANGUAGE,
  87     INPUT_ENCODING,
  88   };
  89
  90   enum Method {
  91     GET,
  92     POST
  93   };
  94
  95   // Key/value of a Param node.
  96   typedef std::pair<std::string, std::string> Param;
  97
  98   ParsingContext(TemplateURLParser::ParameterFilter* parameter_filter,
  99                  TemplateURL* url)
 100       : url_(url),
 101         parameter_filter_(parameter_filter),
 102         method_(GET),
 103         suggestion_method_(GET),
 104         is_suggest_url_(false),
 105         derive_image_from_url_(false) {
 106     if (kElementNameToElementTypeMap == NULL)
 107       InitMapping();
 108   }
 109
 110   // Invoked when an element starts.
 111   void PushElement(const std::string& element) {
 112     ElementType type;
 113     if (kElementNameToElementTypeMap->find(element) ==
 114         kElementNameToElementTypeMap->end()) {
 115       type = UNKNOWN;
 116     } else {
 117       type = (*kElementNameToElementTypeMap)[element];
 118     }
 119     elements_.push_back(type);
 120   }
 121
 122   void PopElement() {
 123     elements_.pop_back();
 124   }
 125
 126   // Returns the current ElementType.
 127   ElementType GetKnownType() {
 128     if (elements_.size() == 2 && elements_[0] == OPEN_SEARCH_DESCRIPTION)
 129       return elements_[1];
 130
 131     // We only expect PARAM nodes under the Url node
 132     if (elements_.size() == 3 && elements_[0] == OPEN_SEARCH_DESCRIPTION &&
 133         elements_[1] == URL && elements_[2] == PARAM)
 134       return PARAM;
 135
 136     return UNKNOWN;
 137   }
 138
 139   TemplateURL* template_url() { return url_; }
 140
 141   void AddImageRef(const std::string& type, int width, int height) {
 142     if (width > 0 && height > 0)
 143       current_image_.reset(new TemplateURL::ImageRef(type, width, height));
 144   }
 145
 146   void EndImage() {
 147     current_image_.reset();
 148   }
 149
 150   void SetImageURL(const GURL& url) {
 151     if (current_image_.get()) {
 152       current_image_->url = url;
 153       url_->add_image_ref(*current_image_);
 154       current_image_.reset();
 155     }
 156   }
 157
 158   void ResetString() {
 159     string_.clear();
 160   }
 161
 162   void AppendString(const string16& string) {
 163     string_ += string;
 164   }
 165
 166   const string16& GetString() {
 167     return string_;
 168   }
 169
 170   void ResetExtraParams() {
 171     extra_params_.clear();
 172   }
 173
 174   void AddExtraParams(const std::string& key, const std::string& value) {
 175     if (parameter_filter_ && !parameter_filter_->KeepParameter(key, value))
 176       return;
 177     extra_params_.push_back(Param(key, value));
 178   }
 179
 180   const std::vector<Param>& extra_params() const { return extra_params_; }
 181
 182   void set_is_suggestion(bool value) { is_suggest_url_ = value; }
 183   bool is_suggestion() const { return is_suggest_url_; }
 184
 185   TemplateURLParser::ParameterFilter* parameter_filter() const {
 186     return parameter_filter_;
 187   }
 188
 189   void set_derive_image_from_url(bool derive_image_from_url) {
 190     derive_image_from_url_ = derive_image_from_url;
 191   }
 192
 193   void set_method(Method method) { method_ = method; }
 194   Method method() { return method_; }
 195
 196   void set_suggestion_method(Method method) { suggestion_method_ = method; }
 197   Method suggestion_method() { return suggestion_method_; }
 198
 199   // Builds the image URL from the Template search URL if no image URL has been
 200   // set.
 201   void DeriveImageFromURL() {
 202     if (derive_image_from_url_ &&
 203         url_->GetFaviconURL().is_empty() && url_->url()) {
 204       GURL url(url_->url()->url());  // More url's please...
 205       url_->SetFaviconURL(TemplateURL::GenerateFaviconURL(url));
 206     }
 207   }
 208
 209  private:
 210   static void InitMapping() {
 211     kElementNameToElementTypeMap = new std::map<std::string, ElementType>;
 212     (*kElementNameToElementTypeMap)[kURLElement] = URL;
 213     (*kElementNameToElementTypeMap)[kParamElement] = PARAM;
 214     (*kElementNameToElementTypeMap)[kShortNameElement] = SHORT_NAME;
 215     (*kElementNameToElementTypeMap)[kDescriptionElement] = DESCRIPTION;
 216     (*kElementNameToElementTypeMap)[kImageElement] = IMAGE;
 217     (*kElementNameToElementTypeMap)[kOpenSearchDescriptionElement] =
 218         OPEN_SEARCH_DESCRIPTION;
 219     (*kElementNameToElementTypeMap)[kFirefoxSearchDescriptionElement] =
 220         OPEN_SEARCH_DESCRIPTION;
 221     (*kElementNameToElementTypeMap)[kLanguageElement] =
 222         LANGUAGE;
 223     (*kElementNameToElementTypeMap)[kInputEncodingElement] =
 224         INPUT_ENCODING;
 225   }
 226
 227   // Key is UTF8 encoded.
 228   static std::map<std::string, ElementType>* kElementNameToElementTypeMap;
 229   // TemplateURL supplied to Read method. It's owned by the caller, so we
 230   // don't need to free it.
 231   TemplateURL* url_;
 232   std::vector<ElementType> elements_;
 233   scoped_ptr<TemplateURL::ImageRef> current_image_;
 234
 235   // Character content for the current element.
 236   string16 string_;
 237
 238   TemplateURLParser::ParameterFilter* parameter_filter_;
 239
 240   // The list of parameters parsed in the Param nodes of a Url node.
 241   std::vector<Param> extra_params_;
 242
 243   // The HTTP methods used.
 244   Method method_;
 245   Method suggestion_method_;
 246
 247   // If true, we are currently parsing a suggest URL, otherwise it is an HTML
 248   // search.  Note that we don't need a stack as Url nodes cannot be nested.
 249   bool is_suggest_url_;
 250
 251   // Whether we should derive the image from the URL (when images are data
 252   // URLs).
 253   bool derive_image_from_url_;
 254
 255   DISALLOW_COPY_AND_ASSIGN(ParsingContext);
 256 };
 257
 258 // static
 259 std::map<std::string, ParsingContext::ElementType>*
 260     ParsingContext::kElementNameToElementTypeMap = NULL;
 261
 262 string16 XMLCharToUTF16(const xmlChar* value, int length) {
 263   return UTF8ToUTF16(std::string((const char*)value, length));
 264 }
 265
 266 std::string XMLCharToString(const xmlChar* value) {
 267   return std::string((const char*)value);
 268 }
 269
 270 // Returns true if input_encoding contains a valid input encoding string. This
 271 // doesn't verify that we have a valid encoding for the string, just that the
 272 // string contains characters that constitute a valid input encoding.
 273 bool IsValidEncodingString(const std::string& input_encoding) {
 274   if (input_encoding.empty())
 275     return false;
 276
 277   if (!IsAsciiAlpha(input_encoding[0]))
 278     return false;
 279
 280   for (size_t i = 1, max = input_encoding.size(); i < max; ++i) {
 281     char c = input_encoding[i];
 282     if (!IsAsciiAlpha(c) && !IsAsciiDigit(c) && c != '.' && c != '_' &&
 283         c != '-') {
 284       return false;
 285     }
 286   }
 287   return true;
 288 }
 289
 290 void ParseURL(const xmlChar** atts, ParsingContext* context) {
 291   if (!atts)
 292     return;
 293
 294   TemplateURL* turl = context->template_url();
 295   const xmlChar** attributes = atts;
 296   std::string template_url;
 297   bool is_post = false;
 298   bool is_html_url = false;
 299   bool is_suggest_url = false;
 300   int index_offset = 1;
 301   int page_offset = 1;
 302
 303   while (*attributes) {
 304     std::string name(XMLCharToString(*attributes));
 305     const xmlChar* value = attributes[1];
 306     if (name == kURLTypeAttribute) {
 307       std::string type = XMLCharToString(value);
 308       is_html_url = (type == kHTMLType);
 309       is_suggest_url = (type == kSuggestionType);
 310     } else if (name == kURLTemplateAttribute) {
 311       template_url = XMLCharToString(value);
 312     } else if (name == kURLIndexOffsetAttribute) {
 313       base::StringToInt(XMLCharToString(value), &index_offset);
 314       index_offset = std::max(1, index_offset);
 315     } else if (name == kURLPageOffsetAttribute) {
 316       base::StringToInt(XMLCharToString(value), &page_offset);
 317       page_offset = std::max(1, page_offset);
 318     } else if (name == kParamMethodAttribute) {
 319       is_post = LowerCaseEqualsASCII(XMLCharToString(value), "post");
 320     }
 321     attributes += 2;
 322   }
 323   if (is_html_url) {
 324     turl->SetURL(template_url, index_offset, page_offset);
 325     context->set_is_suggestion(false);
 326     if (is_post)
 327       context->set_method(ParsingContext::POST);
 328   } else if (is_suggest_url) {
 329     turl->SetSuggestionsURL(template_url, index_offset, page_offset);
 330     context->set_is_suggestion(true);
 331     if (is_post)
 332       context->set_suggestion_method(ParsingContext::POST);
 333   }
 334 }
 335
 336 void ParseImage(const xmlChar** atts, ParsingContext* context) {
 337   if (!atts)
 338     return;
 339
 340   const xmlChar** attributes = atts;
 341   int width = 0;
 342   int height = 0;
 343   std::string type;
 344   while (*attributes) {
 345     std::string name(XMLCharToString(*attributes));
 346     const xmlChar* value = attributes[1];
 347     if (name == kImageTypeAttribute) {
 348       type = XMLCharToString(value);
 349     } else if (name == kImageWidthAttribute) {
 350       base::StringToInt(XMLCharToString(value), &width);
 351     } else if (name == kImageHeightAttribute) {
 352       base::StringToInt(XMLCharToString(value), &height);
 353     }
 354     attributes += 2;
 355   }
 356   if (width > 0 && height > 0 && !type.empty()) {
 357     // Valid Image URL.
 358     context->AddImageRef(type, width, height);
 359   }
 360 }
 361
 362 void ParseParam(const xmlChar** atts, ParsingContext* context) {
 363   if (!atts)
 364     return;
 365
 366   const xmlChar** attributes = atts;
 367   std::string key, value;
 368   while (*attributes) {
 369     std::string name(XMLCharToString(*attributes));
 370     const xmlChar* val = attributes[1];
 371     if (name == kParamNameAttribute) {
 372       key = XMLCharToString(val);
 373     } else if (name == kParamValueAttribute) {
 374       value = XMLCharToString(val);
 375     }
 376     attributes += 2;
 377   }
 378   if (!key.empty())
 379     context->AddExtraParams(key, value);
 380 }
 381
 382 static void AppendParamToQuery(const std::string& key,
 383                                const std::string& value,
 384                                std::string* query) {
 385   if (!query->empty())
 386     query->append("&");
 387   if (!key.empty()) {
 388     query->append(key);
 389     query->append("=");
 390   }
 391   query->append(value);
 392 }
 393
 394 void ProcessURLParams(ParsingContext* context) {
 395   TemplateURL* t_url = context->template_url();
 396   const TemplateURLRef* t_url_ref =
 397       context->is_suggestion() ? t_url->suggestions_url() :
 398                                  t_url->url();
 399   if (!t_url_ref)
 400     return;
 401
 402   if (!context->parameter_filter() && context->extra_params().empty())
 403     return;
 404
 405   GURL url(t_url_ref->url());
 406   // If there is a parameter filter, parse the existing URL and remove any
 407   // unwanted parameter.
 408   TemplateURLParser::ParameterFilter* filter = context->parameter_filter();
 409   std::string new_query;
 410   bool modified = false;
 411   if (filter) {
 412     url_parse::Component query = url.parsed_for_possibly_invalid_spec().query;
 413     url_parse::Component key, value;
 414     const char* url_spec = url.spec().c_str();
 415     while (url_parse::ExtractQueryKeyValue(url_spec, &query, &key, &value)) {
 416       std::string key_str(url_spec, key.begin, key.len);
 417       std::string value_str(url_spec, value.begin, value.len);
 418       if (filter->KeepParameter(key_str, value_str)) {
 419         AppendParamToQuery(key_str, value_str, &new_query);
 420       } else {
 421         modified = true;
 422       }
 423     }
 424   }
 425   if (!modified)
 426     new_query = url.query();
 427
 428   // Add the extra parameters if any.
 429   const std::vector<ParsingContext::Param>& params = context->extra_params();
 430   if (!params.empty()) {
 431     modified = true;
 432     std::vector<ParsingContext::Param>::const_iterator iter;
 433     for (iter = params.begin(); iter != params.end(); ++iter)
 434       AppendParamToQuery(iter->first, iter->second, &new_query);
 435   }
 436
 437   if (modified) {
 438     GURL::Replacements repl;
 439     repl.SetQueryStr(new_query);
 440     url = url.ReplaceComponents(repl);
 441     if (context->is_suggestion()) {
 442       t_url->SetSuggestionsURL(url.spec(),
 443                                t_url_ref->index_offset(),
 444                                t_url_ref->page_offset());
 445     } else {
 446       t_url->SetURL(url.spec(),
 447                     t_url_ref->index_offset(),
 448                     t_url_ref->page_offset());
 449     }
 450   }
 451 }
 452
 453 void StartElementImpl(void *ctx, const xmlChar *name, const xmlChar **atts) {
 454   ParsingContext* context = reinterpret_cast<ParsingContext*>(ctx);
 455   std::string node_name((const char*)name);
 456   PruneNamespace(&node_name);
 457   context->PushElement(node_name);
 458   switch (context->GetKnownType()) {
 459     case ParsingContext::URL:
 460       context->ResetExtraParams();
 461       ParseURL(atts, context);
 462       break;
 463     case ParsingContext::IMAGE:
 464       ParseImage(atts, context);
 465       break;
 466     case ParsingContext::PARAM:
 467       ParseParam(atts, context);
 468       break;
 469     default:
 470       break;
 471   }
 472   context->ResetString();
 473 }
 474
 475 void EndElementImpl(void *ctx, const xmlChar *name) {
 476   ParsingContext* context = reinterpret_cast<ParsingContext*>(ctx);
 477   switch (context->GetKnownType()) {
 478     case ParsingContext::SHORT_NAME:
 479       context->template_url()->set_short_name(context->GetString());
 480       break;
 481     case ParsingContext::DESCRIPTION:
 482       context->template_url()->set_description(context->GetString());
 483       break;
 484     case ParsingContext::IMAGE: {
 485       GURL image_url(UTF16ToUTF8(context->GetString()));
 486       if (image_url.SchemeIs(chrome::kDataScheme)) {
 487         // TODO (jcampan): bug 1169256: when dealing with data URL, we need to
 488         // decode the data URL in the renderer. For now, we'll just point to the
 489         // favicon from the URL.
 490         context->set_derive_image_from_url(true);
 491       } else {
 492         context->SetImageURL(image_url);
 493       }
 494       context->EndImage();
 495       break;
 496     }
 497     case ParsingContext::LANGUAGE:
 498       context->template_url()->add_language(context->GetString());
 499       break;
 500     case ParsingContext::INPUT_ENCODING: {
 501       std::string input_encoding = UTF16ToASCII(context->GetString());
 502       if (IsValidEncodingString(input_encoding))
 503         context->template_url()->add_input_encoding(input_encoding);
 504       break;
 505     }
 506     case ParsingContext::URL:
 507       ProcessURLParams(context);
 508       break;
 509     default:
 510       break;
 511   }
 512   context->ResetString();
 513   context->PopElement();
 514 }
 515
 516 void CharactersImpl(void *ctx, const xmlChar *ch, int len) {
 517   ParsingContext* context = reinterpret_cast<ParsingContext*>(ctx);
 518   context->AppendString(XMLCharToUTF16(ch, len));
 519 }
 520
 521 // Returns true if the ref is null, or the url wrapped by ref is
 522 // valid with a spec of http/https.
 523 bool IsHTTPRef(const TemplateURLRef* ref) {
 524   if (ref == NULL)
 525     return true;
 526   GURL url(ref->url());
 527   return (url.is_valid() && (url.SchemeIs(chrome::kHttpScheme) ||
 528                              url.SchemeIs(chrome::kHttpsScheme)));
 529 }
 530
 531 // Returns true if the TemplateURL is legal. A legal TemplateURL is one
 532 // where all URLs have a spec of http/https.
 533 bool IsLegal(TemplateURL* url) {
 534   if (!IsHTTPRef(url->url()) || !IsHTTPRef(url->suggestions_url()))
 535     return false;
 536   // Make sure all the image refs are legal.
 537   const std::vector<TemplateURL::ImageRef>& image_refs = url->image_refs();
 538   for (size_t i = 0; i < image_refs.size(); i++) {
 539     GURL image_url(image_refs[i].url);
 540     if (!image_url.is_valid() ||
 541         !(image_url.SchemeIs(chrome::kHttpScheme) ||
 542           image_url.SchemeIs(chrome::kHttpsScheme))) {
 543       return false;
 544     }
 545   }
 546   return true;
 547 }
 548
 549 }  // namespace
 550
 551 // static
 552 bool TemplateURLParser::Parse(const unsigned char* data, size_t length,
 553                               TemplateURLParser::ParameterFilter* param_filter,
 554                               TemplateURL* url) {
 555   DCHECK(url);
 556   // xmlSubstituteEntitiesDefault(1) makes it so that &amp; isn't mapped to
 557   // &#38; . Unfortunately xmlSubstituteEntitiesDefault effects global state.
 558   // If this becomes problematic we'll need to provide our own entity
 559   // type for &amp;, or strip out &#34; by hand after parsing.
 560   int last_sub_entities_value = xmlSubstituteEntitiesDefault(1);
 561   ParsingContext context(param_filter, url);
 562   xmlSAXHandler sax_handler;
 563   memset(&sax_handler, 0, sizeof(sax_handler));
 564   sax_handler.startElement = &StartElementImpl;
 565   sax_handler.endElement = &EndElementImpl;
 566   sax_handler.characters = &CharactersImpl;
 567   xmlSAXUserParseMemory(&sax_handler, &context,
 568                         reinterpret_cast<const char*>(data),
 569                         static_cast<int>(length));
 570   xmlSubstituteEntitiesDefault(last_sub_entities_value);
 571   // If the image was a data URL, use the favicon from the search URL instead.
 572   // (see TODO inEndElementImpl()).
 573   context.DeriveImageFromURL();
 574
 575   // TODO(jcampan): http://b/issue?id=1196285 we do not support search engines
 576   //                that use POST yet.
 577   if (context.method() == ParsingContext::POST)
 578     return false;
 579   if (context.suggestion_method() == ParsingContext::POST)
 580     url->SetSuggestionsURL("", 0, 0);
 581
 582   if (!url->short_name().empty() && !url->description().empty()) {
 583     // So far so good, make sure the urls are http.
 584     return IsLegal(url);
 585   }
 586   return false;
 587 }