url/url_util.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "url/url_util.h"
   6
   7 #include <string.h>
   8 #include <vector>
   9
  10 #include "base/logging.h"
  11 #include "url/url_canon_internal.h"
  12 #include "url/url_file.h"
  13 #include "url/url_util_internal.h"
  14
  15 namespace url {
  16
  17 const char kFileScheme[] = "file";
  18 const char kFileSystemScheme[] = "filesystem";
  19 const char kMailtoScheme[] = "mailto";
  20
  21 namespace {
  22
  23 // ASCII-specific tolower.  The standard library's tolower is locale sensitive,
  24 // so we don't want to use it here.
  25 template<class Char>
  26 inline Char ToLowerASCII(Char c) {
  27   return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c;
  28 }
  29
  30 // Backend for LowerCaseEqualsASCII.
  31 template<typename Iter>
  32 inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char* b) {
  33   for (Iter it = a_begin; it != a_end; ++it, ++b) {
  34     if (!*b || ToLowerASCII(*it) != *b)
  35       return false;
  36   }
  37   return *b == 0;
  38 }
  39
  40 const int kNumStandardURLSchemes = 8;
  41 const char* kStandardURLSchemes[kNumStandardURLSchemes] = {
  42   "http",
  43   "https",
  44   kFileScheme,  // Yes, file urls can have a hostname!
  45   "ftp",
  46   "gopher",
  47   "ws",  // WebSocket.
  48   "wss",  // WebSocket secure.
  49   kFileSystemScheme,
  50 };
  51
  52 // List of the currently installed standard schemes. This list is lazily
  53 // initialized by InitStandardSchemes and is leaked on shutdown to prevent
  54 // any destructors from being called that will slow us down or cause problems.
  55 std::vector<const char*>* standard_schemes = NULL;
  56
  57 // See the LockStandardSchemes declaration in the header.
  58 bool standard_schemes_locked = false;
  59
  60 // Ensures that the standard_schemes list is initialized, does nothing if it
  61 // already has values.
  62 void InitStandardSchemes() {
  63   if (standard_schemes)
  64     return;
  65   standard_schemes = new std::vector<const char*>;
  66   for (int i = 0; i < kNumStandardURLSchemes; i++)
  67     standard_schemes->push_back(kStandardURLSchemes[i]);
  68 }
  69
  70 // Given a string and a range inside the string, compares it to the given
  71 // lower-case |compare_to| buffer.
  72 template<typename CHAR>
  73 inline bool DoCompareSchemeComponent(const CHAR* spec,
  74                                      const Component& component,
  75                                      const char* compare_to) {
  76   if (!component.is_nonempty())
  77     return compare_to[0] == 0;  // When component is empty, match empty scheme.
  78   return LowerCaseEqualsASCII(&spec[component.begin],
  79                               &spec[component.end()],
  80                               compare_to);
  81 }
  82
  83 // Returns true if the given scheme identified by |scheme| within |spec| is one
  84 // of the registered "standard" schemes.
  85 template<typename CHAR>
  86 bool DoIsStandard(const CHAR* spec, const Component& scheme) {
  87   if (!scheme.is_nonempty())
  88     return false;  // Empty or invalid schemes are non-standard.
  89
  90   InitStandardSchemes();
  91   for (size_t i = 0; i < standard_schemes->size(); i++) {
  92     if (LowerCaseEqualsASCII(&spec[scheme.begin], &spec[scheme.end()],
  93                              standard_schemes->at(i)))
  94       return true;
  95   }
  96   return false;
  97 }
  98
  99 template<typename CHAR>
 100 bool DoFindAndCompareScheme(const CHAR* str,
 101                             int str_len,
 102                             const char* compare,
 103                             Component* found_scheme) {
 104   // Before extracting scheme, canonicalize the URL to remove any whitespace.
 105   // This matches the canonicalization done in DoCanonicalize function.
 106   RawCanonOutputT<CHAR> whitespace_buffer;
 107   int spec_len;
 108   const CHAR* spec = RemoveURLWhitespace(str, str_len,
 109                                          &whitespace_buffer, &spec_len);
 110
 111   Component our_scheme;
 112   if (!ExtractScheme(spec, spec_len, &our_scheme)) {
 113     // No scheme.
 114     if (found_scheme)
 115       *found_scheme = Component();
 116     return false;
 117   }
 118   if (found_scheme)
 119     *found_scheme = our_scheme;
 120   return DoCompareSchemeComponent(spec, our_scheme, compare);
 121 }
 122
 123 template<typename CHAR>
 124 bool DoCanonicalize(const CHAR* in_spec,
 125                     int in_spec_len,
 126                     bool trim_path_end,
 127                     CharsetConverter* charset_converter,
 128                     CanonOutput* output,
 129                     Parsed* output_parsed) {
 130   // Remove any whitespace from the middle of the relative URL, possibly
 131   // copying to the new buffer.
 132   RawCanonOutputT<CHAR> whitespace_buffer;
 133   int spec_len;
 134   const CHAR* spec = RemoveURLWhitespace(in_spec, in_spec_len,
 135                                          &whitespace_buffer, &spec_len);
 136
 137   Parsed parsed_input;
 138 #ifdef WIN32
 139   // For Windows, we allow things that look like absolute Windows paths to be
 140   // fixed up magically to file URLs. This is done for IE compatability. For
 141   // example, this will change "c:/foo" into a file URL rather than treating
 142   // it as a URL with the protocol "c". It also works for UNC ("\\foo\bar.txt").
 143   // There is similar logic in url_canon_relative.cc for
 144   //
 145   // For Max & Unix, we don't do this (the equivalent would be "/foo/bar" which
 146   // has no meaning as an absolute path name. This is because browsers on Mac
 147   // & Unix don't generally do this, so there is no compatibility reason for
 148   // doing so.
 149   if (DoesBeginUNCPath(spec, 0, spec_len, false) ||
 150       DoesBeginWindowsDriveSpec(spec, 0, spec_len)) {
 151     ParseFileURL(spec, spec_len, &parsed_input);
 152     return CanonicalizeFileURL(spec, spec_len, parsed_input, charset_converter,
 153                                output, output_parsed);
 154   }
 155 #endif
 156
 157   Component scheme;
 158   if (!ExtractScheme(spec, spec_len, &scheme))
 159     return false;
 160
 161   // This is the parsed version of the input URL, we have to canonicalize it
 162   // before storing it in our object.
 163   bool success;
 164   if (DoCompareSchemeComponent(spec, scheme, kFileScheme)) {
 165     // File URLs are special.
 166     ParseFileURL(spec, spec_len, &parsed_input);
 167     success = CanonicalizeFileURL(spec, spec_len, parsed_input,
 168                                   charset_converter, output, output_parsed);
 169   } else if (DoCompareSchemeComponent(spec, scheme, kFileSystemScheme)) {
 170     // Filesystem URLs are special.
 171     ParseFileSystemURL(spec, spec_len, &parsed_input);
 172     success = CanonicalizeFileSystemURL(spec, spec_len, parsed_input,
 173                                         charset_converter, output,
 174                                         output_parsed);
 175
 176   } else if (DoIsStandard(spec, scheme)) {
 177     // All "normal" URLs.
 178     ParseStandardURL(spec, spec_len, &parsed_input);
 179     success = CanonicalizeStandardURL(spec, spec_len, parsed_input,
 180                                       charset_converter, output, output_parsed);
 181
 182   } else if (DoCompareSchemeComponent(spec, scheme, kMailtoScheme)) {
 183     // Mailto are treated like a standard url with only a scheme, path, query
 184     ParseMailtoURL(spec, spec_len, &parsed_input);
 185     success = CanonicalizeMailtoURL(spec, spec_len, parsed_input, output,
 186                                     output_parsed);
 187
 188   } else {
 189     // "Weird" URLs like data: and javascript:
 190     ParsePathURL(spec, spec_len, trim_path_end, &parsed_input);
 191     success = CanonicalizePathURL(spec, spec_len, parsed_input, output,
 192                                   output_parsed);
 193   }
 194   return success;
 195 }
 196
 197 template<typename CHAR>
 198 bool DoResolveRelative(const char* base_spec,
 199                        int base_spec_len,
 200                        const Parsed& base_parsed,
 201                        const CHAR* in_relative,
 202                        int in_relative_length,
 203                        CharsetConverter* charset_converter,
 204                        CanonOutput* output,
 205                        Parsed* output_parsed) {
 206   // Remove any whitespace from the middle of the relative URL, possibly
 207   // copying to the new buffer.
 208   RawCanonOutputT<CHAR> whitespace_buffer;
 209   int relative_length;
 210   const CHAR* relative = RemoveURLWhitespace(in_relative, in_relative_length,
 211                                              &whitespace_buffer,
 212                                              &relative_length);
 213   bool base_is_authority_based = false;
 214   bool base_is_hierarchical = false;
 215   if (base_spec &&
 216       base_parsed.scheme.is_nonempty()) {
 217     int after_scheme = base_parsed.scheme.end() + 1;  // Skip past the colon.
 218     int num_slashes = CountConsecutiveSlashes(base_spec, after_scheme,
 219                                               base_spec_len);
 220     base_is_authority_based = num_slashes > 1;
 221     base_is_hierarchical = num_slashes > 0;
 222   }
 223
 224   bool standard_base_scheme =
 225       base_parsed.scheme.is_nonempty() &&
 226       DoIsStandard(base_spec, base_parsed.scheme);
 227
 228   bool is_relative;
 229   Component relative_component;
 230   if (!IsRelativeURL(base_spec, base_parsed, relative, relative_length,
 231                      (base_is_hierarchical || standard_base_scheme),
 232                      &is_relative, &relative_component)) {
 233     // Error resolving.
 234     return false;
 235   }
 236
 237   // Pretend for a moment that |base_spec| is a standard URL. Normally
 238   // non-standard URLs are treated as PathURLs, but if the base has an
 239   // authority we would like to preserve it.
 240   if (is_relative && base_is_authority_based && !standard_base_scheme) {
 241     Parsed base_parsed_authority;
 242     ParseStandardURL(base_spec, base_spec_len, &base_parsed_authority);
 243     if (base_parsed_authority.host.is_nonempty()) {
 244       bool did_resolve_succeed =
 245           ResolveRelativeURL(base_spec, base_parsed_authority, false, relative,
 246                              relative_component, charset_converter, output,
 247                              output_parsed);
 248       // The output_parsed is incorrect at this point (because it was built
 249       // based on base_parsed_authority instead of base_parsed) and needs to be
 250       // re-created.
 251       ParsePathURL(output->data(), output->length(), true,
 252                    output_parsed);
 253       return did_resolve_succeed;
 254     }
 255   } else if (is_relative) {
 256     // Relative, resolve and canonicalize.
 257     bool file_base_scheme = base_parsed.scheme.is_nonempty() &&
 258         DoCompareSchemeComponent(base_spec, base_parsed.scheme, kFileScheme);
 259     return ResolveRelativeURL(base_spec, base_parsed, file_base_scheme, relative,
 260                               relative_component, charset_converter, output,
 261                               output_parsed);
 262   }
 263
 264   // Not relative, canonicalize the input.
 265   return DoCanonicalize(relative, relative_length, true, charset_converter,
 266                         output, output_parsed);
 267 }
 268
 269 template<typename CHAR>
 270 bool DoReplaceComponents(const char* spec,
 271                          int spec_len,
 272                          const Parsed& parsed,
 273                          const Replacements<CHAR>& replacements,
 274                          CharsetConverter* charset_converter,
 275                          CanonOutput* output,
 276                          Parsed* out_parsed) {
 277   // If the scheme is overridden, just do a simple string substitution and
 278   // reparse the whole thing. There are lots of edge cases that we really don't
 279   // want to deal with. Like what happens if I replace "http://e:8080/foo"
 280   // with a file. Does it become "file:///E:/8080/foo" where the port number
 281   // becomes part of the path? Parsing that string as a file URL says "yes"
 282   // but almost no sane rule for dealing with the components individually would
 283   // come up with that.
 284   //
 285   // Why allow these crazy cases at all? Programatically, there is almost no
 286   // case for replacing the scheme. The most common case for hitting this is
 287   // in JS when building up a URL using the location object. In this case, the
 288   // JS code expects the string substitution behavior:
 289   //   http://www.w3.org/TR/2008/WD-html5-20080610/structured.html#common3
 290   if (replacements.IsSchemeOverridden()) {
 291     // Canonicalize the new scheme so it is 8-bit and can be concatenated with
 292     // the existing spec.
 293     RawCanonOutput<128> scheme_replaced;
 294     Component scheme_replaced_parsed;
 295     CanonicalizeScheme(replacements.sources().scheme,
 296                        replacements.components().scheme,
 297                        &scheme_replaced, &scheme_replaced_parsed);
 298
 299     // We can assume that the input is canonicalized, which means it always has
 300     // a colon after the scheme (or where the scheme would be).
 301     int spec_after_colon = parsed.scheme.is_valid() ? parsed.scheme.end() + 1
 302                                                     : 1;
 303     if (spec_len - spec_after_colon > 0) {
 304       scheme_replaced.Append(&spec[spec_after_colon],
 305                              spec_len - spec_after_colon);
 306     }
 307
 308     // We now need to completely re-parse the resulting string since its meaning
 309     // may have changed with the different scheme.
 310     RawCanonOutput<128> recanonicalized;
 311     Parsed recanonicalized_parsed;
 312     DoCanonicalize(scheme_replaced.data(), scheme_replaced.length(), true,
 313                    charset_converter,
 314                    &recanonicalized, &recanonicalized_parsed);
 315
 316     // Recurse using the version with the scheme already replaced. This will now
 317     // use the replacement rules for the new scheme.
 318     //
 319     // Warning: this code assumes that ReplaceComponents will re-check all
 320     // components for validity. This is because we can't fail if DoCanonicalize
 321     // failed above since theoretically the thing making it fail could be
 322     // getting replaced here. If ReplaceComponents didn't re-check everything,
 323     // we wouldn't know if something *not* getting replaced is a problem.
 324     // If the scheme-specific replacers are made more intelligent so they don't
 325     // re-check everything, we should instead recanonicalize the whole thing
 326     // after this call to check validity (this assumes replacing the scheme is
 327     // much much less common than other types of replacements, like clearing the
 328     // ref).
 329     Replacements<CHAR> replacements_no_scheme = replacements;
 330     replacements_no_scheme.SetScheme(NULL, Component());
 331     return DoReplaceComponents(recanonicalized.data(), recanonicalized.length(),
 332                                recanonicalized_parsed, replacements_no_scheme,
 333                                charset_converter, output, out_parsed);
 334   }
 335
 336   // If we get here, then we know the scheme doesn't need to be replaced, so can
 337   // just key off the scheme in the spec to know how to do the replacements.
 338   if (DoCompareSchemeComponent(spec, parsed.scheme, kFileScheme)) {
 339     return ReplaceFileURL(spec, parsed, replacements, charset_converter, output,
 340                           out_parsed);
 341   }
 342   if (DoCompareSchemeComponent(spec, parsed.scheme, kFileSystemScheme)) {
 343     return ReplaceFileSystemURL(spec, parsed, replacements, charset_converter,
 344                                 output, out_parsed);
 345   }
 346   if (DoIsStandard(spec, parsed.scheme)) {
 347     return ReplaceStandardURL(spec, parsed, replacements, charset_converter,
 348                               output, out_parsed);
 349   }
 350   if (DoCompareSchemeComponent(spec, parsed.scheme, kMailtoScheme)) {
 351     return ReplaceMailtoURL(spec, parsed, replacements, output, out_parsed);
 352   }
 353
 354   // Default is a path URL.
 355   return ReplacePathURL(spec, parsed, replacements, output, out_parsed);
 356 }
 357
 358 }  // namespace
 359
 360 void Initialize() {
 361   InitStandardSchemes();
 362 }
 363
 364 void Shutdown() {
 365   if (standard_schemes) {
 366     delete standard_schemes;
 367     standard_schemes = NULL;
 368   }
 369 }
 370
 371 void AddStandardScheme(const char* new_scheme) {
 372   // If this assert triggers, it means you've called AddStandardScheme after
 373   // LockStandardSchemes have been called (see the header file for
 374   // LockStandardSchemes for more).
 375   //
 376   // This normally means you're trying to set up a new standard scheme too late
 377   // in your application's init process. Locate where your app does this
 378   // initialization and calls LockStandardScheme, and add your new standard
 379   // scheme there.
 380   DCHECK(!standard_schemes_locked) <<
 381       "Trying to add a standard scheme after the list has been locked.";
 382
 383   size_t scheme_len = strlen(new_scheme);
 384   if (scheme_len == 0)
 385     return;
 386
 387   // Dulicate the scheme into a new buffer and add it to the list of standard
 388   // schemes. This pointer will be leaked on shutdown.
 389   char* dup_scheme = new char[scheme_len + 1];
 390   memcpy(dup_scheme, new_scheme, scheme_len + 1);
 391
 392   InitStandardSchemes();
 393   standard_schemes->push_back(dup_scheme);
 394 }
 395
 396 void LockStandardSchemes() {
 397   standard_schemes_locked = true;
 398 }
 399
 400 bool IsStandard(const char* spec, const Component& scheme) {
 401   return DoIsStandard(spec, scheme);
 402 }
 403
 404 bool IsStandard(const base::char16* spec, const Component& scheme) {
 405   return DoIsStandard(spec, scheme);
 406 }
 407
 408 bool FindAndCompareScheme(const char* str,
 409                           int str_len,
 410                           const char* compare,
 411                           Component* found_scheme) {
 412   return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
 413 }
 414
 415 bool FindAndCompareScheme(const base::char16* str,
 416                           int str_len,
 417                           const char* compare,
 418                           Component* found_scheme) {
 419   return DoFindAndCompareScheme(str, str_len, compare, found_scheme);
 420 }
 421
 422 bool Canonicalize(const char* spec,
 423                   int spec_len,
 424                   bool trim_path_end,
 425                   CharsetConverter* charset_converter,
 426                   CanonOutput* output,
 427                   Parsed* output_parsed) {
 428   return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
 429                         output, output_parsed);
 430 }
 431
 432 bool Canonicalize(const base::char16* spec,
 433                   int spec_len,
 434                   bool trim_path_end,
 435                   CharsetConverter* charset_converter,
 436                   CanonOutput* output,
 437                   Parsed* output_parsed) {
 438   return DoCanonicalize(spec, spec_len, trim_path_end, charset_converter,
 439                         output, output_parsed);
 440 }
 441
 442 bool ResolveRelative(const char* base_spec,
 443                      int base_spec_len,
 444                      const Parsed& base_parsed,
 445                      const char* relative,
 446                      int relative_length,
 447                      CharsetConverter* charset_converter,
 448                      CanonOutput* output,
 449                      Parsed* output_parsed) {
 450   return DoResolveRelative(base_spec, base_spec_len, base_parsed,
 451                            relative, relative_length,
 452                            charset_converter, output, output_parsed);
 453 }
 454
 455 bool ResolveRelative(const char* base_spec,
 456                      int base_spec_len,
 457                      const Parsed& base_parsed,
 458                      const base::char16* relative,
 459                      int relative_length,
 460                      CharsetConverter* charset_converter,
 461                      CanonOutput* output,
 462                      Parsed* output_parsed) {
 463   return DoResolveRelative(base_spec, base_spec_len, base_parsed,
 464                            relative, relative_length,
 465                            charset_converter, output, output_parsed);
 466 }
 467
 468 bool ReplaceComponents(const char* spec,
 469                        int spec_len,
 470                        const Parsed& parsed,
 471                        const Replacements<char>& replacements,
 472                        CharsetConverter* charset_converter,
 473                        CanonOutput* output,
 474                        Parsed* out_parsed) {
 475   return DoReplaceComponents(spec, spec_len, parsed, replacements,
 476                              charset_converter, output, out_parsed);
 477 }
 478
 479 bool ReplaceComponents(const char* spec,
 480                        int spec_len,
 481                        const Parsed& parsed,
 482                        const Replacements<base::char16>& replacements,
 483                        CharsetConverter* charset_converter,
 484                        CanonOutput* output,
 485                        Parsed* out_parsed) {
 486   return DoReplaceComponents(spec, spec_len, parsed, replacements,
 487                              charset_converter, output, out_parsed);
 488 }
 489
 490 // Front-ends for LowerCaseEqualsASCII.
 491 bool LowerCaseEqualsASCII(const char* a_begin,
 492                           const char* a_end,
 493                           const char* b) {
 494   return DoLowerCaseEqualsASCII(a_begin, a_end, b);
 495 }
 496
 497 bool LowerCaseEqualsASCII(const char* a_begin,
 498                           const char* a_end,
 499                           const char* b_begin,
 500                           const char* b_end) {
 501   while (a_begin != a_end && b_begin != b_end &&
 502          ToLowerASCII(*a_begin) == *b_begin) {
 503     a_begin++;
 504     b_begin++;
 505   }
 506   return a_begin == a_end && b_begin == b_end;
 507 }
 508
 509 bool LowerCaseEqualsASCII(const base::char16* a_begin,
 510                           const base::char16* a_end,
 511                           const char* b) {
 512   return DoLowerCaseEqualsASCII(a_begin, a_end, b);
 513 }
 514
 515 void DecodeURLEscapeSequences(const char* input,
 516                               int length,
 517                               CanonOutputW* output) {
 518   RawCanonOutputT<char> unescaped_chars;
 519   for (int i = 0; i < length; i++) {
 520     if (input[i] == '%') {
 521       unsigned char ch;
 522       if (DecodeEscaped(input, &i, length, &ch)) {
 523         unescaped_chars.push_back(ch);
 524       } else {
 525         // Invalid escape sequence, copy the percent literal.
 526         unescaped_chars.push_back('%');
 527       }
 528     } else {
 529       // Regular non-escaped 8-bit character.
 530       unescaped_chars.push_back(input[i]);
 531     }
 532   }
 533
 534   // Convert that 8-bit to UTF-16. It's not clear IE does this at all to
 535   // JavaScript URLs, but Firefox and Safari do.
 536   for (int i = 0; i < unescaped_chars.length(); i++) {
 537     unsigned char uch = static_cast<unsigned char>(unescaped_chars.at(i));
 538     if (uch < 0x80) {
 539       // Non-UTF-8, just append directly
 540       output->push_back(uch);
 541     } else {
 542       // next_ch will point to the last character of the decoded
 543       // character.
 544       int next_character = i;
 545       unsigned code_point;
 546       if (ReadUTFChar(unescaped_chars.data(), &next_character,
 547                       unescaped_chars.length(), &code_point)) {
 548         // Valid UTF-8 character, convert to UTF-16.
 549         AppendUTF16Value(code_point, output);
 550         i = next_character;
 551       } else {
 552         // If there are any sequences that are not valid UTF-8, we keep
 553         // invalid code points and promote to UTF-16. We copy all characters
 554         // from the current position to the end of the identified sequence.
 555         while (i < next_character) {
 556           output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
 557           i++;
 558         }
 559         output->push_back(static_cast<unsigned char>(unescaped_chars.at(i)));
 560       }
 561     }
 562   }
 563 }
 564
 565 void EncodeURIComponent(const char* input, int length, CanonOutput* output) {
 566   for (int i = 0; i < length; ++i) {
 567     unsigned char c = static_cast<unsigned char>(input[i]);
 568     if (IsComponentChar(c))
 569       output->push_back(c);
 570     else
 571       AppendEscapedChar(c, output);
 572   }
 573 }
 574
 575 bool CompareSchemeComponent(const char* spec,
 576                             const Component& component,
 577                             const char* compare_to) {
 578   return DoCompareSchemeComponent(spec, component, compare_to);
 579 }
 580
 581 bool CompareSchemeComponent(const base::char16* spec,
 582                             const Component& component,
 583                             const char* compare_to) {
 584   return DoCompareSchemeComponent(spec, component, compare_to);
 585 }
 586
 587 }  // namespace url