net/base/escape.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/base/escape.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/logging.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/strings/string_piece.h"
  12 #include "base/strings/string_util.h"
  13 #include "base/strings/utf_offset_string_conversions.h"
  14 #include "base/strings/utf_string_conversions.h"
  15
  16 namespace net {
  17
  18 namespace {
  19
  20 const char kHexString[] = "0123456789ABCDEF";
  21 inline char IntToHex(int i) {
  22   DCHECK_GE(i, 0) << i << " not a hex value";
  23   DCHECK_LE(i, 15) << i << " not a hex value";
  24   return kHexString[i];
  25 }
  26
  27 // A fast bit-vector map for ascii characters.
  28 //
  29 // Internally stores 256 bits in an array of 8 ints.
  30 // Does quick bit-flicking to lookup needed characters.
  31 struct Charmap {
  32   bool Contains(unsigned char c) const {
  33     return ((map[c >> 5] & (1 << (c & 31))) != 0);
  34   }
  35
  36   uint32 map[8];
  37 };
  38
  39 // Given text to escape and a Charmap defining which values to escape,
  40 // return an escaped string.  If use_plus is true, spaces are converted
  41 // to +, otherwise, if spaces are in the charmap, they are converted to
  42 // %20.
  43 std::string Escape(const std::string& text, const Charmap& charmap,
  44                    bool use_plus) {
  45   std::string escaped;
  46   escaped.reserve(text.length() * 3);
  47   for (unsigned int i = 0; i < text.length(); ++i) {
  48     unsigned char c = static_cast<unsigned char>(text[i]);
  49     if (use_plus && ' ' == c) {
  50       escaped.push_back('+');
  51     } else if (charmap.Contains(c)) {
  52       escaped.push_back('%');
  53       escaped.push_back(IntToHex(c >> 4));
  54       escaped.push_back(IntToHex(c & 0xf));
  55     } else {
  56       escaped.push_back(c);
  57     }
  58   }
  59   return escaped;
  60 }
  61
  62 // Contains nonzero when the corresponding character is unescapable for normal
  63 // URLs. These characters are the ones that may change the parsing of a URL, so
  64 // we don't want to unescape them sometimes. In many case we won't want to
  65 // unescape spaces, but that is controlled by parameters to Unescape*.
  66 //
  67 // The basic rule is that we can't unescape anything that would changing parsing
  68 // like # or ?. We also can't unescape &, =, or + since that could be part of a
  69 // query and that could change the server's parsing of the query. Nor can we
  70 // unescape \ since src/url/ will convert it to a /.
  71 //
  72 // Lastly, we can't unescape anything that doesn't have a canonical
  73 // representation in a URL. This means that unescaping will change the URL, and
  74 // you could get different behavior if you copy and paste the URL, or press
  75 // enter in the URL bar. The list of characters that fall into this category
  76 // are the ones labeled PASS (allow either escaped or unescaped) in the big
  77 // lookup table at the top of url/url_canon_path.cc.  Also, characters
  78 // that have CHAR_QUERY set in url/url_canon_internal.cc but are not
  79 // allowed in query strings according to http://www.ietf.org/rfc/rfc3261.txt are
  80 // not unescaped, to avoid turning a valid url according to spec into an
  81 // invalid one.
  82 const char kUrlUnescape[128] = {
  83 //   NULL, control chars...
  84      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  85      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  86 //  ' ' !  "  #  $  %  &  '  (  )  *  +  ,  -  .  /
  87      0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
  88 //   0  1  2  3  4  5  6  7  8  9  :  ;  <  =  >  ?
  89      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0,
  90 //   @  A  B  C  D  E  F  G  H  I  J  K  L  M  N  O
  91      0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  92 //   P  Q  R  S  T  U  V  W  X  Y  Z  [  \  ]  ^  _
  93      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,
  94 //   `  a  b  c  d  e  f  g  h  i  j  k  l  m  n  o
  95      0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  96 //   p  q  r  s  t  u  v  w  x  y  z  {  |  }  ~  <NBSP>
  97      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0
  98 };
  99
 100 // Attempts to unescape the sequence at |index| within |escaped_text|.  If
 101 // successful, sets |value| to the unescaped value.  Returns whether
 102 // unescaping succeeded.
 103 template<typename STR>
 104 bool UnescapeUnsignedCharAtIndex(const STR& escaped_text,
 105                                  size_t index,
 106                                  unsigned char* value) {
 107   if ((index + 2) >= escaped_text.size())
 108     return false;
 109   if (escaped_text[index] != '%')
 110     return false;
 111   const typename STR::value_type most_sig_digit(
 112       static_cast<typename STR::value_type>(escaped_text[index + 1]));
 113   const typename STR::value_type least_sig_digit(
 114       static_cast<typename STR::value_type>(escaped_text[index + 2]));
 115   if (IsHexDigit(most_sig_digit) && IsHexDigit(least_sig_digit)) {
 116     *value = HexDigitToInt(most_sig_digit) * 16 +
 117       HexDigitToInt(least_sig_digit);
 118     return true;
 119   }
 120   return false;
 121 }
 122
 123 template<typename STR>
 124 STR UnescapeURLWithOffsetsImpl(const STR& escaped_text,
 125                                UnescapeRule::Type rules,
 126                                std::vector<size_t>* offsets_for_adjustment) {
 127   if (offsets_for_adjustment) {
 128     std::for_each(offsets_for_adjustment->begin(),
 129                   offsets_for_adjustment->end(),
 130                   base::LimitOffset<STR>(escaped_text.length()));
 131   }
 132   // Do not unescape anything, return the |escaped_text| text.
 133   if (rules == UnescapeRule::NONE)
 134     return escaped_text;
 135
 136   // The output of the unescaping is always smaller than the input, so we can
 137   // reserve the input size to make sure we have enough buffer and don't have
 138   // to allocate in the loop below.
 139   STR result;
 140   result.reserve(escaped_text.length());
 141
 142   // Locations of adjusted text.
 143   net::internal::AdjustEncodingOffset::Adjustments adjustments;
 144   for (size_t i = 0, max = escaped_text.size(); i < max; ++i) {
 145     if (static_cast<unsigned char>(escaped_text[i]) >= 128) {
 146       // Non ASCII character, append as is.
 147       result.push_back(escaped_text[i]);
 148       continue;
 149     }
 150
 151     unsigned char first_byte;
 152     if (UnescapeUnsignedCharAtIndex(escaped_text, i, &first_byte)) {
 153       // Per http://tools.ietf.org/html/rfc3987#section-4.1, the following BiDi
 154       // control characters are not allowed to appear unescaped in URLs:
 155       //
 156       // U+200E LEFT-TO-RIGHT MARK         (%E2%80%8E)
 157       // U+200F RIGHT-TO-LEFT MARK         (%E2%80%8F)
 158       // U+202A LEFT-TO-RIGHT EMBEDDING    (%E2%80%AA)
 159       // U+202B RIGHT-TO-LEFT EMBEDDING    (%E2%80%AB)
 160       // U+202C POP DIRECTIONAL FORMATTING (%E2%80%AC)
 161       // U+202D LEFT-TO-RIGHT OVERRIDE     (%E2%80%AD)
 162       // U+202E RIGHT-TO-LEFT OVERRIDE     (%E2%80%AE)
 163       //
 164       // Additionally, the Unicode Technical Report (TR9) as referenced by RFC
 165       // 3987 above has since added some new BiDi control characters.
 166       // http://www.unicode.org/reports/tr9
 167       //
 168       // U+061C ARABIC LETTER MARK         (%D8%9C)
 169       // U+2066 LEFT-TO-RIGHT ISOLATE      (%E2%81%A6)
 170       // U+2067 RIGHT-TO-LEFT ISOLATE      (%E2%81%A7)
 171       // U+2068 FIRST STRONG ISOLATE       (%E2%81%A8)
 172       // U+2069 POP DIRECTIONAL ISOLATE    (%E2%81%A9)
 173
 174       unsigned char second_byte;
 175       // Check for ALM.
 176       if ((first_byte == 0xD8) &&
 177           UnescapeUnsignedCharAtIndex(escaped_text, i + 3, &second_byte) &&
 178           (second_byte == 0x9c)) {
 179         result.append(escaped_text, i, 6);
 180         i += 5;
 181         continue;
 182       }
 183
 184       // Check for other BiDi control characters.
 185       if ((first_byte == 0xE2) &&
 186           UnescapeUnsignedCharAtIndex(escaped_text, i + 3, &second_byte) &&
 187           ((second_byte == 0x80) || (second_byte == 0x81))) {
 188         unsigned char third_byte;
 189         if (UnescapeUnsignedCharAtIndex(escaped_text, i + 6, &third_byte) &&
 190             ((second_byte == 0x80) ?
 191              ((third_byte == 0x8E) || (third_byte == 0x8F) ||
 192               ((third_byte >= 0xAA) && (third_byte <= 0xAE))) :
 193              ((third_byte >= 0xA6) && (third_byte <= 0xA9)))) {
 194           result.append(escaped_text, i, 9);
 195           i += 8;
 196           continue;
 197         }
 198       }
 199
 200       if (first_byte >= 0x80 ||  // Unescape all high-bit characters.
 201           // For 7-bit characters, the lookup table tells us all valid chars.
 202           (kUrlUnescape[first_byte] ||
 203            // ...and we allow some additional unescaping when flags are set.
 204            (first_byte == ' ' && (rules & UnescapeRule::SPACES)) ||
 205            // Allow any of the prohibited but non-control characters when
 206            // we're doing "special" chars.
 207            (first_byte > ' ' && (rules & UnescapeRule::URL_SPECIAL_CHARS)) ||
 208            // Additionally allow control characters if requested.
 209            (first_byte < ' ' && (rules & UnescapeRule::CONTROL_CHARS)))) {
 210         // Use the unescaped version of the character.
 211         adjustments.push_back(i);
 212         result.push_back(first_byte);
 213         i += 2;
 214       } else {
 215         // Keep escaped. Append a percent and we'll get the following two
 216         // digits on the next loops through.
 217         result.push_back('%');
 218       }
 219     } else if ((rules & UnescapeRule::REPLACE_PLUS_WITH_SPACE) &&
 220                escaped_text[i] == '+') {
 221       result.push_back(' ');
 222     } else {
 223       // Normal case for unescaped characters.
 224       result.push_back(escaped_text[i]);
 225     }
 226   }
 227
 228   // Make offset adjustment.
 229   if (offsets_for_adjustment && !adjustments.empty()) {
 230     std::for_each(offsets_for_adjustment->begin(),
 231                    offsets_for_adjustment->end(),
 232                    net::internal::AdjustEncodingOffset(adjustments));
 233   }
 234
 235   return result;
 236 }
 237
 238 template <class str>
 239 void AppendEscapedCharForHTMLImpl(typename str::value_type c, str* output) {
 240   static const struct {
 241     char key;
 242     const char* replacement;
 243   } kCharsToEscape[] = {
 244     { '<', "&lt;" },
 245     { '>', "&gt;" },
 246     { '&', "&amp;" },
 247     { '"', "&quot;" },
 248     { '\'', "&#39;" },
 249   };
 250   size_t k;
 251   for (k = 0; k < ARRAYSIZE_UNSAFE(kCharsToEscape); ++k) {
 252     if (c == kCharsToEscape[k].key) {
 253       const char* p = kCharsToEscape[k].replacement;
 254       while (*p)
 255         output->push_back(*p++);
 256       break;
 257     }
 258   }
 259   if (k == ARRAYSIZE_UNSAFE(kCharsToEscape))
 260     output->push_back(c);
 261 }
 262
 263 template <class str>
 264 str EscapeForHTMLImpl(const str& input) {
 265   str result;
 266   result.reserve(input.size());  // Optimize for no escaping.
 267
 268   for (typename str::const_iterator i = input.begin(); i != input.end(); ++i)
 269     AppendEscapedCharForHTMLImpl(*i, &result);
 270
 271   return result;
 272 }
 273
 274 // Everything except alphanumerics and !'()*-._~
 275 // See RFC 2396 for the list of reserved characters.
 276 static const Charmap kQueryCharmap = {{
 277   0xffffffffL, 0xfc00987dL, 0x78000001L, 0xb8000001L,
 278   0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL
 279 }};
 280
 281 // non-printable, non-7bit, and (including space)  "#%:<>?[\]^`{|}
 282 static const Charmap kPathCharmap = {{
 283   0xffffffffL, 0xd400002dL, 0x78000000L, 0xb8000001L,
 284   0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL
 285 }};
 286
 287 // non-printable, non-7bit, and (including space) ?>=<;+'&%$#"![\]^`{|}
 288 static const Charmap kUrlEscape = {{
 289   0xffffffffL, 0xf80008fdL, 0x78000001L, 0xb8000001L,
 290   0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL
 291 }};
 292
 293 // non-7bit
 294 static const Charmap kNonASCIICharmap = {{
 295   0x00000000L, 0x00000000L, 0x00000000L, 0x00000000L,
 296   0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL
 297 }};
 298
 299 // Everything except alphanumerics, the reserved characters(;/?:@&=+$,) and
 300 // !'()*-._~%
 301 static const Charmap kExternalHandlerCharmap = {{
 302   0xffffffffL, 0x5000080dL, 0x68000000L, 0xb8000001L,
 303   0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL
 304 }};
 305
 306 }  // namespace
 307
 308 std::string EscapeQueryParamValue(const std::string& text, bool use_plus) {
 309   return Escape(text, kQueryCharmap, use_plus);
 310 }
 311
 312 std::string EscapePath(const std::string& path) {
 313   return Escape(path, kPathCharmap, false);
 314 }
 315
 316 std::string EscapeUrlEncodedData(const std::string& path, bool use_plus) {
 317   return Escape(path, kUrlEscape, use_plus);
 318 }
 319
 320 std::string EscapeNonASCII(const std::string& input) {
 321   return Escape(input, kNonASCIICharmap, false);
 322 }
 323
 324 std::string EscapeExternalHandlerValue(const std::string& text) {
 325   return Escape(text, kExternalHandlerCharmap, false);
 326 }
 327
 328 void AppendEscapedCharForHTML(char c, std::string* output) {
 329   AppendEscapedCharForHTMLImpl(c, output);
 330 }
 331
 332 std::string EscapeForHTML(const std::string& input) {
 333   return EscapeForHTMLImpl(input);
 334 }
 335
 336 base::string16 EscapeForHTML(const base::string16& input) {
 337   return EscapeForHTMLImpl(input);
 338 }
 339
 340 std::string UnescapeURLComponent(const std::string& escaped_text,
 341                                  UnescapeRule::Type rules) {
 342   return UnescapeURLWithOffsetsImpl(escaped_text, rules, NULL);
 343 }
 344
 345 base::string16 UnescapeURLComponent(const base::string16& escaped_text,
 346                                     UnescapeRule::Type rules) {
 347   return UnescapeURLWithOffsetsImpl(escaped_text, rules, NULL);
 348 }
 349
 350 base::string16 UnescapeAndDecodeUTF8URLComponent(
 351     const std::string& text,
 352     UnescapeRule::Type rules,
 353     size_t* offset_for_adjustment) {
 354   std::vector<size_t> offsets;
 355   if (offset_for_adjustment)
 356     offsets.push_back(*offset_for_adjustment);
 357   base::string16 result =
 358       UnescapeAndDecodeUTF8URLComponentWithOffsets(text, rules, &offsets);
 359   if (offset_for_adjustment)
 360     *offset_for_adjustment = offsets[0];
 361   return result;
 362 }
 363
 364 base::string16 UnescapeAndDecodeUTF8URLComponentWithOffsets(
 365     const std::string& text,
 366     UnescapeRule::Type rules,
 367     std::vector<size_t>* offsets_for_adjustment) {
 368   base::string16 result;
 369   std::vector<size_t> original_offsets;
 370   if (offsets_for_adjustment)
 371     original_offsets = *offsets_for_adjustment;
 372   std::string unescaped_url(
 373       UnescapeURLWithOffsetsImpl(text, rules, offsets_for_adjustment));
 374   if (base::UTF8ToUTF16AndAdjustOffsets(unescaped_url.data(),
 375                                         unescaped_url.length(),
 376                                         &result, offsets_for_adjustment))
 377     return result;  // Character set looks like it's valid.
 378
 379   // Not valid.  Return the escaped version.  Undo our changes to
 380   // |offset_for_adjustment| since we haven't changed the string after all.
 381   if (offsets_for_adjustment)
 382     *offsets_for_adjustment = original_offsets;
 383   return base::UTF8ToUTF16AndAdjustOffsets(text, offsets_for_adjustment);
 384 }
 385
 386 base::string16 UnescapeForHTML(const base::string16& input) {
 387   static const struct {
 388     const char* ampersand_code;
 389     const char replacement;
 390   } kEscapeToChars[] = {
 391     { "&lt;", '<' },
 392     { "&gt;", '>' },
 393     { "&amp;", '&' },
 394     { "&quot;", '"' },
 395     { "&#39;", '\''},
 396   };
 397
 398   if (input.find(base::ASCIIToUTF16("&")) == std::string::npos)
 399     return input;
 400
 401   base::string16 ampersand_chars[ARRAYSIZE_UNSAFE(kEscapeToChars)];
 402   base::string16 text(input);
 403   for (base::string16::iterator iter = text.begin();
 404        iter != text.end(); ++iter) {
 405     if (*iter == '&') {
 406       // Potential ampersand encode char.
 407       size_t index = iter - text.begin();
 408       for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kEscapeToChars); i++) {
 409         if (ampersand_chars[i].empty()) {
 410           ampersand_chars[i] =
 411               base::ASCIIToUTF16(kEscapeToChars[i].ampersand_code);
 412         }
 413         if (text.find(ampersand_chars[i], index) == index) {
 414           text.replace(iter, iter + ampersand_chars[i].length(),
 415                        1, kEscapeToChars[i].replacement);
 416           break;
 417         }
 418       }
 419     }
 420   }
 421   return text;
 422 }
 423
 424 namespace internal {
 425
 426 AdjustEncodingOffset::AdjustEncodingOffset(const Adjustments& adjustments)
 427   : adjustments(adjustments) {}
 428
 429 void AdjustEncodingOffset::operator()(size_t& offset) {
 430   // For each encoded character occurring before an offset subtract 2.
 431   if (offset == base::string16::npos)
 432     return;
 433   size_t adjusted_offset = offset;
 434   for (Adjustments::const_iterator i = adjustments.begin();
 435        i != adjustments.end(); ++i) {
 436     size_t location = *i;
 437     if (offset <= location) {
 438       offset = adjusted_offset;
 439       return;
 440     }
 441     if (offset <= (location + 2)) {
 442       offset = base::string16::npos;
 443       return;
 444     }
 445     adjusted_offset -= 2;
 446   }
 447   offset = adjusted_offset;
 448 }
 449
 450 }  // namespace internal
 451
 452 }  // namespace net