1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/base/net_util.h"
10 #include "base/i18n/time_formatting.h"
11 #include "base/json/string_escape.h"
12 #include "base/lazy_instance.h"
13 #include "base/logging.h"
14 #include "base/memory/singleton.h"
15 #include "base/stl_util.h"
16 #include "base/strings/string_tokenizer.h"
17 #include "base/strings/string_util.h"
18 #include "base/strings/utf_offset_string_conversions.h"
19 #include "base/strings/utf_string_conversions.h"
20 #include "base/time/time.h"
22 #include "third_party/icu/source/common/unicode/uidna.h"
23 #include "third_party/icu/source/common/unicode/uniset.h"
24 #include "third_party/icu/source/common/unicode/uscript.h"
25 #include "third_party/icu/source/common/unicode/uset.h"
26 #include "third_party/icu/source/i18n/unicode/datefmt.h"
27 #include "third_party/icu/source/i18n/unicode/regex.h"
28 #include "third_party/icu/source/i18n/unicode/ulocdata.h"
36 typedef std::vector
<size_t> Offsets
;
38 // Does some simple normalization of scripts so we can allow certain scripts
40 // TODO(brettw) bug 880223: we should allow some other languages to be
41 // oombined such as Chinese and Latin. We will probably need a more
42 // complicated system of language pairs to have more fine-grained control.
43 UScriptCode
NormalizeScript(UScriptCode code
) {
45 case USCRIPT_KATAKANA
:
46 case USCRIPT_HIRAGANA
:
47 case USCRIPT_KATAKANA_OR_HIRAGANA
:
48 case USCRIPT_HANGUL
: // This one is arguable.
55 bool IsIDNComponentInSingleScript(const base::char16
* str
, int str_len
) {
56 UScriptCode first_script
= USCRIPT_INVALID_CODE
;
62 U16_NEXT(str
, i
, str_len
, code_point
);
64 UErrorCode err
= U_ZERO_ERROR
;
65 UScriptCode cur_script
= uscript_getScript(code_point
, &err
);
66 if (err
!= U_ZERO_ERROR
)
67 return false; // Report mixed on error.
68 cur_script
= NormalizeScript(cur_script
);
70 // TODO(brettw) We may have to check for USCRIPT_INHERENT as well.
71 if (is_first
&& cur_script
!= USCRIPT_COMMON
) {
72 first_script
= cur_script
;
75 if (cur_script
!= USCRIPT_COMMON
&& cur_script
!= first_script
)
82 // Check if the script of a language can be 'safely' mixed with
83 // Latin letters in the ASCII range.
84 bool IsCompatibleWithASCIILetters(const std::string
& lang
) {
85 // For now, just list Chinese, Japanese and Korean (positive list).
86 // An alternative is negative-listing (languages using Greek and
87 // Cyrillic letters), but it can be more dangerous.
88 return !lang
.substr(0, 2).compare("zh") ||
89 !lang
.substr(0, 2).compare("ja") ||
90 !lang
.substr(0, 2).compare("ko");
93 typedef std::map
<std::string
, icu::UnicodeSet
*> LangToExemplarSetMap
;
95 class LangToExemplarSet
{
97 static LangToExemplarSet
* GetInstance() {
98 return Singleton
<LangToExemplarSet
>::get();
102 LangToExemplarSetMap map
;
103 LangToExemplarSet() { }
104 ~LangToExemplarSet() {
105 STLDeleteContainerPairSecondPointers(map
.begin(), map
.end());
108 friend class Singleton
<LangToExemplarSet
>;
109 friend struct DefaultSingletonTraits
<LangToExemplarSet
>;
110 friend bool GetExemplarSetForLang(const std::string
&, icu::UnicodeSet
**);
111 friend void SetExemplarSetForLang(const std::string
&, icu::UnicodeSet
*);
113 DISALLOW_COPY_AND_ASSIGN(LangToExemplarSet
);
116 bool GetExemplarSetForLang(const std::string
& lang
,
117 icu::UnicodeSet
** lang_set
) {
118 const LangToExemplarSetMap
& map
= LangToExemplarSet::GetInstance()->map
;
119 LangToExemplarSetMap::const_iterator pos
= map
.find(lang
);
120 if (pos
!= map
.end()) {
121 *lang_set
= pos
->second
;
127 void SetExemplarSetForLang(const std::string
& lang
,
128 icu::UnicodeSet
* lang_set
) {
129 LangToExemplarSetMap
& map
= LangToExemplarSet::GetInstance()->map
;
130 map
.insert(std::make_pair(lang
, lang_set
));
133 static base::LazyInstance
<base::Lock
>::Leaky
134 g_lang_set_lock
= LAZY_INSTANCE_INITIALIZER
;
136 // Returns true if all the characters in component_characters are used by
137 // the language |lang|.
138 bool IsComponentCoveredByLang(const icu::UnicodeSet
& component_characters
,
139 const std::string
& lang
) {
140 CR_DEFINE_STATIC_LOCAL(
141 const icu::UnicodeSet
, kASCIILetters
, ('a', 'z'));
142 icu::UnicodeSet
* lang_set
= nullptr;
143 // We're called from both the UI thread and the history thread.
145 base::AutoLock
lock(g_lang_set_lock
.Get());
146 if (!GetExemplarSetForLang(lang
, &lang_set
)) {
147 UErrorCode status
= U_ZERO_ERROR
;
148 ULocaleData
* uld
= ulocdata_open(lang
.c_str(), &status
);
149 // TODO(jungshik) Turn this check on when the ICU data file is
150 // rebuilt with the minimal subset of locale data for languages
151 // to which Chrome is not localized but which we offer in the list
152 // of languages selectable for Accept-Languages. With the rebuilt ICU
153 // data, ulocdata_open never should fall back to the default locale.
155 // DCHECK(U_SUCCESS(status) && status != U_USING_DEFAULT_WARNING);
156 if (U_SUCCESS(status
) && status
!= U_USING_DEFAULT_WARNING
) {
157 lang_set
= reinterpret_cast<icu::UnicodeSet
*>(ulocdata_getExemplarSet(
158 uld
, nullptr, 0, ULOCDATA_ES_STANDARD
, &status
));
159 // On success, if |lang| is compatible with ASCII Latin letters, add
161 if (lang_set
&& IsCompatibleWithASCIILetters(lang
))
162 lang_set
->addAll(kASCIILetters
);
166 lang_set
= new icu::UnicodeSet(1, 0);
169 SetExemplarSetForLang(lang
, lang_set
);
173 return !lang_set
->isEmpty() && lang_set
->containsAll(component_characters
);
176 // Returns true if the given Unicode host component is safe to display to the
178 bool IsIDNComponentSafe(const base::char16
* str
,
180 const std::string
& languages
) {
181 // Most common cases (non-IDN) do not reach here so that we don't
182 // need a fast return path.
183 // TODO(jungshik) : Check if there's any character inappropriate
184 // (although allowed) for domain names.
185 // See http://www.unicode.org/reports/tr39/#IDN_Security_Profiles and
186 // http://www.unicode.org/reports/tr39/data/xidmodifications.txt
187 // For now, we borrow the list from Mozilla and tweaked it slightly.
188 // (e.g. Characters like U+00A0, U+3000, U+3002 are omitted because
189 // they're gonna be canonicalized to U+0020 and full stop before
191 // The original list is available at
192 // http://kb.mozillazine.org/Network.IDN.blacklist_chars and
193 // at http://mxr.mozilla.org/seamonkey/source/modules/libpref/src/init/all.js#703
195 UErrorCode status
= U_ZERO_ERROR
;
196 #ifdef U_WCHAR_IS_UTF16
197 icu::UnicodeSet
dangerous_characters(icu::UnicodeString(
198 L
"[[\\ \u00ad\u00bc\u00bd\u01c3\u0337\u0338"
199 L
"\u05c3\u05f4\u06d4\u0702\u115f\u1160][\u2000-\u200b]"
200 L
"[\u2024\u2027\u2028\u2029\u2039\u203a\u2044\u205f]"
201 L
"[\u2154-\u2156][\u2159-\u215b][\u215f\u2215\u23ae"
202 L
"\u29f6\u29f8\u2afb\u2afd][\u2ff0-\u2ffb][\u3014"
203 L
"\u3015\u3033\u3164\u321d\u321e\u33ae\u33af\u33c6\u33df\ufe14"
204 L
"\ufe15\ufe3f\ufe5d\ufe5e\ufeff\uff0e\uff06\uff61\uffa0\ufff9]"
205 L
"[\ufffa-\ufffd]]"), status
);
206 DCHECK(U_SUCCESS(status
));
207 icu::RegexMatcher
dangerous_patterns(icu::UnicodeString(
208 // Lone katakana no, so, or n
209 L
"[^\\p{Katakana}][\u30ce\u30f3\u30bd][^\\p{Katakana}]"
210 // Repeating Japanese accent characters
211 L
"|[\u3099\u309a\u309b\u309c][\u3099\u309a\u309b\u309c]"),
214 icu::UnicodeSet
dangerous_characters(icu::UnicodeString(
215 "[[\\u0020\\u00ad\\u00bc\\u00bd\\u01c3\\u0337\\u0338"
216 "\\u05c3\\u05f4\\u06d4\\u0702\\u115f\\u1160][\\u2000-\\u200b]"
217 "[\\u2024\\u2027\\u2028\\u2029\\u2039\\u203a\\u2044\\u205f]"
218 "[\\u2154-\\u2156][\\u2159-\\u215b][\\u215f\\u2215\\u23ae"
219 "\\u29f6\\u29f8\\u2afb\\u2afd][\\u2ff0-\\u2ffb][\\u3014"
220 "\\u3015\\u3033\\u3164\\u321d\\u321e\\u33ae\\u33af\\u33c6\\u33df\\ufe14"
221 "\\ufe15\\ufe3f\\ufe5d\\ufe5e\\ufeff\\uff0e\\uff06\\uff61\\uffa0\\ufff9]"
222 "[\\ufffa-\\ufffd]]", -1, US_INV
), status
);
223 DCHECK(U_SUCCESS(status
));
224 icu::RegexMatcher
dangerous_patterns(icu::UnicodeString(
225 // Lone katakana no, so, or n
226 "[^\\p{Katakana}][\\u30ce\\u30f3\\u30bd][^\\p{Katakana}]"
227 // Repeating Japanese accent characters
228 "|[\\u3099\\u309a\\u309b\\u309c][\\u3099\\u309a\\u309b\\u309c]"),
231 DCHECK(U_SUCCESS(status
));
232 icu::UnicodeSet component_characters
;
233 icu::UnicodeString
component_string(str
, str_len
);
234 component_characters
.addAll(component_string
);
235 if (dangerous_characters
.containsSome(component_characters
))
238 DCHECK(U_SUCCESS(status
));
239 dangerous_patterns
.reset(component_string
);
240 if (dangerous_patterns
.find())
243 // If the language list is empty, the result is completely determined
244 // by whether a component is a single script or not. This will block
245 // even "safe" script mixing cases like <Chinese, Latin-ASCII> that are
246 // allowed with |languages| (while it blocks Chinese + Latin letters with
247 // an accent as should be the case), but we want to err on the safe side
248 // when |languages| is empty.
249 if (languages
.empty())
250 return IsIDNComponentInSingleScript(str
, str_len
);
252 // |common_characters| is made up of ASCII numbers, hyphen, plus and
253 // underscore that are used across scripts and allowed in domain names.
254 // (sync'd with characters allowed in url_canon_host with square
255 // brackets excluded.) See kHostCharLookup[] array in url_canon_host.cc.
256 icu::UnicodeSet
common_characters(UNICODE_STRING_SIMPLE("[[0-9]\\-_+\\ ]"),
258 DCHECK(U_SUCCESS(status
));
259 // Subtract common characters because they're always allowed so that
260 // we just have to check if a language-specific set contains
262 component_characters
.removeAll(common_characters
);
264 base::StringTokenizer
t(languages
, ",");
265 while (t
.GetNext()) {
266 if (IsComponentCoveredByLang(component_characters
, t
.token()))
272 // A wrapper to use LazyInstance<>::Leaky with ICU's UIDNA, a C pointer to
273 // a UTS46/IDNA 2008 handling object opened with uidna_openUTS46().
275 // We use UTS46 with BiDiCheck to migrate from IDNA 2003 to IDNA 2008 with
276 // the backward compatibility in mind. What it does:
278 // 1. Use the up-to-date Unicode data.
279 // 2. Define a case folding/mapping with the up-to-date Unicode data as
281 // 3. Use transitional mechanism for 4 deviation characters (sharp-s,
282 // final sigma, ZWJ and ZWNJ) for now.
283 // 4. Continue to allow symbols and punctuations.
284 // 5. Apply new BiDi check rules more permissive than the IDNA 2003 BiDI rules.
285 // 6. Do not apply STD3 rules
286 // 7. Do not allow unassigned code points.
288 // It also closely matches what IE 10 does except for the BiDi check (
289 // http://goo.gl/3XBhqw ).
290 // See http://http://unicode.org/reports/tr46/ and references therein
292 struct UIDNAWrapper
{
294 UErrorCode err
= U_ZERO_ERROR
;
295 // TODO(jungshik): Change options as different parties (browsers,
296 // registrars, search engines) converge toward a consensus.
297 value
= uidna_openUTS46(UIDNA_CHECK_BIDI
, &err
);
305 static base::LazyInstance
<UIDNAWrapper
>::Leaky
306 g_uidna
= LAZY_INSTANCE_INITIALIZER
;
308 // Converts one component of a host (between dots) to IDN if safe. The result
309 // will be APPENDED to the given output string and will be the same as the input
310 // if it is not IDN or the IDN is unsafe to display. Returns whether any
311 // conversion was performed.
312 bool IDNToUnicodeOneComponent(const base::char16
* comp
,
314 const std::string
& languages
,
315 base::string16
* out
) {
320 // Only transform if the input can be an IDN component.
321 static const base::char16 kIdnPrefix
[] = {'x', 'n', '-', '-'};
322 if ((comp_len
> arraysize(kIdnPrefix
)) &&
323 !memcmp(comp
, kIdnPrefix
, arraysize(kIdnPrefix
) * sizeof(base::char16
))) {
324 UIDNA
* uidna
= g_uidna
.Get().value
;
325 DCHECK(uidna
!= NULL
);
326 size_t original_length
= out
->length();
327 int output_length
= 64;
328 UIDNAInfo info
= UIDNA_INFO_INITIALIZER
;
331 out
->resize(original_length
+ output_length
);
332 status
= U_ZERO_ERROR
;
333 // This returns the actual length required. If this is more than 64
334 // code units, |status| will be U_BUFFER_OVERFLOW_ERROR and we'll try
335 // the conversion again, but with a sufficiently large buffer.
336 output_length
= uidna_labelToUnicode(
337 uidna
, comp
, static_cast<int32_t>(comp_len
), &(*out
)[original_length
],
338 output_length
, &info
, &status
);
339 } while ((status
== U_BUFFER_OVERFLOW_ERROR
&& info
.errors
== 0));
341 if (U_SUCCESS(status
) && info
.errors
== 0) {
342 // Converted successfully. Ensure that the converted component
343 // can be safely displayed to the user.
344 out
->resize(original_length
+ output_length
);
345 if (IsIDNComponentSafe(out
->data() + original_length
, output_length
,
350 // Something went wrong. Revert to original string.
351 out
->resize(original_length
);
354 // We get here with no IDN or on error, in which case we just append the
356 out
->append(comp
, comp_len
);
360 // TODO(brettw) bug 734373: check the scripts for each host component and
361 // don't un-IDN-ize if there is more than one. Alternatively, only IDN for
362 // scripts that the user has installed. For now, just put the entire
363 // path through IDN. Maybe this feature can be implemented in ICU itself?
365 // We may want to skip this step in the case of file URLs to allow unicode
366 // UNC hostnames regardless of encodings.
367 base::string16
IDNToUnicodeWithAdjustments(
368 const std::string
& host
,
369 const std::string
& languages
,
370 base::OffsetAdjuster::Adjustments
* adjustments
) {
372 adjustments
->clear();
373 // Convert the ASCII input to a base::string16 for ICU.
374 base::string16 input16
;
375 input16
.reserve(host
.length());
376 input16
.insert(input16
.end(), host
.begin(), host
.end());
378 // Do each component of the host separately, since we enforce script matching
379 // on a per-component basis.
380 base::string16 out16
;
382 for (size_t component_start
= 0, component_end
;
383 component_start
< input16
.length();
384 component_start
= component_end
+ 1) {
385 // Find the end of the component.
386 component_end
= input16
.find('.', component_start
);
387 if (component_end
== base::string16::npos
)
388 component_end
= input16
.length(); // For getting the last component.
389 size_t component_length
= component_end
- component_start
;
390 size_t new_component_start
= out16
.length();
391 bool converted_idn
= false;
392 if (component_end
> component_start
) {
393 // Add the substring that we just found.
394 converted_idn
= IDNToUnicodeOneComponent(
395 input16
.data() + component_start
, component_length
, languages
,
398 size_t new_component_length
= out16
.length() - new_component_start
;
400 if (converted_idn
&& adjustments
) {
401 adjustments
->push_back(base::OffsetAdjuster::Adjustment(
402 component_start
, component_length
, new_component_length
));
405 // Need to add the dot we just found (if we found one).
406 if (component_end
< input16
.length())
407 out16
.push_back('.');
413 // If |component| is valid, its begin is incremented by |delta|.
414 void AdjustComponent(int delta
, url::Component
* component
) {
415 if (!component
->is_valid())
418 DCHECK(delta
>= 0 || component
->begin
>= -delta
);
419 component
->begin
+= delta
;
422 // Adjusts all the components of |parsed| by |delta|, except for the scheme.
423 void AdjustAllComponentsButScheme(int delta
, url::Parsed
* parsed
) {
424 AdjustComponent(delta
, &(parsed
->username
));
425 AdjustComponent(delta
, &(parsed
->password
));
426 AdjustComponent(delta
, &(parsed
->host
));
427 AdjustComponent(delta
, &(parsed
->port
));
428 AdjustComponent(delta
, &(parsed
->path
));
429 AdjustComponent(delta
, &(parsed
->query
));
430 AdjustComponent(delta
, &(parsed
->ref
));
433 // Helper for FormatUrlWithOffsets().
434 base::string16
FormatViewSourceUrl(
436 const std::string
& languages
,
437 FormatUrlTypes format_types
,
438 UnescapeRule::Type unescape_rules
,
439 url::Parsed
* new_parsed
,
441 base::OffsetAdjuster::Adjustments
* adjustments
) {
443 const char kViewSource
[] = "view-source:";
444 const size_t kViewSourceLength
= arraysize(kViewSource
) - 1;
446 // Format the underlying URL and record adjustments.
447 const std::string
& url_str(url
.possibly_invalid_spec());
448 adjustments
->clear();
449 base::string16
result(base::ASCIIToUTF16(kViewSource
) +
450 FormatUrlWithAdjustments(GURL(url_str
.substr(kViewSourceLength
)),
451 languages
, format_types
, unescape_rules
,
452 new_parsed
, prefix_end
, adjustments
));
453 // Revise |adjustments| by shifting to the offsets to prefix that the above
454 // call to FormatUrl didn't get to see.
455 for (base::OffsetAdjuster::Adjustments::iterator it
= adjustments
->begin();
456 it
!= adjustments
->end(); ++it
)
457 it
->original_offset
+= kViewSourceLength
;
459 // Adjust positions of the parsed components.
460 if (new_parsed
->scheme
.is_nonempty()) {
461 // Assume "view-source:real-scheme" as a scheme.
462 new_parsed
->scheme
.len
+= kViewSourceLength
;
464 new_parsed
->scheme
.begin
= 0;
465 new_parsed
->scheme
.len
= kViewSourceLength
- 1;
467 AdjustAllComponentsButScheme(kViewSourceLength
, new_parsed
);
470 *prefix_end
+= kViewSourceLength
;
475 class AppendComponentTransform
{
477 AppendComponentTransform() {}
478 virtual ~AppendComponentTransform() {}
480 virtual base::string16
Execute(
481 const std::string
& component_text
,
482 base::OffsetAdjuster::Adjustments
* adjustments
) const = 0;
484 // NOTE: No DISALLOW_COPY_AND_ASSIGN here, since gcc < 4.3.0 requires an
485 // accessible copy constructor in order to call AppendFormattedComponent()
486 // with an inline temporary (see http://gcc.gnu.org/bugs/#cxx%5Frvalbind ).
489 class HostComponentTransform
: public AppendComponentTransform
{
491 explicit HostComponentTransform(const std::string
& languages
)
492 : languages_(languages
) {
496 base::string16
Execute(
497 const std::string
& component_text
,
498 base::OffsetAdjuster::Adjustments
* adjustments
) const override
{
499 return IDNToUnicodeWithAdjustments(component_text
, languages_
,
503 const std::string
& languages_
;
506 class NonHostComponentTransform
: public AppendComponentTransform
{
508 explicit NonHostComponentTransform(UnescapeRule::Type unescape_rules
)
509 : unescape_rules_(unescape_rules
) {
513 base::string16
Execute(
514 const std::string
& component_text
,
515 base::OffsetAdjuster::Adjustments
* adjustments
) const override
{
516 return (unescape_rules_
== UnescapeRule::NONE
) ?
517 base::UTF8ToUTF16WithAdjustments(component_text
, adjustments
) :
518 UnescapeAndDecodeUTF8URLComponentWithAdjustments(component_text
,
519 unescape_rules_
, adjustments
);
522 const UnescapeRule::Type unescape_rules_
;
525 // Transforms the portion of |spec| covered by |original_component| according to
526 // |transform|. Appends the result to |output|. If |output_component| is
527 // non-NULL, its start and length are set to the transformed component's new
528 // start and length. If |adjustments| is non-NULL, appends adjustments (if
529 // any) that reflect the transformation the original component underwent to
530 // become the transformed value appended to |output|.
531 void AppendFormattedComponent(const std::string
& spec
,
532 const url::Component
& original_component
,
533 const AppendComponentTransform
& transform
,
534 base::string16
* output
,
535 url::Component
* output_component
,
536 base::OffsetAdjuster::Adjustments
* adjustments
) {
538 if (original_component
.is_nonempty()) {
539 size_t original_component_begin
=
540 static_cast<size_t>(original_component
.begin
);
541 size_t output_component_begin
= output
->length();
542 std::string
component_str(spec
, original_component_begin
,
543 static_cast<size_t>(original_component
.len
));
545 // Transform |component_str| and modify |adjustments| appropriately.
546 base::OffsetAdjuster::Adjustments component_transform_adjustments
;
548 transform
.Execute(component_str
, &component_transform_adjustments
));
550 // Shift all the adjustments made for this component so the offsets are
551 // valid for the original string and add them to |adjustments|.
552 for (base::OffsetAdjuster::Adjustments::iterator comp_iter
=
553 component_transform_adjustments
.begin();
554 comp_iter
!= component_transform_adjustments
.end(); ++comp_iter
)
555 comp_iter
->original_offset
+= original_component_begin
;
557 adjustments
->insert(adjustments
->end(),
558 component_transform_adjustments
.begin(),
559 component_transform_adjustments
.end());
562 // Set positions of the parsed component.
563 if (output_component
) {
564 output_component
->begin
= static_cast<int>(output_component_begin
);
565 output_component
->len
=
566 static_cast<int>(output
->length() - output_component_begin
);
568 } else if (output_component
) {
569 output_component
->reset();
575 const FormatUrlType kFormatUrlOmitNothing
= 0;
576 const FormatUrlType kFormatUrlOmitUsernamePassword
= 1 << 0;
577 const FormatUrlType kFormatUrlOmitHTTP
= 1 << 1;
578 const FormatUrlType kFormatUrlOmitTrailingSlashOnBareHostname
= 1 << 2;
579 const FormatUrlType kFormatUrlOmitAll
= kFormatUrlOmitUsernamePassword
|
580 kFormatUrlOmitHTTP
| kFormatUrlOmitTrailingSlashOnBareHostname
;
582 base::string16
IDNToUnicode(const std::string
& host
,
583 const std::string
& languages
) {
584 return IDNToUnicodeWithAdjustments(host
, languages
, NULL
);
587 std::string
GetDirectoryListingEntry(const base::string16
& name
,
588 const std::string
& raw_bytes
,
593 result
.append("<script>addRow(");
594 base::EscapeJSONString(name
, true, &result
);
596 if (raw_bytes
.empty()) {
597 base::EscapeJSONString(EscapePath(base::UTF16ToUTF8(name
)), true, &result
);
599 base::EscapeJSONString(EscapePath(raw_bytes
), true, &result
);
602 result
.append(",1,");
604 result
.append(",0,");
607 // Negative size means unknown or not applicable (e.g. directory).
608 base::string16 size_string
;
610 size_string
= FormatBytesUnlocalized(size
);
611 base::EscapeJSONString(size_string
, true, &result
);
615 base::string16 modified_str
;
616 // |modified| can be NULL in FTP listings.
617 if (!modified
.is_null()) {
618 modified_str
= base::TimeFormatShortDateAndTime(modified
);
620 base::EscapeJSONString(modified_str
, true, &result
);
622 result
.append(");</script>\n");
627 void AppendFormattedHost(const GURL
& url
,
628 const std::string
& languages
,
629 base::string16
* output
) {
630 AppendFormattedComponent(url
.possibly_invalid_spec(),
631 url
.parsed_for_possibly_invalid_spec().host
,
632 HostComponentTransform(languages
), output
, NULL
, NULL
);
635 base::string16
FormatUrlWithOffsets(
637 const std::string
& languages
,
638 FormatUrlTypes format_types
,
639 UnescapeRule::Type unescape_rules
,
640 url::Parsed
* new_parsed
,
642 std::vector
<size_t>* offsets_for_adjustment
) {
643 base::OffsetAdjuster::Adjustments adjustments
;
644 const base::string16
& format_url_return_value
=
645 FormatUrlWithAdjustments(url
, languages
, format_types
, unescape_rules
,
646 new_parsed
, prefix_end
, &adjustments
);
647 base::OffsetAdjuster::AdjustOffsets(adjustments
, offsets_for_adjustment
);
648 if (offsets_for_adjustment
) {
650 offsets_for_adjustment
->begin(),
651 offsets_for_adjustment
->end(),
652 base::LimitOffset
<std::string
>(format_url_return_value
.length()));
654 return format_url_return_value
;
657 base::string16
FormatUrlWithAdjustments(
659 const std::string
& languages
,
660 FormatUrlTypes format_types
,
661 UnescapeRule::Type unescape_rules
,
662 url::Parsed
* new_parsed
,
664 base::OffsetAdjuster::Adjustments
* adjustments
) {
665 DCHECK(adjustments
!= NULL
);
666 adjustments
->clear();
667 url::Parsed parsed_temp
;
669 new_parsed
= &parsed_temp
;
671 *new_parsed
= url::Parsed();
673 // Special handling for view-source:. Don't use content::kViewSourceScheme
674 // because this library shouldn't depend on chrome.
675 const char kViewSource
[] = "view-source";
676 // Reject "view-source:view-source:..." to avoid deep recursion.
677 const char kViewSourceTwice
[] = "view-source:view-source:";
678 if (url
.SchemeIs(kViewSource
) &&
679 !StartsWithASCII(url
.possibly_invalid_spec(), kViewSourceTwice
, false)) {
680 return FormatViewSourceUrl(url
, languages
, format_types
,
681 unescape_rules
, new_parsed
, prefix_end
,
685 // We handle both valid and invalid URLs (this will give us the spec
686 // regardless of validity).
687 const std::string
& spec
= url
.possibly_invalid_spec();
688 const url::Parsed
& parsed
= url
.parsed_for_possibly_invalid_spec();
690 // Scheme & separators. These are ASCII.
691 base::string16 url_string
;
693 url_string
.end(), spec
.begin(),
694 spec
.begin() + parsed
.CountCharactersBefore(url::Parsed::USERNAME
, true));
695 const char kHTTP
[] = "http://";
696 const char kFTP
[] = "ftp.";
697 // url_fixer::FixupURL() treats "ftp.foo.com" as ftp://ftp.foo.com. This
698 // means that if we trim "http://" off a URL whose host starts with "ftp." and
699 // the user inputs this into any field subject to fixup (which is basically
700 // all input fields), the meaning would be changed. (In fact, often the
701 // formatted URL is directly pre-filled into an input field.) For this reason
702 // we avoid stripping "http://" in this case.
703 bool omit_http
= (format_types
& kFormatUrlOmitHTTP
) &&
704 EqualsASCII(url_string
, kHTTP
) &&
705 !StartsWithASCII(url
.host(), kFTP
, true);
706 new_parsed
->scheme
= parsed
.scheme
;
708 // Username & password.
709 if ((format_types
& kFormatUrlOmitUsernamePassword
) != 0) {
710 // Remove the username and password fields. We don't want to display those
711 // to the user since they can be used for attacks,
712 // e.g. "http://google.com:search@evil.ru/"
713 new_parsed
->username
.reset();
714 new_parsed
->password
.reset();
715 // Update the adjustments based on removed username and/or password.
716 if (parsed
.username
.is_nonempty() || parsed
.password
.is_nonempty()) {
717 if (parsed
.username
.is_nonempty() && parsed
.password
.is_nonempty()) {
718 // The seeming off-by-two is to account for the ':' after the username
719 // and '@' after the password.
720 adjustments
->push_back(base::OffsetAdjuster::Adjustment(
721 static_cast<size_t>(parsed
.username
.begin
),
722 static_cast<size_t>(parsed
.username
.len
+ parsed
.password
.len
+ 2),
725 const url::Component
* nonempty_component
=
726 parsed
.username
.is_nonempty() ? &parsed
.username
: &parsed
.password
;
727 // The seeming off-by-one is to account for the '@' after the
728 // username/password.
729 adjustments
->push_back(base::OffsetAdjuster::Adjustment(
730 static_cast<size_t>(nonempty_component
->begin
),
731 static_cast<size_t>(nonempty_component
->len
+ 1),
736 AppendFormattedComponent(spec
, parsed
.username
,
737 NonHostComponentTransform(unescape_rules
),
738 &url_string
, &new_parsed
->username
, adjustments
);
739 if (parsed
.password
.is_valid())
740 url_string
.push_back(':');
741 AppendFormattedComponent(spec
, parsed
.password
,
742 NonHostComponentTransform(unescape_rules
),
743 &url_string
, &new_parsed
->password
, adjustments
);
744 if (parsed
.username
.is_valid() || parsed
.password
.is_valid())
745 url_string
.push_back('@');
748 *prefix_end
= static_cast<size_t>(url_string
.length());
751 AppendFormattedComponent(spec
, parsed
.host
, HostComponentTransform(languages
),
752 &url_string
, &new_parsed
->host
, adjustments
);
755 if (parsed
.port
.is_nonempty()) {
756 url_string
.push_back(':');
757 new_parsed
->port
.begin
= url_string
.length();
758 url_string
.insert(url_string
.end(),
759 spec
.begin() + parsed
.port
.begin
,
760 spec
.begin() + parsed
.port
.end());
761 new_parsed
->port
.len
= url_string
.length() - new_parsed
->port
.begin
;
763 new_parsed
->port
.reset();
766 // Path & query. Both get the same general unescape & convert treatment.
767 if (!(format_types
& kFormatUrlOmitTrailingSlashOnBareHostname
) ||
768 !CanStripTrailingSlash(url
)) {
769 AppendFormattedComponent(spec
, parsed
.path
,
770 NonHostComponentTransform(unescape_rules
),
771 &url_string
, &new_parsed
->path
, adjustments
);
773 if (parsed
.path
.len
> 0) {
774 adjustments
->push_back(base::OffsetAdjuster::Adjustment(
775 parsed
.path
.begin
, parsed
.path
.len
, 0));
778 if (parsed
.query
.is_valid())
779 url_string
.push_back('?');
780 AppendFormattedComponent(spec
, parsed
.query
,
781 NonHostComponentTransform(unescape_rules
),
782 &url_string
, &new_parsed
->query
, adjustments
);
784 // Ref. This is valid, unescaped UTF-8, so we can just convert.
785 if (parsed
.ref
.is_valid())
786 url_string
.push_back('#');
787 AppendFormattedComponent(spec
, parsed
.ref
,
788 NonHostComponentTransform(UnescapeRule::NONE
),
789 &url_string
, &new_parsed
->ref
, adjustments
);
791 // If we need to strip out http do it after the fact.
792 if (omit_http
&& StartsWith(url_string
, base::ASCIIToUTF16(kHTTP
), true)) {
793 const size_t kHTTPSize
= arraysize(kHTTP
) - 1;
794 url_string
= url_string
.substr(kHTTPSize
);
795 // Because offsets in the |adjustments| are already calculated with respect
796 // to the string with the http:// prefix in it, those offsets remain correct
797 // after stripping the prefix. The only thing necessary is to add an
798 // adjustment to reflect the stripped prefix.
799 adjustments
->insert(adjustments
->begin(),
800 base::OffsetAdjuster::Adjustment(0, kHTTPSize
, 0));
803 *prefix_end
-= kHTTPSize
;
805 // Adjust new_parsed.
806 DCHECK(new_parsed
->scheme
.is_valid());
807 int delta
= -(new_parsed
->scheme
.len
+ 3); // +3 for ://.
808 new_parsed
->scheme
.reset();
809 AdjustAllComponentsButScheme(delta
, new_parsed
);
815 base::string16
FormatUrl(const GURL
& url
,
816 const std::string
& languages
,
817 FormatUrlTypes format_types
,
818 UnescapeRule::Type unescape_rules
,
819 url::Parsed
* new_parsed
,
821 size_t* offset_for_adjustment
) {
823 if (offset_for_adjustment
)
824 offsets
.push_back(*offset_for_adjustment
);
825 base::string16 result
= FormatUrlWithOffsets(url
, languages
, format_types
,
826 unescape_rules
, new_parsed
, prefix_end
, &offsets
);
827 if (offset_for_adjustment
)
828 *offset_for_adjustment
= offsets
[0];