base/i18n/icu_string_conversions_unittest.cc

   1 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6 #include <stdarg.h>
   7
   8 #include <limits>
   9 #include <sstream>
  10
  11 #include "base/basictypes.h"
  12 #include "base/i18n/icu_string_conversions.h"
  13 #include "base/logging.h"
  14 #include "base/utf_string_conversions.h"
  15 #include "testing/gtest/include/gtest/gtest.h"
  16
  17 namespace base {
  18
  19 namespace {
  20
  21 // Given a null-terminated string of wchar_t with each wchar_t representing
  22 // a UTF-16 code unit, returns a string16 made up of wchar_t's in the input.
  23 // Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF)
  24 // should be represented as a surrogate pair (two UTF-16 units)
  25 // *even* where wchar_t is 32-bit (Linux and Mac).
  26 //
  27 // This is to help write tests for functions with string16 params until
  28 // the C++ 0x UTF-16 literal is well-supported by compilers.
  29 string16 BuildString16(const wchar_t* s) {
  30 #if defined(WCHAR_T_IS_UTF16)
  31   return string16(s);
  32 #elif defined(WCHAR_T_IS_UTF32)
  33   string16 u16;
  34   while (*s != 0) {
  35     DCHECK(static_cast<unsigned int>(*s) <= 0xFFFFu);
  36     u16.push_back(*s++);
  37   }
  38   return u16;
  39 #endif
  40 }
  41
  42 const wchar_t* const kConvertRoundtripCases[] = {
  43   L"Google Video",
  44   // "网页 图片 资讯更多 »"
  45   L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb",
  46   //  "Παγκόσμιος Ιστός"
  47   L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9"
  48   L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2",
  49   // "Поиск страниц на русском"
  50   L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442"
  51   L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430"
  52   L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c",
  53   // "전체서비스"
  54   L"\xc804\xccb4\xc11c\xbe44\xc2a4",
  55
  56   // Test characters that take more than 16 bits. This will depend on whether
  57   // wchar_t is 16 or 32 bits.
  58 #if defined(WCHAR_T_IS_UTF16)
  59   L"\xd800\xdf00",
  60   // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
  61   L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44",
  62 #elif defined(WCHAR_T_IS_UTF32)
  63   L"\x10300",
  64   // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
  65   L"\x11d40\x11d41\x11d42\x11d43\x11d44",
  66 #endif
  67 };
  68
  69 }  // namespace
  70
  71 TEST(ICUStringConversionsTest, ConvertCodepageUTF8) {
  72   // Make sure WideToCodepage works like WideToUTF8.
  73   for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) {
  74     std::string expected(WideToUTF8(kConvertRoundtripCases[i]));
  75     std::string utf8;
  76     EXPECT_TRUE(WideToCodepage(kConvertRoundtripCases[i], kCodepageUTF8,
  77                                OnStringConversionError::SKIP, &utf8));
  78     EXPECT_EQ(expected, utf8);
  79   }
  80 }
  81
  82 // kConverterCodepageCases is not comprehensive. There are a number of cases
  83 // to add if we really want to have a comprehensive coverage of various
  84 // codepages and their 'idiosyncrasies'. Currently, the only implementation
  85 // for CodepageTo* and *ToCodepage uses ICU, which has a very extensive
  86 // set of tests for the charset conversion. So, we can get away with a
  87 // relatively small number of cases listed below.
  88 //
  89 // Note about |u16_wide| in the following struct.
  90 // On Windows, the field is always identical to |wide|. On Mac and Linux,
  91 // it's identical as long as there's no character outside the
  92 // BMP (<= U+FFFF). When there is, it is different from |wide| and
  93 // is not a real wide string (UTF-32 string) in that each wchar_t in
  94 // the string is a UTF-16 code unit zero-extended to be 32-bit
  95 // even when the code unit belongs to a surrogate pair.
  96 // For instance, a Unicode string (U+0041 U+010000) is represented as
  97 // L"\x0041\xD800\xDC00" instead of L"\x0041\x10000".
  98 // To avoid the clutter, |u16_wide| will be set to NULL
  99 // if it's identical to |wide| on *all* platforms.
 100
 101 static const struct {
 102   const char* codepage_name;
 103   const char* encoded;
 104   OnStringConversionError::Type on_error;
 105   bool success;
 106   const wchar_t* wide;
 107   const wchar_t* u16_wide;
 108 } kConvertCodepageCases[] = {
 109   // Test a case where the input cannot be decoded, using SKIP, FAIL
 110   // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't.
 111   {"big5",
 112    "\xA7\x41\xA6",
 113    OnStringConversionError::FAIL,
 114    false,
 115    L"",
 116    NULL},
 117   {"big5",
 118    "\xA7\x41\xA6",
 119    OnStringConversionError::SKIP,
 120    true,
 121    L"\x4F60",
 122    NULL},
 123   {"big5",
 124    "\xA7\x41\xA6",
 125    OnStringConversionError::SUBSTITUTE,
 126    true,
 127    L"\x4F60\xFFFD",
 128    NULL},
 129   // Arabic (ISO-8859)
 130   {"iso-8859-6",
 131    "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF" " "
 132    "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2",
 133    OnStringConversionError::FAIL,
 134    true,
 135    L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F" L" "
 136    L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652",
 137    NULL},
 138   // Chinese Simplified (GB2312)
 139   {"gb2312",
 140    "\xC4\xE3\xBA\xC3",
 141    OnStringConversionError::FAIL,
 142    true,
 143    L"\x4F60\x597D",
 144    NULL},
 145   // Chinese (GB18030) : 4 byte sequences mapped to BMP characters
 146   {"gb18030",
 147    "\x81\x30\x84\x36\xA1\xA7",
 148    OnStringConversionError::FAIL,
 149    true,
 150    L"\x00A5\x00A8",
 151    NULL},
 152   // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000)
 153   {"gb18030",
 154    "\x95\x32\x82\x36\xD2\xBB",
 155    OnStringConversionError::FAIL,
 156    true,
 157 #if defined(WCHAR_T_IS_UTF16)
 158    L"\xD840\xDC00\x4E00",
 159 #elif defined(WCHAR_T_IS_UTF32)
 160    L"\x20000\x4E00",
 161 #endif
 162    L"\xD840\xDC00\x4E00"},
 163   {"big5",
 164    "\xA7\x41\xA6\x6E",
 165    OnStringConversionError::FAIL,
 166    true,
 167    L"\x4F60\x597D",
 168    NULL},
 169   // Greek (ISO-8859)
 170   {"iso-8859-7",
 171    "\xE3\xE5\xE9\xDC" " " "\xF3\xEF\xF5",
 172    OnStringConversionError::FAIL,
 173    true,
 174    L"\x03B3\x03B5\x03B9\x03AC" L" " L"\x03C3\x03BF\x03C5",
 175    NULL},
 176   // Hebrew (Windows)
 177   {"windows-1255",
 178    "\xF9\xD1\xC8\xEC\xE5\xC9\xED",
 179    OnStringConversionError::FAIL,
 180    true,
 181    L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD",
 182    NULL},
 183   // Hindi Devanagari (ISCII)
 184   {"iscii-dev",
 185    "\xEF\x42" "\xC6\xCC\xD7\xE8\xB3\xDA\xCF",
 186    OnStringConversionError::FAIL,
 187    true,
 188    L"\x0928\x092E\x0938\x094D\x0915\x093E\x0930",
 189    NULL},
 190   // Korean (EUC)
 191   {"euc-kr",
 192    "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4",
 193    OnStringConversionError::FAIL,
 194    true,
 195    L"\xC548\xB155\xD558\xC138\xC694",
 196    NULL},
 197   // Japanese (EUC)
 198   {"euc-jp",
 199    "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8F\xB0\xA1\x8E\xA6",
 200    OnStringConversionError::FAIL,
 201    true,
 202    L"\x3053\x3093\x306B\x3061\x306F\x4E00\x4E02\xFF66",
 203    NULL},
 204   // Japanese (ISO-2022)
 205   {"iso-2022-jp",
 206    "\x1B$B" "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C" "\x1B(B"
 207    "ab" "\x1B(J" "\x5C\x7E#$" "\x1B(B",
 208    OnStringConversionError::FAIL,
 209    true,
 210    L"\x3053\x3093\x306B\x3061\x306F\x4E00" L"ab\x00A5\x203E#$",
 211    NULL},
 212   // Japanese (Shift-JIS)
 213   {"sjis",
 214    "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6",
 215    OnStringConversionError::FAIL,
 216    true,
 217    L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66",
 218    NULL},
 219   // Russian (KOI8)
 220   {"koi8-r",
 221    "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5",
 222    OnStringConversionError::FAIL,
 223    true,
 224    L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
 225    L"\x0443\x0439\x0442\x0435",
 226    NULL},
 227   // Thai (windows-874)
 228   {"windows-874",
 229    "\xCA\xC7\xD1\xCA\xB4\xD5" "\xA4\xC3\xD1\xBA",
 230    OnStringConversionError::FAIL,
 231    true,
 232    L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35"
 233    L"\x0E04\x0E23\x0e31\x0E1A",
 234    NULL},
 235 };
 236
 237 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndWide) {
 238   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) {
 239     std::wstring wide;
 240     bool success = CodepageToWide(kConvertCodepageCases[i].encoded,
 241                                   kConvertCodepageCases[i].codepage_name,
 242                                   kConvertCodepageCases[i].on_error,
 243                                   &wide);
 244     EXPECT_EQ(kConvertCodepageCases[i].success, success);
 245     EXPECT_EQ(kConvertCodepageCases[i].wide, wide);
 246
 247     // When decoding was successful and nothing was skipped, we also check the
 248     // reverse conversion. Not all conversions are round-trippable, but
 249     // kConverterCodepageCases does not have any one-way conversion at the
 250     // moment.
 251     if (success &&
 252         kConvertCodepageCases[i].on_error ==
 253             OnStringConversionError::FAIL) {
 254       std::string encoded;
 255       success = WideToCodepage(wide, kConvertCodepageCases[i].codepage_name,
 256                                kConvertCodepageCases[i].on_error, &encoded);
 257       EXPECT_EQ(kConvertCodepageCases[i].success, success);
 258       EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
 259     }
 260   }
 261
 262   // The above cases handled codepage->wide errors, but not wide->codepage.
 263   // Test that here.
 264   std::string encoded("Temp data");  // Make sure the string gets cleared.
 265
 266   // First test going to an encoding that can not represent that character.
 267   EXPECT_FALSE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
 268                               OnStringConversionError::FAIL, &encoded));
 269   EXPECT_TRUE(encoded.empty());
 270   EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
 271                              OnStringConversionError::SKIP, &encoded));
 272   EXPECT_STREQ("Chinese", encoded.c_str());
 273   // From Unicode, SUBSTITUTE is the same as SKIP for now.
 274   EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
 275                              OnStringConversionError::SUBSTITUTE,
 276                              &encoded));
 277   EXPECT_STREQ("Chinese", encoded.c_str());
 278
 279 #if defined(WCHAR_T_IS_UTF16)
 280   // When we're in UTF-16 mode, test an invalid UTF-16 character in the input.
 281   EXPECT_FALSE(WideToCodepage(L"a\xd800z", "iso-8859-1",
 282                               OnStringConversionError::FAIL, &encoded));
 283   EXPECT_TRUE(encoded.empty());
 284   EXPECT_TRUE(WideToCodepage(L"a\xd800z", "iso-8859-1",
 285                              OnStringConversionError::SKIP, &encoded));
 286   EXPECT_STREQ("az", encoded.c_str());
 287 #endif  // WCHAR_T_IS_UTF16
 288
 289   // Invalid characters should fail.
 290   EXPECT_TRUE(WideToCodepage(L"a\xffffz", "iso-8859-1",
 291                              OnStringConversionError::SKIP, &encoded));
 292   EXPECT_STREQ("az", encoded.c_str());
 293
 294   // Invalid codepages should fail.
 295   EXPECT_FALSE(WideToCodepage(L"Hello, world", "awesome-8571-2",
 296                               OnStringConversionError::SKIP, &encoded));
 297 }
 298
 299 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) {
 300   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) {
 301     string16 utf16;
 302     bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded,
 303                                    kConvertCodepageCases[i].codepage_name,
 304                                    kConvertCodepageCases[i].on_error,
 305                                    &utf16);
 306     string16 utf16_expected;
 307     if (kConvertCodepageCases[i].u16_wide == NULL)
 308       utf16_expected = BuildString16(kConvertCodepageCases[i].wide);
 309     else
 310       utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide);
 311     EXPECT_EQ(kConvertCodepageCases[i].success, success);
 312     EXPECT_EQ(utf16_expected, utf16);
 313
 314     // When decoding was successful and nothing was skipped, we also check the
 315     // reverse conversion. See also the corresponding comment in
 316     // ConvertBetweenCodepageAndWide.
 317     if (success &&
 318         kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) {
 319       std::string encoded;
 320       success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name,
 321                                 kConvertCodepageCases[i].on_error, &encoded);
 322       EXPECT_EQ(kConvertCodepageCases[i].success, success);
 323       EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
 324     }
 325   }
 326 }
 327
 328 }  // namespace base