test/UnitTests/UnicodeUtilsTest.cpp

   1 // TortoiseGit - a Windows shell extension for easy version control
   2
   3 // Copyright (C) 2020, 2023 - TortoiseGit
   4 // Copyright (C) 2011-2012 - TortoiseSVN
   5
   6 // This program is free software; you can redistribute it and/or
   7 // modify it under the terms of the GNU General Public License
   8 // as published by the Free Software Foundation; either version 2
   9 // of the License, or (at your option) any later version.
  10
  11 // This program is distributed in the hope that it will be useful,
  12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 // GNU General Public License for more details.
  15
  16 // You should have received a copy of the GNU General Public License
  17 // along with this program; if not, write to the Free Software Foundation,
  18 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19 //
  20 #include "stdafx.h"
  21
  22 #pragma warning(push)
  23 #pragma warning(disable:4566)
  24 TEST(UnicodeUtils, CString)
  25 {
  26         CStringA result = CUnicodeUtils::GetUTF8(L"");
  27         EXPECT_EQ(0, result.GetLength());
  28         EXPECT_STREQ(result, "");
  29         CStringW resultW = CUnicodeUtils::GetUnicode("");
  30         EXPECT_EQ(0, resultW.GetLength());
  31         EXPECT_STREQ(resultW, L"");
  32
  33         result = CUnicodeUtils::GetUTF8(L"Iñtërnâtiônàlizætiøn");
  34         EXPECT_EQ(27, result.GetLength());
  35         EXPECT_STREQ(result, "\x49\xC3\xB1\x74\xC3\xAB\x72\x6E\xC3\xA2\x74\x69\xC3\xB4\x6E\xC3\xA0\x6C\x69\x7A\xC3\xA6\x74\x69\xC3\xB8\x6E");
  36         resultW = CUnicodeUtils::GetUnicode(result);
  37         EXPECT_STREQ(resultW, L"Iñtërnâtiônàlizætiøn");
  38         EXPECT_EQ(20, resultW.GetLength());
  39
  40         result = CUnicodeUtils::GetUTF8(L"<value>退订</value>");
  41         resultW = CUnicodeUtils::GetUnicode(result);
  42         EXPECT_STREQ(resultW, L"<value>退订</value>");
  43
  44         result = CUnicodeUtils::GetUTF8(L"äöü");
  45         EXPECT_EQ(6, result.GetLength());
  46         resultW = CUnicodeUtils::GetUnicode(result);
  47         EXPECT_STREQ(resultW, L"äöü");
  48         EXPECT_EQ(3, resultW.GetLength());
  49
  50         resultW = CUnicodeUtils::GetUnicode("\xE4\xF6\xFC\xDF", 1252);
  51         EXPECT_EQ(4, resultW.GetLength());
  52         EXPECT_STREQ(resultW, L"äöüß");
  53         result = CUnicodeUtils::GetUTF8(resultW);
  54         EXPECT_STREQ(result, "\xC3\xA4\xC3\xB6\xC3\xBC\xC3\x9F");
  55
  56         result = CUnicodeUtils::GetUTF8(L"Продолжить выполнение скрипта?");
  57         resultW = CUnicodeUtils::GetUnicode(result);
  58         EXPECT_STREQ(resultW, L"Продолжить выполнение скрипта?");
  59
  60         result = CUnicodeUtils::GetUTF8(L"dvostruki klik za automtsko uključivanje alfa");
  61         resultW = CUnicodeUtils::GetUnicode(result);
  62         EXPECT_STREQ(resultW, L"dvostruki klik za automtsko uključivanje alfa");
  63
  64         result = CUnicodeUtils::GetUTF8(L"包含有错误的结构。");
  65         resultW = CUnicodeUtils::GetUnicode(result);
  66         EXPECT_STREQ(resultW, L"包含有错误的结构。");
  67
  68         result = CUnicodeUtils::GetUTF8(L"个文件，共有 %2!d! 个文件");
  69         resultW = CUnicodeUtils::GetUnicode(result);
  70         EXPECT_STREQ(resultW, L"个文件，共有 %2!d! 个文件");
  71
  72         result = CUnicodeUtils::GetUTF8(L"は予期せぬオブジェクトを含んでいます。");
  73         resultW = CUnicodeUtils::GetUnicode(result);
  74         EXPECT_STREQ(resultW, L"は予期せぬオブジェクトを含んでいます。");
  75
  76         result = CUnicodeUtils::GetUTF8(L"Verify that the correct path and file name are given.");
  77         EXPECT_STREQ(result, "Verify that the correct path and file name are given.");
  78         resultW = CUnicodeUtils::GetUnicode(result);
  79         EXPECT_STREQ(resultW, L"Verify that the correct path and file name are given.");
  80
  81         result = CUnicodeUtils::GetUTF8(L"\U0002070e"); // 𠜎 is 4-byte utf8
  82         EXPECT_EQ(4, result.GetLength());
  83         EXPECT_STREQ(result, "\xf0\xa0\x9c\x8e");
  84         resultW = CUnicodeUtils::GetUnicode("\xf0\xa0\x9c\x8e");
  85         EXPECT_STREQ(resultW, L"\U0002070e");
  86         EXPECT_EQ(2, resultW.GetLength());
  87
  88         resultW = CUnicodeUtils::GetUnicode("\xfe");
  89         EXPECT_STREQ(resultW, L"\uFFFD");
  90         EXPECT_EQ(1, resultW.GetLength());
  91
  92         resultW = CUnicodeUtils::GetUnicode("\xc3\x28"); // Invalid 2 Octet Sequence
  93         EXPECT_STREQ(resultW, L"\uFFFD(");
  94         EXPECT_EQ(2, resultW.GetLength());
  95 }
  96
  97 TEST(UnicodeUtils, Std)
  98 {
  99         std::string result = CUnicodeUtils::StdGetUTF8(L"");
 100         EXPECT_EQ(0u, result.size());
 101         EXPECT_STREQ(result.c_str(), "");
 102         std::wstring resultW = CUnicodeUtils::StdGetUnicode("");
 103         EXPECT_EQ(0u, resultW.size());
 104         EXPECT_STREQ(resultW.c_str(), L"");
 105
 106         result = CUnicodeUtils::StdGetUTF8(L"Iñtërnâtiônàlizætiøn");
 107         EXPECT_EQ(27u, result.size());
 108         EXPECT_STREQ(result.c_str(), "\x49\xC3\xB1\x74\xC3\xAB\x72\x6E\xC3\xA2\x74\x69\xC3\xB4\x6E\xC3\xA0\x6C\x69\x7A\xC3\xA6\x74\x69\xC3\xB8\x6E");
 109         resultW = CUnicodeUtils::StdGetUnicode(result);
 110         EXPECT_STREQ(resultW.c_str(), L"Iñtërnâtiônàlizætiøn");
 111         EXPECT_EQ(20u, resultW.size());
 112
 113         result = CUnicodeUtils::StdGetUTF8(L"<value>退订</value>");
 114         resultW = CUnicodeUtils::StdGetUnicode(result);
 115         EXPECT_STREQ(resultW.c_str(), L"<value>退订</value>");
 116
 117         result = CUnicodeUtils::StdGetUTF8(L"äöü");
 118         EXPECT_EQ(6u, result.size());
 119         resultW = CUnicodeUtils::StdGetUnicode(result);
 120         EXPECT_STREQ(resultW.c_str(), L"äöü");
 121         EXPECT_EQ(3u, resultW.size());
 122
 123         result = CUnicodeUtils::StdGetUTF8(L"äöüß");
 124         EXPECT_STREQ(result.c_str(), "\xC3\xA4\xC3\xB6\xC3\xBC\xC3\x9F");
 125
 126         result = CUnicodeUtils::StdGetUTF8(L"Продолжить выполнение скрипта?");
 127         resultW = CUnicodeUtils::StdGetUnicode(result);
 128         EXPECT_STREQ(resultW.c_str(), L"Продолжить выполнение скрипта?");
 129
 130         result = CUnicodeUtils::StdGetUTF8(L"dvostruki klik za automtsko uključivanje alfa");
 131         resultW = CUnicodeUtils::StdGetUnicode(result);
 132         EXPECT_STREQ(resultW.c_str(), L"dvostruki klik za automtsko uključivanje alfa");
 133
 134         result = CUnicodeUtils::StdGetUTF8(L"包含有错误的结构。");
 135         resultW = CUnicodeUtils::StdGetUnicode(result);
 136         EXPECT_STREQ(resultW.c_str(), L"包含有错误的结构。");
 137
 138         result = CUnicodeUtils::StdGetUTF8(L"个文件，共有 %2!d! 个文件");
 139         resultW = CUnicodeUtils::StdGetUnicode(result);
 140         EXPECT_STREQ(resultW.c_str(), L"个文件，共有 %2!d! 个文件");
 141
 142         result = CUnicodeUtils::StdGetUTF8(L"は予期せぬオブジェクトを含んでいます。");
 143         resultW = CUnicodeUtils::StdGetUnicode(result);
 144         EXPECT_STREQ(resultW.c_str(), L"は予期せぬオブジェクトを含んでいます。");
 145
 146         result = CUnicodeUtils::StdGetUTF8(L"Verify that the correct path and file name are given.");
 147         EXPECT_STREQ(result.c_str(), "Verify that the correct path and file name are given.");
 148         resultW = CUnicodeUtils::StdGetUnicode(result);
 149         EXPECT_STREQ(resultW.c_str(), L"Verify that the correct path and file name are given.");
 150
 151         result = CUnicodeUtils::StdGetUTF8(L"\U0002070e"); // 𠜎 is 4-byte utf8
 152         EXPECT_EQ(4u, result.size());
 153         EXPECT_STREQ(result.c_str(), "\xf0\xa0\x9c\x8e");
 154         resultW = CUnicodeUtils::StdGetUnicode("\xf0\xa0\x9c\x8e");
 155         EXPECT_STREQ(resultW.c_str(), L"\U0002070e");
 156         EXPECT_EQ(2u, resultW.size());
 157
 158         resultW = CUnicodeUtils::StdGetUnicode("\xfe");
 159         EXPECT_STREQ(resultW.c_str(), L"\uFFFD");
 160         EXPECT_EQ(1u, resultW.size());
 161
 162         resultW = CUnicodeUtils::StdGetUnicode("\xc3\x28"); // Invalid 2 Octet Sequence
 163         EXPECT_STREQ(resultW.c_str(), L"\uFFFD(");
 164         EXPECT_EQ(2u, resultW.size());
 165 }
 166 #pragma warning(pop)