*.js files: keep the style consistent
[TortoiseGit.git] / src / Utils / UnicodeUtils.cpp
blobe4fa3364e2aab8cb17c414c5c2209e7f47b09498
1 // TortoiseGit - a Windows shell extension for easy version control
3 // Copyright (C) 2003-2006, 2008 - TortoiseSVN
5 // This program is free software; you can redistribute it and/or
6 // modify it under the terms of the GNU General Public License
7 // as published by the Free Software Foundation; either version 2
8 // of the License, or (at your option) any later version.
10 // This program is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
15 // You should have received a copy of the GNU General Public License
16 // along with this program; if not, write to the Free Software Foundation,
17 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 #include "stdafx.h"
20 #include "unicodeutils.h"
22 CUnicodeUtils::CUnicodeUtils(void)
26 CUnicodeUtils::~CUnicodeUtils(void)
30 #if defined(_MFC_VER) || defined(CSTRING_AVAILABLE)
32 struct CodeMap
34 int m_Code;
35 TCHAR * m_CodeName;
37 int CUnicodeUtils::GetCPCode(CString &codename)
39 static CodeMap map[]=
41 {037, _T("IBM037")},// IBM EBCDIC US-Canada
42 {437, _T("IBM437")},// OEM United States
43 {500, _T("IBM500")},// IBM EBCDIC International
44 {708, _T("ASMO-708")},// Arabic (ASMO 708)
45 {709, _T("Arabic")},// (ASMO-449+, BCON V4)
46 {710, _T("Arabic")},// - Transparent Arabic
47 {720, _T("DOS-720")},// Arabic (Transparent ASMO); Arabic (DOS)
48 {737, _T("ibm737")},// OEM Greek (formerly 437G); Greek (DOS)
49 {775, _T("ibm775")},// OEM Baltic; Baltic (DOS)
50 {850, _T("ibm850")},// OEM Multilingual Latin 1; Western European (DOS)
51 {852, _T("ibm852")},// OEM Latin 2; Central European (DOS)
52 {855, _T("IBM855")},// OEM Cyrillic (primarily Russian)
53 {857, _T("ibm857")},// OEM Turkish; Turkish (DOS)
54 {858, _T("IBM00858")},// OEM Multilingual Latin 1 + Euro symbol
55 {860, _T("IBM860")},// OEM Portuguese; Portuguese (DOS)
56 {861, _T("ibm861")},// OEM Icelandic; Icelandic (DOS)
57 {862, _T("DOS-862")},// OEM Hebrew; Hebrew (DOS)
58 {863, _T("IBM863")},// OEM French Canadian; French Canadian (DOS)
59 {864, _T("IBM864")},// OEM Arabic; Arabic (864)
60 {865, _T("IBM865")},// OEM Nordic; Nordic (DOS)
61 {866, _T("cp866")},// OEM Russian; Cyrillic (DOS)
62 {869, _T("ibm869")},// OEM Modern Greek; Greek, Modern (DOS)
63 {870, _T("IBM870")},// IBM EBCDIC Multilingual/ROECE (Latin 2); IBM EBCDIC Multilingual Latin 2
64 {874, _T("windows-874")},// ANSI/OEM Thai (same as 28605, ISO 8859-15); Thai (Windows)
65 {875, _T("cp875")},// IBM EBCDIC Greek Modern
66 {932, _T("shift_jis")},// ANSI/OEM Japanese; Japanese (Shift-JIS)
67 {936, _T("gb2312")},// ANSI/OEM Simplified Chinese (PRC, Singapore); Chinese Simplified (GB2312)
68 {949, _T("ks_c_5601-1987")},// ANSI/OEM Korean (Unified Hangul Code)
69 {949, _T("cp949")},// ANSI/OEM Korean (Unified Hangul Code)
70 {950, _T("big5")},// ANSI/OEM Traditional Chinese (Taiwan; Hong Kong SAR, PRC); Chinese Traditional (Big5)
71 {1026,_T("IBM1026")},// IBM EBCDIC Turkish (Latin 5)
72 {1047,_T("IBM01047")},// IBM EBCDIC Latin 1/Open System
73 {1140,_T("IBM01140")},// IBM EBCDIC US-Canada (037 + Euro symbol); IBM EBCDIC (US-Canada-Euro)
74 {1141, _T("IBM01141")},// IBM EBCDIC Germany (20273 + Euro symbol); IBM EBCDIC (Germany-Euro)
75 {1142, _T("IBM01142")},// IBM EBCDIC Denmark-Norway (20277 + Euro symbol); IBM EBCDIC (Denmark-Norway-Euro)
76 {1143, _T("IBM01143")},// IBM EBCDIC Finland-Sweden (20278 + Euro symbol); IBM EBCDIC (Finland-Sweden-Euro)
77 {1144, _T("IBM01144")},// IBM EBCDIC Italy (20280 + Euro symbol); IBM EBCDIC (Italy-Euro)
78 {1145, _T("IBM01145")},// IBM EBCDIC Latin America-Spain (20284 + Euro symbol); IBM EBCDIC (Spain-Euro)
79 {1146, _T("IBM01146")},// IBM EBCDIC United Kingdom (20285 + Euro symbol); IBM EBCDIC (UK-Euro)
80 {1147, _T("IBM01147")},// IBM EBCDIC France (20297 + Euro symbol); IBM EBCDIC (France-Euro)
81 {1148, _T("IBM01148")},// IBM EBCDIC International (500 + Euro symbol); IBM EBCDIC (International-Euro)
82 {1149, _T("IBM01149")},// IBM EBCDIC Icelandic (20871 + Euro symbol); IBM EBCDIC (Icelandic-Euro)
83 {1200, _T("utf-16")},// Unicode UTF-16, little endian byte order (BMP of ISO 10646); available only to managed applications
84 {1201, _T("unicodeFFFE")},// Unicode UTF-16, big endian byte order; available only to managed applications
85 {1250, _T("windows-1250")},// ANSI Central European; Central European (Windows)
86 {1251, _T("windows-1251")},// ANSI Cyrillic; Cyrillic (Windows)
87 {1251, _T("cp1251")},
88 {1251, _T("cp-1251")},
89 {1251, _T("cp_1251")},
90 {1252, _T("windows-1252")},// ANSI Latin 1; Western European (Windows)
91 {1253, _T("windows-1253")},// ANSI Greek; Greek (Windows)
92 {1254, _T("windows-1254")},// ANSI Turkish; Turkish (Windows)
93 {1255, _T("windows-1255")},// ANSI Hebrew; Hebrew (Windows)
94 {1256, _T("windows-1256")},// ANSI Arabic; Arabic (Windows)
95 {1257, _T("windows-1257")},// ANSI Baltic; Baltic (Windows)
96 {1258, _T("windows-1258")},// ANSI/OEM Vietnamese; Vietnamese (Windows)
97 {1361, _T("Johab")},// Korean (Johab)
98 {10000,_T("macintosh")},// MAC Roman; Western European (Mac)
99 {10001, _T("x-mac-japanese")},// Japanese (Mac)
100 {10002, _T("x-mac-chinesetrad")},// MAC Traditional Chinese (Big5); Chinese Traditional (Mac)
101 {10003, _T("x-mac-korean")},// Korean (Mac)
102 {10004, _T("x-mac-arabic")},// Arabic (Mac)
103 {10005, _T("x-mac-hebrew")},// Hebrew (Mac)
104 {10006, _T("x-mac-greek")},// Greek (Mac)
105 {10007, _T("x-mac-cyrillic")},// Cyrillic (Mac)
106 {10008, _T("x-mac-chinesesimp")},// MAC Simplified Chinese (GB 2312); Chinese Simplified (Mac)
107 {10010, _T("x-mac-romanian")},// Romanian (Mac)
108 {10017, _T("x-mac-ukrainian")},// Ukrainian (Mac)
109 {10021, _T("x-mac-thai")},// Thai (Mac)
110 {10029, _T("x-mac-ce")},// MAC Latin 2; Central European (Mac)
111 {10079, _T("x-mac-icelandic")},// Icelandic (Mac)
112 {10081, _T("x-mac-turkish")},// Turkish (Mac)
113 {10082, _T("x-mac-croatian")},// Croatian (Mac)
114 {12000, _T("utf-32")},// Unicode UTF-32, little endian byte order; available only to managed applications
115 {12001, _T("utf-32BE")},// Unicode UTF-32, big endian byte order; available only to managed applications
116 {20000, _T("x-Chinese_CNS")},// CNS Taiwan; Chinese Traditional (CNS)
117 {20001, _T("x-cp20001")},// TCA Taiwan
118 {20002, _T("x_Chinese-Eten")},// Eten Taiwan; Chinese Traditional (Eten)
119 {20003, _T("x-cp20003")},// IBM5550 Taiwan
120 {20004, _T("x-cp20004")},// TeleText Taiwan
121 {20005, _T("x-cp20005")},// Wang Taiwan
122 {20105, _T("x-IA5")},// IA5 (IRV International Alphabet No. 5, 7-bit); Western European (IA5)
123 {20106, _T("x-IA5-German")},// IA5 German (7-bit)
124 {20107, _T("x-IA5-Swedish")},// IA5 Swedish (7-bit)
125 {20108, _T("x-IA5-Norwegian")},// IA5 Norwegian (7-bit)
126 {20127, _T("us-ascii")},// US-ASCII (7-bit)
127 {20261, _T("x-cp20261")},// T.61
128 {20269, _T("x-cp20269")},// ISO 6937 Non-Spacing Accent
129 {20273, _T("IBM273")},// IBM EBCDIC Germany
130 {20277, _T("IBM277")},//IBM EBCDIC Denmark-Norway
131 {20278, _T("IBM278")},// IBM EBCDIC Finland-Sweden
132 {20280, _T("IBM280")},// IBM EBCDIC Italy
133 {20284, _T("IBM284")},// IBM EBCDIC Latin America-Spain
134 {20285, _T("IBM285")},// IBM EBCDIC United Kingdom
135 {20290, _T("IBM290")},// IBM EBCDIC Japanese Katakana Extended
136 {20297, _T("IBM297")},// IBM EBCDIC France
137 {20420, _T("IBM420")},// IBM EBCDIC Arabic
138 {20423, _T("IBM423")},// IBM EBCDIC Greek
139 {20424, _T("IBM424")},// IBM EBCDIC Hebrew
140 {20833, _T("x-EBCDIC-KoreanExtended")},// IBM EBCDIC Korean Extended
141 {20838, _T("IBM-Thai")},// IBM EBCDIC Thai
142 {20866, _T("koi8-r")},// Russian (KOI8-R); Cyrillic (KOI8-R)
143 {20871, _T("IBM871")},// IBM EBCDIC Icelandic
144 {20880, _T("IBM880")},// IBM EBCDIC Cyrillic Russian
145 {20905, _T("IBM905")},// IBM EBCDIC Turkish
146 {20924, _T("IBM00924")},// IBM EBCDIC Latin 1/Open System (1047 + Euro symbol)
147 {20932, _T("EUC-JP")},// Japanese (JIS 0208-1990 and 0121-1990)
148 {20936, _T("x-cp20936")},// Simplified Chinese (GB2312); Chinese Simplified (GB2312-80)
149 {20949, _T("x-cp20949")},// Korean Wansung
150 {21025, _T("cp1025")},// IBM EBCDIC Cyrillic Serbian-Bulgarian
151 {21027, _T("21027")},// (deprecated)
152 {21866, _T("koi8-u")},// Ukrainian (KOI8-U); Cyrillic (KOI8-U)
153 {28591, _T("iso-8859-1")},// ISO 8859-1 Latin 1; Western European (ISO)
154 {28592, _T("iso-8859-2")},// ISO 8859-2 Central European; Central European (ISO)
155 {28593, _T("iso-8859-3")},// ISO 8859-3 Latin 3
156 {28594, _T("iso-8859-4")},// ISO 8859-4 Baltic
157 {28595, _T("iso-8859-5")},// ISO 8859-5 Cyrillic
158 {28596, _T("iso-8859-6")},// ISO 8859-6 Arabic
159 {28597, _T("iso-8859-7")},// ISO 8859-7 Greek
160 {28598, _T("iso-8859-8")},// ISO 8859-8 Hebrew; Hebrew (ISO-Visual)
161 {28599, _T("iso-8859-9")},// ISO 8859-9 Turkish
162 {28603, _T("iso-8859-13")},// ISO 8859-13 Estonian
163 {28605, _T("iso-8859-15")},// ISO 8859-15 Latin 9
164 {29001, _T("x-Europa")},// Europa 3
165 {38598, _T("iso-8859-8-i")},// ISO 8859-8 Hebrew; Hebrew (ISO-Logical)
166 {50220, _T("iso-2022-jp")},// ISO 2022 Japanese with no halfwidth Katakana; Japanese (JIS)
167 {50221, _T("csISO2022JP")},// ISO 2022 Japanese with halfwidth Katakana; Japanese (JIS-Allow 1 byte Kana)
168 {50222, _T("iso-2022-jp")},// ISO 2022 Japanese JIS X 0201-1989; Japanese (JIS-Allow 1 byte Kana - SO/SI)
169 {50225, _T("iso-2022-kr")},// ISO 2022 Korean
170 {50227, _T("x-cp50227")},// ISO 2022 Simplified Chinese; Chinese Simplified (ISO 2022)
171 {50229, _T("ISO")},// 2022 Traditional Chinese
172 {50930, _T("EBCDIC")},// Japanese (Katakana) Extended
173 {50931, _T("EBCDIC")},// US-Canada and Japanese
174 {50933, _T("EBCDIC")},// Korean Extended and Korean
175 {50935, _T("EBCDIC")},// Simplified Chinese Extended and Simplified Chinese
176 {50936, _T("EBCDIC")},// Simplified Chinese
177 {50937, _T("EBCDIC")},// US-Canada and Traditional Chinese
178 {50939, _T("EBCDIC")},// Japanese (Latin) Extended and Japanese
179 {51932, _T("euc-jp")},// EUC Japanese
180 {51936, _T("EUC-CN")},// EUC Simplified Chinese; Chinese Simplified (EUC)
181 {51949, _T("euc-kr")},// EUC Korean
182 {51950, _T("EUC")},// Traditional Chinese
183 {52936, _T("hz-gb-2312")},// HZ-GB2312 Simplified Chinese; Chinese Simplified (HZ)
184 {54936, _T("GB18030")},// Windows XP and later: GB18030 Simplified Chinese (4 byte); Chinese Simplified (GB18030)
185 {57002, _T("x-iscii-de")},// ISCII Devanagari
186 {57003, _T("x-iscii-be")},// ISCII Bengali
187 {57004, _T("x-iscii-ta")},// ISCII Tamil
188 {57005, _T("x-iscii-te")},// ISCII Telugu
189 {57006, _T("x-iscii-as")},// ISCII Assamese
190 {57007, _T("x-iscii-or")},// ISCII Oriya
191 {57008, _T("x-iscii-ka")},// ISCII Kannada
192 {57009, _T("x-iscii-ma")},// ISCII Malayalam
193 {57010, _T("x-iscii-gu")},// ISCII Gujarati
194 {57011, _T("x-iscii-pa")},// ISCII Punjabi
195 {65000, _T("utf-7")},// Unicode (UTF-7)
196 {65001, _T("utf-8")},// Unicode (UTF-8)
197 {0,NULL}
200 static CodeMap *p=map;
201 codename=codename.MakeLower();
202 while(p->m_CodeName != NULL)
204 CString str = p->m_CodeName;
205 str=str.MakeLower();
207 if( str == codename)
208 return p->m_Code;
209 ++p;
212 return CP_UTF8;
215 CStringA CUnicodeUtils::GetUTF8(const CStringW& string)
217 return GetMulti(string,CP_UTF8);
220 CStringA CUnicodeUtils::GetMulti(const CStringW& string,int acp)
222 char * buf;
223 CStringA retVal;
224 int len = string.GetLength();
225 if (len==0)
226 return retVal;
227 buf = retVal.GetBuffer(len*4 + 1);
228 // SecureZeroMemory(buf, (string.GetLength()*4 + 1)*sizeof(char));
229 int lengthIncTerminator = WideCharToMultiByte(acp, 0, string, -1, buf, len*4, NULL, NULL);
230 retVal.ReleaseBuffer(lengthIncTerminator-1);
231 return retVal;
235 CStringA CUnicodeUtils::GetUTF8(const CStringA& string)
237 WCHAR * buf;
238 int len = string.GetLength();
239 if (len==0)
240 return CStringA();
241 buf = new WCHAR[len*4 + 1];
242 SecureZeroMemory(buf, (len*4 + 1)*sizeof(WCHAR));
243 MultiByteToWideChar(CP_ACP, 0, string, -1, buf, len*4);
244 CStringW temp = CStringW(buf);
245 delete [] buf;
246 return (CUnicodeUtils::GetUTF8(temp));
249 CString CUnicodeUtils::GetUnicode(const CStringA& string, int acp)
251 WCHAR * buf;
252 int len = string.GetLength();
253 if (len==0)
254 return CString();
255 buf = new WCHAR[len*4 + 1];
256 SecureZeroMemory(buf, (len*4 + 1)*sizeof(WCHAR));
257 MultiByteToWideChar(acp, 0, string, -1, buf, len*4);
258 CString ret = CString(buf);
259 delete [] buf;
260 return ret;
263 CStringA CUnicodeUtils::ConvertWCHARStringToUTF8(const CString& string)
265 CStringA sRet;
266 char * buf;
267 buf = new char[string.GetLength()+1];
268 if (buf)
270 int i=0;
271 for ( ; i<string.GetLength(); ++i)
273 buf[i] = (char)string.GetAt(i);
275 buf[i] = 0;
276 sRet = CStringA(buf);
277 delete [] buf;
279 return sRet;
282 #endif //_MFC_VER
284 #ifdef UNICODE
285 std::string CUnicodeUtils::StdGetUTF8(const wide_string& wide)
287 int len = (int)wide.size();
288 if (len==0)
289 return std::string();
290 int size = len*4;
291 char * narrow = new char[size];
292 int ret = WideCharToMultiByte(CP_UTF8, 0, wide.c_str(), len, narrow, size-1, NULL, NULL);
293 narrow[ret] = 0;
294 std::string sRet = std::string(narrow);
295 delete [] narrow;
296 return sRet;
299 wide_string CUnicodeUtils::StdGetUnicode(const std::string& multibyte)
301 int len = (int)multibyte.size();
302 if (len==0)
303 return wide_string();
304 int size = len*4;
305 wchar_t * wide = new wchar_t[size];
306 int ret = MultiByteToWideChar(CP_UTF8, 0, multibyte.c_str(), len, wide, size - 1);
307 wide[ret] = 0;
308 wide_string sRet = wide_string(wide);
309 delete [] wide;
310 return sRet;
312 #endif
314 std::string WideToMultibyte(const wide_string& wide)
316 char * narrow = new char[wide.length()*3+2];
317 BOOL defaultCharUsed;
318 int ret = (int)WideCharToMultiByte(CP_ACP, 0, wide.c_str(), (int)wide.size(), narrow, (int)wide.length()*3 - 1, ".", &defaultCharUsed);
319 narrow[ret] = 0;
320 std::string str = narrow;
321 delete[] narrow;
322 return str;
325 std::string WideToUTF8(const wide_string& wide)
327 char * narrow = new char[wide.length()*3+2];
328 int ret = (int)WideCharToMultiByte(CP_UTF8, 0, wide.c_str(), (int)wide.size(), narrow, (int)wide.length()*3 - 1, NULL, NULL);
329 narrow[ret] = 0;
330 std::string str = narrow;
331 delete[] narrow;
332 return str;
335 wide_string MultibyteToWide(const std::string& multibyte)
337 size_t length = multibyte.length();
338 if (length == 0)
339 return wide_string();
341 wchar_t * wide = new wchar_t[multibyte.length()*2+2];
342 if (wide == NULL)
343 return wide_string();
344 int ret = (int)MultiByteToWideChar(CP_ACP, 0, multibyte.c_str(), (int)multibyte.size(), wide, (int)length*2 - 1);
345 wide[ret] = 0;
346 wide_string str = wide;
347 delete[] wide;
348 return str;
351 wide_string UTF8ToWide(const std::string& multibyte)
353 size_t length = multibyte.length();
354 if (length == 0)
355 return wide_string();
357 wchar_t * wide = new wchar_t[length*2+2];
358 if (wide == NULL)
359 return wide_string();
360 int ret = (int)MultiByteToWideChar(CP_UTF8, 0, multibyte.c_str(), (int)multibyte.size(), wide, (int)length*2 - 1);
361 wide[ret] = 0;
362 wide_string str = wide;
363 delete[] wide;
364 return str;
366 #ifdef UNICODE
367 stdstring UTF8ToString(const std::string& string) {return UTF8ToWide(string);}
368 std::string StringToUTF8(const stdstring& string) {return WideToUTF8(string);}
369 #else
370 stdstring UTF8ToString(const std::string& string) {return WideToMultibyte(UTF8ToWide(string));}
371 std::string StringToUTF8(const stdstring& string) {return WideToUTF8(MultibyteToWide(string));}
372 #endif
375 #pragma warning(push)
376 #pragma warning(disable: 4200)
377 struct STRINGRESOURCEIMAGE
379 WORD nLength;
380 WCHAR achString[];
382 #pragma warning(pop) // C4200
384 int LoadStringEx(HINSTANCE hInstance, UINT uID, LPTSTR lpBuffer, int nBufferMax, WORD wLanguage)
386 const STRINGRESOURCEIMAGE* pImage;
387 const STRINGRESOURCEIMAGE* pImageEnd;
388 ULONG nResourceSize;
389 HGLOBAL hGlobal;
390 UINT iIndex;
391 #ifndef UNICODE
392 BOOL defaultCharUsed;
393 #endif
394 int ret;
396 if (lpBuffer == NULL)
397 return 0;
398 lpBuffer[0] = 0;
399 HRSRC hResource = FindResourceEx(hInstance, RT_STRING, MAKEINTRESOURCE(((uID>>4)+1)), wLanguage);
400 if (!hResource)
402 //try the default language before giving up!
403 hResource = FindResource(hInstance, MAKEINTRESOURCE(((uID>>4)+1)), RT_STRING);
404 if (!hResource)
405 return 0;
407 hGlobal = LoadResource(hInstance, hResource);
408 if (!hGlobal)
409 return 0;
410 pImage = (const STRINGRESOURCEIMAGE*)::LockResource(hGlobal);
411 if(!pImage)
412 return 0;
414 nResourceSize = ::SizeofResource(hInstance, hResource);
415 pImageEnd = (const STRINGRESOURCEIMAGE*)(LPBYTE(pImage)+nResourceSize);
416 iIndex = uID&0x000f;
418 while ((iIndex > 0) && (pImage < pImageEnd))
420 pImage = (const STRINGRESOURCEIMAGE*)(LPBYTE(pImage)+(sizeof(STRINGRESOURCEIMAGE)+(pImage->nLength*sizeof(WCHAR))));
421 iIndex--;
423 if (pImage >= pImageEnd)
424 return 0;
425 if (pImage->nLength == 0)
426 return 0;
427 #ifdef UNICODE
428 ret = pImage->nLength;
429 if (ret > nBufferMax)
430 ret = nBufferMax;
431 wcsncpy_s((wchar_t *)lpBuffer, nBufferMax, pImage->achString, ret);
432 lpBuffer[ret] = 0;
433 #else
434 ret = WideCharToMultiByte(CP_ACP, 0, pImage->achString, pImage->nLength, (LPSTR)lpBuffer, nBufferMax-1, ".", &defaultCharUsed);
435 lpBuffer[ret] = 0;
436 #endif
437 return ret;