1 /* Some conversion functions for handling UTF-8
3 * copyright Marcoen Hirschberg (2004,2005)
5 * I got all the info from:
6 * http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
8 * http://en.wikipedia.org/wiki/Unicode
14 #include "rbunicode.h"
21 #define CODEPAGE_DIR "/.rockbox/codepages"
22 static int default_codepage
= 0;
23 static int loaded_cp_table
= 0;
25 #ifdef HAVE_LCD_BITMAP
27 #define MAX_CP_TABLE_SIZE 32768
31 ISO_8859_1
= 0, ISO_8859_7
, ISO_8859_8
, WIN_1251
,
32 ISO_8859_11
, WIN_1256
, ISO_8859_9
, ISO_8859_2
,
33 SJIS
, GB_2312
, KSX_1001
, BIG_5
, UTF_8
, NUM_CODEPAGES
35 static const char *filename
[NUM_TABLES
] =
37 CODEPAGE_DIR
"/iso.cp",
38 CODEPAGE_DIR
"/932.cp", /* SJIS */
39 CODEPAGE_DIR
"/936.cp", /* GB2312 */
40 CODEPAGE_DIR
"/949.cp", /* KSX1001 */
41 CODEPAGE_DIR
"/950.cp" /* BIG5 */
43 static const char cp_2_table
[NUM_CODEPAGES
] =
45 0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 0
48 #else /* !HAVE_LCD_BITMAP, reduced support */
50 #define MAX_CP_TABLE_SIZE 512
54 ISO_8859_1
= 0, ISO_8859_7
, WIN_1251
,
55 ISO_8859_9
, ISO_8859_2
, UTF_8
, NUM_CODEPAGES
57 static const char *filename
[NUM_TABLES
] =
59 CODEPAGE_DIR
"/isomini.cp",
61 static const char cp_2_table
[NUM_CODEPAGES
] =
68 static unsigned short codepage_table
[MAX_CP_TABLE_SIZE
];
70 static const unsigned char utf8comp
[6] =
72 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
75 /* Load codepage file into memory */
76 static int load_cp_table(int cp
)
79 int table
= cp_2_table
[cp
];
83 if (table
== 0 || table
== loaded_cp_table
)
86 file
= open(filename
[table
-1], O_RDONLY
|O_BINARY
);
89 DEBUGF("Can't open codepage file: %s.cp\n", filename
[table
-1]);
93 tablesize
= filesize(file
) / 2;
95 if (tablesize
> MAX_CP_TABLE_SIZE
) {
96 DEBUGF("Invalid codepage file: %s.cp\n", filename
[table
-1]);
101 while (i
< tablesize
) {
102 if (!read(file
, tmp
, 2)) {
103 DEBUGF("Can't read from codepage file: %s.cp\n",
108 codepage_table
[i
++] = (tmp
[1] << 8) | tmp
[0];
111 loaded_cp_table
= table
;
116 /* Encode a UCS value as UTF-8 and return a pointer after this UTF-8 char. */
117 unsigned char* utf8encode(unsigned long ucs
, unsigned char *utf8
)
122 while (ucs
>> (5*tail
+ 6))
125 *utf8
++ = (ucs
>> (6*tail
)) | utf8comp
[tail
];
127 *utf8
++ = ((ucs
>> (6*tail
)) & (MASK
^ 0xFF)) | COMP
;
132 /* Recode an iso encoded string to UTF-8 */
133 unsigned char* iso_decode(const unsigned char *iso
, unsigned char *utf8
,
136 unsigned short ucs
, tmp
;
138 if (cp
== -1) /* use default codepage */
139 cp
= default_codepage
;
141 if (!load_cp_table(cp
)) cp
= 0;
144 if (*iso
< 128 || cp
== UTF_8
) /* Already UTF-8 */
149 /* cp tells us which codepage to convert from */
151 case ISO_8859_7
: /* Greek */
152 case WIN_1251
: /* Cyrillic */
153 case ISO_8859_9
: /* Turkish */
154 case ISO_8859_2
: /* Latin Extended */
155 #ifdef HAVE_LCD_BITMAP
156 case ISO_8859_8
: /* Hebrew */
157 case ISO_8859_11
: /* Thai */
158 case WIN_1256
: /* Arabic */
160 tmp
= ((cp
-1)*128) + (*iso
++ - 128);
161 ucs
= codepage_table
[tmp
];
164 #ifdef HAVE_LCD_BITMAP
165 case SJIS
: /* Japanese */
166 if (*iso
> 0xA0 && *iso
< 0xE0) {
167 tmp
= *iso
++ | (0xA100 - 0x8000);
168 ucs
= codepage_table
[tmp
];
172 case GB_2312
: /* Simplified Chinese */
173 case KSX_1001
: /* Korean */
174 case BIG_5
: /* Traditional Chinese */
175 if (count
< 1 || !iso
[1]) {
180 /* we assume all cjk strings are written
181 in big endian order */
185 ucs
= codepage_table
[tmp
];
188 #endif /* HAVE_LCD_BITMAP */
195 if (ucs
== 0) /* unknown char, use replacement char */
197 utf8
= utf8encode(ucs
, utf8
);
203 /* Recode a UTF-16 string with little-endian byte ordering to UTF-8 */
204 unsigned char* utf16LEdecode(const unsigned char *utf16
, unsigned char *utf8
,
210 /* Check for a surrogate pair */
211 if (utf16
[1] >= 0xD8 && utf16
[1] < 0xE0) {
212 ucs
= 0x10000 + ((utf16
[0] << 10) | ((utf16
[1] - 0xD8) << 18)
213 | utf16
[2] | ((utf16
[3] - 0xDC) << 8));
217 ucs
= (utf16
[0] | (utf16
[1] << 8));
221 utf8
= utf8encode(ucs
, utf8
);
226 /* Recode a UTF-16 string with big-endian byte ordering to UTF-8 */
227 unsigned char* utf16BEdecode(const unsigned char *utf16
, unsigned char *utf8
,
233 if (*utf16
>= 0xD8 && *utf16
< 0xE0) { /* Check for a surrogate pair */
234 ucs
= 0x10000 + (((utf16
[0] - 0xD8) << 18) | (utf16
[1] << 10)
235 | ((utf16
[2] - 0xDC) << 8) | utf16
[3]);
239 ucs
= (utf16
[0] << 8) | utf16
[1];
243 utf8
= utf8encode(ucs
, utf8
);
248 #if 0 /* currently unused */
249 /* Recode any UTF-16 string to UTF-8 */
250 unsigned char* utf16decode(const unsigned char *utf16
, unsigned char *utf8
,
255 ucs
= *(utf16
++) << 8;
258 if (ucs
== 0xFEFF) /* Check for BOM */
259 return utf16BEdecode(utf16
, utf8
, count
-1);
260 else if (ucs
== 0xFFFE)
261 return utf16LEdecode(utf16
, utf8
, count
-1);
262 else { /* ADDME: Should default be LE or BE? */
264 return utf16BEdecode(utf16
, utf8
, count
);
269 /* Return the number of UTF-8 chars in a string */
270 unsigned long utf8length(const unsigned char *utf8
)
275 if ((*utf8
++ & MASK
) != COMP
)
281 /* Decode 1 UTF-8 char and return a pointer to the next char. */
282 const unsigned char* utf8decode(const unsigned char *utf8
, unsigned short *ucs
)
284 unsigned char c
= *utf8
++;
288 if ((c
<= 0x7f) || (c
>= 0xc2)) {
289 /* Start of new character. */
290 if (c
< 0x80) { /* U-00000000 - U-0000007F, 1 byte */
292 } else if (c
< 0xe0) { /* U-00000080 - U-000007FF, 2 bytes */
295 } else if (c
< 0xf0) { /* U-00000800 - U-0000FFFF, 3 bytes */
298 } else if (c
< 0xf5) { /* U-00010000 - U-001FFFFF, 4 bytes */
306 while (tail
-- && ((c
= *utf8
++) != 0)) {
307 if ((c
& 0xc0) == 0x80) {
308 /* Valid continuation character. */
309 code
= (code
<< 6) | (c
& 0x3f);
312 /* Invalid continuation char */
319 /* Invalid UTF-8 char */
322 /* currently we don't support chars above U-FFFF */
323 *ucs
= (code
< 0x10000) ? code
: 0xfffd;
327 void set_codepage(int cp
)
329 default_codepage
= cp
;
333 /* seek to a given char in a utf8 string and
334 return its start position in the string */
335 int utf8seek(const unsigned char* utf8
, int offset
)
341 while ((utf8
[pos
] & MASK
) == COMP
)