2 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
33 * The functions in this file convert from the standard multibyte forms
34 * to the wide character forms used internally by libc. Unfortunately,
35 * this approach means that we need a method for each and every encoding.
43 #include <sys/types.h>
44 #include "localedef.h"
46 static int towide_none(wchar_t *, const char *, unsigned);
47 static int towide_utf8(wchar_t *, const char *, unsigned);
48 static int towide_big5(wchar_t *, const char *, unsigned);
49 static int towide_gbk(wchar_t *, const char *, unsigned);
50 static int towide_gb2312(wchar_t *, const char *, unsigned);
51 static int towide_gb18030(wchar_t *, const char *, unsigned);
52 static int towide_mskanji(wchar_t *, const char *, unsigned);
53 static int towide_euccn(wchar_t *, const char *, unsigned);
54 static int towide_eucjp(wchar_t *, const char *, unsigned);
55 static int towide_euckr(wchar_t *, const char *, unsigned);
56 static int towide_euctw(wchar_t *, const char *, unsigned);
58 static int tomb_none(char *, wchar_t);
59 static int tomb_utf8(char *, wchar_t);
60 static int tomb_mbs(char *, wchar_t);
62 static int (*_towide
)(wchar_t *, const char *, unsigned) = towide_none
;
63 static int (*_tomb
)(char *, wchar_t) = tomb_none
;
64 static char _encoding_buffer
[20] = {'N','O','N','E'};
65 static const char *_encoding
= _encoding_buffer
;
66 static int _nbits
= 7;
69 * Table of supported encodings. We only bother to list the multibyte
70 * encodings here, because single byte locales are handed by "NONE".
74 /* the name that the underlying libc implemenation uses */
76 /* the maximum number of bits required for priorities */
78 int (*towide
)(wchar_t *, const char *, unsigned);
79 int (*tomb
)(char *, wchar_t);
82 * UTF8 values max out at 0x1fffff (although in theory there could
83 * be later extensions, but it won't happen.) This means we only need
84 * 21 bits to be able to encode the entire range of priorities.
86 { "UTF-8", "UTF-8", 21, towide_utf8
, tomb_utf8
},
87 { "UTF8", "UTF-8", 21, towide_utf8
, tomb_utf8
},
88 { "utf8", "UTF-8", 21, towide_utf8
, tomb_utf8
},
89 { "utf-8", "UTF-8", 21, towide_utf8
, tomb_utf8
},
91 { "EUC-CN", "EUC-CN", 16, towide_euccn
, tomb_mbs
},
92 { "eucCN", "EUC-CN", 16, towide_euccn
, tomb_mbs
},
94 * Becuase the 3-byte form of EUC-JP use the same leading byte,
95 * only 17 bits required to provide unique priorities. (The low
96 * bit of that first byte is set.) By setting this value low,
97 * we can get by with only 3 bytes in the strxfrm expansion.
99 { "EUC-JP", "EUC-JP", 17, towide_eucjp
, tomb_mbs
},
100 { "eucJP", "EUC-JP", 17, towide_eucjp
, tomb_mbs
},
102 { "EUC-KR", "EUC-KR", 16, towide_euckr
, tomb_mbs
},
103 { "eucKR", "EUC-KR", 16, towide_euckr
, tomb_mbs
},
105 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
106 * high order byte is 0x8E. However, with 4 byte encodings,
107 * the third byte will be A0-B0. So we only need to consider
108 * the lower order 24 bits for collation.
110 { "EUC-TW", "EUC-TW", 24, towide_euctw
, tomb_mbs
},
111 { "eucTW", "EUC-TW", 24, towide_euctw
, tomb_mbs
},
113 { "MS_Kanji", "MSKanji", 16, towide_mskanji
, tomb_mbs
},
114 { "MSKanji", "MSKanji", 16, towide_mskanji
, tomb_mbs
},
115 { "PCK", "MSKanji", 16, towide_mskanji
, tomb_mbs
},
116 { "SJIS", "MSKanji", 16, towide_mskanji
, tomb_mbs
},
117 { "Shift_JIS", "MSKanji", 16, towide_mskanji
, tomb_mbs
},
119 { "BIG5", "BIG5", 16, towide_big5
, tomb_mbs
},
120 { "big5", "BIG5", 16, towide_big5
, tomb_mbs
},
121 { "Big5", "BIG5", 16, towide_big5
, tomb_mbs
},
123 { "GBK", "GBK", 16, towide_gbk
, tomb_mbs
},
126 * GB18030 can get away with just 31 bits. This is because the
127 * high order bit is always set for 4 byte values, and the
128 * at least one of the other bits in that 4 byte value will
131 { "GB18030", "GB18030", 31, towide_gb18030
, tomb_mbs
},
134 * This should probably be an aliase for euc-cn, or vice versa.
136 { "GB2312", "GB2312", 16, towide_gb2312
, tomb_mbs
},
138 { NULL
, NULL
, 0, 0, 0 },
142 show_mb(const char *mb
)
146 /* ASCII stuff we just print */
147 if (isascii(*mb
) && isgraph(*mb
)) {
155 (void) snprintf(scr
, sizeof (scr
), "\\x%02x", *mb
);
156 (void) strlcat(buf
, scr
, sizeof (buf
));
162 static char *widemsg
;
164 __printflike(1, 2) void
165 werr(const char *fmt
, ...)
171 (void) vasprintf(&msg
, fmt
, va
);
179 * This is used for 8-bit encodings.
182 towide_none(wchar_t *c
, const char *mb
, unsigned n __unused
)
184 if (mb_cur_max
!= 1) {
185 werr("invalid or unsupported multibyte locale");
193 tomb_none(char *mb
, wchar_t wc
)
195 if (mb_cur_max
!= 1) {
196 werr("invalid or unsupported multibyte locale");
199 *(uint8_t *)mb
= (wc
& 0xff);
205 * UTF-8 stores wide characters in UTF-32 form.
208 towide_utf8(wchar_t *wc
, const char *mb
, unsigned n
)
212 wchar_t lv
; /* lowest legal value */
214 const uint8_t *s
= (const uint8_t *)mb
;
218 if ((c
& 0x80) == 0) {
222 } else if ((c
& 0xe0) == 0xc0) {
223 /* u80-u7ff - two bytes encoded */
227 } else if ((c
& 0xf0) == 0xe0) {
228 /* u800-uffff - three bytes encoded */
232 } else if ((c
& 0xf8) == 0xf0) {
233 /* u1000-u1fffff - four bytes encoded */
238 /* 5 and 6 byte encodings are not legal unicode */
239 werr("utf8 encoding too large (%s)", show_mb(mb
));
243 werr("incomplete utf8 sequence (%s)", show_mb(mb
));
247 for (i
= 1; i
< nb
; i
++) {
248 if (((s
[i
]) & 0xc0) != 0x80) {
249 werr("illegal utf8 byte (%x)", s
[i
]);
257 werr("illegal redundant utf8 encoding (%s)", show_mb(mb
));
265 tomb_utf8(char *mb
, wchar_t wc
)
267 uint8_t *s
= (uint8_t *)mb
;
280 } else if (wc
<= 0xffff) {
283 } else if (wc
<= 0x1fffff) {
287 werr("illegal uf8 char (%x)", wc
);
290 for (i
= cnt
- 1; i
; i
--) {
291 s
[i
] = (wc
& 0x3f) | 0x80;
300 * Several encodings share a simplistic dual byte encoding. In these
301 * forms, they all indicate that a two byte sequence is to be used if
302 * the first byte has its high bit set. They all store this simple
303 * encoding as a 16-bit value, although a great many of the possible
304 * code points are not used in most character sets. This gives a possible
305 * set of just over 32,000 valid code points.
307 * 0x00 - 0x7f - 1 byte encoding
308 * 0x80 - 0x7fff - illegal
309 * 0x8000 - 0xffff - 2 byte encoding
313 towide_dbcs(wchar_t *wc
, const char *mb
, unsigned n
)
317 c
= *(const uint8_t *)mb
;
319 if ((c
& 0x80) == 0) {
325 werr("incomplete character sequence (%s)", show_mb(mb
));
329 /* Store both bytes as a single 16-bit wide. */
331 c
|= (uint8_t)(mb
[1]);
337 * Most multibyte locales just convert the wide character to the multibyte
338 * form by stripping leading null bytes, and writing the 32-bit quantity
339 * in big-endian order.
342 tomb_mbs(char *mb
, wchar_t wc
)
344 uint8_t *s
= (uint8_t *)mb
;
347 if ((wc
& 0xff000000U
) != 0) {
349 } else if ((wc
& 0x00ff0000U
) != 0) {
351 } else if ((wc
& 0x0000ff00U
) != 0) {
362 /* ensure null termination */
369 * big5 is a simple dual byte character set.
372 towide_big5(wchar_t *wc
, const char *mb
, unsigned n
)
374 return (towide_dbcs(wc
, mb
, n
));
378 * GBK encodes wides in the same way that big5 does, the high order
379 * bit of the first byte indicates a double byte character.
382 towide_gbk(wchar_t *wc
, const char *mb
, unsigned n
)
384 return (towide_dbcs(wc
, mb
, n
));
388 * GB2312 is another DBCS. Its cleaner than others in that the second
389 * byte does not encode ASCII, but it supports characters.
392 towide_gb2312(wchar_t *wc
, const char *mb
, unsigned n
)
394 return (towide_dbcs(wc
, mb
, n
));
398 * GB18030. This encodes as 8, 16, or 32-bits.
399 * 7-bit values are in 1 byte, 4 byte sequences are used when
400 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
403 towide_gb18030(wchar_t *wc
, const char *mb
, unsigned n
)
407 c
= *(const uint8_t *)mb
;
409 if ((c
& 0x80) == 0) {
415 werr("incomplete character sequence (%s)", show_mb(mb
));
419 /* pull in the second byte */
421 c
|= (uint8_t)(mb
[1]);
423 if (((c
& 0xff) >= 0x30) && ((c
& 0xff) <= 0x39)) {
425 werr("incomplete 4-byte character sequence (%s)",
430 c
|= (uint8_t)(mb
[2]);
432 c
|= (uint8_t)(mb
[3]);
442 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
443 * also has a range of single byte characters above 0x80. (0xa1-0xdf).
446 towide_mskanji(wchar_t *wc
, const char *mb
, unsigned n
)
450 c
= *(const uint8_t *)mb
;
452 if ((c
< 0x80) || ((c
> 0xa0) && (c
< 0xe0))) {
459 werr("incomplete character sequence (%s)", show_mb(mb
));
463 /* Store both bytes as a single 16-bit wide. */
465 c
|= (uint8_t)(mb
[1]);
471 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional
472 * variable data to encode these, but we're going to treat each as independent
473 * instead. Its the only way we can sensibly move forward.
475 * Note that the way in which the different EUC forms vary is how wide
476 * CS2 and CS3 are and what the first byte of them is.
479 towide_euc_impl(wchar_t *wc
, const char *mb
, unsigned n
,
480 uint8_t cs2
, uint8_t cs2width
, uint8_t cs3
, uint8_t cs3width
)
486 c
= *(const uint8_t *)mb
;
489 * All variations of EUC encode 7-bit ASCII as one byte, and use
490 * additional bytes for more than that.
492 if ((c
& 0x80) == 0) {
499 * All EUC variants reserve 0xa1-0xff to identify CS1, which
500 * is always two bytes wide. Note that unused CS will be zero,
501 * and that cannot be true because we know that the high order
506 } else if (c
== cs2
) {
508 } else if (c
== cs3
) {
512 if ((int)n
< width
) {
513 werr("incomplete character sequence (%s)", show_mb(mb
));
517 for (i
= 1; i
< width
; i
++) {
518 /* pull in the next byte */
520 c
|= (uint8_t)(mb
[i
]);
528 * EUC-CN encodes as follows:
530 * Code set 0 (ASCII): 0x21-0x7E
531 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
536 towide_euccn(wchar_t *wc
, const char *mb
, unsigned n
)
538 return (towide_euc_impl(wc
, mb
, n
, 0x8e, 4, 0, 0));
542 * EUC-JP encodes as follows:
544 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E
545 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE
546 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF
547 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE
550 towide_eucjp(wchar_t *wc
, const char *mb
, unsigned n
)
552 return (towide_euc_impl(wc
, mb
, n
, 0x8e, 2, 0x8f, 3));
556 * EUC-KR encodes as follows:
558 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E
559 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE
564 towide_euckr(wchar_t *wc
, const char *mb
, unsigned n
)
566 return (towide_euc_impl(wc
, mb
, n
, 0, 0, 0, 0));
570 * EUC-TW encodes as follows:
572 * Code set 0 (ASCII): 0x21-0x7E
573 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
574 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE
578 towide_euctw(wchar_t *wc
, const char *mb
, unsigned n
)
580 return (towide_euc_impl(wc
, mb
, n
, 0x8e, 4, 0, 0));
584 * Public entry points.
588 to_wide(wchar_t *wc
, const char *mb
)
590 /* this won't fail hard */
591 return (_towide(wc
, mb
, strlen(mb
)));
595 to_mb(char *mb
, wchar_t wc
)
599 if ((rv
= _tomb(mb
, wc
)) < 0) {
608 to_mb_string(const wchar_t *wcs
)
614 mbs
= malloc((wcslen(wcs
) * mb_cur_max
) + 1);
616 errf("out of memory");
621 if ((len
= to_mb(ptr
, *wcs
)) < 0) {
634 set_wide_encoding(const char *encoding
)
638 _towide
= towide_none
;
642 snprintf(_encoding_buffer
, sizeof(_encoding_buffer
), "NONE:%s",
644 for (i
= 0; mb_encodings
[i
].name
; i
++) {
645 if (strcasecmp(encoding
, mb_encodings
[i
].name
) == 0) {
646 _towide
= mb_encodings
[i
].towide
;
647 _tomb
= mb_encodings
[i
].tomb
;
648 _encoding
= mb_encodings
[i
].cname
;
649 _nbits
= mb_encodings
[i
].nbits
;
656 get_wide_encoding(void)
664 return ((int)((1U << _nbits
) - 1));