2 * Copyright (c) 2011, 2012
3 * Zhihao Yuan. All rights reserved.
5 * See the LICENSE file for redistribution information.
9 static const char sccsid
[] = "$Id: encoding.c,v 1.4 2011/12/13 19:40:52 zy Exp $";
12 #include <sys/types.h>
14 int looks_utf8
__P((const char *, size_t));
15 int looks_utf16
__P((const char *, size_t));
16 int decode_utf8
__P((const char *));
17 int decode_utf16
__P((const char *, int));
19 #define F 0 /* character never appears in text */
20 #define T 1 /* character appears in plain ASCII text */
21 #define I 2 /* character appears in ISO-8859 text */
22 #define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
24 static char text_chars
[256] = {
25 /* BEL BS HT LF FF CR */
26 F
, F
, F
, F
, F
, F
, F
, T
, T
, T
, T
, F
, T
, T
, F
, F
, /* 0x0X */
28 F
, F
, F
, F
, F
, F
, F
, F
, F
, F
, F
, T
, F
, F
, F
, F
, /* 0x1X */
29 T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, /* 0x2X */
30 T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, /* 0x3X */
31 T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, /* 0x4X */
32 T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, /* 0x5X */
33 T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, /* 0x6X */
34 T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, T
, F
, /* 0x7X */
36 X
, X
, X
, X
, X
, T
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, /* 0x8X */
37 X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, X
, /* 0x9X */
38 I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, /* 0xaX */
39 I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, /* 0xbX */
40 I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, /* 0xcX */
41 I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, /* 0xdX */
42 I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, /* 0xeX */
43 I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
, I
/* 0xfX */
48 * Decide whether some text looks like UTF-8. Returns:
51 * 0: uses odd control characters, so doesn't look like text
53 * 2: definitely UTF-8 text (valid high-bit set bytes)
55 * Based on RFC 3629. UTF-8 with BOM is not accepted.
57 * PUBLIC: int looks_utf8 __P((const char *, size_t));
60 looks_utf8(const char *ibuf
, size_t nbytes
)
62 const u_char
*buf
= (u_char
*)ibuf
;
65 int gotone
= 0, ctrl
= 0;
67 for (i
= 0; i
< nbytes
; i
++) {
68 if ((buf
[i
] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */
70 * Even if the whole file is valid UTF-8 sequences,
71 * still reject it if it uses weird control characters.
74 if (text_chars
[buf
[i
]] != T
)
76 } else if ((buf
[i
] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
78 } else { /* 11xxxxxx begins UTF-8 */
81 if ((buf
[i
] & 0x20) == 0) /* 110xxxxx */
82 if (buf
[i
] > 0xC1) /* C0, C1 */
85 else if ((buf
[i
] & 0x10) == 0) /* 1110xxxx */
87 else if ((buf
[i
] & 0x08) == 0) /* 11110xxx */
90 else return -1; /* F5, F6, F7 */
92 return -1; /* F8~FF */
94 for (n
= 0; n
< following
; n
++) {
99 if (buf
[i
] & 0x40) /* 10xxxxxx */
107 return ctrl
? 0 : (gotone
? 2 : 1);
112 * Decide whether some text looks like UTF-16. Returns:
115 * 1: Little-endian UTF-16
116 * 2: Big-endian UTF-16
118 * PUBLIC: int looks_utf16 __P((const char *, size_t));
121 looks_utf16(const char *ibuf
, size_t nbytes
)
123 const u_char
*buf
= (u_char
*)ibuf
;
133 bom
= buf
[0] << 8 ^ buf
[1];
136 else if (bom
== 0xFEFF)
141 for (i
= 2; i
+ 1 < nbytes
; i
+= 2) {
143 c
= buf
[i
] << 8 ^ buf
[i
+ 1];
145 c
= buf
[i
] ^ buf
[i
+ 1] << 8;
148 if (c
< 0xD800 || c
> 0xDFFF)
149 if (c
< 128 && text_chars
[c
] != T
)
159 else if (c
< 0xDC00 || c
> 0xDFFF)
173 * Decode a UTF-8 character from byte string to Unicode.
174 * Returns -1 if the first byte is a not UTF-8 leader.
176 * Based on RFC 3629, but without error detection.
178 * PUBLIC: int decode_utf8 __P((const char *));
180 int decode_utf8(const char *ibuf
) {
181 const u_char
*buf
= (u_char
*)ibuf
;
184 if ((buf
[0] & 0x80) == 0)
186 else if ((buf
[0] & 0x40) == 0);
188 if ((buf
[0] & 0x20) == 0)
189 u
= (buf
[0] ^ 0xC0) << 6 ^ (buf
[1] ^ 0x80);
190 else if ((buf
[0] & 0x10) == 0)
191 u
= (buf
[0] ^ 0xE0) << 12 ^ (buf
[1] ^ 0x80) << 6
193 else if (((buf
[0] & 0x08) == 0))
194 u
= (buf
[0] ^ 0xF0) << 18 ^ (buf
[1] ^ 0x80) << 12
195 ^ (buf
[2] ^ 0x80) << 6 ^ (buf
[3] ^ 0x80);
202 * Decode a UTF-16 character from byte string to Unicode.
203 * Returns -1 if the first unsigned integer is invalid.
205 * No error detection on supplementary bytes.
207 * PUBLIC: int decode_utf16 __P((const char *, int));
209 int decode_utf16(const char* ibuf
, int bigend
) {
210 const u_char
*buf
= (u_char
*)ibuf
;
215 w1
= buf
[0] << 8 ^ buf
[1];
217 w1
= buf
[0] ^ buf
[1] << 8;
219 if (w1
< 0xD800 || w1
> 0xDFFF)
221 else if (w1
> 0xDBFF);
224 w2
= buf
[2] << 8 ^ buf
[3];
226 w2
= buf
[2] ^ buf
[3] << 8;
227 u
= ((w1
^ 0xD800) << 10 ^ (w2
^ 0xDC00)) + 0x10000;