2 * conversion functions between pg_wchar and multibyte streams.
4 * src/backend/utils/mb/wchar.c
7 /* can be used in either frontend or backend */
9 #include "postgres_fe.h"
14 #include "mb/pg_wchar.h"
18 * Operations on multi-byte encodings are driven by a table of helper
21 * To add an encoding support, define mblen(), dsplen() and verifier() for
22 * the encoding. For server-encodings, also define mb2wchar() and wchar2mb()
23 * conversion functions.
25 * These functions generally assume that their input is validly formed.
26 * The "verifier" functions, further down in the file, have to be more
29 * We expect that mblen() does not need to examine more than the first byte
30 * of the character to discover the correct length. GB18030 is an exception
31 * to that rule, though, as it also looks at second byte. But even that
32 * behaves in a predictable way, if you only pass the first byte: it will
33 * treat 4-byte encoded characters as two 2-byte encoded characters, which is
34 * good enough for all current uses.
36 * Note: for the display output of psql to work properly, the return values
37 * of the dsplen functions must conform to the Unicode standard. In particular
38 * the NUL character is zero width and control characters are generally
39 * width -1. It is recommended that non-ASCII encodings refer their ASCII
40 * subset to the ASCII routines to ensure consistency.
47 pg_ascii2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
51 while (len
> 0 && *from
)
62 pg_ascii_mblen(const unsigned char *s
)
68 pg_ascii_dsplen(const unsigned char *s
)
72 if (*s
< 0x20 || *s
== 0x7f)
82 pg_euc2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
86 while (len
> 0 && *from
)
88 if (*from
== SS2
&& len
>= 2) /* JIS X 0201 (so called "1 byte
92 *to
= (SS2
<< 8) | *from
++;
95 else if (*from
== SS3
&& len
>= 3) /* JIS X 0212 KANJI */
98 *to
= (SS3
<< 16) | (*from
++ << 8);
102 else if (IS_HIGHBIT_SET(*from
) && len
>= 2) /* JIS X 0208 KANJI */
108 else /* must be ASCII */
121 pg_euc_mblen(const unsigned char *s
)
129 else if (IS_HIGHBIT_SET(*s
))
137 pg_euc_dsplen(const unsigned char *s
)
145 else if (IS_HIGHBIT_SET(*s
))
148 len
= pg_ascii_dsplen(s
);
156 pg_eucjp2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
158 return pg_euc2wchar_with_len(from
, to
, len
);
162 pg_eucjp_mblen(const unsigned char *s
)
164 return pg_euc_mblen(s
);
168 pg_eucjp_dsplen(const unsigned char *s
)
176 else if (IS_HIGHBIT_SET(*s
))
179 len
= pg_ascii_dsplen(s
);
187 pg_euckr2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
189 return pg_euc2wchar_with_len(from
, to
, len
);
193 pg_euckr_mblen(const unsigned char *s
)
195 return pg_euc_mblen(s
);
199 pg_euckr_dsplen(const unsigned char *s
)
201 return pg_euc_dsplen(s
);
209 pg_euccn2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
213 while (len
> 0 && *from
)
215 if (*from
== SS2
&& len
>= 3) /* code set 2 (unused?) */
218 *to
= (SS2
<< 16) | (*from
++ << 8);
222 else if (*from
== SS3
&& len
>= 3) /* code set 3 (unused ?) */
225 *to
= (SS3
<< 16) | (*from
++ << 8);
229 else if (IS_HIGHBIT_SET(*from
) && len
>= 2) /* code set 1 */
248 pg_euccn_mblen(const unsigned char *s
)
252 if (IS_HIGHBIT_SET(*s
))
260 pg_euccn_dsplen(const unsigned char *s
)
264 if (IS_HIGHBIT_SET(*s
))
267 len
= pg_ascii_dsplen(s
);
276 pg_euctw2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
280 while (len
> 0 && *from
)
282 if (*from
== SS2
&& len
>= 4) /* code set 2 */
285 *to
= (((uint32
) SS2
) << 24) | (*from
++ << 16);
290 else if (*from
== SS3
&& len
>= 3) /* code set 3 (unused?) */
293 *to
= (SS3
<< 16) | (*from
++ << 8);
297 else if (IS_HIGHBIT_SET(*from
) && len
>= 2) /* code set 2 */
316 pg_euctw_mblen(const unsigned char *s
)
324 else if (IS_HIGHBIT_SET(*s
))
332 pg_euctw_dsplen(const unsigned char *s
)
340 else if (IS_HIGHBIT_SET(*s
))
343 len
= pg_ascii_dsplen(s
);
348 * Convert pg_wchar to EUC_* encoding.
349 * caller must allocate enough space for "to", including a trailing zero!
350 * len: length of from.
351 * "from" not necessarily null terminated.
354 pg_wchar2euc_with_len(const pg_wchar
*from
, unsigned char *to
, int len
)
358 while (len
> 0 && *from
)
362 if ((c
= (*from
>> 24)))
365 *to
++ = (*from
>> 16) & 0xff;
366 *to
++ = (*from
>> 8) & 0xff;
367 *to
++ = *from
& 0xff;
370 else if ((c
= (*from
>> 16)))
373 *to
++ = (*from
>> 8) & 0xff;
374 *to
++ = *from
& 0xff;
377 else if ((c
= (*from
>> 8)))
380 *to
++ = *from
& 0xff;
400 pg_johab_mblen(const unsigned char *s
)
402 return pg_euc_mblen(s
);
406 pg_johab_dsplen(const unsigned char *s
)
408 return pg_euc_dsplen(s
);
412 * convert UTF8 string to pg_wchar (UCS-4)
413 * caller must allocate enough space for "to", including a trailing zero!
414 * len: length of from.
415 * "from" not necessarily null terminated.
418 pg_utf2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
426 while (len
> 0 && *from
)
428 if ((*from
& 0x80) == 0)
433 else if ((*from
& 0xe0) == 0xc0)
436 break; /* drop trailing incomplete char */
439 *to
= (c1
<< 6) | c2
;
442 else if ((*from
& 0xf0) == 0xe0)
445 break; /* drop trailing incomplete char */
449 *to
= (c1
<< 12) | (c2
<< 6) | c3
;
452 else if ((*from
& 0xf8) == 0xf0)
455 break; /* drop trailing incomplete char */
460 *to
= (c1
<< 18) | (c2
<< 12) | (c3
<< 6) | c4
;
465 /* treat a bogus char as length 1; not ours to raise error */
478 * Map a Unicode code point to UTF-8. utf8string must have 4 bytes of
482 unicode_to_utf8(pg_wchar c
, unsigned char *utf8string
)
490 utf8string
[0] = 0xC0 | ((c
>> 6) & 0x1F);
491 utf8string
[1] = 0x80 | (c
& 0x3F);
493 else if (c
<= 0xFFFF)
495 utf8string
[0] = 0xE0 | ((c
>> 12) & 0x0F);
496 utf8string
[1] = 0x80 | ((c
>> 6) & 0x3F);
497 utf8string
[2] = 0x80 | (c
& 0x3F);
501 utf8string
[0] = 0xF0 | ((c
>> 18) & 0x07);
502 utf8string
[1] = 0x80 | ((c
>> 12) & 0x3F);
503 utf8string
[2] = 0x80 | ((c
>> 6) & 0x3F);
504 utf8string
[3] = 0x80 | (c
& 0x3F);
511 * Trivial conversion from pg_wchar to UTF-8.
512 * caller should allocate enough space for "to"
513 * len: length of from.
514 * "from" not necessarily null terminated.
517 pg_wchar2utf_with_len(const pg_wchar
*from
, unsigned char *to
, int len
)
521 while (len
> 0 && *from
)
525 unicode_to_utf8(*from
, to
);
526 char_len
= pg_utf_mblen(to
);
537 * Return the byte length of a UTF8 character pointed to by s
539 * Note: in the current implementation we do not support UTF8 sequences
540 * of more than 4 bytes; hence do NOT return a value larger than 4.
541 * We return "1" for any leading byte that is either flat-out illegal or
542 * indicates a length larger than we support.
544 * pg_utf2wchar_with_len(), utf8_to_unicode(), pg_utf8_islegal(), and perhaps
545 * other places would need to be fixed to change this.
548 pg_utf_mblen(const unsigned char *s
)
552 if ((*s
& 0x80) == 0)
554 else if ((*s
& 0xe0) == 0xc0)
556 else if ((*s
& 0xf0) == 0xe0)
558 else if ((*s
& 0xf8) == 0xf0)
561 else if ((*s
& 0xfc) == 0xf8)
563 else if ((*s
& 0xfe) == 0xfc)
572 * This is an implementation of wcwidth() and wcswidth() as defined in
573 * "The Single UNIX Specification, Version 2, The Open Group, 1997"
574 * <http://www.unix.org/online.html>
576 * Markus Kuhn -- 2001-09-08 -- public domain
578 * customised for PostgreSQL
580 * original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
585 unsigned short first
;
589 /* auxiliary function for binary search in interval table */
591 mbbisearch(pg_wchar ucs
, const struct mbinterval
*table
, int max
)
596 if (ucs
< table
[0].first
|| ucs
> table
[max
].last
)
600 mid
= (min
+ max
) / 2;
601 if (ucs
> table
[mid
].last
)
603 else if (ucs
< table
[mid
].first
)
613 /* The following functions define the column width of an ISO 10646
614 * character as follows:
616 * - The null character (U+0000) has a column width of 0.
618 * - Other C0/C1 control characters and DEL will lead to a return
621 * - Non-spacing and enclosing combining characters (general
622 * category code Mn or Me in the Unicode database) have a
625 * - Other format characters (general category code Cf in the Unicode
626 * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
628 * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
629 * have a column width of 0.
631 * - Spacing characters in the East Asian Wide (W) or East Asian
632 * FullWidth (F) category as defined in Unicode Technical
633 * Report #11 have a column width of 2.
635 * - All remaining characters (including all printable
636 * ISO 8859-1 and WGL4 characters, Unicode control characters,
637 * etc.) have a column width of 1.
639 * This implementation assumes that wchar_t characters are encoded
644 ucs_wcwidth(pg_wchar ucs
)
646 #include "common/unicode_combining_table.h"
648 /* test for 8-bit control characters */
652 if (ucs
< 0x20 || (ucs
>= 0x7f && ucs
< 0xa0) || ucs
> 0x0010ffff)
655 /* binary search in table of non-spacing characters */
656 if (mbbisearch(ucs
, combining
,
657 sizeof(combining
) / sizeof(struct mbinterval
) - 1))
661 * if we arrive here, ucs is not a combining or C0/C1 control character
666 (ucs
<= 0x115f || /* Hangul Jamo init. consonants */
667 (ucs
>= 0x2e80 && ucs
<= 0xa4cf && (ucs
& ~0x0011) != 0x300a &&
668 ucs
!= 0x303f) || /* CJK ... Yi */
669 (ucs
>= 0xac00 && ucs
<= 0xd7a3) || /* Hangul Syllables */
670 (ucs
>= 0xf900 && ucs
<= 0xfaff) || /* CJK Compatibility
672 (ucs
>= 0xfe30 && ucs
<= 0xfe6f) || /* CJK Compatibility Forms */
673 (ucs
>= 0xff00 && ucs
<= 0xff5f) || /* Fullwidth Forms */
674 (ucs
>= 0xffe0 && ucs
<= 0xffe6) ||
675 (ucs
>= 0x20000 && ucs
<= 0x2ffff)));
679 * Convert a UTF-8 character to a Unicode code point.
680 * This is a one-character version of pg_utf2wchar_with_len.
682 * No error checks here, c must point to a long-enough string.
685 utf8_to_unicode(const unsigned char *c
)
687 if ((*c
& 0x80) == 0)
688 return (pg_wchar
) c
[0];
689 else if ((*c
& 0xe0) == 0xc0)
690 return (pg_wchar
) (((c
[0] & 0x1f) << 6) |
692 else if ((*c
& 0xf0) == 0xe0)
693 return (pg_wchar
) (((c
[0] & 0x0f) << 12) |
694 ((c
[1] & 0x3f) << 6) |
696 else if ((*c
& 0xf8) == 0xf0)
697 return (pg_wchar
) (((c
[0] & 0x07) << 18) |
698 ((c
[1] & 0x3f) << 12) |
699 ((c
[2] & 0x3f) << 6) |
702 /* that is an invalid code on purpose */
707 pg_utf_dsplen(const unsigned char *s
)
709 return ucs_wcwidth(utf8_to_unicode(s
));
713 * convert mule internal code to pg_wchar
714 * caller should allocate enough space for "to"
715 * len: length of from.
716 * "from" not necessarily null terminated.
719 pg_mule2wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
723 while (len
> 0 && *from
)
725 if (IS_LC1(*from
) && len
>= 2)
731 else if (IS_LCPRV1(*from
) && len
>= 3)
738 else if (IS_LC2(*from
) && len
>= 3)
745 else if (IS_LCPRV2(*from
) && len
>= 4)
755 *to
= (unsigned char) *from
++;
766 * convert pg_wchar to mule internal code
767 * caller should allocate enough space for "to"
768 * len: length of from.
769 * "from" not necessarily null terminated.
772 pg_wchar2mule_with_len(const pg_wchar
*from
, unsigned char *to
, int len
)
776 while (len
> 0 && *from
)
780 lb
= (*from
>> 16) & 0xff;
784 *to
++ = *from
& 0xff;
790 *to
++ = (*from
>> 8) & 0xff;
791 *to
++ = *from
& 0xff;
794 else if (IS_LCPRV1_A_RANGE(lb
))
798 *to
++ = *from
& 0xff;
801 else if (IS_LCPRV1_B_RANGE(lb
))
805 *to
++ = *from
& 0xff;
808 else if (IS_LCPRV2_A_RANGE(lb
))
812 *to
++ = (*from
>> 8) & 0xff;
813 *to
++ = *from
& 0xff;
816 else if (IS_LCPRV2_B_RANGE(lb
))
820 *to
++ = (*from
>> 8) & 0xff;
821 *to
++ = *from
& 0xff;
826 *to
++ = *from
& 0xff;
837 pg_mule_mblen(const unsigned char *s
)
843 else if (IS_LCPRV1(*s
))
847 else if (IS_LCPRV2(*s
))
850 len
= 1; /* assume ASCII */
855 pg_mule_dsplen(const unsigned char *s
)
860 * Note: it's not really appropriate to assume that all multibyte charsets
861 * are double-wide on screen. But this seems an okay approximation for
862 * the MULE charsets we currently support.
867 else if (IS_LCPRV1(*s
))
871 else if (IS_LCPRV2(*s
))
874 len
= 1; /* assume ASCII */
883 pg_latin12wchar_with_len(const unsigned char *from
, pg_wchar
*to
, int len
)
887 while (len
> 0 && *from
)
898 * Trivial conversion from pg_wchar to single byte encoding. Just ignores
900 * caller should allocate enough space for "to"
901 * len: length of from.
902 * "from" not necessarily null terminated.
905 pg_wchar2single_with_len(const pg_wchar
*from
, unsigned char *to
, int len
)
909 while (len
> 0 && *from
)
920 pg_latin1_mblen(const unsigned char *s
)
926 pg_latin1_dsplen(const unsigned char *s
)
928 return pg_ascii_dsplen(s
);
935 pg_sjis_mblen(const unsigned char *s
)
939 if (*s
>= 0xa1 && *s
<= 0xdf)
940 len
= 1; /* 1 byte kana? */
941 else if (IS_HIGHBIT_SET(*s
))
942 len
= 2; /* kanji? */
944 len
= 1; /* should be ASCII */
949 pg_sjis_dsplen(const unsigned char *s
)
953 if (*s
>= 0xa1 && *s
<= 0xdf)
954 len
= 1; /* 1 byte kana? */
955 else if (IS_HIGHBIT_SET(*s
))
956 len
= 2; /* kanji? */
958 len
= pg_ascii_dsplen(s
); /* should be ASCII */
966 pg_big5_mblen(const unsigned char *s
)
970 if (IS_HIGHBIT_SET(*s
))
971 len
= 2; /* kanji? */
973 len
= 1; /* should be ASCII */
978 pg_big5_dsplen(const unsigned char *s
)
982 if (IS_HIGHBIT_SET(*s
))
983 len
= 2; /* kanji? */
985 len
= pg_ascii_dsplen(s
); /* should be ASCII */
993 pg_gbk_mblen(const unsigned char *s
)
997 if (IS_HIGHBIT_SET(*s
))
998 len
= 2; /* kanji? */
1000 len
= 1; /* should be ASCII */
1005 pg_gbk_dsplen(const unsigned char *s
)
1009 if (IS_HIGHBIT_SET(*s
))
1010 len
= 2; /* kanji? */
1012 len
= pg_ascii_dsplen(s
); /* should be ASCII */
1020 pg_uhc_mblen(const unsigned char *s
)
1024 if (IS_HIGHBIT_SET(*s
))
1025 len
= 2; /* 2byte? */
1027 len
= 1; /* should be ASCII */
1032 pg_uhc_dsplen(const unsigned char *s
)
1036 if (IS_HIGHBIT_SET(*s
))
1037 len
= 2; /* 2byte? */
1039 len
= pg_ascii_dsplen(s
); /* should be ASCII */
1045 * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
1049 * Unlike all other mblen() functions, this also looks at the second byte of
1050 * the input. However, if you only pass the first byte of a multi-byte
1051 * string, and \0 as the second byte, this still works in a predictable way:
1052 * a 4-byte character will be reported as two 2-byte characters. That's
1053 * enough for all current uses, as a client-only encoding. It works that
1054 * way, because in any valid 4-byte GB18030-encoded character, the third and
1055 * fourth byte look like a 2-byte encoded character, when looked at
1059 pg_gb18030_mblen(const unsigned char *s
)
1063 if (!IS_HIGHBIT_SET(*s
))
1064 len
= 1; /* ASCII */
1065 else if (*(s
+ 1) >= 0x30 && *(s
+ 1) <= 0x39)
1073 pg_gb18030_dsplen(const unsigned char *s
)
1077 if (IS_HIGHBIT_SET(*s
))
1080 len
= pg_ascii_dsplen(s
); /* ASCII */
1085 *-------------------------------------------------------------------
1086 * multibyte sequence validators
1088 * These functions accept "s", a pointer to the first byte of a string,
1089 * and "len", the remaining length of the string. If there is a validly
1090 * encoded character beginning at *s, return its length in bytes; else
1093 * The functions can assume that len > 0 and that *s != '\0', but they must
1094 * test for and reject zeroes in any additional bytes of a multibyte character.
1096 * Note that this definition allows the function for a single-byte
1097 * encoding to be just "return 1".
1098 *-------------------------------------------------------------------
1102 pg_ascii_verifier(const unsigned char *s
, int len
)
1107 #define IS_EUC_RANGE_VALID(c) ((c) >= 0xa1 && (c) <= 0xfe)
1110 pg_eucjp_verifier(const unsigned char *s
, int len
)
1120 case SS2
: /* JIS X 0201 */
1125 if (c2
< 0xa1 || c2
> 0xdf)
1129 case SS3
: /* JIS X 0212 */
1134 if (!IS_EUC_RANGE_VALID(c2
))
1137 if (!IS_EUC_RANGE_VALID(c2
))
1142 if (IS_HIGHBIT_SET(c1
)) /* JIS X 0208? */
1147 if (!IS_EUC_RANGE_VALID(c1
))
1150 if (!IS_EUC_RANGE_VALID(c2
))
1165 pg_euckr_verifier(const unsigned char *s
, int len
)
1173 if (IS_HIGHBIT_SET(c1
))
1178 if (!IS_EUC_RANGE_VALID(c1
))
1181 if (!IS_EUC_RANGE_VALID(c2
))
1193 /* EUC-CN byte sequences are exactly same as EUC-KR */
1194 #define pg_euccn_verifier pg_euckr_verifier
1197 pg_euctw_verifier(const unsigned char *s
, int len
)
1207 case SS2
: /* CNS 11643 Plane 1-7 */
1212 if (c2
< 0xa1 || c2
> 0xa7)
1215 if (!IS_EUC_RANGE_VALID(c2
))
1218 if (!IS_EUC_RANGE_VALID(c2
))
1222 case SS3
: /* unused */
1226 if (IS_HIGHBIT_SET(c1
)) /* CNS 11643 Plane 1 */
1231 /* no further range check on c1? */
1233 if (!IS_EUC_RANGE_VALID(c2
))
1247 pg_johab_verifier(const unsigned char *s
, int len
)
1253 l
= mbl
= pg_johab_mblen(s
);
1258 if (!IS_HIGHBIT_SET(*s
))
1264 if (!IS_EUC_RANGE_VALID(c
))
1271 pg_mule_verifier(const unsigned char *s
, int len
)
1277 l
= mbl
= pg_mule_mblen(s
);
1285 if (!IS_HIGHBIT_SET(c
))
1292 pg_latin1_verifier(const unsigned char *s
, int len
)
1298 pg_sjis_verifier(const unsigned char *s
, int len
)
1305 l
= mbl
= pg_sjis_mblen(s
);
1310 if (l
== 1) /* pg_sjis_mblen already verified it */
1315 if (!ISSJISHEAD(c1
) || !ISSJISTAIL(c2
))
1321 pg_big5_verifier(const unsigned char *s
, int len
)
1326 l
= mbl
= pg_big5_mblen(s
);
1341 pg_gbk_verifier(const unsigned char *s
, int len
)
1346 l
= mbl
= pg_gbk_mblen(s
);
1361 pg_uhc_verifier(const unsigned char *s
, int len
)
1366 l
= mbl
= pg_uhc_mblen(s
);
1381 pg_gb18030_verifier(const unsigned char *s
, int len
)
1385 if (!IS_HIGHBIT_SET(*s
))
1387 else if (len
>= 4 && *(s
+ 1) >= 0x30 && *(s
+ 1) <= 0x39)
1389 /* Should be 4-byte, validate remaining bytes */
1390 if (*s
>= 0x81 && *s
<= 0xfe &&
1391 *(s
+ 2) >= 0x81 && *(s
+ 2) <= 0xfe &&
1392 *(s
+ 3) >= 0x30 && *(s
+ 3) <= 0x39)
1397 else if (len
>= 2 && *s
>= 0x81 && *s
<= 0xfe)
1399 /* Should be 2-byte, validate */
1400 if ((*(s
+ 1) >= 0x40 && *(s
+ 1) <= 0x7e) ||
1401 (*(s
+ 1) >= 0x80 && *(s
+ 1) <= 0xfe))
1412 pg_utf8_verifier(const unsigned char *s
, int len
)
1414 int l
= pg_utf_mblen(s
);
1419 if (!pg_utf8_islegal(s
, l
))
1426 * Check for validity of a single UTF-8 encoded character
1428 * This directly implements the rules in RFC3629. The bizarre-looking
1429 * restrictions on the second byte are meant to ensure that there isn't
1430 * more than one encoding of a given Unicode character point; that is,
1431 * you may not use a longer-than-necessary byte sequence with high order
1432 * zero bits to represent a character that would fit in fewer bytes.
1433 * To do otherwise is to create security hazards (eg, create an apparent
1434 * non-ASCII character that decodes to plain ASCII).
1436 * length is assumed to have been obtained by pg_utf_mblen(), and the
1437 * caller must have checked that that many bytes are present in the buffer.
1440 pg_utf8_islegal(const unsigned char *source
, int length
)
1447 /* reject lengths 5 and 6 for now */
1451 if (a
< 0x80 || a
> 0xBF)
1456 if (a
< 0x80 || a
> 0xBF)
1464 if (a
< 0xA0 || a
> 0xBF)
1468 if (a
< 0x80 || a
> 0x9F)
1472 if (a
< 0x90 || a
> 0xBF)
1476 if (a
< 0x80 || a
> 0x8F)
1480 if (a
< 0x80 || a
> 0xBF)
1487 if (a
>= 0x80 && a
< 0xC2)
1499 * Generic character incrementer function.
1501 * Not knowing anything about the properties of the encoding in use, we just
1502 * keep incrementing the last byte until we get a validly-encoded result,
1503 * or we run out of values to try. We don't bother to try incrementing
1504 * higher-order bytes, so there's no growth in runtime for wider characters.
1505 * (If we did try to do that, we'd need to consider the likelihood that 255
1506 * is not a valid final byte in the encoding.)
1509 pg_generic_charinc(unsigned char *charptr
, int len
)
1511 unsigned char *lastbyte
= charptr
+ len
- 1;
1512 mbverifier mbverify
;
1514 /* We can just invoke the character verifier directly. */
1515 mbverify
= pg_wchar_table
[GetDatabaseEncoding()].mbverify
;
1517 while (*lastbyte
< (unsigned char) 255)
1520 if ((*mbverify
) (charptr
, len
) == len
)
1528 * UTF-8 character incrementer function.
1530 * For a one-byte character less than 0x7F, we just increment the byte.
1532 * For a multibyte character, every byte but the first must fall between 0x80
1533 * and 0xBF; and the first byte must be between 0xC0 and 0xF4. We increment
1534 * the last byte that's not already at its maximum value. If we can't find a
1535 * byte that's less than the maximum allowable value, we simply fail. We also
1536 * need some special-case logic to skip regions used for surrogate pair
1537 * handling, as those should not occur in valid UTF-8.
1539 * Note that we don't reset lower-order bytes back to their minimums, since
1540 * we can't afford to make an exhaustive search (see make_greater_string).
1543 pg_utf8_increment(unsigned char *charptr
, int length
)
1546 unsigned char limit
;
1551 /* reject lengths 5 and 6 for now */
1591 if (a
== 0x7F || a
== 0xDF || a
== 0xEF || a
== 0xF4)
1601 * EUC-JP character incrementer function.
1603 * If the sequence starts with SS2 (0x8e), it must be a two-byte sequence
1604 * representing JIS X 0201 characters with the second byte ranging between
1605 * 0xa1 and 0xdf. We just increment the last byte if it's less than 0xdf,
1606 * and otherwise rewrite the whole sequence to 0xa1 0xa1.
1608 * If the sequence starts with SS3 (0x8f), it must be a three-byte sequence
1609 * in which the last two bytes range between 0xa1 and 0xfe. The last byte
1610 * is incremented if possible, otherwise the second-to-last byte.
1612 * If the sequence starts with a value other than the above and its MSB
1613 * is set, it must be a two-byte sequence representing JIS X 0208 characters
1614 * with both bytes ranging between 0xa1 and 0xfe. The last byte is
1615 * incremented if possible, otherwise the second-to-last byte.
1617 * Otherwise, the sequence is a single-byte ASCII character. It is
1618 * incremented up to 0x7f.
1621 pg_eucjp_increment(unsigned char *charptr
, int length
)
1631 case SS2
: /* JIS X 0201 */
1638 charptr
[0] = charptr
[1] = 0xa1;
1645 case SS3
: /* JIS X 0212 */
1649 for (i
= 2; i
> 0; i
--)
1664 /* Out of 3-byte code region */
1668 if (IS_HIGHBIT_SET(c1
)) /* JIS X 0208? */
1673 for (i
= 1; i
>= 0; i
--)
1688 /* Out of 2 byte code region */
1692 { /* ASCII, single byte */
1702 #endif /* !FRONTEND */
1706 *-------------------------------------------------------------------
1707 * encoding info table
1708 * XXX must be sorted by the same order as enum pg_enc (in mb/pg_wchar.h)
1709 *-------------------------------------------------------------------
1711 const pg_wchar_tbl pg_wchar_table
[] = {
1712 {pg_ascii2wchar_with_len
, pg_wchar2single_with_len
, pg_ascii_mblen
, pg_ascii_dsplen
, pg_ascii_verifier
, 1}, /* PG_SQL_ASCII */
1713 {pg_eucjp2wchar_with_len
, pg_wchar2euc_with_len
, pg_eucjp_mblen
, pg_eucjp_dsplen
, pg_eucjp_verifier
, 3}, /* PG_EUC_JP */
1714 {pg_euccn2wchar_with_len
, pg_wchar2euc_with_len
, pg_euccn_mblen
, pg_euccn_dsplen
, pg_euccn_verifier
, 2}, /* PG_EUC_CN */
1715 {pg_euckr2wchar_with_len
, pg_wchar2euc_with_len
, pg_euckr_mblen
, pg_euckr_dsplen
, pg_euckr_verifier
, 3}, /* PG_EUC_KR */
1716 {pg_euctw2wchar_with_len
, pg_wchar2euc_with_len
, pg_euctw_mblen
, pg_euctw_dsplen
, pg_euctw_verifier
, 4}, /* PG_EUC_TW */
1717 {pg_eucjp2wchar_with_len
, pg_wchar2euc_with_len
, pg_eucjp_mblen
, pg_eucjp_dsplen
, pg_eucjp_verifier
, 3}, /* PG_EUC_JIS_2004 */
1718 {pg_utf2wchar_with_len
, pg_wchar2utf_with_len
, pg_utf_mblen
, pg_utf_dsplen
, pg_utf8_verifier
, 4}, /* PG_UTF8 */
1719 {pg_mule2wchar_with_len
, pg_wchar2mule_with_len
, pg_mule_mblen
, pg_mule_dsplen
, pg_mule_verifier
, 4}, /* PG_MULE_INTERNAL */
1720 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN1 */
1721 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN2 */
1722 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN3 */
1723 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN4 */
1724 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN5 */
1725 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN6 */
1726 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN7 */
1727 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN8 */
1728 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN9 */
1729 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_LATIN10 */
1730 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1256 */
1731 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1258 */
1732 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN866 */
1733 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN874 */
1734 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_KOI8R */
1735 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1251 */
1736 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1252 */
1737 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* ISO-8859-5 */
1738 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* ISO-8859-6 */
1739 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* ISO-8859-7 */
1740 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* ISO-8859-8 */
1741 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1250 */
1742 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1253 */
1743 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1254 */
1744 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1255 */
1745 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_WIN1257 */
1746 {pg_latin12wchar_with_len
, pg_wchar2single_with_len
, pg_latin1_mblen
, pg_latin1_dsplen
, pg_latin1_verifier
, 1}, /* PG_KOI8U */
1747 {0, 0, pg_sjis_mblen
, pg_sjis_dsplen
, pg_sjis_verifier
, 2}, /* PG_SJIS */
1748 {0, 0, pg_big5_mblen
, pg_big5_dsplen
, pg_big5_verifier
, 2}, /* PG_BIG5 */
1749 {0, 0, pg_gbk_mblen
, pg_gbk_dsplen
, pg_gbk_verifier
, 2}, /* PG_GBK */
1750 {0, 0, pg_uhc_mblen
, pg_uhc_dsplen
, pg_uhc_verifier
, 2}, /* PG_UHC */
1751 {0, 0, pg_gb18030_mblen
, pg_gb18030_dsplen
, pg_gb18030_verifier
, 4}, /* PG_GB18030 */
1752 {0, 0, pg_johab_mblen
, pg_johab_dsplen
, pg_johab_verifier
, 3}, /* PG_JOHAB */
1753 {0, 0, pg_sjis_mblen
, pg_sjis_dsplen
, pg_sjis_verifier
, 2} /* PG_SHIFT_JIS_2004 */
1756 /* returns the byte length of a word for mule internal code */
1758 pg_mic_mblen(const unsigned char *mbstr
)
1760 return pg_mule_mblen(mbstr
);
1764 * Returns the byte length of a multibyte character.
1767 pg_encoding_mblen(int encoding
, const char *mbstr
)
1769 return (PG_VALID_ENCODING(encoding
) ?
1770 pg_wchar_table
[encoding
].mblen((const unsigned char *) mbstr
) :
1771 pg_wchar_table
[PG_SQL_ASCII
].mblen((const unsigned char *) mbstr
));
1775 * Returns the display length of a multibyte character.
1778 pg_encoding_dsplen(int encoding
, const char *mbstr
)
1780 return (PG_VALID_ENCODING(encoding
) ?
1781 pg_wchar_table
[encoding
].dsplen((const unsigned char *) mbstr
) :
1782 pg_wchar_table
[PG_SQL_ASCII
].dsplen((const unsigned char *) mbstr
));
1786 * Verify the first multibyte character of the given string.
1787 * Return its byte length if good, -1 if bad. (See comments above for
1788 * full details of the mbverify API.)
1791 pg_encoding_verifymb(int encoding
, const char *mbstr
, int len
)
1793 return (PG_VALID_ENCODING(encoding
) ?
1794 pg_wchar_table
[encoding
].mbverify((const unsigned char *) mbstr
, len
) :
1795 pg_wchar_table
[PG_SQL_ASCII
].mbverify((const unsigned char *) mbstr
, len
));
1799 * fetch maximum length of a given encoding
1802 pg_encoding_max_length(int encoding
)
1804 Assert(PG_VALID_ENCODING(encoding
));
1806 return pg_wchar_table
[encoding
].maxmblen
;
1812 * fetch maximum length of the encoding for the current database
1815 pg_database_encoding_max_length(void)
1817 return pg_wchar_table
[GetDatabaseEncoding()].maxmblen
;
1821 * get the character incrementer for the encoding for the current database
1823 mbcharacter_incrementer
1824 pg_database_encoding_character_incrementer(void)
1827 * Eventually it might be best to add a field to pg_wchar_table[], but for
1828 * now we just use a switch.
1830 switch (GetDatabaseEncoding())
1833 return pg_utf8_increment
;
1836 return pg_eucjp_increment
;
1839 return pg_generic_charinc
;
1844 * Verify mbstr to make sure that it is validly encoded in the current
1845 * database encoding. Otherwise same as pg_verify_mbstr().
1848 pg_verifymbstr(const char *mbstr
, int len
, bool noError
)
1851 pg_verify_mbstr_len(GetDatabaseEncoding(), mbstr
, len
, noError
) >= 0;
1855 * Verify mbstr to make sure that it is validly encoded in the specified
1859 pg_verify_mbstr(int encoding
, const char *mbstr
, int len
, bool noError
)
1861 return pg_verify_mbstr_len(encoding
, mbstr
, len
, noError
) >= 0;
1865 * Verify mbstr to make sure that it is validly encoded in the specified
1868 * mbstr is not necessarily zero terminated; length of mbstr is
1871 * If OK, return length of string in the encoding.
1872 * If a problem is found, return -1 when noError is
1873 * true; when noError is false, ereport() a descriptive message.
1876 pg_verify_mbstr_len(int encoding
, const char *mbstr
, int len
, bool noError
)
1878 mbverifier mbverify
;
1881 Assert(PG_VALID_ENCODING(encoding
));
1884 * In single-byte encodings, we need only reject nulls (\0).
1886 if (pg_encoding_max_length(encoding
) <= 1)
1888 const char *nullpos
= memchr(mbstr
, 0, len
);
1890 if (nullpos
== NULL
)
1894 report_invalid_encoding(encoding
, nullpos
, 1);
1897 /* fetch function pointer just once */
1898 mbverify
= pg_wchar_table
[encoding
].mbverify
;
1906 /* fast path for ASCII-subset characters */
1907 if (!IS_HIGHBIT_SET(*mbstr
))
1918 report_invalid_encoding(encoding
, mbstr
, len
);
1921 l
= (*mbverify
) ((const unsigned char *) mbstr
, len
);
1927 report_invalid_encoding(encoding
, mbstr
, len
);
1938 * check_encoding_conversion_args: check arguments of a conversion function
1940 * "expected" arguments can be either an encoding ID or -1 to indicate that
1941 * the caller will check whether it accepts the ID.
1943 * Note: the errors here are not really user-facing, so elog instead of
1944 * ereport seems sufficient. Also, we trust that the "expected" encoding
1945 * arguments are valid encoding IDs, but we don't trust the actuals.
1948 check_encoding_conversion_args(int src_encoding
,
1951 int expected_src_encoding
,
1952 int expected_dest_encoding
)
1954 if (!PG_VALID_ENCODING(src_encoding
))
1955 elog(ERROR
, "invalid source encoding ID: %d", src_encoding
);
1956 if (src_encoding
!= expected_src_encoding
&& expected_src_encoding
>= 0)
1957 elog(ERROR
, "expected source encoding \"%s\", but got \"%s\"",
1958 pg_enc2name_tbl
[expected_src_encoding
].name
,
1959 pg_enc2name_tbl
[src_encoding
].name
);
1960 if (!PG_VALID_ENCODING(dest_encoding
))
1961 elog(ERROR
, "invalid destination encoding ID: %d", dest_encoding
);
1962 if (dest_encoding
!= expected_dest_encoding
&& expected_dest_encoding
>= 0)
1963 elog(ERROR
, "expected destination encoding \"%s\", but got \"%s\"",
1964 pg_enc2name_tbl
[expected_dest_encoding
].name
,
1965 pg_enc2name_tbl
[dest_encoding
].name
);
1967 elog(ERROR
, "encoding conversion length must not be negative");
1971 * report_invalid_encoding: complain about invalid multibyte character
1973 * note: len is remaining length of string, not length of character;
1974 * len must be greater than zero, as we always examine the first byte.
1977 report_invalid_encoding(int encoding
, const char *mbstr
, int len
)
1979 int l
= pg_encoding_mblen(encoding
, mbstr
);
1980 char buf
[8 * 5 + 1];
1985 jlimit
= Min(l
, len
);
1986 jlimit
= Min(jlimit
, 8); /* prevent buffer overrun */
1988 for (j
= 0; j
< jlimit
; j
++)
1990 p
+= sprintf(p
, "0x%02x", (unsigned char) mbstr
[j
]);
1992 p
+= sprintf(p
, " ");
1996 (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE
),
1997 errmsg("invalid byte sequence for encoding \"%s\": %s",
1998 pg_enc2name_tbl
[encoding
].name
,
2003 * report_untranslatable_char: complain about untranslatable character
2005 * note: len is remaining length of string, not length of character;
2006 * len must be greater than zero, as we always examine the first byte.
2009 report_untranslatable_char(int src_encoding
, int dest_encoding
,
2010 const char *mbstr
, int len
)
2012 int l
= pg_encoding_mblen(src_encoding
, mbstr
);
2013 char buf
[8 * 5 + 1];
2018 jlimit
= Min(l
, len
);
2019 jlimit
= Min(jlimit
, 8); /* prevent buffer overrun */
2021 for (j
= 0; j
< jlimit
; j
++)
2023 p
+= sprintf(p
, "0x%02x", (unsigned char) mbstr
[j
]);
2025 p
+= sprintf(p
, " ");
2029 (errcode(ERRCODE_UNTRANSLATABLE_CHARACTER
),
2030 errmsg("character with byte sequence %s in encoding \"%s\" has no equivalent in encoding \"%s\"",
2032 pg_enc2name_tbl
[src_encoding
].name
,
2033 pg_enc2name_tbl
[dest_encoding
].name
)));
2036 #endif /* !FRONTEND */