1 /* CPP Library - charsets
2 Copyright (C) 1998-2024 Free Software Foundation, Inc.
4 Broken out of c-lex.cc Apr 2003, adding valid C99 UCN ranges.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
25 /* Character set handling for C-family languages.
27 Terminological note: In what follows, "charset" or "character set"
28 will be taken to mean both an abstract set of characters and an
29 encoding for that set.
31 The C99 standard discusses two character sets: source and execution.
32 The source character set is used for internal processing in translation
33 phases 1 through 4; the execution character set is used thereafter.
34 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36 of these terms). Furthermore, the "basic character set" (listed in
37 5.2.1p3) is to be encoded in each with values one byte wide, and is
38 to appear in the initial shift state.
40 It is not explicitly mentioned, but there is also a "wide execution
41 character set" used to encode wide character constants and wide
42 string literals; this is supposed to be the result of applying the
43 standard library function mbstowcs() to an equivalent narrow string
44 (6.4.5p5). However, the behavior of hexadecimal and octal
45 \-escapes is at odds with this; they are supposed to be translated
46 directly to wchar_t values (6.4.4.4p5,6).
48 The source character set is not necessarily the character set used
49 to encode physical source files on disk; translation phase 1 converts
50 from whatever that encoding is to the source character set.
52 The presence of universal character names in C99 (6.4.3 et seq.)
53 forces the source character set to be isomorphic to ISO 10646,
54 that is, Unicode. There is no such constraint on the execution
55 character set; note also that the conversion from source to
56 execution character set does not occur for identifiers (5.1.1.2p1#5).
58 For convenience of implementation, the source character set's
59 encoding of the basic character set should be identical to the
60 execution character set OF THE HOST SYSTEM's encoding of the basic
61 character set, and it should not be a state-dependent encoding.
63 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64 depending on whether the host is based on ASCII or EBCDIC (see
65 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66 Technical Report #16). With limited exceptions, it relies on the
67 system library's iconv() primitive to do charset conversion
68 (specified in SUSv2). */
71 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72 below, which are guarded only by if statements with compile-time
73 constant conditions, do not cause link errors. */
74 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76 #define iconv_close(x) (void)0
80 #if HOST_CHARSET == HOST_CHARSET_ASCII
81 #define SOURCE_CHARSET "UTF-8"
82 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84 #define SOURCE_CHARSET "UTF-EBCDIC"
85 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
87 #error "Unrecognized basic host character set"
94 /* This structure is used for a resizable string buffer throughout. */
95 /* Don't call it strbuf, as that conflicts with unistd.h on systems
96 such as DYNIX/ptx where unistd.h includes stropts.h. */
104 /* This is enough to hold any string that fits on a single 80-column
105 line, even if iconv quadruples its size (e.g. conversion from
106 ASCII to UTF-32) rounded up to a power of two. */
107 #define OUTBUF_BLOCK_SIZE 256
109 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110 logic. This is because a depressing number of systems lack iconv,
111 or have have iconv libraries that do not do these conversions, so
112 we need a fallback implementation for them. To ensure the fallback
113 doesn't break due to neglect, it is used on all systems.
115 UTF-32 encoding is nice and simple: a four-byte binary number,
116 constrained to the range 00000000-7FFFFFFF to avoid questions of
117 signedness. We do have to cope with big- and little-endian
120 UTF-16 encoding uses two-byte binary numbers, again in big- and
121 little-endian variants, for all values in the 00000000-0000FFFF
122 range. Values in the 00010000-0010FFFF range are encoded as pairs
123 of two-byte numbers, called "surrogate pairs": given a number S in
124 this range, it is mapped to a pair (H, L) as follows:
126 H = (S - 0x10000) / 0x400 + 0xD800
127 L = (S - 0x10000) % 0x400 + 0xDC00
129 Two-byte values in the D800...DFFF range are ill-formed except as a
130 component of a surrogate pair. Even if the encoding within a
131 two-byte value is little-endian, the H member of the surrogate pair
134 There is no way to encode values in the 00110000-7FFFFFFF range,
135 which is not currently a problem as there are no assigned code
136 points in that range; however, the author expects that it will
137 eventually become necessary to abandon UTF-16 due to this
138 limitation. Note also that, because of these pairs, UTF-16 does
139 not meet the requirements of the C standard for a wide character
140 encoding (see 3.7.3 and 6.4.4.4p11).
142 UTF-8 encoding looks like this:
144 value range encoded as
145 00000000-0000007F 0xxxxxxx
146 00000080-000007FF 110xxxxx 10xxxxxx
147 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
152 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154 never occur. Note also that any value that can be encoded by a
155 given row of the table can also be encoded by all successive rows,
156 but this is not done; only the shortest possible encoding for any
157 given value is valid. For instance, the character 07C0 could be
158 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159 FC 80 80 80 9F 80. Only the first is valid.
161 An implementation note: the transformation from UTF-16 to UTF-8, or
162 vice versa, is easiest done by using UTF-32 as an intermediary. */
164 /* Internal primitives which go from an UTF-8 byte stream to native-endian
165 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166 operation in several places below. */
168 one_utf8_to_cppchar (const uchar
**inbufp
, size_t *inbytesleftp
,
171 static const uchar masks
[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172 static const uchar patns
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
175 const uchar
*inbuf
= *inbufp
;
178 if (*inbytesleftp
< 1)
190 /* The number of leading 1-bits in the first byte indicates how many
192 for (nbytes
= 2; nbytes
< 7; nbytes
++)
193 if ((c
& ~masks
[nbytes
-1]) == patns
[nbytes
-1])
198 if (*inbytesleftp
< nbytes
)
201 c
= (c
& masks
[nbytes
-1]);
203 for (i
= 1; i
< nbytes
; i
++)
205 cppchar_t n
= *inbuf
++;
206 if ((n
& 0xC0) != 0x80)
208 c
= ((c
<< 6) + (n
& 0x3F));
211 /* Make sure the shortest possible encoding was used. */
212 if (c
<= 0x7F && nbytes
> 1) return EILSEQ
;
213 if (c
<= 0x7FF && nbytes
> 2) return EILSEQ
;
214 if (c
<= 0xFFFF && nbytes
> 3) return EILSEQ
;
215 if (c
<= 0x1FFFFF && nbytes
> 4) return EILSEQ
;
216 if (c
<= 0x3FFFFFF && nbytes
> 5) return EILSEQ
;
218 /* Make sure the character is valid. */
219 if (c
> 0x7FFFFFFF || (c
>= 0xD800 && c
<= 0xDFFF)) return EILSEQ
;
223 *inbytesleftp
-= nbytes
;
228 one_cppchar_to_utf8 (cppchar_t c
, uchar
**outbufp
, size_t *outbytesleftp
)
230 static const uchar masks
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231 static const uchar limits
[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
233 uchar buf
[6], *p
= &buf
[6];
234 uchar
*outbuf
= *outbufp
;
243 *--p
= ((c
& 0x3F) | 0x80);
247 while (c
>= 0x3F || (c
& limits
[nbytes
-1]));
248 *--p
= (c
| masks
[nbytes
-1]);
251 if (*outbytesleftp
< nbytes
)
256 *outbytesleftp
-= nbytes
;
261 /* The following four functions transform one character between the two
262 encodings named in the function name. All have the signature
263 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264 uchar **outbufp, size_t *outbytesleftp)
266 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267 interpreted as a boolean indicating whether big-endian or
268 little-endian encoding is to be used for the member of the pair
271 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
274 The return value is either 0 for success, or an errno value for
275 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276 input sequence), ir EINVAL (incomplete input sequence). */
279 one_utf8_to_utf32 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
280 uchar
**outbufp
, size_t *outbytesleftp
)
286 /* Check for space first, since we know exactly how much we need. */
287 if (*outbytesleftp
< 4)
290 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
295 outbuf
[bigend
? 3 : 0] = (s
& 0x000000FF);
296 outbuf
[bigend
? 2 : 1] = (s
& 0x0000FF00) >> 8;
297 outbuf
[bigend
? 1 : 2] = (s
& 0x00FF0000) >> 16;
298 outbuf
[bigend
? 0 : 3] = (s
& 0xFF000000) >> 24;
306 one_utf32_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
307 uchar
**outbufp
, size_t *outbytesleftp
)
313 if (*inbytesleftp
< 4)
318 s
= inbuf
[bigend
? 0 : 3] << 24;
319 s
+= inbuf
[bigend
? 1 : 2] << 16;
320 s
+= inbuf
[bigend
? 2 : 1] << 8;
321 s
+= inbuf
[bigend
? 3 : 0];
323 if (s
>= 0x7FFFFFFF || (s
>= 0xD800 && s
<= 0xDFFF))
326 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
336 one_utf8_to_utf16 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
337 uchar
**outbufp
, size_t *outbytesleftp
)
341 const uchar
*save_inbuf
= *inbufp
;
342 size_t save_inbytesleft
= *inbytesleftp
;
343 uchar
*outbuf
= *outbufp
;
345 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
351 *inbufp
= save_inbuf
;
352 *inbytesleftp
= save_inbytesleft
;
358 if (*outbytesleftp
< 2)
360 *inbufp
= save_inbuf
;
361 *inbytesleftp
= save_inbytesleft
;
364 outbuf
[bigend
? 1 : 0] = (s
& 0x00FF);
365 outbuf
[bigend
? 0 : 1] = (s
& 0xFF00) >> 8;
375 if (*outbytesleftp
< 4)
377 *inbufp
= save_inbuf
;
378 *inbytesleftp
= save_inbytesleft
;
382 hi
= (s
- 0x10000) / 0x400 + 0xD800;
383 lo
= (s
- 0x10000) % 0x400 + 0xDC00;
385 /* Even if we are little-endian, put the high surrogate first.
386 ??? Matches practice? */
387 outbuf
[bigend
? 1 : 0] = (hi
& 0x00FF);
388 outbuf
[bigend
? 0 : 1] = (hi
& 0xFF00) >> 8;
389 outbuf
[bigend
? 3 : 2] = (lo
& 0x00FF);
390 outbuf
[bigend
? 2 : 3] = (lo
& 0xFF00) >> 8;
399 one_utf16_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
400 uchar
**outbufp
, size_t *outbytesleftp
)
403 const uchar
*inbuf
= *inbufp
;
406 if (*inbytesleftp
< 2)
408 s
= inbuf
[bigend
? 0 : 1] << 8;
409 s
+= inbuf
[bigend
? 1 : 0];
411 /* Low surrogate without immediately preceding high surrogate is invalid. */
412 if (s
>= 0xDC00 && s
<= 0xDFFF)
414 /* High surrogate must have a following low surrogate. */
415 else if (s
>= 0xD800 && s
<= 0xDBFF)
417 cppchar_t hi
= s
, lo
;
418 if (*inbytesleftp
< 4)
421 lo
= inbuf
[bigend
? 2 : 3] << 8;
422 lo
+= inbuf
[bigend
? 3 : 2];
424 if (lo
< 0xDC00 || lo
> 0xDFFF)
427 s
= (hi
- 0xD800) * 0x400 + (lo
- 0xDC00) + 0x10000;
430 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
434 /* Success - update the input pointers (one_cppchar_to_utf8 has done
435 the output pointers for us). */
450 /* Special routine which just counts number of characters in the
451 string, what exactly is stored into the output doesn't matter
452 as long as it is one uchar per character. */
455 one_count_chars (iconv_t
, const uchar
**inbufp
, size_t *inbytesleftp
,
456 uchar
**outbufp
, size_t *outbytesleftp
)
461 /* Check for space first, since we know exactly how much we need. */
462 if (*outbytesleftp
< 1)
465 #if HOST_CHARSET == HOST_CHARSET_ASCII
466 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
470 if (*inbytesleftp
< 1)
472 static const uchar utf_ebcdic_map
[256] = {
473 /* See table 4 in http://unicode.org/reports/tr16/tr16-7.2.html */
474 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
475 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
476 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
477 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
478 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1, 1, 1, 1, 1,
479 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1, 1, 1, 1, 1, 1,
480 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1, 1, 1, 1, 1,
481 9, 9, 9, 9, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
482 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
483 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,
484 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 2, 2,
485 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 1, 3, 3,
486 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3,
487 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 4, 4, 4, 4,
488 1, 4, 1, 1, 1, 1, 1, 1, 1, 1, 4, 4, 4, 5, 5, 5,
489 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 6, 6, 7, 7, 0
491 rval
= utf_ebcdic_map
[**inbufp
];
498 if (*inbytesleftp
< rval
)
500 for (int i
= 1; i
< rval
; ++i
)
501 if (utf_ebcdic_map
[(*inbufp
)[i
]] != 9)
504 *inbytesleftp
-= rval
;
516 /* Helper routine for the next few functions. The 'const' on
517 one_conversion means that we promise not to modify what function is
518 pointed to, which lets the inliner see through it. */
521 conversion_loop (int (*const one_conversion
)(iconv_t
, const uchar
**, size_t *,
523 iconv_t cd
, const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
527 size_t inbytesleft
, outbytesleft
;
532 outbuf
= to
->text
+ to
->len
;
533 outbytesleft
= to
->asize
- to
->len
;
538 rval
= one_conversion (cd
, &inbuf
, &inbytesleft
,
539 &outbuf
, &outbytesleft
);
540 while (inbytesleft
&& !rval
);
542 if (__builtin_expect (inbytesleft
== 0, 1))
544 to
->len
= to
->asize
- outbytesleft
;
553 outbytesleft
+= OUTBUF_BLOCK_SIZE
;
554 to
->asize
+= OUTBUF_BLOCK_SIZE
;
555 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
556 outbuf
= to
->text
+ to
->asize
- outbytesleft
;
561 /* These functions convert entire strings between character sets.
562 They all have the signature
564 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
566 The input string FROM is converted as specified by the function
567 name plus the iconv descriptor CD (which may be fake), and the
568 result appended to TO. On any error, false is returned, otherwise true. */
570 /* These four use the custom conversion code above. */
572 convert_utf8_utf16 (iconv_t cd
, const uchar
*from
, size_t flen
,
573 struct _cpp_strbuf
*to
)
575 return conversion_loop (one_utf8_to_utf16
, cd
, from
, flen
, to
);
579 convert_utf8_utf32 (iconv_t cd
, const uchar
*from
, size_t flen
,
580 struct _cpp_strbuf
*to
)
582 return conversion_loop (one_utf8_to_utf32
, cd
, from
, flen
, to
);
586 convert_utf16_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
587 struct _cpp_strbuf
*to
)
589 return conversion_loop (one_utf16_to_utf8
, cd
, from
, flen
, to
);
593 convert_utf32_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
594 struct _cpp_strbuf
*to
)
596 return conversion_loop (one_utf32_to_utf8
, cd
, from
, flen
, to
);
599 /* Magic conversion which just counts characters from input, so
600 only to->len is significant. */
602 convert_count_chars (iconv_t cd
, const uchar
*from
,
603 size_t flen
, struct _cpp_strbuf
*to
)
605 return conversion_loop (one_count_chars
, cd
, from
, flen
, to
);
608 /* Identity conversion, used when we have no alternative. */
610 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED
,
611 const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
613 if (to
->len
+ flen
> to
->asize
)
615 to
->asize
= to
->len
+ flen
;
616 to
->asize
+= to
->asize
/ 4;
617 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
619 memcpy (to
->text
+ to
->len
, from
, flen
);
624 /* And this one uses the system iconv primitive. It's a little
625 different, since iconv's interface is a little different. */
628 #define CONVERT_ICONV_GROW_BUFFER \
630 outbytesleft += OUTBUF_BLOCK_SIZE; \
631 to->asize += OUTBUF_BLOCK_SIZE; \
632 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
633 outbuf = (char *)to->text + to->asize - outbytesleft; \
637 convert_using_iconv (iconv_t cd
, const uchar
*from
, size_t flen
,
638 struct _cpp_strbuf
*to
)
640 ICONV_CONST
char *inbuf
;
642 size_t inbytesleft
, outbytesleft
;
644 /* Reset conversion descriptor and check that it is valid. */
645 if (iconv (cd
, 0, 0, 0, 0) == (size_t)-1)
648 inbuf
= (ICONV_CONST
char *)from
;
650 outbuf
= (char *)to
->text
+ to
->len
;
651 outbytesleft
= to
->asize
- to
->len
;
655 iconv (cd
, &inbuf
, &inbytesleft
, &outbuf
, &outbytesleft
);
656 if (__builtin_expect (inbytesleft
== 0, 1))
658 /* Close out any shift states, returning to the initial state. */
659 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
664 CONVERT_ICONV_GROW_BUFFER
;
665 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
669 to
->len
= to
->asize
- outbytesleft
;
675 CONVERT_ICONV_GROW_BUFFER
;
679 #define convert_using_iconv 0 /* prevent undefined symbol error below */
682 /* Arrange for the above custom conversion logic to be used automatically
683 when conversion between a suitable pair of character sets is requested. */
685 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
686 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
688 struct cpp_conversion
694 static const struct cpp_conversion conversion_tab
[] = {
695 { "UTF-8/UTF-32LE", convert_utf8_utf32
, (iconv_t
)0 },
696 { "UTF-8/UTF-32BE", convert_utf8_utf32
, (iconv_t
)1 },
697 { "UTF-8/UTF-16LE", convert_utf8_utf16
, (iconv_t
)0 },
698 { "UTF-8/UTF-16BE", convert_utf8_utf16
, (iconv_t
)1 },
699 { "UTF-32LE/UTF-8", convert_utf32_utf8
, (iconv_t
)0 },
700 { "UTF-32BE/UTF-8", convert_utf32_utf8
, (iconv_t
)1 },
701 { "UTF-16LE/UTF-8", convert_utf16_utf8
, (iconv_t
)0 },
702 { "UTF-16BE/UTF-8", convert_utf16_utf8
, (iconv_t
)1 },
705 /* Subroutine of cpp_init_iconv: initialize and return a
706 cset_converter structure for conversion from FROM to TO. If
707 iconv_open() fails, issue an error and return an identity
708 converter. Silently return an identity converter if FROM and TO
711 PFILE is only used for generating diagnostics; setting it to NULL
712 suppresses diagnostics. */
714 static struct cset_converter
715 init_iconv_desc (cpp_reader
*pfile
, const char *to
, const char *from
)
717 struct cset_converter ret
;
724 if (!strcasecmp (to
, from
))
726 ret
.func
= convert_no_conversion
;
727 ret
.cd
= (iconv_t
) -1;
732 pair
= (char *) alloca(strlen(to
) + strlen(from
) + 2);
737 for (i
= 0; i
< ARRAY_SIZE (conversion_tab
); i
++)
738 if (!strcasecmp (pair
, conversion_tab
[i
].pair
))
740 ret
.func
= conversion_tab
[i
].func
;
741 ret
.cd
= conversion_tab
[i
].fake_cd
;
746 /* No custom converter - try iconv. */
749 ret
.func
= convert_using_iconv
;
750 ret
.cd
= iconv_open (to
, from
);
753 if (ret
.cd
== (iconv_t
) -1)
758 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME should be DL_SORRY */
759 "conversion from %s to %s not supported by iconv",
762 cpp_errno (pfile
, CPP_DL_ERROR
, "iconv_open");
764 ret
.func
= convert_no_conversion
;
771 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME: should be DL_SORRY */
772 "no iconv implementation, cannot convert from %s to %s",
775 ret
.func
= convert_no_conversion
;
776 ret
.cd
= (iconv_t
) -1;
783 /* If charset conversion is requested, initialize iconv(3) descriptors
784 for conversion from the source character set to the execution
785 character sets. If iconv is not present in the C library, and
786 conversion is requested, issue an error. */
789 cpp_init_iconv (cpp_reader
*pfile
)
791 const char *ncset
= CPP_OPTION (pfile
, narrow_charset
);
792 const char *wcset
= CPP_OPTION (pfile
, wide_charset
);
793 const char *default_wcset
;
795 bool be
= CPP_OPTION (pfile
, bytes_big_endian
);
797 if (CPP_OPTION (pfile
, wchar_precision
) >= 32)
798 default_wcset
= be
? "UTF-32BE" : "UTF-32LE";
799 else if (CPP_OPTION (pfile
, wchar_precision
) >= 16)
800 default_wcset
= be
? "UTF-16BE" : "UTF-16LE";
802 /* This effectively means that wide strings are not supported,
803 so don't do any conversion at all. */
804 default_wcset
= SOURCE_CHARSET
;
807 ncset
= SOURCE_CHARSET
;
809 wcset
= default_wcset
;
811 pfile
->narrow_cset_desc
= init_iconv_desc (pfile
, ncset
, SOURCE_CHARSET
);
812 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
813 pfile
->utf8_cset_desc
= init_iconv_desc (pfile
, "UTF-8", SOURCE_CHARSET
);
814 pfile
->utf8_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
815 pfile
->char16_cset_desc
= init_iconv_desc (pfile
,
816 be
? "UTF-16BE" : "UTF-16LE",
818 pfile
->char16_cset_desc
.width
= 16;
819 pfile
->char32_cset_desc
= init_iconv_desc (pfile
,
820 be
? "UTF-32BE" : "UTF-32LE",
822 pfile
->char32_cset_desc
.width
= 32;
823 pfile
->wide_cset_desc
= init_iconv_desc (pfile
, wcset
, SOURCE_CHARSET
);
824 pfile
->wide_cset_desc
.width
= CPP_OPTION (pfile
, wchar_precision
);
827 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
829 _cpp_destroy_iconv (cpp_reader
*pfile
)
833 if (pfile
->narrow_cset_desc
.func
== convert_using_iconv
)
834 iconv_close (pfile
->narrow_cset_desc
.cd
);
835 if (pfile
->utf8_cset_desc
.func
== convert_using_iconv
)
836 iconv_close (pfile
->utf8_cset_desc
.cd
);
837 if (pfile
->char16_cset_desc
.func
== convert_using_iconv
)
838 iconv_close (pfile
->char16_cset_desc
.cd
);
839 if (pfile
->char32_cset_desc
.func
== convert_using_iconv
)
840 iconv_close (pfile
->char32_cset_desc
.cd
);
841 if (pfile
->wide_cset_desc
.func
== convert_using_iconv
)
842 iconv_close (pfile
->wide_cset_desc
.cd
);
846 /* Utility routine for use by a full compiler. C is a character taken
847 from the *basic* source character set, encoded in the host's
848 execution encoding. Convert it to (the target's) execution
849 encoding, and return that value.
851 Issues an internal error if C's representation in the narrow
852 execution character set fails to be a single-byte value (C99
853 5.2.1p3: "The representation of each member of the source and
854 execution character sets shall fit in a byte.") May also issue an
855 internal error if C fails to be a member of the basic source
856 character set (testing this exactly is too hard, especially when
857 the host character set is EBCDIC). */
859 cpp_host_to_exec_charset (cpp_reader
*pfile
, cppchar_t c
)
862 struct _cpp_strbuf tbuf
;
864 /* This test is merely an approximation, but it suffices to catch
865 the most important thing, which is that we don't get handed a
866 character outside the unibyte range of the host character set. */
867 if (c
> LAST_POSSIBLY_BASIC_SOURCE_CHAR
)
869 cpp_error (pfile
, CPP_DL_ICE
,
870 "character 0x%lx is not in the basic source character set\n",
875 /* Being a character in the unibyte range of the host character set,
876 we can safely splat it into a one-byte buffer and trust that that
877 is a well-formed string. */
880 /* This should never need to reallocate, but just in case... */
882 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
885 if (!APPLY_CONVERSION (pfile
->narrow_cset_desc
, sbuf
, 1, &tbuf
))
887 cpp_errno (pfile
, CPP_DL_ICE
, "converting to execution character set");
892 cpp_error (pfile
, CPP_DL_ICE
,
893 "character 0x%lx is not unibyte in execution character set",
904 /* cpp_substring_ranges's constructor. */
906 cpp_substring_ranges::cpp_substring_ranges () :
911 m_ranges
= XNEWVEC (source_range
, m_alloc_ranges
);
914 /* cpp_substring_ranges's destructor. */
916 cpp_substring_ranges::~cpp_substring_ranges ()
921 /* Add RANGE to the vector of source_range information. */
924 cpp_substring_ranges::add_range (source_range range
)
926 if (m_num_ranges
>= m_alloc_ranges
)
930 = (source_range
*)xrealloc (m_ranges
,
931 sizeof (source_range
) * m_alloc_ranges
);
933 m_ranges
[m_num_ranges
++] = range
;
936 /* Read NUM ranges from LOC_READER, adding them to the vector of source_range
940 cpp_substring_ranges::add_n_ranges (int num
,
941 cpp_string_location_reader
&loc_reader
)
943 for (int i
= 0; i
< num
; i
++)
944 add_range (loc_reader
.get_next ());
949 /* Utility routine that computes a mask of the form 0000...111... with
952 width_to_mask (size_t width
)
954 width
= MIN (width
, BITS_PER_CPPCHAR_T
);
955 if (width
>= CHAR_BIT
* sizeof (size_t))
958 return ((size_t) 1 << width
) - 1;
961 /* A large table of unicode character information. */
963 /* Valid in a C99 identifier? */
965 /* Valid in a C99 identifier, but not as the first character? */
967 /* Valid in a C++ identifier? */
969 /* Valid in a C11/C++11 identifier? */
971 /* Valid in a C11/C++11 identifier, but not as the first character? */
973 /* Valid in a C++23 identifier? */
975 /* Valid in a C++23 identifier, but not as the first character? */
977 /* NFC representation is not valid in an identifier? */
979 /* Might be valid NFC form? */
981 /* Might be valid NFKC form? */
983 /* Certain preceding characters might make it not valid NFC/NKFC form? */
988 /* Bitmap of flags above. */
989 unsigned short flags
;
990 /* Combining class of the character. */
991 unsigned char combine
;
992 /* Last character in the range described by this entry. */
997 /* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */
998 #define UCS_LIMIT 0x10FFFF
1000 #include "uname2c.h"
1002 static const char hangul_syllables
[][4] = {
1004 "G", "GG", "N", "D", "DD", "R", "M", "B", "BB", "S", "SS", "",
1005 "J", "JJ", "C", "K", "T", "P", "H",
1007 "A", "AE", "YA", "YAE", "EO", "E", "YEO", "YE", "O", "WA", "WAE",
1008 "OE", "YO", "U", "WEO", "WE", "WI", "YU", "EU", "YI", "I",
1010 "", "G", "GG", "GS", "N", "NJ", "NH", "D", "L", "LG", "LM", "LB",
1011 "LS", "LT", "LP", "LH", "M", "B", "BS", "S", "SS", "NG", "J", "C",
1015 static const short hangul_count
[6] = { 19, 21, 28 };
1017 /* Used for Unicode loose matching rule UAX44-LM2 matching. */
1025 /* Map NAME, a Unicode character name or correction/control/alternate
1026 alias, to a Unicode codepoint, or return (cppchar_t) -1 if
1027 not found. This uses a space optimized radix tree precomputed
1028 by the makeuname2c utility, with binary format documented in its
1029 source makeuname2c.cc. */
1032 _cpp_uname2c (const char *name
, size_t len
, const unsigned char *n
,
1033 struct uname2c_data
*data
)
1039 size_t key_len
, len_adj
;
1040 bool has_value
= *n
& 0x40;
1041 bool has_children
, no_sibling
= false;
1042 cppchar_t codepoint
= -1;
1043 const unsigned char *child
= NULL
;
1048 k
= ' ' + (*n
++ & 0x3f);
1054 key_len
= *n
++ & 0x3f;
1055 key
= &uname2c_dict
[*n
++];
1060 codepoint
= *n
+ (n
[1] << 8) + ((n
[2] & 0x1f) << 16);
1061 has_children
= n
[2] & 0x80;
1062 no_sibling
= n
[2] & 0x40;
1066 has_children
= true;
1069 unsigned int shift
= 0;
1070 size_t child_off
= 0;
1074 child_off
|= (*n
& 0x7f) << shift
;
1077 while ((*n
++ & 0x80) != 0);
1078 child
= n
+ child_off
;
1080 if (__builtin_expect (data
== NULL
, 1))
1082 ret
= memcmp (name
, key
, len
> key_len
? key_len
: len
);
1087 const char *p
= name
, *q
= key
;
1091 if ((size_t) (p
- name
) == len
|| (size_t) (q
- key
) == key_len
)
1100 /* This is the hard case. Only medial hyphens
1101 should be removed, where medial means preceded
1102 and followed by alnum. */
1103 if (ISALNUM (q
== key
? data
->prev_char
: q
[-1]))
1105 if (q
+ 1 == key
+ key_len
)
1107 /* We don't know what the next letter will be.
1108 It could be ISALNUM, then we are supposed
1109 to omit it, or it could be a space and then
1110 we should not omit it and need to compare it.
1111 Fortunately the only 3 names with hyphen
1112 followed by non-letter are
1113 U+0F0A TIBETAN MARK BKA- SHOG YIG MGO
1114 U+0FD0 TIBETAN MARK BKA- SHOG GI MGO RGYAN
1115 U+0FD0 TIBETAN MARK BSKA- SHOG GI MGO RGYAN
1116 Furthermore, prefixes of NR2 generated
1117 ranges all end with a hyphen, but the generated
1118 part is then followed by alpha-numeric.
1119 So, let's just assume that - at the end of
1120 key is always followed by alphanumeric and
1121 so should be omitted.
1122 makeuname2c.cc verifies that this is true. */
1126 else if (ISALNUM (q
[1]))
1139 /* If we don't consume the whole key, signal a mismatch,
1140 but always with ret = 1, so that we keep looking through
1142 ret
= q
< key
+ key_len
;
1150 else if (codepoint
>= 0xd800
1151 && codepoint
< 0xd800 + ARRAY_SIZE (uname2c_generated
))
1155 if (codepoint
== 0xd800)
1157 /* NR1 - Hangul syllables. */
1158 size_t start
= 0, end
, i
, j
;
1159 int this_len
, max_len
;
1162 for (i
= 0; i
< 3; ++i
)
1164 end
= start
+ hangul_count
[i
];
1167 for (j
= start
; j
< end
; j
++)
1169 this_len
= strlen (hangul_syllables
[j
]);
1170 if (len
>= (size_t) this_len
1171 && this_len
> max_len
1172 && memcmp (name
, hangul_syllables
[j
],
1176 winner
[i
] = j
- start
;
1185 if (__builtin_expect (data
!= NULL
, 0))
1187 memcpy (data
->canon_name
, key
, key_len
);
1188 data
->canon_name
[key_len
] = '\0';
1189 for (i
= 0, start
= 0; i
< 3; ++i
)
1191 strcat (data
->canon_name
,
1192 hangul_syllables
[start
+ winner
[i
]]);
1193 start
+= hangul_count
[i
];
1196 return (0xac00 + 21 * 28 * winner
[0]
1197 + 28 * winner
[1] + winner
[2]);
1201 /* NR2 - prefix followed by hexadecimal codepoint. */
1205 if (len
< 4 || len
> 5)
1207 p
= uname2c_pairs
+ uname2c_generated
[codepoint
- 0xd800];
1209 for (i
= 0; i
< len
; ++i
)
1212 if (!ISXDIGIT (name
[i
]))
1214 codepoint
+= hex_value (name
[i
]);
1219 else if (codepoint
<= p
[1])
1221 if (__builtin_expect (data
!= NULL
, 0))
1223 memcpy (data
->canon_name
, key
, key_len
);
1224 memcpy (data
->canon_name
+ key_len
, name
, len
);
1225 data
->canon_name
[key_len
+ len
] = '\0';
1232 else if (__builtin_expect (data
!= NULL
, 0))
1236 memcpy (data
->canon_name
, key
, key_len
);
1237 data
->canon_name
[key_len
] = '\0';
1242 struct uname2c_data save
= *data
;
1243 memcpy (data
->canon_name
, key
, key_len
);
1244 data
->canon_name
+= key_len
;
1245 data
->prev_char
= key
[key_len
- 1];
1246 codepoint
= _cpp_uname2c (name
+ len_adj
, len
- len_adj
,
1248 if (codepoint
!= (cppchar_t
) -1)
1253 else if (len
== len_adj
)
1255 else if (!has_children
)
1265 if (no_sibling
|| (!has_value
&& *n
== 0xff))
1272 /* Try to do a loose name lookup according to Unicode loose matching rule
1273 UAX44-LM2. First ignore medial hyphens, whitespace, underscore
1274 characters and convert to upper case. */
1277 _cpp_uname2c_uax44_lm2 (const char *name
, size_t len
, char *canon_name
)
1279 char name_after_uax44_lm2
[uname2c_max_name_len
];
1280 char *q
= name_after_uax44_lm2
;
1283 for (p
= name
; p
< name
+ len
; p
++)
1284 if (*p
== '_' || *p
== ' ')
1286 else if (*p
== '-' && p
!= name
&& ISALNUM (p
[-1]) && ISALNUM (p
[1]))
1288 else if (q
== name_after_uax44_lm2
+ uname2c_max_name_len
)
1290 else if (ISLOWER (*p
))
1291 *q
++ = TOUPPER (*p
);
1295 struct uname2c_data data
;
1296 data
.canon_name
= canon_name
;
1297 data
.prev_char
= ' ';
1298 /* Hangul Jungseong O- E after UAX44-LM2 should be HANGULJUNGSEONGO-E
1299 and so should match U+1180. */
1300 if (q
- name_after_uax44_lm2
== sizeof ("HANGULJUNGSEONGO-E") - 1
1301 && memcmp (name_after_uax44_lm2
, "HANGULJUNGSEONGO-E",
1302 sizeof ("HANGULJUNGSEONGO-E") - 1) == 0)
1304 name_after_uax44_lm2
[sizeof ("HANGULJUNGSEONGO") - 1] = 'E';
1308 = _cpp_uname2c (name_after_uax44_lm2
, q
- name_after_uax44_lm2
,
1309 uname2c_tree
, &data
);
1311 /* Unicode UAX44-LM2 exception:
1312 U+116C HANGUL JUNGSEONG OE
1313 U+1180 HANGUL JUNGSEONG O-E
1314 We remove all medial hyphens when we shouldn't remote the U+1180 one.
1315 The U+1180 entry sorts before U+116C lexicographilly, so we get U+1180
1316 in both cases. Thus, if result is U+1180, check if user's name doesn't
1317 have a hyphen there and adjust. */
1318 if (result
== 0x1180)
1320 while (p
[-1] == ' ' || p
[-1] == '_')
1322 gcc_assert (TOUPPER (p
[-1]) == 'E');
1324 while (p
[-1] == ' ' || p
[-1] == '_')
1329 memcpy (canon_name
+ sizeof ("HANGUL JUNGSEONG O") - 1, "E", 2);
1335 /* Returns flags representing the XID properties of the given codepoint. */
1337 cpp_check_xid_property (cppchar_t c
)
1339 // fast path for ASCII
1342 if (('A' <= c
&& c
<= 'Z') || ('a' <= c
&& c
<= 'z'))
1343 return CPP_XID_START
| CPP_XID_CONTINUE
;
1344 if (('0' <= c
&& c
<= '9') || c
== '_')
1345 return CPP_XID_CONTINUE
;
1353 mx
= ARRAY_SIZE (ucnranges
) - 1;
1357 if (c
<= ucnranges
[md
].end
)
1363 unsigned short flags
= ucnranges
[mn
].flags
;
1366 return CPP_XID_START
| CPP_XID_CONTINUE
;
1368 return CPP_XID_CONTINUE
;
1372 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
1373 the start of an identifier, and 0 if C is not valid in an
1374 identifier. We assume C has already gone through the checks of
1375 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
1376 algorithm is a simple binary search on the table defined in
1380 ucn_valid_in_identifier (cpp_reader
*pfile
, cppchar_t c
,
1381 struct normalize_state
*nst
)
1384 unsigned short valid_flags
, invalid_start_flags
;
1390 mx
= ARRAY_SIZE (ucnranges
) - 1;
1394 if (c
<= ucnranges
[md
].end
)
1400 /* When -pedantic, we require the character to have been listed by
1401 the standard for the current language. Otherwise, we accept the
1402 union of the acceptable sets for all supported language versions. */
1403 valid_flags
= C99
| CXX
| C11
| CXX23
;
1404 if (CPP_PEDANTIC (pfile
))
1406 if (CPP_OPTION (pfile
, xid_identifiers
))
1407 valid_flags
= CXX23
;
1408 else if (CPP_OPTION (pfile
, c11_identifiers
))
1410 else if (CPP_OPTION (pfile
, c99
))
1413 if (! (ucnranges
[mn
].flags
& valid_flags
))
1417 if (ucnranges
[mn
].combine
!= 0 && ucnranges
[mn
].combine
< nst
->prev_class
)
1418 nst
->level
= normalized_none
;
1419 else if (ucnranges
[mn
].flags
& CTX
)
1422 cppchar_t p
= nst
->previous
;
1424 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
1425 and are combined algorithmically from a sequence of the form
1426 1100-1112 1161-1175 11A8-11C2
1427 (if the third is not present, it is treated as 11A7, which is not
1428 really a valid character).
1429 Unfortunately, C99 allows (only) the NFC form, but C++ allows
1430 only the combining characters. */
1431 if (c
>= 0x1161 && c
<= 0x1175)
1432 safe
= p
< 0x1100 || p
> 0x1112;
1433 else if (c
>= 0x11A8 && c
<= 0x11C2)
1434 safe
= (p
< 0xAC00 || p
> 0xD7A3 || (p
- 0xAC00) % 28 != 0);
1436 safe
= check_nfc (pfile
, c
, p
);
1439 if ((c
>= 0x1161 && c
<= 0x1175) || (c
>= 0x11A8 && c
<= 0x11C2))
1440 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
1442 nst
->level
= normalized_none
;
1445 else if (ucnranges
[mn
].flags
& NKC
)
1447 else if (ucnranges
[mn
].flags
& NFC
)
1448 nst
->level
= MAX (nst
->level
, normalized_C
);
1449 else if (ucnranges
[mn
].flags
& CID
)
1450 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
1452 nst
->level
= normalized_none
;
1453 if (ucnranges
[mn
].combine
== 0)
1455 nst
->prev_class
= ucnranges
[mn
].combine
;
1457 if (!CPP_PEDANTIC (pfile
))
1459 /* If not -pedantic, accept as character that may
1460 begin an identifier a union of characters allowed
1461 at that position in each of the character sets. */
1462 if ((ucnranges
[mn
].flags
& (C99
| N99
)) == C99
1463 || (ucnranges
[mn
].flags
& CXX
) != 0
1464 || (ucnranges
[mn
].flags
& (C11
| N11
)) == C11
1465 || (ucnranges
[mn
].flags
& (CXX23
| NXX23
)) == CXX23
)
1470 if (CPP_OPTION (pfile
, xid_identifiers
))
1471 invalid_start_flags
= NXX23
;
1472 else if (CPP_OPTION (pfile
, c11_identifiers
))
1473 invalid_start_flags
= N11
;
1474 else if (CPP_OPTION (pfile
, c99
))
1475 invalid_start_flags
= N99
;
1477 invalid_start_flags
= 0;
1479 /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
1480 UCN combining characters may not begin identifiers. */
1481 if (ucnranges
[mn
].flags
& invalid_start_flags
)
1487 /* Increment char_range->m_finish by a single character. */
1490 extend_char_range (source_range
*char_range
,
1491 cpp_string_location_reader
*loc_reader
)
1495 gcc_assert (char_range
);
1496 char_range
->m_finish
= loc_reader
->get_next ().m_finish
;
1500 /* [lex.charset]: The character designated by the universal character
1501 name \UNNNNNNNN is that character whose character short name in
1502 ISO/IEC 10646 is NNNNNNNN; the character designated by the
1503 universal character name \uNNNN is that character whose character
1504 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
1505 for a universal character name corresponds to a surrogate code point
1506 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
1507 Additionally, if the hexadecimal value for a universal-character-name
1508 outside a character or string literal corresponds to a control character
1509 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
1510 character in the basic source character set, the program is ill-formed.
1512 C99 6.4.3: A universal character name shall not specify a character
1513 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
1514 or 0060 (`), nor one in the range D800 through DFFF inclusive.
1516 If the hexadecimal value is larger than the upper bound of the UCS
1517 codespace specified in ISO/IEC 10646, a pedantic warning is issued
1518 in all versions of C and in the C++20 or later versions of C++.
1520 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
1521 buffer end is delimited by a non-hex digit. Returns false if the
1522 UCN has not been consumed, true otherwise.
1524 The value of the UCN, whether valid or invalid, is returned in *CP.
1525 Diagnostics are emitted for invalid values. PSTR is updated to point
1526 one beyond the UCN, or to the syntactically invalid character.
1528 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
1529 an identifier, or 2 otherwise.
1531 If LOC_READER is non-NULL, then position information is
1532 read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
1535 _cpp_valid_ucn (cpp_reader
*pfile
, const uchar
**pstr
,
1536 const uchar
*limit
, int identifier_pos
,
1537 struct normalize_state
*nst
, cppchar_t
*cp
,
1538 source_range
*char_range
,
1539 cpp_string_location_reader
*loc_reader
)
1541 cppchar_t result
, c
;
1542 unsigned int length
;
1543 const uchar
*str
= *pstr
;
1544 const uchar
*base
= str
- 2;
1545 bool delimited
= false, named
= false;
1547 if (!CPP_OPTION (pfile
, cplusplus
) && !CPP_OPTION (pfile
, c99
))
1548 cpp_error (pfile
, CPP_DL_WARNING
,
1549 "universal character names are only valid in C++ and C99");
1550 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
) > 0
1551 && !CPP_OPTION (pfile
, cplusplus
))
1552 cpp_error (pfile
, CPP_DL_WARNING
,
1553 "C99's universal character names are incompatible with C90");
1554 else if (CPP_WTRADITIONAL (pfile
) && identifier_pos
== 0)
1555 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1556 "the meaning of '\\%c' is different in traditional C",
1566 || CPP_OPTION (pfile
, delimited_escape_seqs
)
1567 || !CPP_OPTION (pfile
, std
)))
1570 /* Magic value to indicate no digits seen. */
1573 extend_char_range (char_range
, loc_reader
);
1576 else if (str
[-1] == 'U')
1578 else if (str
[-1] == 'N')
1582 && !CPP_OPTION (pfile
, delimited_escape_seqs
)
1583 && CPP_OPTION (pfile
, std
))
1588 if (str
== limit
|| *str
!= '{')
1595 cpp_error (pfile
, CPP_DL_ERROR
, "'\\N' not followed by '{'");
1601 extend_char_range (char_range
, loc_reader
);
1603 const uchar
*name
= str
;
1611 if (!ISIDNUM (c
) && c
!= ' ' && c
!= '-')
1613 if (ISLOWER (c
) || c
== '_')
1616 extend_char_range (char_range
, loc_reader
);
1620 if (str
< limit
&& *str
== '}')
1622 if (identifier_pos
&& name
== str
)
1624 cpp_warning (pfile
, CPP_W_UNICODE
,
1625 "empty named universal character escape "
1626 "sequence; treating it as separate tokens");
1631 cpp_error (pfile
, CPP_DL_ERROR
,
1632 "empty named universal character escape sequence");
1633 else if ((!identifier_pos
|| strict
)
1634 && !CPP_OPTION (pfile
, delimited_escape_seqs
)
1635 && CPP_OPTION (pfile
, cpp_pedantic
))
1636 cpp_error (pfile
, CPP_DL_PEDWARN
,
1637 "named universal character escapes are only valid "
1643 /* If the name is longer than maximum length of a Unicode
1644 name, it can't be strictly valid. */
1645 if ((size_t) (str
- name
) > uname2c_max_name_len
|| !strict
)
1648 result
= _cpp_uname2c ((const char *) name
, str
- name
,
1649 uname2c_tree
, NULL
);
1650 if (result
== (cppchar_t
) -1)
1654 && (!CPP_OPTION (pfile
, delimited_escape_seqs
)
1656 ret
= cpp_warning (pfile
, CPP_W_UNICODE
,
1657 "\\N{%.*s} is not a valid "
1658 "universal character; treating it "
1659 "as separate tokens",
1660 (int) (str
- name
), name
);
1662 cpp_error (pfile
, CPP_DL_ERROR
,
1663 "\\N{%.*s} is not a valid universal "
1664 "character", (int) (str
- name
), name
);
1666 /* Try to do a loose name lookup according to
1667 Unicode loose matching rule UAX44-LM2. */
1668 char canon_name
[uname2c_max_name_len
+ 1];
1669 result
= _cpp_uname2c_uax44_lm2 ((const char *) name
,
1670 str
- name
, canon_name
);
1671 if (result
!= (cppchar_t
) -1 && ret
)
1672 cpp_error (pfile
, CPP_DL_NOTE
,
1673 "did you mean \\N{%s}?", canon_name
);
1677 && (!CPP_OPTION (pfile
, delimited_escape_seqs
)
1686 extend_char_range (char_range
, loc_reader
);
1688 else if (identifier_pos
)
1690 cpp_warning (pfile
, CPP_W_UNICODE
,
1691 "'\\N{' not terminated with '}' after %.*s; "
1692 "treating it as separate tokens",
1693 (int) (str
- base
), base
);
1699 cpp_error (pfile
, CPP_DL_ERROR
,
1700 "'\\N{' not terminated with '}' after %.*s",
1701 (int) (str
- base
), base
);
1708 cpp_error (pfile
, CPP_DL_ICE
, "In _cpp_valid_ucn but not a UCN");
1721 extend_char_range (char_range
, loc_reader
);
1725 /* Accept arbitrary number of leading zeros.
1726 16 is another magic value, smaller than 32 above
1727 and bigger than 8, so that upon encountering first
1728 non-zero digit we can count 8 digits and after that
1729 or in overflow bit and ensure length doesn't decrease
1730 to 0, as delimited escape sequence doesn't have upper
1731 bound on the number of hex digits. */
1733 else if (length
== 16 - 8)
1735 /* Make sure we detect overflows. */
1736 result
|= 0x8000000;
1741 result
= (result
<< 4) + hex_value (c
);
1745 if (delimited
&& str
< limit
&& *str
== '}')
1747 if (length
== 32 && identifier_pos
)
1749 cpp_warning (pfile
, CPP_W_UNICODE
,
1750 "empty delimited escape sequence; "
1751 "treating it as separate tokens");
1755 else if (length
== 32)
1756 cpp_error (pfile
, CPP_DL_ERROR
,
1757 "empty delimited escape sequence");
1758 else if (!CPP_OPTION (pfile
, delimited_escape_seqs
)
1759 && CPP_OPTION (pfile
, cpp_pedantic
))
1760 cpp_error (pfile
, CPP_DL_PEDWARN
,
1761 "delimited escape sequences are only valid in C++23");
1765 extend_char_range (char_range
, loc_reader
);
1768 /* Partial UCNs are not valid in strings, but decompose into
1769 multiple tokens in identifiers, so we can't give a helpful
1770 error message in that case. */
1771 if (length
&& identifier_pos
)
1774 cpp_warning (pfile
, CPP_W_UNICODE
,
1775 "'\\u{' not terminated with '}' after %.*s; "
1776 "treating it as separate tokens",
1777 (int) (str
- base
), base
);
1786 cpp_error (pfile
, CPP_DL_ERROR
,
1787 "incomplete universal character name %.*s",
1788 (int) (str
- base
), base
);
1790 cpp_error (pfile
, CPP_DL_ERROR
,
1791 "'\\u{' not terminated with '}' after %.*s",
1792 (int) (str
- base
), base
);
1795 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1796 hex escapes so that this also works with EBCDIC hosts.
1797 C++0x permits everything below 0xa0 within literals;
1798 ucn_valid_in_identifier will complain about identifiers. */
1799 else if ((result
< 0xa0
1800 && !CPP_OPTION (pfile
, cplusplus
)
1801 && (result
!= 0x24 && result
!= 0x40 && result
!= 0x60))
1802 || (result
& 0x80000000)
1803 || (result
>= 0xD800 && result
<= 0xDFFF))
1805 cpp_error (pfile
, CPP_DL_ERROR
,
1806 "%.*s is not a valid universal character",
1807 (int) (str
- base
), base
);
1810 else if (identifier_pos
&& result
== 0x24
1811 && CPP_OPTION (pfile
, dollars_in_ident
))
1813 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1815 CPP_OPTION (pfile
, warn_dollars
) = 0;
1816 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1818 NORMALIZE_STATE_UPDATE_IDNUM (nst
, result
);
1820 else if (identifier_pos
)
1822 int validity
= ucn_valid_in_identifier (pfile
, result
, nst
);
1825 cpp_error (pfile
, CPP_DL_ERROR
,
1826 "universal character %.*s is not valid in an identifier",
1827 (int) (str
- base
), base
);
1828 else if (validity
== 2 && identifier_pos
== 1)
1829 cpp_error (pfile
, CPP_DL_ERROR
,
1830 "universal character %.*s is not valid at the start of an identifier",
1831 (int) (str
- base
), base
);
1833 else if (result
> UCS_LIMIT
1834 && (!CPP_OPTION (pfile
, cplusplus
)
1835 || CPP_OPTION (pfile
, lang
) > CLK_CXX17
))
1836 cpp_error (pfile
, CPP_DL_PEDWARN
,
1837 "%.*s is outside the UCS codespace",
1838 (int) (str
- base
), base
);
1844 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1845 it to the execution character set and write the result into TBUF,
1846 if TBUF is non-NULL.
1847 An advanced pointer is returned. Issues all relevant diagnostics.
1848 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1849 contains the location of the character so far: location information
1850 is read from *LOC_READER, and *RANGES is updated accordingly. */
1851 static const uchar
*
1852 convert_ucn (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1853 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1854 source_range char_range
,
1855 cpp_string_location_reader
*loc_reader
,
1856 cpp_substring_ranges
*ranges
)
1861 size_t bytesleft
= 6;
1863 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
1865 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1866 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
1868 from
++; /* Skip u/U/N. */
1870 /* The u/U is part of the spelling of this character. */
1871 extend_char_range (&char_range
, loc_reader
);
1873 _cpp_valid_ucn (pfile
, &from
, limit
, 0, &nst
,
1874 &ucn
, &char_range
, loc_reader
);
1876 rval
= one_cppchar_to_utf8 (ucn
, &bufp
, &bytesleft
);
1880 cpp_errno (pfile
, CPP_DL_ERROR
,
1881 "converting UCN to source character set");
1886 if (!APPLY_CONVERSION (cvt
, buf
, 6 - bytesleft
, tbuf
))
1887 cpp_errno (pfile
, CPP_DL_ERROR
,
1888 "converting UCN to execution character set");
1892 int num_encoded_bytes
= 6 - bytesleft
;
1893 for (int i
= 0; i
< num_encoded_bytes
; i
++)
1894 ranges
->add_range (char_range
);
1901 /* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded
1902 extended characters rather than UCNs. If the return value is TRUE, then a
1903 character was successfully decoded and stored in *CP; *PSTR has been
1904 updated to point one past the valid UTF-8 sequence. Diagnostics may have
1905 been emitted if the character parsed is not allowed in the current context.
1906 If the return value is FALSE, then *PSTR has not been modified and *CP may
1907 equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it
1908 may, when processing an identifier in C mode, equal a codepoint that was
1909 validly encoded but is not allowed to appear in an identifier. In either
1910 case, no diagnostic is emitted, and the return value of FALSE should cause
1911 a new token to be formed.
1913 _cpp_valid_utf8 can be called when lexing a potential identifier, or a
1914 CPP_OTHER token or for the purposes of -Winvalid-utf8 warning in string or
1915 character literals. NST is unused when not in a potential identifier.
1917 As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for
1918 the start of an identifier, or 2 otherwise. */
1921 _cpp_valid_utf8 (cpp_reader
*pfile
,
1925 struct normalize_state
*nst
,
1928 const uchar
*base
= *pstr
;
1929 size_t inbytesleft
= limit
- base
;
1930 if (one_utf8_to_cppchar (pstr
, &inbytesleft
, cp
))
1932 /* No diagnostic here as this byte will rather become a
1940 switch (ucn_valid_in_identifier (pfile
, *cp
, nst
))
1944 /* In C++, this is an error for invalid character in an identifier
1945 because logically, the UTF-8 was converted to a UCN during
1946 translation phase 1 (even though we don't physically do it that
1947 way). In C, this byte rather becomes grammatically a separate
1950 if (CPP_OPTION (pfile
, cplusplus
))
1951 cpp_error (pfile
, CPP_DL_ERROR
,
1952 "extended character %.*s is not valid in an identifier",
1953 (int) (*pstr
- base
), base
);
1963 if (identifier_pos
== 1)
1965 /* This is treated the same way in C++ or C99 -- lexed as an
1966 identifier which is then invalid because an identifier is
1967 not allowed to start with this character. */
1968 cpp_error (pfile
, CPP_DL_ERROR
,
1969 "extended character %.*s is not valid at the start of an identifier",
1970 (int) (*pstr
- base
), base
);
1979 /* Return true iff BUFFER of size NUM_BYTES is validly-encoded UTF-8. */
1982 cpp_valid_utf8_p (const char *buffer
, size_t num_bytes
)
1984 const uchar
*iter
= (const uchar
*)buffer
;
1985 size_t bytesleft
= num_bytes
;
1986 while (bytesleft
> 0)
1988 /* one_utf8_to_cppchar implements 5-byte and 6 byte sequences as per
1989 RFC 2279, but this has been superceded by RFC 3629, which
1990 restricts UTF-8 to 1-byte through 4-byte sequences, and
1991 states "the octet values C0, C1, F5 to FF never appear".
1993 Reject such values. */
1998 int err
= one_utf8_to_cppchar (&iter
, &bytesleft
, &cp
);
2002 /* Additionally, Unicode declares that all codepoints above 0010FFFF are
2003 invalid because they cannot be represented in UTF-16.
2005 Reject such values.*/
2009 /* No problems encountered. */
2013 /* Subroutine of convert_hex and convert_oct. N is the representation
2014 in the execution character set of a numeric escape; write it into the
2015 string buffer TBUF and update the end-of-string pointer therein. WIDE
2016 is true if it's a wide string that's being assembled in TBUF. This
2017 function issues no diagnostics and never fails. */
2019 emit_numeric_escape (cpp_reader
*pfile
, cppchar_t n
,
2020 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
2022 size_t width
= cvt
.width
;
2024 if (width
!= CPP_OPTION (pfile
, char_precision
))
2026 /* We have to render this into the target byte order, which may not
2027 be our byte order. */
2028 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
2029 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
2030 size_t cmask
= width_to_mask (cwidth
);
2031 size_t nbwc
= width
/ cwidth
;
2033 size_t off
= tbuf
->len
;
2036 if (tbuf
->len
+ nbwc
> tbuf
->asize
)
2038 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
2039 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
2042 for (i
= 0; i
< nbwc
; i
++)
2046 tbuf
->text
[off
+ (bigend
? nbwc
- i
- 1 : i
)] = c
;
2052 /* Note: this code does not handle the case where the target
2053 and host have a different number of bits in a byte. */
2054 if (tbuf
->len
+ 1 > tbuf
->asize
)
2056 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
2057 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
2059 tbuf
->text
[tbuf
->len
++] = n
;
2063 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
2064 character set and write it into the string buffer TBUF (if non-NULL).
2065 Returns an advanced pointer, and issues diagnostics as necessary.
2066 No character set translation occurs; this routine always produces the
2067 execution-set character with numeric value equal to the given hex
2068 number. You can, e.g. generate surrogate pairs this way.
2069 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
2070 contains the location of the character so far: location information
2071 is read from *LOC_READER, and *RANGES is updated accordingly. */
2072 static const uchar
*
2073 convert_hex (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
2074 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
2075 source_range char_range
,
2076 cpp_string_location_reader
*loc_reader
,
2077 cpp_substring_ranges
*ranges
)
2079 cppchar_t c
, n
= 0, overflow
= 0;
2080 int digits_found
= 0;
2081 size_t width
= cvt
.width
;
2082 size_t mask
= width_to_mask (width
);
2083 bool delimited
= false;
2084 const uchar
*base
= from
- 1;
2086 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
2087 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
2089 if (CPP_WTRADITIONAL (pfile
))
2090 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
2091 "the meaning of '\\x' is different in traditional C");
2096 /* The 'x' is part of the spelling of this character. */
2097 extend_char_range (&char_range
, loc_reader
);
2099 if (from
< limit
&& *from
== '{')
2103 extend_char_range (&char_range
, loc_reader
);
2106 while (from
< limit
)
2112 extend_char_range (&char_range
, loc_reader
);
2113 overflow
|= n
^ (n
<< 4 >> 4);
2114 n
= (n
<< 4) + hex_value (c
);
2118 if (delimited
&& from
< limit
&& *from
== '}')
2123 cpp_error (pfile
, CPP_DL_ERROR
,
2124 "empty delimited escape sequence");
2127 else if (!CPP_OPTION (pfile
, delimited_escape_seqs
)
2128 && CPP_OPTION (pfile
, cpp_pedantic
))
2129 cpp_error (pfile
, CPP_DL_PEDWARN
,
2130 "delimited escape sequences are only valid in C++23");
2132 extend_char_range (&char_range
, loc_reader
);
2137 cpp_error (pfile
, CPP_DL_ERROR
,
2138 "\\x used with no following hex digits");
2143 cpp_error (pfile
, CPP_DL_ERROR
,
2144 "'\\x{' not terminated with '}' after %.*s",
2145 (int) (from
- base
), base
);
2149 if (overflow
| (n
!= (n
& mask
)))
2151 cpp_error (pfile
, CPP_DL_PEDWARN
,
2152 "hex escape sequence out of range");
2157 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
2159 ranges
->add_range (char_range
);
2164 /* Convert an octal escape, pointed to by FROM, to the execution
2165 character set and write it into the string buffer TBUF. Returns an
2166 advanced pointer, and issues diagnostics as necessary.
2167 No character set translation occurs; this routine always produces the
2168 execution-set character with numeric value equal to the given octal
2170 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
2171 contains the location of the character so far: location information
2172 is read from *LOC_READER, and *RANGES is updated accordingly. */
2173 static const uchar
*
2174 convert_oct (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
2175 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
2176 source_range char_range
,
2177 cpp_string_location_reader
*loc_reader
,
2178 cpp_substring_ranges
*ranges
)
2181 cppchar_t c
, n
= 0, overflow
= 0;
2182 size_t width
= cvt
.width
;
2183 size_t mask
= width_to_mask (width
);
2184 bool delimited
= false;
2185 const uchar
*base
= from
- 1;
2187 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
2188 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
2190 if (from
< limit
&& *from
== 'o')
2193 extend_char_range (&char_range
, loc_reader
);
2194 if (from
== limit
|| *from
!= '{')
2195 cpp_error (pfile
, CPP_DL_ERROR
, "'\\o' not followed by '{'");
2199 extend_char_range (&char_range
, loc_reader
);
2204 while (from
< limit
&& count
++ < 3)
2207 if (c
< '0' || c
> '7')
2210 extend_char_range (&char_range
, loc_reader
);
2214 overflow
|= n
^ (n
<< 3 >> 3);
2216 n
= (n
<< 3) + c
- '0';
2221 if (from
< limit
&& *from
== '}')
2226 cpp_error (pfile
, CPP_DL_ERROR
,
2227 "empty delimited escape sequence");
2230 else if (!CPP_OPTION (pfile
, delimited_escape_seqs
)
2231 && CPP_OPTION (pfile
, cpp_pedantic
))
2232 cpp_error (pfile
, CPP_DL_PEDWARN
,
2233 "delimited escape sequences are only valid in C++23");
2234 extend_char_range (&char_range
, loc_reader
);
2238 cpp_error (pfile
, CPP_DL_ERROR
,
2239 "'\\o{' not terminated with '}' after %.*s",
2240 (int) (from
- base
), base
);
2245 if (overflow
| (n
!= (n
& mask
)))
2247 cpp_error (pfile
, CPP_DL_PEDWARN
,
2248 "octal escape sequence out of range");
2253 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
2255 ranges
->add_range (char_range
);
2260 /* Convert an escape sequence (pointed to by FROM) to its value on
2261 the target, and to the execution character set. Do not scan past
2262 LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
2263 Returns an advanced pointer. Handles all relevant diagnostics.
2264 If LOC_READER is non-NULL, then RANGES must be non-NULL: location
2265 information is read from *LOC_READER, and *RANGES is updated
2267 static const uchar
*
2268 convert_escape (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
2269 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
2270 cpp_string_location_reader
*loc_reader
,
2271 cpp_substring_ranges
*ranges
, bool uneval
)
2273 /* Values of \a \b \e \f \n \r \t \v respectively. */
2274 #if HOST_CHARSET == HOST_CHARSET_ASCII
2275 static const uchar charconsts
[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
2276 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
2277 static const uchar charconsts
[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
2279 #error "unknown host character set"
2284 /* Record the location of the backslash. */
2285 source_range char_range
;
2287 char_range
= loc_reader
->get_next ();
2292 /* UCNs, hex escapes, and octal escapes are processed separately. */
2293 case 'u': case 'U': case 'N':
2294 return convert_ucn (pfile
, from
, limit
, tbuf
, cvt
,
2295 char_range
, loc_reader
, ranges
);
2298 if (uneval
&& CPP_PEDANTIC (pfile
))
2299 cpp_error (pfile
, CPP_DL_PEDWARN
,
2300 "numeric escape sequence in unevaluated string: "
2302 return convert_hex (pfile
, from
, limit
, tbuf
, cvt
,
2303 char_range
, loc_reader
, ranges
);
2305 case '0': case '1': case '2': case '3':
2306 case '4': case '5': case '6': case '7':
2308 if (uneval
&& CPP_PEDANTIC (pfile
))
2309 cpp_error (pfile
, CPP_DL_PEDWARN
,
2310 "numeric escape sequence in unevaluated string: "
2312 return convert_oct (pfile
, from
, limit
, tbuf
, cvt
,
2313 char_range
, loc_reader
, ranges
);
2315 /* Various letter escapes. Get the appropriate host-charset
2317 case '\\': case '\'': case '"': case '?': break;
2319 case '(': case '{': case '[': case '%':
2320 /* '\(', etc, can be used at the beginning of a line in a long
2321 string split onto multiple lines with \-newline, to prevent
2322 Emacs or other text editors from getting confused. '\%' can
2323 be used to prevent SCCS from mangling printf format strings. */
2324 if (CPP_PEDANTIC (pfile
))
2328 case 'b': c
= charconsts
[1]; break;
2329 case 'f': c
= charconsts
[3]; break;
2330 case 'n': c
= charconsts
[4]; break;
2331 case 'r': c
= charconsts
[5]; break;
2332 case 't': c
= charconsts
[6]; break;
2333 case 'v': c
= charconsts
[7]; break;
2336 if (CPP_WTRADITIONAL (pfile
))
2337 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
2338 "the meaning of '\\a' is different in traditional C");
2343 if (CPP_PEDANTIC (pfile
))
2344 cpp_error (pfile
, CPP_DL_PEDWARN
,
2345 "non-ISO-standard escape sequence, '\\%c'", (int) c
);
2352 cpp_error (pfile
, CPP_DL_PEDWARN
,
2353 "unknown escape sequence: '\\%c'", (int) c
);
2356 encoding_rich_location
rich_loc (pfile
);
2358 /* diagnostic.cc does not support "%03o". When it does, this
2359 code can use %03o directly in the diagnostic again. */
2361 sprintf(buf
, "%03o", (int) c
);
2362 cpp_error_at (pfile
, CPP_DL_PEDWARN
, &rich_loc
,
2363 "unknown escape sequence: '\\%s'", buf
);
2368 /* Now convert what we have to the execution character set. */
2369 if (!APPLY_CONVERSION (cvt
, &c
, 1, tbuf
))
2370 cpp_errno (pfile
, CPP_DL_ERROR
,
2371 "converting escape sequence to execution character set");
2375 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
2376 ranges
->add_range (char_range
);
2382 /* TYPE is a token type. The return value is the conversion needed to
2383 convert from source to execution character set for the given type. */
2384 static struct cset_converter
2385 converter_for_type (cpp_reader
*pfile
, enum cpp_ttype type
)
2390 return pfile
->narrow_cset_desc
;
2392 case CPP_UTF8STRING
:
2393 return pfile
->utf8_cset_desc
;
2396 return pfile
->char16_cset_desc
;
2399 return pfile
->char32_cset_desc
;
2402 return pfile
->wide_cset_desc
;
2406 /* FROM is an array of cpp_string structures of length COUNT. These
2407 are to be converted from the source to the execution character set,
2408 escape sequences translated, and finally all are to be
2409 concatenated. WIDE indicates whether or not to produce a wide
2410 string. If TO is non-NULL, the result is written into TO.
2411 If LOC_READERS and OUT are non-NULL, then location information
2412 is read from LOC_READERS (which must be an array of length COUNT),
2413 and location information is written to *RANGES.
2415 Returns true for success, false for failure. */
2418 cpp_interpret_string_1 (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
2419 cpp_string
*to
, enum cpp_ttype type
,
2420 cpp_string_location_reader
*loc_readers
,
2421 cpp_substring_ranges
*out
)
2423 struct _cpp_strbuf tbuf
;
2424 const uchar
*p
, *base
, *limit
;
2426 struct cset_converter cvt
= converter_for_type (pfile
, type
);
2428 /* loc_readers and out must either be both NULL, or both be non-NULL. */
2429 gcc_assert ((loc_readers
!= NULL
) == (out
!= NULL
));
2433 tbuf
.asize
= MAX (OUTBUF_BLOCK_SIZE
, from
->len
);
2434 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
2438 cpp_string_location_reader
*loc_reader
= NULL
;
2439 for (i
= 0; i
< count
; i
++)
2442 loc_reader
= &loc_readers
[i
];
2449 loc_reader
->get_next ();
2454 loc_reader
->get_next ();
2457 else if (*p
== 'L' || *p
== 'U') p
++;
2460 const uchar
*prefix
;
2462 /* Skip over 'R"'. */
2466 loc_reader
->get_next ();
2467 loc_reader
->get_next ();
2474 loc_reader
->get_next ();
2478 loc_reader
->get_next ();
2479 limit
= from
[i
].text
+ from
[i
].len
;
2480 if (limit
>= p
+ (p
- prefix
) + 1)
2481 limit
-= (p
- prefix
) + 1;
2483 /* Raw strings are all normal characters; these can be fed
2484 directly to convert_cset. */
2486 if (!APPLY_CONVERSION (cvt
, p
, limit
- p
, &tbuf
))
2491 /* If generating source ranges, assume we have a 1:1
2492 correspondence between bytes in the source encoding and bytes
2493 in the execution encoding (e.g. if we have a UTF-8 to UTF-8
2494 conversion), so that this run of bytes in the source file
2495 corresponds to a run of bytes in the execution string.
2496 This requirement is guaranteed by an early-reject in
2497 cpp_interpret_string_ranges. */
2498 gcc_assert (cvt
.func
== convert_no_conversion
);
2499 out
->add_n_ranges (limit
- p
, *loc_reader
);
2505 /* If we don't now have a leading quote, something has gone wrong.
2506 This can occur if cpp_interpret_string_ranges is handling a
2507 stringified macro argument, but should not be possible otherwise. */
2508 if (*p
!= '"' && *p
!= '\'')
2510 gcc_assert (out
!= NULL
);
2511 cpp_error (pfile
, CPP_DL_ERROR
, "missing open quote");
2517 /* Skip leading quote. */
2520 loc_reader
->get_next ();
2522 limit
= from
[i
].text
+ from
[i
].len
- 1; /* Skip trailing quote. */
2527 while (p
< limit
&& *p
!= '\\')
2531 /* We have a run of normal characters; these can be fed
2532 directly to convert_cset. */
2534 if (!APPLY_CONVERSION (cvt
, base
, p
- base
, &tbuf
))
2536 /* Similar to above: assumes we have a 1:1 correspondence
2537 between bytes in the source encoding and bytes in the
2538 execution encoding. */
2541 gcc_assert (cvt
.func
== convert_no_conversion
);
2542 out
->add_n_ranges (p
- base
, *loc_reader
);
2548 struct _cpp_strbuf
*tbuf_ptr
= to
? &tbuf
: NULL
;
2549 p
= convert_escape (pfile
, p
+ 1, limit
, tbuf_ptr
, cvt
,
2550 loc_reader
, out
, type
== CPP_UNEVAL_STRING
);
2556 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
2558 emit_numeric_escape (pfile
, 0, &tbuf
, cvt
);
2559 tbuf
.text
= XRESIZEVEC (uchar
, tbuf
.text
, tbuf
.len
);
2560 to
->text
= tbuf
.text
;
2563 /* Use the location of the trailing quote as the location of the
2567 source_range range
= loc_reader
->get_next ();
2568 out
->add_range (range
);
2574 cpp_errno (pfile
, CPP_DL_ERROR
, "converting to execution character set");
2580 /* FROM is an array of cpp_string structures of length COUNT. These
2581 are to be converted from the source to the execution character set,
2582 escape sequences translated, and finally all are to be
2583 concatenated. WIDE indicates whether or not to produce a wide
2584 string. The result is written into TO. Returns true for success,
2585 false for failure. */
2587 cpp_interpret_string (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
2588 cpp_string
*to
, enum cpp_ttype type
)
2590 return cpp_interpret_string_1 (pfile
, from
, count
, to
, type
, NULL
, NULL
);
2593 /* A "do nothing" diagnostic-handling callback for use by
2594 cpp_interpret_string_ranges, so that it can temporarily suppress
2595 diagnostic-handling. */
2598 noop_diagnostic_cb (cpp_reader
*, enum cpp_diagnostic_level
,
2599 enum cpp_warning_reason
, rich_location
*,
2600 const char *, va_list *)
2606 /* This function mimics the behavior of cpp_interpret_string, but
2607 rather than generating a string in the execution character set,
2608 *OUT is written to with the source code ranges of the characters
2610 FROM and LOC_READERS should both be arrays of length COUNT.
2611 Returns NULL for success, or an error message for failure. */
2614 cpp_interpret_string_ranges (cpp_reader
*pfile
, const cpp_string
*from
,
2615 cpp_string_location_reader
*loc_readers
,
2617 cpp_substring_ranges
*out
,
2618 enum cpp_ttype type
)
2620 /* There are a couple of cases in the range-handling in
2621 cpp_interpret_string_1 that rely on there being a 1:1 correspondence
2622 between bytes in the source encoding and bytes in the execution
2623 encoding, so that each byte in the execution string can correspond
2624 to the location of a byte in the source string.
2626 This holds for the typical case of a UTF-8 to UTF-8 conversion.
2627 Enforce this requirement by only attempting to track substring
2628 locations if we have source encoding == execution encoding.
2630 This is a stronger condition than we need, since we could e.g.
2631 have ASCII to EBCDIC (with 1 byte per character before and after),
2632 but it seems to be a reasonable restriction. */
2633 struct cset_converter cvt
= converter_for_type (pfile
, type
);
2634 if (cvt
.func
!= convert_no_conversion
)
2635 return "execution character set != source character set";
2637 /* For on-demand strings we have already lexed the strings, so there
2638 should be no diagnostics. However, if we have bogus source location
2639 data (or stringified macro arguments), the attempt to lex the
2640 strings could fail with an diagnostic. Temporarily install an
2641 diagnostic-handler to catch the diagnostic, so that it can lead to this call
2642 failing, rather than being emitted as a user-visible diagnostic.
2643 If an diagnostic does occur, we should see it via the return value of
2644 cpp_interpret_string_1. */
2645 bool (*saved_diagnostic_handler
) (cpp_reader
*, enum cpp_diagnostic_level
,
2646 enum cpp_warning_reason
, rich_location
*,
2647 const char *, va_list *)
2648 ATTRIBUTE_FPTR_PRINTF(5,0);
2650 saved_diagnostic_handler
= pfile
->cb
.diagnostic
;
2651 pfile
->cb
.diagnostic
= noop_diagnostic_cb
;
2653 bool result
= cpp_interpret_string_1 (pfile
, from
, count
, NULL
, type
,
2656 /* Restore the saved diagnostic-handler. */
2657 pfile
->cb
.diagnostic
= saved_diagnostic_handler
;
2660 return "cpp_interpret_string_1 failed";
2666 /* Subroutine of do_line and do_linemarker. Convert escape sequences
2667 in a string, but do not perform character set conversion. */
2669 cpp_interpret_string_notranslate (cpp_reader
*pfile
, const cpp_string
*from
,
2670 size_t count
, cpp_string
*to
,
2671 enum cpp_ttype type
)
2673 struct cset_converter save_narrow_cset_desc
= pfile
->narrow_cset_desc
;
2676 pfile
->narrow_cset_desc
.func
= convert_no_conversion
;
2677 pfile
->narrow_cset_desc
.cd
= (iconv_t
) -1;
2678 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
2680 retval
= cpp_interpret_string (pfile
, from
, count
, to
,
2681 type
== CPP_UNEVAL_STRING
2682 ? CPP_UNEVAL_STRING
: CPP_STRING
);
2684 pfile
->narrow_cset_desc
= save_narrow_cset_desc
;
2689 /* Return number of source characters in STR. */
2691 count_source_chars (cpp_reader
*pfile
, cpp_string str
, cpp_ttype type
)
2693 cpp_string str2
= { 0, 0 };
2694 bool (*saved_diagnostic_handler
) (cpp_reader
*, enum cpp_diagnostic_level
,
2695 enum cpp_warning_reason
, rich_location
*,
2696 const char *, va_list *)
2697 ATTRIBUTE_FPTR_PRINTF(5,0);
2698 saved_diagnostic_handler
= pfile
->cb
.diagnostic
;
2699 pfile
->cb
.diagnostic
= noop_diagnostic_cb
;
2700 convert_f save_func
= pfile
->narrow_cset_desc
.func
;
2701 pfile
->narrow_cset_desc
.func
= convert_count_chars
;
2702 bool ret
= cpp_interpret_string (pfile
, &str
, 1, &str2
, type
);
2703 pfile
->narrow_cset_desc
.func
= save_func
;
2704 pfile
->cb
.diagnostic
= saved_diagnostic_handler
;
2707 if (str2
.text
!= str
.text
)
2708 free ((void *)str2
.text
);
2715 /* Subroutine of cpp_interpret_charconst which performs the conversion
2716 to a number, for narrow strings. STR is the string structure returned
2717 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
2718 cpp_interpret_charconst. TOKEN is the token. */
2720 narrow_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
2721 unsigned int *pchars_seen
, int *unsignedp
,
2722 const cpp_token
*token
)
2724 enum cpp_ttype type
= token
->type
;
2725 size_t width
= CPP_OPTION (pfile
, char_precision
);
2726 size_t max_chars
= CPP_OPTION (pfile
, int_precision
) / width
;
2727 size_t mask
= width_to_mask (width
);
2729 cppchar_t result
, c
;
2731 bool diagnosed
= false;
2733 /* The value of a multi-character character constant, or a
2734 single-character character constant whose representation in the
2735 execution character set is more than one byte long, is
2736 implementation defined. This implementation defines it to be the
2737 number formed by interpreting the byte sequence in memory as a
2738 big-endian binary number. If overflow occurs, the high bytes are
2739 lost, and a warning is issued.
2741 We don't want to process the NUL terminator handed back by
2742 cpp_interpret_string. */
2744 for (i
= 0; i
< str
.len
- 1; i
++)
2746 c
= str
.text
[i
] & mask
;
2747 if (width
< BITS_PER_CPPCHAR_T
)
2748 result
= (result
<< width
) | c
;
2753 if (type
== CPP_UTF8CHAR
)
2755 else if (i
> 1 && CPP_OPTION (pfile
, cplusplus
) && CPP_PEDANTIC (pfile
))
2757 /* C++ as a DR since
2758 P1854R4 - Making non-encodable string literals ill-formed
2759 makes multi-character narrow character literals if any of the
2760 characters in the literal isn't encodable in char/unsigned char
2761 ill-formed. We need to count the number of c-chars and compare
2763 unsigned src_chars
= count_source_chars (pfile
, token
->val
.str
, type
);
2767 if (str
.len
> src_chars
)
2771 = cpp_error (pfile
, CPP_DL_PEDWARN
,
2772 "character not encodable in a single execution "
2773 "character code unit");
2776 = cpp_error (pfile
, CPP_DL_PEDWARN
,
2777 "at least one character in a multi-character "
2778 "literal not encodable in a single execution "
2779 "character code unit");
2780 if (diagnosed
&& i
> max_chars
)
2786 /* Already diagnosed above. */;
2787 else if (i
> max_chars
)
2790 = count_source_chars (pfile
, token
->val
.str
,
2791 type
== CPP_UTF8CHAR
? CPP_CHAR
: type
);
2793 if (type
!= CPP_UTF8CHAR
)
2794 cpp_error (pfile
, CPP_DL_WARNING
,
2795 "multi-character literal with %ld characters exceeds "
2796 "'int' size of %ld bytes", (long) i
, (long) max_chars
);
2797 else if (src_chars
> 2)
2798 cpp_error (pfile
, CPP_DL_ERROR
,
2799 "multi-character literal cannot have an encoding prefix");
2801 cpp_error (pfile
, CPP_DL_ERROR
,
2802 "character not encodable in a single code unit");
2805 else if (i
> 1 && CPP_OPTION (pfile
, warn_multichar
))
2806 cpp_warning (pfile
, CPP_W_MULTICHAR
, "multi-character character constant");
2808 /* Multichar constants are of type int and therefore signed. */
2811 else if (type
== CPP_UTF8CHAR
)
2812 unsigned_p
= CPP_OPTION (pfile
, unsigned_utf8char
);
2814 unsigned_p
= CPP_OPTION (pfile
, unsigned_char
);
2816 /* Truncate the constant to its natural width, and simultaneously
2817 sign- or zero-extend to the full width of cppchar_t.
2818 For single-character constants, the value is WIDTH bits wide.
2819 For multi-character constants, the value is INT_PRECISION bits wide. */
2821 width
= CPP_OPTION (pfile
, int_precision
);
2822 if (width
< BITS_PER_CPPCHAR_T
)
2824 mask
= ((cppchar_t
) 1 << width
) - 1;
2825 if (unsigned_p
|| !(result
& (1 << (width
- 1))))
2831 *unsignedp
= unsigned_p
;
2835 /* Subroutine of cpp_interpret_charconst which performs the conversion
2836 to a number, for wide strings. STR is the string structure returned
2837 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
2838 cpp_interpret_charconst. TOKEN is the token. */
2840 wide_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
2841 unsigned int *pchars_seen
, int *unsignedp
,
2842 const cpp_token
*token
)
2844 enum cpp_ttype type
= token
->type
;
2845 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
2846 size_t width
= converter_for_type (pfile
, type
).width
;
2847 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
2848 size_t mask
= width_to_mask (width
);
2849 size_t cmask
= width_to_mask (cwidth
);
2850 size_t nbwc
= width
/ cwidth
;
2852 cppchar_t result
= 0, c
;
2854 if (str
.len
<= nbwc
)
2856 /* Error recovery, if no errors have been diagnosed previously,
2857 there should be at least two wide characters. Empty literals
2858 are diagnosed earlier and we can get just the zero terminator
2859 only if there were errors diagnosed during conversion. */
2865 /* This is finicky because the string is in the target's byte order,
2866 which may not be our byte order. Only the last character, ignoring
2867 the NUL terminator, is relevant. */
2868 off
= str
.len
- (nbwc
* 2);
2870 for (i
= 0; i
< nbwc
; i
++)
2872 c
= bigend
? str
.text
[off
+ i
] : str
.text
[off
+ nbwc
- i
- 1];
2873 result
= (result
<< cwidth
) | (c
& cmask
);
2876 /* Wide character constants have type wchar_t, and a single
2877 character exactly fills a wchar_t, so a multi-character wide
2878 character constant is guaranteed to overflow. */
2879 if (str
.len
> nbwc
* 2)
2881 cpp_diagnostic_level level
= CPP_DL_WARNING
;
2883 = count_source_chars (pfile
, token
->val
.str
, CPP_CHAR
);
2885 if (CPP_OPTION (pfile
, cplusplus
)
2886 && (type
== CPP_CHAR16
2887 || type
== CPP_CHAR32
2888 /* In C++23 this is error even for L'ab'. */
2889 || (type
== CPP_WCHAR
2890 && CPP_OPTION (pfile
, size_t_literals
))))
2891 level
= CPP_DL_ERROR
;
2893 cpp_error (pfile
, level
,
2894 "multi-character literal cannot have an encoding prefix");
2896 cpp_error (pfile
, level
,
2897 "character not encodable in a single code unit");
2900 /* Truncate the constant to its natural width, and simultaneously
2901 sign- or zero-extend to the full width of cppchar_t. */
2902 if (width
< BITS_PER_CPPCHAR_T
)
2904 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
2905 || CPP_OPTION (pfile
, unsigned_wchar
)
2906 || !(result
& (1 << (width
- 1))))
2912 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
2913 || CPP_OPTION (pfile
, unsigned_wchar
))
2922 /* Interpret a (possibly wide) character constant in TOKEN.
2923 PCHARS_SEEN points to a variable that is filled in with the number
2924 of characters seen, and UNSIGNEDP to a variable that indicates
2925 whether the result has signed type. */
2927 cpp_interpret_charconst (cpp_reader
*pfile
, const cpp_token
*token
,
2928 unsigned int *pchars_seen
, int *unsignedp
)
2930 cpp_string str
= { 0, 0 };
2931 bool wide
= (token
->type
!= CPP_CHAR
&& token
->type
!= CPP_UTF8CHAR
);
2932 int u8
= 2 * int(token
->type
== CPP_UTF8CHAR
);
2935 /* An empty constant will appear as L'', u'', U'', u8'', or '' */
2936 if (token
->val
.str
.len
== (size_t) (2 + wide
+ u8
))
2938 cpp_error (pfile
, CPP_DL_ERROR
, "empty character constant");
2943 else if (!cpp_interpret_string (pfile
, &token
->val
.str
, 1, &str
,
2952 result
= wide_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
2955 result
= narrow_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
2958 if (str
.text
!= token
->val
.str
.text
)
2959 free ((void *)str
.text
);
2964 /* Convert an identifier denoted by ID and LEN, which might contain
2965 UCN escapes or UTF-8 multibyte chars, to the source character set,
2966 either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually
2967 a valid identifier. */
2969 _cpp_interpret_identifier (cpp_reader
*pfile
, const uchar
*id
, size_t len
)
2971 /* It turns out that a UCN escape always turns into fewer characters
2972 than the escape itself, so we can allocate a temporary in advance. */
2973 uchar
* buf
= (uchar
*) alloca (len
+ 1);
2977 for (idp
= 0; idp
< len
; idp
++)
2978 if (id
[idp
] != '\\')
2982 unsigned length
= id
[idp
+ 1] == 'u' ? 4 : 8;
2983 cppchar_t value
= 0;
2984 size_t bufleft
= len
- (bufp
- buf
);
2986 bool delimited
= false;
2989 if (id
[idp
- 1] == 'N' && id
[idp
] == '{')
2992 const uchar
*name
= &id
[idp
];
2994 && (ISIDNUM (id
[idp
]) || id
[idp
] == ' ' || id
[idp
] == '-'))
2998 value
= _cpp_uname2c ((const char *) name
, &id
[idp
] - name
,
2999 uname2c_tree
, NULL
);
3000 if (value
== (cppchar_t
) -1)
3008 if (length
== 4 && id
[idp
] == '{')
3013 while (length
&& idp
< len
&& ISXDIGIT (id
[idp
]))
3015 value
= (value
<< 4) + hex_value (id
[idp
]);
3020 if (!delimited
|| id
[idp
] != '}')
3024 /* Special case for EBCDIC: if the identifier contains
3025 a '$' specified using a UCN, translate it to EBCDIC. */
3032 rval
= one_cppchar_to_utf8 (value
, &bufp
, &bufleft
);
3036 cpp_errno (pfile
, CPP_DL_ERROR
,
3037 "converting UCN to source character set");
3042 return CPP_HASHNODE (ht_lookup (pfile
->hash_table
,
3043 buf
, bufp
- buf
, HT_ALLOC
));
3047 /* Utility to strip a UTF-8 byte order marking from the beginning
3048 of a buffer. Returns the number of bytes to skip, which currently
3049 will be either 0 or 3. */
3051 cpp_check_utf8_bom (const char *data
, size_t data_length
)
3054 #if HOST_CHARSET == HOST_CHARSET_ASCII
3055 const unsigned char *udata
= (const unsigned char *) data
;
3056 if (data_length
>= 3 && udata
[0] == 0xef && udata
[1] == 0xbb
3057 && udata
[2] == 0xbf)
3065 /* Convert an input buffer (containing the complete contents of one
3066 source file) from INPUT_CHARSET to the source character set. INPUT
3067 points to the input buffer, SIZE is its allocated size, and LEN is
3068 the length of the meaningful data within the buffer. The
3069 translated buffer is returned, *ST_SIZE is set to the length of
3070 the meaningful data within the translated buffer, and *BUFFER_START
3071 is set to the start of the returned buffer. *BUFFER_START may
3072 differ from the return value in the case of a BOM or other ignored
3075 INPUT is expected to have been allocated with xmalloc. This
3076 function will either set *BUFFER_START to INPUT, or free it and set
3077 *BUFFER_START to a pointer to another xmalloc-allocated block of
3080 PFILE is only used to generate diagnostics; setting it to NULL suppresses
3081 diagnostics, and causes a return of NULL if there was any error instead. */
3084 _cpp_convert_input (cpp_reader
*pfile
, const char *input_charset
,
3085 uchar
*input
, size_t size
, size_t len
,
3086 const unsigned char **buffer_start
, off_t
*st_size
)
3088 struct cset_converter input_cset
;
3089 struct _cpp_strbuf to
;
3090 unsigned char *buffer
;
3092 input_cset
= init_iconv_desc (pfile
, SOURCE_CHARSET
, input_charset
);
3093 if (input_cset
.func
== convert_no_conversion
)
3101 to
.asize
= MAX (65536, len
);
3102 to
.text
= XNEWVEC (uchar
, to
.asize
);
3105 const bool ok
= APPLY_CONVERSION (input_cset
, input
, len
, &to
);
3108 /* Clean up the mess. */
3109 if (input_cset
.func
== convert_using_iconv
)
3110 iconv_close (input_cset
.cd
);
3112 /* Handle conversion failure. */
3117 XDELETEVEC (to
.text
);
3118 *buffer_start
= NULL
;
3122 cpp_error (pfile
, CPP_DL_ERROR
, "failure to convert %s to %s",
3123 input_charset
, SOURCE_CHARSET
);
3127 /* Resize buffer if we allocated substantially too much, or if we
3128 haven't enough space for the \n-terminator or following
3129 15 bytes of padding (used to quiet warnings from valgrind or
3130 Address Sanitizer, when the optimized lexer accesses aligned
3131 16-byte memory chunks, including the bytes after the malloced,
3132 area, and stops lexing on '\n'). */
3133 if (to
.len
+ 4096 < to
.asize
|| to
.len
+ 16 > to
.asize
)
3134 to
.text
= XRESIZEVEC (uchar
, to
.text
, to
.len
+ 16);
3136 memset (to
.text
+ to
.len
, '\0', 16);
3138 /* If the file is using old-school Mac line endings (\r only),
3139 terminate with another \r, not an \n, so that we do not mistake
3140 the \r\n sequence for a single DOS line ending and erroneously
3141 issue the "No newline at end of file" diagnostic. */
3142 if (to
.len
&& to
.text
[to
.len
- 1] == '\r')
3143 to
.text
[to
.len
] = '\r';
3145 to
.text
[to
.len
] = '\n';
3150 /* Ignore a UTF-8 BOM if we see one and the source charset is UTF-8. Note
3151 that glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
3152 BOM -- however, even if it did, we would still need this code due
3153 to the 'convert_no_conversion' case. */
3154 const int bom_len
= cpp_check_utf8_bom ((const char *) to
.text
, to
.len
);
3155 *st_size
-= bom_len
;
3158 *buffer_start
= to
.text
;
3162 /* Decide on the default encoding to assume for input files. */
3164 _cpp_default_encoding (void)
3166 const char *current_encoding
= NULL
;
3168 /* We disable this because the default codeset is 7-bit ASCII on
3169 most platforms, and this causes conversion failures on every
3170 file in GCC that happens to have one of the upper 128 characters
3171 in it -- most likely, as part of the name of a contributor.
3172 We should definitely recognize in-band markers of file encoding,
3174 - the appropriate Unicode byte-order mark (FE FF) to recognize
3175 UTF16 and UCS4 (in both big-endian and little-endian flavors)
3177 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
3178 distinguish ASCII and EBCDIC.
3179 - now we can parse something like "#pragma GCC encoding <xyz>
3180 on the first line, or even Emacs/VIM's mode line tags (there's
3181 a problem here in that VIM uses the last line, and Emacs has
3182 its more elaborate "local variables" convention).
3183 - investigate whether Java has another common convention, which
3184 would be friendly to support.
3185 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
3186 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
3187 setlocale (LC_CTYPE
, "");
3188 current_encoding
= nl_langinfo (CODESET
);
3190 if (current_encoding
== NULL
|| *current_encoding
== '\0')
3191 current_encoding
= SOURCE_CHARSET
;
3193 return current_encoding
;
3196 /* Check if the configured input charset requires no conversion, other than
3197 possibly stripping a UTF-8 BOM. */
3198 bool cpp_input_conversion_is_trivial (const char *input_charset
)
3200 return !strcasecmp (input_charset
, SOURCE_CHARSET
);
3203 /* Implementation of class cpp_string_location_reader. */
3205 /* Constructor for cpp_string_location_reader. */
3207 cpp_string_location_reader::
3208 cpp_string_location_reader (location_t src_loc
,
3209 line_maps
*line_table
)
3211 src_loc
= get_range_from_loc (line_table
, src_loc
).m_start
;
3213 /* SRC_LOC might be a macro location. It only makes sense to do
3214 column-by-column calculations on ordinary maps, so get the
3215 corresponding location in an ordinary map. */
3217 = linemap_resolve_location (line_table
, src_loc
,
3218 LRK_SPELLING_LOCATION
, NULL
);
3220 const line_map_ordinary
*map
3221 = linemap_check_ordinary (linemap_lookup (line_table
, m_loc
));
3222 m_offset_per_column
= (1 << map
->m_range_bits
);
3225 /* Get the range of the next source byte. */
3228 cpp_string_location_reader::get_next ()
3230 source_range result
;
3231 result
.m_start
= m_loc
;
3232 result
.m_finish
= m_loc
;
3233 if (m_loc
<= LINE_MAP_MAX_LOCATION_WITH_COLS
)
3234 m_loc
+= m_offset_per_column
;
3238 cpp_display_width_computation::
3239 cpp_display_width_computation (const char *data
, int data_length
,
3240 const cpp_char_column_policy
&policy
) :
3243 m_bytes_left (data_length
),
3247 gcc_assert (policy
.m_tabstop
> 0);
3248 gcc_assert (policy
.m_width_cb
);
3252 /* The main implementation function for class cpp_display_width_computation.
3253 m_next points on entry to the start of the UTF-8 encoding of the next
3254 character, and is updated to point just after the last byte of the encoding.
3255 m_bytes_left contains on entry the remaining size of the buffer into which
3256 m_next points, and this is also updated accordingly. If m_next does not
3257 point to a valid UTF-8-encoded sequence, then it will be treated as a single
3258 byte with display width 1. m_cur_display_col is the current display column,
3259 relative to which tab stops should be expanded. Returns the display width of
3260 the codepoint just processed.
3261 If OUT is non-NULL, it is populated. */
3264 cpp_display_width_computation::process_next_codepoint (cpp_decoded_char
*out
)
3270 out
->m_start_byte
= m_next
;
3272 if (*m_next
== '\t')
3276 next_width
= m_policy
.m_tabstop
- (m_display_cols
% m_policy
.m_tabstop
);
3280 out
->m_valid_ch
= true;
3283 else if (one_utf8_to_cppchar ((const uchar
**) &m_next
, &m_bytes_left
, &c
)
3286 /* Input is not convertible to UTF-8. This could be fine, e.g. in a
3287 string literal, so don't complain. Just treat it as if it has a width
3291 next_width
= m_policy
.m_undecoded_byte_width
;
3293 out
->m_valid_ch
= false;
3297 /* one_utf8_to_cppchar() has updated m_next and m_bytes_left for us. */
3298 next_width
= m_policy
.m_width_cb (c
);
3302 out
->m_valid_ch
= true;
3307 out
->m_next_byte
= m_next
;
3309 m_display_cols
+= next_width
;
3313 /* Utility to advance the byte stream by the minimum amount needed to consume
3314 N display columns. Returns the number of display columns that were
3315 actually skipped. This could be less than N, if there was not enough data,
3316 or more than N, if the last character to be skipped had a sufficiently large
3319 cpp_display_width_computation::advance_display_cols (int n
)
3321 const int start
= m_display_cols
;
3322 const int target
= start
+ n
;
3323 while (m_display_cols
< target
&& !done ())
3324 process_next_codepoint (NULL
);
3325 return m_display_cols
- start
;
3328 /* For the string of length DATA_LENGTH bytes that begins at DATA, compute
3329 how many display columns are occupied by the first COLUMN bytes. COLUMN
3330 may exceed DATA_LENGTH, in which case the phantom bytes at the end are
3331 treated as if they have display width 1. Tabs are expanded to the next tab
3332 stop, relative to the start of DATA, and non-printable-ASCII characters
3333 will be escaped as per POLICY. */
3336 cpp_byte_column_to_display_column (const char *data
, int data_length
,
3338 const cpp_char_column_policy
&policy
)
3340 const int offset
= MAX (0, column
- data_length
);
3341 cpp_display_width_computation
dw (data
, column
- offset
, policy
);
3343 dw
.process_next_codepoint (NULL
);
3344 return dw
.display_cols_processed () + offset
;
3347 /* For the string of length DATA_LENGTH bytes that begins at DATA, compute
3348 the least number of bytes that will result in at least DISPLAY_COL display
3349 columns. The return value may exceed DATA_LENGTH if the entire string does
3350 not occupy enough display columns. Non-printable-ASCII characters
3351 will be escaped as per POLICY. */
3354 cpp_display_column_to_byte_column (const char *data
, int data_length
,
3356 const cpp_char_column_policy
&policy
)
3358 cpp_display_width_computation
dw (data
, data_length
, policy
);
3359 const int avail_display
= dw
.advance_display_cols (display_col
);
3360 return dw
.bytes_processed () + MAX (0, display_col
- avail_display
);
3363 template <typename PropertyType
>
3365 get_cppchar_property (cppchar_t c
,
3366 const cppchar_t
*range_ends
,
3367 const PropertyType
*range_values
,
3369 PropertyType default_value
)
3371 if (__builtin_expect (c
<= range_ends
[0], true))
3372 return range_values
[0];
3374 /* Binary search the tables. */
3376 static const int end
= num_ranges
;
3377 int len
= end
- begin
;
3381 int middle
= begin
+ half
;
3382 if (c
> range_ends
[middle
])
3391 if (__builtin_expect (begin
!= end
, true))
3392 return range_values
[begin
];
3394 return default_value
;
3397 /* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc,
3398 because that will inspect the user's locale, and in particular in an ASCII
3399 locale, it will not return anything useful for extended characters. But GCC
3400 in other respects (see e.g. _cpp_default_encoding()) behaves as if
3401 everything is UTF-8. We also make some tweaks that are useful for the way
3402 GCC needs to use this data, e.g. tabs and other control characters should be
3403 treated as having width 1. The lookup tables are generated from
3404 contrib/unicode/gen_wcwidth.py and were made by simply calling glibc
3405 wcwidth() on all codepoints, then applying the small tweaks. These tables
3406 are not highly optimized, but for the present purpose of outputting
3407 diagnostics, they are sufficient. */
3409 #include "generated_cpp_wcwidth.h"
3412 cpp_wcwidth (cppchar_t c
)
3414 const size_t num_ranges
3415 = sizeof wcwidth_range_ends
/ sizeof (*wcwidth_range_ends
);
3416 return get_cppchar_property
<unsigned char > (c
,
3417 &wcwidth_range_ends
[0],
3423 #include "combining-chars.inc"
3426 cpp_is_combining_char (cppchar_t c
)
3428 const size_t num_ranges
3429 = sizeof combining_range_ends
/ sizeof (*combining_range_ends
);
3430 return get_cppchar_property
<bool> (c
,
3431 &combining_range_ends
[0],
3437 #include "printable-chars.inc"
3440 cpp_is_printable_char (cppchar_t c
)
3442 const size_t num_ranges
3443 = sizeof printable_range_ends
/ sizeof (*printable_range_ends
);
3444 return get_cppchar_property
<bool> (c
,
3445 &printable_range_ends
[0],