1 /* CPP Library - charsets
2 Copyright (C) 1998-2023 Free Software Foundation, Inc.
4 Broken out of c-lex.cc Apr 2003, adding valid C99 UCN ranges.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
25 /* Character set handling for C-family languages.
27 Terminological note: In what follows, "charset" or "character set"
28 will be taken to mean both an abstract set of characters and an
29 encoding for that set.
31 The C99 standard discusses two character sets: source and execution.
32 The source character set is used for internal processing in translation
33 phases 1 through 4; the execution character set is used thereafter.
34 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36 of these terms). Furthermore, the "basic character set" (listed in
37 5.2.1p3) is to be encoded in each with values one byte wide, and is
38 to appear in the initial shift state.
40 It is not explicitly mentioned, but there is also a "wide execution
41 character set" used to encode wide character constants and wide
42 string literals; this is supposed to be the result of applying the
43 standard library function mbstowcs() to an equivalent narrow string
44 (6.4.5p5). However, the behavior of hexadecimal and octal
45 \-escapes is at odds with this; they are supposed to be translated
46 directly to wchar_t values (6.4.4.4p5,6).
48 The source character set is not necessarily the character set used
49 to encode physical source files on disk; translation phase 1 converts
50 from whatever that encoding is to the source character set.
52 The presence of universal character names in C99 (6.4.3 et seq.)
53 forces the source character set to be isomorphic to ISO 10646,
54 that is, Unicode. There is no such constraint on the execution
55 character set; note also that the conversion from source to
56 execution character set does not occur for identifiers (5.1.1.2p1#5).
58 For convenience of implementation, the source character set's
59 encoding of the basic character set should be identical to the
60 execution character set OF THE HOST SYSTEM's encoding of the basic
61 character set, and it should not be a state-dependent encoding.
63 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64 depending on whether the host is based on ASCII or EBCDIC (see
65 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66 Technical Report #16). With limited exceptions, it relies on the
67 system library's iconv() primitive to do charset conversion
68 (specified in SUSv2). */
71 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72 below, which are guarded only by if statements with compile-time
73 constant conditions, do not cause link errors. */
74 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76 #define iconv_close(x) (void)0
80 #if HOST_CHARSET == HOST_CHARSET_ASCII
81 #define SOURCE_CHARSET "UTF-8"
82 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84 #define SOURCE_CHARSET "UTF-EBCDIC"
85 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
87 #error "Unrecognized basic host character set"
94 /* This structure is used for a resizable string buffer throughout. */
95 /* Don't call it strbuf, as that conflicts with unistd.h on systems
96 such as DYNIX/ptx where unistd.h includes stropts.h. */
104 /* This is enough to hold any string that fits on a single 80-column
105 line, even if iconv quadruples its size (e.g. conversion from
106 ASCII to UTF-32) rounded up to a power of two. */
107 #define OUTBUF_BLOCK_SIZE 256
109 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110 logic. This is because a depressing number of systems lack iconv,
111 or have have iconv libraries that do not do these conversions, so
112 we need a fallback implementation for them. To ensure the fallback
113 doesn't break due to neglect, it is used on all systems.
115 UTF-32 encoding is nice and simple: a four-byte binary number,
116 constrained to the range 00000000-7FFFFFFF to avoid questions of
117 signedness. We do have to cope with big- and little-endian
120 UTF-16 encoding uses two-byte binary numbers, again in big- and
121 little-endian variants, for all values in the 00000000-0000FFFF
122 range. Values in the 00010000-0010FFFF range are encoded as pairs
123 of two-byte numbers, called "surrogate pairs": given a number S in
124 this range, it is mapped to a pair (H, L) as follows:
126 H = (S - 0x10000) / 0x400 + 0xD800
127 L = (S - 0x10000) % 0x400 + 0xDC00
129 Two-byte values in the D800...DFFF range are ill-formed except as a
130 component of a surrogate pair. Even if the encoding within a
131 two-byte value is little-endian, the H member of the surrogate pair
134 There is no way to encode values in the 00110000-7FFFFFFF range,
135 which is not currently a problem as there are no assigned code
136 points in that range; however, the author expects that it will
137 eventually become necessary to abandon UTF-16 due to this
138 limitation. Note also that, because of these pairs, UTF-16 does
139 not meet the requirements of the C standard for a wide character
140 encoding (see 3.7.3 and 6.4.4.4p11).
142 UTF-8 encoding looks like this:
144 value range encoded as
145 00000000-0000007F 0xxxxxxx
146 00000080-000007FF 110xxxxx 10xxxxxx
147 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
152 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154 never occur. Note also that any value that can be encoded by a
155 given row of the table can also be encoded by all successive rows,
156 but this is not done; only the shortest possible encoding for any
157 given value is valid. For instance, the character 07C0 could be
158 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159 FC 80 80 80 9F 80. Only the first is valid.
161 An implementation note: the transformation from UTF-16 to UTF-8, or
162 vice versa, is easiest done by using UTF-32 as an intermediary. */
164 /* Internal primitives which go from an UTF-8 byte stream to native-endian
165 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166 operation in several places below. */
168 one_utf8_to_cppchar (const uchar
**inbufp
, size_t *inbytesleftp
,
171 static const uchar masks
[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172 static const uchar patns
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
175 const uchar
*inbuf
= *inbufp
;
178 if (*inbytesleftp
< 1)
190 /* The number of leading 1-bits in the first byte indicates how many
192 for (nbytes
= 2; nbytes
< 7; nbytes
++)
193 if ((c
& ~masks
[nbytes
-1]) == patns
[nbytes
-1])
198 if (*inbytesleftp
< nbytes
)
201 c
= (c
& masks
[nbytes
-1]);
203 for (i
= 1; i
< nbytes
; i
++)
205 cppchar_t n
= *inbuf
++;
206 if ((n
& 0xC0) != 0x80)
208 c
= ((c
<< 6) + (n
& 0x3F));
211 /* Make sure the shortest possible encoding was used. */
212 if (c
<= 0x7F && nbytes
> 1) return EILSEQ
;
213 if (c
<= 0x7FF && nbytes
> 2) return EILSEQ
;
214 if (c
<= 0xFFFF && nbytes
> 3) return EILSEQ
;
215 if (c
<= 0x1FFFFF && nbytes
> 4) return EILSEQ
;
216 if (c
<= 0x3FFFFFF && nbytes
> 5) return EILSEQ
;
218 /* Make sure the character is valid. */
219 if (c
> 0x7FFFFFFF || (c
>= 0xD800 && c
<= 0xDFFF)) return EILSEQ
;
223 *inbytesleftp
-= nbytes
;
228 one_cppchar_to_utf8 (cppchar_t c
, uchar
**outbufp
, size_t *outbytesleftp
)
230 static const uchar masks
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231 static const uchar limits
[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
233 uchar buf
[6], *p
= &buf
[6];
234 uchar
*outbuf
= *outbufp
;
243 *--p
= ((c
& 0x3F) | 0x80);
247 while (c
>= 0x3F || (c
& limits
[nbytes
-1]));
248 *--p
= (c
| masks
[nbytes
-1]);
251 if (*outbytesleftp
< nbytes
)
256 *outbytesleftp
-= nbytes
;
261 /* The following four functions transform one character between the two
262 encodings named in the function name. All have the signature
263 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264 uchar **outbufp, size_t *outbytesleftp)
266 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267 interpreted as a boolean indicating whether big-endian or
268 little-endian encoding is to be used for the member of the pair
271 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
274 The return value is either 0 for success, or an errno value for
275 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276 input sequence), ir EINVAL (incomplete input sequence). */
279 one_utf8_to_utf32 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
280 uchar
**outbufp
, size_t *outbytesleftp
)
286 /* Check for space first, since we know exactly how much we need. */
287 if (*outbytesleftp
< 4)
290 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
295 outbuf
[bigend
? 3 : 0] = (s
& 0x000000FF);
296 outbuf
[bigend
? 2 : 1] = (s
& 0x0000FF00) >> 8;
297 outbuf
[bigend
? 1 : 2] = (s
& 0x00FF0000) >> 16;
298 outbuf
[bigend
? 0 : 3] = (s
& 0xFF000000) >> 24;
306 one_utf32_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
307 uchar
**outbufp
, size_t *outbytesleftp
)
313 if (*inbytesleftp
< 4)
318 s
= inbuf
[bigend
? 0 : 3] << 24;
319 s
+= inbuf
[bigend
? 1 : 2] << 16;
320 s
+= inbuf
[bigend
? 2 : 1] << 8;
321 s
+= inbuf
[bigend
? 3 : 0];
323 if (s
>= 0x7FFFFFFF || (s
>= 0xD800 && s
<= 0xDFFF))
326 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
336 one_utf8_to_utf16 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
337 uchar
**outbufp
, size_t *outbytesleftp
)
341 const uchar
*save_inbuf
= *inbufp
;
342 size_t save_inbytesleft
= *inbytesleftp
;
343 uchar
*outbuf
= *outbufp
;
345 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
351 *inbufp
= save_inbuf
;
352 *inbytesleftp
= save_inbytesleft
;
358 if (*outbytesleftp
< 2)
360 *inbufp
= save_inbuf
;
361 *inbytesleftp
= save_inbytesleft
;
364 outbuf
[bigend
? 1 : 0] = (s
& 0x00FF);
365 outbuf
[bigend
? 0 : 1] = (s
& 0xFF00) >> 8;
375 if (*outbytesleftp
< 4)
377 *inbufp
= save_inbuf
;
378 *inbytesleftp
= save_inbytesleft
;
382 hi
= (s
- 0x10000) / 0x400 + 0xD800;
383 lo
= (s
- 0x10000) % 0x400 + 0xDC00;
385 /* Even if we are little-endian, put the high surrogate first.
386 ??? Matches practice? */
387 outbuf
[bigend
? 1 : 0] = (hi
& 0x00FF);
388 outbuf
[bigend
? 0 : 1] = (hi
& 0xFF00) >> 8;
389 outbuf
[bigend
? 3 : 2] = (lo
& 0x00FF);
390 outbuf
[bigend
? 2 : 3] = (lo
& 0xFF00) >> 8;
399 one_utf16_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
400 uchar
**outbufp
, size_t *outbytesleftp
)
403 const uchar
*inbuf
= *inbufp
;
406 if (*inbytesleftp
< 2)
408 s
= inbuf
[bigend
? 0 : 1] << 8;
409 s
+= inbuf
[bigend
? 1 : 0];
411 /* Low surrogate without immediately preceding high surrogate is invalid. */
412 if (s
>= 0xDC00 && s
<= 0xDFFF)
414 /* High surrogate must have a following low surrogate. */
415 else if (s
>= 0xD800 && s
<= 0xDBFF)
417 cppchar_t hi
= s
, lo
;
418 if (*inbytesleftp
< 4)
421 lo
= inbuf
[bigend
? 2 : 3] << 8;
422 lo
+= inbuf
[bigend
? 3 : 2];
424 if (lo
< 0xDC00 || lo
> 0xDFFF)
427 s
= (hi
- 0xD800) * 0x400 + (lo
- 0xDC00) + 0x10000;
430 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
434 /* Success - update the input pointers (one_cppchar_to_utf8 has done
435 the output pointers for us). */
449 /* Helper routine for the next few functions. The 'const' on
450 one_conversion means that we promise not to modify what function is
451 pointed to, which lets the inliner see through it. */
454 conversion_loop (int (*const one_conversion
)(iconv_t
, const uchar
**, size_t *,
456 iconv_t cd
, const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
460 size_t inbytesleft
, outbytesleft
;
465 outbuf
= to
->text
+ to
->len
;
466 outbytesleft
= to
->asize
- to
->len
;
471 rval
= one_conversion (cd
, &inbuf
, &inbytesleft
,
472 &outbuf
, &outbytesleft
);
473 while (inbytesleft
&& !rval
);
475 if (__builtin_expect (inbytesleft
== 0, 1))
477 to
->len
= to
->asize
- outbytesleft
;
486 outbytesleft
+= OUTBUF_BLOCK_SIZE
;
487 to
->asize
+= OUTBUF_BLOCK_SIZE
;
488 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
489 outbuf
= to
->text
+ to
->asize
- outbytesleft
;
494 /* These functions convert entire strings between character sets.
495 They all have the signature
497 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
499 The input string FROM is converted as specified by the function
500 name plus the iconv descriptor CD (which may be fake), and the
501 result appended to TO. On any error, false is returned, otherwise true. */
503 /* These four use the custom conversion code above. */
505 convert_utf8_utf16 (iconv_t cd
, const uchar
*from
, size_t flen
,
506 struct _cpp_strbuf
*to
)
508 return conversion_loop (one_utf8_to_utf16
, cd
, from
, flen
, to
);
512 convert_utf8_utf32 (iconv_t cd
, const uchar
*from
, size_t flen
,
513 struct _cpp_strbuf
*to
)
515 return conversion_loop (one_utf8_to_utf32
, cd
, from
, flen
, to
);
519 convert_utf16_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
520 struct _cpp_strbuf
*to
)
522 return conversion_loop (one_utf16_to_utf8
, cd
, from
, flen
, to
);
526 convert_utf32_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
527 struct _cpp_strbuf
*to
)
529 return conversion_loop (one_utf32_to_utf8
, cd
, from
, flen
, to
);
532 /* Identity conversion, used when we have no alternative. */
534 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED
,
535 const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
537 if (to
->len
+ flen
> to
->asize
)
539 to
->asize
= to
->len
+ flen
;
540 to
->asize
+= to
->asize
/ 4;
541 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
543 memcpy (to
->text
+ to
->len
, from
, flen
);
548 /* And this one uses the system iconv primitive. It's a little
549 different, since iconv's interface is a little different. */
552 #define CONVERT_ICONV_GROW_BUFFER \
554 outbytesleft += OUTBUF_BLOCK_SIZE; \
555 to->asize += OUTBUF_BLOCK_SIZE; \
556 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
557 outbuf = (char *)to->text + to->asize - outbytesleft; \
561 convert_using_iconv (iconv_t cd
, const uchar
*from
, size_t flen
,
562 struct _cpp_strbuf
*to
)
564 ICONV_CONST
char *inbuf
;
566 size_t inbytesleft
, outbytesleft
;
568 /* Reset conversion descriptor and check that it is valid. */
569 if (iconv (cd
, 0, 0, 0, 0) == (size_t)-1)
572 inbuf
= (ICONV_CONST
char *)from
;
574 outbuf
= (char *)to
->text
+ to
->len
;
575 outbytesleft
= to
->asize
- to
->len
;
579 iconv (cd
, &inbuf
, &inbytesleft
, &outbuf
, &outbytesleft
);
580 if (__builtin_expect (inbytesleft
== 0, 1))
582 /* Close out any shift states, returning to the initial state. */
583 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
588 CONVERT_ICONV_GROW_BUFFER
;
589 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
593 to
->len
= to
->asize
- outbytesleft
;
599 CONVERT_ICONV_GROW_BUFFER
;
603 #define convert_using_iconv 0 /* prevent undefined symbol error below */
606 /* Arrange for the above custom conversion logic to be used automatically
607 when conversion between a suitable pair of character sets is requested. */
609 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
610 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
612 struct cpp_conversion
618 static const struct cpp_conversion conversion_tab
[] = {
619 { "UTF-8/UTF-32LE", convert_utf8_utf32
, (iconv_t
)0 },
620 { "UTF-8/UTF-32BE", convert_utf8_utf32
, (iconv_t
)1 },
621 { "UTF-8/UTF-16LE", convert_utf8_utf16
, (iconv_t
)0 },
622 { "UTF-8/UTF-16BE", convert_utf8_utf16
, (iconv_t
)1 },
623 { "UTF-32LE/UTF-8", convert_utf32_utf8
, (iconv_t
)0 },
624 { "UTF-32BE/UTF-8", convert_utf32_utf8
, (iconv_t
)1 },
625 { "UTF-16LE/UTF-8", convert_utf16_utf8
, (iconv_t
)0 },
626 { "UTF-16BE/UTF-8", convert_utf16_utf8
, (iconv_t
)1 },
629 /* Subroutine of cpp_init_iconv: initialize and return a
630 cset_converter structure for conversion from FROM to TO. If
631 iconv_open() fails, issue an error and return an identity
632 converter. Silently return an identity converter if FROM and TO
635 PFILE is only used for generating diagnostics; setting it to NULL
636 suppresses diagnostics. */
638 static struct cset_converter
639 init_iconv_desc (cpp_reader
*pfile
, const char *to
, const char *from
)
641 struct cset_converter ret
;
648 if (!strcasecmp (to
, from
))
650 ret
.func
= convert_no_conversion
;
651 ret
.cd
= (iconv_t
) -1;
656 pair
= (char *) alloca(strlen(to
) + strlen(from
) + 2);
661 for (i
= 0; i
< ARRAY_SIZE (conversion_tab
); i
++)
662 if (!strcasecmp (pair
, conversion_tab
[i
].pair
))
664 ret
.func
= conversion_tab
[i
].func
;
665 ret
.cd
= conversion_tab
[i
].fake_cd
;
670 /* No custom converter - try iconv. */
673 ret
.func
= convert_using_iconv
;
674 ret
.cd
= iconv_open (to
, from
);
677 if (ret
.cd
== (iconv_t
) -1)
682 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME should be DL_SORRY */
683 "conversion from %s to %s not supported by iconv",
686 cpp_errno (pfile
, CPP_DL_ERROR
, "iconv_open");
688 ret
.func
= convert_no_conversion
;
695 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME: should be DL_SORRY */
696 "no iconv implementation, cannot convert from %s to %s",
699 ret
.func
= convert_no_conversion
;
700 ret
.cd
= (iconv_t
) -1;
707 /* If charset conversion is requested, initialize iconv(3) descriptors
708 for conversion from the source character set to the execution
709 character sets. If iconv is not present in the C library, and
710 conversion is requested, issue an error. */
713 cpp_init_iconv (cpp_reader
*pfile
)
715 const char *ncset
= CPP_OPTION (pfile
, narrow_charset
);
716 const char *wcset
= CPP_OPTION (pfile
, wide_charset
);
717 const char *default_wcset
;
719 bool be
= CPP_OPTION (pfile
, bytes_big_endian
);
721 if (CPP_OPTION (pfile
, wchar_precision
) >= 32)
722 default_wcset
= be
? "UTF-32BE" : "UTF-32LE";
723 else if (CPP_OPTION (pfile
, wchar_precision
) >= 16)
724 default_wcset
= be
? "UTF-16BE" : "UTF-16LE";
726 /* This effectively means that wide strings are not supported,
727 so don't do any conversion at all. */
728 default_wcset
= SOURCE_CHARSET
;
731 ncset
= SOURCE_CHARSET
;
733 wcset
= default_wcset
;
735 pfile
->narrow_cset_desc
= init_iconv_desc (pfile
, ncset
, SOURCE_CHARSET
);
736 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
737 pfile
->utf8_cset_desc
= init_iconv_desc (pfile
, "UTF-8", SOURCE_CHARSET
);
738 pfile
->utf8_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
739 pfile
->char16_cset_desc
= init_iconv_desc (pfile
,
740 be
? "UTF-16BE" : "UTF-16LE",
742 pfile
->char16_cset_desc
.width
= 16;
743 pfile
->char32_cset_desc
= init_iconv_desc (pfile
,
744 be
? "UTF-32BE" : "UTF-32LE",
746 pfile
->char32_cset_desc
.width
= 32;
747 pfile
->wide_cset_desc
= init_iconv_desc (pfile
, wcset
, SOURCE_CHARSET
);
748 pfile
->wide_cset_desc
.width
= CPP_OPTION (pfile
, wchar_precision
);
751 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
753 _cpp_destroy_iconv (cpp_reader
*pfile
)
757 if (pfile
->narrow_cset_desc
.func
== convert_using_iconv
)
758 iconv_close (pfile
->narrow_cset_desc
.cd
);
759 if (pfile
->utf8_cset_desc
.func
== convert_using_iconv
)
760 iconv_close (pfile
->utf8_cset_desc
.cd
);
761 if (pfile
->char16_cset_desc
.func
== convert_using_iconv
)
762 iconv_close (pfile
->char16_cset_desc
.cd
);
763 if (pfile
->char32_cset_desc
.func
== convert_using_iconv
)
764 iconv_close (pfile
->char32_cset_desc
.cd
);
765 if (pfile
->wide_cset_desc
.func
== convert_using_iconv
)
766 iconv_close (pfile
->wide_cset_desc
.cd
);
770 /* Utility routine for use by a full compiler. C is a character taken
771 from the *basic* source character set, encoded in the host's
772 execution encoding. Convert it to (the target's) execution
773 encoding, and return that value.
775 Issues an internal error if C's representation in the narrow
776 execution character set fails to be a single-byte value (C99
777 5.2.1p3: "The representation of each member of the source and
778 execution character sets shall fit in a byte.") May also issue an
779 internal error if C fails to be a member of the basic source
780 character set (testing this exactly is too hard, especially when
781 the host character set is EBCDIC). */
783 cpp_host_to_exec_charset (cpp_reader
*pfile
, cppchar_t c
)
786 struct _cpp_strbuf tbuf
;
788 /* This test is merely an approximation, but it suffices to catch
789 the most important thing, which is that we don't get handed a
790 character outside the unibyte range of the host character set. */
791 if (c
> LAST_POSSIBLY_BASIC_SOURCE_CHAR
)
793 cpp_error (pfile
, CPP_DL_ICE
,
794 "character 0x%lx is not in the basic source character set\n",
799 /* Being a character in the unibyte range of the host character set,
800 we can safely splat it into a one-byte buffer and trust that that
801 is a well-formed string. */
804 /* This should never need to reallocate, but just in case... */
806 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
809 if (!APPLY_CONVERSION (pfile
->narrow_cset_desc
, sbuf
, 1, &tbuf
))
811 cpp_errno (pfile
, CPP_DL_ICE
, "converting to execution character set");
816 cpp_error (pfile
, CPP_DL_ICE
,
817 "character 0x%lx is not unibyte in execution character set",
828 /* cpp_substring_ranges's constructor. */
830 cpp_substring_ranges::cpp_substring_ranges () :
835 m_ranges
= XNEWVEC (source_range
, m_alloc_ranges
);
838 /* cpp_substring_ranges's destructor. */
840 cpp_substring_ranges::~cpp_substring_ranges ()
845 /* Add RANGE to the vector of source_range information. */
848 cpp_substring_ranges::add_range (source_range range
)
850 if (m_num_ranges
>= m_alloc_ranges
)
854 = (source_range
*)xrealloc (m_ranges
,
855 sizeof (source_range
) * m_alloc_ranges
);
857 m_ranges
[m_num_ranges
++] = range
;
860 /* Read NUM ranges from LOC_READER, adding them to the vector of source_range
864 cpp_substring_ranges::add_n_ranges (int num
,
865 cpp_string_location_reader
&loc_reader
)
867 for (int i
= 0; i
< num
; i
++)
868 add_range (loc_reader
.get_next ());
873 /* Utility routine that computes a mask of the form 0000...111... with
876 width_to_mask (size_t width
)
878 width
= MIN (width
, BITS_PER_CPPCHAR_T
);
879 if (width
>= CHAR_BIT
* sizeof (size_t))
882 return ((size_t) 1 << width
) - 1;
885 /* A large table of unicode character information. */
887 /* Valid in a C99 identifier? */
889 /* Valid in a C99 identifier, but not as the first character? */
891 /* Valid in a C++ identifier? */
893 /* Valid in a C11/C++11 identifier? */
895 /* Valid in a C11/C++11 identifier, but not as the first character? */
897 /* Valid in a C++23 identifier? */
899 /* Valid in a C++23 identifier, but not as the first character? */
901 /* NFC representation is not valid in an identifier? */
903 /* Might be valid NFC form? */
905 /* Might be valid NFKC form? */
907 /* Certain preceding characters might make it not valid NFC/NKFC form? */
912 /* Bitmap of flags above. */
913 unsigned short flags
;
914 /* Combining class of the character. */
915 unsigned char combine
;
916 /* Last character in the range described by this entry. */
921 /* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */
922 #define UCS_LIMIT 0x10FFFF
926 static const char hangul_syllables
[][4] = {
928 "G", "GG", "N", "D", "DD", "R", "M", "B", "BB", "S", "SS", "",
929 "J", "JJ", "C", "K", "T", "P", "H",
931 "A", "AE", "YA", "YAE", "EO", "E", "YEO", "YE", "O", "WA", "WAE",
932 "OE", "YO", "U", "WEO", "WE", "WI", "YU", "EU", "YI", "I",
934 "", "G", "GG", "GS", "N", "NJ", "NH", "D", "L", "LG", "LM", "LB",
935 "LS", "LT", "LP", "LH", "M", "B", "BS", "S", "SS", "NG", "J", "C",
939 static const short hangul_count
[6] = { 19, 21, 28 };
941 /* Used for Unicode loose matching rule UAX44-LM2 matching. */
949 /* Map NAME, a Unicode character name or correction/control/alternate
950 alias, to a Unicode codepoint, or return (cppchar_t) -1 if
951 not found. This uses a space optimized radix tree precomputed
952 by the makeuname2c utility, with binary format documented in its
953 source makeuname2c.cc. */
956 _cpp_uname2c (const char *name
, size_t len
, const unsigned char *n
,
957 struct uname2c_data
*data
)
963 size_t key_len
, len_adj
;
964 bool has_value
= *n
& 0x40;
965 bool has_children
, no_sibling
= false;
966 cppchar_t codepoint
= -1;
967 const unsigned char *child
= NULL
;
972 k
= ' ' + (*n
++ & 0x3f);
978 key_len
= *n
++ & 0x3f;
979 key
= &uname2c_dict
[*n
++];
984 codepoint
= *n
+ (n
[1] << 8) + ((n
[2] & 0x1f) << 16);
985 has_children
= n
[2] & 0x80;
986 no_sibling
= n
[2] & 0x40;
993 unsigned int shift
= 0;
994 size_t child_off
= 0;
998 child_off
|= (*n
& 0x7f) << shift
;
1001 while ((*n
++ & 0x80) != 0);
1002 child
= n
+ child_off
;
1004 if (__builtin_expect (data
== NULL
, 1))
1006 ret
= memcmp (name
, key
, len
> key_len
? key_len
: len
);
1011 const char *p
= name
, *q
= key
;
1015 if ((size_t) (p
- name
) == len
|| (size_t) (q
- key
) == key_len
)
1024 /* This is the hard case. Only medial hyphens
1025 should be removed, where medial means preceded
1026 and followed by alnum. */
1027 if (ISALNUM (q
== key
? data
->prev_char
: q
[-1]))
1029 if (q
+ 1 == key
+ key_len
)
1031 /* We don't know what the next letter will be.
1032 It could be ISALNUM, then we are supposed
1033 to omit it, or it could be a space and then
1034 we should not omit it and need to compare it.
1035 Fortunately the only 3 names with hyphen
1036 followed by non-letter are
1037 U+0F0A TIBETAN MARK BKA- SHOG YIG MGO
1038 U+0FD0 TIBETAN MARK BKA- SHOG GI MGO RGYAN
1039 U+0FD0 TIBETAN MARK BSKA- SHOG GI MGO RGYAN
1040 Furthermore, prefixes of NR2 generated
1041 ranges all end with a hyphen, but the generated
1042 part is then followed by alpha-numeric.
1043 So, let's just assume that - at the end of
1044 key is always followed by alphanumeric and
1045 so should be omitted.
1046 makeuname2c.cc verifies that this is true. */
1050 else if (ISALNUM (q
[1]))
1063 /* If we don't consume the whole key, signal a mismatch,
1064 but always with ret = 1, so that we keep looking through
1066 ret
= q
< key
+ key_len
;
1074 else if (codepoint
>= 0xd800
1075 && codepoint
< 0xd800 + ARRAY_SIZE (uname2c_generated
))
1079 if (codepoint
== 0xd800)
1081 /* NR1 - Hangul syllables. */
1082 size_t start
= 0, end
, i
, j
;
1083 int this_len
, max_len
;
1086 for (i
= 0; i
< 3; ++i
)
1088 end
= start
+ hangul_count
[i
];
1091 for (j
= start
; j
< end
; j
++)
1093 this_len
= strlen (hangul_syllables
[j
]);
1094 if (len
>= (size_t) this_len
1095 && this_len
> max_len
1096 && memcmp (name
, hangul_syllables
[j
],
1100 winner
[i
] = j
- start
;
1109 if (__builtin_expect (data
!= NULL
, 0))
1111 memcpy (data
->canon_name
, key
, key_len
);
1112 data
->canon_name
[key_len
] = '\0';
1113 for (i
= 0, start
= 0; i
< 3; ++i
)
1115 strcat (data
->canon_name
,
1116 hangul_syllables
[start
+ winner
[i
]]);
1117 start
+= hangul_count
[i
];
1120 return (0xac00 + 21 * 28 * winner
[0]
1121 + 28 * winner
[1] + winner
[2]);
1125 /* NR2 - prefix followed by hexadecimal codepoint. */
1129 if (len
< 4 || len
> 5)
1131 p
= uname2c_pairs
+ uname2c_generated
[codepoint
- 0xd800];
1133 for (i
= 0; i
< len
; ++i
)
1136 if (!ISXDIGIT (name
[i
]))
1138 codepoint
+= hex_value (name
[i
]);
1143 else if (codepoint
<= p
[1])
1145 if (__builtin_expect (data
!= NULL
, 0))
1147 memcpy (data
->canon_name
, key
, key_len
);
1148 memcpy (data
->canon_name
+ key_len
, name
, len
);
1149 data
->canon_name
[key_len
+ len
] = '\0';
1156 else if (__builtin_expect (data
!= NULL
, 0))
1160 memcpy (data
->canon_name
, key
, key_len
);
1161 data
->canon_name
[key_len
] = '\0';
1166 struct uname2c_data save
= *data
;
1167 memcpy (data
->canon_name
, key
, key_len
);
1168 data
->canon_name
+= key_len
;
1169 data
->prev_char
= key
[key_len
- 1];
1170 codepoint
= _cpp_uname2c (name
+ len_adj
, len
- len_adj
,
1172 if (codepoint
!= (cppchar_t
) -1)
1177 else if (len
== len_adj
)
1179 else if (!has_children
)
1189 if (no_sibling
|| (!has_value
&& *n
== 0xff))
1196 /* Try to do a loose name lookup according to Unicode loose matching rule
1197 UAX44-LM2. First ignore medial hyphens, whitespace, underscore
1198 characters and convert to upper case. */
1201 _cpp_uname2c_uax44_lm2 (const char *name
, size_t len
, char *canon_name
)
1203 char name_after_uax44_lm2
[uname2c_max_name_len
];
1204 char *q
= name_after_uax44_lm2
;
1207 for (p
= name
; p
< name
+ len
; p
++)
1208 if (*p
== '_' || *p
== ' ')
1210 else if (*p
== '-' && p
!= name
&& ISALNUM (p
[-1]) && ISALNUM (p
[1]))
1212 else if (q
== name_after_uax44_lm2
+ uname2c_max_name_len
)
1214 else if (ISLOWER (*p
))
1215 *q
++ = TOUPPER (*p
);
1219 struct uname2c_data data
;
1220 data
.canon_name
= canon_name
;
1221 data
.prev_char
= ' ';
1222 /* Hangul Jungseong O- E after UAX44-LM2 should be HANGULJUNGSEONGO-E
1223 and so should match U+1180. */
1224 if (q
- name_after_uax44_lm2
== sizeof ("HANGULJUNGSEONGO-E") - 1
1225 && memcmp (name_after_uax44_lm2
, "HANGULJUNGSEONGO-E",
1226 sizeof ("HANGULJUNGSEONGO-E") - 1) == 0)
1228 name_after_uax44_lm2
[sizeof ("HANGULJUNGSEONGO") - 1] = 'E';
1232 = _cpp_uname2c (name_after_uax44_lm2
, q
- name_after_uax44_lm2
,
1233 uname2c_tree
, &data
);
1235 /* Unicode UAX44-LM2 exception:
1236 U+116C HANGUL JUNGSEONG OE
1237 U+1180 HANGUL JUNGSEONG O-E
1238 We remove all medial hyphens when we shouldn't remote the U+1180 one.
1239 The U+1180 entry sorts before U+116C lexicographilly, so we get U+1180
1240 in both cases. Thus, if result is U+1180, check if user's name doesn't
1241 have a hyphen there and adjust. */
1242 if (result
== 0x1180)
1244 while (p
[-1] == ' ' || p
[-1] == '_')
1246 gcc_assert (TOUPPER (p
[-1]) == 'E');
1248 while (p
[-1] == ' ' || p
[-1] == '_')
1253 memcpy (canon_name
+ sizeof ("HANGUL JUNGSEONG O") - 1, "E", 2);
1260 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
1261 the start of an identifier, and 0 if C is not valid in an
1262 identifier. We assume C has already gone through the checks of
1263 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
1264 algorithm is a simple binary search on the table defined in
1268 ucn_valid_in_identifier (cpp_reader
*pfile
, cppchar_t c
,
1269 struct normalize_state
*nst
)
1272 unsigned short valid_flags
, invalid_start_flags
;
1278 mx
= ARRAY_SIZE (ucnranges
) - 1;
1282 if (c
<= ucnranges
[md
].end
)
1288 /* When -pedantic, we require the character to have been listed by
1289 the standard for the current language. Otherwise, we accept the
1290 union of the acceptable sets for all supported language versions. */
1291 valid_flags
= C99
| CXX
| C11
| CXX23
;
1292 if (CPP_PEDANTIC (pfile
))
1294 if (CPP_OPTION (pfile
, xid_identifiers
))
1295 valid_flags
= CXX23
;
1296 else if (CPP_OPTION (pfile
, c11_identifiers
))
1298 else if (CPP_OPTION (pfile
, c99
))
1301 if (! (ucnranges
[mn
].flags
& valid_flags
))
1305 if (ucnranges
[mn
].combine
!= 0 && ucnranges
[mn
].combine
< nst
->prev_class
)
1306 nst
->level
= normalized_none
;
1307 else if (ucnranges
[mn
].flags
& CTX
)
1310 cppchar_t p
= nst
->previous
;
1312 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
1313 and are combined algorithmically from a sequence of the form
1314 1100-1112 1161-1175 11A8-11C2
1315 (if the third is not present, it is treated as 11A7, which is not
1316 really a valid character).
1317 Unfortunately, C99 allows (only) the NFC form, but C++ allows
1318 only the combining characters. */
1319 if (c
>= 0x1161 && c
<= 0x1175)
1320 safe
= p
< 0x1100 || p
> 0x1112;
1321 else if (c
>= 0x11A8 && c
<= 0x11C2)
1322 safe
= (p
< 0xAC00 || p
> 0xD7A3 || (p
- 0xAC00) % 28 != 0);
1324 safe
= check_nfc (pfile
, c
, p
);
1327 if ((c
>= 0x1161 && c
<= 0x1175) || (c
>= 0x11A8 && c
<= 0x11C2))
1328 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
1330 nst
->level
= normalized_none
;
1333 else if (ucnranges
[mn
].flags
& NKC
)
1335 else if (ucnranges
[mn
].flags
& NFC
)
1336 nst
->level
= MAX (nst
->level
, normalized_C
);
1337 else if (ucnranges
[mn
].flags
& CID
)
1338 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
1340 nst
->level
= normalized_none
;
1341 if (ucnranges
[mn
].combine
== 0)
1343 nst
->prev_class
= ucnranges
[mn
].combine
;
1345 if (!CPP_PEDANTIC (pfile
))
1347 /* If not -pedantic, accept as character that may
1348 begin an identifier a union of characters allowed
1349 at that position in each of the character sets. */
1350 if ((ucnranges
[mn
].flags
& (C99
| N99
)) == C99
1351 || (ucnranges
[mn
].flags
& CXX
) != 0
1352 || (ucnranges
[mn
].flags
& (C11
| N11
)) == C11
1353 || (ucnranges
[mn
].flags
& (CXX23
| NXX23
)) == CXX23
)
1358 if (CPP_OPTION (pfile
, xid_identifiers
))
1359 invalid_start_flags
= NXX23
;
1360 else if (CPP_OPTION (pfile
, c11_identifiers
))
1361 invalid_start_flags
= N11
;
1362 else if (CPP_OPTION (pfile
, c99
))
1363 invalid_start_flags
= N99
;
1365 invalid_start_flags
= 0;
1367 /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
1368 UCN combining characters may not begin identifiers. */
1369 if (ucnranges
[mn
].flags
& invalid_start_flags
)
1375 /* Increment char_range->m_finish by a single character. */
1378 extend_char_range (source_range
*char_range
,
1379 cpp_string_location_reader
*loc_reader
)
1383 gcc_assert (char_range
);
1384 char_range
->m_finish
= loc_reader
->get_next ().m_finish
;
1388 /* [lex.charset]: The character designated by the universal character
1389 name \UNNNNNNNN is that character whose character short name in
1390 ISO/IEC 10646 is NNNNNNNN; the character designated by the
1391 universal character name \uNNNN is that character whose character
1392 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
1393 for a universal character name corresponds to a surrogate code point
1394 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
1395 Additionally, if the hexadecimal value for a universal-character-name
1396 outside a character or string literal corresponds to a control character
1397 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
1398 character in the basic source character set, the program is ill-formed.
1400 C99 6.4.3: A universal character name shall not specify a character
1401 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
1402 or 0060 (`), nor one in the range D800 through DFFF inclusive.
1404 If the hexadecimal value is larger than the upper bound of the UCS
1405 codespace specified in ISO/IEC 10646, a pedantic warning is issued
1406 in all versions of C and in the C++20 or later versions of C++.
1408 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
1409 buffer end is delimited by a non-hex digit. Returns false if the
1410 UCN has not been consumed, true otherwise.
1412 The value of the UCN, whether valid or invalid, is returned in *CP.
1413 Diagnostics are emitted for invalid values. PSTR is updated to point
1414 one beyond the UCN, or to the syntactically invalid character.
1416 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
1417 an identifier, or 2 otherwise.
1419 If LOC_READER is non-NULL, then position information is
1420 read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
1423 _cpp_valid_ucn (cpp_reader
*pfile
, const uchar
**pstr
,
1424 const uchar
*limit
, int identifier_pos
,
1425 struct normalize_state
*nst
, cppchar_t
*cp
,
1426 source_range
*char_range
,
1427 cpp_string_location_reader
*loc_reader
)
1429 cppchar_t result
, c
;
1430 unsigned int length
;
1431 const uchar
*str
= *pstr
;
1432 const uchar
*base
= str
- 2;
1433 bool delimited
= false, named
= false;
1435 if (!CPP_OPTION (pfile
, cplusplus
) && !CPP_OPTION (pfile
, c99
))
1436 cpp_error (pfile
, CPP_DL_WARNING
,
1437 "universal character names are only valid in C++ and C99");
1438 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
) > 0
1439 && !CPP_OPTION (pfile
, cplusplus
))
1440 cpp_error (pfile
, CPP_DL_WARNING
,
1441 "C99's universal character names are incompatible with C90");
1442 else if (CPP_WTRADITIONAL (pfile
) && identifier_pos
== 0)
1443 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1444 "the meaning of '\\%c' is different in traditional C",
1454 || CPP_OPTION (pfile
, delimited_escape_seqs
)
1455 || !CPP_OPTION (pfile
, std
)))
1458 /* Magic value to indicate no digits seen. */
1461 extend_char_range (char_range
, loc_reader
);
1464 else if (str
[-1] == 'U')
1466 else if (str
[-1] == 'N')
1470 && !CPP_OPTION (pfile
, delimited_escape_seqs
)
1471 && CPP_OPTION (pfile
, std
))
1476 if (str
== limit
|| *str
!= '{')
1483 cpp_error (pfile
, CPP_DL_ERROR
, "'\\N' not followed by '{'");
1489 extend_char_range (char_range
, loc_reader
);
1491 const uchar
*name
= str
;
1499 if (!ISIDNUM (c
) && c
!= ' ' && c
!= '-')
1501 if (ISLOWER (c
) || c
== '_')
1504 extend_char_range (char_range
, loc_reader
);
1508 if (str
< limit
&& *str
== '}')
1510 if (identifier_pos
&& name
== str
)
1512 cpp_warning (pfile
, CPP_W_UNICODE
,
1513 "empty named universal character escape "
1514 "sequence; treating it as separate tokens");
1519 cpp_error (pfile
, CPP_DL_ERROR
,
1520 "empty named universal character escape sequence");
1521 else if ((!identifier_pos
|| strict
)
1522 && !CPP_OPTION (pfile
, delimited_escape_seqs
)
1523 && CPP_OPTION (pfile
, cpp_pedantic
))
1524 cpp_error (pfile
, CPP_DL_PEDWARN
,
1525 "named universal character escapes are only valid "
1531 /* If the name is longer than maximum length of a Unicode
1532 name, it can't be strictly valid. */
1533 if ((size_t) (str
- name
) > uname2c_max_name_len
|| !strict
)
1536 result
= _cpp_uname2c ((const char *) name
, str
- name
,
1537 uname2c_tree
, NULL
);
1538 if (result
== (cppchar_t
) -1)
1542 && (!CPP_OPTION (pfile
, delimited_escape_seqs
)
1544 ret
= cpp_warning (pfile
, CPP_W_UNICODE
,
1545 "\\N{%.*s} is not a valid "
1546 "universal character; treating it "
1547 "as separate tokens",
1548 (int) (str
- name
), name
);
1550 cpp_error (pfile
, CPP_DL_ERROR
,
1551 "\\N{%.*s} is not a valid universal "
1552 "character", (int) (str
- name
), name
);
1554 /* Try to do a loose name lookup according to
1555 Unicode loose matching rule UAX44-LM2. */
1556 char canon_name
[uname2c_max_name_len
+ 1];
1557 result
= _cpp_uname2c_uax44_lm2 ((const char *) name
,
1558 str
- name
, canon_name
);
1559 if (result
!= (cppchar_t
) -1 && ret
)
1560 cpp_error (pfile
, CPP_DL_NOTE
,
1561 "did you mean \\N{%s}?", canon_name
);
1565 && (!CPP_OPTION (pfile
, delimited_escape_seqs
)
1574 extend_char_range (char_range
, loc_reader
);
1576 else if (identifier_pos
)
1578 cpp_warning (pfile
, CPP_W_UNICODE
,
1579 "'\\N{' not terminated with '}' after %.*s; "
1580 "treating it as separate tokens",
1581 (int) (str
- base
), base
);
1587 cpp_error (pfile
, CPP_DL_ERROR
,
1588 "'\\N{' not terminated with '}' after %.*s",
1589 (int) (str
- base
), base
);
1596 cpp_error (pfile
, CPP_DL_ICE
, "In _cpp_valid_ucn but not a UCN");
1609 extend_char_range (char_range
, loc_reader
);
1613 /* Accept arbitrary number of leading zeros.
1614 16 is another magic value, smaller than 32 above
1615 and bigger than 8, so that upon encountering first
1616 non-zero digit we can count 8 digits and after that
1617 or in overflow bit and ensure length doesn't decrease
1618 to 0, as delimited escape sequence doesn't have upper
1619 bound on the number of hex digits. */
1621 else if (length
== 16 - 8)
1623 /* Make sure we detect overflows. */
1624 result
|= 0x8000000;
1629 result
= (result
<< 4) + hex_value (c
);
1633 if (delimited
&& str
< limit
&& *str
== '}')
1635 if (length
== 32 && identifier_pos
)
1637 cpp_warning (pfile
, CPP_W_UNICODE
,
1638 "empty delimited escape sequence; "
1639 "treating it as separate tokens");
1643 else if (length
== 32)
1644 cpp_error (pfile
, CPP_DL_ERROR
,
1645 "empty delimited escape sequence");
1646 else if (!CPP_OPTION (pfile
, delimited_escape_seqs
)
1647 && CPP_OPTION (pfile
, cpp_pedantic
))
1648 cpp_error (pfile
, CPP_DL_PEDWARN
,
1649 "delimited escape sequences are only valid in C++23");
1653 extend_char_range (char_range
, loc_reader
);
1656 /* Partial UCNs are not valid in strings, but decompose into
1657 multiple tokens in identifiers, so we can't give a helpful
1658 error message in that case. */
1659 if (length
&& identifier_pos
)
1662 cpp_warning (pfile
, CPP_W_UNICODE
,
1663 "'\\u{' not terminated with '}' after %.*s; "
1664 "treating it as separate tokens",
1665 (int) (str
- base
), base
);
1674 cpp_error (pfile
, CPP_DL_ERROR
,
1675 "incomplete universal character name %.*s",
1676 (int) (str
- base
), base
);
1678 cpp_error (pfile
, CPP_DL_ERROR
,
1679 "'\\u{' not terminated with '}' after %.*s",
1680 (int) (str
- base
), base
);
1683 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1684 hex escapes so that this also works with EBCDIC hosts.
1685 C++0x permits everything below 0xa0 within literals;
1686 ucn_valid_in_identifier will complain about identifiers. */
1687 else if ((result
< 0xa0
1688 && !CPP_OPTION (pfile
, cplusplus
)
1689 && (result
!= 0x24 && result
!= 0x40 && result
!= 0x60))
1690 || (result
& 0x80000000)
1691 || (result
>= 0xD800 && result
<= 0xDFFF))
1693 cpp_error (pfile
, CPP_DL_ERROR
,
1694 "%.*s is not a valid universal character",
1695 (int) (str
- base
), base
);
1698 else if (identifier_pos
&& result
== 0x24
1699 && CPP_OPTION (pfile
, dollars_in_ident
))
1701 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1703 CPP_OPTION (pfile
, warn_dollars
) = 0;
1704 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1706 NORMALIZE_STATE_UPDATE_IDNUM (nst
, result
);
1708 else if (identifier_pos
)
1710 int validity
= ucn_valid_in_identifier (pfile
, result
, nst
);
1713 cpp_error (pfile
, CPP_DL_ERROR
,
1714 "universal character %.*s is not valid in an identifier",
1715 (int) (str
- base
), base
);
1716 else if (validity
== 2 && identifier_pos
== 1)
1717 cpp_error (pfile
, CPP_DL_ERROR
,
1718 "universal character %.*s is not valid at the start of an identifier",
1719 (int) (str
- base
), base
);
1721 else if (result
> UCS_LIMIT
1722 && (!CPP_OPTION (pfile
, cplusplus
)
1723 || CPP_OPTION (pfile
, lang
) > CLK_CXX17
))
1724 cpp_error (pfile
, CPP_DL_PEDWARN
,
1725 "%.*s is outside the UCS codespace",
1726 (int) (str
- base
), base
);
1732 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1733 it to the execution character set and write the result into TBUF,
1734 if TBUF is non-NULL.
1735 An advanced pointer is returned. Issues all relevant diagnostics.
1736 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1737 contains the location of the character so far: location information
1738 is read from *LOC_READER, and *RANGES is updated accordingly. */
1739 static const uchar
*
1740 convert_ucn (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1741 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1742 source_range char_range
,
1743 cpp_string_location_reader
*loc_reader
,
1744 cpp_substring_ranges
*ranges
)
1749 size_t bytesleft
= 6;
1751 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
1753 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1754 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
1756 from
++; /* Skip u/U/N. */
1758 /* The u/U is part of the spelling of this character. */
1759 extend_char_range (&char_range
, loc_reader
);
1761 _cpp_valid_ucn (pfile
, &from
, limit
, 0, &nst
,
1762 &ucn
, &char_range
, loc_reader
);
1764 rval
= one_cppchar_to_utf8 (ucn
, &bufp
, &bytesleft
);
1768 cpp_errno (pfile
, CPP_DL_ERROR
,
1769 "converting UCN to source character set");
1774 if (!APPLY_CONVERSION (cvt
, buf
, 6 - bytesleft
, tbuf
))
1775 cpp_errno (pfile
, CPP_DL_ERROR
,
1776 "converting UCN to execution character set");
1780 int num_encoded_bytes
= 6 - bytesleft
;
1781 for (int i
= 0; i
< num_encoded_bytes
; i
++)
1782 ranges
->add_range (char_range
);
1789 /* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded
1790 extended characters rather than UCNs. If the return value is TRUE, then a
1791 character was successfully decoded and stored in *CP; *PSTR has been
1792 updated to point one past the valid UTF-8 sequence. Diagnostics may have
1793 been emitted if the character parsed is not allowed in the current context.
1794 If the return value is FALSE, then *PSTR has not been modified and *CP may
1795 equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it
1796 may, when processing an identifier in C mode, equal a codepoint that was
1797 validly encoded but is not allowed to appear in an identifier. In either
1798 case, no diagnostic is emitted, and the return value of FALSE should cause
1799 a new token to be formed.
1801 _cpp_valid_utf8 can be called when lexing a potential identifier, or a
1802 CPP_OTHER token or for the purposes of -Winvalid-utf8 warning in string or
1803 character literals. NST is unused when not in a potential identifier.
1805 As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for
1806 the start of an identifier, or 2 otherwise. */
1809 _cpp_valid_utf8 (cpp_reader
*pfile
,
1813 struct normalize_state
*nst
,
1816 const uchar
*base
= *pstr
;
1817 size_t inbytesleft
= limit
- base
;
1818 if (one_utf8_to_cppchar (pstr
, &inbytesleft
, cp
))
1820 /* No diagnostic here as this byte will rather become a
1828 switch (ucn_valid_in_identifier (pfile
, *cp
, nst
))
1832 /* In C++, this is an error for invalid character in an identifier
1833 because logically, the UTF-8 was converted to a UCN during
1834 translation phase 1 (even though we don't physically do it that
1835 way). In C, this byte rather becomes grammatically a separate
1838 if (CPP_OPTION (pfile
, cplusplus
))
1839 cpp_error (pfile
, CPP_DL_ERROR
,
1840 "extended character %.*s is not valid in an identifier",
1841 (int) (*pstr
- base
), base
);
1851 if (identifier_pos
== 1)
1853 /* This is treated the same way in C++ or C99 -- lexed as an
1854 identifier which is then invalid because an identifier is
1855 not allowed to start with this character. */
1856 cpp_error (pfile
, CPP_DL_ERROR
,
1857 "extended character %.*s is not valid at the start of an identifier",
1858 (int) (*pstr
- base
), base
);
1867 /* Return true iff BUFFER of size NUM_BYTES is validly-encoded UTF-8. */
1870 cpp_valid_utf8_p (const char *buffer
, size_t num_bytes
)
1872 const uchar
*iter
= (const uchar
*)buffer
;
1873 size_t bytesleft
= num_bytes
;
1874 while (bytesleft
> 0)
1876 /* one_utf8_to_cppchar implements 5-byte and 6 byte sequences as per
1877 RFC 2279, but this has been superceded by RFC 3629, which
1878 restricts UTF-8 to 1-byte through 4-byte sequences, and
1879 states "the octet values C0, C1, F5 to FF never appear".
1881 Reject such values. */
1886 int err
= one_utf8_to_cppchar (&iter
, &bytesleft
, &cp
);
1890 /* Additionally, Unicode declares that all codepoints above 0010FFFF are
1891 invalid because they cannot be represented in UTF-16.
1893 Reject such values.*/
1897 /* No problems encountered. */
1901 /* Subroutine of convert_hex and convert_oct. N is the representation
1902 in the execution character set of a numeric escape; write it into the
1903 string buffer TBUF and update the end-of-string pointer therein. WIDE
1904 is true if it's a wide string that's being assembled in TBUF. This
1905 function issues no diagnostics and never fails. */
1907 emit_numeric_escape (cpp_reader
*pfile
, cppchar_t n
,
1908 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1910 size_t width
= cvt
.width
;
1912 if (width
!= CPP_OPTION (pfile
, char_precision
))
1914 /* We have to render this into the target byte order, which may not
1915 be our byte order. */
1916 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
1917 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
1918 size_t cmask
= width_to_mask (cwidth
);
1919 size_t nbwc
= width
/ cwidth
;
1921 size_t off
= tbuf
->len
;
1924 if (tbuf
->len
+ nbwc
> tbuf
->asize
)
1926 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
1927 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
1930 for (i
= 0; i
< nbwc
; i
++)
1934 tbuf
->text
[off
+ (bigend
? nbwc
- i
- 1 : i
)] = c
;
1940 /* Note: this code does not handle the case where the target
1941 and host have a different number of bits in a byte. */
1942 if (tbuf
->len
+ 1 > tbuf
->asize
)
1944 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
1945 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
1947 tbuf
->text
[tbuf
->len
++] = n
;
1951 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
1952 character set and write it into the string buffer TBUF (if non-NULL).
1953 Returns an advanced pointer, and issues diagnostics as necessary.
1954 No character set translation occurs; this routine always produces the
1955 execution-set character with numeric value equal to the given hex
1956 number. You can, e.g. generate surrogate pairs this way.
1957 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1958 contains the location of the character so far: location information
1959 is read from *LOC_READER, and *RANGES is updated accordingly. */
1960 static const uchar
*
1961 convert_hex (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1962 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1963 source_range char_range
,
1964 cpp_string_location_reader
*loc_reader
,
1965 cpp_substring_ranges
*ranges
)
1967 cppchar_t c
, n
= 0, overflow
= 0;
1968 int digits_found
= 0;
1969 size_t width
= cvt
.width
;
1970 size_t mask
= width_to_mask (width
);
1971 bool delimited
= false;
1972 const uchar
*base
= from
- 1;
1974 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1975 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
1977 if (CPP_WTRADITIONAL (pfile
))
1978 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1979 "the meaning of '\\x' is different in traditional C");
1984 /* The 'x' is part of the spelling of this character. */
1985 extend_char_range (&char_range
, loc_reader
);
1987 if (from
< limit
&& *from
== '{')
1991 extend_char_range (&char_range
, loc_reader
);
1994 while (from
< limit
)
2000 extend_char_range (&char_range
, loc_reader
);
2001 overflow
|= n
^ (n
<< 4 >> 4);
2002 n
= (n
<< 4) + hex_value (c
);
2006 if (delimited
&& from
< limit
&& *from
== '}')
2011 cpp_error (pfile
, CPP_DL_ERROR
,
2012 "empty delimited escape sequence");
2015 else if (!CPP_OPTION (pfile
, delimited_escape_seqs
)
2016 && CPP_OPTION (pfile
, cpp_pedantic
))
2017 cpp_error (pfile
, CPP_DL_PEDWARN
,
2018 "delimited escape sequences are only valid in C++23");
2020 extend_char_range (&char_range
, loc_reader
);
2025 cpp_error (pfile
, CPP_DL_ERROR
,
2026 "\\x used with no following hex digits");
2031 cpp_error (pfile
, CPP_DL_ERROR
,
2032 "'\\x{' not terminated with '}' after %.*s",
2033 (int) (from
- base
), base
);
2037 if (overflow
| (n
!= (n
& mask
)))
2039 cpp_error (pfile
, CPP_DL_PEDWARN
,
2040 "hex escape sequence out of range");
2045 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
2047 ranges
->add_range (char_range
);
2052 /* Convert an octal escape, pointed to by FROM, to the execution
2053 character set and write it into the string buffer TBUF. Returns an
2054 advanced pointer, and issues diagnostics as necessary.
2055 No character set translation occurs; this routine always produces the
2056 execution-set character with numeric value equal to the given octal
2058 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
2059 contains the location of the character so far: location information
2060 is read from *LOC_READER, and *RANGES is updated accordingly. */
2061 static const uchar
*
2062 convert_oct (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
2063 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
2064 source_range char_range
,
2065 cpp_string_location_reader
*loc_reader
,
2066 cpp_substring_ranges
*ranges
)
2069 cppchar_t c
, n
= 0, overflow
= 0;
2070 size_t width
= cvt
.width
;
2071 size_t mask
= width_to_mask (width
);
2072 bool delimited
= false;
2073 const uchar
*base
= from
- 1;
2075 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
2076 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
2078 if (from
< limit
&& *from
== 'o')
2081 extend_char_range (&char_range
, loc_reader
);
2082 if (from
== limit
|| *from
!= '{')
2083 cpp_error (pfile
, CPP_DL_ERROR
, "'\\o' not followed by '{'");
2087 extend_char_range (&char_range
, loc_reader
);
2092 while (from
< limit
&& count
++ < 3)
2095 if (c
< '0' || c
> '7')
2098 extend_char_range (&char_range
, loc_reader
);
2102 overflow
|= n
^ (n
<< 3 >> 3);
2104 n
= (n
<< 3) + c
- '0';
2109 if (from
< limit
&& *from
== '}')
2114 cpp_error (pfile
, CPP_DL_ERROR
,
2115 "empty delimited escape sequence");
2118 else if (!CPP_OPTION (pfile
, delimited_escape_seqs
)
2119 && CPP_OPTION (pfile
, cpp_pedantic
))
2120 cpp_error (pfile
, CPP_DL_PEDWARN
,
2121 "delimited escape sequences are only valid in C++23");
2122 extend_char_range (&char_range
, loc_reader
);
2126 cpp_error (pfile
, CPP_DL_ERROR
,
2127 "'\\o{' not terminated with '}' after %.*s",
2128 (int) (from
- base
), base
);
2133 if (overflow
| (n
!= (n
& mask
)))
2135 cpp_error (pfile
, CPP_DL_PEDWARN
,
2136 "octal escape sequence out of range");
2141 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
2143 ranges
->add_range (char_range
);
2148 /* Convert an escape sequence (pointed to by FROM) to its value on
2149 the target, and to the execution character set. Do not scan past
2150 LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
2151 Returns an advanced pointer. Handles all relevant diagnostics.
2152 If LOC_READER is non-NULL, then RANGES must be non-NULL: location
2153 information is read from *LOC_READER, and *RANGES is updated
2155 static const uchar
*
2156 convert_escape (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
2157 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
2158 cpp_string_location_reader
*loc_reader
,
2159 cpp_substring_ranges
*ranges
, bool uneval
)
2161 /* Values of \a \b \e \f \n \r \t \v respectively. */
2162 #if HOST_CHARSET == HOST_CHARSET_ASCII
2163 static const uchar charconsts
[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
2164 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
2165 static const uchar charconsts
[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
2167 #error "unknown host character set"
2172 /* Record the location of the backslash. */
2173 source_range char_range
;
2175 char_range
= loc_reader
->get_next ();
2180 /* UCNs, hex escapes, and octal escapes are processed separately. */
2181 case 'u': case 'U': case 'N':
2182 return convert_ucn (pfile
, from
, limit
, tbuf
, cvt
,
2183 char_range
, loc_reader
, ranges
);
2186 if (uneval
&& CPP_PEDANTIC (pfile
))
2187 cpp_error (pfile
, CPP_DL_PEDWARN
,
2188 "numeric escape sequence in unevaluated string: "
2190 return convert_hex (pfile
, from
, limit
, tbuf
, cvt
,
2191 char_range
, loc_reader
, ranges
);
2193 case '0': case '1': case '2': case '3':
2194 case '4': case '5': case '6': case '7':
2196 if (uneval
&& CPP_PEDANTIC (pfile
))
2197 cpp_error (pfile
, CPP_DL_PEDWARN
,
2198 "numeric escape sequence in unevaluated string: "
2200 return convert_oct (pfile
, from
, limit
, tbuf
, cvt
,
2201 char_range
, loc_reader
, ranges
);
2203 /* Various letter escapes. Get the appropriate host-charset
2205 case '\\': case '\'': case '"': case '?': break;
2207 case '(': case '{': case '[': case '%':
2208 /* '\(', etc, can be used at the beginning of a line in a long
2209 string split onto multiple lines with \-newline, to prevent
2210 Emacs or other text editors from getting confused. '\%' can
2211 be used to prevent SCCS from mangling printf format strings. */
2212 if (CPP_PEDANTIC (pfile
))
2216 case 'b': c
= charconsts
[1]; break;
2217 case 'f': c
= charconsts
[3]; break;
2218 case 'n': c
= charconsts
[4]; break;
2219 case 'r': c
= charconsts
[5]; break;
2220 case 't': c
= charconsts
[6]; break;
2221 case 'v': c
= charconsts
[7]; break;
2224 if (CPP_WTRADITIONAL (pfile
))
2225 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
2226 "the meaning of '\\a' is different in traditional C");
2231 if (CPP_PEDANTIC (pfile
))
2232 cpp_error (pfile
, CPP_DL_PEDWARN
,
2233 "non-ISO-standard escape sequence, '\\%c'", (int) c
);
2240 cpp_error (pfile
, CPP_DL_PEDWARN
,
2241 "unknown escape sequence: '\\%c'", (int) c
);
2244 encoding_rich_location
rich_loc (pfile
);
2246 /* diagnostic.cc does not support "%03o". When it does, this
2247 code can use %03o directly in the diagnostic again. */
2249 sprintf(buf
, "%03o", (int) c
);
2250 cpp_error_at (pfile
, CPP_DL_PEDWARN
, &rich_loc
,
2251 "unknown escape sequence: '\\%s'", buf
);
2256 /* Now convert what we have to the execution character set. */
2257 if (!APPLY_CONVERSION (cvt
, &c
, 1, tbuf
))
2258 cpp_errno (pfile
, CPP_DL_ERROR
,
2259 "converting escape sequence to execution character set");
2263 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
2264 ranges
->add_range (char_range
);
2270 /* TYPE is a token type. The return value is the conversion needed to
2271 convert from source to execution character set for the given type. */
2272 static struct cset_converter
2273 converter_for_type (cpp_reader
*pfile
, enum cpp_ttype type
)
2278 return pfile
->narrow_cset_desc
;
2280 case CPP_UTF8STRING
:
2281 return pfile
->utf8_cset_desc
;
2284 return pfile
->char16_cset_desc
;
2287 return pfile
->char32_cset_desc
;
2290 return pfile
->wide_cset_desc
;
2294 /* FROM is an array of cpp_string structures of length COUNT. These
2295 are to be converted from the source to the execution character set,
2296 escape sequences translated, and finally all are to be
2297 concatenated. WIDE indicates whether or not to produce a wide
2298 string. If TO is non-NULL, the result is written into TO.
2299 If LOC_READERS and OUT are non-NULL, then location information
2300 is read from LOC_READERS (which must be an array of length COUNT),
2301 and location information is written to *RANGES.
2303 Returns true for success, false for failure. */
2306 cpp_interpret_string_1 (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
2307 cpp_string
*to
, enum cpp_ttype type
,
2308 cpp_string_location_reader
*loc_readers
,
2309 cpp_substring_ranges
*out
)
2311 struct _cpp_strbuf tbuf
;
2312 const uchar
*p
, *base
, *limit
;
2314 struct cset_converter cvt
= converter_for_type (pfile
, type
);
2316 /* loc_readers and out must either be both NULL, or both be non-NULL. */
2317 gcc_assert ((loc_readers
!= NULL
) == (out
!= NULL
));
2321 tbuf
.asize
= MAX (OUTBUF_BLOCK_SIZE
, from
->len
);
2322 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
2326 cpp_string_location_reader
*loc_reader
= NULL
;
2327 for (i
= 0; i
< count
; i
++)
2330 loc_reader
= &loc_readers
[i
];
2337 loc_reader
->get_next ();
2342 loc_reader
->get_next ();
2345 else if (*p
== 'L' || *p
== 'U') p
++;
2348 const uchar
*prefix
;
2350 /* Skip over 'R"'. */
2354 loc_reader
->get_next ();
2355 loc_reader
->get_next ();
2362 loc_reader
->get_next ();
2366 loc_reader
->get_next ();
2367 limit
= from
[i
].text
+ from
[i
].len
;
2368 if (limit
>= p
+ (p
- prefix
) + 1)
2369 limit
-= (p
- prefix
) + 1;
2371 /* Raw strings are all normal characters; these can be fed
2372 directly to convert_cset. */
2374 if (!APPLY_CONVERSION (cvt
, p
, limit
- p
, &tbuf
))
2379 /* If generating source ranges, assume we have a 1:1
2380 correspondence between bytes in the source encoding and bytes
2381 in the execution encoding (e.g. if we have a UTF-8 to UTF-8
2382 conversion), so that this run of bytes in the source file
2383 corresponds to a run of bytes in the execution string.
2384 This requirement is guaranteed by an early-reject in
2385 cpp_interpret_string_ranges. */
2386 gcc_assert (cvt
.func
== convert_no_conversion
);
2387 out
->add_n_ranges (limit
- p
, *loc_reader
);
2393 /* If we don't now have a leading quote, something has gone wrong.
2394 This can occur if cpp_interpret_string_ranges is handling a
2395 stringified macro argument, but should not be possible otherwise. */
2396 if (*p
!= '"' && *p
!= '\'')
2398 gcc_assert (out
!= NULL
);
2399 cpp_error (pfile
, CPP_DL_ERROR
, "missing open quote");
2405 /* Skip leading quote. */
2408 loc_reader
->get_next ();
2410 limit
= from
[i
].text
+ from
[i
].len
- 1; /* Skip trailing quote. */
2415 while (p
< limit
&& *p
!= '\\')
2419 /* We have a run of normal characters; these can be fed
2420 directly to convert_cset. */
2422 if (!APPLY_CONVERSION (cvt
, base
, p
- base
, &tbuf
))
2424 /* Similar to above: assumes we have a 1:1 correspondence
2425 between bytes in the source encoding and bytes in the
2426 execution encoding. */
2429 gcc_assert (cvt
.func
== convert_no_conversion
);
2430 out
->add_n_ranges (p
- base
, *loc_reader
);
2436 struct _cpp_strbuf
*tbuf_ptr
= to
? &tbuf
: NULL
;
2437 p
= convert_escape (pfile
, p
+ 1, limit
, tbuf_ptr
, cvt
,
2438 loc_reader
, out
, type
== CPP_UNEVAL_STRING
);
2444 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
2446 emit_numeric_escape (pfile
, 0, &tbuf
, cvt
);
2447 tbuf
.text
= XRESIZEVEC (uchar
, tbuf
.text
, tbuf
.len
);
2448 to
->text
= tbuf
.text
;
2451 /* Use the location of the trailing quote as the location of the
2455 source_range range
= loc_reader
->get_next ();
2456 out
->add_range (range
);
2462 cpp_errno (pfile
, CPP_DL_ERROR
, "converting to execution character set");
2468 /* FROM is an array of cpp_string structures of length COUNT. These
2469 are to be converted from the source to the execution character set,
2470 escape sequences translated, and finally all are to be
2471 concatenated. WIDE indicates whether or not to produce a wide
2472 string. The result is written into TO. Returns true for success,
2473 false for failure. */
2475 cpp_interpret_string (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
2476 cpp_string
*to
, enum cpp_ttype type
)
2478 return cpp_interpret_string_1 (pfile
, from
, count
, to
, type
, NULL
, NULL
);
2481 /* A "do nothing" diagnostic-handling callback for use by
2482 cpp_interpret_string_ranges, so that it can temporarily suppress
2483 diagnostic-handling. */
2486 noop_diagnostic_cb (cpp_reader
*, enum cpp_diagnostic_level
,
2487 enum cpp_warning_reason
, rich_location
*,
2488 const char *, va_list *)
2494 /* This function mimics the behavior of cpp_interpret_string, but
2495 rather than generating a string in the execution character set,
2496 *OUT is written to with the source code ranges of the characters
2498 FROM and LOC_READERS should both be arrays of length COUNT.
2499 Returns NULL for success, or an error message for failure. */
2502 cpp_interpret_string_ranges (cpp_reader
*pfile
, const cpp_string
*from
,
2503 cpp_string_location_reader
*loc_readers
,
2505 cpp_substring_ranges
*out
,
2506 enum cpp_ttype type
)
2508 /* There are a couple of cases in the range-handling in
2509 cpp_interpret_string_1 that rely on there being a 1:1 correspondence
2510 between bytes in the source encoding and bytes in the execution
2511 encoding, so that each byte in the execution string can correspond
2512 to the location of a byte in the source string.
2514 This holds for the typical case of a UTF-8 to UTF-8 conversion.
2515 Enforce this requirement by only attempting to track substring
2516 locations if we have source encoding == execution encoding.
2518 This is a stronger condition than we need, since we could e.g.
2519 have ASCII to EBCDIC (with 1 byte per character before and after),
2520 but it seems to be a reasonable restriction. */
2521 struct cset_converter cvt
= converter_for_type (pfile
, type
);
2522 if (cvt
.func
!= convert_no_conversion
)
2523 return "execution character set != source character set";
2525 /* For on-demand strings we have already lexed the strings, so there
2526 should be no diagnostics. However, if we have bogus source location
2527 data (or stringified macro arguments), the attempt to lex the
2528 strings could fail with an diagnostic. Temporarily install an
2529 diagnostic-handler to catch the diagnostic, so that it can lead to this call
2530 failing, rather than being emitted as a user-visible diagnostic.
2531 If an diagnostic does occur, we should see it via the return value of
2532 cpp_interpret_string_1. */
2533 bool (*saved_diagnostic_handler
) (cpp_reader
*, enum cpp_diagnostic_level
,
2534 enum cpp_warning_reason
, rich_location
*,
2535 const char *, va_list *)
2536 ATTRIBUTE_FPTR_PRINTF(5,0);
2538 saved_diagnostic_handler
= pfile
->cb
.diagnostic
;
2539 pfile
->cb
.diagnostic
= noop_diagnostic_cb
;
2541 bool result
= cpp_interpret_string_1 (pfile
, from
, count
, NULL
, type
,
2544 /* Restore the saved diagnostic-handler. */
2545 pfile
->cb
.diagnostic
= saved_diagnostic_handler
;
2548 return "cpp_interpret_string_1 failed";
2554 /* Subroutine of do_line and do_linemarker. Convert escape sequences
2555 in a string, but do not perform character set conversion. */
2557 cpp_interpret_string_notranslate (cpp_reader
*pfile
, const cpp_string
*from
,
2558 size_t count
, cpp_string
*to
,
2559 enum cpp_ttype type
)
2561 struct cset_converter save_narrow_cset_desc
= pfile
->narrow_cset_desc
;
2564 pfile
->narrow_cset_desc
.func
= convert_no_conversion
;
2565 pfile
->narrow_cset_desc
.cd
= (iconv_t
) -1;
2566 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
2568 retval
= cpp_interpret_string (pfile
, from
, count
, to
,
2569 type
== CPP_UNEVAL_STRING
2570 ? CPP_UNEVAL_STRING
: CPP_STRING
);
2572 pfile
->narrow_cset_desc
= save_narrow_cset_desc
;
2577 /* Subroutine of cpp_interpret_charconst which performs the conversion
2578 to a number, for narrow strings. STR is the string structure returned
2579 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
2580 cpp_interpret_charconst. TYPE is the token type. */
2582 narrow_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
2583 unsigned int *pchars_seen
, int *unsignedp
,
2584 enum cpp_ttype type
)
2586 size_t width
= CPP_OPTION (pfile
, char_precision
);
2587 size_t max_chars
= CPP_OPTION (pfile
, int_precision
) / width
;
2588 size_t mask
= width_to_mask (width
);
2590 cppchar_t result
, c
;
2593 /* The value of a multi-character character constant, or a
2594 single-character character constant whose representation in the
2595 execution character set is more than one byte long, is
2596 implementation defined. This implementation defines it to be the
2597 number formed by interpreting the byte sequence in memory as a
2598 big-endian binary number. If overflow occurs, the high bytes are
2599 lost, and a warning is issued.
2601 We don't want to process the NUL terminator handed back by
2602 cpp_interpret_string. */
2604 for (i
= 0; i
< str
.len
- 1; i
++)
2606 c
= str
.text
[i
] & mask
;
2607 if (width
< BITS_PER_CPPCHAR_T
)
2608 result
= (result
<< width
) | c
;
2613 if (type
== CPP_UTF8CHAR
)
2618 cpp_error (pfile
, type
== CPP_UTF8CHAR
? CPP_DL_ERROR
: CPP_DL_WARNING
,
2619 "character constant too long for its type");
2621 else if (i
> 1 && CPP_OPTION (pfile
, warn_multichar
))
2622 cpp_warning (pfile
, CPP_W_MULTICHAR
, "multi-character character constant");
2624 /* Multichar constants are of type int and therefore signed. */
2627 else if (type
== CPP_UTF8CHAR
)
2628 unsigned_p
= CPP_OPTION (pfile
, unsigned_utf8char
);
2630 unsigned_p
= CPP_OPTION (pfile
, unsigned_char
);
2632 /* Truncate the constant to its natural width, and simultaneously
2633 sign- or zero-extend to the full width of cppchar_t.
2634 For single-character constants, the value is WIDTH bits wide.
2635 For multi-character constants, the value is INT_PRECISION bits wide. */
2637 width
= CPP_OPTION (pfile
, int_precision
);
2638 if (width
< BITS_PER_CPPCHAR_T
)
2640 mask
= ((cppchar_t
) 1 << width
) - 1;
2641 if (unsigned_p
|| !(result
& (1 << (width
- 1))))
2647 *unsignedp
= unsigned_p
;
2651 /* Subroutine of cpp_interpret_charconst which performs the conversion
2652 to a number, for wide strings. STR is the string structure returned
2653 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
2654 cpp_interpret_charconst. TYPE is the token type. */
2656 wide_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
2657 unsigned int *pchars_seen
, int *unsignedp
,
2658 enum cpp_ttype type
)
2660 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
2661 size_t width
= converter_for_type (pfile
, type
).width
;
2662 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
2663 size_t mask
= width_to_mask (width
);
2664 size_t cmask
= width_to_mask (cwidth
);
2665 size_t nbwc
= width
/ cwidth
;
2667 cppchar_t result
= 0, c
;
2669 if (str
.len
<= nbwc
)
2671 /* Error recovery, if no errors have been diagnosed previously,
2672 there should be at least two wide characters. Empty literals
2673 are diagnosed earlier and we can get just the zero terminator
2674 only if there were errors diagnosed during conversion. */
2680 /* This is finicky because the string is in the target's byte order,
2681 which may not be our byte order. Only the last character, ignoring
2682 the NUL terminator, is relevant. */
2683 off
= str
.len
- (nbwc
* 2);
2685 for (i
= 0; i
< nbwc
; i
++)
2687 c
= bigend
? str
.text
[off
+ i
] : str
.text
[off
+ nbwc
- i
- 1];
2688 result
= (result
<< cwidth
) | (c
& cmask
);
2691 /* Wide character constants have type wchar_t, and a single
2692 character exactly fills a wchar_t, so a multi-character wide
2693 character constant is guaranteed to overflow. */
2694 if (str
.len
> nbwc
* 2)
2695 cpp_error (pfile
, (CPP_OPTION (pfile
, cplusplus
)
2696 && (type
== CPP_CHAR16
2697 || type
== CPP_CHAR32
2698 /* In C++23 this is error even for L'ab'. */
2699 || (type
== CPP_WCHAR
2700 && CPP_OPTION (pfile
, size_t_literals
))))
2701 ? CPP_DL_ERROR
: CPP_DL_WARNING
,
2702 "character constant too long for its type");
2704 /* Truncate the constant to its natural width, and simultaneously
2705 sign- or zero-extend to the full width of cppchar_t. */
2706 if (width
< BITS_PER_CPPCHAR_T
)
2708 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
2709 || CPP_OPTION (pfile
, unsigned_wchar
)
2710 || !(result
& (1 << (width
- 1))))
2716 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
2717 || CPP_OPTION (pfile
, unsigned_wchar
))
2726 /* Interpret a (possibly wide) character constant in TOKEN.
2727 PCHARS_SEEN points to a variable that is filled in with the number
2728 of characters seen, and UNSIGNEDP to a variable that indicates
2729 whether the result has signed type. */
2731 cpp_interpret_charconst (cpp_reader
*pfile
, const cpp_token
*token
,
2732 unsigned int *pchars_seen
, int *unsignedp
)
2734 cpp_string str
= { 0, 0 };
2735 bool wide
= (token
->type
!= CPP_CHAR
&& token
->type
!= CPP_UTF8CHAR
);
2736 int u8
= 2 * int(token
->type
== CPP_UTF8CHAR
);
2739 /* An empty constant will appear as L'', u'', U'', u8'', or '' */
2740 if (token
->val
.str
.len
== (size_t) (2 + wide
+ u8
))
2742 cpp_error (pfile
, CPP_DL_ERROR
, "empty character constant");
2747 else if (!cpp_interpret_string (pfile
, &token
->val
.str
, 1, &str
,
2756 result
= wide_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
2759 result
= narrow_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
2762 if (str
.text
!= token
->val
.str
.text
)
2763 free ((void *)str
.text
);
2768 /* Convert an identifier denoted by ID and LEN, which might contain
2769 UCN escapes or UTF-8 multibyte chars, to the source character set,
2770 either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually
2771 a valid identifier. */
2773 _cpp_interpret_identifier (cpp_reader
*pfile
, const uchar
*id
, size_t len
)
2775 /* It turns out that a UCN escape always turns into fewer characters
2776 than the escape itself, so we can allocate a temporary in advance. */
2777 uchar
* buf
= (uchar
*) alloca (len
+ 1);
2781 for (idp
= 0; idp
< len
; idp
++)
2782 if (id
[idp
] != '\\')
2786 unsigned length
= id
[idp
+ 1] == 'u' ? 4 : 8;
2787 cppchar_t value
= 0;
2788 size_t bufleft
= len
- (bufp
- buf
);
2790 bool delimited
= false;
2793 if (id
[idp
- 1] == 'N' && id
[idp
] == '{')
2796 const uchar
*name
= &id
[idp
];
2798 && (ISIDNUM (id
[idp
]) || id
[idp
] == ' ' || id
[idp
] == '-'))
2802 value
= _cpp_uname2c ((const char *) name
, &id
[idp
] - name
,
2803 uname2c_tree
, NULL
);
2804 if (value
== (cppchar_t
) -1)
2812 if (length
== 4 && id
[idp
] == '{')
2817 while (length
&& idp
< len
&& ISXDIGIT (id
[idp
]))
2819 value
= (value
<< 4) + hex_value (id
[idp
]);
2824 if (!delimited
|| id
[idp
] != '}')
2828 /* Special case for EBCDIC: if the identifier contains
2829 a '$' specified using a UCN, translate it to EBCDIC. */
2836 rval
= one_cppchar_to_utf8 (value
, &bufp
, &bufleft
);
2840 cpp_errno (pfile
, CPP_DL_ERROR
,
2841 "converting UCN to source character set");
2846 return CPP_HASHNODE (ht_lookup (pfile
->hash_table
,
2847 buf
, bufp
- buf
, HT_ALLOC
));
2851 /* Utility to strip a UTF-8 byte order marking from the beginning
2852 of a buffer. Returns the number of bytes to skip, which currently
2853 will be either 0 or 3. */
2855 cpp_check_utf8_bom (const char *data
, size_t data_length
)
2858 #if HOST_CHARSET == HOST_CHARSET_ASCII
2859 const unsigned char *udata
= (const unsigned char *) data
;
2860 if (data_length
>= 3 && udata
[0] == 0xef && udata
[1] == 0xbb
2861 && udata
[2] == 0xbf)
2869 /* Convert an input buffer (containing the complete contents of one
2870 source file) from INPUT_CHARSET to the source character set. INPUT
2871 points to the input buffer, SIZE is its allocated size, and LEN is
2872 the length of the meaningful data within the buffer. The
2873 translated buffer is returned, *ST_SIZE is set to the length of
2874 the meaningful data within the translated buffer, and *BUFFER_START
2875 is set to the start of the returned buffer. *BUFFER_START may
2876 differ from the return value in the case of a BOM or other ignored
2879 INPUT is expected to have been allocated with xmalloc. This
2880 function will either set *BUFFER_START to INPUT, or free it and set
2881 *BUFFER_START to a pointer to another xmalloc-allocated block of
2884 PFILE is only used to generate diagnostics; setting it to NULL suppresses
2885 diagnostics, and causes a return of NULL if there was any error instead. */
2888 _cpp_convert_input (cpp_reader
*pfile
, const char *input_charset
,
2889 uchar
*input
, size_t size
, size_t len
,
2890 const unsigned char **buffer_start
, off_t
*st_size
)
2892 struct cset_converter input_cset
;
2893 struct _cpp_strbuf to
;
2894 unsigned char *buffer
;
2896 input_cset
= init_iconv_desc (pfile
, SOURCE_CHARSET
, input_charset
);
2897 if (input_cset
.func
== convert_no_conversion
)
2905 to
.asize
= MAX (65536, len
);
2906 to
.text
= XNEWVEC (uchar
, to
.asize
);
2909 const bool ok
= APPLY_CONVERSION (input_cset
, input
, len
, &to
);
2912 /* Clean up the mess. */
2913 if (input_cset
.func
== convert_using_iconv
)
2914 iconv_close (input_cset
.cd
);
2916 /* Handle conversion failure. */
2921 XDELETEVEC (to
.text
);
2922 *buffer_start
= NULL
;
2926 cpp_error (pfile
, CPP_DL_ERROR
, "failure to convert %s to %s",
2927 input_charset
, SOURCE_CHARSET
);
2931 /* Resize buffer if we allocated substantially too much, or if we
2932 haven't enough space for the \n-terminator or following
2933 15 bytes of padding (used to quiet warnings from valgrind or
2934 Address Sanitizer, when the optimized lexer accesses aligned
2935 16-byte memory chunks, including the bytes after the malloced,
2936 area, and stops lexing on '\n'). */
2937 if (to
.len
+ 4096 < to
.asize
|| to
.len
+ 16 > to
.asize
)
2938 to
.text
= XRESIZEVEC (uchar
, to
.text
, to
.len
+ 16);
2940 memset (to
.text
+ to
.len
, '\0', 16);
2942 /* If the file is using old-school Mac line endings (\r only),
2943 terminate with another \r, not an \n, so that we do not mistake
2944 the \r\n sequence for a single DOS line ending and erroneously
2945 issue the "No newline at end of file" diagnostic. */
2946 if (to
.len
&& to
.text
[to
.len
- 1] == '\r')
2947 to
.text
[to
.len
] = '\r';
2949 to
.text
[to
.len
] = '\n';
2954 /* Ignore a UTF-8 BOM if we see one and the source charset is UTF-8. Note
2955 that glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
2956 BOM -- however, even if it did, we would still need this code due
2957 to the 'convert_no_conversion' case. */
2958 const int bom_len
= cpp_check_utf8_bom ((const char *) to
.text
, to
.len
);
2959 *st_size
-= bom_len
;
2962 *buffer_start
= to
.text
;
2966 /* Decide on the default encoding to assume for input files. */
2968 _cpp_default_encoding (void)
2970 const char *current_encoding
= NULL
;
2972 /* We disable this because the default codeset is 7-bit ASCII on
2973 most platforms, and this causes conversion failures on every
2974 file in GCC that happens to have one of the upper 128 characters
2975 in it -- most likely, as part of the name of a contributor.
2976 We should definitely recognize in-band markers of file encoding,
2978 - the appropriate Unicode byte-order mark (FE FF) to recognize
2979 UTF16 and UCS4 (in both big-endian and little-endian flavors)
2981 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
2982 distinguish ASCII and EBCDIC.
2983 - now we can parse something like "#pragma GCC encoding <xyz>
2984 on the first line, or even Emacs/VIM's mode line tags (there's
2985 a problem here in that VIM uses the last line, and Emacs has
2986 its more elaborate "local variables" convention).
2987 - investigate whether Java has another common convention, which
2988 would be friendly to support.
2989 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
2990 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
2991 setlocale (LC_CTYPE
, "");
2992 current_encoding
= nl_langinfo (CODESET
);
2994 if (current_encoding
== NULL
|| *current_encoding
== '\0')
2995 current_encoding
= SOURCE_CHARSET
;
2997 return current_encoding
;
3000 /* Check if the configured input charset requires no conversion, other than
3001 possibly stripping a UTF-8 BOM. */
3002 bool cpp_input_conversion_is_trivial (const char *input_charset
)
3004 return !strcasecmp (input_charset
, SOURCE_CHARSET
);
3007 /* Implementation of class cpp_string_location_reader. */
3009 /* Constructor for cpp_string_location_reader. */
3011 cpp_string_location_reader::
3012 cpp_string_location_reader (location_t src_loc
,
3013 line_maps
*line_table
)
3015 src_loc
= get_range_from_loc (line_table
, src_loc
).m_start
;
3017 /* SRC_LOC might be a macro location. It only makes sense to do
3018 column-by-column calculations on ordinary maps, so get the
3019 corresponding location in an ordinary map. */
3021 = linemap_resolve_location (line_table
, src_loc
,
3022 LRK_SPELLING_LOCATION
, NULL
);
3024 const line_map_ordinary
*map
3025 = linemap_check_ordinary (linemap_lookup (line_table
, m_loc
));
3026 m_offset_per_column
= (1 << map
->m_range_bits
);
3029 /* Get the range of the next source byte. */
3032 cpp_string_location_reader::get_next ()
3034 source_range result
;
3035 result
.m_start
= m_loc
;
3036 result
.m_finish
= m_loc
;
3037 if (m_loc
<= LINE_MAP_MAX_LOCATION_WITH_COLS
)
3038 m_loc
+= m_offset_per_column
;
3042 cpp_display_width_computation::
3043 cpp_display_width_computation (const char *data
, int data_length
,
3044 const cpp_char_column_policy
&policy
) :
3047 m_bytes_left (data_length
),
3051 gcc_assert (policy
.m_tabstop
> 0);
3052 gcc_assert (policy
.m_width_cb
);
3056 /* The main implementation function for class cpp_display_width_computation.
3057 m_next points on entry to the start of the UTF-8 encoding of the next
3058 character, and is updated to point just after the last byte of the encoding.
3059 m_bytes_left contains on entry the remaining size of the buffer into which
3060 m_next points, and this is also updated accordingly. If m_next does not
3061 point to a valid UTF-8-encoded sequence, then it will be treated as a single
3062 byte with display width 1. m_cur_display_col is the current display column,
3063 relative to which tab stops should be expanded. Returns the display width of
3064 the codepoint just processed.
3065 If OUT is non-NULL, it is populated. */
3068 cpp_display_width_computation::process_next_codepoint (cpp_decoded_char
*out
)
3074 out
->m_start_byte
= m_next
;
3076 if (*m_next
== '\t')
3080 next_width
= m_policy
.m_tabstop
- (m_display_cols
% m_policy
.m_tabstop
);
3084 out
->m_valid_ch
= true;
3087 else if (one_utf8_to_cppchar ((const uchar
**) &m_next
, &m_bytes_left
, &c
)
3090 /* Input is not convertible to UTF-8. This could be fine, e.g. in a
3091 string literal, so don't complain. Just treat it as if it has a width
3095 next_width
= m_policy
.m_undecoded_byte_width
;
3097 out
->m_valid_ch
= false;
3101 /* one_utf8_to_cppchar() has updated m_next and m_bytes_left for us. */
3102 next_width
= m_policy
.m_width_cb (c
);
3106 out
->m_valid_ch
= true;
3111 out
->m_next_byte
= m_next
;
3113 m_display_cols
+= next_width
;
3117 /* Utility to advance the byte stream by the minimum amount needed to consume
3118 N display columns. Returns the number of display columns that were
3119 actually skipped. This could be less than N, if there was not enough data,
3120 or more than N, if the last character to be skipped had a sufficiently large
3123 cpp_display_width_computation::advance_display_cols (int n
)
3125 const int start
= m_display_cols
;
3126 const int target
= start
+ n
;
3127 while (m_display_cols
< target
&& !done ())
3128 process_next_codepoint (NULL
);
3129 return m_display_cols
- start
;
3132 /* For the string of length DATA_LENGTH bytes that begins at DATA, compute
3133 how many display columns are occupied by the first COLUMN bytes. COLUMN
3134 may exceed DATA_LENGTH, in which case the phantom bytes at the end are
3135 treated as if they have display width 1. Tabs are expanded to the next tab
3136 stop, relative to the start of DATA, and non-printable-ASCII characters
3137 will be escaped as per POLICY. */
3140 cpp_byte_column_to_display_column (const char *data
, int data_length
,
3142 const cpp_char_column_policy
&policy
)
3144 const int offset
= MAX (0, column
- data_length
);
3145 cpp_display_width_computation
dw (data
, column
- offset
, policy
);
3147 dw
.process_next_codepoint (NULL
);
3148 return dw
.display_cols_processed () + offset
;
3151 /* For the string of length DATA_LENGTH bytes that begins at DATA, compute
3152 the least number of bytes that will result in at least DISPLAY_COL display
3153 columns. The return value may exceed DATA_LENGTH if the entire string does
3154 not occupy enough display columns. Non-printable-ASCII characters
3155 will be escaped as per POLICY. */
3158 cpp_display_column_to_byte_column (const char *data
, int data_length
,
3160 const cpp_char_column_policy
&policy
)
3162 cpp_display_width_computation
dw (data
, data_length
, policy
);
3163 const int avail_display
= dw
.advance_display_cols (display_col
);
3164 return dw
.bytes_processed () + MAX (0, display_col
- avail_display
);
3167 template <typename PropertyType
>
3169 get_cppchar_property (cppchar_t c
,
3170 const cppchar_t
*range_ends
,
3171 const PropertyType
*range_values
,
3173 PropertyType default_value
)
3175 if (__builtin_expect (c
<= range_ends
[0], true))
3176 return range_values
[0];
3178 /* Binary search the tables. */
3180 static const int end
= num_ranges
;
3181 int len
= end
- begin
;
3185 int middle
= begin
+ half
;
3186 if (c
> range_ends
[middle
])
3195 if (__builtin_expect (begin
!= end
, true))
3196 return range_values
[begin
];
3198 return default_value
;
3201 /* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc,
3202 because that will inspect the user's locale, and in particular in an ASCII
3203 locale, it will not return anything useful for extended characters. But GCC
3204 in other respects (see e.g. _cpp_default_encoding()) behaves as if
3205 everything is UTF-8. We also make some tweaks that are useful for the way
3206 GCC needs to use this data, e.g. tabs and other control characters should be
3207 treated as having width 1. The lookup tables are generated from
3208 contrib/unicode/gen_wcwidth.py and were made by simply calling glibc
3209 wcwidth() on all codepoints, then applying the small tweaks. These tables
3210 are not highly optimized, but for the present purpose of outputting
3211 diagnostics, they are sufficient. */
3213 #include "generated_cpp_wcwidth.h"
3216 cpp_wcwidth (cppchar_t c
)
3218 const size_t num_ranges
3219 = sizeof wcwidth_range_ends
/ sizeof (*wcwidth_range_ends
);
3220 return get_cppchar_property
<unsigned char > (c
,
3221 &wcwidth_range_ends
[0],
3227 #include "combining-chars.inc"
3230 cpp_is_combining_char (cppchar_t c
)
3232 const size_t num_ranges
3233 = sizeof combining_range_ends
/ sizeof (*combining_range_ends
);
3234 return get_cppchar_property
<bool> (c
,
3235 &combining_range_ends
[0],
3241 #include "printable-chars.inc"
3244 cpp_is_printable_char (cppchar_t c
)
3246 const size_t num_ranges
3247 = sizeof printable_range_ends
/ sizeof (*printable_range_ends
);
3248 return get_cppchar_property
<bool> (c
,
3249 &printable_range_ends
[0],