1 /* CPP Library - charsets
2 Copyright (C) 1998-2021 Free Software Foundation, Inc.
4 Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
25 /* Character set handling for C-family languages.
27 Terminological note: In what follows, "charset" or "character set"
28 will be taken to mean both an abstract set of characters and an
29 encoding for that set.
31 The C99 standard discusses two character sets: source and execution.
32 The source character set is used for internal processing in translation
33 phases 1 through 4; the execution character set is used thereafter.
34 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36 of these terms). Furthermore, the "basic character set" (listed in
37 5.2.1p3) is to be encoded in each with values one byte wide, and is
38 to appear in the initial shift state.
40 It is not explicitly mentioned, but there is also a "wide execution
41 character set" used to encode wide character constants and wide
42 string literals; this is supposed to be the result of applying the
43 standard library function mbstowcs() to an equivalent narrow string
44 (6.4.5p5). However, the behavior of hexadecimal and octal
45 \-escapes is at odds with this; they are supposed to be translated
46 directly to wchar_t values (6.4.4.4p5,6).
48 The source character set is not necessarily the character set used
49 to encode physical source files on disk; translation phase 1 converts
50 from whatever that encoding is to the source character set.
52 The presence of universal character names in C99 (6.4.3 et seq.)
53 forces the source character set to be isomorphic to ISO 10646,
54 that is, Unicode. There is no such constraint on the execution
55 character set; note also that the conversion from source to
56 execution character set does not occur for identifiers (5.1.1.2p1#5).
58 For convenience of implementation, the source character set's
59 encoding of the basic character set should be identical to the
60 execution character set OF THE HOST SYSTEM's encoding of the basic
61 character set, and it should not be a state-dependent encoding.
63 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64 depending on whether the host is based on ASCII or EBCDIC (see
65 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66 Technical Report #16). With limited exceptions, it relies on the
67 system library's iconv() primitive to do charset conversion
68 (specified in SUSv2). */
71 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72 below, which are guarded only by if statements with compile-time
73 constant conditions, do not cause link errors. */
74 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76 #define iconv_close(x) (void)0
80 #if HOST_CHARSET == HOST_CHARSET_ASCII
81 #define SOURCE_CHARSET "UTF-8"
82 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84 #define SOURCE_CHARSET "UTF-EBCDIC"
85 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
87 #error "Unrecognized basic host character set"
94 /* This structure is used for a resizable string buffer throughout. */
95 /* Don't call it strbuf, as that conflicts with unistd.h on systems
96 such as DYNIX/ptx where unistd.h includes stropts.h. */
104 /* This is enough to hold any string that fits on a single 80-column
105 line, even if iconv quadruples its size (e.g. conversion from
106 ASCII to UTF-32) rounded up to a power of two. */
107 #define OUTBUF_BLOCK_SIZE 256
109 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110 logic. This is because a depressing number of systems lack iconv,
111 or have have iconv libraries that do not do these conversions, so
112 we need a fallback implementation for them. To ensure the fallback
113 doesn't break due to neglect, it is used on all systems.
115 UTF-32 encoding is nice and simple: a four-byte binary number,
116 constrained to the range 00000000-7FFFFFFF to avoid questions of
117 signedness. We do have to cope with big- and little-endian
120 UTF-16 encoding uses two-byte binary numbers, again in big- and
121 little-endian variants, for all values in the 00000000-0000FFFF
122 range. Values in the 00010000-0010FFFF range are encoded as pairs
123 of two-byte numbers, called "surrogate pairs": given a number S in
124 this range, it is mapped to a pair (H, L) as follows:
126 H = (S - 0x10000) / 0x400 + 0xD800
127 L = (S - 0x10000) % 0x400 + 0xDC00
129 Two-byte values in the D800...DFFF range are ill-formed except as a
130 component of a surrogate pair. Even if the encoding within a
131 two-byte value is little-endian, the H member of the surrogate pair
134 There is no way to encode values in the 00110000-7FFFFFFF range,
135 which is not currently a problem as there are no assigned code
136 points in that range; however, the author expects that it will
137 eventually become necessary to abandon UTF-16 due to this
138 limitation. Note also that, because of these pairs, UTF-16 does
139 not meet the requirements of the C standard for a wide character
140 encoding (see 3.7.3 and 6.4.4.4p11).
142 UTF-8 encoding looks like this:
144 value range encoded as
145 00000000-0000007F 0xxxxxxx
146 00000080-000007FF 110xxxxx 10xxxxxx
147 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
152 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154 never occur. Note also that any value that can be encoded by a
155 given row of the table can also be encoded by all successive rows,
156 but this is not done; only the shortest possible encoding for any
157 given value is valid. For instance, the character 07C0 could be
158 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159 FC 80 80 80 9F 80. Only the first is valid.
161 An implementation note: the transformation from UTF-16 to UTF-8, or
162 vice versa, is easiest done by using UTF-32 as an intermediary. */
164 /* Internal primitives which go from an UTF-8 byte stream to native-endian
165 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166 operation in several places below. */
168 one_utf8_to_cppchar (const uchar
**inbufp
, size_t *inbytesleftp
,
171 static const uchar masks
[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172 static const uchar patns
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
175 const uchar
*inbuf
= *inbufp
;
178 if (*inbytesleftp
< 1)
190 /* The number of leading 1-bits in the first byte indicates how many
192 for (nbytes
= 2; nbytes
< 7; nbytes
++)
193 if ((c
& ~masks
[nbytes
-1]) == patns
[nbytes
-1])
198 if (*inbytesleftp
< nbytes
)
201 c
= (c
& masks
[nbytes
-1]);
203 for (i
= 1; i
< nbytes
; i
++)
205 cppchar_t n
= *inbuf
++;
206 if ((n
& 0xC0) != 0x80)
208 c
= ((c
<< 6) + (n
& 0x3F));
211 /* Make sure the shortest possible encoding was used. */
212 if (c
<= 0x7F && nbytes
> 1) return EILSEQ
;
213 if (c
<= 0x7FF && nbytes
> 2) return EILSEQ
;
214 if (c
<= 0xFFFF && nbytes
> 3) return EILSEQ
;
215 if (c
<= 0x1FFFFF && nbytes
> 4) return EILSEQ
;
216 if (c
<= 0x3FFFFFF && nbytes
> 5) return EILSEQ
;
218 /* Make sure the character is valid. */
219 if (c
> 0x7FFFFFFF || (c
>= 0xD800 && c
<= 0xDFFF)) return EILSEQ
;
223 *inbytesleftp
-= nbytes
;
228 one_cppchar_to_utf8 (cppchar_t c
, uchar
**outbufp
, size_t *outbytesleftp
)
230 static const uchar masks
[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231 static const uchar limits
[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
233 uchar buf
[6], *p
= &buf
[6];
234 uchar
*outbuf
= *outbufp
;
243 *--p
= ((c
& 0x3F) | 0x80);
247 while (c
>= 0x3F || (c
& limits
[nbytes
-1]));
248 *--p
= (c
| masks
[nbytes
-1]);
251 if (*outbytesleftp
< nbytes
)
256 *outbytesleftp
-= nbytes
;
261 /* The following four functions transform one character between the two
262 encodings named in the function name. All have the signature
263 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264 uchar **outbufp, size_t *outbytesleftp)
266 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267 interpreted as a boolean indicating whether big-endian or
268 little-endian encoding is to be used for the member of the pair
271 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
274 The return value is either 0 for success, or an errno value for
275 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276 input sequence), ir EINVAL (incomplete input sequence). */
279 one_utf8_to_utf32 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
280 uchar
**outbufp
, size_t *outbytesleftp
)
286 /* Check for space first, since we know exactly how much we need. */
287 if (*outbytesleftp
< 4)
290 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
295 outbuf
[bigend
? 3 : 0] = (s
& 0x000000FF);
296 outbuf
[bigend
? 2 : 1] = (s
& 0x0000FF00) >> 8;
297 outbuf
[bigend
? 1 : 2] = (s
& 0x00FF0000) >> 16;
298 outbuf
[bigend
? 0 : 3] = (s
& 0xFF000000) >> 24;
306 one_utf32_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
307 uchar
**outbufp
, size_t *outbytesleftp
)
313 if (*inbytesleftp
< 4)
318 s
= inbuf
[bigend
? 0 : 3] << 24;
319 s
+= inbuf
[bigend
? 1 : 2] << 16;
320 s
+= inbuf
[bigend
? 2 : 1] << 8;
321 s
+= inbuf
[bigend
? 3 : 0];
323 if (s
>= 0x7FFFFFFF || (s
>= 0xD800 && s
<= 0xDFFF))
326 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
336 one_utf8_to_utf16 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
337 uchar
**outbufp
, size_t *outbytesleftp
)
341 const uchar
*save_inbuf
= *inbufp
;
342 size_t save_inbytesleft
= *inbytesleftp
;
343 uchar
*outbuf
= *outbufp
;
345 rval
= one_utf8_to_cppchar (inbufp
, inbytesleftp
, &s
);
351 *inbufp
= save_inbuf
;
352 *inbytesleftp
= save_inbytesleft
;
358 if (*outbytesleftp
< 2)
360 *inbufp
= save_inbuf
;
361 *inbytesleftp
= save_inbytesleft
;
364 outbuf
[bigend
? 1 : 0] = (s
& 0x00FF);
365 outbuf
[bigend
? 0 : 1] = (s
& 0xFF00) >> 8;
375 if (*outbytesleftp
< 4)
377 *inbufp
= save_inbuf
;
378 *inbytesleftp
= save_inbytesleft
;
382 hi
= (s
- 0x10000) / 0x400 + 0xD800;
383 lo
= (s
- 0x10000) % 0x400 + 0xDC00;
385 /* Even if we are little-endian, put the high surrogate first.
386 ??? Matches practice? */
387 outbuf
[bigend
? 1 : 0] = (hi
& 0x00FF);
388 outbuf
[bigend
? 0 : 1] = (hi
& 0xFF00) >> 8;
389 outbuf
[bigend
? 3 : 2] = (lo
& 0x00FF);
390 outbuf
[bigend
? 2 : 3] = (lo
& 0xFF00) >> 8;
399 one_utf16_to_utf8 (iconv_t bigend
, const uchar
**inbufp
, size_t *inbytesleftp
,
400 uchar
**outbufp
, size_t *outbytesleftp
)
403 const uchar
*inbuf
= *inbufp
;
406 if (*inbytesleftp
< 2)
408 s
= inbuf
[bigend
? 0 : 1] << 8;
409 s
+= inbuf
[bigend
? 1 : 0];
411 /* Low surrogate without immediately preceding high surrogate is invalid. */
412 if (s
>= 0xDC00 && s
<= 0xDFFF)
414 /* High surrogate must have a following low surrogate. */
415 else if (s
>= 0xD800 && s
<= 0xDBFF)
417 cppchar_t hi
= s
, lo
;
418 if (*inbytesleftp
< 4)
421 lo
= inbuf
[bigend
? 2 : 3] << 8;
422 lo
+= inbuf
[bigend
? 3 : 2];
424 if (lo
< 0xDC00 || lo
> 0xDFFF)
427 s
= (hi
- 0xD800) * 0x400 + (lo
- 0xDC00) + 0x10000;
430 rval
= one_cppchar_to_utf8 (s
, outbufp
, outbytesleftp
);
434 /* Success - update the input pointers (one_cppchar_to_utf8 has done
435 the output pointers for us). */
449 /* Helper routine for the next few functions. The 'const' on
450 one_conversion means that we promise not to modify what function is
451 pointed to, which lets the inliner see through it. */
454 conversion_loop (int (*const one_conversion
)(iconv_t
, const uchar
**, size_t *,
456 iconv_t cd
, const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
460 size_t inbytesleft
, outbytesleft
;
465 outbuf
= to
->text
+ to
->len
;
466 outbytesleft
= to
->asize
- to
->len
;
471 rval
= one_conversion (cd
, &inbuf
, &inbytesleft
,
472 &outbuf
, &outbytesleft
);
473 while (inbytesleft
&& !rval
);
475 if (__builtin_expect (inbytesleft
== 0, 1))
477 to
->len
= to
->asize
- outbytesleft
;
486 outbytesleft
+= OUTBUF_BLOCK_SIZE
;
487 to
->asize
+= OUTBUF_BLOCK_SIZE
;
488 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
489 outbuf
= to
->text
+ to
->asize
- outbytesleft
;
494 /* These functions convert entire strings between character sets.
495 They all have the signature
497 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
499 The input string FROM is converted as specified by the function
500 name plus the iconv descriptor CD (which may be fake), and the
501 result appended to TO. On any error, false is returned, otherwise true. */
503 /* These four use the custom conversion code above. */
505 convert_utf8_utf16 (iconv_t cd
, const uchar
*from
, size_t flen
,
506 struct _cpp_strbuf
*to
)
508 return conversion_loop (one_utf8_to_utf16
, cd
, from
, flen
, to
);
512 convert_utf8_utf32 (iconv_t cd
, const uchar
*from
, size_t flen
,
513 struct _cpp_strbuf
*to
)
515 return conversion_loop (one_utf8_to_utf32
, cd
, from
, flen
, to
);
519 convert_utf16_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
520 struct _cpp_strbuf
*to
)
522 return conversion_loop (one_utf16_to_utf8
, cd
, from
, flen
, to
);
526 convert_utf32_utf8 (iconv_t cd
, const uchar
*from
, size_t flen
,
527 struct _cpp_strbuf
*to
)
529 return conversion_loop (one_utf32_to_utf8
, cd
, from
, flen
, to
);
532 /* Identity conversion, used when we have no alternative. */
534 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED
,
535 const uchar
*from
, size_t flen
, struct _cpp_strbuf
*to
)
537 if (to
->len
+ flen
> to
->asize
)
539 to
->asize
= to
->len
+ flen
;
540 to
->asize
+= to
->asize
/ 4;
541 to
->text
= XRESIZEVEC (uchar
, to
->text
, to
->asize
);
543 memcpy (to
->text
+ to
->len
, from
, flen
);
548 /* And this one uses the system iconv primitive. It's a little
549 different, since iconv's interface is a little different. */
552 #define CONVERT_ICONV_GROW_BUFFER \
554 outbytesleft += OUTBUF_BLOCK_SIZE; \
555 to->asize += OUTBUF_BLOCK_SIZE; \
556 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
557 outbuf = (char *)to->text + to->asize - outbytesleft; \
561 convert_using_iconv (iconv_t cd
, const uchar
*from
, size_t flen
,
562 struct _cpp_strbuf
*to
)
564 ICONV_CONST
char *inbuf
;
566 size_t inbytesleft
, outbytesleft
;
568 /* Reset conversion descriptor and check that it is valid. */
569 if (iconv (cd
, 0, 0, 0, 0) == (size_t)-1)
572 inbuf
= (ICONV_CONST
char *)from
;
574 outbuf
= (char *)to
->text
+ to
->len
;
575 outbytesleft
= to
->asize
- to
->len
;
579 iconv (cd
, &inbuf
, &inbytesleft
, &outbuf
, &outbytesleft
);
580 if (__builtin_expect (inbytesleft
== 0, 1))
582 /* Close out any shift states, returning to the initial state. */
583 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
588 CONVERT_ICONV_GROW_BUFFER
;
589 if (iconv (cd
, 0, 0, &outbuf
, &outbytesleft
) == (size_t)-1)
593 to
->len
= to
->asize
- outbytesleft
;
599 CONVERT_ICONV_GROW_BUFFER
;
603 #define convert_using_iconv 0 /* prevent undefined symbol error below */
606 /* Arrange for the above custom conversion logic to be used automatically
607 when conversion between a suitable pair of character sets is requested. */
609 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
610 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
612 struct cpp_conversion
618 static const struct cpp_conversion conversion_tab
[] = {
619 { "UTF-8/UTF-32LE", convert_utf8_utf32
, (iconv_t
)0 },
620 { "UTF-8/UTF-32BE", convert_utf8_utf32
, (iconv_t
)1 },
621 { "UTF-8/UTF-16LE", convert_utf8_utf16
, (iconv_t
)0 },
622 { "UTF-8/UTF-16BE", convert_utf8_utf16
, (iconv_t
)1 },
623 { "UTF-32LE/UTF-8", convert_utf32_utf8
, (iconv_t
)0 },
624 { "UTF-32BE/UTF-8", convert_utf32_utf8
, (iconv_t
)1 },
625 { "UTF-16LE/UTF-8", convert_utf16_utf8
, (iconv_t
)0 },
626 { "UTF-16BE/UTF-8", convert_utf16_utf8
, (iconv_t
)1 },
629 /* Subroutine of cpp_init_iconv: initialize and return a
630 cset_converter structure for conversion from FROM to TO. If
631 iconv_open() fails, issue an error and return an identity
632 converter. Silently return an identity converter if FROM and TO
634 static struct cset_converter
635 init_iconv_desc (cpp_reader
*pfile
, const char *to
, const char *from
)
637 struct cset_converter ret
;
644 if (!strcasecmp (to
, from
))
646 ret
.func
= convert_no_conversion
;
647 ret
.cd
= (iconv_t
) -1;
652 pair
= (char *) alloca(strlen(to
) + strlen(from
) + 2);
657 for (i
= 0; i
< ARRAY_SIZE (conversion_tab
); i
++)
658 if (!strcasecmp (pair
, conversion_tab
[i
].pair
))
660 ret
.func
= conversion_tab
[i
].func
;
661 ret
.cd
= conversion_tab
[i
].fake_cd
;
666 /* No custom converter - try iconv. */
669 ret
.func
= convert_using_iconv
;
670 ret
.cd
= iconv_open (to
, from
);
673 if (ret
.cd
== (iconv_t
) -1)
676 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME should be DL_SORRY */
677 "conversion from %s to %s not supported by iconv",
680 cpp_errno (pfile
, CPP_DL_ERROR
, "iconv_open");
682 ret
.func
= convert_no_conversion
;
687 cpp_error (pfile
, CPP_DL_ERROR
, /* FIXME: should be DL_SORRY */
688 "no iconv implementation, cannot convert from %s to %s",
690 ret
.func
= convert_no_conversion
;
691 ret
.cd
= (iconv_t
) -1;
697 /* If charset conversion is requested, initialize iconv(3) descriptors
698 for conversion from the source character set to the execution
699 character sets. If iconv is not present in the C library, and
700 conversion is requested, issue an error. */
703 cpp_init_iconv (cpp_reader
*pfile
)
705 const char *ncset
= CPP_OPTION (pfile
, narrow_charset
);
706 const char *wcset
= CPP_OPTION (pfile
, wide_charset
);
707 const char *default_wcset
;
709 bool be
= CPP_OPTION (pfile
, bytes_big_endian
);
711 if (CPP_OPTION (pfile
, wchar_precision
) >= 32)
712 default_wcset
= be
? "UTF-32BE" : "UTF-32LE";
713 else if (CPP_OPTION (pfile
, wchar_precision
) >= 16)
714 default_wcset
= be
? "UTF-16BE" : "UTF-16LE";
716 /* This effectively means that wide strings are not supported,
717 so don't do any conversion at all. */
718 default_wcset
= SOURCE_CHARSET
;
721 ncset
= SOURCE_CHARSET
;
723 wcset
= default_wcset
;
725 pfile
->narrow_cset_desc
= init_iconv_desc (pfile
, ncset
, SOURCE_CHARSET
);
726 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
727 pfile
->utf8_cset_desc
= init_iconv_desc (pfile
, "UTF-8", SOURCE_CHARSET
);
728 pfile
->utf8_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
729 pfile
->char16_cset_desc
= init_iconv_desc (pfile
,
730 be
? "UTF-16BE" : "UTF-16LE",
732 pfile
->char16_cset_desc
.width
= 16;
733 pfile
->char32_cset_desc
= init_iconv_desc (pfile
,
734 be
? "UTF-32BE" : "UTF-32LE",
736 pfile
->char32_cset_desc
.width
= 32;
737 pfile
->wide_cset_desc
= init_iconv_desc (pfile
, wcset
, SOURCE_CHARSET
);
738 pfile
->wide_cset_desc
.width
= CPP_OPTION (pfile
, wchar_precision
);
741 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
743 _cpp_destroy_iconv (cpp_reader
*pfile
)
747 if (pfile
->narrow_cset_desc
.func
== convert_using_iconv
)
748 iconv_close (pfile
->narrow_cset_desc
.cd
);
749 if (pfile
->utf8_cset_desc
.func
== convert_using_iconv
)
750 iconv_close (pfile
->utf8_cset_desc
.cd
);
751 if (pfile
->char16_cset_desc
.func
== convert_using_iconv
)
752 iconv_close (pfile
->char16_cset_desc
.cd
);
753 if (pfile
->char32_cset_desc
.func
== convert_using_iconv
)
754 iconv_close (pfile
->char32_cset_desc
.cd
);
755 if (pfile
->wide_cset_desc
.func
== convert_using_iconv
)
756 iconv_close (pfile
->wide_cset_desc
.cd
);
760 /* Utility routine for use by a full compiler. C is a character taken
761 from the *basic* source character set, encoded in the host's
762 execution encoding. Convert it to (the target's) execution
763 encoding, and return that value.
765 Issues an internal error if C's representation in the narrow
766 execution character set fails to be a single-byte value (C99
767 5.2.1p3: "The representation of each member of the source and
768 execution character sets shall fit in a byte.") May also issue an
769 internal error if C fails to be a member of the basic source
770 character set (testing this exactly is too hard, especially when
771 the host character set is EBCDIC). */
773 cpp_host_to_exec_charset (cpp_reader
*pfile
, cppchar_t c
)
776 struct _cpp_strbuf tbuf
;
778 /* This test is merely an approximation, but it suffices to catch
779 the most important thing, which is that we don't get handed a
780 character outside the unibyte range of the host character set. */
781 if (c
> LAST_POSSIBLY_BASIC_SOURCE_CHAR
)
783 cpp_error (pfile
, CPP_DL_ICE
,
784 "character 0x%lx is not in the basic source character set\n",
789 /* Being a character in the unibyte range of the host character set,
790 we can safely splat it into a one-byte buffer and trust that that
791 is a well-formed string. */
794 /* This should never need to reallocate, but just in case... */
796 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
799 if (!APPLY_CONVERSION (pfile
->narrow_cset_desc
, sbuf
, 1, &tbuf
))
801 cpp_errno (pfile
, CPP_DL_ICE
, "converting to execution character set");
806 cpp_error (pfile
, CPP_DL_ICE
,
807 "character 0x%lx is not unibyte in execution character set",
818 /* cpp_substring_ranges's constructor. */
820 cpp_substring_ranges::cpp_substring_ranges () :
825 m_ranges
= XNEWVEC (source_range
, m_alloc_ranges
);
828 /* cpp_substring_ranges's destructor. */
830 cpp_substring_ranges::~cpp_substring_ranges ()
835 /* Add RANGE to the vector of source_range information. */
838 cpp_substring_ranges::add_range (source_range range
)
840 if (m_num_ranges
>= m_alloc_ranges
)
844 = (source_range
*)xrealloc (m_ranges
,
845 sizeof (source_range
) * m_alloc_ranges
);
847 m_ranges
[m_num_ranges
++] = range
;
850 /* Read NUM ranges from LOC_READER, adding them to the vector of source_range
854 cpp_substring_ranges::add_n_ranges (int num
,
855 cpp_string_location_reader
&loc_reader
)
857 for (int i
= 0; i
< num
; i
++)
858 add_range (loc_reader
.get_next ());
863 /* Utility routine that computes a mask of the form 0000...111... with
866 width_to_mask (size_t width
)
868 width
= MIN (width
, BITS_PER_CPPCHAR_T
);
869 if (width
>= CHAR_BIT
* sizeof (size_t))
872 return ((size_t) 1 << width
) - 1;
875 /* A large table of unicode character information. */
877 /* Valid in a C99 identifier? */
879 /* Valid in a C99 identifier, but not as the first character? */
881 /* Valid in a C++ identifier? */
883 /* Valid in a C11/C++11 identifier? */
885 /* Valid in a C11/C++11 identifier, but not as the first character? */
887 /* NFC representation is not valid in an identifier? */
889 /* Might be valid NFC form? */
891 /* Might be valid NFKC form? */
893 /* Certain preceding characters might make it not valid NFC/NKFC form? */
898 /* Bitmap of flags above. */
899 unsigned short flags
;
900 /* Combining class of the character. */
901 unsigned char combine
;
902 /* Last character in the range described by this entry. */
907 /* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive. */
908 #define UCS_LIMIT 0x10FFFF
910 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
911 the start of an identifier, and 0 if C is not valid in an
912 identifier. We assume C has already gone through the checks of
913 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
914 algorithm is a simple binary search on the table defined in
918 ucn_valid_in_identifier (cpp_reader
*pfile
, cppchar_t c
,
919 struct normalize_state
*nst
)
922 unsigned short valid_flags
, invalid_start_flags
;
928 mx
= ARRAY_SIZE (ucnranges
) - 1;
932 if (c
<= ucnranges
[md
].end
)
938 /* When -pedantic, we require the character to have been listed by
939 the standard for the current language. Otherwise, we accept the
940 union of the acceptable sets for all supported language versions. */
941 valid_flags
= C99
| CXX
| C11
;
942 if (CPP_PEDANTIC (pfile
))
944 if (CPP_OPTION (pfile
, c11_identifiers
))
946 else if (CPP_OPTION (pfile
, c99
))
948 else if (CPP_OPTION (pfile
, cplusplus
))
951 if (! (ucnranges
[mn
].flags
& valid_flags
))
953 if (CPP_OPTION (pfile
, c11_identifiers
))
954 invalid_start_flags
= N11
;
955 else if (CPP_OPTION (pfile
, c99
))
956 invalid_start_flags
= N99
;
958 invalid_start_flags
= 0;
961 if (ucnranges
[mn
].combine
!= 0 && ucnranges
[mn
].combine
< nst
->prev_class
)
962 nst
->level
= normalized_none
;
963 else if (ucnranges
[mn
].flags
& CTX
)
966 cppchar_t p
= nst
->previous
;
968 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
969 and are combined algorithmically from a sequence of the form
970 1100-1112 1161-1175 11A8-11C2
971 (if the third is not present, it is treated as 11A7, which is not
972 really a valid character).
973 Unfortunately, C99 allows (only) the NFC form, but C++ allows
974 only the combining characters. */
975 if (c
>= 0x1161 && c
<= 0x1175)
976 safe
= p
< 0x1100 || p
> 0x1112;
977 else if (c
>= 0x11A8 && c
<= 0x11C2)
978 safe
= (p
< 0xAC00 || p
> 0xD7A3 || (p
- 0xAC00) % 28 != 0);
980 safe
= check_nfc (pfile
, c
, p
);
983 if ((c
>= 0x1161 && c
<= 0x1175) || (c
>= 0x11A8 && c
<= 0x11C2))
984 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
986 nst
->level
= normalized_none
;
989 else if (ucnranges
[mn
].flags
& NKC
)
991 else if (ucnranges
[mn
].flags
& NFC
)
992 nst
->level
= MAX (nst
->level
, normalized_C
);
993 else if (ucnranges
[mn
].flags
& CID
)
994 nst
->level
= MAX (nst
->level
, normalized_identifier_C
);
996 nst
->level
= normalized_none
;
997 if (ucnranges
[mn
].combine
== 0)
999 nst
->prev_class
= ucnranges
[mn
].combine
;
1001 /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
1002 UCN combining characters may not begin identifiers. */
1003 if (ucnranges
[mn
].flags
& invalid_start_flags
)
1009 /* [lex.charset]: The character designated by the universal character
1010 name \UNNNNNNNN is that character whose character short name in
1011 ISO/IEC 10646 is NNNNNNNN; the character designated by the
1012 universal character name \uNNNN is that character whose character
1013 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
1014 for a universal character name corresponds to a surrogate code point
1015 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
1016 Additionally, if the hexadecimal value for a universal-character-name
1017 outside a character or string literal corresponds to a control character
1018 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
1019 character in the basic source character set, the program is ill-formed.
1021 C99 6.4.3: A universal character name shall not specify a character
1022 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
1023 or 0060 (`), nor one in the range D800 through DFFF inclusive.
1025 If the hexadecimal value is larger than the upper bound of the UCS
1026 codespace specified in ISO/IEC 10646, a pedantic warning is issued
1027 in all versions of C and in the C++20 or later versions of C++.
1029 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
1030 buffer end is delimited by a non-hex digit. Returns false if the
1031 UCN has not been consumed, true otherwise.
1033 The value of the UCN, whether valid or invalid, is returned in *CP.
1034 Diagnostics are emitted for invalid values. PSTR is updated to point
1035 one beyond the UCN, or to the syntactically invalid character.
1037 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
1038 an identifier, or 2 otherwise.
1040 If LOC_READER is non-NULL, then position information is
1041 read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
1044 _cpp_valid_ucn (cpp_reader
*pfile
, const uchar
**pstr
,
1045 const uchar
*limit
, int identifier_pos
,
1046 struct normalize_state
*nst
, cppchar_t
*cp
,
1047 source_range
*char_range
,
1048 cpp_string_location_reader
*loc_reader
)
1050 cppchar_t result
, c
;
1051 unsigned int length
;
1052 const uchar
*str
= *pstr
;
1053 const uchar
*base
= str
- 2;
1055 if (!CPP_OPTION (pfile
, cplusplus
) && !CPP_OPTION (pfile
, c99
))
1056 cpp_error (pfile
, CPP_DL_WARNING
,
1057 "universal character names are only valid in C++ and C99");
1058 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
) > 0
1059 && !CPP_OPTION (pfile
, cplusplus
))
1060 cpp_error (pfile
, CPP_DL_WARNING
,
1061 "C99's universal character names are incompatible with C90");
1062 else if (CPP_WTRADITIONAL (pfile
) && identifier_pos
== 0)
1063 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1064 "the meaning of '\\%c' is different in traditional C",
1069 else if (str
[-1] == 'U')
1073 cpp_error (pfile
, CPP_DL_ICE
, "In _cpp_valid_ucn but not a UCN");
1086 gcc_assert (char_range
);
1087 char_range
->m_finish
= loc_reader
->get_next ().m_finish
;
1089 result
= (result
<< 4) + hex_value (c
);
1091 while (--length
&& str
< limit
);
1093 /* Partial UCNs are not valid in strings, but decompose into
1094 multiple tokens in identifiers, so we can't give a helpful
1095 error message in that case. */
1096 if (length
&& identifier_pos
)
1105 cpp_error (pfile
, CPP_DL_ERROR
,
1106 "incomplete universal character name %.*s",
1107 (int) (str
- base
), base
);
1110 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1111 hex escapes so that this also works with EBCDIC hosts.
1112 C++0x permits everything below 0xa0 within literals;
1113 ucn_valid_in_identifier will complain about identifiers. */
1114 else if ((result
< 0xa0
1115 && !CPP_OPTION (pfile
, cplusplus
)
1116 && (result
!= 0x24 && result
!= 0x40 && result
!= 0x60))
1117 || (result
& 0x80000000)
1118 || (result
>= 0xD800 && result
<= 0xDFFF))
1120 cpp_error (pfile
, CPP_DL_ERROR
,
1121 "%.*s is not a valid universal character",
1122 (int) (str
- base
), base
);
1125 else if (identifier_pos
&& result
== 0x24
1126 && CPP_OPTION (pfile
, dollars_in_ident
))
1128 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1130 CPP_OPTION (pfile
, warn_dollars
) = 0;
1131 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1133 NORMALIZE_STATE_UPDATE_IDNUM (nst
, result
);
1135 else if (identifier_pos
)
1137 int validity
= ucn_valid_in_identifier (pfile
, result
, nst
);
1140 cpp_error (pfile
, CPP_DL_ERROR
,
1141 "universal character %.*s is not valid in an identifier",
1142 (int) (str
- base
), base
);
1143 else if (validity
== 2 && identifier_pos
== 1)
1144 cpp_error (pfile
, CPP_DL_ERROR
,
1145 "universal character %.*s is not valid at the start of an identifier",
1146 (int) (str
- base
), base
);
1148 else if (result
> UCS_LIMIT
1149 && (!CPP_OPTION (pfile
, cplusplus
)
1150 || CPP_OPTION (pfile
, lang
) > CLK_CXX17
))
1151 cpp_error (pfile
, CPP_DL_PEDWARN
,
1152 "%.*s is outside the UCS codespace",
1153 (int) (str
- base
), base
);
1159 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1160 it to the execution character set and write the result into TBUF,
1161 if TBUF is non-NULL.
1162 An advanced pointer is returned. Issues all relevant diagnostics.
1163 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1164 contains the location of the character so far: location information
1165 is read from *LOC_READER, and *RANGES is updated accordingly. */
1166 static const uchar
*
1167 convert_ucn (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1168 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1169 source_range char_range
,
1170 cpp_string_location_reader
*loc_reader
,
1171 cpp_substring_ranges
*ranges
)
1176 size_t bytesleft
= 6;
1178 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
1180 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1181 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
1183 from
++; /* Skip u/U. */
1186 /* The u/U is part of the spelling of this character. */
1187 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
1189 _cpp_valid_ucn (pfile
, &from
, limit
, 0, &nst
,
1190 &ucn
, &char_range
, loc_reader
);
1192 rval
= one_cppchar_to_utf8 (ucn
, &bufp
, &bytesleft
);
1196 cpp_errno (pfile
, CPP_DL_ERROR
,
1197 "converting UCN to source character set");
1202 if (!APPLY_CONVERSION (cvt
, buf
, 6 - bytesleft
, tbuf
))
1203 cpp_errno (pfile
, CPP_DL_ERROR
,
1204 "converting UCN to execution character set");
1208 int num_encoded_bytes
= 6 - bytesleft
;
1209 for (int i
= 0; i
< num_encoded_bytes
; i
++)
1210 ranges
->add_range (char_range
);
1217 /* Performs a similar task as _cpp_valid_ucn, but parses UTF-8-encoded
1218 extended characters rather than UCNs. If the return value is TRUE, then a
1219 character was successfully decoded and stored in *CP; *PSTR has been
1220 updated to point one past the valid UTF-8 sequence. Diagnostics may have
1221 been emitted if the character parsed is not allowed in the current context.
1222 If the return value is FALSE, then *PSTR has not been modified and *CP may
1223 equal 0, to indicate that *PSTR does not form a valid UTF-8 sequence, or it
1224 may, when processing an identifier in C mode, equal a codepoint that was
1225 validly encoded but is not allowed to appear in an identifier. In either
1226 case, no diagnostic is emitted, and the return value of FALSE should cause
1227 a new token to be formed.
1229 Unlike _cpp_valid_ucn, this will never be called when lexing a string; only
1230 a potential identifier, or a CPP_OTHER token. NST is unused in the latter
1233 As in _cpp_valid_ucn, IDENTIFIER_POS is 0 when not in an identifier, 1 for
1234 the start of an identifier, or 2 otherwise. */
1237 _cpp_valid_utf8 (cpp_reader
*pfile
,
1241 struct normalize_state
*nst
,
1244 const uchar
*base
= *pstr
;
1245 size_t inbytesleft
= limit
- base
;
1246 if (one_utf8_to_cppchar (pstr
, &inbytesleft
, cp
))
1248 /* No diagnostic here as this byte will rather become a
1256 switch (ucn_valid_in_identifier (pfile
, *cp
, nst
))
1260 /* In C++, this is an error for invalid character in an identifier
1261 because logically, the UTF-8 was converted to a UCN during
1262 translation phase 1 (even though we don't physically do it that
1263 way). In C, this byte rather becomes grammatically a separate
1266 if (CPP_OPTION (pfile
, cplusplus
))
1267 cpp_error (pfile
, CPP_DL_ERROR
,
1268 "extended character %.*s is not valid in an identifier",
1269 (int) (*pstr
- base
), base
);
1279 if (identifier_pos
== 1)
1281 /* This is treated the same way in C++ or C99 -- lexed as an
1282 identifier which is then invalid because an identifier is
1283 not allowed to start with this character. */
1284 cpp_error (pfile
, CPP_DL_ERROR
,
1285 "extended character %.*s is not valid at the start of an identifier",
1286 (int) (*pstr
- base
), base
);
1295 /* Subroutine of convert_hex and convert_oct. N is the representation
1296 in the execution character set of a numeric escape; write it into the
1297 string buffer TBUF and update the end-of-string pointer therein. WIDE
1298 is true if it's a wide string that's being assembled in TBUF. This
1299 function issues no diagnostics and never fails. */
1301 emit_numeric_escape (cpp_reader
*pfile
, cppchar_t n
,
1302 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
)
1304 size_t width
= cvt
.width
;
1306 if (width
!= CPP_OPTION (pfile
, char_precision
))
1308 /* We have to render this into the target byte order, which may not
1309 be our byte order. */
1310 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
1311 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
1312 size_t cmask
= width_to_mask (cwidth
);
1313 size_t nbwc
= width
/ cwidth
;
1315 size_t off
= tbuf
->len
;
1318 if (tbuf
->len
+ nbwc
> tbuf
->asize
)
1320 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
1321 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
1324 for (i
= 0; i
< nbwc
; i
++)
1328 tbuf
->text
[off
+ (bigend
? nbwc
- i
- 1 : i
)] = c
;
1334 /* Note: this code does not handle the case where the target
1335 and host have a different number of bits in a byte. */
1336 if (tbuf
->len
+ 1 > tbuf
->asize
)
1338 tbuf
->asize
+= OUTBUF_BLOCK_SIZE
;
1339 tbuf
->text
= XRESIZEVEC (uchar
, tbuf
->text
, tbuf
->asize
);
1341 tbuf
->text
[tbuf
->len
++] = n
;
1345 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
1346 character set and write it into the string buffer TBUF (if non-NULL).
1347 Returns an advanced pointer, and issues diagnostics as necessary.
1348 No character set translation occurs; this routine always produces the
1349 execution-set character with numeric value equal to the given hex
1350 number. You can, e.g. generate surrogate pairs this way.
1351 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1352 contains the location of the character so far: location information
1353 is read from *LOC_READER, and *RANGES is updated accordingly. */
1354 static const uchar
*
1355 convert_hex (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1356 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1357 source_range char_range
,
1358 cpp_string_location_reader
*loc_reader
,
1359 cpp_substring_ranges
*ranges
)
1361 cppchar_t c
, n
= 0, overflow
= 0;
1362 int digits_found
= 0;
1363 size_t width
= cvt
.width
;
1364 size_t mask
= width_to_mask (width
);
1366 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1367 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
1369 if (CPP_WTRADITIONAL (pfile
))
1370 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1371 "the meaning of '\\x' is different in traditional C");
1376 /* The 'x' is part of the spelling of this character. */
1378 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
1380 while (from
< limit
)
1387 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
1388 overflow
|= n
^ (n
<< 4 >> 4);
1389 n
= (n
<< 4) + hex_value (c
);
1395 cpp_error (pfile
, CPP_DL_ERROR
,
1396 "\\x used with no following hex digits");
1400 if (overflow
| (n
!= (n
& mask
)))
1402 cpp_error (pfile
, CPP_DL_PEDWARN
,
1403 "hex escape sequence out of range");
1408 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
1410 ranges
->add_range (char_range
);
1415 /* Convert an octal escape, pointed to by FROM, to the execution
1416 character set and write it into the string buffer TBUF. Returns an
1417 advanced pointer, and issues diagnostics as necessary.
1418 No character set translation occurs; this routine always produces the
1419 execution-set character with numeric value equal to the given octal
1421 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1422 contains the location of the character so far: location information
1423 is read from *LOC_READER, and *RANGES is updated accordingly. */
1424 static const uchar
*
1425 convert_oct (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1426 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1427 source_range char_range
,
1428 cpp_string_location_reader
*loc_reader
,
1429 cpp_substring_ranges
*ranges
)
1433 size_t width
= cvt
.width
;
1434 size_t mask
= width_to_mask (width
);
1435 bool overflow
= false;
1437 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1438 gcc_assert ((loc_reader
!= NULL
) == (ranges
!= NULL
));
1440 while (from
< limit
&& count
++ < 3)
1443 if (c
< '0' || c
> '7')
1447 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
1448 overflow
|= n
^ (n
<< 3 >> 3);
1449 n
= (n
<< 3) + c
- '0';
1452 if (n
!= (n
& mask
))
1454 cpp_error (pfile
, CPP_DL_PEDWARN
,
1455 "octal escape sequence out of range");
1460 emit_numeric_escape (pfile
, n
, tbuf
, cvt
);
1462 ranges
->add_range (char_range
);
1467 /* Convert an escape sequence (pointed to by FROM) to its value on
1468 the target, and to the execution character set. Do not scan past
1469 LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
1470 Returns an advanced pointer. Handles all relevant diagnostics.
1471 If LOC_READER is non-NULL, then RANGES must be non-NULL: location
1472 information is read from *LOC_READER, and *RANGES is updated
1474 static const uchar
*
1475 convert_escape (cpp_reader
*pfile
, const uchar
*from
, const uchar
*limit
,
1476 struct _cpp_strbuf
*tbuf
, struct cset_converter cvt
,
1477 cpp_string_location_reader
*loc_reader
,
1478 cpp_substring_ranges
*ranges
)
1480 /* Values of \a \b \e \f \n \r \t \v respectively. */
1481 #if HOST_CHARSET == HOST_CHARSET_ASCII
1482 static const uchar charconsts
[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
1483 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
1484 static const uchar charconsts
[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
1486 #error "unknown host character set"
1491 /* Record the location of the backslash. */
1492 source_range char_range
;
1494 char_range
= loc_reader
->get_next ();
1499 /* UCNs, hex escapes, and octal escapes are processed separately. */
1501 return convert_ucn (pfile
, from
, limit
, tbuf
, cvt
,
1502 char_range
, loc_reader
, ranges
);
1505 return convert_hex (pfile
, from
, limit
, tbuf
, cvt
,
1506 char_range
, loc_reader
, ranges
);
1509 case '0': case '1': case '2': case '3':
1510 case '4': case '5': case '6': case '7':
1511 return convert_oct (pfile
, from
, limit
, tbuf
, cvt
,
1512 char_range
, loc_reader
, ranges
);
1514 /* Various letter escapes. Get the appropriate host-charset
1516 case '\\': case '\'': case '"': case '?': break;
1518 case '(': case '{': case '[': case '%':
1519 /* '\(', etc, can be used at the beginning of a line in a long
1520 string split onto multiple lines with \-newline, to prevent
1521 Emacs or other text editors from getting confused. '\%' can
1522 be used to prevent SCCS from mangling printf format strings. */
1523 if (CPP_PEDANTIC (pfile
))
1527 case 'b': c
= charconsts
[1]; break;
1528 case 'f': c
= charconsts
[3]; break;
1529 case 'n': c
= charconsts
[4]; break;
1530 case 'r': c
= charconsts
[5]; break;
1531 case 't': c
= charconsts
[6]; break;
1532 case 'v': c
= charconsts
[7]; break;
1535 if (CPP_WTRADITIONAL (pfile
))
1536 cpp_warning (pfile
, CPP_W_TRADITIONAL
,
1537 "the meaning of '\\a' is different in traditional C");
1542 if (CPP_PEDANTIC (pfile
))
1543 cpp_error (pfile
, CPP_DL_PEDWARN
,
1544 "non-ISO-standard escape sequence, '\\%c'", (int) c
);
1551 cpp_error (pfile
, CPP_DL_PEDWARN
,
1552 "unknown escape sequence: '\\%c'", (int) c
);
1555 /* diagnostic.c does not support "%03o". When it does, this
1556 code can use %03o directly in the diagnostic again. */
1558 sprintf(buf
, "%03o", (int) c
);
1559 cpp_error (pfile
, CPP_DL_PEDWARN
,
1560 "unknown escape sequence: '\\%s'", buf
);
1565 /* Now convert what we have to the execution character set. */
1566 if (!APPLY_CONVERSION (cvt
, &c
, 1, tbuf
))
1567 cpp_errno (pfile
, CPP_DL_ERROR
,
1568 "converting escape sequence to execution character set");
1572 char_range
.m_finish
= loc_reader
->get_next ().m_finish
;
1573 ranges
->add_range (char_range
);
1579 /* TYPE is a token type. The return value is the conversion needed to
1580 convert from source to execution character set for the given type. */
1581 static struct cset_converter
1582 converter_for_type (cpp_reader
*pfile
, enum cpp_ttype type
)
1587 return pfile
->narrow_cset_desc
;
1589 case CPP_UTF8STRING
:
1590 return pfile
->utf8_cset_desc
;
1593 return pfile
->char16_cset_desc
;
1596 return pfile
->char32_cset_desc
;
1599 return pfile
->wide_cset_desc
;
1603 /* FROM is an array of cpp_string structures of length COUNT. These
1604 are to be converted from the source to the execution character set,
1605 escape sequences translated, and finally all are to be
1606 concatenated. WIDE indicates whether or not to produce a wide
1607 string. If TO is non-NULL, the result is written into TO.
1608 If LOC_READERS and OUT are non-NULL, then location information
1609 is read from LOC_READERS (which must be an array of length COUNT),
1610 and location information is written to *RANGES.
1612 Returns true for success, false for failure. */
1615 cpp_interpret_string_1 (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
1616 cpp_string
*to
, enum cpp_ttype type
,
1617 cpp_string_location_reader
*loc_readers
,
1618 cpp_substring_ranges
*out
)
1620 struct _cpp_strbuf tbuf
;
1621 const uchar
*p
, *base
, *limit
;
1623 struct cset_converter cvt
= converter_for_type (pfile
, type
);
1625 /* loc_readers and out must either be both NULL, or both be non-NULL. */
1626 gcc_assert ((loc_readers
!= NULL
) == (out
!= NULL
));
1630 tbuf
.asize
= MAX (OUTBUF_BLOCK_SIZE
, from
->len
);
1631 tbuf
.text
= XNEWVEC (uchar
, tbuf
.asize
);
1635 cpp_string_location_reader
*loc_reader
= NULL
;
1636 for (i
= 0; i
< count
; i
++)
1639 loc_reader
= &loc_readers
[i
];
1646 loc_reader
->get_next ();
1651 loc_reader
->get_next ();
1654 else if (*p
== 'L' || *p
== 'U') p
++;
1657 const uchar
*prefix
;
1659 /* Skip over 'R"'. */
1663 loc_reader
->get_next ();
1664 loc_reader
->get_next ();
1671 loc_reader
->get_next ();
1675 loc_reader
->get_next ();
1676 limit
= from
[i
].text
+ from
[i
].len
;
1677 if (limit
>= p
+ (p
- prefix
) + 1)
1678 limit
-= (p
- prefix
) + 1;
1680 /* Raw strings are all normal characters; these can be fed
1681 directly to convert_cset. */
1683 if (!APPLY_CONVERSION (cvt
, p
, limit
- p
, &tbuf
))
1688 /* If generating source ranges, assume we have a 1:1
1689 correspondence between bytes in the source encoding and bytes
1690 in the execution encoding (e.g. if we have a UTF-8 to UTF-8
1691 conversion), so that this run of bytes in the source file
1692 corresponds to a run of bytes in the execution string.
1693 This requirement is guaranteed by an early-reject in
1694 cpp_interpret_string_ranges. */
1695 gcc_assert (cvt
.func
== convert_no_conversion
);
1696 out
->add_n_ranges (limit
- p
, *loc_reader
);
1702 /* If we don't now have a leading quote, something has gone wrong.
1703 This can occur if cpp_interpret_string_ranges is handling a
1704 stringified macro argument, but should not be possible otherwise. */
1705 if (*p
!= '"' && *p
!= '\'')
1707 gcc_assert (out
!= NULL
);
1708 cpp_error (pfile
, CPP_DL_ERROR
, "missing open quote");
1714 /* Skip leading quote. */
1717 loc_reader
->get_next ();
1719 limit
= from
[i
].text
+ from
[i
].len
- 1; /* Skip trailing quote. */
1724 while (p
< limit
&& *p
!= '\\')
1728 /* We have a run of normal characters; these can be fed
1729 directly to convert_cset. */
1731 if (!APPLY_CONVERSION (cvt
, base
, p
- base
, &tbuf
))
1733 /* Similar to above: assumes we have a 1:1 correspondence
1734 between bytes in the source encoding and bytes in the
1735 execution encoding. */
1738 gcc_assert (cvt
.func
== convert_no_conversion
);
1739 out
->add_n_ranges (p
- base
, *loc_reader
);
1745 struct _cpp_strbuf
*tbuf_ptr
= to
? &tbuf
: NULL
;
1746 p
= convert_escape (pfile
, p
+ 1, limit
, tbuf_ptr
, cvt
,
1753 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
1755 emit_numeric_escape (pfile
, 0, &tbuf
, cvt
);
1756 tbuf
.text
= XRESIZEVEC (uchar
, tbuf
.text
, tbuf
.len
);
1757 to
->text
= tbuf
.text
;
1760 /* Use the location of the trailing quote as the location of the
1764 source_range range
= loc_reader
->get_next ();
1765 out
->add_range (range
);
1771 cpp_errno (pfile
, CPP_DL_ERROR
, "converting to execution character set");
1777 /* FROM is an array of cpp_string structures of length COUNT. These
1778 are to be converted from the source to the execution character set,
1779 escape sequences translated, and finally all are to be
1780 concatenated. WIDE indicates whether or not to produce a wide
1781 string. The result is written into TO. Returns true for success,
1782 false for failure. */
1784 cpp_interpret_string (cpp_reader
*pfile
, const cpp_string
*from
, size_t count
,
1785 cpp_string
*to
, enum cpp_ttype type
)
1787 return cpp_interpret_string_1 (pfile
, from
, count
, to
, type
, NULL
, NULL
);
1790 /* A "do nothing" diagnostic-handling callback for use by
1791 cpp_interpret_string_ranges, so that it can temporarily suppress
1792 diagnostic-handling. */
1795 noop_diagnostic_cb (cpp_reader
*, enum cpp_diagnostic_level
,
1796 enum cpp_warning_reason
, rich_location
*,
1797 const char *, va_list *)
1803 /* This function mimics the behavior of cpp_interpret_string, but
1804 rather than generating a string in the execution character set,
1805 *OUT is written to with the source code ranges of the characters
1807 FROM and LOC_READERS should both be arrays of length COUNT.
1808 Returns NULL for success, or an error message for failure. */
1811 cpp_interpret_string_ranges (cpp_reader
*pfile
, const cpp_string
*from
,
1812 cpp_string_location_reader
*loc_readers
,
1814 cpp_substring_ranges
*out
,
1815 enum cpp_ttype type
)
1817 /* There are a couple of cases in the range-handling in
1818 cpp_interpret_string_1 that rely on there being a 1:1 correspondence
1819 between bytes in the source encoding and bytes in the execution
1820 encoding, so that each byte in the execution string can correspond
1821 to the location of a byte in the source string.
1823 This holds for the typical case of a UTF-8 to UTF-8 conversion.
1824 Enforce this requirement by only attempting to track substring
1825 locations if we have source encoding == execution encoding.
1827 This is a stronger condition than we need, since we could e.g.
1828 have ASCII to EBCDIC (with 1 byte per character before and after),
1829 but it seems to be a reasonable restriction. */
1830 struct cset_converter cvt
= converter_for_type (pfile
, type
);
1831 if (cvt
.func
!= convert_no_conversion
)
1832 return "execution character set != source character set";
1834 /* For on-demand strings we have already lexed the strings, so there
1835 should be no diagnostics. However, if we have bogus source location
1836 data (or stringified macro arguments), the attempt to lex the
1837 strings could fail with an diagnostic. Temporarily install an
1838 diagnostic-handler to catch the diagnostic, so that it can lead to this call
1839 failing, rather than being emitted as a user-visible diagnostic.
1840 If an diagnostic does occur, we should see it via the return value of
1841 cpp_interpret_string_1. */
1842 bool (*saved_diagnostic_handler
) (cpp_reader
*, enum cpp_diagnostic_level
,
1843 enum cpp_warning_reason
, rich_location
*,
1844 const char *, va_list *)
1845 ATTRIBUTE_FPTR_PRINTF(5,0);
1847 saved_diagnostic_handler
= pfile
->cb
.diagnostic
;
1848 pfile
->cb
.diagnostic
= noop_diagnostic_cb
;
1850 bool result
= cpp_interpret_string_1 (pfile
, from
, count
, NULL
, type
,
1853 /* Restore the saved diagnostic-handler. */
1854 pfile
->cb
.diagnostic
= saved_diagnostic_handler
;
1857 return "cpp_interpret_string_1 failed";
1863 /* Subroutine of do_line and do_linemarker. Convert escape sequences
1864 in a string, but do not perform character set conversion. */
1866 cpp_interpret_string_notranslate (cpp_reader
*pfile
, const cpp_string
*from
,
1867 size_t count
, cpp_string
*to
,
1868 enum cpp_ttype type ATTRIBUTE_UNUSED
)
1870 struct cset_converter save_narrow_cset_desc
= pfile
->narrow_cset_desc
;
1873 pfile
->narrow_cset_desc
.func
= convert_no_conversion
;
1874 pfile
->narrow_cset_desc
.cd
= (iconv_t
) -1;
1875 pfile
->narrow_cset_desc
.width
= CPP_OPTION (pfile
, char_precision
);
1877 retval
= cpp_interpret_string (pfile
, from
, count
, to
, CPP_STRING
);
1879 pfile
->narrow_cset_desc
= save_narrow_cset_desc
;
1884 /* Subroutine of cpp_interpret_charconst which performs the conversion
1885 to a number, for narrow strings. STR is the string structure returned
1886 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1887 cpp_interpret_charconst. TYPE is the token type. */
1889 narrow_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
1890 unsigned int *pchars_seen
, int *unsignedp
,
1891 enum cpp_ttype type
)
1893 size_t width
= CPP_OPTION (pfile
, char_precision
);
1894 size_t max_chars
= CPP_OPTION (pfile
, int_precision
) / width
;
1895 size_t mask
= width_to_mask (width
);
1897 cppchar_t result
, c
;
1900 /* The value of a multi-character character constant, or a
1901 single-character character constant whose representation in the
1902 execution character set is more than one byte long, is
1903 implementation defined. This implementation defines it to be the
1904 number formed by interpreting the byte sequence in memory as a
1905 big-endian binary number. If overflow occurs, the high bytes are
1906 lost, and a warning is issued.
1908 We don't want to process the NUL terminator handed back by
1909 cpp_interpret_string. */
1911 for (i
= 0; i
< str
.len
- 1; i
++)
1913 c
= str
.text
[i
] & mask
;
1914 if (width
< BITS_PER_CPPCHAR_T
)
1915 result
= (result
<< width
) | c
;
1920 if (type
== CPP_UTF8CHAR
)
1925 cpp_error (pfile
, type
== CPP_UTF8CHAR
? CPP_DL_ERROR
: CPP_DL_WARNING
,
1926 "character constant too long for its type");
1928 else if (i
> 1 && CPP_OPTION (pfile
, warn_multichar
))
1929 cpp_warning (pfile
, CPP_W_MULTICHAR
, "multi-character character constant");
1931 /* Multichar constants are of type int and therefore signed. */
1934 else if (type
== CPP_UTF8CHAR
&& !CPP_OPTION (pfile
, cplusplus
))
1937 unsigned_p
= CPP_OPTION (pfile
, unsigned_char
);
1939 /* Truncate the constant to its natural width, and simultaneously
1940 sign- or zero-extend to the full width of cppchar_t.
1941 For single-character constants, the value is WIDTH bits wide.
1942 For multi-character constants, the value is INT_PRECISION bits wide. */
1944 width
= CPP_OPTION (pfile
, int_precision
);
1945 if (width
< BITS_PER_CPPCHAR_T
)
1947 mask
= ((cppchar_t
) 1 << width
) - 1;
1948 if (unsigned_p
|| !(result
& (1 << (width
- 1))))
1954 *unsignedp
= unsigned_p
;
1958 /* Subroutine of cpp_interpret_charconst which performs the conversion
1959 to a number, for wide strings. STR is the string structure returned
1960 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1961 cpp_interpret_charconst. TYPE is the token type. */
1963 wide_str_to_charconst (cpp_reader
*pfile
, cpp_string str
,
1964 unsigned int *pchars_seen
, int *unsignedp
,
1965 enum cpp_ttype type
)
1967 bool bigend
= CPP_OPTION (pfile
, bytes_big_endian
);
1968 size_t width
= converter_for_type (pfile
, type
).width
;
1969 size_t cwidth
= CPP_OPTION (pfile
, char_precision
);
1970 size_t mask
= width_to_mask (width
);
1971 size_t cmask
= width_to_mask (cwidth
);
1972 size_t nbwc
= width
/ cwidth
;
1974 cppchar_t result
= 0, c
;
1976 if (str
.len
<= nbwc
)
1978 /* Error recovery, if no errors have been diagnosed previously,
1979 there should be at least two wide characters. Empty literals
1980 are diagnosed earlier and we can get just the zero terminator
1981 only if there were errors diagnosed during conversion. */
1987 /* This is finicky because the string is in the target's byte order,
1988 which may not be our byte order. Only the last character, ignoring
1989 the NUL terminator, is relevant. */
1990 off
= str
.len
- (nbwc
* 2);
1992 for (i
= 0; i
< nbwc
; i
++)
1994 c
= bigend
? str
.text
[off
+ i
] : str
.text
[off
+ nbwc
- i
- 1];
1995 result
= (result
<< cwidth
) | (c
& cmask
);
1998 /* Wide character constants have type wchar_t, and a single
1999 character exactly fills a wchar_t, so a multi-character wide
2000 character constant is guaranteed to overflow. */
2001 if (str
.len
> nbwc
* 2)
2002 cpp_error (pfile
, (CPP_OPTION (pfile
, cplusplus
)
2003 && (type
== CPP_CHAR16
|| type
== CPP_CHAR32
))
2004 ? CPP_DL_ERROR
: CPP_DL_WARNING
,
2005 "character constant too long for its type");
2007 /* Truncate the constant to its natural width, and simultaneously
2008 sign- or zero-extend to the full width of cppchar_t. */
2009 if (width
< BITS_PER_CPPCHAR_T
)
2011 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
2012 || CPP_OPTION (pfile
, unsigned_wchar
)
2013 || !(result
& (1 << (width
- 1))))
2019 if (type
== CPP_CHAR16
|| type
== CPP_CHAR32
2020 || CPP_OPTION (pfile
, unsigned_wchar
))
2029 /* Interpret a (possibly wide) character constant in TOKEN.
2030 PCHARS_SEEN points to a variable that is filled in with the number
2031 of characters seen, and UNSIGNEDP to a variable that indicates
2032 whether the result has signed type. */
2034 cpp_interpret_charconst (cpp_reader
*pfile
, const cpp_token
*token
,
2035 unsigned int *pchars_seen
, int *unsignedp
)
2037 cpp_string str
= { 0, 0 };
2038 bool wide
= (token
->type
!= CPP_CHAR
&& token
->type
!= CPP_UTF8CHAR
);
2039 int u8
= 2 * int(token
->type
== CPP_UTF8CHAR
);
2042 /* An empty constant will appear as L'', u'', U'', u8'', or '' */
2043 if (token
->val
.str
.len
== (size_t) (2 + wide
+ u8
))
2045 cpp_error (pfile
, CPP_DL_ERROR
, "empty character constant");
2050 else if (!cpp_interpret_string (pfile
, &token
->val
.str
, 1, &str
,
2059 result
= wide_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
2062 result
= narrow_str_to_charconst (pfile
, str
, pchars_seen
, unsignedp
,
2065 if (str
.text
!= token
->val
.str
.text
)
2066 free ((void *)str
.text
);
2071 /* Convert an identifier denoted by ID and LEN, which might contain
2072 UCN escapes or UTF-8 multibyte chars, to the source character set,
2073 either UTF-8 or UTF-EBCDIC. Assumes that the identifier is actually
2074 a valid identifier. */
2076 _cpp_interpret_identifier (cpp_reader
*pfile
, const uchar
*id
, size_t len
)
2078 /* It turns out that a UCN escape always turns into fewer characters
2079 than the escape itself, so we can allocate a temporary in advance. */
2080 uchar
* buf
= (uchar
*) alloca (len
+ 1);
2084 for (idp
= 0; idp
< len
; idp
++)
2085 if (id
[idp
] != '\\')
2089 unsigned length
= id
[idp
+1] == 'u' ? 4 : 8;
2090 cppchar_t value
= 0;
2091 size_t bufleft
= len
- (bufp
- buf
);
2095 while (length
&& idp
< len
&& ISXDIGIT (id
[idp
]))
2097 value
= (value
<< 4) + hex_value (id
[idp
]);
2103 /* Special case for EBCDIC: if the identifier contains
2104 a '$' specified using a UCN, translate it to EBCDIC. */
2111 rval
= one_cppchar_to_utf8 (value
, &bufp
, &bufleft
);
2115 cpp_errno (pfile
, CPP_DL_ERROR
,
2116 "converting UCN to source character set");
2121 return CPP_HASHNODE (ht_lookup (pfile
->hash_table
,
2122 buf
, bufp
- buf
, HT_ALLOC
));
2125 /* Convert an input buffer (containing the complete contents of one
2126 source file) from INPUT_CHARSET to the source character set. INPUT
2127 points to the input buffer, SIZE is its allocated size, and LEN is
2128 the length of the meaningful data within the buffer. The
2129 translated buffer is returned, *ST_SIZE is set to the length of
2130 the meaningful data within the translated buffer, and *BUFFER_START
2131 is set to the start of the returned buffer. *BUFFER_START may
2132 differ from the return value in the case of a BOM or other ignored
2135 INPUT is expected to have been allocated with xmalloc. This
2136 function will either set *BUFFER_START to INPUT, or free it and set
2137 *BUFFER_START to a pointer to another xmalloc-allocated block of
2140 _cpp_convert_input (cpp_reader
*pfile
, const char *input_charset
,
2141 uchar
*input
, size_t size
, size_t len
,
2142 const unsigned char **buffer_start
, off_t
*st_size
)
2144 struct cset_converter input_cset
;
2145 struct _cpp_strbuf to
;
2146 unsigned char *buffer
;
2148 input_cset
= init_iconv_desc (pfile
, SOURCE_CHARSET
, input_charset
);
2149 if (input_cset
.func
== convert_no_conversion
)
2157 to
.asize
= MAX (65536, len
);
2158 to
.text
= XNEWVEC (uchar
, to
.asize
);
2161 if (!APPLY_CONVERSION (input_cset
, input
, len
, &to
))
2162 cpp_error (pfile
, CPP_DL_ERROR
,
2163 "failure to convert %s to %s",
2164 CPP_OPTION (pfile
, input_charset
), SOURCE_CHARSET
);
2169 /* Clean up the mess. */
2170 if (input_cset
.func
== convert_using_iconv
)
2171 iconv_close (input_cset
.cd
);
2173 /* Resize buffer if we allocated substantially too much, or if we
2174 haven't enough space for the \n-terminator or following
2175 15 bytes of padding (used to quiet warnings from valgrind or
2176 Address Sanitizer, when the optimized lexer accesses aligned
2177 16-byte memory chunks, including the bytes after the malloced,
2178 area, and stops lexing on '\n'). */
2179 if (to
.len
+ 4096 < to
.asize
|| to
.len
+ 16 > to
.asize
)
2180 to
.text
= XRESIZEVEC (uchar
, to
.text
, to
.len
+ 16);
2182 memset (to
.text
+ to
.len
, '\0', 16);
2184 /* If the file is using old-school Mac line endings (\r only),
2185 terminate with another \r, not an \n, so that we do not mistake
2186 the \r\n sequence for a single DOS line ending and erroneously
2187 issue the "No newline at end of file" diagnostic. */
2188 if (to
.len
&& to
.text
[to
.len
- 1] == '\r')
2189 to
.text
[to
.len
] = '\r';
2191 to
.text
[to
.len
] = '\n';
2195 #if HOST_CHARSET == HOST_CHARSET_ASCII
2196 /* The HOST_CHARSET test just above ensures that the source charset
2197 is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
2198 glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
2199 BOM -- however, even if it did, we would still need this code due
2200 to the 'convert_no_conversion' case. */
2201 if (to
.len
>= 3 && to
.text
[0] == 0xef && to
.text
[1] == 0xbb
2202 && to
.text
[2] == 0xbf)
2209 *buffer_start
= to
.text
;
2213 /* Decide on the default encoding to assume for input files. */
2215 _cpp_default_encoding (void)
2217 const char *current_encoding
= NULL
;
2219 /* We disable this because the default codeset is 7-bit ASCII on
2220 most platforms, and this causes conversion failures on every
2221 file in GCC that happens to have one of the upper 128 characters
2222 in it -- most likely, as part of the name of a contributor.
2223 We should definitely recognize in-band markers of file encoding,
2225 - the appropriate Unicode byte-order mark (FE FF) to recognize
2226 UTF16 and UCS4 (in both big-endian and little-endian flavors)
2228 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
2229 distinguish ASCII and EBCDIC.
2230 - now we can parse something like "#pragma GCC encoding <xyz>
2231 on the first line, or even Emacs/VIM's mode line tags (there's
2232 a problem here in that VIM uses the last line, and Emacs has
2233 its more elaborate "local variables" convention).
2234 - investigate whether Java has another common convention, which
2235 would be friendly to support.
2236 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
2237 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
2238 setlocale (LC_CTYPE
, "");
2239 current_encoding
= nl_langinfo (CODESET
);
2241 if (current_encoding
== NULL
|| *current_encoding
== '\0')
2242 current_encoding
= SOURCE_CHARSET
;
2244 return current_encoding
;
2247 /* Implementation of class cpp_string_location_reader. */
2249 /* Constructor for cpp_string_location_reader. */
2251 cpp_string_location_reader::
2252 cpp_string_location_reader (location_t src_loc
,
2253 line_maps
*line_table
)
2255 src_loc
= get_range_from_loc (line_table
, src_loc
).m_start
;
2257 /* SRC_LOC might be a macro location. It only makes sense to do
2258 column-by-column calculations on ordinary maps, so get the
2259 corresponding location in an ordinary map. */
2261 = linemap_resolve_location (line_table
, src_loc
,
2262 LRK_SPELLING_LOCATION
, NULL
);
2264 const line_map_ordinary
*map
2265 = linemap_check_ordinary (linemap_lookup (line_table
, m_loc
));
2266 m_offset_per_column
= (1 << map
->m_range_bits
);
2269 /* Get the range of the next source byte. */
2272 cpp_string_location_reader::get_next ()
2274 source_range result
;
2275 result
.m_start
= m_loc
;
2276 result
.m_finish
= m_loc
;
2277 if (m_loc
<= LINE_MAP_MAX_LOCATION_WITH_COLS
)
2278 m_loc
+= m_offset_per_column
;
2282 cpp_display_width_computation::
2283 cpp_display_width_computation (const char *data
, int data_length
, int tabstop
) :
2286 m_bytes_left (data_length
),
2287 m_tabstop (tabstop
),
2290 gcc_assert (m_tabstop
> 0);
2294 /* The main implementation function for class cpp_display_width_computation.
2295 m_next points on entry to the start of the UTF-8 encoding of the next
2296 character, and is updated to point just after the last byte of the encoding.
2297 m_bytes_left contains on entry the remaining size of the buffer into which
2298 m_next points, and this is also updated accordingly. If m_next does not
2299 point to a valid UTF-8-encoded sequence, then it will be treated as a single
2300 byte with display width 1. m_cur_display_col is the current display column,
2301 relative to which tab stops should be expanded. Returns the display width of
2302 the codepoint just processed. */
2305 cpp_display_width_computation::process_next_codepoint ()
2310 if (*m_next
== '\t')
2314 next_width
= m_tabstop
- (m_display_cols
% m_tabstop
);
2316 else if (one_utf8_to_cppchar ((const uchar
**) &m_next
, &m_bytes_left
, &c
)
2319 /* Input is not convertible to UTF-8. This could be fine, e.g. in a
2320 string literal, so don't complain. Just treat it as if it has a width
2328 /* one_utf8_to_cppchar() has updated m_next and m_bytes_left for us. */
2329 next_width
= cpp_wcwidth (c
);
2332 m_display_cols
+= next_width
;
2336 /* Utility to advance the byte stream by the minimum amount needed to consume
2337 N display columns. Returns the number of display columns that were
2338 actually skipped. This could be less than N, if there was not enough data,
2339 or more than N, if the last character to be skipped had a sufficiently large
2342 cpp_display_width_computation::advance_display_cols (int n
)
2344 const int start
= m_display_cols
;
2345 const int target
= start
+ n
;
2346 while (m_display_cols
< target
&& !done ())
2347 process_next_codepoint ();
2348 return m_display_cols
- start
;
2351 /* For the string of length DATA_LENGTH bytes that begins at DATA, compute
2352 how many display columns are occupied by the first COLUMN bytes. COLUMN
2353 may exceed DATA_LENGTH, in which case the phantom bytes at the end are
2354 treated as if they have display width 1. Tabs are expanded to the next tab
2355 stop, relative to the start of DATA. */
2358 cpp_byte_column_to_display_column (const char *data
, int data_length
,
2359 int column
, int tabstop
)
2361 const int offset
= MAX (0, column
- data_length
);
2362 cpp_display_width_computation
dw (data
, column
- offset
, tabstop
);
2364 dw
.process_next_codepoint ();
2365 return dw
.display_cols_processed () + offset
;
2368 /* For the string of length DATA_LENGTH bytes that begins at DATA, compute
2369 the least number of bytes that will result in at least DISPLAY_COL display
2370 columns. The return value may exceed DATA_LENGTH if the entire string does
2371 not occupy enough display columns. */
2374 cpp_display_column_to_byte_column (const char *data
, int data_length
,
2375 int display_col
, int tabstop
)
2377 cpp_display_width_computation
dw (data
, data_length
, tabstop
);
2378 const int avail_display
= dw
.advance_display_cols (display_col
);
2379 return dw
.bytes_processed () + MAX (0, display_col
- avail_display
);
2382 /* Our own version of wcwidth(). We don't use the actual wcwidth() in glibc,
2383 because that will inspect the user's locale, and in particular in an ASCII
2384 locale, it will not return anything useful for extended characters. But GCC
2385 in other respects (see e.g. _cpp_default_encoding()) behaves as if
2386 everything is UTF-8. We also make some tweaks that are useful for the way
2387 GCC needs to use this data, e.g. tabs and other control characters should be
2388 treated as having width 1. The lookup tables are generated from
2389 contrib/unicode/gen_wcwidth.py and were made by simply calling glibc
2390 wcwidth() on all codepoints, then applying the small tweaks. These tables
2391 are not highly optimized, but for the present purpose of outputting
2392 diagnostics, they are sufficient. */
2394 #include "generated_cpp_wcwidth.h"
2395 int cpp_wcwidth (cppchar_t c
)
2397 if (__builtin_expect (c
<= wcwidth_range_ends
[0], true))
2398 return wcwidth_widths
[0];
2400 /* Binary search the tables. */
2402 static const int end
2403 = sizeof wcwidth_range_ends
/ sizeof (*wcwidth_range_ends
);
2404 int len
= end
- begin
;
2408 int middle
= begin
+ half
;
2409 if (c
> wcwidth_range_ends
[middle
])
2418 if (__builtin_expect (begin
!= end
, true))
2419 return wcwidth_widths
[begin
];