2016-09-29 Sandra Loosemore <sandra@codesourcery.com>
[official-gcc.git] / libcpp / charset.c
blobe77270a18eb957e29f4a5e391a4969f1a2fb0ce0
1 /* CPP Library - charsets
2 Copyright (C) 1998-2016 Free Software Foundation, Inc.
4 Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
9 later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
20 #include "config.h"
21 #include "system.h"
22 #include "cpplib.h"
23 #include "internal.h"
25 /* Character set handling for C-family languages.
27 Terminological note: In what follows, "charset" or "character set"
28 will be taken to mean both an abstract set of characters and an
29 encoding for that set.
31 The C99 standard discusses two character sets: source and execution.
32 The source character set is used for internal processing in translation
33 phases 1 through 4; the execution character set is used thereafter.
34 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36 of these terms). Furthermore, the "basic character set" (listed in
37 5.2.1p3) is to be encoded in each with values one byte wide, and is
38 to appear in the initial shift state.
40 It is not explicitly mentioned, but there is also a "wide execution
41 character set" used to encode wide character constants and wide
42 string literals; this is supposed to be the result of applying the
43 standard library function mbstowcs() to an equivalent narrow string
44 (6.4.5p5). However, the behavior of hexadecimal and octal
45 \-escapes is at odds with this; they are supposed to be translated
46 directly to wchar_t values (6.4.4.4p5,6).
48 The source character set is not necessarily the character set used
49 to encode physical source files on disk; translation phase 1 converts
50 from whatever that encoding is to the source character set.
52 The presence of universal character names in C99 (6.4.3 et seq.)
53 forces the source character set to be isomorphic to ISO 10646,
54 that is, Unicode. There is no such constraint on the execution
55 character set; note also that the conversion from source to
56 execution character set does not occur for identifiers (5.1.1.2p1#5).
58 For convenience of implementation, the source character set's
59 encoding of the basic character set should be identical to the
60 execution character set OF THE HOST SYSTEM's encoding of the basic
61 character set, and it should not be a state-dependent encoding.
63 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64 depending on whether the host is based on ASCII or EBCDIC (see
65 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66 Technical Report #16). With limited exceptions, it relies on the
67 system library's iconv() primitive to do charset conversion
68 (specified in SUSv2). */
70 #if !HAVE_ICONV
71 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72 below, which are guarded only by if statements with compile-time
73 constant conditions, do not cause link errors. */
74 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76 #define iconv_close(x) (void)0
77 #define ICONV_CONST
78 #endif
80 #if HOST_CHARSET == HOST_CHARSET_ASCII
81 #define SOURCE_CHARSET "UTF-8"
82 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84 #define SOURCE_CHARSET "UTF-EBCDIC"
85 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
86 #else
87 #error "Unrecognized basic host character set"
88 #endif
90 #ifndef EILSEQ
91 #define EILSEQ EINVAL
92 #endif
94 /* This structure is used for a resizable string buffer throughout. */
95 /* Don't call it strbuf, as that conflicts with unistd.h on systems
96 such as DYNIX/ptx where unistd.h includes stropts.h. */
97 struct _cpp_strbuf
99 uchar *text;
100 size_t asize;
101 size_t len;
104 /* This is enough to hold any string that fits on a single 80-column
105 line, even if iconv quadruples its size (e.g. conversion from
106 ASCII to UTF-32) rounded up to a power of two. */
107 #define OUTBUF_BLOCK_SIZE 256
109 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110 logic. This is because a depressing number of systems lack iconv,
111 or have have iconv libraries that do not do these conversions, so
112 we need a fallback implementation for them. To ensure the fallback
113 doesn't break due to neglect, it is used on all systems.
115 UTF-32 encoding is nice and simple: a four-byte binary number,
116 constrained to the range 00000000-7FFFFFFF to avoid questions of
117 signedness. We do have to cope with big- and little-endian
118 variants.
120 UTF-16 encoding uses two-byte binary numbers, again in big- and
121 little-endian variants, for all values in the 00000000-0000FFFF
122 range. Values in the 00010000-0010FFFF range are encoded as pairs
123 of two-byte numbers, called "surrogate pairs": given a number S in
124 this range, it is mapped to a pair (H, L) as follows:
126 H = (S - 0x10000) / 0x400 + 0xD800
127 L = (S - 0x10000) % 0x400 + 0xDC00
129 Two-byte values in the D800...DFFF range are ill-formed except as a
130 component of a surrogate pair. Even if the encoding within a
131 two-byte value is little-endian, the H member of the surrogate pair
132 comes first.
134 There is no way to encode values in the 00110000-7FFFFFFF range,
135 which is not currently a problem as there are no assigned code
136 points in that range; however, the author expects that it will
137 eventually become necessary to abandon UTF-16 due to this
138 limitation. Note also that, because of these pairs, UTF-16 does
139 not meet the requirements of the C standard for a wide character
140 encoding (see 3.7.3 and 6.4.4.4p11).
142 UTF-8 encoding looks like this:
144 value range encoded as
145 00000000-0000007F 0xxxxxxx
146 00000080-000007FF 110xxxxx 10xxxxxx
147 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
152 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154 never occur. Note also that any value that can be encoded by a
155 given row of the table can also be encoded by all successive rows,
156 but this is not done; only the shortest possible encoding for any
157 given value is valid. For instance, the character 07C0 could be
158 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159 FC 80 80 80 9F 80. Only the first is valid.
161 An implementation note: the transformation from UTF-16 to UTF-8, or
162 vice versa, is easiest done by using UTF-32 as an intermediary. */
164 /* Internal primitives which go from an UTF-8 byte stream to native-endian
165 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166 operation in several places below. */
167 static inline int
168 one_utf8_to_cppchar (const uchar **inbufp, size_t *inbytesleftp,
169 cppchar_t *cp)
171 static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172 static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
174 cppchar_t c;
175 const uchar *inbuf = *inbufp;
176 size_t nbytes, i;
178 if (*inbytesleftp < 1)
179 return EINVAL;
181 c = *inbuf;
182 if (c < 0x80)
184 *cp = c;
185 *inbytesleftp -= 1;
186 *inbufp += 1;
187 return 0;
190 /* The number of leading 1-bits in the first byte indicates how many
191 bytes follow. */
192 for (nbytes = 2; nbytes < 7; nbytes++)
193 if ((c & ~masks[nbytes-1]) == patns[nbytes-1])
194 goto found;
195 return EILSEQ;
196 found:
198 if (*inbytesleftp < nbytes)
199 return EINVAL;
201 c = (c & masks[nbytes-1]);
202 inbuf++;
203 for (i = 1; i < nbytes; i++)
205 cppchar_t n = *inbuf++;
206 if ((n & 0xC0) != 0x80)
207 return EILSEQ;
208 c = ((c << 6) + (n & 0x3F));
211 /* Make sure the shortest possible encoding was used. */
212 if (c <= 0x7F && nbytes > 1) return EILSEQ;
213 if (c <= 0x7FF && nbytes > 2) return EILSEQ;
214 if (c <= 0xFFFF && nbytes > 3) return EILSEQ;
215 if (c <= 0x1FFFFF && nbytes > 4) return EILSEQ;
216 if (c <= 0x3FFFFFF && nbytes > 5) return EILSEQ;
218 /* Make sure the character is valid. */
219 if (c > 0x7FFFFFFF || (c >= 0xD800 && c <= 0xDFFF)) return EILSEQ;
221 *cp = c;
222 *inbufp = inbuf;
223 *inbytesleftp -= nbytes;
224 return 0;
227 static inline int
228 one_cppchar_to_utf8 (cppchar_t c, uchar **outbufp, size_t *outbytesleftp)
230 static const uchar masks[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231 static const uchar limits[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
232 size_t nbytes;
233 uchar buf[6], *p = &buf[6];
234 uchar *outbuf = *outbufp;
236 nbytes = 1;
237 if (c < 0x80)
238 *--p = c;
239 else
243 *--p = ((c & 0x3F) | 0x80);
244 c >>= 6;
245 nbytes++;
247 while (c >= 0x3F || (c & limits[nbytes-1]));
248 *--p = (c | masks[nbytes-1]);
251 if (*outbytesleftp < nbytes)
252 return E2BIG;
254 while (p < &buf[6])
255 *outbuf++ = *p++;
256 *outbytesleftp -= nbytes;
257 *outbufp = outbuf;
258 return 0;
261 /* The following four functions transform one character between the two
262 encodings named in the function name. All have the signature
263 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264 uchar **outbufp, size_t *outbytesleftp)
266 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267 interpreted as a boolean indicating whether big-endian or
268 little-endian encoding is to be used for the member of the pair
269 that is not UTF-8.
271 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
272 do for iconv.
274 The return value is either 0 for success, or an errno value for
275 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276 input sequence), ir EINVAL (incomplete input sequence). */
278 static inline int
279 one_utf8_to_utf32 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
280 uchar **outbufp, size_t *outbytesleftp)
282 uchar *outbuf;
283 cppchar_t s = 0;
284 int rval;
286 /* Check for space first, since we know exactly how much we need. */
287 if (*outbytesleftp < 4)
288 return E2BIG;
290 rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s);
291 if (rval)
292 return rval;
294 outbuf = *outbufp;
295 outbuf[bigend ? 3 : 0] = (s & 0x000000FF);
296 outbuf[bigend ? 2 : 1] = (s & 0x0000FF00) >> 8;
297 outbuf[bigend ? 1 : 2] = (s & 0x00FF0000) >> 16;
298 outbuf[bigend ? 0 : 3] = (s & 0xFF000000) >> 24;
300 *outbufp += 4;
301 *outbytesleftp -= 4;
302 return 0;
305 static inline int
306 one_utf32_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
307 uchar **outbufp, size_t *outbytesleftp)
309 cppchar_t s;
310 int rval;
311 const uchar *inbuf;
313 if (*inbytesleftp < 4)
314 return EINVAL;
316 inbuf = *inbufp;
318 s = inbuf[bigend ? 0 : 3] << 24;
319 s += inbuf[bigend ? 1 : 2] << 16;
320 s += inbuf[bigend ? 2 : 1] << 8;
321 s += inbuf[bigend ? 3 : 0];
323 if (s >= 0x7FFFFFFF || (s >= 0xD800 && s <= 0xDFFF))
324 return EILSEQ;
326 rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp);
327 if (rval)
328 return rval;
330 *inbufp += 4;
331 *inbytesleftp -= 4;
332 return 0;
335 static inline int
336 one_utf8_to_utf16 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
337 uchar **outbufp, size_t *outbytesleftp)
339 int rval;
340 cppchar_t s = 0;
341 const uchar *save_inbuf = *inbufp;
342 size_t save_inbytesleft = *inbytesleftp;
343 uchar *outbuf = *outbufp;
345 rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s);
346 if (rval)
347 return rval;
349 if (s > 0x0010FFFF)
351 *inbufp = save_inbuf;
352 *inbytesleftp = save_inbytesleft;
353 return EILSEQ;
356 if (s <= 0xFFFF)
358 if (*outbytesleftp < 2)
360 *inbufp = save_inbuf;
361 *inbytesleftp = save_inbytesleft;
362 return E2BIG;
364 outbuf[bigend ? 1 : 0] = (s & 0x00FF);
365 outbuf[bigend ? 0 : 1] = (s & 0xFF00) >> 8;
367 *outbufp += 2;
368 *outbytesleftp -= 2;
369 return 0;
371 else
373 cppchar_t hi, lo;
375 if (*outbytesleftp < 4)
377 *inbufp = save_inbuf;
378 *inbytesleftp = save_inbytesleft;
379 return E2BIG;
382 hi = (s - 0x10000) / 0x400 + 0xD800;
383 lo = (s - 0x10000) % 0x400 + 0xDC00;
385 /* Even if we are little-endian, put the high surrogate first.
386 ??? Matches practice? */
387 outbuf[bigend ? 1 : 0] = (hi & 0x00FF);
388 outbuf[bigend ? 0 : 1] = (hi & 0xFF00) >> 8;
389 outbuf[bigend ? 3 : 2] = (lo & 0x00FF);
390 outbuf[bigend ? 2 : 3] = (lo & 0xFF00) >> 8;
392 *outbufp += 4;
393 *outbytesleftp -= 4;
394 return 0;
398 static inline int
399 one_utf16_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
400 uchar **outbufp, size_t *outbytesleftp)
402 cppchar_t s;
403 const uchar *inbuf = *inbufp;
404 int rval;
406 if (*inbytesleftp < 2)
407 return EINVAL;
408 s = inbuf[bigend ? 0 : 1] << 8;
409 s += inbuf[bigend ? 1 : 0];
411 /* Low surrogate without immediately preceding high surrogate is invalid. */
412 if (s >= 0xDC00 && s <= 0xDFFF)
413 return EILSEQ;
414 /* High surrogate must have a following low surrogate. */
415 else if (s >= 0xD800 && s <= 0xDBFF)
417 cppchar_t hi = s, lo;
418 if (*inbytesleftp < 4)
419 return EINVAL;
421 lo = inbuf[bigend ? 2 : 3] << 8;
422 lo += inbuf[bigend ? 3 : 2];
424 if (lo < 0xDC00 || lo > 0xDFFF)
425 return EILSEQ;
427 s = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000;
430 rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp);
431 if (rval)
432 return rval;
434 /* Success - update the input pointers (one_cppchar_to_utf8 has done
435 the output pointers for us). */
436 if (s <= 0xFFFF)
438 *inbufp += 2;
439 *inbytesleftp -= 2;
441 else
443 *inbufp += 4;
444 *inbytesleftp -= 4;
446 return 0;
449 /* Helper routine for the next few functions. The 'const' on
450 one_conversion means that we promise not to modify what function is
451 pointed to, which lets the inliner see through it. */
453 static inline bool
454 conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
455 uchar **, size_t *),
456 iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to)
458 const uchar *inbuf;
459 uchar *outbuf;
460 size_t inbytesleft, outbytesleft;
461 int rval;
463 inbuf = from;
464 inbytesleft = flen;
465 outbuf = to->text + to->len;
466 outbytesleft = to->asize - to->len;
468 for (;;)
471 rval = one_conversion (cd, &inbuf, &inbytesleft,
472 &outbuf, &outbytesleft);
473 while (inbytesleft && !rval);
475 if (__builtin_expect (inbytesleft == 0, 1))
477 to->len = to->asize - outbytesleft;
478 return true;
480 if (rval != E2BIG)
482 errno = rval;
483 return false;
486 outbytesleft += OUTBUF_BLOCK_SIZE;
487 to->asize += OUTBUF_BLOCK_SIZE;
488 to->text = XRESIZEVEC (uchar, to->text, to->asize);
489 outbuf = to->text + to->asize - outbytesleft;
494 /* These functions convert entire strings between character sets.
495 They all have the signature
497 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
499 The input string FROM is converted as specified by the function
500 name plus the iconv descriptor CD (which may be fake), and the
501 result appended to TO. On any error, false is returned, otherwise true. */
503 /* These four use the custom conversion code above. */
504 static bool
505 convert_utf8_utf16 (iconv_t cd, const uchar *from, size_t flen,
506 struct _cpp_strbuf *to)
508 return conversion_loop (one_utf8_to_utf16, cd, from, flen, to);
511 static bool
512 convert_utf8_utf32 (iconv_t cd, const uchar *from, size_t flen,
513 struct _cpp_strbuf *to)
515 return conversion_loop (one_utf8_to_utf32, cd, from, flen, to);
518 static bool
519 convert_utf16_utf8 (iconv_t cd, const uchar *from, size_t flen,
520 struct _cpp_strbuf *to)
522 return conversion_loop (one_utf16_to_utf8, cd, from, flen, to);
525 static bool
526 convert_utf32_utf8 (iconv_t cd, const uchar *from, size_t flen,
527 struct _cpp_strbuf *to)
529 return conversion_loop (one_utf32_to_utf8, cd, from, flen, to);
532 /* Identity conversion, used when we have no alternative. */
533 static bool
534 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
535 const uchar *from, size_t flen, struct _cpp_strbuf *to)
537 if (to->len + flen > to->asize)
539 to->asize = to->len + flen;
540 to->asize += to->asize / 4;
541 to->text = XRESIZEVEC (uchar, to->text, to->asize);
543 memcpy (to->text + to->len, from, flen);
544 to->len += flen;
545 return true;
548 /* And this one uses the system iconv primitive. It's a little
549 different, since iconv's interface is a little different. */
550 #if HAVE_ICONV
552 #define CONVERT_ICONV_GROW_BUFFER \
553 do { \
554 outbytesleft += OUTBUF_BLOCK_SIZE; \
555 to->asize += OUTBUF_BLOCK_SIZE; \
556 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
557 outbuf = (char *)to->text + to->asize - outbytesleft; \
558 } while (0)
560 static bool
561 convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
562 struct _cpp_strbuf *to)
564 ICONV_CONST char *inbuf;
565 char *outbuf;
566 size_t inbytesleft, outbytesleft;
568 /* Reset conversion descriptor and check that it is valid. */
569 if (iconv (cd, 0, 0, 0, 0) == (size_t)-1)
570 return false;
572 inbuf = (ICONV_CONST char *)from;
573 inbytesleft = flen;
574 outbuf = (char *)to->text + to->len;
575 outbytesleft = to->asize - to->len;
577 for (;;)
579 iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
580 if (__builtin_expect (inbytesleft == 0, 1))
582 /* Close out any shift states, returning to the initial state. */
583 if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
585 if (errno != E2BIG)
586 return false;
588 CONVERT_ICONV_GROW_BUFFER;
589 if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
590 return false;
593 to->len = to->asize - outbytesleft;
594 return true;
596 if (errno != E2BIG)
597 return false;
599 CONVERT_ICONV_GROW_BUFFER;
602 #else
603 #define convert_using_iconv 0 /* prevent undefined symbol error below */
604 #endif
606 /* Arrange for the above custom conversion logic to be used automatically
607 when conversion between a suitable pair of character sets is requested. */
609 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
610 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
612 struct cpp_conversion
614 const char *pair;
615 convert_f func;
616 iconv_t fake_cd;
618 static const struct cpp_conversion conversion_tab[] = {
619 { "UTF-8/UTF-32LE", convert_utf8_utf32, (iconv_t)0 },
620 { "UTF-8/UTF-32BE", convert_utf8_utf32, (iconv_t)1 },
621 { "UTF-8/UTF-16LE", convert_utf8_utf16, (iconv_t)0 },
622 { "UTF-8/UTF-16BE", convert_utf8_utf16, (iconv_t)1 },
623 { "UTF-32LE/UTF-8", convert_utf32_utf8, (iconv_t)0 },
624 { "UTF-32BE/UTF-8", convert_utf32_utf8, (iconv_t)1 },
625 { "UTF-16LE/UTF-8", convert_utf16_utf8, (iconv_t)0 },
626 { "UTF-16BE/UTF-8", convert_utf16_utf8, (iconv_t)1 },
629 /* Subroutine of cpp_init_iconv: initialize and return a
630 cset_converter structure for conversion from FROM to TO. If
631 iconv_open() fails, issue an error and return an identity
632 converter. Silently return an identity converter if FROM and TO
633 are identical. */
634 static struct cset_converter
635 init_iconv_desc (cpp_reader *pfile, const char *to, const char *from)
637 struct cset_converter ret;
638 char *pair;
639 size_t i;
641 if (!strcasecmp (to, from))
643 ret.func = convert_no_conversion;
644 ret.cd = (iconv_t) -1;
645 ret.width = -1;
646 return ret;
649 pair = (char *) alloca(strlen(to) + strlen(from) + 2);
651 strcpy(pair, from);
652 strcat(pair, "/");
653 strcat(pair, to);
654 for (i = 0; i < ARRAY_SIZE (conversion_tab); i++)
655 if (!strcasecmp (pair, conversion_tab[i].pair))
657 ret.func = conversion_tab[i].func;
658 ret.cd = conversion_tab[i].fake_cd;
659 ret.width = -1;
660 return ret;
663 /* No custom converter - try iconv. */
664 if (HAVE_ICONV)
666 ret.func = convert_using_iconv;
667 ret.cd = iconv_open (to, from);
668 ret.width = -1;
670 if (ret.cd == (iconv_t) -1)
672 if (errno == EINVAL)
673 cpp_error (pfile, CPP_DL_ERROR, /* FIXME should be DL_SORRY */
674 "conversion from %s to %s not supported by iconv",
675 from, to);
676 else
677 cpp_errno (pfile, CPP_DL_ERROR, "iconv_open");
679 ret.func = convert_no_conversion;
682 else
684 cpp_error (pfile, CPP_DL_ERROR, /* FIXME: should be DL_SORRY */
685 "no iconv implementation, cannot convert from %s to %s",
686 from, to);
687 ret.func = convert_no_conversion;
688 ret.cd = (iconv_t) -1;
689 ret.width = -1;
691 return ret;
694 /* If charset conversion is requested, initialize iconv(3) descriptors
695 for conversion from the source character set to the execution
696 character sets. If iconv is not present in the C library, and
697 conversion is requested, issue an error. */
699 void
700 cpp_init_iconv (cpp_reader *pfile)
702 const char *ncset = CPP_OPTION (pfile, narrow_charset);
703 const char *wcset = CPP_OPTION (pfile, wide_charset);
704 const char *default_wcset;
706 bool be = CPP_OPTION (pfile, bytes_big_endian);
708 if (CPP_OPTION (pfile, wchar_precision) >= 32)
709 default_wcset = be ? "UTF-32BE" : "UTF-32LE";
710 else if (CPP_OPTION (pfile, wchar_precision) >= 16)
711 default_wcset = be ? "UTF-16BE" : "UTF-16LE";
712 else
713 /* This effectively means that wide strings are not supported,
714 so don't do any conversion at all. */
715 default_wcset = SOURCE_CHARSET;
717 if (!ncset)
718 ncset = SOURCE_CHARSET;
719 if (!wcset)
720 wcset = default_wcset;
722 pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET);
723 pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
724 pfile->utf8_cset_desc = init_iconv_desc (pfile, "UTF-8", SOURCE_CHARSET);
725 pfile->utf8_cset_desc.width = CPP_OPTION (pfile, char_precision);
726 pfile->char16_cset_desc = init_iconv_desc (pfile,
727 be ? "UTF-16BE" : "UTF-16LE",
728 SOURCE_CHARSET);
729 pfile->char16_cset_desc.width = 16;
730 pfile->char32_cset_desc = init_iconv_desc (pfile,
731 be ? "UTF-32BE" : "UTF-32LE",
732 SOURCE_CHARSET);
733 pfile->char32_cset_desc.width = 32;
734 pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET);
735 pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision);
738 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
739 void
740 _cpp_destroy_iconv (cpp_reader *pfile)
742 if (HAVE_ICONV)
744 if (pfile->narrow_cset_desc.func == convert_using_iconv)
745 iconv_close (pfile->narrow_cset_desc.cd);
746 if (pfile->utf8_cset_desc.func == convert_using_iconv)
747 iconv_close (pfile->utf8_cset_desc.cd);
748 if (pfile->char16_cset_desc.func == convert_using_iconv)
749 iconv_close (pfile->char16_cset_desc.cd);
750 if (pfile->char32_cset_desc.func == convert_using_iconv)
751 iconv_close (pfile->char32_cset_desc.cd);
752 if (pfile->wide_cset_desc.func == convert_using_iconv)
753 iconv_close (pfile->wide_cset_desc.cd);
757 /* Utility routine for use by a full compiler. C is a character taken
758 from the *basic* source character set, encoded in the host's
759 execution encoding. Convert it to (the target's) execution
760 encoding, and return that value.
762 Issues an internal error if C's representation in the narrow
763 execution character set fails to be a single-byte value (C99
764 5.2.1p3: "The representation of each member of the source and
765 execution character sets shall fit in a byte.") May also issue an
766 internal error if C fails to be a member of the basic source
767 character set (testing this exactly is too hard, especially when
768 the host character set is EBCDIC). */
769 cppchar_t
770 cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c)
772 uchar sbuf[1];
773 struct _cpp_strbuf tbuf;
775 /* This test is merely an approximation, but it suffices to catch
776 the most important thing, which is that we don't get handed a
777 character outside the unibyte range of the host character set. */
778 if (c > LAST_POSSIBLY_BASIC_SOURCE_CHAR)
780 cpp_error (pfile, CPP_DL_ICE,
781 "character 0x%lx is not in the basic source character set\n",
782 (unsigned long)c);
783 return 0;
786 /* Being a character in the unibyte range of the host character set,
787 we can safely splat it into a one-byte buffer and trust that that
788 is a well-formed string. */
789 sbuf[0] = c;
791 /* This should never need to reallocate, but just in case... */
792 tbuf.asize = 1;
793 tbuf.text = XNEWVEC (uchar, tbuf.asize);
794 tbuf.len = 0;
796 if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf))
798 cpp_errno (pfile, CPP_DL_ICE, "converting to execution character set");
799 return 0;
801 if (tbuf.len != 1)
803 cpp_error (pfile, CPP_DL_ICE,
804 "character 0x%lx is not unibyte in execution character set",
805 (unsigned long)c);
806 return 0;
808 c = tbuf.text[0];
809 free(tbuf.text);
810 return c;
815 /* cpp_substring_ranges's constructor. */
817 cpp_substring_ranges::cpp_substring_ranges () :
818 m_ranges (NULL),
819 m_num_ranges (0),
820 m_alloc_ranges (8)
822 m_ranges = XNEWVEC (source_range, m_alloc_ranges);
825 /* cpp_substring_ranges's destructor. */
827 cpp_substring_ranges::~cpp_substring_ranges ()
829 free (m_ranges);
832 /* Add RANGE to the vector of source_range information. */
834 void
835 cpp_substring_ranges::add_range (source_range range)
837 if (m_num_ranges >= m_alloc_ranges)
839 m_alloc_ranges *= 2;
840 m_ranges
841 = (source_range *)xrealloc (m_ranges,
842 sizeof (source_range) * m_alloc_ranges);
844 m_ranges[m_num_ranges++] = range;
847 /* Read NUM ranges from LOC_READER, adding them to the vector of source_range
848 information. */
850 void
851 cpp_substring_ranges::add_n_ranges (int num,
852 cpp_string_location_reader &loc_reader)
854 for (int i = 0; i < num; i++)
855 add_range (loc_reader.get_next ());
860 /* Utility routine that computes a mask of the form 0000...111... with
861 WIDTH 1-bits. */
862 static inline size_t
863 width_to_mask (size_t width)
865 width = MIN (width, BITS_PER_CPPCHAR_T);
866 if (width >= CHAR_BIT * sizeof (size_t))
867 return ~(size_t) 0;
868 else
869 return ((size_t) 1 << width) - 1;
872 /* A large table of unicode character information. */
873 enum {
874 /* Valid in a C99 identifier? */
875 C99 = 1,
876 /* Valid in a C99 identifier, but not as the first character? */
877 N99 = 2,
878 /* Valid in a C++ identifier? */
879 CXX = 4,
880 /* Valid in a C11/C++11 identifier? */
881 C11 = 8,
882 /* Valid in a C11/C++11 identifier, but not as the first character? */
883 N11 = 16,
884 /* NFC representation is not valid in an identifier? */
885 CID = 32,
886 /* Might be valid NFC form? */
887 NFC = 64,
888 /* Might be valid NFKC form? */
889 NKC = 128,
890 /* Certain preceding characters might make it not valid NFC/NKFC form? */
891 CTX = 256
894 struct ucnrange {
895 /* Bitmap of flags above. */
896 unsigned short flags;
897 /* Combining class of the character. */
898 unsigned char combine;
899 /* Last character in the range described by this entry. */
900 unsigned int end;
902 #include "ucnid.h"
904 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
905 the start of an identifier, and 0 if C is not valid in an
906 identifier. We assume C has already gone through the checks of
907 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
908 algorithm is a simple binary search on the table defined in
909 ucnid.h. */
911 static int
912 ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c,
913 struct normalize_state *nst)
915 int mn, mx, md;
916 unsigned short valid_flags, invalid_start_flags;
918 if (c > 0x10FFFF)
919 return 0;
921 mn = 0;
922 mx = ARRAY_SIZE (ucnranges) - 1;
923 while (mx != mn)
925 md = (mn + mx) / 2;
926 if (c <= ucnranges[md].end)
927 mx = md;
928 else
929 mn = md + 1;
932 /* When -pedantic, we require the character to have been listed by
933 the standard for the current language. Otherwise, we accept the
934 union of the acceptable sets for all supported language versions. */
935 valid_flags = C99 | CXX | C11;
936 if (CPP_PEDANTIC (pfile))
938 if (CPP_OPTION (pfile, c11_identifiers))
939 valid_flags = C11;
940 else if (CPP_OPTION (pfile, c99))
941 valid_flags = C99;
942 else if (CPP_OPTION (pfile, cplusplus))
943 valid_flags = CXX;
945 if (! (ucnranges[mn].flags & valid_flags))
946 return 0;
947 if (CPP_OPTION (pfile, c11_identifiers))
948 invalid_start_flags = N11;
949 else if (CPP_OPTION (pfile, c99))
950 invalid_start_flags = N99;
951 else
952 invalid_start_flags = 0;
954 /* Update NST. */
955 if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class)
956 nst->level = normalized_none;
957 else if (ucnranges[mn].flags & CTX)
959 bool safe;
960 cppchar_t p = nst->previous;
962 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
963 and are combined algorithmically from a sequence of the form
964 1100-1112 1161-1175 11A8-11C2
965 (if the third is not present, it is treated as 11A7, which is not
966 really a valid character).
967 Unfortunately, C99 allows (only) the NFC form, but C++ allows
968 only the combining characters. */
969 if (c >= 0x1161 && c <= 0x1175)
970 safe = p < 0x1100 || p > 0x1112;
971 else if (c >= 0x11A8 && c <= 0x11C2)
972 safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0);
973 else
974 safe = check_nfc (pfile, c, p);
975 if (!safe)
977 if ((c >= 0x1161 && c <= 0x1175) || (c >= 0x11A8 && c <= 0x11C2))
978 nst->level = MAX (nst->level, normalized_identifier_C);
979 else
980 nst->level = normalized_none;
983 else if (ucnranges[mn].flags & NKC)
985 else if (ucnranges[mn].flags & NFC)
986 nst->level = MAX (nst->level, normalized_C);
987 else if (ucnranges[mn].flags & CID)
988 nst->level = MAX (nst->level, normalized_identifier_C);
989 else
990 nst->level = normalized_none;
991 if (ucnranges[mn].combine == 0)
992 nst->previous = c;
993 nst->prev_class = ucnranges[mn].combine;
995 /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
996 UCN combining characters may not begin identifiers. */
997 if (ucnranges[mn].flags & invalid_start_flags)
998 return 2;
1000 return 1;
1003 /* [lex.charset]: The character designated by the universal character
1004 name \UNNNNNNNN is that character whose character short name in
1005 ISO/IEC 10646 is NNNNNNNN; the character designated by the
1006 universal character name \uNNNN is that character whose character
1007 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
1008 for a universal character name corresponds to a surrogate code point
1009 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
1010 Additionally, if the hexadecimal value for a universal-character-name
1011 outside a character or string literal corresponds to a control character
1012 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
1013 character in the basic source character set, the program is ill-formed.
1015 C99 6.4.3: A universal character name shall not specify a character
1016 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
1017 or 0060 (`), nor one in the range D800 through DFFF inclusive.
1019 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
1020 buffer end is delimited by a non-hex digit. Returns false if the
1021 UCN has not been consumed, true otherwise.
1023 The value of the UCN, whether valid or invalid, is returned in *CP.
1024 Diagnostics are emitted for invalid values. PSTR is updated to point
1025 one beyond the UCN, or to the syntactically invalid character.
1027 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
1028 an identifier, or 2 otherwise.
1030 If LOC_READER is non-NULL, then position information is
1031 read from *LOC_READER and CHAR_RANGE->m_finish is updated accordingly. */
1033 bool
1034 _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
1035 const uchar *limit, int identifier_pos,
1036 struct normalize_state *nst, cppchar_t *cp,
1037 source_range *char_range,
1038 cpp_string_location_reader *loc_reader)
1040 cppchar_t result, c;
1041 unsigned int length;
1042 const uchar *str = *pstr;
1043 const uchar *base = str - 2;
1045 if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
1046 cpp_error (pfile, CPP_DL_WARNING,
1047 "universal character names are only valid in C++ and C99");
1048 else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
1049 && !CPP_OPTION (pfile, cplusplus))
1050 cpp_error (pfile, CPP_DL_WARNING,
1051 "C99's universal character names are incompatible with C90");
1052 else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
1053 cpp_warning (pfile, CPP_W_TRADITIONAL,
1054 "the meaning of '\\%c' is different in traditional C",
1055 (int) str[-1]);
1057 if (str[-1] == 'u')
1058 length = 4;
1059 else if (str[-1] == 'U')
1060 length = 8;
1061 else
1063 cpp_error (pfile, CPP_DL_ICE, "In _cpp_valid_ucn but not a UCN");
1064 length = 4;
1067 result = 0;
1070 c = *str;
1071 if (!ISXDIGIT (c))
1072 break;
1073 str++;
1074 if (loc_reader)
1076 gcc_assert (char_range);
1077 char_range->m_finish = loc_reader->get_next ().m_finish;
1079 result = (result << 4) + hex_value (c);
1081 while (--length && str < limit);
1083 /* Partial UCNs are not valid in strings, but decompose into
1084 multiple tokens in identifiers, so we can't give a helpful
1085 error message in that case. */
1086 if (length && identifier_pos)
1088 *cp = 0;
1089 return false;
1092 *pstr = str;
1093 if (length)
1095 cpp_error (pfile, CPP_DL_ERROR,
1096 "incomplete universal character name %.*s",
1097 (int) (str - base), base);
1098 result = 1;
1100 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1101 hex escapes so that this also works with EBCDIC hosts.
1102 C++0x permits everything below 0xa0 within literals;
1103 ucn_valid_in_identifier will complain about identifiers. */
1104 else if ((result < 0xa0
1105 && !CPP_OPTION (pfile, cplusplus)
1106 && (result != 0x24 && result != 0x40 && result != 0x60))
1107 || (result & 0x80000000)
1108 || (result >= 0xD800 && result <= 0xDFFF))
1110 cpp_error (pfile, CPP_DL_ERROR,
1111 "%.*s is not a valid universal character",
1112 (int) (str - base), base);
1113 result = 1;
1115 else if (identifier_pos && result == 0x24
1116 && CPP_OPTION (pfile, dollars_in_ident))
1118 if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1120 CPP_OPTION (pfile, warn_dollars) = 0;
1121 cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
1123 NORMALIZE_STATE_UPDATE_IDNUM (nst, result);
1125 else if (identifier_pos)
1127 int validity = ucn_valid_in_identifier (pfile, result, nst);
1129 if (validity == 0)
1130 cpp_error (pfile, CPP_DL_ERROR,
1131 "universal character %.*s is not valid in an identifier",
1132 (int) (str - base), base);
1133 else if (validity == 2 && identifier_pos == 1)
1134 cpp_error (pfile, CPP_DL_ERROR,
1135 "universal character %.*s is not valid at the start of an identifier",
1136 (int) (str - base), base);
1139 *cp = result;
1140 return true;
1143 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1144 it to the execution character set and write the result into TBUF,
1145 if TBUF is non-NULL.
1146 An advanced pointer is returned. Issues all relevant diagnostics.
1147 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1148 contains the location of the character so far: location information
1149 is read from *LOC_READER, and *RANGES is updated accordingly. */
1150 static const uchar *
1151 convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
1152 struct _cpp_strbuf *tbuf, struct cset_converter cvt,
1153 source_range char_range,
1154 cpp_string_location_reader *loc_reader,
1155 cpp_substring_ranges *ranges)
1157 cppchar_t ucn;
1158 uchar buf[6];
1159 uchar *bufp = buf;
1160 size_t bytesleft = 6;
1161 int rval;
1162 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
1164 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1165 gcc_assert ((loc_reader != NULL) == (ranges != NULL));
1167 from++; /* Skip u/U. */
1169 if (loc_reader)
1170 /* The u/U is part of the spelling of this character. */
1171 char_range.m_finish = loc_reader->get_next ().m_finish;
1173 _cpp_valid_ucn (pfile, &from, limit, 0, &nst,
1174 &ucn, &char_range, loc_reader);
1176 rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft);
1177 if (rval)
1179 errno = rval;
1180 cpp_errno (pfile, CPP_DL_ERROR,
1181 "converting UCN to source character set");
1183 else
1185 if (tbuf)
1186 if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
1187 cpp_errno (pfile, CPP_DL_ERROR,
1188 "converting UCN to execution character set");
1190 if (loc_reader)
1192 int num_encoded_bytes = 6 - bytesleft;
1193 for (int i = 0; i < num_encoded_bytes; i++)
1194 ranges->add_range (char_range);
1198 return from;
1201 /* Subroutine of convert_hex and convert_oct. N is the representation
1202 in the execution character set of a numeric escape; write it into the
1203 string buffer TBUF and update the end-of-string pointer therein. WIDE
1204 is true if it's a wide string that's being assembled in TBUF. This
1205 function issues no diagnostics and never fails. */
1206 static void
1207 emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
1208 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1210 size_t width = cvt.width;
1212 if (width != CPP_OPTION (pfile, char_precision))
1214 /* We have to render this into the target byte order, which may not
1215 be our byte order. */
1216 bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1217 size_t cwidth = CPP_OPTION (pfile, char_precision);
1218 size_t cmask = width_to_mask (cwidth);
1219 size_t nbwc = width / cwidth;
1220 size_t i;
1221 size_t off = tbuf->len;
1222 cppchar_t c;
1224 if (tbuf->len + nbwc > tbuf->asize)
1226 tbuf->asize += OUTBUF_BLOCK_SIZE;
1227 tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1230 for (i = 0; i < nbwc; i++)
1232 c = n & cmask;
1233 n >>= cwidth;
1234 tbuf->text[off + (bigend ? nbwc - i - 1 : i)] = c;
1236 tbuf->len += nbwc;
1238 else
1240 /* Note: this code does not handle the case where the target
1241 and host have a different number of bits in a byte. */
1242 if (tbuf->len + 1 > tbuf->asize)
1244 tbuf->asize += OUTBUF_BLOCK_SIZE;
1245 tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1247 tbuf->text[tbuf->len++] = n;
1251 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
1252 character set and write it into the string buffer TBUF (if non-NULL).
1253 Returns an advanced pointer, and issues diagnostics as necessary.
1254 No character set translation occurs; this routine always produces the
1255 execution-set character with numeric value equal to the given hex
1256 number. You can, e.g. generate surrogate pairs this way.
1257 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1258 contains the location of the character so far: location information
1259 is read from *LOC_READER, and *RANGES is updated accordingly. */
1260 static const uchar *
1261 convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
1262 struct _cpp_strbuf *tbuf, struct cset_converter cvt,
1263 source_range char_range,
1264 cpp_string_location_reader *loc_reader,
1265 cpp_substring_ranges *ranges)
1267 cppchar_t c, n = 0, overflow = 0;
1268 int digits_found = 0;
1269 size_t width = cvt.width;
1270 size_t mask = width_to_mask (width);
1272 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1273 gcc_assert ((loc_reader != NULL) == (ranges != NULL));
1275 if (CPP_WTRADITIONAL (pfile))
1276 cpp_warning (pfile, CPP_W_TRADITIONAL,
1277 "the meaning of '\\x' is different in traditional C");
1279 /* Skip 'x'. */
1280 from++;
1282 /* The 'x' is part of the spelling of this character. */
1283 if (loc_reader)
1284 char_range.m_finish = loc_reader->get_next ().m_finish;
1286 while (from < limit)
1288 c = *from;
1289 if (! hex_p (c))
1290 break;
1291 from++;
1292 if (loc_reader)
1293 char_range.m_finish = loc_reader->get_next ().m_finish;
1294 overflow |= n ^ (n << 4 >> 4);
1295 n = (n << 4) + hex_value (c);
1296 digits_found = 1;
1299 if (!digits_found)
1301 cpp_error (pfile, CPP_DL_ERROR,
1302 "\\x used with no following hex digits");
1303 return from;
1306 if (overflow | (n != (n & mask)))
1308 cpp_error (pfile, CPP_DL_PEDWARN,
1309 "hex escape sequence out of range");
1310 n &= mask;
1313 if (tbuf)
1314 emit_numeric_escape (pfile, n, tbuf, cvt);
1315 if (ranges)
1316 ranges->add_range (char_range);
1318 return from;
1321 /* Convert an octal escape, pointed to by FROM, to the execution
1322 character set and write it into the string buffer TBUF. Returns an
1323 advanced pointer, and issues diagnostics as necessary.
1324 No character set translation occurs; this routine always produces the
1325 execution-set character with numeric value equal to the given octal
1326 number.
1327 If LOC_READER is non-NULL, then RANGES must be non-NULL and CHAR_RANGE
1328 contains the location of the character so far: location information
1329 is read from *LOC_READER, and *RANGES is updated accordingly. */
1330 static const uchar *
1331 convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
1332 struct _cpp_strbuf *tbuf, struct cset_converter cvt,
1333 source_range char_range,
1334 cpp_string_location_reader *loc_reader,
1335 cpp_substring_ranges *ranges)
1337 size_t count = 0;
1338 cppchar_t c, n = 0;
1339 size_t width = cvt.width;
1340 size_t mask = width_to_mask (width);
1341 bool overflow = false;
1343 /* loc_reader and ranges must either be both NULL, or both be non-NULL. */
1344 gcc_assert ((loc_reader != NULL) == (ranges != NULL));
1346 while (from < limit && count++ < 3)
1348 c = *from;
1349 if (c < '0' || c > '7')
1350 break;
1351 from++;
1352 if (loc_reader)
1353 char_range.m_finish = loc_reader->get_next ().m_finish;
1354 overflow |= n ^ (n << 3 >> 3);
1355 n = (n << 3) + c - '0';
1358 if (n != (n & mask))
1360 cpp_error (pfile, CPP_DL_PEDWARN,
1361 "octal escape sequence out of range");
1362 n &= mask;
1365 if (tbuf)
1366 emit_numeric_escape (pfile, n, tbuf, cvt);
1367 if (ranges)
1368 ranges->add_range (char_range);
1370 return from;
1373 /* Convert an escape sequence (pointed to by FROM) to its value on
1374 the target, and to the execution character set. Do not scan past
1375 LIMIT. Write the converted value into TBUF, if TBUF is non-NULL.
1376 Returns an advanced pointer. Handles all relevant diagnostics.
1377 If LOC_READER is non-NULL, then RANGES must be non-NULL: location
1378 information is read from *LOC_READER, and *RANGES is updated
1379 accordingly. */
1380 static const uchar *
1381 convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
1382 struct _cpp_strbuf *tbuf, struct cset_converter cvt,
1383 cpp_string_location_reader *loc_reader,
1384 cpp_substring_ranges *ranges)
1386 /* Values of \a \b \e \f \n \r \t \v respectively. */
1387 #if HOST_CHARSET == HOST_CHARSET_ASCII
1388 static const uchar charconsts[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
1389 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
1390 static const uchar charconsts[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
1391 #else
1392 #error "unknown host character set"
1393 #endif
1395 uchar c;
1397 /* Record the location of the backslash. */
1398 source_range char_range;
1399 if (loc_reader)
1400 char_range = loc_reader->get_next ();
1402 c = *from;
1403 switch (c)
1405 /* UCNs, hex escapes, and octal escapes are processed separately. */
1406 case 'u': case 'U':
1407 return convert_ucn (pfile, from, limit, tbuf, cvt,
1408 char_range, loc_reader, ranges);
1410 case 'x':
1411 return convert_hex (pfile, from, limit, tbuf, cvt,
1412 char_range, loc_reader, ranges);
1413 break;
1415 case '0': case '1': case '2': case '3':
1416 case '4': case '5': case '6': case '7':
1417 return convert_oct (pfile, from, limit, tbuf, cvt,
1418 char_range, loc_reader, ranges);
1420 /* Various letter escapes. Get the appropriate host-charset
1421 value into C. */
1422 case '\\': case '\'': case '"': case '?': break;
1424 case '(': case '{': case '[': case '%':
1425 /* '\(', etc, can be used at the beginning of a line in a long
1426 string split onto multiple lines with \-newline, to prevent
1427 Emacs or other text editors from getting confused. '\%' can
1428 be used to prevent SCCS from mangling printf format strings. */
1429 if (CPP_PEDANTIC (pfile))
1430 goto unknown;
1431 break;
1433 case 'b': c = charconsts[1]; break;
1434 case 'f': c = charconsts[3]; break;
1435 case 'n': c = charconsts[4]; break;
1436 case 'r': c = charconsts[5]; break;
1437 case 't': c = charconsts[6]; break;
1438 case 'v': c = charconsts[7]; break;
1440 case 'a':
1441 if (CPP_WTRADITIONAL (pfile))
1442 cpp_warning (pfile, CPP_W_TRADITIONAL,
1443 "the meaning of '\\a' is different in traditional C");
1444 c = charconsts[0];
1445 break;
1447 case 'e': case 'E':
1448 if (CPP_PEDANTIC (pfile))
1449 cpp_error (pfile, CPP_DL_PEDWARN,
1450 "non-ISO-standard escape sequence, '\\%c'", (int) c);
1451 c = charconsts[2];
1452 break;
1454 default:
1455 unknown:
1456 if (ISGRAPH (c))
1457 cpp_error (pfile, CPP_DL_PEDWARN,
1458 "unknown escape sequence: '\\%c'", (int) c);
1459 else
1461 /* diagnostic.c does not support "%03o". When it does, this
1462 code can use %03o directly in the diagnostic again. */
1463 char buf[32];
1464 sprintf(buf, "%03o", (int) c);
1465 cpp_error (pfile, CPP_DL_PEDWARN,
1466 "unknown escape sequence: '\\%s'", buf);
1470 if (tbuf)
1471 /* Now convert what we have to the execution character set. */
1472 if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
1473 cpp_errno (pfile, CPP_DL_ERROR,
1474 "converting escape sequence to execution character set");
1476 if (loc_reader)
1478 char_range.m_finish = loc_reader->get_next ().m_finish;
1479 ranges->add_range (char_range);
1482 return from + 1;
1485 /* TYPE is a token type. The return value is the conversion needed to
1486 convert from source to execution character set for the given type. */
1487 static struct cset_converter
1488 converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
1490 switch (type)
1492 default:
1493 return pfile->narrow_cset_desc;
1494 case CPP_UTF8CHAR:
1495 case CPP_UTF8STRING:
1496 return pfile->utf8_cset_desc;
1497 case CPP_CHAR16:
1498 case CPP_STRING16:
1499 return pfile->char16_cset_desc;
1500 case CPP_CHAR32:
1501 case CPP_STRING32:
1502 return pfile->char32_cset_desc;
1503 case CPP_WCHAR:
1504 case CPP_WSTRING:
1505 return pfile->wide_cset_desc;
1509 /* FROM is an array of cpp_string structures of length COUNT. These
1510 are to be converted from the source to the execution character set,
1511 escape sequences translated, and finally all are to be
1512 concatenated. WIDE indicates whether or not to produce a wide
1513 string. If TO is non-NULL, the result is written into TO.
1514 If LOC_READERS and OUT are non-NULL, then location information
1515 is read from LOC_READERS (which must be an array of length COUNT),
1516 and location information is written to *RANGES.
1518 Returns true for success, false for failure. */
1520 static bool
1521 cpp_interpret_string_1 (cpp_reader *pfile, const cpp_string *from, size_t count,
1522 cpp_string *to, enum cpp_ttype type,
1523 cpp_string_location_reader *loc_readers,
1524 cpp_substring_ranges *out)
1526 struct _cpp_strbuf tbuf;
1527 const uchar *p, *base, *limit;
1528 size_t i;
1529 struct cset_converter cvt = converter_for_type (pfile, type);
1531 /* loc_readers and out must either be both NULL, or both be non-NULL. */
1532 gcc_assert ((loc_readers != NULL) == (out != NULL));
1534 if (to)
1536 tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
1537 tbuf.text = XNEWVEC (uchar, tbuf.asize);
1538 tbuf.len = 0;
1541 cpp_string_location_reader *loc_reader = NULL;
1542 for (i = 0; i < count; i++)
1544 if (loc_readers)
1545 loc_reader = &loc_readers[i];
1547 p = from[i].text;
1548 if (*p == 'u')
1550 p++;
1551 if (loc_reader)
1552 loc_reader->get_next ();
1553 if (*p == '8')
1555 p++;
1556 if (loc_reader)
1557 loc_reader->get_next ();
1560 else if (*p == 'L' || *p == 'U') p++;
1561 if (*p == 'R')
1563 const uchar *prefix;
1565 /* Skip over 'R"'. */
1566 p += 2;
1567 prefix = p;
1568 while (*p != '(')
1569 p++;
1570 p++;
1571 limit = from[i].text + from[i].len;
1572 if (limit >= p + (p - prefix) + 1)
1573 limit -= (p - prefix) + 1;
1575 /* Raw strings are all normal characters; these can be fed
1576 directly to convert_cset. */
1577 if (to)
1578 if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
1579 goto fail;
1581 if (loc_reader)
1583 /* If generating source ranges, assume we have a 1:1
1584 correspondence between bytes in the source encoding and bytes
1585 in the execution encoding (e.g. if we have a UTF-8 to UTF-8
1586 conversion), so that this run of bytes in the source file
1587 corresponds to a run of bytes in the execution string.
1588 This requirement is guaranteed by an early-reject in
1589 cpp_interpret_string_ranges. */
1590 gcc_assert (cvt.func == convert_no_conversion);
1591 out->add_n_ranges (limit - p, *loc_reader);
1594 continue;
1597 /* If we don't now have a leading quote, something has gone wrong.
1598 This can occur if cpp_interpret_string_ranges is handling a
1599 stringified macro argument, but should not be possible otherwise. */
1600 if (*p != '"' && *p != '\'')
1602 gcc_assert (out != NULL);
1603 cpp_error (pfile, CPP_DL_ERROR, "missing open quote");
1604 if (to)
1605 free (tbuf.text);
1606 return false;
1609 /* Skip leading quote. */
1610 p++;
1611 if (loc_reader)
1612 loc_reader->get_next ();
1614 limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
1616 for (;;)
1618 base = p;
1619 while (p < limit && *p != '\\')
1620 p++;
1621 if (p > base)
1623 /* We have a run of normal characters; these can be fed
1624 directly to convert_cset. */
1625 if (to)
1626 if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
1627 goto fail;
1628 /* Similar to above: assumes we have a 1:1 correspondence
1629 between bytes in the source encoding and bytes in the
1630 execution encoding. */
1631 if (loc_reader)
1633 gcc_assert (cvt.func == convert_no_conversion);
1634 out->add_n_ranges (p - base, *loc_reader);
1637 if (p >= limit)
1638 break;
1640 struct _cpp_strbuf *tbuf_ptr = to ? &tbuf : NULL;
1641 p = convert_escape (pfile, p + 1, limit, tbuf_ptr, cvt,
1642 loc_reader, out);
1646 if (to)
1648 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
1649 structure. */
1650 emit_numeric_escape (pfile, 0, &tbuf, cvt);
1651 tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
1652 to->text = tbuf.text;
1653 to->len = tbuf.len;
1655 /* Use the location of the trailing quote as the location of the
1656 NUL-terminator. */
1657 if (loc_reader)
1659 source_range range = loc_reader->get_next ();
1660 out->add_range (range);
1663 return true;
1665 fail:
1666 cpp_errno (pfile, CPP_DL_ERROR, "converting to execution character set");
1667 if (to)
1668 free (tbuf.text);
1669 return false;
1672 /* FROM is an array of cpp_string structures of length COUNT. These
1673 are to be converted from the source to the execution character set,
1674 escape sequences translated, and finally all are to be
1675 concatenated. WIDE indicates whether or not to produce a wide
1676 string. The result is written into TO. Returns true for success,
1677 false for failure. */
1678 bool
1679 cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
1680 cpp_string *to, enum cpp_ttype type)
1682 return cpp_interpret_string_1 (pfile, from, count, to, type, NULL, NULL);
1685 /* A "do nothing" error-handling callback for use by
1686 cpp_interpret_string_ranges, so that it can temporarily suppress
1687 error-handling. */
1689 static bool
1690 noop_error_cb (cpp_reader *, int, int, rich_location *,
1691 const char *, va_list *)
1693 /* no-op. */
1694 return true;
1697 /* This function mimics the behavior of cpp_interpret_string, but
1698 rather than generating a string in the execution character set,
1699 *OUT is written to with the source code ranges of the characters
1700 in such a string.
1701 FROM and LOC_READERS should both be arrays of length COUNT.
1702 Returns NULL for success, or an error message for failure. */
1704 const char *
1705 cpp_interpret_string_ranges (cpp_reader *pfile, const cpp_string *from,
1706 cpp_string_location_reader *loc_readers,
1707 size_t count,
1708 cpp_substring_ranges *out,
1709 enum cpp_ttype type)
1711 /* There are a couple of cases in the range-handling in
1712 cpp_interpret_string_1 that rely on there being a 1:1 correspondence
1713 between bytes in the source encoding and bytes in the execution
1714 encoding, so that each byte in the execution string can correspond
1715 to the location of a byte in the source string.
1717 This holds for the typical case of a UTF-8 to UTF-8 conversion.
1718 Enforce this requirement by only attempting to track substring
1719 locations if we have source encoding == execution encoding.
1721 This is a stronger condition than we need, since we could e.g.
1722 have ASCII to EBCDIC (with 1 byte per character before and after),
1723 but it seems to be a reasonable restriction. */
1724 struct cset_converter cvt = converter_for_type (pfile, type);
1725 if (cvt.func != convert_no_conversion)
1726 return "execution character set != source character set";
1728 /* For on-demand strings we have already lexed the strings, so there
1729 should be no errors. However, if we have bogus source location
1730 data (or stringified macro arguments), the attempt to lex the
1731 strings could fail with an error. Temporarily install an
1732 error-handler to catch the error, so that it can lead to this call
1733 failing, rather than being emitted as a user-visible diagnostic.
1734 If an error does occur, we should see it via the return value of
1735 cpp_interpret_string_1. */
1736 bool (*saved_error_handler) (cpp_reader *, int, int, rich_location *,
1737 const char *, va_list *)
1738 ATTRIBUTE_FPTR_PRINTF(5,0);
1740 saved_error_handler = pfile->cb.error;
1741 pfile->cb.error = noop_error_cb;
1743 bool result = cpp_interpret_string_1 (pfile, from, count, NULL, type,
1744 loc_readers, out);
1746 /* Restore the saved error-handler. */
1747 pfile->cb.error = saved_error_handler;
1749 if (!result)
1750 return "cpp_interpret_string_1 failed";
1752 /* Success. */
1753 return NULL;
1756 /* Subroutine of do_line and do_linemarker. Convert escape sequences
1757 in a string, but do not perform character set conversion. */
1758 bool
1759 cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from,
1760 size_t count, cpp_string *to,
1761 enum cpp_ttype type ATTRIBUTE_UNUSED)
1763 struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc;
1764 bool retval;
1766 pfile->narrow_cset_desc.func = convert_no_conversion;
1767 pfile->narrow_cset_desc.cd = (iconv_t) -1;
1768 pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
1770 retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING);
1772 pfile->narrow_cset_desc = save_narrow_cset_desc;
1773 return retval;
1777 /* Subroutine of cpp_interpret_charconst which performs the conversion
1778 to a number, for narrow strings. STR is the string structure returned
1779 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1780 cpp_interpret_charconst. */
1781 static cppchar_t
1782 narrow_str_to_charconst (cpp_reader *pfile, cpp_string str,
1783 unsigned int *pchars_seen, int *unsignedp)
1785 size_t width = CPP_OPTION (pfile, char_precision);
1786 size_t max_chars = CPP_OPTION (pfile, int_precision) / width;
1787 size_t mask = width_to_mask (width);
1788 size_t i;
1789 cppchar_t result, c;
1790 bool unsigned_p;
1792 /* The value of a multi-character character constant, or a
1793 single-character character constant whose representation in the
1794 execution character set is more than one byte long, is
1795 implementation defined. This implementation defines it to be the
1796 number formed by interpreting the byte sequence in memory as a
1797 big-endian binary number. If overflow occurs, the high bytes are
1798 lost, and a warning is issued.
1800 We don't want to process the NUL terminator handed back by
1801 cpp_interpret_string. */
1802 result = 0;
1803 for (i = 0; i < str.len - 1; i++)
1805 c = str.text[i] & mask;
1806 if (width < BITS_PER_CPPCHAR_T)
1807 result = (result << width) | c;
1808 else
1809 result = c;
1812 if (i > max_chars)
1814 i = max_chars;
1815 cpp_error (pfile, CPP_DL_WARNING,
1816 "character constant too long for its type");
1818 else if (i > 1 && CPP_OPTION (pfile, warn_multichar))
1819 cpp_warning (pfile, CPP_W_MULTICHAR, "multi-character character constant");
1821 /* Multichar constants are of type int and therefore signed. */
1822 if (i > 1)
1823 unsigned_p = 0;
1824 else
1825 unsigned_p = CPP_OPTION (pfile, unsigned_char);
1827 /* Truncate the constant to its natural width, and simultaneously
1828 sign- or zero-extend to the full width of cppchar_t.
1829 For single-character constants, the value is WIDTH bits wide.
1830 For multi-character constants, the value is INT_PRECISION bits wide. */
1831 if (i > 1)
1832 width = CPP_OPTION (pfile, int_precision);
1833 if (width < BITS_PER_CPPCHAR_T)
1835 mask = ((cppchar_t) 1 << width) - 1;
1836 if (unsigned_p || !(result & (1 << (width - 1))))
1837 result &= mask;
1838 else
1839 result |= ~mask;
1841 *pchars_seen = i;
1842 *unsignedp = unsigned_p;
1843 return result;
1846 /* Subroutine of cpp_interpret_charconst which performs the conversion
1847 to a number, for wide strings. STR is the string structure returned
1848 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1849 cpp_interpret_charconst. TYPE is the token type. */
1850 static cppchar_t
1851 wide_str_to_charconst (cpp_reader *pfile, cpp_string str,
1852 unsigned int *pchars_seen, int *unsignedp,
1853 enum cpp_ttype type)
1855 bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1856 size_t width = converter_for_type (pfile, type).width;
1857 size_t cwidth = CPP_OPTION (pfile, char_precision);
1858 size_t mask = width_to_mask (width);
1859 size_t cmask = width_to_mask (cwidth);
1860 size_t nbwc = width / cwidth;
1861 size_t off, i;
1862 cppchar_t result = 0, c;
1864 /* This is finicky because the string is in the target's byte order,
1865 which may not be our byte order. Only the last character, ignoring
1866 the NUL terminator, is relevant. */
1867 off = str.len - (nbwc * 2);
1868 result = 0;
1869 for (i = 0; i < nbwc; i++)
1871 c = bigend ? str.text[off + i] : str.text[off + nbwc - i - 1];
1872 result = (result << cwidth) | (c & cmask);
1875 /* Wide character constants have type wchar_t, and a single
1876 character exactly fills a wchar_t, so a multi-character wide
1877 character constant is guaranteed to overflow. */
1878 if (str.len > nbwc * 2)
1879 cpp_error (pfile, CPP_DL_WARNING,
1880 "character constant too long for its type");
1882 /* Truncate the constant to its natural width, and simultaneously
1883 sign- or zero-extend to the full width of cppchar_t. */
1884 if (width < BITS_PER_CPPCHAR_T)
1886 if (type == CPP_CHAR16 || type == CPP_CHAR32
1887 || CPP_OPTION (pfile, unsigned_wchar)
1888 || !(result & (1 << (width - 1))))
1889 result &= mask;
1890 else
1891 result |= ~mask;
1894 if (type == CPP_CHAR16 || type == CPP_CHAR32
1895 || CPP_OPTION (pfile, unsigned_wchar))
1896 *unsignedp = 1;
1897 else
1898 *unsignedp = 0;
1900 *pchars_seen = 1;
1901 return result;
1904 /* Interpret a (possibly wide) character constant in TOKEN.
1905 PCHARS_SEEN points to a variable that is filled in with the number
1906 of characters seen, and UNSIGNEDP to a variable that indicates
1907 whether the result has signed type. */
1908 cppchar_t
1909 cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token,
1910 unsigned int *pchars_seen, int *unsignedp)
1912 cpp_string str = { 0, 0 };
1913 bool wide = (token->type != CPP_CHAR && token->type != CPP_UTF8CHAR);
1914 int u8 = 2 * int(token->type == CPP_UTF8CHAR);
1915 cppchar_t result;
1917 /* An empty constant will appear as L'', u'', U'', u8'', or '' */
1918 if (token->val.str.len == (size_t) (2 + wide + u8))
1920 cpp_error (pfile, CPP_DL_ERROR, "empty character constant");
1921 *pchars_seen = 0;
1922 *unsignedp = 0;
1923 return 0;
1925 else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str,
1926 token->type))
1928 *pchars_seen = 0;
1929 *unsignedp = 0;
1930 return 0;
1933 if (wide)
1934 result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
1935 token->type);
1936 else
1937 result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp);
1939 if (str.text != token->val.str.text)
1940 free ((void *)str.text);
1942 return result;
1945 /* Convert an identifier denoted by ID and LEN, which might contain
1946 UCN escapes, to the source character set, either UTF-8 or
1947 UTF-EBCDIC. Assumes that the identifier is actually a valid identifier. */
1948 cpp_hashnode *
1949 _cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len)
1951 /* It turns out that a UCN escape always turns into fewer characters
1952 than the escape itself, so we can allocate a temporary in advance. */
1953 uchar * buf = (uchar *) alloca (len + 1);
1954 uchar * bufp = buf;
1955 size_t idp;
1957 for (idp = 0; idp < len; idp++)
1958 if (id[idp] != '\\')
1959 *bufp++ = id[idp];
1960 else
1962 unsigned length = id[idp+1] == 'u' ? 4 : 8;
1963 cppchar_t value = 0;
1964 size_t bufleft = len - (bufp - buf);
1965 int rval;
1967 idp += 2;
1968 while (length && idp < len && ISXDIGIT (id[idp]))
1970 value = (value << 4) + hex_value (id[idp]);
1971 idp++;
1972 length--;
1974 idp--;
1976 /* Special case for EBCDIC: if the identifier contains
1977 a '$' specified using a UCN, translate it to EBCDIC. */
1978 if (value == 0x24)
1980 *bufp++ = '$';
1981 continue;
1984 rval = one_cppchar_to_utf8 (value, &bufp, &bufleft);
1985 if (rval)
1987 errno = rval;
1988 cpp_errno (pfile, CPP_DL_ERROR,
1989 "converting UCN to source character set");
1990 break;
1994 return CPP_HASHNODE (ht_lookup (pfile->hash_table,
1995 buf, bufp - buf, HT_ALLOC));
1998 /* Convert an input buffer (containing the complete contents of one
1999 source file) from INPUT_CHARSET to the source character set. INPUT
2000 points to the input buffer, SIZE is its allocated size, and LEN is
2001 the length of the meaningful data within the buffer. The
2002 translated buffer is returned, *ST_SIZE is set to the length of
2003 the meaningful data within the translated buffer, and *BUFFER_START
2004 is set to the start of the returned buffer. *BUFFER_START may
2005 differ from the return value in the case of a BOM or other ignored
2006 marker information.
2008 INPUT is expected to have been allocated with xmalloc. This
2009 function will either set *BUFFER_START to INPUT, or free it and set
2010 *BUFFER_START to a pointer to another xmalloc-allocated block of
2011 memory. */
2012 uchar *
2013 _cpp_convert_input (cpp_reader *pfile, const char *input_charset,
2014 uchar *input, size_t size, size_t len,
2015 const unsigned char **buffer_start, off_t *st_size)
2017 struct cset_converter input_cset;
2018 struct _cpp_strbuf to;
2019 unsigned char *buffer;
2021 input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset);
2022 if (input_cset.func == convert_no_conversion)
2024 to.text = input;
2025 to.asize = size;
2026 to.len = len;
2028 else
2030 to.asize = MAX (65536, len);
2031 to.text = XNEWVEC (uchar, to.asize);
2032 to.len = 0;
2034 if (!APPLY_CONVERSION (input_cset, input, len, &to))
2035 cpp_error (pfile, CPP_DL_ERROR,
2036 "failure to convert %s to %s",
2037 CPP_OPTION (pfile, input_charset), SOURCE_CHARSET);
2039 free (input);
2042 /* Clean up the mess. */
2043 if (input_cset.func == convert_using_iconv)
2044 iconv_close (input_cset.cd);
2046 /* Resize buffer if we allocated substantially too much, or if we
2047 haven't enough space for the \n-terminator or following
2048 15 bytes of padding (used to quiet warnings from valgrind or
2049 Address Sanitizer, when the optimized lexer accesses aligned
2050 16-byte memory chunks, including the bytes after the malloced,
2051 area, and stops lexing on '\n'). */
2052 if (to.len + 4096 < to.asize || to.len + 16 > to.asize)
2053 to.text = XRESIZEVEC (uchar, to.text, to.len + 16);
2055 memset (to.text + to.len, '\0', 16);
2057 /* If the file is using old-school Mac line endings (\r only),
2058 terminate with another \r, not an \n, so that we do not mistake
2059 the \r\n sequence for a single DOS line ending and erroneously
2060 issue the "No newline at end of file" diagnostic. */
2061 if (to.len && to.text[to.len - 1] == '\r')
2062 to.text[to.len] = '\r';
2063 else
2064 to.text[to.len] = '\n';
2066 buffer = to.text;
2067 *st_size = to.len;
2068 #if HOST_CHARSET == HOST_CHARSET_ASCII
2069 /* The HOST_CHARSET test just above ensures that the source charset
2070 is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
2071 glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
2072 BOM -- however, even if it did, we would still need this code due
2073 to the 'convert_no_conversion' case. */
2074 if (to.len >= 3 && to.text[0] == 0xef && to.text[1] == 0xbb
2075 && to.text[2] == 0xbf)
2077 *st_size -= 3;
2078 buffer += 3;
2080 #endif
2082 *buffer_start = to.text;
2083 return buffer;
2086 /* Decide on the default encoding to assume for input files. */
2087 const char *
2088 _cpp_default_encoding (void)
2090 const char *current_encoding = NULL;
2092 /* We disable this because the default codeset is 7-bit ASCII on
2093 most platforms, and this causes conversion failures on every
2094 file in GCC that happens to have one of the upper 128 characters
2095 in it -- most likely, as part of the name of a contributor.
2096 We should definitely recognize in-band markers of file encoding,
2097 like:
2098 - the appropriate Unicode byte-order mark (FE FF) to recognize
2099 UTF16 and UCS4 (in both big-endian and little-endian flavors)
2100 and UTF8
2101 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
2102 distinguish ASCII and EBCDIC.
2103 - now we can parse something like "#pragma GCC encoding <xyz>
2104 on the first line, or even Emacs/VIM's mode line tags (there's
2105 a problem here in that VIM uses the last line, and Emacs has
2106 its more elaborate "local variables" convention).
2107 - investigate whether Java has another common convention, which
2108 would be friendly to support.
2109 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
2110 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
2111 setlocale (LC_CTYPE, "");
2112 current_encoding = nl_langinfo (CODESET);
2113 #endif
2114 if (current_encoding == NULL || *current_encoding == '\0')
2115 current_encoding = SOURCE_CHARSET;
2117 return current_encoding;
2120 /* Implementation of class cpp_string_location_reader. */
2122 /* Constructor for cpp_string_location_reader. */
2124 cpp_string_location_reader::
2125 cpp_string_location_reader (source_location src_loc,
2126 line_maps *line_table)
2127 : m_line_table (line_table)
2129 src_loc = get_range_from_loc (line_table, src_loc).m_start;
2131 /* SRC_LOC might be a macro location. It only makes sense to do
2132 column-by-column calculations on ordinary maps, so get the
2133 corresponding location in an ordinary map. */
2134 m_loc
2135 = linemap_resolve_location (line_table, src_loc,
2136 LRK_SPELLING_LOCATION, NULL);
2138 const line_map_ordinary *map
2139 = linemap_check_ordinary (linemap_lookup (line_table, m_loc));
2140 m_offset_per_column = (1 << map->m_range_bits);
2143 /* Get the range of the next source byte. */
2145 source_range
2146 cpp_string_location_reader::get_next ()
2148 source_range result;
2149 result.m_start = m_loc;
2150 result.m_finish = m_loc;
2151 if (m_loc <= LINE_MAP_MAX_LOCATION_WITH_COLS)
2152 m_loc += m_offset_per_column;
2153 return result;