[AArch64] arm_neon.h - add vpaddd_f64, vpaddd_s64, vpaddd_u64 intrinsics
[official-gcc.git] / libcpp / charset.c
1 /* CPP Library - charsets
2 Copyright (C) 1998-2014 Free Software Foundation, Inc.
3
4 Broken out of c-lex.c Apr 2003, adding valid C99 UCN ranges.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
9 later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "cpplib.h"
23 #include "internal.h"
24
25 /* Character set handling for C-family languages.
26
27 Terminological note: In what follows, "charset" or "character set"
28 will be taken to mean both an abstract set of characters and an
29 encoding for that set.
30
31 The C99 standard discusses two character sets: source and execution.
32 The source character set is used for internal processing in translation
33 phases 1 through 4; the execution character set is used thereafter.
34 Both are required by 5.2.1.2p1 to be multibyte encodings, not wide
35 character encodings (see 3.7.2, 3.7.3 for the standardese meanings
36 of these terms). Furthermore, the "basic character set" (listed in
37 5.2.1p3) is to be encoded in each with values one byte wide, and is
38 to appear in the initial shift state.
39
40 It is not explicitly mentioned, but there is also a "wide execution
41 character set" used to encode wide character constants and wide
42 string literals; this is supposed to be the result of applying the
43 standard library function mbstowcs() to an equivalent narrow string
44 (6.4.5p5). However, the behavior of hexadecimal and octal
45 \-escapes is at odds with this; they are supposed to be translated
46 directly to wchar_t values (6.4.4.4p5,6).
47
48 The source character set is not necessarily the character set used
49 to encode physical source files on disk; translation phase 1 converts
50 from whatever that encoding is to the source character set.
51
52 The presence of universal character names in C99 (6.4.3 et seq.)
53 forces the source character set to be isomorphic to ISO 10646,
54 that is, Unicode. There is no such constraint on the execution
55 character set; note also that the conversion from source to
56 execution character set does not occur for identifiers (5.1.1.2p1#5).
57
58 For convenience of implementation, the source character set's
59 encoding of the basic character set should be identical to the
60 execution character set OF THE HOST SYSTEM's encoding of the basic
61 character set, and it should not be a state-dependent encoding.
62
63 cpplib uses UTF-8 or UTF-EBCDIC for the source character set,
64 depending on whether the host is based on ASCII or EBCDIC (see
65 respectively Unicode section 2.3/ISO10646 Amendment 2, and Unicode
66 Technical Report #16). With limited exceptions, it relies on the
67 system library's iconv() primitive to do charset conversion
68 (specified in SUSv2). */
69
70 #if !HAVE_ICONV
71 /* Make certain that the uses of iconv(), iconv_open(), iconv_close()
72 below, which are guarded only by if statements with compile-time
73 constant conditions, do not cause link errors. */
74 #define iconv_open(x, y) (errno = EINVAL, (iconv_t)-1)
75 #define iconv(a,b,c,d,e) (errno = EINVAL, (size_t)-1)
76 #define iconv_close(x) (void)0
77 #define ICONV_CONST
78 #endif
79
80 #if HOST_CHARSET == HOST_CHARSET_ASCII
81 #define SOURCE_CHARSET "UTF-8"
82 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0x7e
83 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
84 #define SOURCE_CHARSET "UTF-EBCDIC"
85 #define LAST_POSSIBLY_BASIC_SOURCE_CHAR 0xFF
86 #else
87 #error "Unrecognized basic host character set"
88 #endif
89
90 #ifndef EILSEQ
91 #define EILSEQ EINVAL
92 #endif
93
94 /* This structure is used for a resizable string buffer throughout. */
95 /* Don't call it strbuf, as that conflicts with unistd.h on systems
96 such as DYNIX/ptx where unistd.h includes stropts.h. */
97 struct _cpp_strbuf
98 {
99 uchar *text;
100 size_t asize;
101 size_t len;
102 };
103
104 /* This is enough to hold any string that fits on a single 80-column
105 line, even if iconv quadruples its size (e.g. conversion from
106 ASCII to UTF-32) rounded up to a power of two. */
107 #define OUTBUF_BLOCK_SIZE 256
108
109 /* Conversions between UTF-8 and UTF-16/32 are implemented by custom
110 logic. This is because a depressing number of systems lack iconv,
111 or have have iconv libraries that do not do these conversions, so
112 we need a fallback implementation for them. To ensure the fallback
113 doesn't break due to neglect, it is used on all systems.
114
115 UTF-32 encoding is nice and simple: a four-byte binary number,
116 constrained to the range 00000000-7FFFFFFF to avoid questions of
117 signedness. We do have to cope with big- and little-endian
118 variants.
119
120 UTF-16 encoding uses two-byte binary numbers, again in big- and
121 little-endian variants, for all values in the 00000000-0000FFFF
122 range. Values in the 00010000-0010FFFF range are encoded as pairs
123 of two-byte numbers, called "surrogate pairs": given a number S in
124 this range, it is mapped to a pair (H, L) as follows:
125
126 H = (S - 0x10000) / 0x400 + 0xD800
127 L = (S - 0x10000) % 0x400 + 0xDC00
128
129 Two-byte values in the D800...DFFF range are ill-formed except as a
130 component of a surrogate pair. Even if the encoding within a
131 two-byte value is little-endian, the H member of the surrogate pair
132 comes first.
133
134 There is no way to encode values in the 00110000-7FFFFFFF range,
135 which is not currently a problem as there are no assigned code
136 points in that range; however, the author expects that it will
137 eventually become necessary to abandon UTF-16 due to this
138 limitation. Note also that, because of these pairs, UTF-16 does
139 not meet the requirements of the C standard for a wide character
140 encoding (see 3.7.3 and 6.4.4.4p11).
141
142 UTF-8 encoding looks like this:
143
144 value range encoded as
145 00000000-0000007F 0xxxxxxx
146 00000080-000007FF 110xxxxx 10xxxxxx
147 00000800-0000FFFF 1110xxxx 10xxxxxx 10xxxxxx
148 00010000-001FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
149 00200000-03FFFFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
150 04000000-7FFFFFFF 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
151
152 Values in the 0000D800 ... 0000DFFF range (surrogates) are invalid,
153 which means that three-byte sequences ED xx yy, with A0 <= xx <= BF,
154 never occur. Note also that any value that can be encoded by a
155 given row of the table can also be encoded by all successive rows,
156 but this is not done; only the shortest possible encoding for any
157 given value is valid. For instance, the character 07C0 could be
158 encoded as any of DF 80, E0 9F 80, F0 80 9F 80, F8 80 80 9F 80, or
159 FC 80 80 80 9F 80. Only the first is valid.
160
161 An implementation note: the transformation from UTF-16 to UTF-8, or
162 vice versa, is easiest done by using UTF-32 as an intermediary. */
163
164 /* Internal primitives which go from an UTF-8 byte stream to native-endian
165 UTF-32 in a cppchar_t, or vice versa; this avoids an extra marshal/unmarshal
166 operation in several places below. */
167 static inline int
168 one_utf8_to_cppchar (const uchar **inbufp, size_t *inbytesleftp,
169 cppchar_t *cp)
170 {
171 static const uchar masks[6] = { 0x7F, 0x1F, 0x0F, 0x07, 0x03, 0x01 };
172 static const uchar patns[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
173
174 cppchar_t c;
175 const uchar *inbuf = *inbufp;
176 size_t nbytes, i;
177
178 if (*inbytesleftp < 1)
179 return EINVAL;
180
181 c = *inbuf;
182 if (c < 0x80)
183 {
184 *cp = c;
185 *inbytesleftp -= 1;
186 *inbufp += 1;
187 return 0;
188 }
189
190 /* The number of leading 1-bits in the first byte indicates how many
191 bytes follow. */
192 for (nbytes = 2; nbytes < 7; nbytes++)
193 if ((c & ~masks[nbytes-1]) == patns[nbytes-1])
194 goto found;
195 return EILSEQ;
196 found:
197
198 if (*inbytesleftp < nbytes)
199 return EINVAL;
200
201 c = (c & masks[nbytes-1]);
202 inbuf++;
203 for (i = 1; i < nbytes; i++)
204 {
205 cppchar_t n = *inbuf++;
206 if ((n & 0xC0) != 0x80)
207 return EILSEQ;
208 c = ((c << 6) + (n & 0x3F));
209 }
210
211 /* Make sure the shortest possible encoding was used. */
212 if (c <= 0x7F && nbytes > 1) return EILSEQ;
213 if (c <= 0x7FF && nbytes > 2) return EILSEQ;
214 if (c <= 0xFFFF && nbytes > 3) return EILSEQ;
215 if (c <= 0x1FFFFF && nbytes > 4) return EILSEQ;
216 if (c <= 0x3FFFFFF && nbytes > 5) return EILSEQ;
217
218 /* Make sure the character is valid. */
219 if (c > 0x7FFFFFFF || (c >= 0xD800 && c <= 0xDFFF)) return EILSEQ;
220
221 *cp = c;
222 *inbufp = inbuf;
223 *inbytesleftp -= nbytes;
224 return 0;
225 }
226
227 static inline int
228 one_cppchar_to_utf8 (cppchar_t c, uchar **outbufp, size_t *outbytesleftp)
229 {
230 static const uchar masks[6] = { 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
231 static const uchar limits[6] = { 0x80, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
232 size_t nbytes;
233 uchar buf[6], *p = &buf[6];
234 uchar *outbuf = *outbufp;
235
236 nbytes = 1;
237 if (c < 0x80)
238 *--p = c;
239 else
240 {
241 do
242 {
243 *--p = ((c & 0x3F) | 0x80);
244 c >>= 6;
245 nbytes++;
246 }
247 while (c >= 0x3F || (c & limits[nbytes-1]));
248 *--p = (c | masks[nbytes-1]);
249 }
250
251 if (*outbytesleftp < nbytes)
252 return E2BIG;
253
254 while (p < &buf[6])
255 *outbuf++ = *p++;
256 *outbytesleftp -= nbytes;
257 *outbufp = outbuf;
258 return 0;
259 }
260
261 /* The following four functions transform one character between the two
262 encodings named in the function name. All have the signature
263 int (*)(iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
264 uchar **outbufp, size_t *outbytesleftp)
265
266 BIGEND must have the value 0 or 1, coerced to (iconv_t); it is
267 interpreted as a boolean indicating whether big-endian or
268 little-endian encoding is to be used for the member of the pair
269 that is not UTF-8.
270
271 INBUFP, INBYTESLEFTP, OUTBUFP, OUTBYTESLEFTP work exactly as they
272 do for iconv.
273
274 The return value is either 0 for success, or an errno value for
275 failure, which may be E2BIG (need more space), EILSEQ (ill-formed
276 input sequence), ir EINVAL (incomplete input sequence). */
277
278 static inline int
279 one_utf8_to_utf32 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
280 uchar **outbufp, size_t *outbytesleftp)
281 {
282 uchar *outbuf;
283 cppchar_t s = 0;
284 int rval;
285
286 /* Check for space first, since we know exactly how much we need. */
287 if (*outbytesleftp < 4)
288 return E2BIG;
289
290 rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s);
291 if (rval)
292 return rval;
293
294 outbuf = *outbufp;
295 outbuf[bigend ? 3 : 0] = (s & 0x000000FF);
296 outbuf[bigend ? 2 : 1] = (s & 0x0000FF00) >> 8;
297 outbuf[bigend ? 1 : 2] = (s & 0x00FF0000) >> 16;
298 outbuf[bigend ? 0 : 3] = (s & 0xFF000000) >> 24;
299
300 *outbufp += 4;
301 *outbytesleftp -= 4;
302 return 0;
303 }
304
305 static inline int
306 one_utf32_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
307 uchar **outbufp, size_t *outbytesleftp)
308 {
309 cppchar_t s;
310 int rval;
311 const uchar *inbuf;
312
313 if (*inbytesleftp < 4)
314 return EINVAL;
315
316 inbuf = *inbufp;
317
318 s = inbuf[bigend ? 0 : 3] << 24;
319 s += inbuf[bigend ? 1 : 2] << 16;
320 s += inbuf[bigend ? 2 : 1] << 8;
321 s += inbuf[bigend ? 3 : 0];
322
323 if (s >= 0x7FFFFFFF || (s >= 0xD800 && s <= 0xDFFF))
324 return EILSEQ;
325
326 rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp);
327 if (rval)
328 return rval;
329
330 *inbufp += 4;
331 *inbytesleftp -= 4;
332 return 0;
333 }
334
335 static inline int
336 one_utf8_to_utf16 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
337 uchar **outbufp, size_t *outbytesleftp)
338 {
339 int rval;
340 cppchar_t s = 0;
341 const uchar *save_inbuf = *inbufp;
342 size_t save_inbytesleft = *inbytesleftp;
343 uchar *outbuf = *outbufp;
344
345 rval = one_utf8_to_cppchar (inbufp, inbytesleftp, &s);
346 if (rval)
347 return rval;
348
349 if (s > 0x0010FFFF)
350 {
351 *inbufp = save_inbuf;
352 *inbytesleftp = save_inbytesleft;
353 return EILSEQ;
354 }
355
356 if (s < 0xFFFF)
357 {
358 if (*outbytesleftp < 2)
359 {
360 *inbufp = save_inbuf;
361 *inbytesleftp = save_inbytesleft;
362 return E2BIG;
363 }
364 outbuf[bigend ? 1 : 0] = (s & 0x00FF);
365 outbuf[bigend ? 0 : 1] = (s & 0xFF00) >> 8;
366
367 *outbufp += 2;
368 *outbytesleftp -= 2;
369 return 0;
370 }
371 else
372 {
373 cppchar_t hi, lo;
374
375 if (*outbytesleftp < 4)
376 {
377 *inbufp = save_inbuf;
378 *inbytesleftp = save_inbytesleft;
379 return E2BIG;
380 }
381
382 hi = (s - 0x10000) / 0x400 + 0xD800;
383 lo = (s - 0x10000) % 0x400 + 0xDC00;
384
385 /* Even if we are little-endian, put the high surrogate first.
386 ??? Matches practice? */
387 outbuf[bigend ? 1 : 0] = (hi & 0x00FF);
388 outbuf[bigend ? 0 : 1] = (hi & 0xFF00) >> 8;
389 outbuf[bigend ? 3 : 2] = (lo & 0x00FF);
390 outbuf[bigend ? 2 : 3] = (lo & 0xFF00) >> 8;
391
392 *outbufp += 4;
393 *outbytesleftp -= 4;
394 return 0;
395 }
396 }
397
398 static inline int
399 one_utf16_to_utf8 (iconv_t bigend, const uchar **inbufp, size_t *inbytesleftp,
400 uchar **outbufp, size_t *outbytesleftp)
401 {
402 cppchar_t s;
403 const uchar *inbuf = *inbufp;
404 int rval;
405
406 if (*inbytesleftp < 2)
407 return EINVAL;
408 s = inbuf[bigend ? 0 : 1] << 8;
409 s += inbuf[bigend ? 1 : 0];
410
411 /* Low surrogate without immediately preceding high surrogate is invalid. */
412 if (s >= 0xDC00 && s <= 0xDFFF)
413 return EILSEQ;
414 /* High surrogate must have a following low surrogate. */
415 else if (s >= 0xD800 && s <= 0xDBFF)
416 {
417 cppchar_t hi = s, lo;
418 if (*inbytesleftp < 4)
419 return EINVAL;
420
421 lo = inbuf[bigend ? 2 : 3] << 8;
422 lo += inbuf[bigend ? 3 : 2];
423
424 if (lo < 0xDC00 || lo > 0xDFFF)
425 return EILSEQ;
426
427 s = (hi - 0xD800) * 0x400 + (lo - 0xDC00) + 0x10000;
428 }
429
430 rval = one_cppchar_to_utf8 (s, outbufp, outbytesleftp);
431 if (rval)
432 return rval;
433
434 /* Success - update the input pointers (one_cppchar_to_utf8 has done
435 the output pointers for us). */
436 if (s <= 0xFFFF)
437 {
438 *inbufp += 2;
439 *inbytesleftp -= 2;
440 }
441 else
442 {
443 *inbufp += 4;
444 *inbytesleftp -= 4;
445 }
446 return 0;
447 }
448
449 /* Helper routine for the next few functions. The 'const' on
450 one_conversion means that we promise not to modify what function is
451 pointed to, which lets the inliner see through it. */
452
453 static inline bool
454 conversion_loop (int (*const one_conversion)(iconv_t, const uchar **, size_t *,
455 uchar **, size_t *),
456 iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to)
457 {
458 const uchar *inbuf;
459 uchar *outbuf;
460 size_t inbytesleft, outbytesleft;
461 int rval;
462
463 inbuf = from;
464 inbytesleft = flen;
465 outbuf = to->text + to->len;
466 outbytesleft = to->asize - to->len;
467
468 for (;;)
469 {
470 do
471 rval = one_conversion (cd, &inbuf, &inbytesleft,
472 &outbuf, &outbytesleft);
473 while (inbytesleft && !rval);
474
475 if (__builtin_expect (inbytesleft == 0, 1))
476 {
477 to->len = to->asize - outbytesleft;
478 return true;
479 }
480 if (rval != E2BIG)
481 {
482 errno = rval;
483 return false;
484 }
485
486 outbytesleft += OUTBUF_BLOCK_SIZE;
487 to->asize += OUTBUF_BLOCK_SIZE;
488 to->text = XRESIZEVEC (uchar, to->text, to->asize);
489 outbuf = to->text + to->asize - outbytesleft;
490 }
491 }
492
493
494 /* These functions convert entire strings between character sets.
495 They all have the signature
496
497 bool (*)(iconv_t cd, const uchar *from, size_t flen, struct _cpp_strbuf *to);
498
499 The input string FROM is converted as specified by the function
500 name plus the iconv descriptor CD (which may be fake), and the
501 result appended to TO. On any error, false is returned, otherwise true. */
502
503 /* These four use the custom conversion code above. */
504 static bool
505 convert_utf8_utf16 (iconv_t cd, const uchar *from, size_t flen,
506 struct _cpp_strbuf *to)
507 {
508 return conversion_loop (one_utf8_to_utf16, cd, from, flen, to);
509 }
510
511 static bool
512 convert_utf8_utf32 (iconv_t cd, const uchar *from, size_t flen,
513 struct _cpp_strbuf *to)
514 {
515 return conversion_loop (one_utf8_to_utf32, cd, from, flen, to);
516 }
517
518 static bool
519 convert_utf16_utf8 (iconv_t cd, const uchar *from, size_t flen,
520 struct _cpp_strbuf *to)
521 {
522 return conversion_loop (one_utf16_to_utf8, cd, from, flen, to);
523 }
524
525 static bool
526 convert_utf32_utf8 (iconv_t cd, const uchar *from, size_t flen,
527 struct _cpp_strbuf *to)
528 {
529 return conversion_loop (one_utf32_to_utf8, cd, from, flen, to);
530 }
531
532 /* Identity conversion, used when we have no alternative. */
533 static bool
534 convert_no_conversion (iconv_t cd ATTRIBUTE_UNUSED,
535 const uchar *from, size_t flen, struct _cpp_strbuf *to)
536 {
537 if (to->len + flen > to->asize)
538 {
539 to->asize = to->len + flen;
540 to->text = XRESIZEVEC (uchar, to->text, to->asize);
541 }
542 memcpy (to->text + to->len, from, flen);
543 to->len += flen;
544 return true;
545 }
546
547 /* And this one uses the system iconv primitive. It's a little
548 different, since iconv's interface is a little different. */
549 #if HAVE_ICONV
550
551 #define CONVERT_ICONV_GROW_BUFFER \
552 do { \
553 outbytesleft += OUTBUF_BLOCK_SIZE; \
554 to->asize += OUTBUF_BLOCK_SIZE; \
555 to->text = XRESIZEVEC (uchar, to->text, to->asize); \
556 outbuf = (char *)to->text + to->asize - outbytesleft; \
557 } while (0)
558
559 static bool
560 convert_using_iconv (iconv_t cd, const uchar *from, size_t flen,
561 struct _cpp_strbuf *to)
562 {
563 ICONV_CONST char *inbuf;
564 char *outbuf;
565 size_t inbytesleft, outbytesleft;
566
567 /* Reset conversion descriptor and check that it is valid. */
568 if (iconv (cd, 0, 0, 0, 0) == (size_t)-1)
569 return false;
570
571 inbuf = (ICONV_CONST char *)from;
572 inbytesleft = flen;
573 outbuf = (char *)to->text + to->len;
574 outbytesleft = to->asize - to->len;
575
576 for (;;)
577 {
578 iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
579 if (__builtin_expect (inbytesleft == 0, 1))
580 {
581 /* Close out any shift states, returning to the initial state. */
582 if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
583 {
584 if (errno != E2BIG)
585 return false;
586
587 CONVERT_ICONV_GROW_BUFFER;
588 if (iconv (cd, 0, 0, &outbuf, &outbytesleft) == (size_t)-1)
589 return false;
590 }
591
592 to->len = to->asize - outbytesleft;
593 return true;
594 }
595 if (errno != E2BIG)
596 return false;
597
598 CONVERT_ICONV_GROW_BUFFER;
599 }
600 }
601 #else
602 #define convert_using_iconv 0 /* prevent undefined symbol error below */
603 #endif
604
605 /* Arrange for the above custom conversion logic to be used automatically
606 when conversion between a suitable pair of character sets is requested. */
607
608 #define APPLY_CONVERSION(CONVERTER, FROM, FLEN, TO) \
609 CONVERTER.func (CONVERTER.cd, FROM, FLEN, TO)
610
611 struct conversion
612 {
613 const char *pair;
614 convert_f func;
615 iconv_t fake_cd;
616 };
617 static const struct conversion conversion_tab[] = {
618 { "UTF-8/UTF-32LE", convert_utf8_utf32, (iconv_t)0 },
619 { "UTF-8/UTF-32BE", convert_utf8_utf32, (iconv_t)1 },
620 { "UTF-8/UTF-16LE", convert_utf8_utf16, (iconv_t)0 },
621 { "UTF-8/UTF-16BE", convert_utf8_utf16, (iconv_t)1 },
622 { "UTF-32LE/UTF-8", convert_utf32_utf8, (iconv_t)0 },
623 { "UTF-32BE/UTF-8", convert_utf32_utf8, (iconv_t)1 },
624 { "UTF-16LE/UTF-8", convert_utf16_utf8, (iconv_t)0 },
625 { "UTF-16BE/UTF-8", convert_utf16_utf8, (iconv_t)1 },
626 };
627
628 /* Subroutine of cpp_init_iconv: initialize and return a
629 cset_converter structure for conversion from FROM to TO. If
630 iconv_open() fails, issue an error and return an identity
631 converter. Silently return an identity converter if FROM and TO
632 are identical. */
633 static struct cset_converter
634 init_iconv_desc (cpp_reader *pfile, const char *to, const char *from)
635 {
636 struct cset_converter ret;
637 char *pair;
638 size_t i;
639
640 if (!strcasecmp (to, from))
641 {
642 ret.func = convert_no_conversion;
643 ret.cd = (iconv_t) -1;
644 ret.width = -1;
645 return ret;
646 }
647
648 pair = (char *) alloca(strlen(to) + strlen(from) + 2);
649
650 strcpy(pair, from);
651 strcat(pair, "/");
652 strcat(pair, to);
653 for (i = 0; i < ARRAY_SIZE (conversion_tab); i++)
654 if (!strcasecmp (pair, conversion_tab[i].pair))
655 {
656 ret.func = conversion_tab[i].func;
657 ret.cd = conversion_tab[i].fake_cd;
658 ret.width = -1;
659 return ret;
660 }
661
662 /* No custom converter - try iconv. */
663 if (HAVE_ICONV)
664 {
665 ret.func = convert_using_iconv;
666 ret.cd = iconv_open (to, from);
667 ret.width = -1;
668
669 if (ret.cd == (iconv_t) -1)
670 {
671 if (errno == EINVAL)
672 cpp_error (pfile, CPP_DL_ERROR, /* FIXME should be DL_SORRY */
673 "conversion from %s to %s not supported by iconv",
674 from, to);
675 else
676 cpp_errno (pfile, CPP_DL_ERROR, "iconv_open");
677
678 ret.func = convert_no_conversion;
679 }
680 }
681 else
682 {
683 cpp_error (pfile, CPP_DL_ERROR, /* FIXME: should be DL_SORRY */
684 "no iconv implementation, cannot convert from %s to %s",
685 from, to);
686 ret.func = convert_no_conversion;
687 ret.cd = (iconv_t) -1;
688 ret.width = -1;
689 }
690 return ret;
691 }
692
693 /* If charset conversion is requested, initialize iconv(3) descriptors
694 for conversion from the source character set to the execution
695 character sets. If iconv is not present in the C library, and
696 conversion is requested, issue an error. */
697
698 void
699 cpp_init_iconv (cpp_reader *pfile)
700 {
701 const char *ncset = CPP_OPTION (pfile, narrow_charset);
702 const char *wcset = CPP_OPTION (pfile, wide_charset);
703 const char *default_wcset;
704
705 bool be = CPP_OPTION (pfile, bytes_big_endian);
706
707 if (CPP_OPTION (pfile, wchar_precision) >= 32)
708 default_wcset = be ? "UTF-32BE" : "UTF-32LE";
709 else if (CPP_OPTION (pfile, wchar_precision) >= 16)
710 default_wcset = be ? "UTF-16BE" : "UTF-16LE";
711 else
712 /* This effectively means that wide strings are not supported,
713 so don't do any conversion at all. */
714 default_wcset = SOURCE_CHARSET;
715
716 if (!ncset)
717 ncset = SOURCE_CHARSET;
718 if (!wcset)
719 wcset = default_wcset;
720
721 pfile->narrow_cset_desc = init_iconv_desc (pfile, ncset, SOURCE_CHARSET);
722 pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
723 pfile->utf8_cset_desc = init_iconv_desc (pfile, "UTF-8", SOURCE_CHARSET);
724 pfile->utf8_cset_desc.width = CPP_OPTION (pfile, char_precision);
725 pfile->char16_cset_desc = init_iconv_desc (pfile,
726 be ? "UTF-16BE" : "UTF-16LE",
727 SOURCE_CHARSET);
728 pfile->char16_cset_desc.width = 16;
729 pfile->char32_cset_desc = init_iconv_desc (pfile,
730 be ? "UTF-32BE" : "UTF-32LE",
731 SOURCE_CHARSET);
732 pfile->char32_cset_desc.width = 32;
733 pfile->wide_cset_desc = init_iconv_desc (pfile, wcset, SOURCE_CHARSET);
734 pfile->wide_cset_desc.width = CPP_OPTION (pfile, wchar_precision);
735 }
736
737 /* Destroy iconv(3) descriptors set up by cpp_init_iconv, if necessary. */
738 void
739 _cpp_destroy_iconv (cpp_reader *pfile)
740 {
741 if (HAVE_ICONV)
742 {
743 if (pfile->narrow_cset_desc.func == convert_using_iconv)
744 iconv_close (pfile->narrow_cset_desc.cd);
745 if (pfile->utf8_cset_desc.func == convert_using_iconv)
746 iconv_close (pfile->utf8_cset_desc.cd);
747 if (pfile->char16_cset_desc.func == convert_using_iconv)
748 iconv_close (pfile->char16_cset_desc.cd);
749 if (pfile->char32_cset_desc.func == convert_using_iconv)
750 iconv_close (pfile->char32_cset_desc.cd);
751 if (pfile->wide_cset_desc.func == convert_using_iconv)
752 iconv_close (pfile->wide_cset_desc.cd);
753 }
754 }
755
756 /* Utility routine for use by a full compiler. C is a character taken
757 from the *basic* source character set, encoded in the host's
758 execution encoding. Convert it to (the target's) execution
759 encoding, and return that value.
760
761 Issues an internal error if C's representation in the narrow
762 execution character set fails to be a single-byte value (C99
763 5.2.1p3: "The representation of each member of the source and
764 execution character sets shall fit in a byte.") May also issue an
765 internal error if C fails to be a member of the basic source
766 character set (testing this exactly is too hard, especially when
767 the host character set is EBCDIC). */
768 cppchar_t
769 cpp_host_to_exec_charset (cpp_reader *pfile, cppchar_t c)
770 {
771 uchar sbuf[1];
772 struct _cpp_strbuf tbuf;
773
774 /* This test is merely an approximation, but it suffices to catch
775 the most important thing, which is that we don't get handed a
776 character outside the unibyte range of the host character set. */
777 if (c > LAST_POSSIBLY_BASIC_SOURCE_CHAR)
778 {
779 cpp_error (pfile, CPP_DL_ICE,
780 "character 0x%lx is not in the basic source character set\n",
781 (unsigned long)c);
782 return 0;
783 }
784
785 /* Being a character in the unibyte range of the host character set,
786 we can safely splat it into a one-byte buffer and trust that that
787 is a well-formed string. */
788 sbuf[0] = c;
789
790 /* This should never need to reallocate, but just in case... */
791 tbuf.asize = 1;
792 tbuf.text = XNEWVEC (uchar, tbuf.asize);
793 tbuf.len = 0;
794
795 if (!APPLY_CONVERSION (pfile->narrow_cset_desc, sbuf, 1, &tbuf))
796 {
797 cpp_errno (pfile, CPP_DL_ICE, "converting to execution character set");
798 return 0;
799 }
800 if (tbuf.len != 1)
801 {
802 cpp_error (pfile, CPP_DL_ICE,
803 "character 0x%lx is not unibyte in execution character set",
804 (unsigned long)c);
805 return 0;
806 }
807 c = tbuf.text[0];
808 free(tbuf.text);
809 return c;
810 }
811
812 \f
813
814 /* Utility routine that computes a mask of the form 0000...111... with
815 WIDTH 1-bits. */
816 static inline size_t
817 width_to_mask (size_t width)
818 {
819 width = MIN (width, BITS_PER_CPPCHAR_T);
820 if (width >= CHAR_BIT * sizeof (size_t))
821 return ~(size_t) 0;
822 else
823 return ((size_t) 1 << width) - 1;
824 }
825
826 /* A large table of unicode character information. */
827 enum {
828 /* Valid in a C99 identifier? */
829 C99 = 1,
830 /* Valid in a C99 identifier, but not as the first character? */
831 N99 = 2,
832 /* Valid in a C++ identifier? */
833 CXX = 4,
834 /* Valid in a C11/C++11 identifier? */
835 C11 = 8,
836 /* Valid in a C11/C++11 identifier, but not as the first character? */
837 N11 = 16,
838 /* NFC representation is not valid in an identifier? */
839 CID = 32,
840 /* Might be valid NFC form? */
841 NFC = 64,
842 /* Might be valid NFKC form? */
843 NKC = 128,
844 /* Certain preceding characters might make it not valid NFC/NKFC form? */
845 CTX = 256
846 };
847
848 struct ucnrange {
849 /* Bitmap of flags above. */
850 unsigned short flags;
851 /* Combining class of the character. */
852 unsigned char combine;
853 /* Last character in the range described by this entry. */
854 unsigned int end;
855 };
856 #include "ucnid.h"
857
858 /* Returns 1 if C is valid in an identifier, 2 if C is valid except at
859 the start of an identifier, and 0 if C is not valid in an
860 identifier. We assume C has already gone through the checks of
861 _cpp_valid_ucn. Also update NST for C if returning nonzero. The
862 algorithm is a simple binary search on the table defined in
863 ucnid.h. */
864
865 static int
866 ucn_valid_in_identifier (cpp_reader *pfile, cppchar_t c,
867 struct normalize_state *nst)
868 {
869 int mn, mx, md;
870 unsigned short valid_flags, invalid_start_flags;
871
872 if (c > 0x10FFFF)
873 return 0;
874
875 mn = 0;
876 mx = ARRAY_SIZE (ucnranges) - 1;
877 while (mx != mn)
878 {
879 md = (mn + mx) / 2;
880 if (c <= ucnranges[md].end)
881 mx = md;
882 else
883 mn = md + 1;
884 }
885
886 /* When -pedantic, we require the character to have been listed by
887 the standard for the current language. Otherwise, we accept the
888 union of the acceptable sets for all supported language versions. */
889 valid_flags = C99 | CXX | C11;
890 if (CPP_PEDANTIC (pfile))
891 {
892 if (CPP_OPTION (pfile, c11_identifiers))
893 valid_flags = C11;
894 else if (CPP_OPTION (pfile, c99))
895 valid_flags = C99;
896 else if (CPP_OPTION (pfile, cplusplus))
897 valid_flags = CXX;
898 }
899 if (! (ucnranges[mn].flags & valid_flags))
900 return 0;
901 if (CPP_OPTION (pfile, c11_identifiers))
902 invalid_start_flags = N11;
903 else if (CPP_OPTION (pfile, c99))
904 invalid_start_flags = N99;
905 else
906 invalid_start_flags = 0;
907
908 /* Update NST. */
909 if (ucnranges[mn].combine != 0 && ucnranges[mn].combine < nst->prev_class)
910 nst->level = normalized_none;
911 else if (ucnranges[mn].flags & CTX)
912 {
913 bool safe;
914 cppchar_t p = nst->previous;
915
916 /* For Hangul, characters in the range AC00-D7A3 are NFC/NFKC,
917 and are combined algorithmically from a sequence of the form
918 1100-1112 1161-1175 11A8-11C2
919 (if the third is not present, it is treated as 11A7, which is not
920 really a valid character).
921 Unfortunately, C99 allows (only) the NFC form, but C++ allows
922 only the combining characters. */
923 if (c >= 0x1161 && c <= 0x1175)
924 safe = p < 0x1100 || p > 0x1112;
925 else if (c >= 0x11A8 && c <= 0x11C2)
926 safe = (p < 0xAC00 || p > 0xD7A3 || (p - 0xAC00) % 28 != 0);
927 else
928 safe = check_nfc (pfile, c, p);
929 if (!safe)
930 {
931 if ((c >= 0x1161 && c <= 0x1175) || (c >= 0x11A8 && c <= 0x11C2))
932 nst->level = MAX (nst->level, normalized_identifier_C);
933 else
934 nst->level = normalized_none;
935 }
936 }
937 else if (ucnranges[mn].flags & NKC)
938 ;
939 else if (ucnranges[mn].flags & NFC)
940 nst->level = MAX (nst->level, normalized_C);
941 else if (ucnranges[mn].flags & CID)
942 nst->level = MAX (nst->level, normalized_identifier_C);
943 else
944 nst->level = normalized_none;
945 if (ucnranges[mn].combine == 0)
946 nst->previous = c;
947 nst->prev_class = ucnranges[mn].combine;
948
949 /* In C99, UCN digits may not begin identifiers. In C11 and C++11,
950 UCN combining characters may not begin identifiers. */
951 if (ucnranges[mn].flags & invalid_start_flags)
952 return 2;
953
954 return 1;
955 }
956
957 /* [lex.charset]: The character designated by the universal character
958 name \UNNNNNNNN is that character whose character short name in
959 ISO/IEC 10646 is NNNNNNNN; the character designated by the
960 universal character name \uNNNN is that character whose character
961 short name in ISO/IEC 10646 is 0000NNNN. If the hexadecimal value
962 for a universal character name corresponds to a surrogate code point
963 (in the range 0xD800-0xDFFF, inclusive), the program is ill-formed.
964 Additionally, if the hexadecimal value for a universal-character-name
965 outside a character or string literal corresponds to a control character
966 (in either of the ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a
967 character in the basic source character set, the program is ill-formed.
968
969 C99 6.4.3: A universal character name shall not specify a character
970 whose short identifier is less than 00A0 other than 0024 ($), 0040 (@),
971 or 0060 (`), nor one in the range D800 through DFFF inclusive.
972
973 *PSTR must be preceded by "\u" or "\U"; it is assumed that the
974 buffer end is delimited by a non-hex digit. Returns zero if the
975 UCN has not been consumed.
976
977 Otherwise the nonzero value of the UCN, whether valid or invalid,
978 is returned. Diagnostics are emitted for invalid values. PSTR
979 is updated to point one beyond the UCN, or to the syntactically
980 invalid character.
981
982 IDENTIFIER_POS is 0 when not in an identifier, 1 for the start of
983 an identifier, or 2 otherwise. */
984
985 cppchar_t
986 _cpp_valid_ucn (cpp_reader *pfile, const uchar **pstr,
987 const uchar *limit, int identifier_pos,
988 struct normalize_state *nst)
989 {
990 cppchar_t result, c;
991 unsigned int length;
992 const uchar *str = *pstr;
993 const uchar *base = str - 2;
994
995 if (!CPP_OPTION (pfile, cplusplus) && !CPP_OPTION (pfile, c99))
996 cpp_error (pfile, CPP_DL_WARNING,
997 "universal character names are only valid in C++ and C99");
998 else if (CPP_WTRADITIONAL (pfile) && identifier_pos == 0)
999 cpp_warning (pfile, CPP_W_TRADITIONAL,
1000 "the meaning of '\\%c' is different in traditional C",
1001 (int) str[-1]);
1002
1003 if (str[-1] == 'u')
1004 length = 4;
1005 else if (str[-1] == 'U')
1006 length = 8;
1007 else
1008 {
1009 cpp_error (pfile, CPP_DL_ICE, "In _cpp_valid_ucn but not a UCN");
1010 length = 4;
1011 }
1012
1013 result = 0;
1014 do
1015 {
1016 c = *str;
1017 if (!ISXDIGIT (c))
1018 break;
1019 str++;
1020 result = (result << 4) + hex_value (c);
1021 }
1022 while (--length && str < limit);
1023
1024 /* Partial UCNs are not valid in strings, but decompose into
1025 multiple tokens in identifiers, so we can't give a helpful
1026 error message in that case. */
1027 if (length && identifier_pos)
1028 return 0;
1029
1030 *pstr = str;
1031 if (length)
1032 {
1033 cpp_error (pfile, CPP_DL_ERROR,
1034 "incomplete universal character name %.*s",
1035 (int) (str - base), base);
1036 result = 1;
1037 }
1038 /* The C99 standard permits $, @ and ` to be specified as UCNs. We use
1039 hex escapes so that this also works with EBCDIC hosts.
1040 C++0x permits everything below 0xa0 within literals;
1041 ucn_valid_in_identifier will complain about identifiers. */
1042 else if ((result < 0xa0
1043 && !CPP_OPTION (pfile, cplusplus)
1044 && (result != 0x24 && result != 0x40 && result != 0x60))
1045 || (result & 0x80000000)
1046 || (result >= 0xD800 && result <= 0xDFFF))
1047 {
1048 cpp_error (pfile, CPP_DL_ERROR,
1049 "%.*s is not a valid universal character",
1050 (int) (str - base), base);
1051 result = 1;
1052 }
1053 else if (identifier_pos && result == 0x24
1054 && CPP_OPTION (pfile, dollars_in_ident))
1055 {
1056 if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1057 {
1058 CPP_OPTION (pfile, warn_dollars) = 0;
1059 cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
1060 }
1061 NORMALIZE_STATE_UPDATE_IDNUM (nst, result);
1062 }
1063 else if (identifier_pos)
1064 {
1065 int validity = ucn_valid_in_identifier (pfile, result, nst);
1066
1067 if (validity == 0)
1068 cpp_error (pfile, CPP_DL_ERROR,
1069 "universal character %.*s is not valid in an identifier",
1070 (int) (str - base), base);
1071 else if (validity == 2 && identifier_pos == 1)
1072 cpp_error (pfile, CPP_DL_ERROR,
1073 "universal character %.*s is not valid at the start of an identifier",
1074 (int) (str - base), base);
1075 }
1076
1077 if (result == 0)
1078 result = 1;
1079
1080 return result;
1081 }
1082
1083 /* Convert an UCN, pointed to by FROM, to UTF-8 encoding, then translate
1084 it to the execution character set and write the result into TBUF.
1085 An advanced pointer is returned. Issues all relevant diagnostics. */
1086 static const uchar *
1087 convert_ucn (cpp_reader *pfile, const uchar *from, const uchar *limit,
1088 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1089 {
1090 cppchar_t ucn;
1091 uchar buf[6];
1092 uchar *bufp = buf;
1093 size_t bytesleft = 6;
1094 int rval;
1095 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
1096
1097 from++; /* Skip u/U. */
1098 ucn = _cpp_valid_ucn (pfile, &from, limit, 0, &nst);
1099
1100 rval = one_cppchar_to_utf8 (ucn, &bufp, &bytesleft);
1101 if (rval)
1102 {
1103 errno = rval;
1104 cpp_errno (pfile, CPP_DL_ERROR,
1105 "converting UCN to source character set");
1106 }
1107 else if (!APPLY_CONVERSION (cvt, buf, 6 - bytesleft, tbuf))
1108 cpp_errno (pfile, CPP_DL_ERROR,
1109 "converting UCN to execution character set");
1110
1111 return from;
1112 }
1113
1114 /* Subroutine of convert_hex and convert_oct. N is the representation
1115 in the execution character set of a numeric escape; write it into the
1116 string buffer TBUF and update the end-of-string pointer therein. WIDE
1117 is true if it's a wide string that's being assembled in TBUF. This
1118 function issues no diagnostics and never fails. */
1119 static void
1120 emit_numeric_escape (cpp_reader *pfile, cppchar_t n,
1121 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1122 {
1123 size_t width = cvt.width;
1124
1125 if (width != CPP_OPTION (pfile, char_precision))
1126 {
1127 /* We have to render this into the target byte order, which may not
1128 be our byte order. */
1129 bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1130 size_t cwidth = CPP_OPTION (pfile, char_precision);
1131 size_t cmask = width_to_mask (cwidth);
1132 size_t nbwc = width / cwidth;
1133 size_t i;
1134 size_t off = tbuf->len;
1135 cppchar_t c;
1136
1137 if (tbuf->len + nbwc > tbuf->asize)
1138 {
1139 tbuf->asize += OUTBUF_BLOCK_SIZE;
1140 tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1141 }
1142
1143 for (i = 0; i < nbwc; i++)
1144 {
1145 c = n & cmask;
1146 n >>= cwidth;
1147 tbuf->text[off + (bigend ? nbwc - i - 1 : i)] = c;
1148 }
1149 tbuf->len += nbwc;
1150 }
1151 else
1152 {
1153 /* Note: this code does not handle the case where the target
1154 and host have a different number of bits in a byte. */
1155 if (tbuf->len + 1 > tbuf->asize)
1156 {
1157 tbuf->asize += OUTBUF_BLOCK_SIZE;
1158 tbuf->text = XRESIZEVEC (uchar, tbuf->text, tbuf->asize);
1159 }
1160 tbuf->text[tbuf->len++] = n;
1161 }
1162 }
1163
1164 /* Convert a hexadecimal escape, pointed to by FROM, to the execution
1165 character set and write it into the string buffer TBUF. Returns an
1166 advanced pointer, and issues diagnostics as necessary.
1167 No character set translation occurs; this routine always produces the
1168 execution-set character with numeric value equal to the given hex
1169 number. You can, e.g. generate surrogate pairs this way. */
1170 static const uchar *
1171 convert_hex (cpp_reader *pfile, const uchar *from, const uchar *limit,
1172 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1173 {
1174 cppchar_t c, n = 0, overflow = 0;
1175 int digits_found = 0;
1176 size_t width = cvt.width;
1177 size_t mask = width_to_mask (width);
1178
1179 if (CPP_WTRADITIONAL (pfile))
1180 cpp_warning (pfile, CPP_W_TRADITIONAL,
1181 "the meaning of '\\x' is different in traditional C");
1182
1183 from++; /* Skip 'x'. */
1184 while (from < limit)
1185 {
1186 c = *from;
1187 if (! hex_p (c))
1188 break;
1189 from++;
1190 overflow |= n ^ (n << 4 >> 4);
1191 n = (n << 4) + hex_value (c);
1192 digits_found = 1;
1193 }
1194
1195 if (!digits_found)
1196 {
1197 cpp_error (pfile, CPP_DL_ERROR,
1198 "\\x used with no following hex digits");
1199 return from;
1200 }
1201
1202 if (overflow | (n != (n & mask)))
1203 {
1204 cpp_error (pfile, CPP_DL_PEDWARN,
1205 "hex escape sequence out of range");
1206 n &= mask;
1207 }
1208
1209 emit_numeric_escape (pfile, n, tbuf, cvt);
1210
1211 return from;
1212 }
1213
1214 /* Convert an octal escape, pointed to by FROM, to the execution
1215 character set and write it into the string buffer TBUF. Returns an
1216 advanced pointer, and issues diagnostics as necessary.
1217 No character set translation occurs; this routine always produces the
1218 execution-set character with numeric value equal to the given octal
1219 number. */
1220 static const uchar *
1221 convert_oct (cpp_reader *pfile, const uchar *from, const uchar *limit,
1222 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1223 {
1224 size_t count = 0;
1225 cppchar_t c, n = 0;
1226 size_t width = cvt.width;
1227 size_t mask = width_to_mask (width);
1228 bool overflow = false;
1229
1230 while (from < limit && count++ < 3)
1231 {
1232 c = *from;
1233 if (c < '0' || c > '7')
1234 break;
1235 from++;
1236 overflow |= n ^ (n << 3 >> 3);
1237 n = (n << 3) + c - '0';
1238 }
1239
1240 if (n != (n & mask))
1241 {
1242 cpp_error (pfile, CPP_DL_PEDWARN,
1243 "octal escape sequence out of range");
1244 n &= mask;
1245 }
1246
1247 emit_numeric_escape (pfile, n, tbuf, cvt);
1248
1249 return from;
1250 }
1251
1252 /* Convert an escape sequence (pointed to by FROM) to its value on
1253 the target, and to the execution character set. Do not scan past
1254 LIMIT. Write the converted value into TBUF. Returns an advanced
1255 pointer. Handles all relevant diagnostics. */
1256 static const uchar *
1257 convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
1258 struct _cpp_strbuf *tbuf, struct cset_converter cvt)
1259 {
1260 /* Values of \a \b \e \f \n \r \t \v respectively. */
1261 #if HOST_CHARSET == HOST_CHARSET_ASCII
1262 static const uchar charconsts[] = { 7, 8, 27, 12, 10, 13, 9, 11 };
1263 #elif HOST_CHARSET == HOST_CHARSET_EBCDIC
1264 static const uchar charconsts[] = { 47, 22, 39, 12, 21, 13, 5, 11 };
1265 #else
1266 #error "unknown host character set"
1267 #endif
1268
1269 uchar c;
1270
1271 c = *from;
1272 switch (c)
1273 {
1274 /* UCNs, hex escapes, and octal escapes are processed separately. */
1275 case 'u': case 'U':
1276 return convert_ucn (pfile, from, limit, tbuf, cvt);
1277
1278 case 'x':
1279 return convert_hex (pfile, from, limit, tbuf, cvt);
1280 break;
1281
1282 case '0': case '1': case '2': case '3':
1283 case '4': case '5': case '6': case '7':
1284 return convert_oct (pfile, from, limit, tbuf, cvt);
1285
1286 /* Various letter escapes. Get the appropriate host-charset
1287 value into C. */
1288 case '\\': case '\'': case '"': case '?': break;
1289
1290 case '(': case '{': case '[': case '%':
1291 /* '\(', etc, can be used at the beginning of a line in a long
1292 string split onto multiple lines with \-newline, to prevent
1293 Emacs or other text editors from getting confused. '\%' can
1294 be used to prevent SCCS from mangling printf format strings. */
1295 if (CPP_PEDANTIC (pfile))
1296 goto unknown;
1297 break;
1298
1299 case 'b': c = charconsts[1]; break;
1300 case 'f': c = charconsts[3]; break;
1301 case 'n': c = charconsts[4]; break;
1302 case 'r': c = charconsts[5]; break;
1303 case 't': c = charconsts[6]; break;
1304 case 'v': c = charconsts[7]; break;
1305
1306 case 'a':
1307 if (CPP_WTRADITIONAL (pfile))
1308 cpp_warning (pfile, CPP_W_TRADITIONAL,
1309 "the meaning of '\\a' is different in traditional C");
1310 c = charconsts[0];
1311 break;
1312
1313 case 'e': case 'E':
1314 if (CPP_PEDANTIC (pfile))
1315 cpp_error (pfile, CPP_DL_PEDWARN,
1316 "non-ISO-standard escape sequence, '\\%c'", (int) c);
1317 c = charconsts[2];
1318 break;
1319
1320 default:
1321 unknown:
1322 if (ISGRAPH (c))
1323 cpp_error (pfile, CPP_DL_PEDWARN,
1324 "unknown escape sequence: '\\%c'", (int) c);
1325 else
1326 {
1327 /* diagnostic.c does not support "%03o". When it does, this
1328 code can use %03o directly in the diagnostic again. */
1329 char buf[32];
1330 sprintf(buf, "%03o", (int) c);
1331 cpp_error (pfile, CPP_DL_PEDWARN,
1332 "unknown escape sequence: '\\%s'", buf);
1333 }
1334 }
1335
1336 /* Now convert what we have to the execution character set. */
1337 if (!APPLY_CONVERSION (cvt, &c, 1, tbuf))
1338 cpp_errno (pfile, CPP_DL_ERROR,
1339 "converting escape sequence to execution character set");
1340
1341 return from + 1;
1342 }
1343 \f
1344 /* TYPE is a token type. The return value is the conversion needed to
1345 convert from source to execution character set for the given type. */
1346 static struct cset_converter
1347 converter_for_type (cpp_reader *pfile, enum cpp_ttype type)
1348 {
1349 switch (type)
1350 {
1351 default:
1352 return pfile->narrow_cset_desc;
1353 case CPP_UTF8STRING:
1354 return pfile->utf8_cset_desc;
1355 case CPP_CHAR16:
1356 case CPP_STRING16:
1357 return pfile->char16_cset_desc;
1358 case CPP_CHAR32:
1359 case CPP_STRING32:
1360 return pfile->char32_cset_desc;
1361 case CPP_WCHAR:
1362 case CPP_WSTRING:
1363 return pfile->wide_cset_desc;
1364 }
1365 }
1366
1367 /* FROM is an array of cpp_string structures of length COUNT. These
1368 are to be converted from the source to the execution character set,
1369 escape sequences translated, and finally all are to be
1370 concatenated. WIDE indicates whether or not to produce a wide
1371 string. The result is written into TO. Returns true for success,
1372 false for failure. */
1373 bool
1374 cpp_interpret_string (cpp_reader *pfile, const cpp_string *from, size_t count,
1375 cpp_string *to, enum cpp_ttype type)
1376 {
1377 struct _cpp_strbuf tbuf;
1378 const uchar *p, *base, *limit;
1379 size_t i;
1380 struct cset_converter cvt = converter_for_type (pfile, type);
1381
1382 tbuf.asize = MAX (OUTBUF_BLOCK_SIZE, from->len);
1383 tbuf.text = XNEWVEC (uchar, tbuf.asize);
1384 tbuf.len = 0;
1385
1386 for (i = 0; i < count; i++)
1387 {
1388 p = from[i].text;
1389 if (*p == 'u')
1390 {
1391 if (*++p == '8')
1392 p++;
1393 }
1394 else if (*p == 'L' || *p == 'U') p++;
1395 if (*p == 'R')
1396 {
1397 const uchar *prefix;
1398
1399 /* Skip over 'R"'. */
1400 p += 2;
1401 prefix = p;
1402 while (*p != '(')
1403 p++;
1404 p++;
1405 limit = from[i].text + from[i].len;
1406 if (limit >= p + (p - prefix) + 1)
1407 limit -= (p - prefix) + 1;
1408
1409 /* Raw strings are all normal characters; these can be fed
1410 directly to convert_cset. */
1411 if (!APPLY_CONVERSION (cvt, p, limit - p, &tbuf))
1412 goto fail;
1413
1414 continue;
1415 }
1416
1417 p++; /* Skip leading quote. */
1418 limit = from[i].text + from[i].len - 1; /* Skip trailing quote. */
1419
1420 for (;;)
1421 {
1422 base = p;
1423 while (p < limit && *p != '\\')
1424 p++;
1425 if (p > base)
1426 {
1427 /* We have a run of normal characters; these can be fed
1428 directly to convert_cset. */
1429 if (!APPLY_CONVERSION (cvt, base, p - base, &tbuf))
1430 goto fail;
1431 }
1432 if (p == limit)
1433 break;
1434
1435 p = convert_escape (pfile, p + 1, limit, &tbuf, cvt);
1436 }
1437 }
1438 /* NUL-terminate the 'to' buffer and translate it to a cpp_string
1439 structure. */
1440 emit_numeric_escape (pfile, 0, &tbuf, cvt);
1441 tbuf.text = XRESIZEVEC (uchar, tbuf.text, tbuf.len);
1442 to->text = tbuf.text;
1443 to->len = tbuf.len;
1444 return true;
1445
1446 fail:
1447 cpp_errno (pfile, CPP_DL_ERROR, "converting to execution character set");
1448 free (tbuf.text);
1449 return false;
1450 }
1451
1452 /* Subroutine of do_line and do_linemarker. Convert escape sequences
1453 in a string, but do not perform character set conversion. */
1454 bool
1455 cpp_interpret_string_notranslate (cpp_reader *pfile, const cpp_string *from,
1456 size_t count, cpp_string *to,
1457 enum cpp_ttype type ATTRIBUTE_UNUSED)
1458 {
1459 struct cset_converter save_narrow_cset_desc = pfile->narrow_cset_desc;
1460 bool retval;
1461
1462 pfile->narrow_cset_desc.func = convert_no_conversion;
1463 pfile->narrow_cset_desc.cd = (iconv_t) -1;
1464 pfile->narrow_cset_desc.width = CPP_OPTION (pfile, char_precision);
1465
1466 retval = cpp_interpret_string (pfile, from, count, to, CPP_STRING);
1467
1468 pfile->narrow_cset_desc = save_narrow_cset_desc;
1469 return retval;
1470 }
1471
1472 \f
1473 /* Subroutine of cpp_interpret_charconst which performs the conversion
1474 to a number, for narrow strings. STR is the string structure returned
1475 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1476 cpp_interpret_charconst. */
1477 static cppchar_t
1478 narrow_str_to_charconst (cpp_reader *pfile, cpp_string str,
1479 unsigned int *pchars_seen, int *unsignedp)
1480 {
1481 size_t width = CPP_OPTION (pfile, char_precision);
1482 size_t max_chars = CPP_OPTION (pfile, int_precision) / width;
1483 size_t mask = width_to_mask (width);
1484 size_t i;
1485 cppchar_t result, c;
1486 bool unsigned_p;
1487
1488 /* The value of a multi-character character constant, or a
1489 single-character character constant whose representation in the
1490 execution character set is more than one byte long, is
1491 implementation defined. This implementation defines it to be the
1492 number formed by interpreting the byte sequence in memory as a
1493 big-endian binary number. If overflow occurs, the high bytes are
1494 lost, and a warning is issued.
1495
1496 We don't want to process the NUL terminator handed back by
1497 cpp_interpret_string. */
1498 result = 0;
1499 for (i = 0; i < str.len - 1; i++)
1500 {
1501 c = str.text[i] & mask;
1502 if (width < BITS_PER_CPPCHAR_T)
1503 result = (result << width) | c;
1504 else
1505 result = c;
1506 }
1507
1508 if (i > max_chars)
1509 {
1510 i = max_chars;
1511 cpp_error (pfile, CPP_DL_WARNING,
1512 "character constant too long for its type");
1513 }
1514 else if (i > 1 && CPP_OPTION (pfile, warn_multichar))
1515 cpp_warning (pfile, CPP_W_MULTICHAR, "multi-character character constant");
1516
1517 /* Multichar constants are of type int and therefore signed. */
1518 if (i > 1)
1519 unsigned_p = 0;
1520 else
1521 unsigned_p = CPP_OPTION (pfile, unsigned_char);
1522
1523 /* Truncate the constant to its natural width, and simultaneously
1524 sign- or zero-extend to the full width of cppchar_t.
1525 For single-character constants, the value is WIDTH bits wide.
1526 For multi-character constants, the value is INT_PRECISION bits wide. */
1527 if (i > 1)
1528 width = CPP_OPTION (pfile, int_precision);
1529 if (width < BITS_PER_CPPCHAR_T)
1530 {
1531 mask = ((cppchar_t) 1 << width) - 1;
1532 if (unsigned_p || !(result & (1 << (width - 1))))
1533 result &= mask;
1534 else
1535 result |= ~mask;
1536 }
1537 *pchars_seen = i;
1538 *unsignedp = unsigned_p;
1539 return result;
1540 }
1541
1542 /* Subroutine of cpp_interpret_charconst which performs the conversion
1543 to a number, for wide strings. STR is the string structure returned
1544 by cpp_interpret_string. PCHARS_SEEN and UNSIGNEDP are as for
1545 cpp_interpret_charconst. TYPE is the token type. */
1546 static cppchar_t
1547 wide_str_to_charconst (cpp_reader *pfile, cpp_string str,
1548 unsigned int *pchars_seen, int *unsignedp,
1549 enum cpp_ttype type)
1550 {
1551 bool bigend = CPP_OPTION (pfile, bytes_big_endian);
1552 size_t width = converter_for_type (pfile, type).width;
1553 size_t cwidth = CPP_OPTION (pfile, char_precision);
1554 size_t mask = width_to_mask (width);
1555 size_t cmask = width_to_mask (cwidth);
1556 size_t nbwc = width / cwidth;
1557 size_t off, i;
1558 cppchar_t result = 0, c;
1559
1560 /* This is finicky because the string is in the target's byte order,
1561 which may not be our byte order. Only the last character, ignoring
1562 the NUL terminator, is relevant. */
1563 off = str.len - (nbwc * 2);
1564 result = 0;
1565 for (i = 0; i < nbwc; i++)
1566 {
1567 c = bigend ? str.text[off + i] : str.text[off + nbwc - i - 1];
1568 result = (result << cwidth) | (c & cmask);
1569 }
1570
1571 /* Wide character constants have type wchar_t, and a single
1572 character exactly fills a wchar_t, so a multi-character wide
1573 character constant is guaranteed to overflow. */
1574 if (str.len > nbwc * 2)
1575 cpp_error (pfile, CPP_DL_WARNING,
1576 "character constant too long for its type");
1577
1578 /* Truncate the constant to its natural width, and simultaneously
1579 sign- or zero-extend to the full width of cppchar_t. */
1580 if (width < BITS_PER_CPPCHAR_T)
1581 {
1582 if (type == CPP_CHAR16 || type == CPP_CHAR32
1583 || CPP_OPTION (pfile, unsigned_wchar)
1584 || !(result & (1 << (width - 1))))
1585 result &= mask;
1586 else
1587 result |= ~mask;
1588 }
1589
1590 if (type == CPP_CHAR16 || type == CPP_CHAR32
1591 || CPP_OPTION (pfile, unsigned_wchar))
1592 *unsignedp = 1;
1593 else
1594 *unsignedp = 0;
1595
1596 *pchars_seen = 1;
1597 return result;
1598 }
1599
1600 /* Interpret a (possibly wide) character constant in TOKEN.
1601 PCHARS_SEEN points to a variable that is filled in with the number
1602 of characters seen, and UNSIGNEDP to a variable that indicates
1603 whether the result has signed type. */
1604 cppchar_t
1605 cpp_interpret_charconst (cpp_reader *pfile, const cpp_token *token,
1606 unsigned int *pchars_seen, int *unsignedp)
1607 {
1608 cpp_string str = { 0, 0 };
1609 bool wide = (token->type != CPP_CHAR);
1610 cppchar_t result;
1611
1612 /* an empty constant will appear as L'', u'', U'' or '' */
1613 if (token->val.str.len == (size_t) (2 + wide))
1614 {
1615 cpp_error (pfile, CPP_DL_ERROR, "empty character constant");
1616 return 0;
1617 }
1618 else if (!cpp_interpret_string (pfile, &token->val.str, 1, &str, token->type))
1619 return 0;
1620
1621 if (wide)
1622 result = wide_str_to_charconst (pfile, str, pchars_seen, unsignedp,
1623 token->type);
1624 else
1625 result = narrow_str_to_charconst (pfile, str, pchars_seen, unsignedp);
1626
1627 if (str.text != token->val.str.text)
1628 free ((void *)str.text);
1629
1630 return result;
1631 }
1632 \f
1633 /* Convert an identifier denoted by ID and LEN, which might contain
1634 UCN escapes, to the source character set, either UTF-8 or
1635 UTF-EBCDIC. Assumes that the identifier is actually a valid identifier. */
1636 cpp_hashnode *
1637 _cpp_interpret_identifier (cpp_reader *pfile, const uchar *id, size_t len)
1638 {
1639 /* It turns out that a UCN escape always turns into fewer characters
1640 than the escape itself, so we can allocate a temporary in advance. */
1641 uchar * buf = (uchar *) alloca (len + 1);
1642 uchar * bufp = buf;
1643 size_t idp;
1644
1645 for (idp = 0; idp < len; idp++)
1646 if (id[idp] != '\\')
1647 *bufp++ = id[idp];
1648 else
1649 {
1650 unsigned length = id[idp+1] == 'u' ? 4 : 8;
1651 cppchar_t value = 0;
1652 size_t bufleft = len - (bufp - buf);
1653 int rval;
1654
1655 idp += 2;
1656 while (length && idp < len && ISXDIGIT (id[idp]))
1657 {
1658 value = (value << 4) + hex_value (id[idp]);
1659 idp++;
1660 length--;
1661 }
1662 idp--;
1663
1664 /* Special case for EBCDIC: if the identifier contains
1665 a '$' specified using a UCN, translate it to EBCDIC. */
1666 if (value == 0x24)
1667 {
1668 *bufp++ = '$';
1669 continue;
1670 }
1671
1672 rval = one_cppchar_to_utf8 (value, &bufp, &bufleft);
1673 if (rval)
1674 {
1675 errno = rval;
1676 cpp_errno (pfile, CPP_DL_ERROR,
1677 "converting UCN to source character set");
1678 break;
1679 }
1680 }
1681
1682 return CPP_HASHNODE (ht_lookup (pfile->hash_table,
1683 buf, bufp - buf, HT_ALLOC));
1684 }
1685 \f
1686 /* Convert an input buffer (containing the complete contents of one
1687 source file) from INPUT_CHARSET to the source character set. INPUT
1688 points to the input buffer, SIZE is its allocated size, and LEN is
1689 the length of the meaningful data within the buffer. The
1690 translated buffer is returned, *ST_SIZE is set to the length of
1691 the meaningful data within the translated buffer, and *BUFFER_START
1692 is set to the start of the returned buffer. *BUFFER_START may
1693 differ from the return value in the case of a BOM or other ignored
1694 marker information.
1695
1696 INPUT is expected to have been allocated with xmalloc. This
1697 function will either set *BUFFER_START to INPUT, or free it and set
1698 *BUFFER_START to a pointer to another xmalloc-allocated block of
1699 memory. */
1700 uchar *
1701 _cpp_convert_input (cpp_reader *pfile, const char *input_charset,
1702 uchar *input, size_t size, size_t len,
1703 const unsigned char **buffer_start, off_t *st_size)
1704 {
1705 struct cset_converter input_cset;
1706 struct _cpp_strbuf to;
1707 unsigned char *buffer;
1708
1709 input_cset = init_iconv_desc (pfile, SOURCE_CHARSET, input_charset);
1710 if (input_cset.func == convert_no_conversion)
1711 {
1712 to.text = input;
1713 to.asize = size;
1714 to.len = len;
1715 }
1716 else
1717 {
1718 to.asize = MAX (65536, len);
1719 to.text = XNEWVEC (uchar, to.asize);
1720 to.len = 0;
1721
1722 if (!APPLY_CONVERSION (input_cset, input, len, &to))
1723 cpp_error (pfile, CPP_DL_ERROR,
1724 "failure to convert %s to %s",
1725 CPP_OPTION (pfile, input_charset), SOURCE_CHARSET);
1726
1727 free (input);
1728 }
1729
1730 /* Clean up the mess. */
1731 if (input_cset.func == convert_using_iconv)
1732 iconv_close (input_cset.cd);
1733
1734 /* Resize buffer if we allocated substantially too much, or if we
1735 haven't enough space for the \n-terminator or following
1736 15 bytes of padding (used to quiet warnings from valgrind or
1737 Address Sanitizer, when the optimized lexer accesses aligned
1738 16-byte memory chunks, including the bytes after the malloced,
1739 area, and stops lexing on '\n'). */
1740 if (to.len + 4096 < to.asize || to.len + 16 > to.asize)
1741 to.text = XRESIZEVEC (uchar, to.text, to.len + 16);
1742
1743 memset (to.text + to.len, '\0', 16);
1744
1745 /* If the file is using old-school Mac line endings (\r only),
1746 terminate with another \r, not an \n, so that we do not mistake
1747 the \r\n sequence for a single DOS line ending and erroneously
1748 issue the "No newline at end of file" diagnostic. */
1749 if (to.len && to.text[to.len - 1] == '\r')
1750 to.text[to.len] = '\r';
1751 else
1752 to.text[to.len] = '\n';
1753
1754 buffer = to.text;
1755 *st_size = to.len;
1756 #if HOST_CHARSET == HOST_CHARSET_ASCII
1757 /* The HOST_CHARSET test just above ensures that the source charset
1758 is UTF-8. So, ignore a UTF-8 BOM if we see one. Note that
1759 glib'c UTF-8 iconv() provider (as of glibc 2.7) does not ignore a
1760 BOM -- however, even if it did, we would still need this code due
1761 to the 'convert_no_conversion' case. */
1762 if (to.len >= 3 && to.text[0] == 0xef && to.text[1] == 0xbb
1763 && to.text[2] == 0xbf)
1764 {
1765 *st_size -= 3;
1766 buffer += 3;
1767 }
1768 #endif
1769
1770 *buffer_start = to.text;
1771 return buffer;
1772 }
1773
1774 /* Decide on the default encoding to assume for input files. */
1775 const char *
1776 _cpp_default_encoding (void)
1777 {
1778 const char *current_encoding = NULL;
1779
1780 /* We disable this because the default codeset is 7-bit ASCII on
1781 most platforms, and this causes conversion failures on every
1782 file in GCC that happens to have one of the upper 128 characters
1783 in it -- most likely, as part of the name of a contributor.
1784 We should definitely recognize in-band markers of file encoding,
1785 like:
1786 - the appropriate Unicode byte-order mark (FE FF) to recognize
1787 UTF16 and UCS4 (in both big-endian and little-endian flavors)
1788 and UTF8
1789 - a "#i", "#d", "/ *", "//", " #p" or "#p" (for #pragma) to
1790 distinguish ASCII and EBCDIC.
1791 - now we can parse something like "#pragma GCC encoding <xyz>
1792 on the first line, or even Emacs/VIM's mode line tags (there's
1793 a problem here in that VIM uses the last line, and Emacs has
1794 its more elaborate "local variables" convention).
1795 - investigate whether Java has another common convention, which
1796 would be friendly to support.
1797 (Zack Weinberg and Paolo Bonzini, May 20th 2004) */
1798 #if defined (HAVE_LOCALE_H) && defined (HAVE_LANGINFO_CODESET) && 0
1799 setlocale (LC_CTYPE, "");
1800 current_encoding = nl_langinfo (CODESET);
1801 #endif
1802 if (current_encoding == NULL || *current_encoding == '\0')
1803 current_encoding = SOURCE_CHARSET;
1804
1805 return current_encoding;
1806 }