In gcc/objc/: 2011-04-12 Nicola Pero <nicola.pero@meta-innovation.com>
[official-gcc.git] / libcpp / lex.c
blobc9b5c95df6a55e32567e59a6c1f452dc3b064c72
1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2007, 2008, 2009, 2010
3 Free Software Foundation, Inc.
4 Contributed by Per Bothner, 1994-95.
5 Based on CCCP program by Paul Rubin, June 1986
6 Adapted to ANSI C, Richard Stallman, Jan 1987
7 Broken out to separate file, Zack Weinberg, Mar 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 3, or (at your option) any
12 later version.
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
23 #include "config.h"
24 #include "system.h"
25 #include "cpplib.h"
26 #include "internal.h"
28 enum spell_type
30 SPELL_OPERATOR = 0,
31 SPELL_IDENT,
32 SPELL_LITERAL,
33 SPELL_NONE
36 struct token_spelling
38 enum spell_type category;
39 const unsigned char *name;
42 static const unsigned char *const digraph_spellings[] =
43 { UC"%:", UC"%:%:", UC"<:", UC":>", UC"<%", UC"%>" };
45 #define OP(e, s) { SPELL_OPERATOR, UC s },
46 #define TK(e, s) { SPELL_ ## s, UC #e },
47 static const struct token_spelling token_spellings[N_TTYPES] = { TTYPE_TABLE };
48 #undef OP
49 #undef TK
51 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
52 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
54 static void add_line_note (cpp_buffer *, const uchar *, unsigned int);
55 static int skip_line_comment (cpp_reader *);
56 static void skip_whitespace (cpp_reader *, cppchar_t);
57 static void lex_string (cpp_reader *, cpp_token *, const uchar *);
58 static void save_comment (cpp_reader *, cpp_token *, const uchar *, cppchar_t);
59 static void store_comment (cpp_reader *, cpp_token *);
60 static void create_literal (cpp_reader *, cpp_token *, const uchar *,
61 unsigned int, enum cpp_ttype);
62 static bool warn_in_comment (cpp_reader *, _cpp_line_note *);
63 static int name_p (cpp_reader *, const cpp_string *);
64 static tokenrun *next_tokenrun (tokenrun *);
66 static _cpp_buff *new_buff (size_t);
69 /* Utility routine:
71 Compares, the token TOKEN to the NUL-terminated string STRING.
72 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
73 int
74 cpp_ideq (const cpp_token *token, const char *string)
76 if (token->type != CPP_NAME)
77 return 0;
79 return !ustrcmp (NODE_NAME (token->val.node.node), (const uchar *) string);
82 /* Record a note TYPE at byte POS into the current cleaned logical
83 line. */
84 static void
85 add_line_note (cpp_buffer *buffer, const uchar *pos, unsigned int type)
87 if (buffer->notes_used == buffer->notes_cap)
89 buffer->notes_cap = buffer->notes_cap * 2 + 200;
90 buffer->notes = XRESIZEVEC (_cpp_line_note, buffer->notes,
91 buffer->notes_cap);
94 buffer->notes[buffer->notes_used].pos = pos;
95 buffer->notes[buffer->notes_used].type = type;
96 buffer->notes_used++;
100 /* Fast path to find line special characters using optimized character
101 scanning algorithms. Anything complicated falls back to the slow
102 path below. Since this loop is very hot it's worth doing these kinds
103 of optimizations.
105 One of the paths through the ifdefs should provide
107 const uchar *search_line_fast (const uchar *s, const uchar *end);
109 Between S and END, search for \n, \r, \\, ?. Return a pointer to
110 the found character.
112 Note that the last character of the buffer is *always* a newline,
113 as forced by _cpp_convert_input. This fact can be used to avoid
114 explicitly looking for the end of the buffer. */
116 /* Configure gives us an ifdef test. */
117 #ifndef WORDS_BIGENDIAN
118 #define WORDS_BIGENDIAN 0
119 #endif
121 /* We'd like the largest integer that fits into a register. There's nothing
122 in <stdint.h> that gives us that. For most hosts this is unsigned long,
123 but MS decided on an LLP64 model. Thankfully when building with GCC we
124 can get the "real" word size. */
125 #ifdef __GNUC__
126 typedef unsigned int word_type __attribute__((__mode__(__word__)));
127 #else
128 typedef unsigned long word_type;
129 #endif
131 /* The code below is only expecting sizes 4 or 8.
132 Die at compile-time if this expectation is violated. */
133 typedef char check_word_type_size
134 [(sizeof(word_type) == 8 || sizeof(word_type) == 4) * 2 - 1];
136 /* Return X with the first N bytes forced to values that won't match one
137 of the interesting characters. Note that NUL is not interesting. */
139 static inline word_type
140 acc_char_mask_misalign (word_type val, unsigned int n)
142 word_type mask = -1;
143 if (WORDS_BIGENDIAN)
144 mask >>= n * 8;
145 else
146 mask <<= n * 8;
147 return val & mask;
150 /* Return X replicated to all byte positions within WORD_TYPE. */
152 static inline word_type
153 acc_char_replicate (uchar x)
155 word_type ret;
157 ret = (x << 24) | (x << 16) | (x << 8) | x;
158 if (sizeof(word_type) == 8)
159 ret = (ret << 16 << 16) | ret;
160 return ret;
163 /* Return non-zero if some byte of VAL is (probably) C. */
165 static inline word_type
166 acc_char_cmp (word_type val, word_type c)
168 #if defined(__GNUC__) && defined(__alpha__)
169 /* We can get exact results using a compare-bytes instruction.
170 Get (val == c) via (0 >= (val ^ c)). */
171 return __builtin_alpha_cmpbge (0, val ^ c);
172 #else
173 word_type magic = 0x7efefefeU;
174 if (sizeof(word_type) == 8)
175 magic = (magic << 16 << 16) | 0xfefefefeU;
176 magic |= 1;
178 val ^= c;
179 return ((val + magic) ^ ~val) & ~magic;
180 #endif
183 /* Given the result of acc_char_cmp is non-zero, return the index of
184 the found character. If this was a false positive, return -1. */
186 static inline int
187 acc_char_index (word_type cmp ATTRIBUTE_UNUSED,
188 word_type val ATTRIBUTE_UNUSED)
190 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
191 /* The cmpbge instruction sets *bits* of the result corresponding to
192 matches in the bytes with no false positives. */
193 return __builtin_ctzl (cmp);
194 #else
195 unsigned int i;
197 /* ??? It would be nice to force unrolling here,
198 and have all of these constants folded. */
199 for (i = 0; i < sizeof(word_type); ++i)
201 uchar c;
202 if (WORDS_BIGENDIAN)
203 c = (val >> (sizeof(word_type) - i - 1) * 8) & 0xff;
204 else
205 c = (val >> i * 8) & 0xff;
207 if (c == '\n' || c == '\r' || c == '\\' || c == '?')
208 return i;
211 return -1;
212 #endif
215 /* A version of the fast scanner using bit fiddling techniques.
217 For 32-bit words, one would normally perform 16 comparisons and
218 16 branches. With this algorithm one performs 24 arithmetic
219 operations and one branch. Whether this is faster with a 32-bit
220 word size is going to be somewhat system dependent.
222 For 64-bit words, we eliminate twice the number of comparisons
223 and branches without increasing the number of arithmetic operations.
224 It's almost certainly going to be a win with 64-bit word size. */
226 static const uchar * search_line_acc_char (const uchar *, const uchar *)
227 ATTRIBUTE_UNUSED;
229 static const uchar *
230 search_line_acc_char (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
232 const word_type repl_nl = acc_char_replicate ('\n');
233 const word_type repl_cr = acc_char_replicate ('\r');
234 const word_type repl_bs = acc_char_replicate ('\\');
235 const word_type repl_qm = acc_char_replicate ('?');
237 unsigned int misalign;
238 const word_type *p;
239 word_type val, t;
241 /* Align the buffer. Mask out any bytes from before the beginning. */
242 p = (word_type *)((uintptr_t)s & -sizeof(word_type));
243 val = *p;
244 misalign = (uintptr_t)s & (sizeof(word_type) - 1);
245 if (misalign)
246 val = acc_char_mask_misalign (val, misalign);
248 /* Main loop. */
249 while (1)
251 t = acc_char_cmp (val, repl_nl);
252 t |= acc_char_cmp (val, repl_cr);
253 t |= acc_char_cmp (val, repl_bs);
254 t |= acc_char_cmp (val, repl_qm);
256 if (__builtin_expect (t != 0, 0))
258 int i = acc_char_index (t, val);
259 if (i >= 0)
260 return (const uchar *)p + i;
263 val = *++p;
267 /* Disable on Solaris 2/x86 until the following problems can be properly
268 autoconfed:
270 The Solaris 8 assembler cannot assemble SSE2/SSE4.2 insns.
271 The Solaris 9 assembler cannot assemble SSE4.2 insns.
272 Before Solaris 9 Update 6, SSE insns cannot be executed.
273 The Solaris 10+ assembler tags objects with the instruction set
274 extensions used, so SSE4.2 executables cannot run on machines that
275 don't support that extension. */
277 #if (GCC_VERSION >= 4005) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
279 /* Replicated character data to be shared between implementations.
280 Recall that outside of a context with vector support we can't
281 define compatible vector types, therefore these are all defined
282 in terms of raw characters. */
283 static const char repl_chars[4][16] __attribute__((aligned(16))) = {
284 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
285 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
286 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
287 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
288 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
289 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
290 { '?', '?', '?', '?', '?', '?', '?', '?',
291 '?', '?', '?', '?', '?', '?', '?', '?' },
294 /* A version of the fast scanner using MMX vectorized byte compare insns.
296 This uses the PMOVMSKB instruction which was introduced with "MMX2",
297 which was packaged into SSE1; it is also present in the AMD 3dNOW-A
298 extension. Mark the function as using "sse" so that we emit a real
299 "emms" instruction, rather than the 3dNOW "femms" instruction. */
301 static const uchar *
302 #ifndef __SSE__
303 __attribute__((__target__("sse")))
304 #endif
305 search_line_mmx (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
307 typedef char v8qi __attribute__ ((__vector_size__ (8)));
308 typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__));
310 const v8qi repl_nl = *(const v8qi *)repl_chars[0];
311 const v8qi repl_cr = *(const v8qi *)repl_chars[1];
312 const v8qi repl_bs = *(const v8qi *)repl_chars[2];
313 const v8qi repl_qm = *(const v8qi *)repl_chars[3];
315 unsigned int misalign, found, mask;
316 const v8qi *p;
317 v8qi data, t, c;
319 /* Align the source pointer. While MMX doesn't generate unaligned data
320 faults, this allows us to safely scan to the end of the buffer without
321 reading beyond the end of the last page. */
322 misalign = (uintptr_t)s & 7;
323 p = (const v8qi *)((uintptr_t)s & -8);
324 data = *p;
326 /* Create a mask for the bytes that are valid within the first
327 16-byte block. The Idea here is that the AND with the mask
328 within the loop is "free", since we need some AND or TEST
329 insn in order to set the flags for the branch anyway. */
330 mask = -1u << misalign;
332 /* Main loop processing 8 bytes at a time. */
333 goto start;
336 data = *++p;
337 mask = -1;
339 start:
340 t = __builtin_ia32_pcmpeqb(data, repl_nl);
341 c = __builtin_ia32_pcmpeqb(data, repl_cr);
342 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
343 c = __builtin_ia32_pcmpeqb(data, repl_bs);
344 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
345 c = __builtin_ia32_pcmpeqb(data, repl_qm);
346 t = (v8qi) __builtin_ia32_por ((__m64)t, (__m64)c);
347 found = __builtin_ia32_pmovmskb (t);
348 found &= mask;
350 while (!found);
352 __builtin_ia32_emms ();
354 /* FOUND contains 1 in bits for which we matched a relevant
355 character. Conversion to the byte index is trivial. */
356 found = __builtin_ctz(found);
357 return (const uchar *)p + found;
360 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
362 static const uchar *
363 #ifndef __SSE2__
364 __attribute__((__target__("sse2")))
365 #endif
366 search_line_sse2 (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
368 typedef char v16qi __attribute__ ((__vector_size__ (16)));
370 const v16qi repl_nl = *(const v16qi *)repl_chars[0];
371 const v16qi repl_cr = *(const v16qi *)repl_chars[1];
372 const v16qi repl_bs = *(const v16qi *)repl_chars[2];
373 const v16qi repl_qm = *(const v16qi *)repl_chars[3];
375 unsigned int misalign, found, mask;
376 const v16qi *p;
377 v16qi data, t;
379 /* Align the source pointer. */
380 misalign = (uintptr_t)s & 15;
381 p = (const v16qi *)((uintptr_t)s & -16);
382 data = *p;
384 /* Create a mask for the bytes that are valid within the first
385 16-byte block. The Idea here is that the AND with the mask
386 within the loop is "free", since we need some AND or TEST
387 insn in order to set the flags for the branch anyway. */
388 mask = -1u << misalign;
390 /* Main loop processing 16 bytes at a time. */
391 goto start;
394 data = *++p;
395 mask = -1;
397 start:
398 t = __builtin_ia32_pcmpeqb128(data, repl_nl);
399 t |= __builtin_ia32_pcmpeqb128(data, repl_cr);
400 t |= __builtin_ia32_pcmpeqb128(data, repl_bs);
401 t |= __builtin_ia32_pcmpeqb128(data, repl_qm);
402 found = __builtin_ia32_pmovmskb128 (t);
403 found &= mask;
405 while (!found);
407 /* FOUND contains 1 in bits for which we matched a relevant
408 character. Conversion to the byte index is trivial. */
409 found = __builtin_ctz(found);
410 return (const uchar *)p + found;
413 #ifdef HAVE_SSE4
414 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
416 static const uchar *
417 #ifndef __SSE4_2__
418 __attribute__((__target__("sse4.2")))
419 #endif
420 search_line_sse42 (const uchar *s, const uchar *end)
422 typedef char v16qi __attribute__ ((__vector_size__ (16)));
423 static const v16qi search = { '\n', '\r', '?', '\\' };
425 uintptr_t si = (uintptr_t)s;
426 uintptr_t index;
428 /* Check for unaligned input. */
429 if (si & 15)
431 if (__builtin_expect (end - s < 16, 0)
432 && __builtin_expect ((si & 0xfff) > 0xff0, 0))
434 /* There are less than 16 bytes left in the buffer, and less
435 than 16 bytes left on the page. Reading 16 bytes at this
436 point might generate a spurious page fault. Defer to the
437 SSE2 implementation, which already handles alignment. */
438 return search_line_sse2 (s, end);
441 /* ??? The builtin doesn't understand that the PCMPESTRI read from
442 memory need not be aligned. */
443 __asm ("%vpcmpestri $0, (%1), %2"
444 : "=c"(index) : "r"(s), "x"(search), "a"(4), "d"(16));
445 if (__builtin_expect (index < 16, 0))
446 goto found;
448 /* Advance the pointer to an aligned address. We will re-scan a
449 few bytes, but we no longer need care for reading past the
450 end of a page, since we're guaranteed a match. */
451 s = (const uchar *)((si + 16) & -16);
454 /* Main loop, processing 16 bytes at a time. By doing the whole loop
455 in inline assembly, we can make proper use of the flags set. */
456 __asm ( "sub $16, %1\n"
457 " .balign 16\n"
458 "0: add $16, %1\n"
459 " %vpcmpestri $0, (%1), %2\n"
460 " jnc 0b"
461 : "=&c"(index), "+r"(s)
462 : "x"(search), "a"(4), "d"(16));
464 found:
465 return s + index;
468 #else
469 /* Work around out-dated assemblers without sse4 support. */
470 #define search_line_sse42 search_line_sse2
471 #endif
473 /* Check the CPU capabilities. */
475 #include "../gcc/config/i386/cpuid.h"
477 typedef const uchar * (*search_line_fast_type) (const uchar *, const uchar *);
478 static search_line_fast_type search_line_fast;
480 static void __attribute__((constructor))
481 init_vectorized_lexer (void)
483 unsigned dummy, ecx = 0, edx = 0;
484 search_line_fast_type impl = search_line_acc_char;
485 int minimum = 0;
487 #if defined(__SSE4_2__)
488 minimum = 3;
489 #elif defined(__SSE2__)
490 minimum = 2;
491 #elif defined(__SSE__) || defined(__3dNOW_A__)
492 minimum = 1;
493 #endif
495 if (minimum == 3)
496 impl = search_line_sse42;
497 else if (__get_cpuid (1, &dummy, &dummy, &ecx, &edx) || minimum == 2)
499 if (minimum == 3 || (ecx & bit_SSE4_2))
500 impl = search_line_sse42;
501 else if (minimum == 2 || (edx & bit_SSE2))
502 impl = search_line_sse2;
503 else if (minimum == 1 || (edx & bit_SSE))
504 impl = search_line_mmx;
506 else if (__get_cpuid (0x80000001, &dummy, &dummy, &dummy, &edx))
508 if (minimum == 1 || edx & bit_3DNOWP)
509 impl = search_line_mmx;
512 search_line_fast = impl;
515 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__)
517 /* A vection of the fast scanner using AltiVec vectorized byte compares. */
518 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
519 so we can't compile this function without -maltivec on the command line
520 (or implied by some other switch). */
522 static const uchar *
523 search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED)
525 typedef __attribute__((altivec(vector))) unsigned char vc;
527 const vc repl_nl = {
528 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
529 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
531 const vc repl_cr = {
532 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
533 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
535 const vc repl_bs = {
536 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
537 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
539 const vc repl_qm = {
540 '?', '?', '?', '?', '?', '?', '?', '?',
541 '?', '?', '?', '?', '?', '?', '?', '?',
543 const vc ones = {
544 -1, -1, -1, -1, -1, -1, -1, -1,
545 -1, -1, -1, -1, -1, -1, -1, -1,
547 const vc zero = { 0 };
549 vc data, mask, t;
551 /* Altivec loads automatically mask addresses with -16. This lets us
552 issue the first load as early as possible. */
553 data = __builtin_vec_ld(0, (const vc *)s);
555 /* Discard bytes before the beginning of the buffer. Do this by
556 beginning with all ones and shifting in zeros according to the
557 mis-alignment. The LVSR instruction pulls the exact shift we
558 want from the address. */
559 mask = __builtin_vec_lvsr(0, s);
560 mask = __builtin_vec_perm(zero, ones, mask);
561 data &= mask;
563 /* While altivec loads mask addresses, we still need to align S so
564 that the offset we compute at the end is correct. */
565 s = (const uchar *)((uintptr_t)s & -16);
567 /* Main loop processing 16 bytes at a time. */
568 goto start;
571 vc m_nl, m_cr, m_bs, m_qm;
573 s += 16;
574 data = __builtin_vec_ld(0, (const vc *)s);
576 start:
577 m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl);
578 m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr);
579 m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs);
580 m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm);
581 t = (m_nl | m_cr) | (m_bs | m_qm);
583 /* T now contains 0xff in bytes for which we matched one of the relevant
584 characters. We want to exit the loop if any byte in T is non-zero.
585 Below is the expansion of vec_any_ne(t, zero). */
587 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero));
590 #define N (sizeof(vc) / sizeof(long))
592 typedef char check_count[(N == 2 || N == 4) * 2 - 1];
593 union {
594 vc v;
595 unsigned long l[N];
596 } u;
597 unsigned long l, i = 0;
599 u.v = t;
601 /* Find the first word of T that is non-zero. */
602 switch (N)
604 case 4:
605 l = u.l[i++];
606 if (l != 0)
607 break;
608 s += sizeof(unsigned long);
609 l = u.l[i++];
610 if (l != 0)
611 break;
612 s += sizeof(unsigned long);
613 case 2:
614 l = u.l[i++];
615 if (l != 0)
616 break;
617 s += sizeof(unsigned long);
618 l = u.l[i];
621 /* L now contains 0xff in bytes for which we matched one of the
622 relevant characters. We can find the byte index by finding
623 its bit index and dividing by 8. */
624 l = __builtin_clzl(l) >> 3;
625 return s + l;
627 #undef N
631 #else
633 /* We only have one accellerated alternative. Use a direct call so that
634 we encourage inlining. */
636 #define search_line_fast search_line_acc_char
638 #endif
640 /* Returns with a logical line that contains no escaped newlines or
641 trigraphs. This is a time-critical inner loop. */
642 void
643 _cpp_clean_line (cpp_reader *pfile)
645 cpp_buffer *buffer;
646 const uchar *s;
647 uchar c, *d, *p;
649 buffer = pfile->buffer;
650 buffer->cur_note = buffer->notes_used = 0;
651 buffer->cur = buffer->line_base = buffer->next_line;
652 buffer->need_line = false;
653 s = buffer->next_line;
655 if (!buffer->from_stage3)
657 const uchar *pbackslash = NULL;
659 /* Fast path. This is the common case of an un-escaped line with
660 no trigraphs. The primary win here is by not writing any
661 data back to memory until we have to. */
662 while (1)
664 /* Perform an optimized search for \n, \r, \\, ?. */
665 s = search_line_fast (s, buffer->rlimit);
667 c = *s;
668 if (c == '\\')
670 /* Record the location of the backslash and continue. */
671 pbackslash = s++;
673 else if (__builtin_expect (c == '?', 0))
675 if (__builtin_expect (s[1] == '?', false)
676 && _cpp_trigraph_map[s[2]])
678 /* Have a trigraph. We may or may not have to convert
679 it. Add a line note regardless, for -Wtrigraphs. */
680 add_line_note (buffer, s, s[2]);
681 if (CPP_OPTION (pfile, trigraphs))
683 /* We do, and that means we have to switch to the
684 slow path. */
685 d = (uchar *) s;
686 *d = _cpp_trigraph_map[s[2]];
687 s += 2;
688 goto slow_path;
691 /* Not a trigraph. Continue on fast-path. */
692 s++;
694 else
695 break;
698 /* This must be \r or \n. We're either done, or we'll be forced
699 to write back to the buffer and continue on the slow path. */
700 d = (uchar *) s;
702 if (__builtin_expect (s == buffer->rlimit, false))
703 goto done;
705 /* DOS line ending? */
706 if (__builtin_expect (c == '\r', false) && s[1] == '\n')
708 s++;
709 if (s == buffer->rlimit)
710 goto done;
713 if (__builtin_expect (pbackslash == NULL, true))
714 goto done;
716 /* Check for escaped newline. */
717 p = d;
718 while (is_nvspace (p[-1]))
719 p--;
720 if (p - 1 != pbackslash)
721 goto done;
723 /* Have an escaped newline; process it and proceed to
724 the slow path. */
725 add_line_note (buffer, p - 1, p != d ? ' ' : '\\');
726 d = p - 2;
727 buffer->next_line = p - 1;
729 slow_path:
730 while (1)
732 c = *++s;
733 *++d = c;
735 if (c == '\n' || c == '\r')
737 /* Handle DOS line endings. */
738 if (c == '\r' && s != buffer->rlimit && s[1] == '\n')
739 s++;
740 if (s == buffer->rlimit)
741 break;
743 /* Escaped? */
744 p = d;
745 while (p != buffer->next_line && is_nvspace (p[-1]))
746 p--;
747 if (p == buffer->next_line || p[-1] != '\\')
748 break;
750 add_line_note (buffer, p - 1, p != d ? ' ': '\\');
751 d = p - 2;
752 buffer->next_line = p - 1;
754 else if (c == '?' && s[1] == '?' && _cpp_trigraph_map[s[2]])
756 /* Add a note regardless, for the benefit of -Wtrigraphs. */
757 add_line_note (buffer, d, s[2]);
758 if (CPP_OPTION (pfile, trigraphs))
760 *d = _cpp_trigraph_map[s[2]];
761 s += 2;
766 else
768 while (*s != '\n' && *s != '\r')
769 s++;
770 d = (uchar *) s;
772 /* Handle DOS line endings. */
773 if (*s == '\r' && s != buffer->rlimit && s[1] == '\n')
774 s++;
777 done:
778 *d = '\n';
779 /* A sentinel note that should never be processed. */
780 add_line_note (buffer, d + 1, '\n');
781 buffer->next_line = s + 1;
784 /* Return true if the trigraph indicated by NOTE should be warned
785 about in a comment. */
786 static bool
787 warn_in_comment (cpp_reader *pfile, _cpp_line_note *note)
789 const uchar *p;
791 /* Within comments we don't warn about trigraphs, unless the
792 trigraph forms an escaped newline, as that may change
793 behavior. */
794 if (note->type != '/')
795 return false;
797 /* If -trigraphs, then this was an escaped newline iff the next note
798 is coincident. */
799 if (CPP_OPTION (pfile, trigraphs))
800 return note[1].pos == note->pos;
802 /* Otherwise, see if this forms an escaped newline. */
803 p = note->pos + 3;
804 while (is_nvspace (*p))
805 p++;
807 /* There might have been escaped newlines between the trigraph and the
808 newline we found. Hence the position test. */
809 return (*p == '\n' && p < note[1].pos);
812 /* Process the notes created by add_line_note as far as the current
813 location. */
814 void
815 _cpp_process_line_notes (cpp_reader *pfile, int in_comment)
817 cpp_buffer *buffer = pfile->buffer;
819 for (;;)
821 _cpp_line_note *note = &buffer->notes[buffer->cur_note];
822 unsigned int col;
824 if (note->pos > buffer->cur)
825 break;
827 buffer->cur_note++;
828 col = CPP_BUF_COLUMN (buffer, note->pos + 1);
830 if (note->type == '\\' || note->type == ' ')
832 if (note->type == ' ' && !in_comment)
833 cpp_error_with_line (pfile, CPP_DL_WARNING, pfile->line_table->highest_line, col,
834 "backslash and newline separated by space");
836 if (buffer->next_line > buffer->rlimit)
838 cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line, col,
839 "backslash-newline at end of file");
840 /* Prevent "no newline at end of file" warning. */
841 buffer->next_line = buffer->rlimit;
844 buffer->line_base = note->pos;
845 CPP_INCREMENT_LINE (pfile, 0);
847 else if (_cpp_trigraph_map[note->type])
849 if (CPP_OPTION (pfile, warn_trigraphs)
850 && (!in_comment || warn_in_comment (pfile, note)))
852 if (CPP_OPTION (pfile, trigraphs))
853 cpp_warning_with_line (pfile, CPP_W_TRIGRAPHS,
854 pfile->line_table->highest_line, col,
855 "trigraph ??%c converted to %c",
856 note->type,
857 (int) _cpp_trigraph_map[note->type]);
858 else
860 cpp_warning_with_line
861 (pfile, CPP_W_TRIGRAPHS,
862 pfile->line_table->highest_line, col,
863 "trigraph ??%c ignored, use -trigraphs to enable",
864 note->type);
868 else if (note->type == 0)
869 /* Already processed in lex_raw_string. */;
870 else
871 abort ();
875 /* Skip a C-style block comment. We find the end of the comment by
876 seeing if an asterisk is before every '/' we encounter. Returns
877 nonzero if comment terminated by EOF, zero otherwise.
879 Buffer->cur points to the initial asterisk of the comment. */
880 bool
881 _cpp_skip_block_comment (cpp_reader *pfile)
883 cpp_buffer *buffer = pfile->buffer;
884 const uchar *cur = buffer->cur;
885 uchar c;
887 cur++;
888 if (*cur == '/')
889 cur++;
891 for (;;)
893 /* People like decorating comments with '*', so check for '/'
894 instead for efficiency. */
895 c = *cur++;
897 if (c == '/')
899 if (cur[-2] == '*')
900 break;
902 /* Warn about potential nested comments, but not if the '/'
903 comes immediately before the true comment delimiter.
904 Don't bother to get it right across escaped newlines. */
905 if (CPP_OPTION (pfile, warn_comments)
906 && cur[0] == '*' && cur[1] != '/')
908 buffer->cur = cur;
909 cpp_warning_with_line (pfile, CPP_W_COMMENTS,
910 pfile->line_table->highest_line,
911 CPP_BUF_COL (buffer),
912 "\"/*\" within comment");
915 else if (c == '\n')
917 unsigned int cols;
918 buffer->cur = cur - 1;
919 _cpp_process_line_notes (pfile, true);
920 if (buffer->next_line >= buffer->rlimit)
921 return true;
922 _cpp_clean_line (pfile);
924 cols = buffer->next_line - buffer->line_base;
925 CPP_INCREMENT_LINE (pfile, cols);
927 cur = buffer->cur;
931 buffer->cur = cur;
932 _cpp_process_line_notes (pfile, true);
933 return false;
936 /* Skip a C++ line comment, leaving buffer->cur pointing to the
937 terminating newline. Handles escaped newlines. Returns nonzero
938 if a multiline comment. */
939 static int
940 skip_line_comment (cpp_reader *pfile)
942 cpp_buffer *buffer = pfile->buffer;
943 source_location orig_line = pfile->line_table->highest_line;
945 while (*buffer->cur != '\n')
946 buffer->cur++;
948 _cpp_process_line_notes (pfile, true);
949 return orig_line != pfile->line_table->highest_line;
952 /* Skips whitespace, saving the next non-whitespace character. */
953 static void
954 skip_whitespace (cpp_reader *pfile, cppchar_t c)
956 cpp_buffer *buffer = pfile->buffer;
957 bool saw_NUL = false;
961 /* Horizontal space always OK. */
962 if (c == ' ' || c == '\t')
964 /* Just \f \v or \0 left. */
965 else if (c == '\0')
966 saw_NUL = true;
967 else if (pfile->state.in_directive && CPP_PEDANTIC (pfile))
968 cpp_error_with_line (pfile, CPP_DL_PEDWARN, pfile->line_table->highest_line,
969 CPP_BUF_COL (buffer),
970 "%s in preprocessing directive",
971 c == '\f' ? "form feed" : "vertical tab");
973 c = *buffer->cur++;
975 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
976 while (is_nvspace (c));
978 if (saw_NUL)
979 cpp_error (pfile, CPP_DL_WARNING, "null character(s) ignored");
981 buffer->cur--;
984 /* See if the characters of a number token are valid in a name (no
985 '.', '+' or '-'). */
986 static int
987 name_p (cpp_reader *pfile, const cpp_string *string)
989 unsigned int i;
991 for (i = 0; i < string->len; i++)
992 if (!is_idchar (string->text[i]))
993 return 0;
995 return 1;
998 /* After parsing an identifier or other sequence, produce a warning about
999 sequences not in NFC/NFKC. */
1000 static void
1001 warn_about_normalization (cpp_reader *pfile,
1002 const cpp_token *token,
1003 const struct normalize_state *s)
1005 if (CPP_OPTION (pfile, warn_normalize) < NORMALIZE_STATE_RESULT (s)
1006 && !pfile->state.skipping)
1008 /* Make sure that the token is printed using UCNs, even
1009 if we'd otherwise happily print UTF-8. */
1010 unsigned char *buf = XNEWVEC (unsigned char, cpp_token_len (token));
1011 size_t sz;
1013 sz = cpp_spell_token (pfile, token, buf, false) - buf;
1014 if (NORMALIZE_STATE_RESULT (s) == normalized_C)
1015 cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
1016 "`%.*s' is not in NFKC", (int) sz, buf);
1017 else
1018 cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
1019 "`%.*s' is not in NFC", (int) sz, buf);
1023 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1024 an identifier. FIRST is TRUE if this starts an identifier. */
1025 static bool
1026 forms_identifier_p (cpp_reader *pfile, int first,
1027 struct normalize_state *state)
1029 cpp_buffer *buffer = pfile->buffer;
1031 if (*buffer->cur == '$')
1033 if (!CPP_OPTION (pfile, dollars_in_ident))
1034 return false;
1036 buffer->cur++;
1037 if (CPP_OPTION (pfile, warn_dollars) && !pfile->state.skipping)
1039 CPP_OPTION (pfile, warn_dollars) = 0;
1040 cpp_error (pfile, CPP_DL_PEDWARN, "'$' in identifier or number");
1043 return true;
1046 /* Is this a syntactically valid UCN? */
1047 if (CPP_OPTION (pfile, extended_identifiers)
1048 && *buffer->cur == '\\'
1049 && (buffer->cur[1] == 'u' || buffer->cur[1] == 'U'))
1051 buffer->cur += 2;
1052 if (_cpp_valid_ucn (pfile, &buffer->cur, buffer->rlimit, 1 + !first,
1053 state))
1054 return true;
1055 buffer->cur -= 2;
1058 return false;
1061 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1062 static cpp_hashnode *
1063 lex_identifier_intern (cpp_reader *pfile, const uchar *base)
1065 cpp_hashnode *result;
1066 const uchar *cur;
1067 unsigned int len;
1068 unsigned int hash = HT_HASHSTEP (0, *base);
1070 cur = base + 1;
1071 while (ISIDNUM (*cur))
1073 hash = HT_HASHSTEP (hash, *cur);
1074 cur++;
1076 len = cur - base;
1077 hash = HT_HASHFINISH (hash, len);
1078 result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1079 base, len, hash, HT_ALLOC));
1081 /* Rarely, identifiers require diagnostics when lexed. */
1082 if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC)
1083 && !pfile->state.skipping, 0))
1085 /* It is allowed to poison the same identifier twice. */
1086 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
1087 cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"",
1088 NODE_NAME (result));
1090 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1091 replacement list of a variadic macro. */
1092 if (result == pfile->spec_nodes.n__VA_ARGS__
1093 && !pfile->state.va_args_ok)
1094 cpp_error (pfile, CPP_DL_PEDWARN,
1095 "__VA_ARGS__ can only appear in the expansion"
1096 " of a C99 variadic macro");
1098 /* For -Wc++-compat, warn about use of C++ named operators. */
1099 if (result->flags & NODE_WARN_OPERATOR)
1100 cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
1101 "identifier \"%s\" is a special operator name in C++",
1102 NODE_NAME (result));
1105 return result;
1108 /* Get the cpp_hashnode of an identifier specified by NAME in
1109 the current cpp_reader object. If none is found, NULL is returned. */
1110 cpp_hashnode *
1111 _cpp_lex_identifier (cpp_reader *pfile, const char *name)
1113 cpp_hashnode *result;
1114 result = lex_identifier_intern (pfile, (uchar *) name);
1115 return result;
1118 /* Lex an identifier starting at BUFFER->CUR - 1. */
1119 static cpp_hashnode *
1120 lex_identifier (cpp_reader *pfile, const uchar *base, bool starts_ucn,
1121 struct normalize_state *nst)
1123 cpp_hashnode *result;
1124 const uchar *cur;
1125 unsigned int len;
1126 unsigned int hash = HT_HASHSTEP (0, *base);
1128 cur = pfile->buffer->cur;
1129 if (! starts_ucn)
1130 while (ISIDNUM (*cur))
1132 hash = HT_HASHSTEP (hash, *cur);
1133 cur++;
1135 pfile->buffer->cur = cur;
1136 if (starts_ucn || forms_identifier_p (pfile, false, nst))
1138 /* Slower version for identifiers containing UCNs (or $). */
1139 do {
1140 while (ISIDNUM (*pfile->buffer->cur))
1142 pfile->buffer->cur++;
1143 NORMALIZE_STATE_UPDATE_IDNUM (nst);
1145 } while (forms_identifier_p (pfile, false, nst));
1146 result = _cpp_interpret_identifier (pfile, base,
1147 pfile->buffer->cur - base);
1149 else
1151 len = cur - base;
1152 hash = HT_HASHFINISH (hash, len);
1154 result = CPP_HASHNODE (ht_lookup_with_hash (pfile->hash_table,
1155 base, len, hash, HT_ALLOC));
1158 /* Rarely, identifiers require diagnostics when lexed. */
1159 if (__builtin_expect ((result->flags & NODE_DIAGNOSTIC)
1160 && !pfile->state.skipping, 0))
1162 /* It is allowed to poison the same identifier twice. */
1163 if ((result->flags & NODE_POISONED) && !pfile->state.poisoned_ok)
1164 cpp_error (pfile, CPP_DL_ERROR, "attempt to use poisoned \"%s\"",
1165 NODE_NAME (result));
1167 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1168 replacement list of a variadic macro. */
1169 if (result == pfile->spec_nodes.n__VA_ARGS__
1170 && !pfile->state.va_args_ok)
1171 cpp_error (pfile, CPP_DL_PEDWARN,
1172 "__VA_ARGS__ can only appear in the expansion"
1173 " of a C99 variadic macro");
1175 /* For -Wc++-compat, warn about use of C++ named operators. */
1176 if (result->flags & NODE_WARN_OPERATOR)
1177 cpp_warning (pfile, CPP_W_CXX_OPERATOR_NAMES,
1178 "identifier \"%s\" is a special operator name in C++",
1179 NODE_NAME (result));
1182 return result;
1185 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1186 static void
1187 lex_number (cpp_reader *pfile, cpp_string *number,
1188 struct normalize_state *nst)
1190 const uchar *cur;
1191 const uchar *base;
1192 uchar *dest;
1194 base = pfile->buffer->cur - 1;
1197 cur = pfile->buffer->cur;
1199 /* N.B. ISIDNUM does not include $. */
1200 while (ISIDNUM (*cur) || *cur == '.' || VALID_SIGN (*cur, cur[-1]))
1202 cur++;
1203 NORMALIZE_STATE_UPDATE_IDNUM (nst);
1206 pfile->buffer->cur = cur;
1208 while (forms_identifier_p (pfile, false, nst));
1210 number->len = cur - base;
1211 dest = _cpp_unaligned_alloc (pfile, number->len + 1);
1212 memcpy (dest, base, number->len);
1213 dest[number->len] = '\0';
1214 number->text = dest;
1217 /* Create a token of type TYPE with a literal spelling. */
1218 static void
1219 create_literal (cpp_reader *pfile, cpp_token *token, const uchar *base,
1220 unsigned int len, enum cpp_ttype type)
1222 uchar *dest = _cpp_unaligned_alloc (pfile, len + 1);
1224 memcpy (dest, base, len);
1225 dest[len] = '\0';
1226 token->type = type;
1227 token->val.str.len = len;
1228 token->val.str.text = dest;
1231 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1232 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1234 static void
1235 bufring_append (cpp_reader *pfile, const uchar *base, size_t len,
1236 _cpp_buff **first_buff_p, _cpp_buff **last_buff_p)
1238 _cpp_buff *first_buff = *first_buff_p;
1239 _cpp_buff *last_buff = *last_buff_p;
1241 if (first_buff == NULL)
1242 first_buff = last_buff = _cpp_get_buff (pfile, len);
1243 else if (len > BUFF_ROOM (last_buff))
1245 size_t room = BUFF_ROOM (last_buff);
1246 memcpy (BUFF_FRONT (last_buff), base, room);
1247 BUFF_FRONT (last_buff) += room;
1248 base += room;
1249 len -= room;
1250 last_buff = _cpp_append_extend_buff (pfile, last_buff, len);
1253 memcpy (BUFF_FRONT (last_buff), base, len);
1254 BUFF_FRONT (last_buff) += len;
1256 *first_buff_p = first_buff;
1257 *last_buff_p = last_buff;
1260 /* Lexes a raw string. The stored string contains the spelling, including
1261 double quotes, delimiter string, '(' and ')', any leading
1262 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1263 literal, or CPP_OTHER if it was not properly terminated.
1265 The spelling is NUL-terminated, but it is not guaranteed that this
1266 is the first NUL since embedded NULs are preserved. */
1268 static void
1269 lex_raw_string (cpp_reader *pfile, cpp_token *token, const uchar *base,
1270 const uchar *cur)
1272 source_location saw_NUL = 0;
1273 const uchar *raw_prefix;
1274 unsigned int raw_prefix_len = 0;
1275 enum cpp_ttype type;
1276 size_t total_len = 0;
1277 _cpp_buff *first_buff = NULL, *last_buff = NULL;
1278 _cpp_line_note *note = &pfile->buffer->notes[pfile->buffer->cur_note];
1280 type = (*base == 'L' ? CPP_WSTRING :
1281 *base == 'U' ? CPP_STRING32 :
1282 *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
1283 : CPP_STRING);
1285 raw_prefix = cur + 1;
1286 while (raw_prefix_len < 16)
1288 switch (raw_prefix[raw_prefix_len])
1290 case ' ': case '(': case ')': case '\\': case '\t':
1291 case '\v': case '\f': case '\n': default:
1292 break;
1293 /* Basic source charset except the above chars. */
1294 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1295 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1296 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1297 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1298 case 'y': case 'z':
1299 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1300 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1301 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1302 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1303 case 'Y': case 'Z':
1304 case '0': case '1': case '2': case '3': case '4': case '5':
1305 case '6': case '7': case '8': case '9':
1306 case '_': case '{': case '}': case '#': case '[': case ']':
1307 case '<': case '>': case '%': case ':': case ';': case '.':
1308 case '?': case '*': case '+': case '-': case '/': case '^':
1309 case '&': case '|': case '~': case '!': case '=': case ',':
1310 case '"': case '\'':
1311 raw_prefix_len++;
1312 continue;
1314 break;
1317 if (raw_prefix[raw_prefix_len] != '(')
1319 int col = CPP_BUF_COLUMN (pfile->buffer, raw_prefix + raw_prefix_len)
1320 + 1;
1321 if (raw_prefix_len == 16)
1322 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, col,
1323 "raw string delimiter longer than 16 characters");
1324 else
1325 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, col,
1326 "invalid character '%c' in raw string delimiter",
1327 (int) raw_prefix[raw_prefix_len]);
1328 pfile->buffer->cur = raw_prefix - 1;
1329 create_literal (pfile, token, base, raw_prefix - 1 - base, CPP_OTHER);
1330 return;
1333 cur = raw_prefix + raw_prefix_len + 1;
1334 for (;;)
1336 #define BUF_APPEND(STR,LEN) \
1337 do { \
1338 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1339 &first_buff, &last_buff); \
1340 total_len += (LEN); \
1341 } while (0);
1343 cppchar_t c;
1345 /* If we previously performed any trigraph or line splicing
1346 transformations, undo them within the body of the raw string. */
1347 while (note->pos < cur)
1348 ++note;
1349 for (; note->pos == cur; ++note)
1351 switch (note->type)
1353 case '\\':
1354 case ' ':
1355 /* Restore backslash followed by newline. */
1356 BUF_APPEND (base, cur - base);
1357 base = cur;
1358 BUF_APPEND ("\\", 1);
1359 after_backslash:
1360 if (note->type == ' ')
1362 /* GNU backslash whitespace newline extension. FIXME
1363 could be any sequence of non-vertical space. When we
1364 can properly restore any such sequence, we should mark
1365 this note as handled so _cpp_process_line_notes
1366 doesn't warn. */
1367 BUF_APPEND (" ", 1);
1370 BUF_APPEND ("\n", 1);
1371 break;
1373 case 0:
1374 /* Already handled. */
1375 break;
1377 default:
1378 if (_cpp_trigraph_map[note->type])
1380 /* Don't warn about this trigraph in
1381 _cpp_process_line_notes, since trigraphs show up as
1382 trigraphs in raw strings. */
1383 uchar type = note->type;
1384 note->type = 0;
1386 if (!CPP_OPTION (pfile, trigraphs))
1387 /* If we didn't convert the trigraph in the first
1388 place, don't do anything now either. */
1389 break;
1391 BUF_APPEND (base, cur - base);
1392 base = cur;
1393 BUF_APPEND ("??", 2);
1395 /* ??/ followed by newline gets two line notes, one for
1396 the trigraph and one for the backslash/newline. */
1397 if (type == '/' && note[1].pos == cur)
1399 if (note[1].type != '\\'
1400 && note[1].type != ' ')
1401 abort ();
1402 BUF_APPEND ("/", 1);
1403 ++note;
1404 goto after_backslash;
1406 /* The ) from ??) could be part of the suffix. */
1407 else if (type == ')'
1408 && strncmp ((const char *) cur+1,
1409 (const char *) raw_prefix,
1410 raw_prefix_len) == 0
1411 && cur[raw_prefix_len+1] == '"')
1413 cur += raw_prefix_len+2;
1414 goto break_outer_loop;
1416 else
1418 /* Skip the replacement character. */
1419 base = ++cur;
1420 BUF_APPEND (&type, 1);
1423 else
1424 abort ();
1425 break;
1428 c = *cur++;
1430 if (c == ')'
1431 && strncmp ((const char *) cur, (const char *) raw_prefix,
1432 raw_prefix_len) == 0
1433 && cur[raw_prefix_len] == '"')
1435 cur += raw_prefix_len + 1;
1436 break;
1438 else if (c == '\n')
1440 if (pfile->state.in_directive
1441 || pfile->state.parsing_args
1442 || pfile->state.in_deferred_pragma)
1444 cur--;
1445 type = CPP_OTHER;
1446 cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 0,
1447 "unterminated raw string");
1448 break;
1451 BUF_APPEND (base, cur - base);
1453 if (pfile->buffer->cur < pfile->buffer->rlimit)
1454 CPP_INCREMENT_LINE (pfile, 0);
1455 pfile->buffer->need_line = true;
1457 pfile->buffer->cur = cur-1;
1458 _cpp_process_line_notes (pfile, false);
1459 if (!_cpp_get_fresh_line (pfile))
1461 source_location src_loc = token->src_loc;
1462 token->type = CPP_EOF;
1463 /* Tell the compiler the line number of the EOF token. */
1464 token->src_loc = pfile->line_table->highest_line;
1465 token->flags = BOL;
1466 if (first_buff != NULL)
1467 _cpp_release_buff (pfile, first_buff);
1468 cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0,
1469 "unterminated raw string");
1470 return;
1473 cur = base = pfile->buffer->cur;
1474 note = &pfile->buffer->notes[pfile->buffer->cur_note];
1476 else if (c == '\0' && !saw_NUL)
1477 LINEMAP_POSITION_FOR_COLUMN (saw_NUL, pfile->line_table,
1478 CPP_BUF_COLUMN (pfile->buffer, cur));
1480 break_outer_loop:
1482 if (saw_NUL && !pfile->state.skipping)
1483 cpp_error_with_line (pfile, CPP_DL_WARNING, saw_NUL, 0,
1484 "null character(s) preserved in literal");
1486 pfile->buffer->cur = cur;
1487 if (first_buff == NULL)
1488 create_literal (pfile, token, base, cur - base, type);
1489 else
1491 uchar *dest = _cpp_unaligned_alloc (pfile, total_len + (cur - base) + 1);
1493 token->type = type;
1494 token->val.str.len = total_len + (cur - base);
1495 token->val.str.text = dest;
1496 last_buff = first_buff;
1497 while (last_buff != NULL)
1499 memcpy (dest, last_buff->base,
1500 BUFF_FRONT (last_buff) - last_buff->base);
1501 dest += BUFF_FRONT (last_buff) - last_buff->base;
1502 last_buff = last_buff->next;
1504 _cpp_release_buff (pfile, first_buff);
1505 memcpy (dest, base, cur - base);
1506 dest[cur - base] = '\0';
1510 /* Lexes a string, character constant, or angle-bracketed header file
1511 name. The stored string contains the spelling, including opening
1512 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1513 'R' modifier. It returns the type of the literal, or CPP_OTHER
1514 if it was not properly terminated, or CPP_LESS for an unterminated
1515 header name which must be relexed as normal tokens.
1517 The spelling is NUL-terminated, but it is not guaranteed that this
1518 is the first NUL since embedded NULs are preserved. */
1519 static void
1520 lex_string (cpp_reader *pfile, cpp_token *token, const uchar *base)
1522 bool saw_NUL = false;
1523 const uchar *cur;
1524 cppchar_t terminator;
1525 enum cpp_ttype type;
1527 cur = base;
1528 terminator = *cur++;
1529 if (terminator == 'L' || terminator == 'U')
1530 terminator = *cur++;
1531 else if (terminator == 'u')
1533 terminator = *cur++;
1534 if (terminator == '8')
1535 terminator = *cur++;
1537 if (terminator == 'R')
1539 lex_raw_string (pfile, token, base, cur);
1540 return;
1542 if (terminator == '"')
1543 type = (*base == 'L' ? CPP_WSTRING :
1544 *base == 'U' ? CPP_STRING32 :
1545 *base == 'u' ? (base[1] == '8' ? CPP_UTF8STRING : CPP_STRING16)
1546 : CPP_STRING);
1547 else if (terminator == '\'')
1548 type = (*base == 'L' ? CPP_WCHAR :
1549 *base == 'U' ? CPP_CHAR32 :
1550 *base == 'u' ? CPP_CHAR16 : CPP_CHAR);
1551 else
1552 terminator = '>', type = CPP_HEADER_NAME;
1554 for (;;)
1556 cppchar_t c = *cur++;
1558 /* In #include-style directives, terminators are not escapable. */
1559 if (c == '\\' && !pfile->state.angled_headers && *cur != '\n')
1560 cur++;
1561 else if (c == terminator)
1562 break;
1563 else if (c == '\n')
1565 cur--;
1566 /* Unmatched quotes always yield undefined behavior, but
1567 greedy lexing means that what appears to be an unterminated
1568 header name may actually be a legitimate sequence of tokens. */
1569 if (terminator == '>')
1571 token->type = CPP_LESS;
1572 return;
1574 type = CPP_OTHER;
1575 break;
1577 else if (c == '\0')
1578 saw_NUL = true;
1581 if (saw_NUL && !pfile->state.skipping)
1582 cpp_error (pfile, CPP_DL_WARNING,
1583 "null character(s) preserved in literal");
1585 if (type == CPP_OTHER && CPP_OPTION (pfile, lang) != CLK_ASM)
1586 cpp_error (pfile, CPP_DL_PEDWARN, "missing terminating %c character",
1587 (int) terminator);
1589 pfile->buffer->cur = cur;
1590 create_literal (pfile, token, base, cur - base, type);
1593 /* Return the comment table. The client may not make any assumption
1594 about the ordering of the table. */
1595 cpp_comment_table *
1596 cpp_get_comments (cpp_reader *pfile)
1598 return &pfile->comments;
1601 /* Append a comment to the end of the comment table. */
1602 static void
1603 store_comment (cpp_reader *pfile, cpp_token *token)
1605 int len;
1607 if (pfile->comments.allocated == 0)
1609 pfile->comments.allocated = 256;
1610 pfile->comments.entries = (cpp_comment *) xmalloc
1611 (pfile->comments.allocated * sizeof (cpp_comment));
1614 if (pfile->comments.count == pfile->comments.allocated)
1616 pfile->comments.allocated *= 2;
1617 pfile->comments.entries = (cpp_comment *) xrealloc
1618 (pfile->comments.entries,
1619 pfile->comments.allocated * sizeof (cpp_comment));
1622 len = token->val.str.len;
1624 /* Copy comment. Note, token may not be NULL terminated. */
1625 pfile->comments.entries[pfile->comments.count].comment =
1626 (char *) xmalloc (sizeof (char) * (len + 1));
1627 memcpy (pfile->comments.entries[pfile->comments.count].comment,
1628 token->val.str.text, len);
1629 pfile->comments.entries[pfile->comments.count].comment[len] = '\0';
1631 /* Set source location. */
1632 pfile->comments.entries[pfile->comments.count].sloc = token->src_loc;
1634 /* Increment the count of entries in the comment table. */
1635 pfile->comments.count++;
1638 /* The stored comment includes the comment start and any terminator. */
1639 static void
1640 save_comment (cpp_reader *pfile, cpp_token *token, const unsigned char *from,
1641 cppchar_t type)
1643 unsigned char *buffer;
1644 unsigned int len, clen, i;
1646 len = pfile->buffer->cur - from + 1; /* + 1 for the initial '/'. */
1648 /* C++ comments probably (not definitely) have moved past a new
1649 line, which we don't want to save in the comment. */
1650 if (is_vspace (pfile->buffer->cur[-1]))
1651 len--;
1653 /* If we are currently in a directive or in argument parsing, then
1654 we need to store all C++ comments as C comments internally, and
1655 so we need to allocate a little extra space in that case.
1657 Note that the only time we encounter a directive here is
1658 when we are saving comments in a "#define". */
1659 clen = ((pfile->state.in_directive || pfile->state.parsing_args)
1660 && type == '/') ? len + 2 : len;
1662 buffer = _cpp_unaligned_alloc (pfile, clen);
1664 token->type = CPP_COMMENT;
1665 token->val.str.len = clen;
1666 token->val.str.text = buffer;
1668 buffer[0] = '/';
1669 memcpy (buffer + 1, from, len - 1);
1671 /* Finish conversion to a C comment, if necessary. */
1672 if ((pfile->state.in_directive || pfile->state.parsing_args) && type == '/')
1674 buffer[1] = '*';
1675 buffer[clen - 2] = '*';
1676 buffer[clen - 1] = '/';
1677 /* As there can be in a C++ comments illegal sequences for C comments
1678 we need to filter them out. */
1679 for (i = 2; i < (clen - 2); i++)
1680 if (buffer[i] == '/' && (buffer[i - 1] == '*' || buffer[i + 1] == '*'))
1681 buffer[i] = '|';
1684 /* Finally store this comment for use by clients of libcpp. */
1685 store_comment (pfile, token);
1688 /* Allocate COUNT tokens for RUN. */
1689 void
1690 _cpp_init_tokenrun (tokenrun *run, unsigned int count)
1692 run->base = XNEWVEC (cpp_token, count);
1693 run->limit = run->base + count;
1694 run->next = NULL;
1697 /* Returns the next tokenrun, or creates one if there is none. */
1698 static tokenrun *
1699 next_tokenrun (tokenrun *run)
1701 if (run->next == NULL)
1703 run->next = XNEW (tokenrun);
1704 run->next->prev = run;
1705 _cpp_init_tokenrun (run->next, 250);
1708 return run->next;
1711 /* Look ahead in the input stream. */
1712 const cpp_token *
1713 cpp_peek_token (cpp_reader *pfile, int index)
1715 cpp_context *context = pfile->context;
1716 const cpp_token *peektok;
1717 int count;
1719 /* First, scan through any pending cpp_context objects. */
1720 while (context->prev)
1722 ptrdiff_t sz = (context->direct_p
1723 ? LAST (context).token - FIRST (context).token
1724 : LAST (context).ptoken - FIRST (context).ptoken);
1726 if (index < (int) sz)
1727 return (context->direct_p
1728 ? FIRST (context).token + index
1729 : *(FIRST (context).ptoken + index));
1731 index -= (int) sz;
1732 context = context->prev;
1735 /* We will have to read some new tokens after all (and do so
1736 without invalidating preceding tokens). */
1737 count = index;
1738 pfile->keep_tokens++;
1742 peektok = _cpp_lex_token (pfile);
1743 if (peektok->type == CPP_EOF)
1744 return peektok;
1746 while (index--);
1748 _cpp_backup_tokens_direct (pfile, count + 1);
1749 pfile->keep_tokens--;
1751 return peektok;
1754 /* Allocate a single token that is invalidated at the same time as the
1755 rest of the tokens on the line. Has its line and col set to the
1756 same as the last lexed token, so that diagnostics appear in the
1757 right place. */
1758 cpp_token *
1759 _cpp_temp_token (cpp_reader *pfile)
1761 cpp_token *old, *result;
1762 ptrdiff_t sz = pfile->cur_run->limit - pfile->cur_token;
1763 ptrdiff_t la = (ptrdiff_t) pfile->lookaheads;
1765 old = pfile->cur_token - 1;
1766 /* Any pre-existing lookaheads must not be clobbered. */
1767 if (la)
1769 if (sz <= la)
1771 tokenrun *next = next_tokenrun (pfile->cur_run);
1773 if (sz < la)
1774 memmove (next->base + 1, next->base,
1775 (la - sz) * sizeof (cpp_token));
1777 next->base[0] = pfile->cur_run->limit[-1];
1780 if (sz > 1)
1781 memmove (pfile->cur_token + 1, pfile->cur_token,
1782 MIN (la, sz - 1) * sizeof (cpp_token));
1785 if (!sz && pfile->cur_token == pfile->cur_run->limit)
1787 pfile->cur_run = next_tokenrun (pfile->cur_run);
1788 pfile->cur_token = pfile->cur_run->base;
1791 result = pfile->cur_token++;
1792 result->src_loc = old->src_loc;
1793 return result;
1796 /* Lex a token into RESULT (external interface). Takes care of issues
1797 like directive handling, token lookahead, multiple include
1798 optimization and skipping. */
1799 const cpp_token *
1800 _cpp_lex_token (cpp_reader *pfile)
1802 cpp_token *result;
1804 for (;;)
1806 if (pfile->cur_token == pfile->cur_run->limit)
1808 pfile->cur_run = next_tokenrun (pfile->cur_run);
1809 pfile->cur_token = pfile->cur_run->base;
1811 /* We assume that the current token is somewhere in the current
1812 run. */
1813 if (pfile->cur_token < pfile->cur_run->base
1814 || pfile->cur_token >= pfile->cur_run->limit)
1815 abort ();
1817 if (pfile->lookaheads)
1819 pfile->lookaheads--;
1820 result = pfile->cur_token++;
1822 else
1823 result = _cpp_lex_direct (pfile);
1825 if (result->flags & BOL)
1827 /* Is this a directive. If _cpp_handle_directive returns
1828 false, it is an assembler #. */
1829 if (result->type == CPP_HASH
1830 /* 6.10.3 p 11: Directives in a list of macro arguments
1831 gives undefined behavior. This implementation
1832 handles the directive as normal. */
1833 && pfile->state.parsing_args != 1)
1835 if (_cpp_handle_directive (pfile, result->flags & PREV_WHITE))
1837 if (pfile->directive_result.type == CPP_PADDING)
1838 continue;
1839 result = &pfile->directive_result;
1842 else if (pfile->state.in_deferred_pragma)
1843 result = &pfile->directive_result;
1845 if (pfile->cb.line_change && !pfile->state.skipping)
1846 pfile->cb.line_change (pfile, result, pfile->state.parsing_args);
1849 /* We don't skip tokens in directives. */
1850 if (pfile->state.in_directive || pfile->state.in_deferred_pragma)
1851 break;
1853 /* Outside a directive, invalidate controlling macros. At file
1854 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
1855 get here and MI optimization works. */
1856 pfile->mi_valid = false;
1858 if (!pfile->state.skipping || result->type == CPP_EOF)
1859 break;
1862 return result;
1865 /* Returns true if a fresh line has been loaded. */
1866 bool
1867 _cpp_get_fresh_line (cpp_reader *pfile)
1869 int return_at_eof;
1871 /* We can't get a new line until we leave the current directive. */
1872 if (pfile->state.in_directive)
1873 return false;
1875 for (;;)
1877 cpp_buffer *buffer = pfile->buffer;
1879 if (!buffer->need_line)
1880 return true;
1882 if (buffer->next_line < buffer->rlimit)
1884 _cpp_clean_line (pfile);
1885 return true;
1888 /* First, get out of parsing arguments state. */
1889 if (pfile->state.parsing_args)
1890 return false;
1892 /* End of buffer. Non-empty files should end in a newline. */
1893 if (buffer->buf != buffer->rlimit
1894 && buffer->next_line > buffer->rlimit
1895 && !buffer->from_stage3)
1897 /* Clip to buffer size. */
1898 buffer->next_line = buffer->rlimit;
1901 return_at_eof = buffer->return_at_eof;
1902 _cpp_pop_buffer (pfile);
1903 if (pfile->buffer == NULL || return_at_eof)
1904 return false;
1908 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
1909 do \
1911 result->type = ELSE_TYPE; \
1912 if (*buffer->cur == CHAR) \
1913 buffer->cur++, result->type = THEN_TYPE; \
1915 while (0)
1917 /* Lex a token into pfile->cur_token, which is also incremented, to
1918 get diagnostics pointing to the correct location.
1920 Does not handle issues such as token lookahead, multiple-include
1921 optimization, directives, skipping etc. This function is only
1922 suitable for use by _cpp_lex_token, and in special cases like
1923 lex_expansion_token which doesn't care for any of these issues.
1925 When meeting a newline, returns CPP_EOF if parsing a directive,
1926 otherwise returns to the start of the token buffer if permissible.
1927 Returns the location of the lexed token. */
1928 cpp_token *
1929 _cpp_lex_direct (cpp_reader *pfile)
1931 cppchar_t c;
1932 cpp_buffer *buffer;
1933 const unsigned char *comment_start;
1934 cpp_token *result = pfile->cur_token++;
1936 fresh_line:
1937 result->flags = 0;
1938 buffer = pfile->buffer;
1939 if (buffer->need_line)
1941 if (pfile->state.in_deferred_pragma)
1943 result->type = CPP_PRAGMA_EOL;
1944 pfile->state.in_deferred_pragma = false;
1945 if (!pfile->state.pragma_allow_expansion)
1946 pfile->state.prevent_expansion--;
1947 return result;
1949 if (!_cpp_get_fresh_line (pfile))
1951 result->type = CPP_EOF;
1952 if (!pfile->state.in_directive)
1954 /* Tell the compiler the line number of the EOF token. */
1955 result->src_loc = pfile->line_table->highest_line;
1956 result->flags = BOL;
1958 return result;
1960 if (!pfile->keep_tokens)
1962 pfile->cur_run = &pfile->base_run;
1963 result = pfile->base_run.base;
1964 pfile->cur_token = result + 1;
1966 result->flags = BOL;
1967 if (pfile->state.parsing_args == 2)
1968 result->flags |= PREV_WHITE;
1970 buffer = pfile->buffer;
1971 update_tokens_line:
1972 result->src_loc = pfile->line_table->highest_line;
1974 skipped_white:
1975 if (buffer->cur >= buffer->notes[buffer->cur_note].pos
1976 && !pfile->overlaid_buffer)
1978 _cpp_process_line_notes (pfile, false);
1979 result->src_loc = pfile->line_table->highest_line;
1981 c = *buffer->cur++;
1983 LINEMAP_POSITION_FOR_COLUMN (result->src_loc, pfile->line_table,
1984 CPP_BUF_COLUMN (buffer, buffer->cur));
1986 switch (c)
1988 case ' ': case '\t': case '\f': case '\v': case '\0':
1989 result->flags |= PREV_WHITE;
1990 skip_whitespace (pfile, c);
1991 goto skipped_white;
1993 case '\n':
1994 if (buffer->cur < buffer->rlimit)
1995 CPP_INCREMENT_LINE (pfile, 0);
1996 buffer->need_line = true;
1997 goto fresh_line;
1999 case '0': case '1': case '2': case '3': case '4':
2000 case '5': case '6': case '7': case '8': case '9':
2002 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2003 result->type = CPP_NUMBER;
2004 lex_number (pfile, &result->val.str, &nst);
2005 warn_about_normalization (pfile, result, &nst);
2006 break;
2009 case 'L':
2010 case 'u':
2011 case 'U':
2012 case 'R':
2013 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2014 wide strings or raw strings. */
2015 if (c == 'L' || CPP_OPTION (pfile, uliterals))
2017 if ((*buffer->cur == '\'' && c != 'R')
2018 || *buffer->cur == '"'
2019 || (*buffer->cur == 'R'
2020 && c != 'R'
2021 && buffer->cur[1] == '"'
2022 && CPP_OPTION (pfile, uliterals))
2023 || (*buffer->cur == '8'
2024 && c == 'u'
2025 && (buffer->cur[1] == '"'
2026 || (buffer->cur[1] == 'R' && buffer->cur[2] == '"'))))
2028 lex_string (pfile, result, buffer->cur - 1);
2029 break;
2032 /* Fall through. */
2034 case '_':
2035 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2036 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2037 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2038 case 's': case 't': case 'v': case 'w': case 'x':
2039 case 'y': case 'z':
2040 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2041 case 'G': case 'H': case 'I': case 'J': case 'K':
2042 case 'M': case 'N': case 'O': case 'P': case 'Q':
2043 case 'S': case 'T': case 'V': case 'W': case 'X':
2044 case 'Y': case 'Z':
2045 result->type = CPP_NAME;
2047 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2048 result->val.node.node = lex_identifier (pfile, buffer->cur - 1, false,
2049 &nst);
2050 warn_about_normalization (pfile, result, &nst);
2053 /* Convert named operators to their proper types. */
2054 if (result->val.node.node->flags & NODE_OPERATOR)
2056 result->flags |= NAMED_OP;
2057 result->type = (enum cpp_ttype) result->val.node.node->directive_index;
2059 break;
2061 case '\'':
2062 case '"':
2063 lex_string (pfile, result, buffer->cur - 1);
2064 break;
2066 case '/':
2067 /* A potential block or line comment. */
2068 comment_start = buffer->cur;
2069 c = *buffer->cur;
2071 if (c == '*')
2073 if (_cpp_skip_block_comment (pfile))
2074 cpp_error (pfile, CPP_DL_ERROR, "unterminated comment");
2076 else if (c == '/' && (CPP_OPTION (pfile, cplusplus_comments)
2077 || cpp_in_system_header (pfile)))
2079 /* Warn about comments only if pedantically GNUC89, and not
2080 in system headers. */
2081 if (CPP_OPTION (pfile, lang) == CLK_GNUC89 && CPP_PEDANTIC (pfile)
2082 && ! buffer->warned_cplusplus_comments)
2084 cpp_error (pfile, CPP_DL_PEDWARN,
2085 "C++ style comments are not allowed in ISO C90");
2086 cpp_error (pfile, CPP_DL_PEDWARN,
2087 "(this will be reported only once per input file)");
2088 buffer->warned_cplusplus_comments = 1;
2091 if (skip_line_comment (pfile) && CPP_OPTION (pfile, warn_comments))
2092 cpp_warning (pfile, CPP_W_COMMENTS, "multi-line comment");
2094 else if (c == '=')
2096 buffer->cur++;
2097 result->type = CPP_DIV_EQ;
2098 break;
2100 else
2102 result->type = CPP_DIV;
2103 break;
2106 if (!pfile->state.save_comments)
2108 result->flags |= PREV_WHITE;
2109 goto update_tokens_line;
2112 /* Save the comment as a token in its own right. */
2113 save_comment (pfile, result, comment_start, c);
2114 break;
2116 case '<':
2117 if (pfile->state.angled_headers)
2119 lex_string (pfile, result, buffer->cur - 1);
2120 if (result->type != CPP_LESS)
2121 break;
2124 result->type = CPP_LESS;
2125 if (*buffer->cur == '=')
2126 buffer->cur++, result->type = CPP_LESS_EQ;
2127 else if (*buffer->cur == '<')
2129 buffer->cur++;
2130 IF_NEXT_IS ('=', CPP_LSHIFT_EQ, CPP_LSHIFT);
2132 else if (CPP_OPTION (pfile, digraphs))
2134 if (*buffer->cur == ':')
2136 buffer->cur++;
2137 result->flags |= DIGRAPH;
2138 result->type = CPP_OPEN_SQUARE;
2140 else if (*buffer->cur == '%')
2142 buffer->cur++;
2143 result->flags |= DIGRAPH;
2144 result->type = CPP_OPEN_BRACE;
2147 break;
2149 case '>':
2150 result->type = CPP_GREATER;
2151 if (*buffer->cur == '=')
2152 buffer->cur++, result->type = CPP_GREATER_EQ;
2153 else if (*buffer->cur == '>')
2155 buffer->cur++;
2156 IF_NEXT_IS ('=', CPP_RSHIFT_EQ, CPP_RSHIFT);
2158 break;
2160 case '%':
2161 result->type = CPP_MOD;
2162 if (*buffer->cur == '=')
2163 buffer->cur++, result->type = CPP_MOD_EQ;
2164 else if (CPP_OPTION (pfile, digraphs))
2166 if (*buffer->cur == ':')
2168 buffer->cur++;
2169 result->flags |= DIGRAPH;
2170 result->type = CPP_HASH;
2171 if (*buffer->cur == '%' && buffer->cur[1] == ':')
2172 buffer->cur += 2, result->type = CPP_PASTE, result->val.token_no = 0;
2174 else if (*buffer->cur == '>')
2176 buffer->cur++;
2177 result->flags |= DIGRAPH;
2178 result->type = CPP_CLOSE_BRACE;
2181 break;
2183 case '.':
2184 result->type = CPP_DOT;
2185 if (ISDIGIT (*buffer->cur))
2187 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2188 result->type = CPP_NUMBER;
2189 lex_number (pfile, &result->val.str, &nst);
2190 warn_about_normalization (pfile, result, &nst);
2192 else if (*buffer->cur == '.' && buffer->cur[1] == '.')
2193 buffer->cur += 2, result->type = CPP_ELLIPSIS;
2194 else if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
2195 buffer->cur++, result->type = CPP_DOT_STAR;
2196 break;
2198 case '+':
2199 result->type = CPP_PLUS;
2200 if (*buffer->cur == '+')
2201 buffer->cur++, result->type = CPP_PLUS_PLUS;
2202 else if (*buffer->cur == '=')
2203 buffer->cur++, result->type = CPP_PLUS_EQ;
2204 break;
2206 case '-':
2207 result->type = CPP_MINUS;
2208 if (*buffer->cur == '>')
2210 buffer->cur++;
2211 result->type = CPP_DEREF;
2212 if (*buffer->cur == '*' && CPP_OPTION (pfile, cplusplus))
2213 buffer->cur++, result->type = CPP_DEREF_STAR;
2215 else if (*buffer->cur == '-')
2216 buffer->cur++, result->type = CPP_MINUS_MINUS;
2217 else if (*buffer->cur == '=')
2218 buffer->cur++, result->type = CPP_MINUS_EQ;
2219 break;
2221 case '&':
2222 result->type = CPP_AND;
2223 if (*buffer->cur == '&')
2224 buffer->cur++, result->type = CPP_AND_AND;
2225 else if (*buffer->cur == '=')
2226 buffer->cur++, result->type = CPP_AND_EQ;
2227 break;
2229 case '|':
2230 result->type = CPP_OR;
2231 if (*buffer->cur == '|')
2232 buffer->cur++, result->type = CPP_OR_OR;
2233 else if (*buffer->cur == '=')
2234 buffer->cur++, result->type = CPP_OR_EQ;
2235 break;
2237 case ':':
2238 result->type = CPP_COLON;
2239 if (*buffer->cur == ':' && CPP_OPTION (pfile, cplusplus))
2240 buffer->cur++, result->type = CPP_SCOPE;
2241 else if (*buffer->cur == '>' && CPP_OPTION (pfile, digraphs))
2243 buffer->cur++;
2244 result->flags |= DIGRAPH;
2245 result->type = CPP_CLOSE_SQUARE;
2247 break;
2249 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ, CPP_MULT); break;
2250 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ, CPP_EQ); break;
2251 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ, CPP_NOT); break;
2252 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ, CPP_XOR); break;
2253 case '#': IF_NEXT_IS ('#', CPP_PASTE, CPP_HASH); result->val.token_no = 0; break;
2255 case '?': result->type = CPP_QUERY; break;
2256 case '~': result->type = CPP_COMPL; break;
2257 case ',': result->type = CPP_COMMA; break;
2258 case '(': result->type = CPP_OPEN_PAREN; break;
2259 case ')': result->type = CPP_CLOSE_PAREN; break;
2260 case '[': result->type = CPP_OPEN_SQUARE; break;
2261 case ']': result->type = CPP_CLOSE_SQUARE; break;
2262 case '{': result->type = CPP_OPEN_BRACE; break;
2263 case '}': result->type = CPP_CLOSE_BRACE; break;
2264 case ';': result->type = CPP_SEMICOLON; break;
2266 /* @ is a punctuator in Objective-C. */
2267 case '@': result->type = CPP_ATSIGN; break;
2269 case '$':
2270 case '\\':
2272 const uchar *base = --buffer->cur;
2273 struct normalize_state nst = INITIAL_NORMALIZE_STATE;
2275 if (forms_identifier_p (pfile, true, &nst))
2277 result->type = CPP_NAME;
2278 result->val.node.node = lex_identifier (pfile, base, true, &nst);
2279 warn_about_normalization (pfile, result, &nst);
2280 break;
2282 buffer->cur++;
2285 default:
2286 create_literal (pfile, result, buffer->cur - 1, 1, CPP_OTHER);
2287 break;
2290 return result;
2293 /* An upper bound on the number of bytes needed to spell TOKEN.
2294 Does not include preceding whitespace. */
2295 unsigned int
2296 cpp_token_len (const cpp_token *token)
2298 unsigned int len;
2300 switch (TOKEN_SPELL (token))
2302 default: len = 6; break;
2303 case SPELL_LITERAL: len = token->val.str.len; break;
2304 case SPELL_IDENT: len = NODE_LEN (token->val.node.node) * 10; break;
2307 return len;
2310 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2311 Return the number of bytes read out of NAME. (There are always
2312 10 bytes written to BUFFER.) */
2314 static size_t
2315 utf8_to_ucn (unsigned char *buffer, const unsigned char *name)
2317 int j;
2318 int ucn_len = 0;
2319 int ucn_len_c;
2320 unsigned t;
2321 unsigned long utf32;
2323 /* Compute the length of the UTF-8 sequence. */
2324 for (t = *name; t & 0x80; t <<= 1)
2325 ucn_len++;
2327 utf32 = *name & (0x7F >> ucn_len);
2328 for (ucn_len_c = 1; ucn_len_c < ucn_len; ucn_len_c++)
2330 utf32 = (utf32 << 6) | (*++name & 0x3F);
2332 /* Ill-formed UTF-8. */
2333 if ((*name & ~0x3F) != 0x80)
2334 abort ();
2337 *buffer++ = '\\';
2338 *buffer++ = 'U';
2339 for (j = 7; j >= 0; j--)
2340 *buffer++ = "0123456789abcdef"[(utf32 >> (4 * j)) & 0xF];
2341 return ucn_len;
2344 /* Given a token TYPE corresponding to a digraph, return a pointer to
2345 the spelling of the digraph. */
2346 static const unsigned char *
2347 cpp_digraph2name (enum cpp_ttype type)
2349 return digraph_spellings[(int) type - (int) CPP_FIRST_DIGRAPH];
2352 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
2353 already contain the enough space to hold the token's spelling.
2354 Returns a pointer to the character after the last character written.
2355 FORSTRING is true if this is to be the spelling after translation
2356 phase 1 (this is different for UCNs).
2357 FIXME: Would be nice if we didn't need the PFILE argument. */
2358 unsigned char *
2359 cpp_spell_token (cpp_reader *pfile, const cpp_token *token,
2360 unsigned char *buffer, bool forstring)
2362 switch (TOKEN_SPELL (token))
2364 case SPELL_OPERATOR:
2366 const unsigned char *spelling;
2367 unsigned char c;
2369 if (token->flags & DIGRAPH)
2370 spelling = cpp_digraph2name (token->type);
2371 else if (token->flags & NAMED_OP)
2372 goto spell_ident;
2373 else
2374 spelling = TOKEN_NAME (token);
2376 while ((c = *spelling++) != '\0')
2377 *buffer++ = c;
2379 break;
2381 spell_ident:
2382 case SPELL_IDENT:
2383 if (forstring)
2385 memcpy (buffer, NODE_NAME (token->val.node.node),
2386 NODE_LEN (token->val.node.node));
2387 buffer += NODE_LEN (token->val.node.node);
2389 else
2391 size_t i;
2392 const unsigned char * name = NODE_NAME (token->val.node.node);
2394 for (i = 0; i < NODE_LEN (token->val.node.node); i++)
2395 if (name[i] & ~0x7F)
2397 i += utf8_to_ucn (buffer, name + i) - 1;
2398 buffer += 10;
2400 else
2401 *buffer++ = NODE_NAME (token->val.node.node)[i];
2403 break;
2405 case SPELL_LITERAL:
2406 memcpy (buffer, token->val.str.text, token->val.str.len);
2407 buffer += token->val.str.len;
2408 break;
2410 case SPELL_NONE:
2411 cpp_error (pfile, CPP_DL_ICE,
2412 "unspellable token %s", TOKEN_NAME (token));
2413 break;
2416 return buffer;
2419 /* Returns TOKEN spelt as a null-terminated string. The string is
2420 freed when the reader is destroyed. Useful for diagnostics. */
2421 unsigned char *
2422 cpp_token_as_text (cpp_reader *pfile, const cpp_token *token)
2424 unsigned int len = cpp_token_len (token) + 1;
2425 unsigned char *start = _cpp_unaligned_alloc (pfile, len), *end;
2427 end = cpp_spell_token (pfile, token, start, false);
2428 end[0] = '\0';
2430 return start;
2433 /* Returns a pointer to a string which spells the token defined by
2434 TYPE and FLAGS. Used by C front ends, which really should move to
2435 using cpp_token_as_text. */
2436 const char *
2437 cpp_type2name (enum cpp_ttype type, unsigned char flags)
2439 if (flags & DIGRAPH)
2440 return (const char *) cpp_digraph2name (type);
2441 else if (flags & NAMED_OP)
2442 return cpp_named_operator2name (type);
2444 return (const char *) token_spellings[type].name;
2447 /* Writes the spelling of token to FP, without any preceding space.
2448 Separated from cpp_spell_token for efficiency - to avoid stdio
2449 double-buffering. */
2450 void
2451 cpp_output_token (const cpp_token *token, FILE *fp)
2453 switch (TOKEN_SPELL (token))
2455 case SPELL_OPERATOR:
2457 const unsigned char *spelling;
2458 int c;
2460 if (token->flags & DIGRAPH)
2461 spelling = cpp_digraph2name (token->type);
2462 else if (token->flags & NAMED_OP)
2463 goto spell_ident;
2464 else
2465 spelling = TOKEN_NAME (token);
2467 c = *spelling;
2469 putc (c, fp);
2470 while ((c = *++spelling) != '\0');
2472 break;
2474 spell_ident:
2475 case SPELL_IDENT:
2477 size_t i;
2478 const unsigned char * name = NODE_NAME (token->val.node.node);
2480 for (i = 0; i < NODE_LEN (token->val.node.node); i++)
2481 if (name[i] & ~0x7F)
2483 unsigned char buffer[10];
2484 i += utf8_to_ucn (buffer, name + i) - 1;
2485 fwrite (buffer, 1, 10, fp);
2487 else
2488 fputc (NODE_NAME (token->val.node.node)[i], fp);
2490 break;
2492 case SPELL_LITERAL:
2493 fwrite (token->val.str.text, 1, token->val.str.len, fp);
2494 break;
2496 case SPELL_NONE:
2497 /* An error, most probably. */
2498 break;
2502 /* Compare two tokens. */
2504 _cpp_equiv_tokens (const cpp_token *a, const cpp_token *b)
2506 if (a->type == b->type && a->flags == b->flags)
2507 switch (TOKEN_SPELL (a))
2509 default: /* Keep compiler happy. */
2510 case SPELL_OPERATOR:
2511 /* token_no is used to track where multiple consecutive ##
2512 tokens were originally located. */
2513 return (a->type != CPP_PASTE || a->val.token_no == b->val.token_no);
2514 case SPELL_NONE:
2515 return (a->type != CPP_MACRO_ARG
2516 || a->val.macro_arg.arg_no == b->val.macro_arg.arg_no);
2517 case SPELL_IDENT:
2518 return a->val.node.node == b->val.node.node;
2519 case SPELL_LITERAL:
2520 return (a->val.str.len == b->val.str.len
2521 && !memcmp (a->val.str.text, b->val.str.text,
2522 a->val.str.len));
2525 return 0;
2528 /* Returns nonzero if a space should be inserted to avoid an
2529 accidental token paste for output. For simplicity, it is
2530 conservative, and occasionally advises a space where one is not
2531 needed, e.g. "." and ".2". */
2533 cpp_avoid_paste (cpp_reader *pfile, const cpp_token *token1,
2534 const cpp_token *token2)
2536 enum cpp_ttype a = token1->type, b = token2->type;
2537 cppchar_t c;
2539 if (token1->flags & NAMED_OP)
2540 a = CPP_NAME;
2541 if (token2->flags & NAMED_OP)
2542 b = CPP_NAME;
2544 c = EOF;
2545 if (token2->flags & DIGRAPH)
2546 c = digraph_spellings[(int) b - (int) CPP_FIRST_DIGRAPH][0];
2547 else if (token_spellings[b].category == SPELL_OPERATOR)
2548 c = token_spellings[b].name[0];
2550 /* Quickly get everything that can paste with an '='. */
2551 if ((int) a <= (int) CPP_LAST_EQ && c == '=')
2552 return 1;
2554 switch (a)
2556 case CPP_GREATER: return c == '>';
2557 case CPP_LESS: return c == '<' || c == '%' || c == ':';
2558 case CPP_PLUS: return c == '+';
2559 case CPP_MINUS: return c == '-' || c == '>';
2560 case CPP_DIV: return c == '/' || c == '*'; /* Comments. */
2561 case CPP_MOD: return c == ':' || c == '>';
2562 case CPP_AND: return c == '&';
2563 case CPP_OR: return c == '|';
2564 case CPP_COLON: return c == ':' || c == '>';
2565 case CPP_DEREF: return c == '*';
2566 case CPP_DOT: return c == '.' || c == '%' || b == CPP_NUMBER;
2567 case CPP_HASH: return c == '#' || c == '%'; /* Digraph form. */
2568 case CPP_NAME: return ((b == CPP_NUMBER
2569 && name_p (pfile, &token2->val.str))
2570 || b == CPP_NAME
2571 || b == CPP_CHAR || b == CPP_STRING); /* L */
2572 case CPP_NUMBER: return (b == CPP_NUMBER || b == CPP_NAME
2573 || c == '.' || c == '+' || c == '-');
2574 /* UCNs */
2575 case CPP_OTHER: return ((token1->val.str.text[0] == '\\'
2576 && b == CPP_NAME)
2577 || (CPP_OPTION (pfile, objc)
2578 && token1->val.str.text[0] == '@'
2579 && (b == CPP_NAME || b == CPP_STRING)));
2580 default: break;
2583 return 0;
2586 /* Output all the remaining tokens on the current line, and a newline
2587 character, to FP. Leading whitespace is removed. If there are
2588 macros, special token padding is not performed. */
2589 void
2590 cpp_output_line (cpp_reader *pfile, FILE *fp)
2592 const cpp_token *token;
2594 token = cpp_get_token (pfile);
2595 while (token->type != CPP_EOF)
2597 cpp_output_token (token, fp);
2598 token = cpp_get_token (pfile);
2599 if (token->flags & PREV_WHITE)
2600 putc (' ', fp);
2603 putc ('\n', fp);
2606 /* Return a string representation of all the remaining tokens on the
2607 current line. The result is allocated using xmalloc and must be
2608 freed by the caller. */
2609 unsigned char *
2610 cpp_output_line_to_string (cpp_reader *pfile, const unsigned char *dir_name)
2612 const cpp_token *token;
2613 unsigned int out = dir_name ? ustrlen (dir_name) : 0;
2614 unsigned int alloced = 120 + out;
2615 unsigned char *result = (unsigned char *) xmalloc (alloced);
2617 /* If DIR_NAME is empty, there are no initial contents. */
2618 if (dir_name)
2620 sprintf ((char *) result, "#%s ", dir_name);
2621 out += 2;
2624 token = cpp_get_token (pfile);
2625 while (token->type != CPP_EOF)
2627 unsigned char *last;
2628 /* Include room for a possible space and the terminating nul. */
2629 unsigned int len = cpp_token_len (token) + 2;
2631 if (out + len > alloced)
2633 alloced *= 2;
2634 if (out + len > alloced)
2635 alloced = out + len;
2636 result = (unsigned char *) xrealloc (result, alloced);
2639 last = cpp_spell_token (pfile, token, &result[out], 0);
2640 out = last - result;
2642 token = cpp_get_token (pfile);
2643 if (token->flags & PREV_WHITE)
2644 result[out++] = ' ';
2647 result[out] = '\0';
2648 return result;
2651 /* Memory buffers. Changing these three constants can have a dramatic
2652 effect on performance. The values here are reasonable defaults,
2653 but might be tuned. If you adjust them, be sure to test across a
2654 range of uses of cpplib, including heavy nested function-like macro
2655 expansion. Also check the change in peak memory usage (NJAMD is a
2656 good tool for this). */
2657 #define MIN_BUFF_SIZE 8000
2658 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
2659 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
2660 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
2662 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
2663 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
2664 #endif
2666 /* Create a new allocation buffer. Place the control block at the end
2667 of the buffer, so that buffer overflows will cause immediate chaos. */
2668 static _cpp_buff *
2669 new_buff (size_t len)
2671 _cpp_buff *result;
2672 unsigned char *base;
2674 if (len < MIN_BUFF_SIZE)
2675 len = MIN_BUFF_SIZE;
2676 len = CPP_ALIGN (len);
2678 base = XNEWVEC (unsigned char, len + sizeof (_cpp_buff));
2679 result = (_cpp_buff *) (base + len);
2680 result->base = base;
2681 result->cur = base;
2682 result->limit = base + len;
2683 result->next = NULL;
2684 return result;
2687 /* Place a chain of unwanted allocation buffers on the free list. */
2688 void
2689 _cpp_release_buff (cpp_reader *pfile, _cpp_buff *buff)
2691 _cpp_buff *end = buff;
2693 while (end->next)
2694 end = end->next;
2695 end->next = pfile->free_buffs;
2696 pfile->free_buffs = buff;
2699 /* Return a free buffer of size at least MIN_SIZE. */
2700 _cpp_buff *
2701 _cpp_get_buff (cpp_reader *pfile, size_t min_size)
2703 _cpp_buff *result, **p;
2705 for (p = &pfile->free_buffs;; p = &(*p)->next)
2707 size_t size;
2709 if (*p == NULL)
2710 return new_buff (min_size);
2711 result = *p;
2712 size = result->limit - result->base;
2713 /* Return a buffer that's big enough, but don't waste one that's
2714 way too big. */
2715 if (size >= min_size && size <= BUFF_SIZE_UPPER_BOUND (min_size))
2716 break;
2719 *p = result->next;
2720 result->next = NULL;
2721 result->cur = result->base;
2722 return result;
2725 /* Creates a new buffer with enough space to hold the uncommitted
2726 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
2727 the excess bytes to the new buffer. Chains the new buffer after
2728 BUFF, and returns the new buffer. */
2729 _cpp_buff *
2730 _cpp_append_extend_buff (cpp_reader *pfile, _cpp_buff *buff, size_t min_extra)
2732 size_t size = EXTENDED_BUFF_SIZE (buff, min_extra);
2733 _cpp_buff *new_buff = _cpp_get_buff (pfile, size);
2735 buff->next = new_buff;
2736 memcpy (new_buff->base, buff->cur, BUFF_ROOM (buff));
2737 return new_buff;
2740 /* Creates a new buffer with enough space to hold the uncommitted
2741 remaining bytes of the buffer pointed to by BUFF, and at least
2742 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
2743 Chains the new buffer before the buffer pointed to by BUFF, and
2744 updates the pointer to point to the new buffer. */
2745 void
2746 _cpp_extend_buff (cpp_reader *pfile, _cpp_buff **pbuff, size_t min_extra)
2748 _cpp_buff *new_buff, *old_buff = *pbuff;
2749 size_t size = EXTENDED_BUFF_SIZE (old_buff, min_extra);
2751 new_buff = _cpp_get_buff (pfile, size);
2752 memcpy (new_buff->base, old_buff->cur, BUFF_ROOM (old_buff));
2753 new_buff->next = old_buff;
2754 *pbuff = new_buff;
2757 /* Free a chain of buffers starting at BUFF. */
2758 void
2759 _cpp_free_buff (_cpp_buff *buff)
2761 _cpp_buff *next;
2763 for (; buff; buff = next)
2765 next = buff->next;
2766 free (buff->base);
2770 /* Allocate permanent, unaligned storage of length LEN. */
2771 unsigned char *
2772 _cpp_unaligned_alloc (cpp_reader *pfile, size_t len)
2774 _cpp_buff *buff = pfile->u_buff;
2775 unsigned char *result = buff->cur;
2777 if (len > (size_t) (buff->limit - result))
2779 buff = _cpp_get_buff (pfile, len);
2780 buff->next = pfile->u_buff;
2781 pfile->u_buff = buff;
2782 result = buff->cur;
2785 buff->cur = result + len;
2786 return result;
2789 /* Allocate permanent, unaligned storage of length LEN from a_buff.
2790 That buffer is used for growing allocations when saving macro
2791 replacement lists in a #define, and when parsing an answer to an
2792 assertion in #assert, #unassert or #if (and therefore possibly
2793 whilst expanding macros). It therefore must not be used by any
2794 code that they might call: specifically the lexer and the guts of
2795 the macro expander.
2797 All existing other uses clearly fit this restriction: storing
2798 registered pragmas during initialization. */
2799 unsigned char *
2800 _cpp_aligned_alloc (cpp_reader *pfile, size_t len)
2802 _cpp_buff *buff = pfile->a_buff;
2803 unsigned char *result = buff->cur;
2805 if (len > (size_t) (buff->limit - result))
2807 buff = _cpp_get_buff (pfile, len);
2808 buff->next = pfile->a_buff;
2809 pfile->a_buff = buff;
2810 result = buff->cur;
2813 buff->cur = result + len;
2814 return result;
2817 /* Say which field of TOK is in use. */
2819 enum cpp_token_fld_kind
2820 cpp_token_val_index (cpp_token *tok)
2822 switch (TOKEN_SPELL (tok))
2824 case SPELL_IDENT:
2825 return CPP_TOKEN_FLD_NODE;
2826 case SPELL_LITERAL:
2827 return CPP_TOKEN_FLD_STR;
2828 case SPELL_OPERATOR:
2829 if (tok->type == CPP_PASTE)
2830 return CPP_TOKEN_FLD_TOKEN_NO;
2831 else
2832 return CPP_TOKEN_FLD_NONE;
2833 case SPELL_NONE:
2834 if (tok->type == CPP_MACRO_ARG)
2835 return CPP_TOKEN_FLD_ARG_NO;
2836 else if (tok->type == CPP_PADDING)
2837 return CPP_TOKEN_FLD_SOURCE;
2838 else if (tok->type == CPP_PRAGMA)
2839 return CPP_TOKEN_FLD_PRAGMA;
2840 /* else fall through */
2841 default:
2842 return CPP_TOKEN_FLD_NONE;