1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2007, 2008, 2009, 2010,
3 2011, 2012 Free Software Foundation, Inc.
4 Contributed by Per Bothner, 1994-95.
5 Based on CCCP program by Paul Rubin, June 1986
6 Adapted to ANSI C, Richard Stallman, Jan 1987
7 Broken out to separate file, Zack Weinberg, Mar 2000
9 This program is free software; you can redistribute it and/or modify it
10 under the terms of the GNU General Public License as published by the
11 Free Software Foundation; either version 3, or (at your option) any
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
38 enum spell_type category
;
39 const unsigned char *name
;
42 static const unsigned char *const digraph_spellings
[] =
43 { UC
"%:", UC
"%:%:", UC
"<:", UC
":>", UC
"<%", UC
"%>" };
45 #define OP(e, s) { SPELL_OPERATOR, UC s },
46 #define TK(e, s) { SPELL_ ## s, UC #e },
47 static const struct token_spelling token_spellings
[N_TTYPES
] = { TTYPE_TABLE
};
51 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
52 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
54 static void add_line_note (cpp_buffer
*, const uchar
*, unsigned int);
55 static int skip_line_comment (cpp_reader
*);
56 static void skip_whitespace (cpp_reader
*, cppchar_t
);
57 static void lex_string (cpp_reader
*, cpp_token
*, const uchar
*);
58 static void save_comment (cpp_reader
*, cpp_token
*, const uchar
*, cppchar_t
);
59 static void store_comment (cpp_reader
*, cpp_token
*);
60 static void create_literal (cpp_reader
*, cpp_token
*, const uchar
*,
61 unsigned int, enum cpp_ttype
);
62 static bool warn_in_comment (cpp_reader
*, _cpp_line_note
*);
63 static int name_p (cpp_reader
*, const cpp_string
*);
64 static tokenrun
*next_tokenrun (tokenrun
*);
66 static _cpp_buff
*new_buff (size_t);
71 Compares, the token TOKEN to the NUL-terminated string STRING.
72 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
74 cpp_ideq (const cpp_token
*token
, const char *string
)
76 if (token
->type
!= CPP_NAME
)
79 return !ustrcmp (NODE_NAME (token
->val
.node
.node
), (const uchar
*) string
);
82 /* Record a note TYPE at byte POS into the current cleaned logical
85 add_line_note (cpp_buffer
*buffer
, const uchar
*pos
, unsigned int type
)
87 if (buffer
->notes_used
== buffer
->notes_cap
)
89 buffer
->notes_cap
= buffer
->notes_cap
* 2 + 200;
90 buffer
->notes
= XRESIZEVEC (_cpp_line_note
, buffer
->notes
,
94 buffer
->notes
[buffer
->notes_used
].pos
= pos
;
95 buffer
->notes
[buffer
->notes_used
].type
= type
;
100 /* Fast path to find line special characters using optimized character
101 scanning algorithms. Anything complicated falls back to the slow
102 path below. Since this loop is very hot it's worth doing these kinds
105 One of the paths through the ifdefs should provide
107 const uchar *search_line_fast (const uchar *s, const uchar *end);
109 Between S and END, search for \n, \r, \\, ?. Return a pointer to
112 Note that the last character of the buffer is *always* a newline,
113 as forced by _cpp_convert_input. This fact can be used to avoid
114 explicitly looking for the end of the buffer. */
116 /* Configure gives us an ifdef test. */
117 #ifndef WORDS_BIGENDIAN
118 #define WORDS_BIGENDIAN 0
121 /* We'd like the largest integer that fits into a register. There's nothing
122 in <stdint.h> that gives us that. For most hosts this is unsigned long,
123 but MS decided on an LLP64 model. Thankfully when building with GCC we
124 can get the "real" word size. */
126 typedef unsigned int word_type
__attribute__((__mode__(__word__
)));
128 typedef unsigned long word_type
;
131 /* The code below is only expecting sizes 4 or 8.
132 Die at compile-time if this expectation is violated. */
133 typedef char check_word_type_size
134 [(sizeof(word_type
) == 8 || sizeof(word_type
) == 4) * 2 - 1];
136 /* Return X with the first N bytes forced to values that won't match one
137 of the interesting characters. Note that NUL is not interesting. */
139 static inline word_type
140 acc_char_mask_misalign (word_type val
, unsigned int n
)
150 /* Return X replicated to all byte positions within WORD_TYPE. */
152 static inline word_type
153 acc_char_replicate (uchar x
)
157 ret
= (x
<< 24) | (x
<< 16) | (x
<< 8) | x
;
158 if (sizeof(word_type
) == 8)
159 ret
= (ret
<< 16 << 16) | ret
;
163 /* Return non-zero if some byte of VAL is (probably) C. */
165 static inline word_type
166 acc_char_cmp (word_type val
, word_type c
)
168 #if defined(__GNUC__) && defined(__alpha__)
169 /* We can get exact results using a compare-bytes instruction.
170 Get (val == c) via (0 >= (val ^ c)). */
171 return __builtin_alpha_cmpbge (0, val
^ c
);
173 word_type magic
= 0x7efefefeU
;
174 if (sizeof(word_type
) == 8)
175 magic
= (magic
<< 16 << 16) | 0xfefefefeU
;
179 return ((val
+ magic
) ^ ~val
) & ~magic
;
183 /* Given the result of acc_char_cmp is non-zero, return the index of
184 the found character. If this was a false positive, return -1. */
187 acc_char_index (word_type cmp ATTRIBUTE_UNUSED
,
188 word_type val ATTRIBUTE_UNUSED
)
190 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
191 /* The cmpbge instruction sets *bits* of the result corresponding to
192 matches in the bytes with no false positives. */
193 return __builtin_ctzl (cmp
);
197 /* ??? It would be nice to force unrolling here,
198 and have all of these constants folded. */
199 for (i
= 0; i
< sizeof(word_type
); ++i
)
203 c
= (val
>> (sizeof(word_type
) - i
- 1) * 8) & 0xff;
205 c
= (val
>> i
* 8) & 0xff;
207 if (c
== '\n' || c
== '\r' || c
== '\\' || c
== '?')
215 /* A version of the fast scanner using bit fiddling techniques.
217 For 32-bit words, one would normally perform 16 comparisons and
218 16 branches. With this algorithm one performs 24 arithmetic
219 operations and one branch. Whether this is faster with a 32-bit
220 word size is going to be somewhat system dependent.
222 For 64-bit words, we eliminate twice the number of comparisons
223 and branches without increasing the number of arithmetic operations.
224 It's almost certainly going to be a win with 64-bit word size. */
226 static const uchar
* search_line_acc_char (const uchar
*, const uchar
*)
230 search_line_acc_char (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
232 const word_type repl_nl
= acc_char_replicate ('\n');
233 const word_type repl_cr
= acc_char_replicate ('\r');
234 const word_type repl_bs
= acc_char_replicate ('\\');
235 const word_type repl_qm
= acc_char_replicate ('?');
237 unsigned int misalign
;
241 /* Align the buffer. Mask out any bytes from before the beginning. */
242 p
= (word_type
*)((uintptr_t)s
& -sizeof(word_type
));
244 misalign
= (uintptr_t)s
& (sizeof(word_type
) - 1);
246 val
= acc_char_mask_misalign (val
, misalign
);
251 t
= acc_char_cmp (val
, repl_nl
);
252 t
|= acc_char_cmp (val
, repl_cr
);
253 t
|= acc_char_cmp (val
, repl_bs
);
254 t
|= acc_char_cmp (val
, repl_qm
);
256 if (__builtin_expect (t
!= 0, 0))
258 int i
= acc_char_index (t
, val
);
260 return (const uchar
*)p
+ i
;
267 /* Disable on Solaris 2/x86 until the following problems can be properly
270 The Solaris 9 assembler cannot assemble SSE4.2 insns.
271 Before Solaris 9 Update 6, SSE insns cannot be executed.
272 The Solaris 10+ assembler tags objects with the instruction set
273 extensions used, so SSE4.2 executables cannot run on machines that
274 don't support that extension. */
276 #if (GCC_VERSION >= 4005) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
278 /* Replicated character data to be shared between implementations.
279 Recall that outside of a context with vector support we can't
280 define compatible vector types, therefore these are all defined
281 in terms of raw characters. */
282 static const char repl_chars
[4][16] __attribute__((aligned(16))) = {
283 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
284 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
285 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
286 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
287 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
288 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
289 { '?', '?', '?', '?', '?', '?', '?', '?',
290 '?', '?', '?', '?', '?', '?', '?', '?' },
293 /* A version of the fast scanner using MMX vectorized byte compare insns.
295 This uses the PMOVMSKB instruction which was introduced with "MMX2",
296 which was packaged into SSE1; it is also present in the AMD MMX
297 extension. Mark the function as using "sse" so that we emit a real
298 "emms" instruction, rather than the 3dNOW "femms" instruction. */
302 __attribute__((__target__("sse")))
304 search_line_mmx (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
306 typedef char v8qi
__attribute__ ((__vector_size__ (8)));
307 typedef int __m64
__attribute__ ((__vector_size__ (8), __may_alias__
));
309 const v8qi repl_nl
= *(const v8qi
*)repl_chars
[0];
310 const v8qi repl_cr
= *(const v8qi
*)repl_chars
[1];
311 const v8qi repl_bs
= *(const v8qi
*)repl_chars
[2];
312 const v8qi repl_qm
= *(const v8qi
*)repl_chars
[3];
314 unsigned int misalign
, found
, mask
;
318 /* Align the source pointer. While MMX doesn't generate unaligned data
319 faults, this allows us to safely scan to the end of the buffer without
320 reading beyond the end of the last page. */
321 misalign
= (uintptr_t)s
& 7;
322 p
= (const v8qi
*)((uintptr_t)s
& -8);
325 /* Create a mask for the bytes that are valid within the first
326 16-byte block. The Idea here is that the AND with the mask
327 within the loop is "free", since we need some AND or TEST
328 insn in order to set the flags for the branch anyway. */
329 mask
= -1u << misalign
;
331 /* Main loop processing 8 bytes at a time. */
339 t
= __builtin_ia32_pcmpeqb(data
, repl_nl
);
340 c
= __builtin_ia32_pcmpeqb(data
, repl_cr
);
341 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
342 c
= __builtin_ia32_pcmpeqb(data
, repl_bs
);
343 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
344 c
= __builtin_ia32_pcmpeqb(data
, repl_qm
);
345 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
346 found
= __builtin_ia32_pmovmskb (t
);
351 __builtin_ia32_emms ();
353 /* FOUND contains 1 in bits for which we matched a relevant
354 character. Conversion to the byte index is trivial. */
355 found
= __builtin_ctz(found
);
356 return (const uchar
*)p
+ found
;
359 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
363 __attribute__((__target__("sse2")))
365 search_line_sse2 (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
367 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
369 const v16qi repl_nl
= *(const v16qi
*)repl_chars
[0];
370 const v16qi repl_cr
= *(const v16qi
*)repl_chars
[1];
371 const v16qi repl_bs
= *(const v16qi
*)repl_chars
[2];
372 const v16qi repl_qm
= *(const v16qi
*)repl_chars
[3];
374 unsigned int misalign
, found
, mask
;
378 /* Align the source pointer. */
379 misalign
= (uintptr_t)s
& 15;
380 p
= (const v16qi
*)((uintptr_t)s
& -16);
383 /* Create a mask for the bytes that are valid within the first
384 16-byte block. The Idea here is that the AND with the mask
385 within the loop is "free", since we need some AND or TEST
386 insn in order to set the flags for the branch anyway. */
387 mask
= -1u << misalign
;
389 /* Main loop processing 16 bytes at a time. */
397 t
= __builtin_ia32_pcmpeqb128(data
, repl_nl
);
398 t
|= __builtin_ia32_pcmpeqb128(data
, repl_cr
);
399 t
|= __builtin_ia32_pcmpeqb128(data
, repl_bs
);
400 t
|= __builtin_ia32_pcmpeqb128(data
, repl_qm
);
401 found
= __builtin_ia32_pmovmskb128 (t
);
406 /* FOUND contains 1 in bits for which we matched a relevant
407 character. Conversion to the byte index is trivial. */
408 found
= __builtin_ctz(found
);
409 return (const uchar
*)p
+ found
;
413 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
417 __attribute__((__target__("sse4.2")))
419 search_line_sse42 (const uchar
*s
, const uchar
*end
)
421 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
422 static const v16qi search
= { '\n', '\r', '?', '\\' };
424 uintptr_t si
= (uintptr_t)s
;
427 /* Check for unaligned input. */
430 if (__builtin_expect (end
- s
< 16, 0)
431 && __builtin_expect ((si
& 0xfff) > 0xff0, 0))
433 /* There are less than 16 bytes left in the buffer, and less
434 than 16 bytes left on the page. Reading 16 bytes at this
435 point might generate a spurious page fault. Defer to the
436 SSE2 implementation, which already handles alignment. */
437 return search_line_sse2 (s
, end
);
440 /* ??? The builtin doesn't understand that the PCMPESTRI read from
441 memory need not be aligned. */
442 __asm ("%vpcmpestri $0, (%1), %2"
443 : "=c"(index
) : "r"(s
), "x"(search
), "a"(4), "d"(16));
444 if (__builtin_expect (index
< 16, 0))
447 /* Advance the pointer to an aligned address. We will re-scan a
448 few bytes, but we no longer need care for reading past the
449 end of a page, since we're guaranteed a match. */
450 s
= (const uchar
*)((si
+ 16) & -16);
453 /* Main loop, processing 16 bytes at a time. By doing the whole loop
454 in inline assembly, we can make proper use of the flags set. */
455 __asm ( "sub $16, %1\n"
458 " %vpcmpestri $0, (%1), %2\n"
460 : "=&c"(index
), "+r"(s
)
461 : "x"(search
), "a"(4), "d"(16));
468 /* Work around out-dated assemblers without sse4 support. */
469 #define search_line_sse42 search_line_sse2
472 /* Check the CPU capabilities. */
474 #include "../gcc/config/i386/cpuid.h"
476 typedef const uchar
* (*search_line_fast_type
) (const uchar
*, const uchar
*);
477 static search_line_fast_type search_line_fast
;
479 #define HAVE_init_vectorized_lexer 1
481 init_vectorized_lexer (void)
483 unsigned dummy
, ecx
= 0, edx
= 0;
484 search_line_fast_type impl
= search_line_acc_char
;
487 #if defined(__SSE4_2__)
489 #elif defined(__SSE2__)
491 #elif defined(__SSE__)
496 impl
= search_line_sse42
;
497 else if (__get_cpuid (1, &dummy
, &dummy
, &ecx
, &edx
) || minimum
== 2)
499 if (minimum
== 3 || (ecx
& bit_SSE4_2
))
500 impl
= search_line_sse42
;
501 else if (minimum
== 2 || (edx
& bit_SSE2
))
502 impl
= search_line_sse2
;
503 else if (minimum
== 1 || (edx
& bit_SSE
))
504 impl
= search_line_mmx
;
506 else if (__get_cpuid (0x80000001, &dummy
, &dummy
, &dummy
, &edx
))
509 || (edx
& (bit_MMXEXT
| bit_CMOV
)) == (bit_MMXEXT
| bit_CMOV
))
510 impl
= search_line_mmx
;
513 search_line_fast
= impl
;
516 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__)
518 /* A vection of the fast scanner using AltiVec vectorized byte compares. */
519 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
520 so we can't compile this function without -maltivec on the command line
521 (or implied by some other switch). */
524 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
526 typedef __attribute__((altivec(vector
))) unsigned char vc
;
529 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
530 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
533 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
534 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
537 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
538 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
541 '?', '?', '?', '?', '?', '?', '?', '?',
542 '?', '?', '?', '?', '?', '?', '?', '?',
545 -1, -1, -1, -1, -1, -1, -1, -1,
546 -1, -1, -1, -1, -1, -1, -1, -1,
548 const vc zero
= { 0 };
552 /* Altivec loads automatically mask addresses with -16. This lets us
553 issue the first load as early as possible. */
554 data
= __builtin_vec_ld(0, (const vc
*)s
);
556 /* Discard bytes before the beginning of the buffer. Do this by
557 beginning with all ones and shifting in zeros according to the
558 mis-alignment. The LVSR instruction pulls the exact shift we
559 want from the address. */
560 mask
= __builtin_vec_lvsr(0, s
);
561 mask
= __builtin_vec_perm(zero
, ones
, mask
);
564 /* While altivec loads mask addresses, we still need to align S so
565 that the offset we compute at the end is correct. */
566 s
= (const uchar
*)((uintptr_t)s
& -16);
568 /* Main loop processing 16 bytes at a time. */
572 vc m_nl
, m_cr
, m_bs
, m_qm
;
575 data
= __builtin_vec_ld(0, (const vc
*)s
);
578 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
579 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
580 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
581 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
582 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
584 /* T now contains 0xff in bytes for which we matched one of the relevant
585 characters. We want to exit the loop if any byte in T is non-zero.
586 Below is the expansion of vec_any_ne(t, zero). */
588 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
591 #define N (sizeof(vc) / sizeof(long))
595 /* Statically assert that N is 2 or 4. */
596 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
598 unsigned long l
, i
= 0;
602 /* Find the first word of T that is non-zero. */
609 s
+= sizeof(unsigned long);
613 s
+= sizeof(unsigned long);
618 s
+= sizeof(unsigned long);
622 /* L now contains 0xff in bytes for which we matched one of the
623 relevant characters. We can find the byte index by finding
624 its bit index and dividing by 8. */
625 l
= __builtin_clzl(l
) >> 3;
632 #elif defined (__ARM_NEON__)
633 #include "arm_neon.h"
636 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
638 const uint8x16_t repl_nl
= vdupq_n_u8 ('\n');
639 const uint8x16_t repl_cr
= vdupq_n_u8 ('\r');
640 const uint8x16_t repl_bs
= vdupq_n_u8 ('\\');
641 const uint8x16_t repl_qm
= vdupq_n_u8 ('?');
642 const uint8x16_t xmask
= (uint8x16_t
) vdupq_n_u64 (0x8040201008040201ULL
);
644 unsigned int misalign
, found
, mask
;
648 /* Align the source pointer. */
649 misalign
= (uintptr_t)s
& 15;
650 p
= (const uint8_t *)((uintptr_t)s
& -16);
653 /* Create a mask for the bytes that are valid within the first
654 16-byte block. The Idea here is that the AND with the mask
655 within the loop is "free", since we need some AND or TEST
656 insn in order to set the flags for the branch anyway. */
657 mask
= (-1u << misalign
) & 0xffff;
659 /* Main loop, processing 16 bytes at a time. */
667 uint8x16_t t
, u
, v
, w
;
674 t
= vceqq_u8 (data
, repl_nl
);
675 u
= vceqq_u8 (data
, repl_cr
);
676 v
= vorrq_u8 (t
, vceqq_u8 (data
, repl_bs
));
677 w
= vorrq_u8 (u
, vceqq_u8 (data
, repl_qm
));
678 t
= vandq_u8 (vorrq_u8 (v
, w
), xmask
);
679 l
= vpadd_u8 (vget_low_u8 (t
), vget_high_u8 (t
));
683 found
= vget_lane_u32 ((uint32x2_t
) vorr_u64 ((uint64x1_t
) n
,
684 vshr_n_u64 ((uint64x1_t
) n
, 24)), 0);
689 /* FOUND contains 1 in bits for which we matched a relevant
690 character. Conversion to the byte index is trivial. */
691 found
= __builtin_ctz (found
);
692 return (const uchar
*)p
+ found
;
697 /* We only have one accellerated alternative. Use a direct call so that
698 we encourage inlining. */
700 #define search_line_fast search_line_acc_char
704 /* Initialize the lexer if needed. */
707 _cpp_init_lexer (void)
709 #ifdef HAVE_init_vectorized_lexer
710 init_vectorized_lexer ();
714 /* Returns with a logical line that contains no escaped newlines or
715 trigraphs. This is a time-critical inner loop. */
717 _cpp_clean_line (cpp_reader
*pfile
)
723 buffer
= pfile
->buffer
;
724 buffer
->cur_note
= buffer
->notes_used
= 0;
725 buffer
->cur
= buffer
->line_base
= buffer
->next_line
;
726 buffer
->need_line
= false;
727 s
= buffer
->next_line
;
729 if (!buffer
->from_stage3
)
731 const uchar
*pbackslash
= NULL
;
733 /* Fast path. This is the common case of an un-escaped line with
734 no trigraphs. The primary win here is by not writing any
735 data back to memory until we have to. */
738 /* Perform an optimized search for \n, \r, \\, ?. */
739 s
= search_line_fast (s
, buffer
->rlimit
);
744 /* Record the location of the backslash and continue. */
747 else if (__builtin_expect (c
== '?', 0))
749 if (__builtin_expect (s
[1] == '?', false)
750 && _cpp_trigraph_map
[s
[2]])
752 /* Have a trigraph. We may or may not have to convert
753 it. Add a line note regardless, for -Wtrigraphs. */
754 add_line_note (buffer
, s
, s
[2]);
755 if (CPP_OPTION (pfile
, trigraphs
))
757 /* We do, and that means we have to switch to the
760 *d
= _cpp_trigraph_map
[s
[2]];
765 /* Not a trigraph. Continue on fast-path. */
772 /* This must be \r or \n. We're either done, or we'll be forced
773 to write back to the buffer and continue on the slow path. */
776 if (__builtin_expect (s
== buffer
->rlimit
, false))
779 /* DOS line ending? */
780 if (__builtin_expect (c
== '\r', false) && s
[1] == '\n')
783 if (s
== buffer
->rlimit
)
787 if (__builtin_expect (pbackslash
== NULL
, true))
790 /* Check for escaped newline. */
792 while (is_nvspace (p
[-1]))
794 if (p
- 1 != pbackslash
)
797 /* Have an escaped newline; process it and proceed to
799 add_line_note (buffer
, p
- 1, p
!= d
? ' ' : '\\');
801 buffer
->next_line
= p
- 1;
809 if (c
== '\n' || c
== '\r')
811 /* Handle DOS line endings. */
812 if (c
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
814 if (s
== buffer
->rlimit
)
819 while (p
!= buffer
->next_line
&& is_nvspace (p
[-1]))
821 if (p
== buffer
->next_line
|| p
[-1] != '\\')
824 add_line_note (buffer
, p
- 1, p
!= d
? ' ': '\\');
826 buffer
->next_line
= p
- 1;
828 else if (c
== '?' && s
[1] == '?' && _cpp_trigraph_map
[s
[2]])
830 /* Add a note regardless, for the benefit of -Wtrigraphs. */
831 add_line_note (buffer
, d
, s
[2]);
832 if (CPP_OPTION (pfile
, trigraphs
))
834 *d
= _cpp_trigraph_map
[s
[2]];
842 while (*s
!= '\n' && *s
!= '\r')
846 /* Handle DOS line endings. */
847 if (*s
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
853 /* A sentinel note that should never be processed. */
854 add_line_note (buffer
, d
+ 1, '\n');
855 buffer
->next_line
= s
+ 1;
858 /* Return true if the trigraph indicated by NOTE should be warned
859 about in a comment. */
861 warn_in_comment (cpp_reader
*pfile
, _cpp_line_note
*note
)
865 /* Within comments we don't warn about trigraphs, unless the
866 trigraph forms an escaped newline, as that may change
868 if (note
->type
!= '/')
871 /* If -trigraphs, then this was an escaped newline iff the next note
873 if (CPP_OPTION (pfile
, trigraphs
))
874 return note
[1].pos
== note
->pos
;
876 /* Otherwise, see if this forms an escaped newline. */
878 while (is_nvspace (*p
))
881 /* There might have been escaped newlines between the trigraph and the
882 newline we found. Hence the position test. */
883 return (*p
== '\n' && p
< note
[1].pos
);
886 /* Process the notes created by add_line_note as far as the current
889 _cpp_process_line_notes (cpp_reader
*pfile
, int in_comment
)
891 cpp_buffer
*buffer
= pfile
->buffer
;
895 _cpp_line_note
*note
= &buffer
->notes
[buffer
->cur_note
];
898 if (note
->pos
> buffer
->cur
)
902 col
= CPP_BUF_COLUMN (buffer
, note
->pos
+ 1);
904 if (note
->type
== '\\' || note
->type
== ' ')
906 if (note
->type
== ' ' && !in_comment
)
907 cpp_error_with_line (pfile
, CPP_DL_WARNING
, pfile
->line_table
->highest_line
, col
,
908 "backslash and newline separated by space");
910 if (buffer
->next_line
> buffer
->rlimit
)
912 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
, col
,
913 "backslash-newline at end of file");
914 /* Prevent "no newline at end of file" warning. */
915 buffer
->next_line
= buffer
->rlimit
;
918 buffer
->line_base
= note
->pos
;
919 CPP_INCREMENT_LINE (pfile
, 0);
921 else if (_cpp_trigraph_map
[note
->type
])
923 if (CPP_OPTION (pfile
, warn_trigraphs
)
924 && (!in_comment
|| warn_in_comment (pfile
, note
)))
926 if (CPP_OPTION (pfile
, trigraphs
))
927 cpp_warning_with_line (pfile
, CPP_W_TRIGRAPHS
,
928 pfile
->line_table
->highest_line
, col
,
929 "trigraph ??%c converted to %c",
931 (int) _cpp_trigraph_map
[note
->type
]);
934 cpp_warning_with_line
935 (pfile
, CPP_W_TRIGRAPHS
,
936 pfile
->line_table
->highest_line
, col
,
937 "trigraph ??%c ignored, use -trigraphs to enable",
942 else if (note
->type
== 0)
943 /* Already processed in lex_raw_string. */;
949 /* Skip a C-style block comment. We find the end of the comment by
950 seeing if an asterisk is before every '/' we encounter. Returns
951 nonzero if comment terminated by EOF, zero otherwise.
953 Buffer->cur points to the initial asterisk of the comment. */
955 _cpp_skip_block_comment (cpp_reader
*pfile
)
957 cpp_buffer
*buffer
= pfile
->buffer
;
958 const uchar
*cur
= buffer
->cur
;
967 /* People like decorating comments with '*', so check for '/'
968 instead for efficiency. */
976 /* Warn about potential nested comments, but not if the '/'
977 comes immediately before the true comment delimiter.
978 Don't bother to get it right across escaped newlines. */
979 if (CPP_OPTION (pfile
, warn_comments
)
980 && cur
[0] == '*' && cur
[1] != '/')
983 cpp_warning_with_line (pfile
, CPP_W_COMMENTS
,
984 pfile
->line_table
->highest_line
,
985 CPP_BUF_COL (buffer
),
986 "\"/*\" within comment");
992 buffer
->cur
= cur
- 1;
993 _cpp_process_line_notes (pfile
, true);
994 if (buffer
->next_line
>= buffer
->rlimit
)
996 _cpp_clean_line (pfile
);
998 cols
= buffer
->next_line
- buffer
->line_base
;
999 CPP_INCREMENT_LINE (pfile
, cols
);
1006 _cpp_process_line_notes (pfile
, true);
1010 /* Skip a C++ line comment, leaving buffer->cur pointing to the
1011 terminating newline. Handles escaped newlines. Returns nonzero
1012 if a multiline comment. */
1014 skip_line_comment (cpp_reader
*pfile
)
1016 cpp_buffer
*buffer
= pfile
->buffer
;
1017 source_location orig_line
= pfile
->line_table
->highest_line
;
1019 while (*buffer
->cur
!= '\n')
1022 _cpp_process_line_notes (pfile
, true);
1023 return orig_line
!= pfile
->line_table
->highest_line
;
1026 /* Skips whitespace, saving the next non-whitespace character. */
1028 skip_whitespace (cpp_reader
*pfile
, cppchar_t c
)
1030 cpp_buffer
*buffer
= pfile
->buffer
;
1031 bool saw_NUL
= false;
1035 /* Horizontal space always OK. */
1036 if (c
== ' ' || c
== '\t')
1038 /* Just \f \v or \0 left. */
1041 else if (pfile
->state
.in_directive
&& CPP_PEDANTIC (pfile
))
1042 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
,
1043 CPP_BUF_COL (buffer
),
1044 "%s in preprocessing directive",
1045 c
== '\f' ? "form feed" : "vertical tab");
1049 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
1050 while (is_nvspace (c
));
1053 cpp_error (pfile
, CPP_DL_WARNING
, "null character(s) ignored");
1058 /* See if the characters of a number token are valid in a name (no
1059 '.', '+' or '-'). */
1061 name_p (cpp_reader
*pfile
, const cpp_string
*string
)
1065 for (i
= 0; i
< string
->len
; i
++)
1066 if (!is_idchar (string
->text
[i
]))
1072 /* After parsing an identifier or other sequence, produce a warning about
1073 sequences not in NFC/NFKC. */
1075 warn_about_normalization (cpp_reader
*pfile
,
1076 const cpp_token
*token
,
1077 const struct normalize_state
*s
)
1079 if (CPP_OPTION (pfile
, warn_normalize
) < NORMALIZE_STATE_RESULT (s
)
1080 && !pfile
->state
.skipping
)
1082 /* Make sure that the token is printed using UCNs, even
1083 if we'd otherwise happily print UTF-8. */
1084 unsigned char *buf
= XNEWVEC (unsigned char, cpp_token_len (token
));
1087 sz
= cpp_spell_token (pfile
, token
, buf
, false) - buf
;
1088 if (NORMALIZE_STATE_RESULT (s
) == normalized_C
)
1089 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1090 "`%.*s' is not in NFKC", (int) sz
, buf
);
1092 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1093 "`%.*s' is not in NFC", (int) sz
, buf
);
1097 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1098 an identifier. FIRST is TRUE if this starts an identifier. */
1100 forms_identifier_p (cpp_reader
*pfile
, int first
,
1101 struct normalize_state
*state
)
1103 cpp_buffer
*buffer
= pfile
->buffer
;
1105 if (*buffer
->cur
== '$')
1107 if (!CPP_OPTION (pfile
, dollars_in_ident
))
1111 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1113 CPP_OPTION (pfile
, warn_dollars
) = 0;
1114 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1120 /* Is this a syntactically valid UCN? */
1121 if (CPP_OPTION (pfile
, extended_identifiers
)
1122 && *buffer
->cur
== '\\'
1123 && (buffer
->cur
[1] == 'u' || buffer
->cur
[1] == 'U'))
1126 if (_cpp_valid_ucn (pfile
, &buffer
->cur
, buffer
->rlimit
, 1 + !first
,
1135 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1136 static cpp_hashnode
*
1137 lex_identifier_intern (cpp_reader
*pfile
, const uchar
*base
)
1139 cpp_hashnode
*result
;
1142 unsigned int hash
= HT_HASHSTEP (0, *base
);
1145 while (ISIDNUM (*cur
))
1147 hash
= HT_HASHSTEP (hash
, *cur
);
1151 hash
= HT_HASHFINISH (hash
, len
);
1152 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1153 base
, len
, hash
, HT_ALLOC
));
1155 /* Rarely, identifiers require diagnostics when lexed. */
1156 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1157 && !pfile
->state
.skipping
, 0))
1159 /* It is allowed to poison the same identifier twice. */
1160 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1161 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1162 NODE_NAME (result
));
1164 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1165 replacement list of a variadic macro. */
1166 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1167 && !pfile
->state
.va_args_ok
)
1168 cpp_error (pfile
, CPP_DL_PEDWARN
,
1169 "__VA_ARGS__ can only appear in the expansion"
1170 " of a C99 variadic macro");
1172 /* For -Wc++-compat, warn about use of C++ named operators. */
1173 if (result
->flags
& NODE_WARN_OPERATOR
)
1174 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1175 "identifier \"%s\" is a special operator name in C++",
1176 NODE_NAME (result
));
1182 /* Get the cpp_hashnode of an identifier specified by NAME in
1183 the current cpp_reader object. If none is found, NULL is returned. */
1185 _cpp_lex_identifier (cpp_reader
*pfile
, const char *name
)
1187 cpp_hashnode
*result
;
1188 result
= lex_identifier_intern (pfile
, (uchar
*) name
);
1192 /* Lex an identifier starting at BUFFER->CUR - 1. */
1193 static cpp_hashnode
*
1194 lex_identifier (cpp_reader
*pfile
, const uchar
*base
, bool starts_ucn
,
1195 struct normalize_state
*nst
)
1197 cpp_hashnode
*result
;
1200 unsigned int hash
= HT_HASHSTEP (0, *base
);
1202 cur
= pfile
->buffer
->cur
;
1204 while (ISIDNUM (*cur
))
1206 hash
= HT_HASHSTEP (hash
, *cur
);
1209 pfile
->buffer
->cur
= cur
;
1210 if (starts_ucn
|| forms_identifier_p (pfile
, false, nst
))
1212 /* Slower version for identifiers containing UCNs (or $). */
1214 while (ISIDNUM (*pfile
->buffer
->cur
))
1216 pfile
->buffer
->cur
++;
1217 NORMALIZE_STATE_UPDATE_IDNUM (nst
);
1219 } while (forms_identifier_p (pfile
, false, nst
));
1220 result
= _cpp_interpret_identifier (pfile
, base
,
1221 pfile
->buffer
->cur
- base
);
1226 hash
= HT_HASHFINISH (hash
, len
);
1228 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1229 base
, len
, hash
, HT_ALLOC
));
1232 /* Rarely, identifiers require diagnostics when lexed. */
1233 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1234 && !pfile
->state
.skipping
, 0))
1236 /* It is allowed to poison the same identifier twice. */
1237 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1238 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1239 NODE_NAME (result
));
1241 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1242 replacement list of a variadic macro. */
1243 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1244 && !pfile
->state
.va_args_ok
)
1245 cpp_error (pfile
, CPP_DL_PEDWARN
,
1246 "__VA_ARGS__ can only appear in the expansion"
1247 " of a C99 variadic macro");
1249 /* For -Wc++-compat, warn about use of C++ named operators. */
1250 if (result
->flags
& NODE_WARN_OPERATOR
)
1251 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1252 "identifier \"%s\" is a special operator name in C++",
1253 NODE_NAME (result
));
1259 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1261 lex_number (cpp_reader
*pfile
, cpp_string
*number
,
1262 struct normalize_state
*nst
)
1268 base
= pfile
->buffer
->cur
- 1;
1271 cur
= pfile
->buffer
->cur
;
1273 /* N.B. ISIDNUM does not include $. */
1274 while (ISIDNUM (*cur
) || *cur
== '.' || VALID_SIGN (*cur
, cur
[-1]))
1277 NORMALIZE_STATE_UPDATE_IDNUM (nst
);
1280 pfile
->buffer
->cur
= cur
;
1282 while (forms_identifier_p (pfile
, false, nst
));
1284 number
->len
= cur
- base
;
1285 dest
= _cpp_unaligned_alloc (pfile
, number
->len
+ 1);
1286 memcpy (dest
, base
, number
->len
);
1287 dest
[number
->len
] = '\0';
1288 number
->text
= dest
;
1291 /* Create a token of type TYPE with a literal spelling. */
1293 create_literal (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1294 unsigned int len
, enum cpp_ttype type
)
1296 uchar
*dest
= _cpp_unaligned_alloc (pfile
, len
+ 1);
1298 memcpy (dest
, base
, len
);
1301 token
->val
.str
.len
= len
;
1302 token
->val
.str
.text
= dest
;
1305 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1306 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1309 bufring_append (cpp_reader
*pfile
, const uchar
*base
, size_t len
,
1310 _cpp_buff
**first_buff_p
, _cpp_buff
**last_buff_p
)
1312 _cpp_buff
*first_buff
= *first_buff_p
;
1313 _cpp_buff
*last_buff
= *last_buff_p
;
1315 if (first_buff
== NULL
)
1316 first_buff
= last_buff
= _cpp_get_buff (pfile
, len
);
1317 else if (len
> BUFF_ROOM (last_buff
))
1319 size_t room
= BUFF_ROOM (last_buff
);
1320 memcpy (BUFF_FRONT (last_buff
), base
, room
);
1321 BUFF_FRONT (last_buff
) += room
;
1324 last_buff
= _cpp_append_extend_buff (pfile
, last_buff
, len
);
1327 memcpy (BUFF_FRONT (last_buff
), base
, len
);
1328 BUFF_FRONT (last_buff
) += len
;
1330 *first_buff_p
= first_buff
;
1331 *last_buff_p
= last_buff
;
1334 /* Lexes a raw string. The stored string contains the spelling, including
1335 double quotes, delimiter string, '(' and ')', any leading
1336 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1337 literal, or CPP_OTHER if it was not properly terminated.
1339 The spelling is NUL-terminated, but it is not guaranteed that this
1340 is the first NUL since embedded NULs are preserved. */
1343 lex_raw_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1346 const uchar
*raw_prefix
;
1347 unsigned int raw_prefix_len
= 0;
1348 enum cpp_ttype type
;
1349 size_t total_len
= 0;
1350 _cpp_buff
*first_buff
= NULL
, *last_buff
= NULL
;
1351 _cpp_line_note
*note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1353 type
= (*base
== 'L' ? CPP_WSTRING
:
1354 *base
== 'U' ? CPP_STRING32
:
1355 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1358 raw_prefix
= cur
+ 1;
1359 while (raw_prefix_len
< 16)
1361 switch (raw_prefix
[raw_prefix_len
])
1363 case ' ': case '(': case ')': case '\\': case '\t':
1364 case '\v': case '\f': case '\n': default:
1366 /* Basic source charset except the above chars. */
1367 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1368 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1369 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1370 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1372 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1373 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1374 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1375 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1377 case '0': case '1': case '2': case '3': case '4': case '5':
1378 case '6': case '7': case '8': case '9':
1379 case '_': case '{': case '}': case '#': case '[': case ']':
1380 case '<': case '>': case '%': case ':': case ';': case '.':
1381 case '?': case '*': case '+': case '-': case '/': case '^':
1382 case '&': case '|': case '~': case '!': case '=': case ',':
1383 case '"': case '\'':
1390 if (raw_prefix
[raw_prefix_len
] != '(')
1392 int col
= CPP_BUF_COLUMN (pfile
->buffer
, raw_prefix
+ raw_prefix_len
)
1394 if (raw_prefix_len
== 16)
1395 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, col
,
1396 "raw string delimiter longer than 16 characters");
1398 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, col
,
1399 "invalid character '%c' in raw string delimiter",
1400 (int) raw_prefix
[raw_prefix_len
]);
1401 pfile
->buffer
->cur
= raw_prefix
- 1;
1402 create_literal (pfile
, token
, base
, raw_prefix
- 1 - base
, CPP_OTHER
);
1406 cur
= raw_prefix
+ raw_prefix_len
+ 1;
1409 #define BUF_APPEND(STR,LEN) \
1411 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1412 &first_buff, &last_buff); \
1413 total_len += (LEN); \
1418 /* If we previously performed any trigraph or line splicing
1419 transformations, undo them within the body of the raw string. */
1420 while (note
->pos
< cur
)
1422 for (; note
->pos
== cur
; ++note
)
1428 /* Restore backslash followed by newline. */
1429 BUF_APPEND (base
, cur
- base
);
1431 BUF_APPEND ("\\", 1);
1433 if (note
->type
== ' ')
1435 /* GNU backslash whitespace newline extension. FIXME
1436 could be any sequence of non-vertical space. When we
1437 can properly restore any such sequence, we should mark
1438 this note as handled so _cpp_process_line_notes
1440 BUF_APPEND (" ", 1);
1443 BUF_APPEND ("\n", 1);
1447 /* Already handled. */
1451 if (_cpp_trigraph_map
[note
->type
])
1453 /* Don't warn about this trigraph in
1454 _cpp_process_line_notes, since trigraphs show up as
1455 trigraphs in raw strings. */
1456 uchar type
= note
->type
;
1459 if (!CPP_OPTION (pfile
, trigraphs
))
1460 /* If we didn't convert the trigraph in the first
1461 place, don't do anything now either. */
1464 BUF_APPEND (base
, cur
- base
);
1466 BUF_APPEND ("??", 2);
1468 /* ??/ followed by newline gets two line notes, one for
1469 the trigraph and one for the backslash/newline. */
1470 if (type
== '/' && note
[1].pos
== cur
)
1472 if (note
[1].type
!= '\\'
1473 && note
[1].type
!= ' ')
1475 BUF_APPEND ("/", 1);
1477 goto after_backslash
;
1479 /* The ) from ??) could be part of the suffix. */
1480 else if (type
== ')'
1481 && strncmp ((const char *) cur
+1,
1482 (const char *) raw_prefix
,
1483 raw_prefix_len
) == 0
1484 && cur
[raw_prefix_len
+1] == '"')
1486 BUF_APPEND (")", 1);
1488 cur
+= raw_prefix_len
+ 2;
1489 goto break_outer_loop
;
1493 /* Skip the replacement character. */
1495 BUF_APPEND (&type
, 1);
1506 && strncmp ((const char *) cur
, (const char *) raw_prefix
,
1507 raw_prefix_len
) == 0
1508 && cur
[raw_prefix_len
] == '"')
1510 cur
+= raw_prefix_len
+ 1;
1515 if (pfile
->state
.in_directive
1516 || pfile
->state
.parsing_args
1517 || pfile
->state
.in_deferred_pragma
)
1521 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, 0,
1522 "unterminated raw string");
1526 BUF_APPEND (base
, cur
- base
);
1528 if (pfile
->buffer
->cur
< pfile
->buffer
->rlimit
)
1529 CPP_INCREMENT_LINE (pfile
, 0);
1530 pfile
->buffer
->need_line
= true;
1532 pfile
->buffer
->cur
= cur
-1;
1533 _cpp_process_line_notes (pfile
, false);
1534 if (!_cpp_get_fresh_line (pfile
))
1536 source_location src_loc
= token
->src_loc
;
1537 token
->type
= CPP_EOF
;
1538 /* Tell the compiler the line number of the EOF token. */
1539 token
->src_loc
= pfile
->line_table
->highest_line
;
1541 if (first_buff
!= NULL
)
1542 _cpp_release_buff (pfile
, first_buff
);
1543 cpp_error_with_line (pfile
, CPP_DL_ERROR
, src_loc
, 0,
1544 "unterminated raw string");
1548 cur
= base
= pfile
->buffer
->cur
;
1549 note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1554 if (CPP_OPTION (pfile
, user_literals
))
1556 /* According to C++11 [lex.ext]p10, a ud-suffix not starting with an
1557 underscore is ill-formed. Since this breaks programs using macros
1558 from inttypes.h, we generate a warning and treat the ud-suffix as a
1559 separate preprocessing token. This approach is under discussion by
1560 the standards committee, and has been adopted as a conforming
1561 extension by other front ends such as clang. */
1564 /* Raise a warning, but do not consume subsequent tokens. */
1565 if (CPP_OPTION (pfile
, warn_literal_suffix
))
1566 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1568 "invalid suffix on literal; C++11 requires "
1569 "a space between literal and identifier");
1571 /* Grab user defined literal suffix. */
1572 else if (*cur
== '_')
1574 type
= cpp_userdef_string_add_type (type
);
1577 while (ISIDNUM (*cur
))
1582 pfile
->buffer
->cur
= cur
;
1583 if (first_buff
== NULL
)
1584 create_literal (pfile
, token
, base
, cur
- base
, type
);
1587 uchar
*dest
= _cpp_unaligned_alloc (pfile
, total_len
+ (cur
- base
) + 1);
1590 token
->val
.str
.len
= total_len
+ (cur
- base
);
1591 token
->val
.str
.text
= dest
;
1592 last_buff
= first_buff
;
1593 while (last_buff
!= NULL
)
1595 memcpy (dest
, last_buff
->base
,
1596 BUFF_FRONT (last_buff
) - last_buff
->base
);
1597 dest
+= BUFF_FRONT (last_buff
) - last_buff
->base
;
1598 last_buff
= last_buff
->next
;
1600 _cpp_release_buff (pfile
, first_buff
);
1601 memcpy (dest
, base
, cur
- base
);
1602 dest
[cur
- base
] = '\0';
1606 /* Lexes a string, character constant, or angle-bracketed header file
1607 name. The stored string contains the spelling, including opening
1608 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1609 'R' modifier. It returns the type of the literal, or CPP_OTHER
1610 if it was not properly terminated, or CPP_LESS for an unterminated
1611 header name which must be relexed as normal tokens.
1613 The spelling is NUL-terminated, but it is not guaranteed that this
1614 is the first NUL since embedded NULs are preserved. */
1616 lex_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
)
1618 bool saw_NUL
= false;
1620 cppchar_t terminator
;
1621 enum cpp_ttype type
;
1624 terminator
= *cur
++;
1625 if (terminator
== 'L' || terminator
== 'U')
1626 terminator
= *cur
++;
1627 else if (terminator
== 'u')
1629 terminator
= *cur
++;
1630 if (terminator
== '8')
1631 terminator
= *cur
++;
1633 if (terminator
== 'R')
1635 lex_raw_string (pfile
, token
, base
, cur
);
1638 if (terminator
== '"')
1639 type
= (*base
== 'L' ? CPP_WSTRING
:
1640 *base
== 'U' ? CPP_STRING32
:
1641 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1643 else if (terminator
== '\'')
1644 type
= (*base
== 'L' ? CPP_WCHAR
:
1645 *base
== 'U' ? CPP_CHAR32
:
1646 *base
== 'u' ? CPP_CHAR16
: CPP_CHAR
);
1648 terminator
= '>', type
= CPP_HEADER_NAME
;
1652 cppchar_t c
= *cur
++;
1654 /* In #include-style directives, terminators are not escapable. */
1655 if (c
== '\\' && !pfile
->state
.angled_headers
&& *cur
!= '\n')
1657 else if (c
== terminator
)
1662 /* Unmatched quotes always yield undefined behavior, but
1663 greedy lexing means that what appears to be an unterminated
1664 header name may actually be a legitimate sequence of tokens. */
1665 if (terminator
== '>')
1667 token
->type
= CPP_LESS
;
1677 if (saw_NUL
&& !pfile
->state
.skipping
)
1678 cpp_error (pfile
, CPP_DL_WARNING
,
1679 "null character(s) preserved in literal");
1681 if (type
== CPP_OTHER
&& CPP_OPTION (pfile
, lang
) != CLK_ASM
)
1682 cpp_error (pfile
, CPP_DL_PEDWARN
, "missing terminating %c character",
1685 if (CPP_OPTION (pfile
, user_literals
))
1687 /* According to C++11 [lex.ext]p10, a ud-suffix not starting with an
1688 underscore is ill-formed. Since this breaks programs using macros
1689 from inttypes.h, we generate a warning and treat the ud-suffix as a
1690 separate preprocessing token. This approach is under discussion by
1691 the standards committee, and has been adopted as a conforming
1692 extension by other front ends such as clang. */
1695 /* Raise a warning, but do not consume subsequent tokens. */
1696 if (CPP_OPTION (pfile
, warn_literal_suffix
))
1697 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1699 "invalid suffix on literal; C++11 requires "
1700 "a space between literal and identifier");
1702 /* Grab user defined literal suffix. */
1703 else if (*cur
== '_')
1705 type
= cpp_userdef_char_add_type (type
);
1706 type
= cpp_userdef_string_add_type (type
);
1709 while (ISIDNUM (*cur
))
1714 pfile
->buffer
->cur
= cur
;
1715 create_literal (pfile
, token
, base
, cur
- base
, type
);
1718 /* Return the comment table. The client may not make any assumption
1719 about the ordering of the table. */
1721 cpp_get_comments (cpp_reader
*pfile
)
1723 return &pfile
->comments
;
1726 /* Append a comment to the end of the comment table. */
1728 store_comment (cpp_reader
*pfile
, cpp_token
*token
)
1732 if (pfile
->comments
.allocated
== 0)
1734 pfile
->comments
.allocated
= 256;
1735 pfile
->comments
.entries
= (cpp_comment
*) xmalloc
1736 (pfile
->comments
.allocated
* sizeof (cpp_comment
));
1739 if (pfile
->comments
.count
== pfile
->comments
.allocated
)
1741 pfile
->comments
.allocated
*= 2;
1742 pfile
->comments
.entries
= (cpp_comment
*) xrealloc
1743 (pfile
->comments
.entries
,
1744 pfile
->comments
.allocated
* sizeof (cpp_comment
));
1747 len
= token
->val
.str
.len
;
1749 /* Copy comment. Note, token may not be NULL terminated. */
1750 pfile
->comments
.entries
[pfile
->comments
.count
].comment
=
1751 (char *) xmalloc (sizeof (char) * (len
+ 1));
1752 memcpy (pfile
->comments
.entries
[pfile
->comments
.count
].comment
,
1753 token
->val
.str
.text
, len
);
1754 pfile
->comments
.entries
[pfile
->comments
.count
].comment
[len
] = '\0';
1756 /* Set source location. */
1757 pfile
->comments
.entries
[pfile
->comments
.count
].sloc
= token
->src_loc
;
1759 /* Increment the count of entries in the comment table. */
1760 pfile
->comments
.count
++;
1763 /* The stored comment includes the comment start and any terminator. */
1765 save_comment (cpp_reader
*pfile
, cpp_token
*token
, const unsigned char *from
,
1768 unsigned char *buffer
;
1769 unsigned int len
, clen
, i
;
1771 len
= pfile
->buffer
->cur
- from
+ 1; /* + 1 for the initial '/'. */
1773 /* C++ comments probably (not definitely) have moved past a new
1774 line, which we don't want to save in the comment. */
1775 if (is_vspace (pfile
->buffer
->cur
[-1]))
1778 /* If we are currently in a directive or in argument parsing, then
1779 we need to store all C++ comments as C comments internally, and
1780 so we need to allocate a little extra space in that case.
1782 Note that the only time we encounter a directive here is
1783 when we are saving comments in a "#define". */
1784 clen
= ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
)
1785 && type
== '/') ? len
+ 2 : len
;
1787 buffer
= _cpp_unaligned_alloc (pfile
, clen
);
1789 token
->type
= CPP_COMMENT
;
1790 token
->val
.str
.len
= clen
;
1791 token
->val
.str
.text
= buffer
;
1794 memcpy (buffer
+ 1, from
, len
- 1);
1796 /* Finish conversion to a C comment, if necessary. */
1797 if ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
) && type
== '/')
1800 buffer
[clen
- 2] = '*';
1801 buffer
[clen
- 1] = '/';
1802 /* As there can be in a C++ comments illegal sequences for C comments
1803 we need to filter them out. */
1804 for (i
= 2; i
< (clen
- 2); i
++)
1805 if (buffer
[i
] == '/' && (buffer
[i
- 1] == '*' || buffer
[i
+ 1] == '*'))
1809 /* Finally store this comment for use by clients of libcpp. */
1810 store_comment (pfile
, token
);
1813 /* Allocate COUNT tokens for RUN. */
1815 _cpp_init_tokenrun (tokenrun
*run
, unsigned int count
)
1817 run
->base
= XNEWVEC (cpp_token
, count
);
1818 run
->limit
= run
->base
+ count
;
1822 /* Returns the next tokenrun, or creates one if there is none. */
1824 next_tokenrun (tokenrun
*run
)
1826 if (run
->next
== NULL
)
1828 run
->next
= XNEW (tokenrun
);
1829 run
->next
->prev
= run
;
1830 _cpp_init_tokenrun (run
->next
, 250);
1836 /* Return the number of not yet processed token in a given
1839 _cpp_remaining_tokens_num_in_context (cpp_context
*context
)
1841 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
1842 return (LAST (context
).token
- FIRST (context
).token
);
1843 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
1844 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
1845 return (LAST (context
).ptoken
- FIRST (context
).ptoken
);
1850 /* Returns the token present at index INDEX in a given context. If
1851 INDEX is zero, the next token to be processed is returned. */
1852 static const cpp_token
*
1853 _cpp_token_from_context_at (cpp_context
*context
, int index
)
1855 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
1856 return &(FIRST (context
).token
[index
]);
1857 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
1858 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
1859 return FIRST (context
).ptoken
[index
];
1864 /* Look ahead in the input stream. */
1866 cpp_peek_token (cpp_reader
*pfile
, int index
)
1868 cpp_context
*context
= pfile
->context
;
1869 const cpp_token
*peektok
;
1872 /* First, scan through any pending cpp_context objects. */
1873 while (context
->prev
)
1875 ptrdiff_t sz
= _cpp_remaining_tokens_num_in_context (context
);
1877 if (index
< (int) sz
)
1878 return _cpp_token_from_context_at (context
, index
);
1880 context
= context
->prev
;
1883 /* We will have to read some new tokens after all (and do so
1884 without invalidating preceding tokens). */
1886 pfile
->keep_tokens
++;
1890 peektok
= _cpp_lex_token (pfile
);
1891 if (peektok
->type
== CPP_EOF
)
1896 _cpp_backup_tokens_direct (pfile
, count
+ 1);
1897 pfile
->keep_tokens
--;
1902 /* Allocate a single token that is invalidated at the same time as the
1903 rest of the tokens on the line. Has its line and col set to the
1904 same as the last lexed token, so that diagnostics appear in the
1907 _cpp_temp_token (cpp_reader
*pfile
)
1909 cpp_token
*old
, *result
;
1910 ptrdiff_t sz
= pfile
->cur_run
->limit
- pfile
->cur_token
;
1911 ptrdiff_t la
= (ptrdiff_t) pfile
->lookaheads
;
1913 old
= pfile
->cur_token
- 1;
1914 /* Any pre-existing lookaheads must not be clobbered. */
1919 tokenrun
*next
= next_tokenrun (pfile
->cur_run
);
1922 memmove (next
->base
+ 1, next
->base
,
1923 (la
- sz
) * sizeof (cpp_token
));
1925 next
->base
[0] = pfile
->cur_run
->limit
[-1];
1929 memmove (pfile
->cur_token
+ 1, pfile
->cur_token
,
1930 MIN (la
, sz
- 1) * sizeof (cpp_token
));
1933 if (!sz
&& pfile
->cur_token
== pfile
->cur_run
->limit
)
1935 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
1936 pfile
->cur_token
= pfile
->cur_run
->base
;
1939 result
= pfile
->cur_token
++;
1940 result
->src_loc
= old
->src_loc
;
1944 /* Lex a token into RESULT (external interface). Takes care of issues
1945 like directive handling, token lookahead, multiple include
1946 optimization and skipping. */
1948 _cpp_lex_token (cpp_reader
*pfile
)
1954 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
1956 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
1957 pfile
->cur_token
= pfile
->cur_run
->base
;
1959 /* We assume that the current token is somewhere in the current
1961 if (pfile
->cur_token
< pfile
->cur_run
->base
1962 || pfile
->cur_token
>= pfile
->cur_run
->limit
)
1965 if (pfile
->lookaheads
)
1967 pfile
->lookaheads
--;
1968 result
= pfile
->cur_token
++;
1971 result
= _cpp_lex_direct (pfile
);
1973 if (result
->flags
& BOL
)
1975 /* Is this a directive. If _cpp_handle_directive returns
1976 false, it is an assembler #. */
1977 if (result
->type
== CPP_HASH
1978 /* 6.10.3 p 11: Directives in a list of macro arguments
1979 gives undefined behavior. This implementation
1980 handles the directive as normal. */
1981 && pfile
->state
.parsing_args
!= 1)
1983 if (_cpp_handle_directive (pfile
, result
->flags
& PREV_WHITE
))
1985 if (pfile
->directive_result
.type
== CPP_PADDING
)
1987 result
= &pfile
->directive_result
;
1990 else if (pfile
->state
.in_deferred_pragma
)
1991 result
= &pfile
->directive_result
;
1993 if (pfile
->cb
.line_change
&& !pfile
->state
.skipping
)
1994 pfile
->cb
.line_change (pfile
, result
, pfile
->state
.parsing_args
);
1997 /* We don't skip tokens in directives. */
1998 if (pfile
->state
.in_directive
|| pfile
->state
.in_deferred_pragma
)
2001 /* Outside a directive, invalidate controlling macros. At file
2002 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
2003 get here and MI optimization works. */
2004 pfile
->mi_valid
= false;
2006 if (!pfile
->state
.skipping
|| result
->type
== CPP_EOF
)
2013 /* Returns true if a fresh line has been loaded. */
2015 _cpp_get_fresh_line (cpp_reader
*pfile
)
2019 /* We can't get a new line until we leave the current directive. */
2020 if (pfile
->state
.in_directive
)
2025 cpp_buffer
*buffer
= pfile
->buffer
;
2027 if (!buffer
->need_line
)
2030 if (buffer
->next_line
< buffer
->rlimit
)
2032 _cpp_clean_line (pfile
);
2036 /* First, get out of parsing arguments state. */
2037 if (pfile
->state
.parsing_args
)
2040 /* End of buffer. Non-empty files should end in a newline. */
2041 if (buffer
->buf
!= buffer
->rlimit
2042 && buffer
->next_line
> buffer
->rlimit
2043 && !buffer
->from_stage3
)
2045 /* Clip to buffer size. */
2046 buffer
->next_line
= buffer
->rlimit
;
2049 return_at_eof
= buffer
->return_at_eof
;
2050 _cpp_pop_buffer (pfile
);
2051 if (pfile
->buffer
== NULL
|| return_at_eof
)
2056 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
2059 result->type = ELSE_TYPE; \
2060 if (*buffer->cur == CHAR) \
2061 buffer->cur++, result->type = THEN_TYPE; \
2065 /* Lex a token into pfile->cur_token, which is also incremented, to
2066 get diagnostics pointing to the correct location.
2068 Does not handle issues such as token lookahead, multiple-include
2069 optimization, directives, skipping etc. This function is only
2070 suitable for use by _cpp_lex_token, and in special cases like
2071 lex_expansion_token which doesn't care for any of these issues.
2073 When meeting a newline, returns CPP_EOF if parsing a directive,
2074 otherwise returns to the start of the token buffer if permissible.
2075 Returns the location of the lexed token. */
2077 _cpp_lex_direct (cpp_reader
*pfile
)
2081 const unsigned char *comment_start
;
2082 cpp_token
*result
= pfile
->cur_token
++;
2086 buffer
= pfile
->buffer
;
2087 if (buffer
->need_line
)
2089 if (pfile
->state
.in_deferred_pragma
)
2091 result
->type
= CPP_PRAGMA_EOL
;
2092 pfile
->state
.in_deferred_pragma
= false;
2093 if (!pfile
->state
.pragma_allow_expansion
)
2094 pfile
->state
.prevent_expansion
--;
2097 if (!_cpp_get_fresh_line (pfile
))
2099 result
->type
= CPP_EOF
;
2100 if (!pfile
->state
.in_directive
)
2102 /* Tell the compiler the line number of the EOF token. */
2103 result
->src_loc
= pfile
->line_table
->highest_line
;
2104 result
->flags
= BOL
;
2108 if (!pfile
->keep_tokens
)
2110 pfile
->cur_run
= &pfile
->base_run
;
2111 result
= pfile
->base_run
.base
;
2112 pfile
->cur_token
= result
+ 1;
2114 result
->flags
= BOL
;
2115 if (pfile
->state
.parsing_args
== 2)
2116 result
->flags
|= PREV_WHITE
;
2118 buffer
= pfile
->buffer
;
2120 result
->src_loc
= pfile
->line_table
->highest_line
;
2123 if (buffer
->cur
>= buffer
->notes
[buffer
->cur_note
].pos
2124 && !pfile
->overlaid_buffer
)
2126 _cpp_process_line_notes (pfile
, false);
2127 result
->src_loc
= pfile
->line_table
->highest_line
;
2131 if (pfile
->forced_token_location_p
)
2132 result
->src_loc
= *pfile
->forced_token_location_p
;
2134 result
->src_loc
= linemap_position_for_column (pfile
->line_table
,
2135 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2139 case ' ': case '\t': case '\f': case '\v': case '\0':
2140 result
->flags
|= PREV_WHITE
;
2141 skip_whitespace (pfile
, c
);
2145 if (buffer
->cur
< buffer
->rlimit
)
2146 CPP_INCREMENT_LINE (pfile
, 0);
2147 buffer
->need_line
= true;
2150 case '0': case '1': case '2': case '3': case '4':
2151 case '5': case '6': case '7': case '8': case '9':
2153 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2154 result
->type
= CPP_NUMBER
;
2155 lex_number (pfile
, &result
->val
.str
, &nst
);
2156 warn_about_normalization (pfile
, result
, &nst
);
2164 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2165 wide strings or raw strings. */
2166 if (c
== 'L' || CPP_OPTION (pfile
, rliterals
)
2167 || (c
!= 'R' && CPP_OPTION (pfile
, uliterals
)))
2169 if ((*buffer
->cur
== '\'' && c
!= 'R')
2170 || *buffer
->cur
== '"'
2171 || (*buffer
->cur
== 'R'
2173 && buffer
->cur
[1] == '"'
2174 && CPP_OPTION (pfile
, rliterals
))
2175 || (*buffer
->cur
== '8'
2177 && (buffer
->cur
[1] == '"'
2178 || (buffer
->cur
[1] == 'R' && buffer
->cur
[2] == '"'
2179 && CPP_OPTION (pfile
, rliterals
)))))
2181 lex_string (pfile
, result
, buffer
->cur
- 1);
2188 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2189 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2190 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2191 case 's': case 't': case 'v': case 'w': case 'x':
2193 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2194 case 'G': case 'H': case 'I': case 'J': case 'K':
2195 case 'M': case 'N': case 'O': case 'P': case 'Q':
2196 case 'S': case 'T': case 'V': case 'W': case 'X':
2198 result
->type
= CPP_NAME
;
2200 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2201 result
->val
.node
.node
= lex_identifier (pfile
, buffer
->cur
- 1, false,
2203 warn_about_normalization (pfile
, result
, &nst
);
2206 /* Convert named operators to their proper types. */
2207 if (result
->val
.node
.node
->flags
& NODE_OPERATOR
)
2209 result
->flags
|= NAMED_OP
;
2210 result
->type
= (enum cpp_ttype
) result
->val
.node
.node
->directive_index
;
2216 lex_string (pfile
, result
, buffer
->cur
- 1);
2220 /* A potential block or line comment. */
2221 comment_start
= buffer
->cur
;
2226 if (_cpp_skip_block_comment (pfile
))
2227 cpp_error (pfile
, CPP_DL_ERROR
, "unterminated comment");
2229 else if (c
== '/' && (CPP_OPTION (pfile
, cplusplus_comments
)
2230 || cpp_in_system_header (pfile
)))
2232 /* Warn about comments only if pedantically GNUC89, and not
2233 in system headers. */
2234 if (CPP_OPTION (pfile
, lang
) == CLK_GNUC89
&& CPP_PEDANTIC (pfile
)
2235 && ! buffer
->warned_cplusplus_comments
)
2237 cpp_error (pfile
, CPP_DL_PEDWARN
,
2238 "C++ style comments are not allowed in ISO C90");
2239 cpp_error (pfile
, CPP_DL_PEDWARN
,
2240 "(this will be reported only once per input file)");
2241 buffer
->warned_cplusplus_comments
= 1;
2244 if (skip_line_comment (pfile
) && CPP_OPTION (pfile
, warn_comments
))
2245 cpp_warning (pfile
, CPP_W_COMMENTS
, "multi-line comment");
2250 result
->type
= CPP_DIV_EQ
;
2255 result
->type
= CPP_DIV
;
2259 if (!pfile
->state
.save_comments
)
2261 result
->flags
|= PREV_WHITE
;
2262 goto update_tokens_line
;
2265 /* Save the comment as a token in its own right. */
2266 save_comment (pfile
, result
, comment_start
, c
);
2270 if (pfile
->state
.angled_headers
)
2272 lex_string (pfile
, result
, buffer
->cur
- 1);
2273 if (result
->type
!= CPP_LESS
)
2277 result
->type
= CPP_LESS
;
2278 if (*buffer
->cur
== '=')
2279 buffer
->cur
++, result
->type
= CPP_LESS_EQ
;
2280 else if (*buffer
->cur
== '<')
2283 IF_NEXT_IS ('=', CPP_LSHIFT_EQ
, CPP_LSHIFT
);
2285 else if (CPP_OPTION (pfile
, digraphs
))
2287 if (*buffer
->cur
== ':')
2290 result
->flags
|= DIGRAPH
;
2291 result
->type
= CPP_OPEN_SQUARE
;
2293 else if (*buffer
->cur
== '%')
2296 result
->flags
|= DIGRAPH
;
2297 result
->type
= CPP_OPEN_BRACE
;
2303 result
->type
= CPP_GREATER
;
2304 if (*buffer
->cur
== '=')
2305 buffer
->cur
++, result
->type
= CPP_GREATER_EQ
;
2306 else if (*buffer
->cur
== '>')
2309 IF_NEXT_IS ('=', CPP_RSHIFT_EQ
, CPP_RSHIFT
);
2314 result
->type
= CPP_MOD
;
2315 if (*buffer
->cur
== '=')
2316 buffer
->cur
++, result
->type
= CPP_MOD_EQ
;
2317 else if (CPP_OPTION (pfile
, digraphs
))
2319 if (*buffer
->cur
== ':')
2322 result
->flags
|= DIGRAPH
;
2323 result
->type
= CPP_HASH
;
2324 if (*buffer
->cur
== '%' && buffer
->cur
[1] == ':')
2325 buffer
->cur
+= 2, result
->type
= CPP_PASTE
, result
->val
.token_no
= 0;
2327 else if (*buffer
->cur
== '>')
2330 result
->flags
|= DIGRAPH
;
2331 result
->type
= CPP_CLOSE_BRACE
;
2337 result
->type
= CPP_DOT
;
2338 if (ISDIGIT (*buffer
->cur
))
2340 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2341 result
->type
= CPP_NUMBER
;
2342 lex_number (pfile
, &result
->val
.str
, &nst
);
2343 warn_about_normalization (pfile
, result
, &nst
);
2345 else if (*buffer
->cur
== '.' && buffer
->cur
[1] == '.')
2346 buffer
->cur
+= 2, result
->type
= CPP_ELLIPSIS
;
2347 else if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2348 buffer
->cur
++, result
->type
= CPP_DOT_STAR
;
2352 result
->type
= CPP_PLUS
;
2353 if (*buffer
->cur
== '+')
2354 buffer
->cur
++, result
->type
= CPP_PLUS_PLUS
;
2355 else if (*buffer
->cur
== '=')
2356 buffer
->cur
++, result
->type
= CPP_PLUS_EQ
;
2360 result
->type
= CPP_MINUS
;
2361 if (*buffer
->cur
== '>')
2364 result
->type
= CPP_DEREF
;
2365 if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2366 buffer
->cur
++, result
->type
= CPP_DEREF_STAR
;
2368 else if (*buffer
->cur
== '-')
2369 buffer
->cur
++, result
->type
= CPP_MINUS_MINUS
;
2370 else if (*buffer
->cur
== '=')
2371 buffer
->cur
++, result
->type
= CPP_MINUS_EQ
;
2375 result
->type
= CPP_AND
;
2376 if (*buffer
->cur
== '&')
2377 buffer
->cur
++, result
->type
= CPP_AND_AND
;
2378 else if (*buffer
->cur
== '=')
2379 buffer
->cur
++, result
->type
= CPP_AND_EQ
;
2383 result
->type
= CPP_OR
;
2384 if (*buffer
->cur
== '|')
2385 buffer
->cur
++, result
->type
= CPP_OR_OR
;
2386 else if (*buffer
->cur
== '=')
2387 buffer
->cur
++, result
->type
= CPP_OR_EQ
;
2391 result
->type
= CPP_COLON
;
2392 if (*buffer
->cur
== ':' && CPP_OPTION (pfile
, cplusplus
))
2393 buffer
->cur
++, result
->type
= CPP_SCOPE
;
2394 else if (*buffer
->cur
== '>' && CPP_OPTION (pfile
, digraphs
))
2397 result
->flags
|= DIGRAPH
;
2398 result
->type
= CPP_CLOSE_SQUARE
;
2402 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ
, CPP_MULT
); break;
2403 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ
, CPP_EQ
); break;
2404 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ
, CPP_NOT
); break;
2405 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ
, CPP_XOR
); break;
2406 case '#': IF_NEXT_IS ('#', CPP_PASTE
, CPP_HASH
); result
->val
.token_no
= 0; break;
2408 case '?': result
->type
= CPP_QUERY
; break;
2409 case '~': result
->type
= CPP_COMPL
; break;
2410 case ',': result
->type
= CPP_COMMA
; break;
2411 case '(': result
->type
= CPP_OPEN_PAREN
; break;
2412 case ')': result
->type
= CPP_CLOSE_PAREN
; break;
2413 case '[': result
->type
= CPP_OPEN_SQUARE
; break;
2414 case ']': result
->type
= CPP_CLOSE_SQUARE
; break;
2415 case '{': result
->type
= CPP_OPEN_BRACE
; break;
2416 case '}': result
->type
= CPP_CLOSE_BRACE
; break;
2417 case ';': result
->type
= CPP_SEMICOLON
; break;
2419 /* @ is a punctuator in Objective-C. */
2420 case '@': result
->type
= CPP_ATSIGN
; break;
2425 const uchar
*base
= --buffer
->cur
;
2426 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2428 if (forms_identifier_p (pfile
, true, &nst
))
2430 result
->type
= CPP_NAME
;
2431 result
->val
.node
.node
= lex_identifier (pfile
, base
, true, &nst
);
2432 warn_about_normalization (pfile
, result
, &nst
);
2439 create_literal (pfile
, result
, buffer
->cur
- 1, 1, CPP_OTHER
);
2446 /* An upper bound on the number of bytes needed to spell TOKEN.
2447 Does not include preceding whitespace. */
2449 cpp_token_len (const cpp_token
*token
)
2453 switch (TOKEN_SPELL (token
))
2455 default: len
= 6; break;
2456 case SPELL_LITERAL
: len
= token
->val
.str
.len
; break;
2457 case SPELL_IDENT
: len
= NODE_LEN (token
->val
.node
.node
) * 10; break;
2463 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2464 Return the number of bytes read out of NAME. (There are always
2465 10 bytes written to BUFFER.) */
2468 utf8_to_ucn (unsigned char *buffer
, const unsigned char *name
)
2474 unsigned long utf32
;
2476 /* Compute the length of the UTF-8 sequence. */
2477 for (t
= *name
; t
& 0x80; t
<<= 1)
2480 utf32
= *name
& (0x7F >> ucn_len
);
2481 for (ucn_len_c
= 1; ucn_len_c
< ucn_len
; ucn_len_c
++)
2483 utf32
= (utf32
<< 6) | (*++name
& 0x3F);
2485 /* Ill-formed UTF-8. */
2486 if ((*name
& ~0x3F) != 0x80)
2492 for (j
= 7; j
>= 0; j
--)
2493 *buffer
++ = "0123456789abcdef"[(utf32
>> (4 * j
)) & 0xF];
2497 /* Given a token TYPE corresponding to a digraph, return a pointer to
2498 the spelling of the digraph. */
2499 static const unsigned char *
2500 cpp_digraph2name (enum cpp_ttype type
)
2502 return digraph_spellings
[(int) type
- (int) CPP_FIRST_DIGRAPH
];
2505 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
2506 already contain the enough space to hold the token's spelling.
2507 Returns a pointer to the character after the last character written.
2508 FORSTRING is true if this is to be the spelling after translation
2509 phase 1 (this is different for UCNs).
2510 FIXME: Would be nice if we didn't need the PFILE argument. */
2512 cpp_spell_token (cpp_reader
*pfile
, const cpp_token
*token
,
2513 unsigned char *buffer
, bool forstring
)
2515 switch (TOKEN_SPELL (token
))
2517 case SPELL_OPERATOR
:
2519 const unsigned char *spelling
;
2522 if (token
->flags
& DIGRAPH
)
2523 spelling
= cpp_digraph2name (token
->type
);
2524 else if (token
->flags
& NAMED_OP
)
2527 spelling
= TOKEN_NAME (token
);
2529 while ((c
= *spelling
++) != '\0')
2538 memcpy (buffer
, NODE_NAME (token
->val
.node
.node
),
2539 NODE_LEN (token
->val
.node
.node
));
2540 buffer
+= NODE_LEN (token
->val
.node
.node
);
2545 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
2547 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
2548 if (name
[i
] & ~0x7F)
2550 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2554 *buffer
++ = NODE_NAME (token
->val
.node
.node
)[i
];
2559 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
2560 buffer
+= token
->val
.str
.len
;
2564 cpp_error (pfile
, CPP_DL_ICE
,
2565 "unspellable token %s", TOKEN_NAME (token
));
2572 /* Returns TOKEN spelt as a null-terminated string. The string is
2573 freed when the reader is destroyed. Useful for diagnostics. */
2575 cpp_token_as_text (cpp_reader
*pfile
, const cpp_token
*token
)
2577 unsigned int len
= cpp_token_len (token
) + 1;
2578 unsigned char *start
= _cpp_unaligned_alloc (pfile
, len
), *end
;
2580 end
= cpp_spell_token (pfile
, token
, start
, false);
2586 /* Returns a pointer to a string which spells the token defined by
2587 TYPE and FLAGS. Used by C front ends, which really should move to
2588 using cpp_token_as_text. */
2590 cpp_type2name (enum cpp_ttype type
, unsigned char flags
)
2592 if (flags
& DIGRAPH
)
2593 return (const char *) cpp_digraph2name (type
);
2594 else if (flags
& NAMED_OP
)
2595 return cpp_named_operator2name (type
);
2597 return (const char *) token_spellings
[type
].name
;
2600 /* Writes the spelling of token to FP, without any preceding space.
2601 Separated from cpp_spell_token for efficiency - to avoid stdio
2602 double-buffering. */
2604 cpp_output_token (const cpp_token
*token
, FILE *fp
)
2606 switch (TOKEN_SPELL (token
))
2608 case SPELL_OPERATOR
:
2610 const unsigned char *spelling
;
2613 if (token
->flags
& DIGRAPH
)
2614 spelling
= cpp_digraph2name (token
->type
);
2615 else if (token
->flags
& NAMED_OP
)
2618 spelling
= TOKEN_NAME (token
);
2623 while ((c
= *++spelling
) != '\0');
2631 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
2633 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
2634 if (name
[i
] & ~0x7F)
2636 unsigned char buffer
[10];
2637 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2638 fwrite (buffer
, 1, 10, fp
);
2641 fputc (NODE_NAME (token
->val
.node
.node
)[i
], fp
);
2646 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
2650 /* An error, most probably. */
2655 /* Compare two tokens. */
2657 _cpp_equiv_tokens (const cpp_token
*a
, const cpp_token
*b
)
2659 if (a
->type
== b
->type
&& a
->flags
== b
->flags
)
2660 switch (TOKEN_SPELL (a
))
2662 default: /* Keep compiler happy. */
2663 case SPELL_OPERATOR
:
2664 /* token_no is used to track where multiple consecutive ##
2665 tokens were originally located. */
2666 return (a
->type
!= CPP_PASTE
|| a
->val
.token_no
== b
->val
.token_no
);
2668 return (a
->type
!= CPP_MACRO_ARG
2669 || a
->val
.macro_arg
.arg_no
== b
->val
.macro_arg
.arg_no
);
2671 return a
->val
.node
.node
== b
->val
.node
.node
;
2673 return (a
->val
.str
.len
== b
->val
.str
.len
2674 && !memcmp (a
->val
.str
.text
, b
->val
.str
.text
,
2681 /* Returns nonzero if a space should be inserted to avoid an
2682 accidental token paste for output. For simplicity, it is
2683 conservative, and occasionally advises a space where one is not
2684 needed, e.g. "." and ".2". */
2686 cpp_avoid_paste (cpp_reader
*pfile
, const cpp_token
*token1
,
2687 const cpp_token
*token2
)
2689 enum cpp_ttype a
= token1
->type
, b
= token2
->type
;
2692 if (token1
->flags
& NAMED_OP
)
2694 if (token2
->flags
& NAMED_OP
)
2698 if (token2
->flags
& DIGRAPH
)
2699 c
= digraph_spellings
[(int) b
- (int) CPP_FIRST_DIGRAPH
][0];
2700 else if (token_spellings
[b
].category
== SPELL_OPERATOR
)
2701 c
= token_spellings
[b
].name
[0];
2703 /* Quickly get everything that can paste with an '='. */
2704 if ((int) a
<= (int) CPP_LAST_EQ
&& c
== '=')
2709 case CPP_GREATER
: return c
== '>';
2710 case CPP_LESS
: return c
== '<' || c
== '%' || c
== ':';
2711 case CPP_PLUS
: return c
== '+';
2712 case CPP_MINUS
: return c
== '-' || c
== '>';
2713 case CPP_DIV
: return c
== '/' || c
== '*'; /* Comments. */
2714 case CPP_MOD
: return c
== ':' || c
== '>';
2715 case CPP_AND
: return c
== '&';
2716 case CPP_OR
: return c
== '|';
2717 case CPP_COLON
: return c
== ':' || c
== '>';
2718 case CPP_DEREF
: return c
== '*';
2719 case CPP_DOT
: return c
== '.' || c
== '%' || b
== CPP_NUMBER
;
2720 case CPP_HASH
: return c
== '#' || c
== '%'; /* Digraph form. */
2721 case CPP_NAME
: return ((b
== CPP_NUMBER
2722 && name_p (pfile
, &token2
->val
.str
))
2724 || b
== CPP_CHAR
|| b
== CPP_STRING
); /* L */
2725 case CPP_NUMBER
: return (b
== CPP_NUMBER
|| b
== CPP_NAME
2726 || c
== '.' || c
== '+' || c
== '-');
2728 case CPP_OTHER
: return ((token1
->val
.str
.text
[0] == '\\'
2730 || (CPP_OPTION (pfile
, objc
)
2731 && token1
->val
.str
.text
[0] == '@'
2732 && (b
== CPP_NAME
|| b
== CPP_STRING
)));
2739 /* Output all the remaining tokens on the current line, and a newline
2740 character, to FP. Leading whitespace is removed. If there are
2741 macros, special token padding is not performed. */
2743 cpp_output_line (cpp_reader
*pfile
, FILE *fp
)
2745 const cpp_token
*token
;
2747 token
= cpp_get_token (pfile
);
2748 while (token
->type
!= CPP_EOF
)
2750 cpp_output_token (token
, fp
);
2751 token
= cpp_get_token (pfile
);
2752 if (token
->flags
& PREV_WHITE
)
2759 /* Return a string representation of all the remaining tokens on the
2760 current line. The result is allocated using xmalloc and must be
2761 freed by the caller. */
2763 cpp_output_line_to_string (cpp_reader
*pfile
, const unsigned char *dir_name
)
2765 const cpp_token
*token
;
2766 unsigned int out
= dir_name
? ustrlen (dir_name
) : 0;
2767 unsigned int alloced
= 120 + out
;
2768 unsigned char *result
= (unsigned char *) xmalloc (alloced
);
2770 /* If DIR_NAME is empty, there are no initial contents. */
2773 sprintf ((char *) result
, "#%s ", dir_name
);
2777 token
= cpp_get_token (pfile
);
2778 while (token
->type
!= CPP_EOF
)
2780 unsigned char *last
;
2781 /* Include room for a possible space and the terminating nul. */
2782 unsigned int len
= cpp_token_len (token
) + 2;
2784 if (out
+ len
> alloced
)
2787 if (out
+ len
> alloced
)
2788 alloced
= out
+ len
;
2789 result
= (unsigned char *) xrealloc (result
, alloced
);
2792 last
= cpp_spell_token (pfile
, token
, &result
[out
], 0);
2793 out
= last
- result
;
2795 token
= cpp_get_token (pfile
);
2796 if (token
->flags
& PREV_WHITE
)
2797 result
[out
++] = ' ';
2804 /* Memory buffers. Changing these three constants can have a dramatic
2805 effect on performance. The values here are reasonable defaults,
2806 but might be tuned. If you adjust them, be sure to test across a
2807 range of uses of cpplib, including heavy nested function-like macro
2808 expansion. Also check the change in peak memory usage (NJAMD is a
2809 good tool for this). */
2810 #define MIN_BUFF_SIZE 8000
2811 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
2812 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
2813 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
2815 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
2816 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
2819 /* Create a new allocation buffer. Place the control block at the end
2820 of the buffer, so that buffer overflows will cause immediate chaos. */
2822 new_buff (size_t len
)
2825 unsigned char *base
;
2827 if (len
< MIN_BUFF_SIZE
)
2828 len
= MIN_BUFF_SIZE
;
2829 len
= CPP_ALIGN (len
);
2831 base
= XNEWVEC (unsigned char, len
+ sizeof (_cpp_buff
));
2832 result
= (_cpp_buff
*) (base
+ len
);
2833 result
->base
= base
;
2835 result
->limit
= base
+ len
;
2836 result
->next
= NULL
;
2840 /* Place a chain of unwanted allocation buffers on the free list. */
2842 _cpp_release_buff (cpp_reader
*pfile
, _cpp_buff
*buff
)
2844 _cpp_buff
*end
= buff
;
2848 end
->next
= pfile
->free_buffs
;
2849 pfile
->free_buffs
= buff
;
2852 /* Return a free buffer of size at least MIN_SIZE. */
2854 _cpp_get_buff (cpp_reader
*pfile
, size_t min_size
)
2856 _cpp_buff
*result
, **p
;
2858 for (p
= &pfile
->free_buffs
;; p
= &(*p
)->next
)
2863 return new_buff (min_size
);
2865 size
= result
->limit
- result
->base
;
2866 /* Return a buffer that's big enough, but don't waste one that's
2868 if (size
>= min_size
&& size
<= BUFF_SIZE_UPPER_BOUND (min_size
))
2873 result
->next
= NULL
;
2874 result
->cur
= result
->base
;
2878 /* Creates a new buffer with enough space to hold the uncommitted
2879 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
2880 the excess bytes to the new buffer. Chains the new buffer after
2881 BUFF, and returns the new buffer. */
2883 _cpp_append_extend_buff (cpp_reader
*pfile
, _cpp_buff
*buff
, size_t min_extra
)
2885 size_t size
= EXTENDED_BUFF_SIZE (buff
, min_extra
);
2886 _cpp_buff
*new_buff
= _cpp_get_buff (pfile
, size
);
2888 buff
->next
= new_buff
;
2889 memcpy (new_buff
->base
, buff
->cur
, BUFF_ROOM (buff
));
2893 /* Creates a new buffer with enough space to hold the uncommitted
2894 remaining bytes of the buffer pointed to by BUFF, and at least
2895 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
2896 Chains the new buffer before the buffer pointed to by BUFF, and
2897 updates the pointer to point to the new buffer. */
2899 _cpp_extend_buff (cpp_reader
*pfile
, _cpp_buff
**pbuff
, size_t min_extra
)
2901 _cpp_buff
*new_buff
, *old_buff
= *pbuff
;
2902 size_t size
= EXTENDED_BUFF_SIZE (old_buff
, min_extra
);
2904 new_buff
= _cpp_get_buff (pfile
, size
);
2905 memcpy (new_buff
->base
, old_buff
->cur
, BUFF_ROOM (old_buff
));
2906 new_buff
->next
= old_buff
;
2910 /* Free a chain of buffers starting at BUFF. */
2912 _cpp_free_buff (_cpp_buff
*buff
)
2916 for (; buff
; buff
= next
)
2923 /* Allocate permanent, unaligned storage of length LEN. */
2925 _cpp_unaligned_alloc (cpp_reader
*pfile
, size_t len
)
2927 _cpp_buff
*buff
= pfile
->u_buff
;
2928 unsigned char *result
= buff
->cur
;
2930 if (len
> (size_t) (buff
->limit
- result
))
2932 buff
= _cpp_get_buff (pfile
, len
);
2933 buff
->next
= pfile
->u_buff
;
2934 pfile
->u_buff
= buff
;
2938 buff
->cur
= result
+ len
;
2942 /* Allocate permanent, unaligned storage of length LEN from a_buff.
2943 That buffer is used for growing allocations when saving macro
2944 replacement lists in a #define, and when parsing an answer to an
2945 assertion in #assert, #unassert or #if (and therefore possibly
2946 whilst expanding macros). It therefore must not be used by any
2947 code that they might call: specifically the lexer and the guts of
2950 All existing other uses clearly fit this restriction: storing
2951 registered pragmas during initialization. */
2953 _cpp_aligned_alloc (cpp_reader
*pfile
, size_t len
)
2955 _cpp_buff
*buff
= pfile
->a_buff
;
2956 unsigned char *result
= buff
->cur
;
2958 if (len
> (size_t) (buff
->limit
- result
))
2960 buff
= _cpp_get_buff (pfile
, len
);
2961 buff
->next
= pfile
->a_buff
;
2962 pfile
->a_buff
= buff
;
2966 buff
->cur
= result
+ len
;
2970 /* Say which field of TOK is in use. */
2972 enum cpp_token_fld_kind
2973 cpp_token_val_index (cpp_token
*tok
)
2975 switch (TOKEN_SPELL (tok
))
2978 return CPP_TOKEN_FLD_NODE
;
2980 return CPP_TOKEN_FLD_STR
;
2981 case SPELL_OPERATOR
:
2982 if (tok
->type
== CPP_PASTE
)
2983 return CPP_TOKEN_FLD_TOKEN_NO
;
2985 return CPP_TOKEN_FLD_NONE
;
2987 if (tok
->type
== CPP_MACRO_ARG
)
2988 return CPP_TOKEN_FLD_ARG_NO
;
2989 else if (tok
->type
== CPP_PADDING
)
2990 return CPP_TOKEN_FLD_SOURCE
;
2991 else if (tok
->type
== CPP_PRAGMA
)
2992 return CPP_TOKEN_FLD_PRAGMA
;
2993 /* else fall through */
2995 return CPP_TOKEN_FLD_NONE
;
2999 /* All tokens lexed in R after calling this function will be forced to have
3000 their source_location the same as the location referenced by P, until
3001 cpp_stop_forcing_token_locations is called for R. */
3004 cpp_force_token_locations (cpp_reader
*r
, source_location
*p
)
3006 r
->forced_token_location_p
= p
;
3009 /* Go back to assigning locations naturally for lexed tokens. */
3012 cpp_stop_forcing_token_locations (cpp_reader
*r
)
3014 r
->forced_token_location_p
= NULL
;