1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000-2016 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
8 This program is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by the
10 Free Software Foundation; either version 3, or (at your option) any
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
37 enum spell_type category
;
38 const unsigned char *name
;
41 static const unsigned char *const digraph_spellings
[] =
42 { UC
"%:", UC
"%:%:", UC
"<:", UC
":>", UC
"<%", UC
"%>" };
44 #define OP(e, s) { SPELL_OPERATOR, UC s },
45 #define TK(e, s) { SPELL_ ## s, UC #e },
46 static const struct token_spelling token_spellings
[N_TTYPES
] = { TTYPE_TABLE
};
50 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
51 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
53 static void add_line_note (cpp_buffer
*, const uchar
*, unsigned int);
54 static int skip_line_comment (cpp_reader
*);
55 static void skip_whitespace (cpp_reader
*, cppchar_t
);
56 static void lex_string (cpp_reader
*, cpp_token
*, const uchar
*);
57 static void save_comment (cpp_reader
*, cpp_token
*, const uchar
*, cppchar_t
);
58 static void store_comment (cpp_reader
*, cpp_token
*);
59 static void create_literal (cpp_reader
*, cpp_token
*, const uchar
*,
60 unsigned int, enum cpp_ttype
);
61 static bool warn_in_comment (cpp_reader
*, _cpp_line_note
*);
62 static int name_p (cpp_reader
*, const cpp_string
*);
63 static tokenrun
*next_tokenrun (tokenrun
*);
65 static _cpp_buff
*new_buff (size_t);
70 Compares, the token TOKEN to the NUL-terminated string STRING.
71 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
73 cpp_ideq (const cpp_token
*token
, const char *string
)
75 if (token
->type
!= CPP_NAME
)
78 return !ustrcmp (NODE_NAME (token
->val
.node
.node
), (const uchar
*) string
);
81 /* Record a note TYPE at byte POS into the current cleaned logical
84 add_line_note (cpp_buffer
*buffer
, const uchar
*pos
, unsigned int type
)
86 if (buffer
->notes_used
== buffer
->notes_cap
)
88 buffer
->notes_cap
= buffer
->notes_cap
* 2 + 200;
89 buffer
->notes
= XRESIZEVEC (_cpp_line_note
, buffer
->notes
,
93 buffer
->notes
[buffer
->notes_used
].pos
= pos
;
94 buffer
->notes
[buffer
->notes_used
].type
= type
;
99 /* Fast path to find line special characters using optimized character
100 scanning algorithms. Anything complicated falls back to the slow
101 path below. Since this loop is very hot it's worth doing these kinds
104 One of the paths through the ifdefs should provide
106 const uchar *search_line_fast (const uchar *s, const uchar *end);
108 Between S and END, search for \n, \r, \\, ?. Return a pointer to
111 Note that the last character of the buffer is *always* a newline,
112 as forced by _cpp_convert_input. This fact can be used to avoid
113 explicitly looking for the end of the buffer. */
115 /* Configure gives us an ifdef test. */
116 #ifndef WORDS_BIGENDIAN
117 #define WORDS_BIGENDIAN 0
120 /* We'd like the largest integer that fits into a register. There's nothing
121 in <stdint.h> that gives us that. For most hosts this is unsigned long,
122 but MS decided on an LLP64 model. Thankfully when building with GCC we
123 can get the "real" word size. */
125 typedef unsigned int word_type
__attribute__((__mode__(__word__
)));
127 typedef unsigned long word_type
;
130 /* The code below is only expecting sizes 4 or 8.
131 Die at compile-time if this expectation is violated. */
132 typedef char check_word_type_size
133 [(sizeof(word_type
) == 8 || sizeof(word_type
) == 4) * 2 - 1];
135 /* Return X with the first N bytes forced to values that won't match one
136 of the interesting characters. Note that NUL is not interesting. */
138 static inline word_type
139 acc_char_mask_misalign (word_type val
, unsigned int n
)
149 /* Return X replicated to all byte positions within WORD_TYPE. */
151 static inline word_type
152 acc_char_replicate (uchar x
)
156 ret
= (x
<< 24) | (x
<< 16) | (x
<< 8) | x
;
157 if (sizeof(word_type
) == 8)
158 ret
= (ret
<< 16 << 16) | ret
;
162 /* Return non-zero if some byte of VAL is (probably) C. */
164 static inline word_type
165 acc_char_cmp (word_type val
, word_type c
)
167 #if defined(__GNUC__) && defined(__alpha__)
168 /* We can get exact results using a compare-bytes instruction.
169 Get (val == c) via (0 >= (val ^ c)). */
170 return __builtin_alpha_cmpbge (0, val
^ c
);
172 word_type magic
= 0x7efefefeU
;
173 if (sizeof(word_type
) == 8)
174 magic
= (magic
<< 16 << 16) | 0xfefefefeU
;
178 return ((val
+ magic
) ^ ~val
) & ~magic
;
182 /* Given the result of acc_char_cmp is non-zero, return the index of
183 the found character. If this was a false positive, return -1. */
186 acc_char_index (word_type cmp ATTRIBUTE_UNUSED
,
187 word_type val ATTRIBUTE_UNUSED
)
189 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
190 /* The cmpbge instruction sets *bits* of the result corresponding to
191 matches in the bytes with no false positives. */
192 return __builtin_ctzl (cmp
);
196 /* ??? It would be nice to force unrolling here,
197 and have all of these constants folded. */
198 for (i
= 0; i
< sizeof(word_type
); ++i
)
202 c
= (val
>> (sizeof(word_type
) - i
- 1) * 8) & 0xff;
204 c
= (val
>> i
* 8) & 0xff;
206 if (c
== '\n' || c
== '\r' || c
== '\\' || c
== '?')
214 /* A version of the fast scanner using bit fiddling techniques.
216 For 32-bit words, one would normally perform 16 comparisons and
217 16 branches. With this algorithm one performs 24 arithmetic
218 operations and one branch. Whether this is faster with a 32-bit
219 word size is going to be somewhat system dependent.
221 For 64-bit words, we eliminate twice the number of comparisons
222 and branches without increasing the number of arithmetic operations.
223 It's almost certainly going to be a win with 64-bit word size. */
225 static const uchar
* search_line_acc_char (const uchar
*, const uchar
*)
229 search_line_acc_char (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
231 const word_type repl_nl
= acc_char_replicate ('\n');
232 const word_type repl_cr
= acc_char_replicate ('\r');
233 const word_type repl_bs
= acc_char_replicate ('\\');
234 const word_type repl_qm
= acc_char_replicate ('?');
236 unsigned int misalign
;
240 /* Align the buffer. Mask out any bytes from before the beginning. */
241 p
= (word_type
*)((uintptr_t)s
& -sizeof(word_type
));
243 misalign
= (uintptr_t)s
& (sizeof(word_type
) - 1);
245 val
= acc_char_mask_misalign (val
, misalign
);
250 t
= acc_char_cmp (val
, repl_nl
);
251 t
|= acc_char_cmp (val
, repl_cr
);
252 t
|= acc_char_cmp (val
, repl_bs
);
253 t
|= acc_char_cmp (val
, repl_qm
);
255 if (__builtin_expect (t
!= 0, 0))
257 int i
= acc_char_index (t
, val
);
259 return (const uchar
*)p
+ i
;
266 /* Disable on Solaris 2/x86 until the following problem can be properly
269 The Solaris 10+ assembler tags objects with the instruction set
270 extensions used, so SSE4.2 executables cannot run on machines that
271 don't support that extension. */
273 #if (GCC_VERSION >= 4005) && (__GNUC__ >= 5 || !defined(__PIC__)) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
275 /* Replicated character data to be shared between implementations.
276 Recall that outside of a context with vector support we can't
277 define compatible vector types, therefore these are all defined
278 in terms of raw characters. */
279 static const char repl_chars
[4][16] __attribute__((aligned(16))) = {
280 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
281 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
282 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
283 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
284 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
285 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
286 { '?', '?', '?', '?', '?', '?', '?', '?',
287 '?', '?', '?', '?', '?', '?', '?', '?' },
290 /* A version of the fast scanner using MMX vectorized byte compare insns.
292 This uses the PMOVMSKB instruction which was introduced with "MMX2",
293 which was packaged into SSE1; it is also present in the AMD MMX
294 extension. Mark the function as using "sse" so that we emit a real
295 "emms" instruction, rather than the 3dNOW "femms" instruction. */
299 __attribute__((__target__("sse")))
301 search_line_mmx (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
303 typedef char v8qi
__attribute__ ((__vector_size__ (8)));
304 typedef int __m64
__attribute__ ((__vector_size__ (8), __may_alias__
));
306 const v8qi repl_nl
= *(const v8qi
*)repl_chars
[0];
307 const v8qi repl_cr
= *(const v8qi
*)repl_chars
[1];
308 const v8qi repl_bs
= *(const v8qi
*)repl_chars
[2];
309 const v8qi repl_qm
= *(const v8qi
*)repl_chars
[3];
311 unsigned int misalign
, found
, mask
;
315 /* Align the source pointer. While MMX doesn't generate unaligned data
316 faults, this allows us to safely scan to the end of the buffer without
317 reading beyond the end of the last page. */
318 misalign
= (uintptr_t)s
& 7;
319 p
= (const v8qi
*)((uintptr_t)s
& -8);
322 /* Create a mask for the bytes that are valid within the first
323 16-byte block. The Idea here is that the AND with the mask
324 within the loop is "free", since we need some AND or TEST
325 insn in order to set the flags for the branch anyway. */
326 mask
= -1u << misalign
;
328 /* Main loop processing 8 bytes at a time. */
336 t
= __builtin_ia32_pcmpeqb(data
, repl_nl
);
337 c
= __builtin_ia32_pcmpeqb(data
, repl_cr
);
338 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
339 c
= __builtin_ia32_pcmpeqb(data
, repl_bs
);
340 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
341 c
= __builtin_ia32_pcmpeqb(data
, repl_qm
);
342 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
343 found
= __builtin_ia32_pmovmskb (t
);
348 __builtin_ia32_emms ();
350 /* FOUND contains 1 in bits for which we matched a relevant
351 character. Conversion to the byte index is trivial. */
352 found
= __builtin_ctz(found
);
353 return (const uchar
*)p
+ found
;
356 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
360 __attribute__((__target__("sse2")))
362 search_line_sse2 (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
364 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
366 const v16qi repl_nl
= *(const v16qi
*)repl_chars
[0];
367 const v16qi repl_cr
= *(const v16qi
*)repl_chars
[1];
368 const v16qi repl_bs
= *(const v16qi
*)repl_chars
[2];
369 const v16qi repl_qm
= *(const v16qi
*)repl_chars
[3];
371 unsigned int misalign
, found
, mask
;
375 /* Align the source pointer. */
376 misalign
= (uintptr_t)s
& 15;
377 p
= (const v16qi
*)((uintptr_t)s
& -16);
380 /* Create a mask for the bytes that are valid within the first
381 16-byte block. The Idea here is that the AND with the mask
382 within the loop is "free", since we need some AND or TEST
383 insn in order to set the flags for the branch anyway. */
384 mask
= -1u << misalign
;
386 /* Main loop processing 16 bytes at a time. */
394 t
= __builtin_ia32_pcmpeqb128(data
, repl_nl
);
395 t
|= __builtin_ia32_pcmpeqb128(data
, repl_cr
);
396 t
|= __builtin_ia32_pcmpeqb128(data
, repl_bs
);
397 t
|= __builtin_ia32_pcmpeqb128(data
, repl_qm
);
398 found
= __builtin_ia32_pmovmskb128 (t
);
403 /* FOUND contains 1 in bits for which we matched a relevant
404 character. Conversion to the byte index is trivial. */
405 found
= __builtin_ctz(found
);
406 return (const uchar
*)p
+ found
;
410 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
414 __attribute__((__target__("sse4.2")))
416 search_line_sse42 (const uchar
*s
, const uchar
*end
)
418 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
419 static const v16qi search
= { '\n', '\r', '?', '\\' };
421 uintptr_t si
= (uintptr_t)s
;
424 /* Check for unaligned input. */
429 if (__builtin_expect (end
- s
< 16, 0)
430 && __builtin_expect ((si
& 0xfff) > 0xff0, 0))
432 /* There are less than 16 bytes left in the buffer, and less
433 than 16 bytes left on the page. Reading 16 bytes at this
434 point might generate a spurious page fault. Defer to the
435 SSE2 implementation, which already handles alignment. */
436 return search_line_sse2 (s
, end
);
439 /* ??? The builtin doesn't understand that the PCMPESTRI read from
440 memory need not be aligned. */
441 sv
= __builtin_ia32_loaddqu ((const char *) s
);
442 index
= __builtin_ia32_pcmpestri128 (search
, 4, sv
, 16, 0);
444 if (__builtin_expect (index
< 16, 0))
447 /* Advance the pointer to an aligned address. We will re-scan a
448 few bytes, but we no longer need care for reading past the
449 end of a page, since we're guaranteed a match. */
450 s
= (const uchar
*)((si
+ 15) & -16);
453 /* Main loop, processing 16 bytes at a time. */
454 #ifdef __GCC_ASM_FLAG_OUTPUTS__
459 /* By using inline assembly instead of the builtin,
460 we can use the result, as well as the flags set. */
461 __asm ("%vpcmpestri\t$0, %2, %3"
462 : "=c"(index
), "=@ccc"(f
)
463 : "m"(*s
), "x"(search
), "a"(4), "d"(16));
471 /* By doing the whole loop in inline assembly,
472 we can make proper use of the flags set. */
473 __asm ( ".balign 16\n"
475 " %vpcmpestri\t$0, (%1), %2\n"
477 : "=&c"(index
), "+r"(s
)
478 : "x"(search
), "a"(4), "d"(16));
486 /* Work around out-dated assemblers without sse4 support. */
487 #define search_line_sse42 search_line_sse2
490 /* Check the CPU capabilities. */
492 #include "../gcc/config/i386/cpuid.h"
494 typedef const uchar
* (*search_line_fast_type
) (const uchar
*, const uchar
*);
495 static search_line_fast_type search_line_fast
;
497 #define HAVE_init_vectorized_lexer 1
499 init_vectorized_lexer (void)
501 unsigned dummy
, ecx
= 0, edx
= 0;
502 search_line_fast_type impl
= search_line_acc_char
;
505 #if defined(__SSE4_2__)
507 #elif defined(__SSE2__)
509 #elif defined(__SSE__)
514 impl
= search_line_sse42
;
515 else if (__get_cpuid (1, &dummy
, &dummy
, &ecx
, &edx
) || minimum
== 2)
517 if (minimum
== 3 || (ecx
& bit_SSE4_2
))
518 impl
= search_line_sse42
;
519 else if (minimum
== 2 || (edx
& bit_SSE2
))
520 impl
= search_line_sse2
;
521 else if (minimum
== 1 || (edx
& bit_SSE
))
522 impl
= search_line_mmx
;
524 else if (__get_cpuid (0x80000001, &dummy
, &dummy
, &dummy
, &edx
))
527 || (edx
& (bit_MMXEXT
| bit_CMOV
)) == (bit_MMXEXT
| bit_CMOV
))
528 impl
= search_line_mmx
;
531 search_line_fast
= impl
;
534 #elif defined(_ARCH_PWR8) && defined(__ALTIVEC__)
536 /* A vection of the fast scanner using AltiVec vectorized byte compares
537 and VSX unaligned loads (when VSX is available). This is otherwise
538 the same as the pre-GCC 5 version. */
540 ATTRIBUTE_NO_SANITIZE_UNDEFINED
542 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
544 typedef __attribute__((altivec(vector
))) unsigned char vc
;
547 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
548 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
551 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
552 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
555 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
556 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
559 '?', '?', '?', '?', '?', '?', '?', '?',
560 '?', '?', '?', '?', '?', '?', '?', '?',
562 const vc zero
= { 0 };
566 /* Main loop processing 16 bytes at a time. */
569 vc m_nl
, m_cr
, m_bs
, m_qm
;
571 data
= *((const vc
*)s
);
574 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
575 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
576 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
577 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
578 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
580 /* T now contains 0xff in bytes for which we matched one of the relevant
581 characters. We want to exit the loop if any byte in T is non-zero.
582 Below is the expansion of vec_any_ne(t, zero). */
584 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
586 /* Restore s to to point to the 16 bytes we just processed. */
590 #define N (sizeof(vc) / sizeof(long))
594 /* Statically assert that N is 2 or 4. */
595 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
597 unsigned long l
, i
= 0;
601 /* Find the first word of T that is non-zero. */
608 s
+= sizeof(unsigned long);
612 s
+= sizeof(unsigned long);
618 s
+= sizeof(unsigned long);
622 /* L now contains 0xff in bytes for which we matched one of the
623 relevant characters. We can find the byte index by finding
624 its bit index and dividing by 8. */
625 #ifdef __BIG_ENDIAN__
626 l
= __builtin_clzl(l
) >> 3;
628 l
= __builtin_ctzl(l
) >> 3;
636 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__) && defined (__BIG_ENDIAN__)
638 /* A vection of the fast scanner using AltiVec vectorized byte compares.
639 This cannot be used for little endian because vec_lvsl/lvsr are
640 deprecated for little endian and the code won't work properly. */
641 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
642 so we can't compile this function without -maltivec on the command line
643 (or implied by some other switch). */
646 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
648 typedef __attribute__((altivec(vector
))) unsigned char vc
;
651 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
652 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
655 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
656 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
659 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
660 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
663 '?', '?', '?', '?', '?', '?', '?', '?',
664 '?', '?', '?', '?', '?', '?', '?', '?',
667 -1, -1, -1, -1, -1, -1, -1, -1,
668 -1, -1, -1, -1, -1, -1, -1, -1,
670 const vc zero
= { 0 };
674 /* Altivec loads automatically mask addresses with -16. This lets us
675 issue the first load as early as possible. */
676 data
= __builtin_vec_ld(0, (const vc
*)s
);
678 /* Discard bytes before the beginning of the buffer. Do this by
679 beginning with all ones and shifting in zeros according to the
680 mis-alignment. The LVSR instruction pulls the exact shift we
681 want from the address. */
682 mask
= __builtin_vec_lvsr(0, s
);
683 mask
= __builtin_vec_perm(zero
, ones
, mask
);
686 /* While altivec loads mask addresses, we still need to align S so
687 that the offset we compute at the end is correct. */
688 s
= (const uchar
*)((uintptr_t)s
& -16);
690 /* Main loop processing 16 bytes at a time. */
694 vc m_nl
, m_cr
, m_bs
, m_qm
;
697 data
= __builtin_vec_ld(0, (const vc
*)s
);
700 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
701 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
702 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
703 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
704 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
706 /* T now contains 0xff in bytes for which we matched one of the relevant
707 characters. We want to exit the loop if any byte in T is non-zero.
708 Below is the expansion of vec_any_ne(t, zero). */
710 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
713 #define N (sizeof(vc) / sizeof(long))
717 /* Statically assert that N is 2 or 4. */
718 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
720 unsigned long l
, i
= 0;
724 /* Find the first word of T that is non-zero. */
731 s
+= sizeof(unsigned long);
735 s
+= sizeof(unsigned long);
740 s
+= sizeof(unsigned long);
744 /* L now contains 0xff in bytes for which we matched one of the
745 relevant characters. We can find the byte index by finding
746 its bit index and dividing by 8. */
747 l
= __builtin_clzl(l
) >> 3;
754 #elif defined (__ARM_NEON)
755 #include "arm_neon.h"
758 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
760 const uint8x16_t repl_nl
= vdupq_n_u8 ('\n');
761 const uint8x16_t repl_cr
= vdupq_n_u8 ('\r');
762 const uint8x16_t repl_bs
= vdupq_n_u8 ('\\');
763 const uint8x16_t repl_qm
= vdupq_n_u8 ('?');
764 const uint8x16_t xmask
= (uint8x16_t
) vdupq_n_u64 (0x8040201008040201ULL
);
766 unsigned int misalign
, found
, mask
;
770 /* Align the source pointer. */
771 misalign
= (uintptr_t)s
& 15;
772 p
= (const uint8_t *)((uintptr_t)s
& -16);
775 /* Create a mask for the bytes that are valid within the first
776 16-byte block. The Idea here is that the AND with the mask
777 within the loop is "free", since we need some AND or TEST
778 insn in order to set the flags for the branch anyway. */
779 mask
= (-1u << misalign
) & 0xffff;
781 /* Main loop, processing 16 bytes at a time. */
789 uint8x16_t t
, u
, v
, w
;
796 t
= vceqq_u8 (data
, repl_nl
);
797 u
= vceqq_u8 (data
, repl_cr
);
798 v
= vorrq_u8 (t
, vceqq_u8 (data
, repl_bs
));
799 w
= vorrq_u8 (u
, vceqq_u8 (data
, repl_qm
));
800 t
= vandq_u8 (vorrq_u8 (v
, w
), xmask
);
801 l
= vpadd_u8 (vget_low_u8 (t
), vget_high_u8 (t
));
805 found
= vget_lane_u32 ((uint32x2_t
) vorr_u64 ((uint64x1_t
) n
,
806 vshr_n_u64 ((uint64x1_t
) n
, 24)), 0);
811 /* FOUND contains 1 in bits for which we matched a relevant
812 character. Conversion to the byte index is trivial. */
813 found
= __builtin_ctz (found
);
814 return (const uchar
*)p
+ found
;
819 /* We only have one accellerated alternative. Use a direct call so that
820 we encourage inlining. */
822 #define search_line_fast search_line_acc_char
826 /* Initialize the lexer if needed. */
829 _cpp_init_lexer (void)
831 #ifdef HAVE_init_vectorized_lexer
832 init_vectorized_lexer ();
836 /* Returns with a logical line that contains no escaped newlines or
837 trigraphs. This is a time-critical inner loop. */
839 _cpp_clean_line (cpp_reader
*pfile
)
845 buffer
= pfile
->buffer
;
846 buffer
->cur_note
= buffer
->notes_used
= 0;
847 buffer
->cur
= buffer
->line_base
= buffer
->next_line
;
848 buffer
->need_line
= false;
849 s
= buffer
->next_line
;
851 if (!buffer
->from_stage3
)
853 const uchar
*pbackslash
= NULL
;
855 /* Fast path. This is the common case of an un-escaped line with
856 no trigraphs. The primary win here is by not writing any
857 data back to memory until we have to. */
860 /* Perform an optimized search for \n, \r, \\, ?. */
861 s
= search_line_fast (s
, buffer
->rlimit
);
866 /* Record the location of the backslash and continue. */
869 else if (__builtin_expect (c
== '?', 0))
871 if (__builtin_expect (s
[1] == '?', false)
872 && _cpp_trigraph_map
[s
[2]])
874 /* Have a trigraph. We may or may not have to convert
875 it. Add a line note regardless, for -Wtrigraphs. */
876 add_line_note (buffer
, s
, s
[2]);
877 if (CPP_OPTION (pfile
, trigraphs
))
879 /* We do, and that means we have to switch to the
882 *d
= _cpp_trigraph_map
[s
[2]];
887 /* Not a trigraph. Continue on fast-path. */
894 /* This must be \r or \n. We're either done, or we'll be forced
895 to write back to the buffer and continue on the slow path. */
898 if (__builtin_expect (s
== buffer
->rlimit
, false))
901 /* DOS line ending? */
902 if (__builtin_expect (c
== '\r', false) && s
[1] == '\n')
905 if (s
== buffer
->rlimit
)
909 if (__builtin_expect (pbackslash
== NULL
, true))
912 /* Check for escaped newline. */
914 while (is_nvspace (p
[-1]))
916 if (p
- 1 != pbackslash
)
919 /* Have an escaped newline; process it and proceed to
921 add_line_note (buffer
, p
- 1, p
!= d
? ' ' : '\\');
923 buffer
->next_line
= p
- 1;
931 if (c
== '\n' || c
== '\r')
933 /* Handle DOS line endings. */
934 if (c
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
936 if (s
== buffer
->rlimit
)
941 while (p
!= buffer
->next_line
&& is_nvspace (p
[-1]))
943 if (p
== buffer
->next_line
|| p
[-1] != '\\')
946 add_line_note (buffer
, p
- 1, p
!= d
? ' ': '\\');
948 buffer
->next_line
= p
- 1;
950 else if (c
== '?' && s
[1] == '?' && _cpp_trigraph_map
[s
[2]])
952 /* Add a note regardless, for the benefit of -Wtrigraphs. */
953 add_line_note (buffer
, d
, s
[2]);
954 if (CPP_OPTION (pfile
, trigraphs
))
956 *d
= _cpp_trigraph_map
[s
[2]];
964 while (*s
!= '\n' && *s
!= '\r')
968 /* Handle DOS line endings. */
969 if (*s
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
975 /* A sentinel note that should never be processed. */
976 add_line_note (buffer
, d
+ 1, '\n');
977 buffer
->next_line
= s
+ 1;
980 /* Return true if the trigraph indicated by NOTE should be warned
981 about in a comment. */
983 warn_in_comment (cpp_reader
*pfile
, _cpp_line_note
*note
)
987 /* Within comments we don't warn about trigraphs, unless the
988 trigraph forms an escaped newline, as that may change
990 if (note
->type
!= '/')
993 /* If -trigraphs, then this was an escaped newline iff the next note
995 if (CPP_OPTION (pfile
, trigraphs
))
996 return note
[1].pos
== note
->pos
;
998 /* Otherwise, see if this forms an escaped newline. */
1000 while (is_nvspace (*p
))
1003 /* There might have been escaped newlines between the trigraph and the
1004 newline we found. Hence the position test. */
1005 return (*p
== '\n' && p
< note
[1].pos
);
1008 /* Process the notes created by add_line_note as far as the current
1011 _cpp_process_line_notes (cpp_reader
*pfile
, int in_comment
)
1013 cpp_buffer
*buffer
= pfile
->buffer
;
1017 _cpp_line_note
*note
= &buffer
->notes
[buffer
->cur_note
];
1020 if (note
->pos
> buffer
->cur
)
1024 col
= CPP_BUF_COLUMN (buffer
, note
->pos
+ 1);
1026 if (note
->type
== '\\' || note
->type
== ' ')
1028 if (note
->type
== ' ' && !in_comment
)
1029 cpp_error_with_line (pfile
, CPP_DL_WARNING
, pfile
->line_table
->highest_line
, col
,
1030 "backslash and newline separated by space");
1032 if (buffer
->next_line
> buffer
->rlimit
)
1034 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
, col
,
1035 "backslash-newline at end of file");
1036 /* Prevent "no newline at end of file" warning. */
1037 buffer
->next_line
= buffer
->rlimit
;
1040 buffer
->line_base
= note
->pos
;
1041 CPP_INCREMENT_LINE (pfile
, 0);
1043 else if (_cpp_trigraph_map
[note
->type
])
1045 if (CPP_OPTION (pfile
, warn_trigraphs
)
1046 && (!in_comment
|| warn_in_comment (pfile
, note
)))
1048 if (CPP_OPTION (pfile
, trigraphs
))
1049 cpp_warning_with_line (pfile
, CPP_W_TRIGRAPHS
,
1050 pfile
->line_table
->highest_line
, col
,
1051 "trigraph ??%c converted to %c",
1053 (int) _cpp_trigraph_map
[note
->type
]);
1056 cpp_warning_with_line
1057 (pfile
, CPP_W_TRIGRAPHS
,
1058 pfile
->line_table
->highest_line
, col
,
1059 "trigraph ??%c ignored, use -trigraphs to enable",
1064 else if (note
->type
== 0)
1065 /* Already processed in lex_raw_string. */;
1071 /* Skip a C-style block comment. We find the end of the comment by
1072 seeing if an asterisk is before every '/' we encounter. Returns
1073 nonzero if comment terminated by EOF, zero otherwise.
1075 Buffer->cur points to the initial asterisk of the comment. */
1077 _cpp_skip_block_comment (cpp_reader
*pfile
)
1079 cpp_buffer
*buffer
= pfile
->buffer
;
1080 const uchar
*cur
= buffer
->cur
;
1089 /* People like decorating comments with '*', so check for '/'
1090 instead for efficiency. */
1098 /* Warn about potential nested comments, but not if the '/'
1099 comes immediately before the true comment delimiter.
1100 Don't bother to get it right across escaped newlines. */
1101 if (CPP_OPTION (pfile
, warn_comments
)
1102 && cur
[0] == '*' && cur
[1] != '/')
1105 cpp_warning_with_line (pfile
, CPP_W_COMMENTS
,
1106 pfile
->line_table
->highest_line
,
1107 CPP_BUF_COL (buffer
),
1108 "\"/*\" within comment");
1114 buffer
->cur
= cur
- 1;
1115 _cpp_process_line_notes (pfile
, true);
1116 if (buffer
->next_line
>= buffer
->rlimit
)
1118 _cpp_clean_line (pfile
);
1120 cols
= buffer
->next_line
- buffer
->line_base
;
1121 CPP_INCREMENT_LINE (pfile
, cols
);
1128 _cpp_process_line_notes (pfile
, true);
1132 /* Skip a C++ line comment, leaving buffer->cur pointing to the
1133 terminating newline. Handles escaped newlines. Returns nonzero
1134 if a multiline comment. */
1136 skip_line_comment (cpp_reader
*pfile
)
1138 cpp_buffer
*buffer
= pfile
->buffer
;
1139 source_location orig_line
= pfile
->line_table
->highest_line
;
1141 while (*buffer
->cur
!= '\n')
1144 _cpp_process_line_notes (pfile
, true);
1145 return orig_line
!= pfile
->line_table
->highest_line
;
1148 /* Skips whitespace, saving the next non-whitespace character. */
1150 skip_whitespace (cpp_reader
*pfile
, cppchar_t c
)
1152 cpp_buffer
*buffer
= pfile
->buffer
;
1153 bool saw_NUL
= false;
1157 /* Horizontal space always OK. */
1158 if (c
== ' ' || c
== '\t')
1160 /* Just \f \v or \0 left. */
1163 else if (pfile
->state
.in_directive
&& CPP_PEDANTIC (pfile
))
1164 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
,
1165 CPP_BUF_COL (buffer
),
1166 "%s in preprocessing directive",
1167 c
== '\f' ? "form feed" : "vertical tab");
1171 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
1172 while (is_nvspace (c
));
1175 cpp_error (pfile
, CPP_DL_WARNING
, "null character(s) ignored");
1180 /* See if the characters of a number token are valid in a name (no
1181 '.', '+' or '-'). */
1183 name_p (cpp_reader
*pfile
, const cpp_string
*string
)
1187 for (i
= 0; i
< string
->len
; i
++)
1188 if (!is_idchar (string
->text
[i
]))
1194 /* After parsing an identifier or other sequence, produce a warning about
1195 sequences not in NFC/NFKC. */
1197 warn_about_normalization (cpp_reader
*pfile
,
1198 const cpp_token
*token
,
1199 const struct normalize_state
*s
)
1201 if (CPP_OPTION (pfile
, warn_normalize
) < NORMALIZE_STATE_RESULT (s
)
1202 && !pfile
->state
.skipping
)
1204 /* Make sure that the token is printed using UCNs, even
1205 if we'd otherwise happily print UTF-8. */
1206 unsigned char *buf
= XNEWVEC (unsigned char, cpp_token_len (token
));
1209 sz
= cpp_spell_token (pfile
, token
, buf
, false) - buf
;
1210 if (NORMALIZE_STATE_RESULT (s
) == normalized_C
)
1211 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1212 "`%.*s' is not in NFKC", (int) sz
, buf
);
1214 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1215 "`%.*s' is not in NFC", (int) sz
, buf
);
1220 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1221 an identifier. FIRST is TRUE if this starts an identifier. */
1223 forms_identifier_p (cpp_reader
*pfile
, int first
,
1224 struct normalize_state
*state
)
1226 cpp_buffer
*buffer
= pfile
->buffer
;
1228 if (*buffer
->cur
== '$')
1230 if (!CPP_OPTION (pfile
, dollars_in_ident
))
1234 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1236 CPP_OPTION (pfile
, warn_dollars
) = 0;
1237 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1243 /* Is this a syntactically valid UCN? */
1244 if (CPP_OPTION (pfile
, extended_identifiers
)
1245 && *buffer
->cur
== '\\'
1246 && (buffer
->cur
[1] == 'u' || buffer
->cur
[1] == 'U'))
1250 if (_cpp_valid_ucn (pfile
, &buffer
->cur
, buffer
->rlimit
, 1 + !first
,
1251 state
, &s
, NULL
, NULL
))
1259 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1260 static cpp_hashnode
*
1261 lex_identifier_intern (cpp_reader
*pfile
, const uchar
*base
)
1263 cpp_hashnode
*result
;
1266 unsigned int hash
= HT_HASHSTEP (0, *base
);
1269 while (ISIDNUM (*cur
))
1271 hash
= HT_HASHSTEP (hash
, *cur
);
1275 hash
= HT_HASHFINISH (hash
, len
);
1276 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1277 base
, len
, hash
, HT_ALLOC
));
1279 /* Rarely, identifiers require diagnostics when lexed. */
1280 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1281 && !pfile
->state
.skipping
, 0))
1283 /* It is allowed to poison the same identifier twice. */
1284 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1285 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1286 NODE_NAME (result
));
1288 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1289 replacement list of a variadic macro. */
1290 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1291 && !pfile
->state
.va_args_ok
)
1293 if (CPP_OPTION (pfile
, cplusplus
))
1294 cpp_error (pfile
, CPP_DL_PEDWARN
,
1295 "__VA_ARGS__ can only appear in the expansion"
1296 " of a C++11 variadic macro");
1298 cpp_error (pfile
, CPP_DL_PEDWARN
,
1299 "__VA_ARGS__ can only appear in the expansion"
1300 " of a C99 variadic macro");
1303 /* For -Wc++-compat, warn about use of C++ named operators. */
1304 if (result
->flags
& NODE_WARN_OPERATOR
)
1305 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1306 "identifier \"%s\" is a special operator name in C++",
1307 NODE_NAME (result
));
1313 /* Get the cpp_hashnode of an identifier specified by NAME in
1314 the current cpp_reader object. If none is found, NULL is returned. */
1316 _cpp_lex_identifier (cpp_reader
*pfile
, const char *name
)
1318 cpp_hashnode
*result
;
1319 result
= lex_identifier_intern (pfile
, (uchar
*) name
);
1323 /* Lex an identifier starting at BUFFER->CUR - 1. */
1324 static cpp_hashnode
*
1325 lex_identifier (cpp_reader
*pfile
, const uchar
*base
, bool starts_ucn
,
1326 struct normalize_state
*nst
, cpp_hashnode
**spelling
)
1328 cpp_hashnode
*result
;
1331 unsigned int hash
= HT_HASHSTEP (0, *base
);
1333 cur
= pfile
->buffer
->cur
;
1336 while (ISIDNUM (*cur
))
1338 hash
= HT_HASHSTEP (hash
, *cur
);
1341 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *(cur
- 1));
1343 pfile
->buffer
->cur
= cur
;
1344 if (starts_ucn
|| forms_identifier_p (pfile
, false, nst
))
1346 /* Slower version for identifiers containing UCNs (or $). */
1348 while (ISIDNUM (*pfile
->buffer
->cur
))
1350 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *pfile
->buffer
->cur
);
1351 pfile
->buffer
->cur
++;
1353 } while (forms_identifier_p (pfile
, false, nst
));
1354 result
= _cpp_interpret_identifier (pfile
, base
,
1355 pfile
->buffer
->cur
- base
);
1356 *spelling
= cpp_lookup (pfile
, base
, pfile
->buffer
->cur
- base
);
1361 hash
= HT_HASHFINISH (hash
, len
);
1363 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1364 base
, len
, hash
, HT_ALLOC
));
1368 /* Rarely, identifiers require diagnostics when lexed. */
1369 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1370 && !pfile
->state
.skipping
, 0))
1372 /* It is allowed to poison the same identifier twice. */
1373 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1374 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1375 NODE_NAME (result
));
1377 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1378 replacement list of a variadic macro. */
1379 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1380 && !pfile
->state
.va_args_ok
)
1382 if (CPP_OPTION (pfile
, cplusplus
))
1383 cpp_error (pfile
, CPP_DL_PEDWARN
,
1384 "__VA_ARGS__ can only appear in the expansion"
1385 " of a C++11 variadic macro");
1387 cpp_error (pfile
, CPP_DL_PEDWARN
,
1388 "__VA_ARGS__ can only appear in the expansion"
1389 " of a C99 variadic macro");
1392 /* For -Wc++-compat, warn about use of C++ named operators. */
1393 if (result
->flags
& NODE_WARN_OPERATOR
)
1394 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1395 "identifier \"%s\" is a special operator name in C++",
1396 NODE_NAME (result
));
1402 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1404 lex_number (cpp_reader
*pfile
, cpp_string
*number
,
1405 struct normalize_state
*nst
)
1411 base
= pfile
->buffer
->cur
- 1;
1414 cur
= pfile
->buffer
->cur
;
1416 /* N.B. ISIDNUM does not include $. */
1417 while (ISIDNUM (*cur
) || *cur
== '.' || DIGIT_SEP (*cur
)
1418 || VALID_SIGN (*cur
, cur
[-1]))
1420 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *cur
);
1423 /* A number can't end with a digit separator. */
1424 while (cur
> pfile
->buffer
->cur
&& DIGIT_SEP (cur
[-1]))
1427 pfile
->buffer
->cur
= cur
;
1429 while (forms_identifier_p (pfile
, false, nst
));
1431 number
->len
= cur
- base
;
1432 dest
= _cpp_unaligned_alloc (pfile
, number
->len
+ 1);
1433 memcpy (dest
, base
, number
->len
);
1434 dest
[number
->len
] = '\0';
1435 number
->text
= dest
;
1438 /* Create a token of type TYPE with a literal spelling. */
1440 create_literal (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1441 unsigned int len
, enum cpp_ttype type
)
1443 uchar
*dest
= _cpp_unaligned_alloc (pfile
, len
+ 1);
1445 memcpy (dest
, base
, len
);
1448 token
->val
.str
.len
= len
;
1449 token
->val
.str
.text
= dest
;
1452 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1453 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1456 bufring_append (cpp_reader
*pfile
, const uchar
*base
, size_t len
,
1457 _cpp_buff
**first_buff_p
, _cpp_buff
**last_buff_p
)
1459 _cpp_buff
*first_buff
= *first_buff_p
;
1460 _cpp_buff
*last_buff
= *last_buff_p
;
1462 if (first_buff
== NULL
)
1463 first_buff
= last_buff
= _cpp_get_buff (pfile
, len
);
1464 else if (len
> BUFF_ROOM (last_buff
))
1466 size_t room
= BUFF_ROOM (last_buff
);
1467 memcpy (BUFF_FRONT (last_buff
), base
, room
);
1468 BUFF_FRONT (last_buff
) += room
;
1471 last_buff
= _cpp_append_extend_buff (pfile
, last_buff
, len
);
1474 memcpy (BUFF_FRONT (last_buff
), base
, len
);
1475 BUFF_FRONT (last_buff
) += len
;
1477 *first_buff_p
= first_buff
;
1478 *last_buff_p
= last_buff
;
1482 /* Returns true if a macro has been defined.
1483 This might not work if compile with -save-temps,
1484 or preprocess separately from compilation. */
1487 is_macro(cpp_reader
*pfile
, const uchar
*base
)
1489 const uchar
*cur
= base
;
1490 if (! ISIDST (*cur
))
1492 unsigned int hash
= HT_HASHSTEP (0, *cur
);
1494 while (ISIDNUM (*cur
))
1496 hash
= HT_HASHSTEP (hash
, *cur
);
1499 hash
= HT_HASHFINISH (hash
, cur
- base
);
1501 cpp_hashnode
*result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1502 base
, cur
- base
, hash
, HT_NO_INSERT
));
1504 return !result
? false : (result
->type
== NT_MACRO
);
1508 /* Lexes a raw string. The stored string contains the spelling, including
1509 double quotes, delimiter string, '(' and ')', any leading
1510 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1511 literal, or CPP_OTHER if it was not properly terminated.
1513 The spelling is NUL-terminated, but it is not guaranteed that this
1514 is the first NUL since embedded NULs are preserved. */
1517 lex_raw_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1520 uchar raw_prefix
[17];
1521 uchar temp_buffer
[18];
1522 const uchar
*orig_base
;
1523 unsigned int raw_prefix_len
= 0, raw_suffix_len
= 0;
1524 enum raw_str_phase
{ RAW_STR_PREFIX
, RAW_STR
, RAW_STR_SUFFIX
};
1525 raw_str_phase phase
= RAW_STR_PREFIX
;
1526 enum cpp_ttype type
;
1527 size_t total_len
= 0;
1528 /* Index into temp_buffer during phases other than RAW_STR,
1529 during RAW_STR phase 17 to tell BUF_APPEND that nothing should
1530 be appended to temp_buffer. */
1531 size_t temp_buffer_len
= 0;
1532 _cpp_buff
*first_buff
= NULL
, *last_buff
= NULL
;
1533 size_t raw_prefix_start
;
1534 _cpp_line_note
*note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1536 type
= (*base
== 'L' ? CPP_WSTRING
:
1537 *base
== 'U' ? CPP_STRING32
:
1538 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1541 #define BUF_APPEND(STR,LEN) \
1543 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1544 &first_buff, &last_buff); \
1545 total_len += (LEN); \
1546 if (__builtin_expect (temp_buffer_len < 17, 0) \
1547 && (const uchar *)(STR) != base \
1550 memcpy (temp_buffer + temp_buffer_len, \
1551 (const uchar *)(STR), (LEN)); \
1552 temp_buffer_len += (LEN); \
1558 raw_prefix_start
= cur
- base
;
1563 /* If we previously performed any trigraph or line splicing
1564 transformations, undo them in between the opening and closing
1566 while (note
->pos
< cur
)
1568 for (; note
->pos
== cur
; ++note
)
1574 /* Restore backslash followed by newline. */
1575 BUF_APPEND (base
, cur
- base
);
1577 BUF_APPEND ("\\", 1);
1579 if (note
->type
== ' ')
1581 /* GNU backslash whitespace newline extension. FIXME
1582 could be any sequence of non-vertical space. When we
1583 can properly restore any such sequence, we should mark
1584 this note as handled so _cpp_process_line_notes
1586 BUF_APPEND (" ", 1);
1589 BUF_APPEND ("\n", 1);
1593 /* Already handled. */
1597 if (_cpp_trigraph_map
[note
->type
])
1599 /* Don't warn about this trigraph in
1600 _cpp_process_line_notes, since trigraphs show up as
1601 trigraphs in raw strings. */
1602 uchar type
= note
->type
;
1605 if (!CPP_OPTION (pfile
, trigraphs
))
1606 /* If we didn't convert the trigraph in the first
1607 place, don't do anything now either. */
1610 BUF_APPEND (base
, cur
- base
);
1612 BUF_APPEND ("??", 2);
1614 /* ??/ followed by newline gets two line notes, one for
1615 the trigraph and one for the backslash/newline. */
1616 if (type
== '/' && note
[1].pos
== cur
)
1618 if (note
[1].type
!= '\\'
1619 && note
[1].type
!= ' ')
1621 BUF_APPEND ("/", 1);
1623 goto after_backslash
;
1627 /* Skip the replacement character. */
1629 BUF_APPEND (&type
, 1);
1640 if (__builtin_expect (temp_buffer_len
< 17, 0))
1641 temp_buffer
[temp_buffer_len
++] = c
;
1644 if (phase
== RAW_STR_PREFIX
)
1646 while (raw_prefix_len
< temp_buffer_len
)
1648 raw_prefix
[raw_prefix_len
] = temp_buffer
[raw_prefix_len
];
1649 switch (raw_prefix
[raw_prefix_len
])
1651 case ' ': case '(': case ')': case '\\': case '\t':
1652 case '\v': case '\f': case '\n': default:
1654 /* Basic source charset except the above chars. */
1655 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1656 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1657 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1658 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1660 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1661 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1662 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1663 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1665 case '0': case '1': case '2': case '3': case '4': case '5':
1666 case '6': case '7': case '8': case '9':
1667 case '_': case '{': case '}': case '#': case '[': case ']':
1668 case '<': case '>': case '%': case ':': case ';': case '.':
1669 case '?': case '*': case '+': case '-': case '/': case '^':
1670 case '&': case '|': case '~': case '!': case '=': case ',':
1671 case '"': case '\'':
1672 if (raw_prefix_len
< 16)
1680 if (raw_prefix
[raw_prefix_len
] != '(')
1682 int col
= CPP_BUF_COLUMN (pfile
->buffer
, cur
) + 1;
1683 if (raw_prefix_len
== 16)
1684 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1685 col
, "raw string delimiter longer "
1686 "than 16 characters");
1687 else if (raw_prefix
[raw_prefix_len
] == '\n')
1688 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1689 col
, "invalid new-line in raw "
1690 "string delimiter");
1692 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1693 col
, "invalid character '%c' in "
1694 "raw string delimiter",
1695 (int) raw_prefix
[raw_prefix_len
]);
1696 pfile
->buffer
->cur
= orig_base
+ raw_prefix_start
- 1;
1697 create_literal (pfile
, token
, orig_base
,
1698 raw_prefix_start
- 1, CPP_OTHER
);
1700 _cpp_release_buff (pfile
, first_buff
);
1703 raw_prefix
[raw_prefix_len
] = '"';
1705 /* Nothing should be appended to temp_buffer during
1707 temp_buffer_len
= 17;
1712 else if (phase
== RAW_STR_SUFFIX
)
1714 while (raw_suffix_len
<= raw_prefix_len
1715 && raw_suffix_len
< temp_buffer_len
1716 && temp_buffer
[raw_suffix_len
] == raw_prefix
[raw_suffix_len
])
1718 if (raw_suffix_len
> raw_prefix_len
)
1720 if (raw_suffix_len
== temp_buffer_len
)
1723 /* Nothing should be appended to temp_buffer during
1725 temp_buffer_len
= 17;
1729 phase
= RAW_STR_SUFFIX
;
1731 temp_buffer_len
= 0;
1735 if (pfile
->state
.in_directive
1736 || (pfile
->state
.parsing_args
1737 && pfile
->buffer
->next_line
>= pfile
->buffer
->rlimit
))
1741 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, 0,
1742 "unterminated raw string");
1746 BUF_APPEND (base
, cur
- base
);
1748 if (pfile
->buffer
->cur
< pfile
->buffer
->rlimit
)
1749 CPP_INCREMENT_LINE (pfile
, 0);
1750 pfile
->buffer
->need_line
= true;
1752 pfile
->buffer
->cur
= cur
-1;
1753 _cpp_process_line_notes (pfile
, false);
1754 if (!_cpp_get_fresh_line (pfile
))
1756 source_location src_loc
= token
->src_loc
;
1757 token
->type
= CPP_EOF
;
1758 /* Tell the compiler the line number of the EOF token. */
1759 token
->src_loc
= pfile
->line_table
->highest_line
;
1761 if (first_buff
!= NULL
)
1762 _cpp_release_buff (pfile
, first_buff
);
1763 cpp_error_with_line (pfile
, CPP_DL_ERROR
, src_loc
, 0,
1764 "unterminated raw string");
1768 cur
= base
= pfile
->buffer
->cur
;
1769 note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1773 if (CPP_OPTION (pfile
, user_literals
))
1775 /* If a string format macro, say from inttypes.h, is placed touching
1776 a string literal it could be parsed as a C++11 user-defined string
1777 literal thus breaking the program.
1778 Try to identify macros with is_macro. A warning is issued. */
1779 if (is_macro (pfile
, cur
))
1781 /* Raise a warning, but do not consume subsequent tokens. */
1782 if (CPP_OPTION (pfile
, warn_literal_suffix
) && !pfile
->state
.skipping
)
1783 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1785 "invalid suffix on literal; C++11 requires "
1786 "a space between literal and string macro");
1788 /* Grab user defined literal suffix. */
1789 else if (ISIDST (*cur
))
1791 type
= cpp_userdef_string_add_type (type
);
1794 while (ISIDNUM (*cur
))
1799 pfile
->buffer
->cur
= cur
;
1800 if (first_buff
== NULL
)
1801 create_literal (pfile
, token
, base
, cur
- base
, type
);
1804 uchar
*dest
= _cpp_unaligned_alloc (pfile
, total_len
+ (cur
- base
) + 1);
1807 token
->val
.str
.len
= total_len
+ (cur
- base
);
1808 token
->val
.str
.text
= dest
;
1809 last_buff
= first_buff
;
1810 while (last_buff
!= NULL
)
1812 memcpy (dest
, last_buff
->base
,
1813 BUFF_FRONT (last_buff
) - last_buff
->base
);
1814 dest
+= BUFF_FRONT (last_buff
) - last_buff
->base
;
1815 last_buff
= last_buff
->next
;
1817 _cpp_release_buff (pfile
, first_buff
);
1818 memcpy (dest
, base
, cur
- base
);
1819 dest
[cur
- base
] = '\0';
1823 /* Lexes a string, character constant, or angle-bracketed header file
1824 name. The stored string contains the spelling, including opening
1825 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1826 'R' modifier. It returns the type of the literal, or CPP_OTHER
1827 if it was not properly terminated, or CPP_LESS for an unterminated
1828 header name which must be relexed as normal tokens.
1830 The spelling is NUL-terminated, but it is not guaranteed that this
1831 is the first NUL since embedded NULs are preserved. */
1833 lex_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
)
1835 bool saw_NUL
= false;
1837 cppchar_t terminator
;
1838 enum cpp_ttype type
;
1841 terminator
= *cur
++;
1842 if (terminator
== 'L' || terminator
== 'U')
1843 terminator
= *cur
++;
1844 else if (terminator
== 'u')
1846 terminator
= *cur
++;
1847 if (terminator
== '8')
1848 terminator
= *cur
++;
1850 if (terminator
== 'R')
1852 lex_raw_string (pfile
, token
, base
, cur
);
1855 if (terminator
== '"')
1856 type
= (*base
== 'L' ? CPP_WSTRING
:
1857 *base
== 'U' ? CPP_STRING32
:
1858 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1860 else if (terminator
== '\'')
1861 type
= (*base
== 'L' ? CPP_WCHAR
:
1862 *base
== 'U' ? CPP_CHAR32
:
1863 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8CHAR
: CPP_CHAR16
)
1866 terminator
= '>', type
= CPP_HEADER_NAME
;
1870 cppchar_t c
= *cur
++;
1872 /* In #include-style directives, terminators are not escapable. */
1873 if (c
== '\\' && !pfile
->state
.angled_headers
&& *cur
!= '\n')
1875 else if (c
== terminator
)
1880 /* Unmatched quotes always yield undefined behavior, but
1881 greedy lexing means that what appears to be an unterminated
1882 header name may actually be a legitimate sequence of tokens. */
1883 if (terminator
== '>')
1885 token
->type
= CPP_LESS
;
1895 if (saw_NUL
&& !pfile
->state
.skipping
)
1896 cpp_error (pfile
, CPP_DL_WARNING
,
1897 "null character(s) preserved in literal");
1899 if (type
== CPP_OTHER
&& CPP_OPTION (pfile
, lang
) != CLK_ASM
)
1900 cpp_error (pfile
, CPP_DL_PEDWARN
, "missing terminating %c character",
1903 if (CPP_OPTION (pfile
, user_literals
))
1905 /* If a string format macro, say from inttypes.h, is placed touching
1906 a string literal it could be parsed as a C++11 user-defined string
1907 literal thus breaking the program.
1908 Try to identify macros with is_macro. A warning is issued. */
1909 if (is_macro (pfile
, cur
))
1911 /* Raise a warning, but do not consume subsequent tokens. */
1912 if (CPP_OPTION (pfile
, warn_literal_suffix
) && !pfile
->state
.skipping
)
1913 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1915 "invalid suffix on literal; C++11 requires "
1916 "a space between literal and string macro");
1918 /* Grab user defined literal suffix. */
1919 else if (ISIDST (*cur
))
1921 type
= cpp_userdef_char_add_type (type
);
1922 type
= cpp_userdef_string_add_type (type
);
1925 while (ISIDNUM (*cur
))
1929 else if (CPP_OPTION (pfile
, cpp_warn_cxx11_compat
)
1930 && is_macro (pfile
, cur
)
1931 && !pfile
->state
.skipping
)
1932 cpp_warning_with_line (pfile
, CPP_W_CXX11_COMPAT
,
1933 token
->src_loc
, 0, "C++11 requires a space "
1934 "between string literal and macro");
1936 pfile
->buffer
->cur
= cur
;
1937 create_literal (pfile
, token
, base
, cur
- base
, type
);
1940 /* Return the comment table. The client may not make any assumption
1941 about the ordering of the table. */
1943 cpp_get_comments (cpp_reader
*pfile
)
1945 return &pfile
->comments
;
1948 /* Append a comment to the end of the comment table. */
1950 store_comment (cpp_reader
*pfile
, cpp_token
*token
)
1954 if (pfile
->comments
.allocated
== 0)
1956 pfile
->comments
.allocated
= 256;
1957 pfile
->comments
.entries
= (cpp_comment
*) xmalloc
1958 (pfile
->comments
.allocated
* sizeof (cpp_comment
));
1961 if (pfile
->comments
.count
== pfile
->comments
.allocated
)
1963 pfile
->comments
.allocated
*= 2;
1964 pfile
->comments
.entries
= (cpp_comment
*) xrealloc
1965 (pfile
->comments
.entries
,
1966 pfile
->comments
.allocated
* sizeof (cpp_comment
));
1969 len
= token
->val
.str
.len
;
1971 /* Copy comment. Note, token may not be NULL terminated. */
1972 pfile
->comments
.entries
[pfile
->comments
.count
].comment
=
1973 (char *) xmalloc (sizeof (char) * (len
+ 1));
1974 memcpy (pfile
->comments
.entries
[pfile
->comments
.count
].comment
,
1975 token
->val
.str
.text
, len
);
1976 pfile
->comments
.entries
[pfile
->comments
.count
].comment
[len
] = '\0';
1978 /* Set source location. */
1979 pfile
->comments
.entries
[pfile
->comments
.count
].sloc
= token
->src_loc
;
1981 /* Increment the count of entries in the comment table. */
1982 pfile
->comments
.count
++;
1985 /* The stored comment includes the comment start and any terminator. */
1987 save_comment (cpp_reader
*pfile
, cpp_token
*token
, const unsigned char *from
,
1990 unsigned char *buffer
;
1991 unsigned int len
, clen
, i
;
1993 len
= pfile
->buffer
->cur
- from
+ 1; /* + 1 for the initial '/'. */
1995 /* C++ comments probably (not definitely) have moved past a new
1996 line, which we don't want to save in the comment. */
1997 if (is_vspace (pfile
->buffer
->cur
[-1]))
2000 /* If we are currently in a directive or in argument parsing, then
2001 we need to store all C++ comments as C comments internally, and
2002 so we need to allocate a little extra space in that case.
2004 Note that the only time we encounter a directive here is
2005 when we are saving comments in a "#define". */
2006 clen
= ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
)
2007 && type
== '/') ? len
+ 2 : len
;
2009 buffer
= _cpp_unaligned_alloc (pfile
, clen
);
2011 token
->type
= CPP_COMMENT
;
2012 token
->val
.str
.len
= clen
;
2013 token
->val
.str
.text
= buffer
;
2016 memcpy (buffer
+ 1, from
, len
- 1);
2018 /* Finish conversion to a C comment, if necessary. */
2019 if ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
) && type
== '/')
2022 buffer
[clen
- 2] = '*';
2023 buffer
[clen
- 1] = '/';
2024 /* As there can be in a C++ comments illegal sequences for C comments
2025 we need to filter them out. */
2026 for (i
= 2; i
< (clen
- 2); i
++)
2027 if (buffer
[i
] == '/' && (buffer
[i
- 1] == '*' || buffer
[i
+ 1] == '*'))
2031 /* Finally store this comment for use by clients of libcpp. */
2032 store_comment (pfile
, token
);
2035 /* Returns true if comment at COMMENT_START is a recognized FALLTHROUGH
2039 fallthrough_comment_p (cpp_reader
*pfile
, const unsigned char *comment_start
)
2041 const unsigned char *from
= comment_start
+ 1;
2042 /* Whole comment contents:
2046 if (*from
== '-' || *from
== '@')
2048 size_t len
= sizeof "fallthrough" - 1;
2049 if ((size_t) (pfile
->buffer
->cur
- from
- 1) < len
)
2051 if (memcmp (from
+ 1, "fallthrough", len
))
2055 if (from
[len
+ 1] != '@')
2061 /* Whole comment contents (regex):
2062 [ \t]*FALL(S | |-)?THR(OUGH|U)\.?[ \t]*
2063 [ \t]*Fall(s | |-)?[Tt]hr(ough|u)\.?[ \t]*
2064 [ \t]*fall(s | |-)?thr(ough|u)\.?[ \t]*
2068 while (*from
== ' ' || *from
== '\t')
2070 unsigned char f
= *from
;
2071 if (f
!= 'F' && f
!= 'f')
2073 if ((size_t) (pfile
->buffer
->cur
- from
) < sizeof "fallthrough")
2075 bool all_upper
= false;
2076 if (f
== 'F' && memcmp (from
+ 1, "ALL", sizeof "ALL" - 1) == 0)
2078 else if (memcmp (from
+ 1, "all", sizeof "all" - 1))
2080 if (from
[sizeof "fall" - 1] == (all_upper
? 'S' : 's')
2081 && from
[sizeof "falls" - 1] == ' ')
2082 from
+= sizeof "falls " - 1;
2083 else if (from
[sizeof "fall" - 1] == ' '
2084 || from
[sizeof "fall" - 1] == '-')
2085 from
+= sizeof "fall " - 1;
2086 else if (from
[sizeof "fall" - 1] != (all_upper
? 'T' : 't'))
2089 from
+= sizeof "fall" - 1;
2090 if ((f
== 'f' || *from
!= 'T') && (all_upper
|| *from
!= 't'))
2092 if ((size_t) (pfile
->buffer
->cur
- from
) < sizeof "thru")
2094 if (memcmp (from
+ 1, all_upper
? "HRU" : "hru", sizeof "hru" - 1))
2096 if ((size_t) (pfile
->buffer
->cur
- from
) < sizeof "through")
2098 if (memcmp (from
+ 1, all_upper
? "HROUGH" : "hrough",
2099 sizeof "hrough" - 1))
2101 from
+= sizeof "through" - 1;
2104 from
+= sizeof "thru" - 1;
2107 while (*from
== ' ' || *from
== '\t')
2110 /* C block comment. */
2111 if (*comment_start
== '*')
2113 if (*from
!= '*' || from
[1] != '/')
2116 /* C++ line comment. */
2117 else if (*from
!= '\n')
2123 /* Allocate COUNT tokens for RUN. */
2125 _cpp_init_tokenrun (tokenrun
*run
, unsigned int count
)
2127 run
->base
= XNEWVEC (cpp_token
, count
);
2128 run
->limit
= run
->base
+ count
;
2132 /* Returns the next tokenrun, or creates one if there is none. */
2134 next_tokenrun (tokenrun
*run
)
2136 if (run
->next
== NULL
)
2138 run
->next
= XNEW (tokenrun
);
2139 run
->next
->prev
= run
;
2140 _cpp_init_tokenrun (run
->next
, 250);
2146 /* Return the number of not yet processed token in a given
2149 _cpp_remaining_tokens_num_in_context (cpp_context
*context
)
2151 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
2152 return (LAST (context
).token
- FIRST (context
).token
);
2153 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
2154 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
2155 return (LAST (context
).ptoken
- FIRST (context
).ptoken
);
2160 /* Returns the token present at index INDEX in a given context. If
2161 INDEX is zero, the next token to be processed is returned. */
2162 static const cpp_token
*
2163 _cpp_token_from_context_at (cpp_context
*context
, int index
)
2165 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
2166 return &(FIRST (context
).token
[index
]);
2167 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
2168 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
2169 return FIRST (context
).ptoken
[index
];
2174 /* Look ahead in the input stream. */
2176 cpp_peek_token (cpp_reader
*pfile
, int index
)
2178 cpp_context
*context
= pfile
->context
;
2179 const cpp_token
*peektok
;
2182 /* First, scan through any pending cpp_context objects. */
2183 while (context
->prev
)
2185 ptrdiff_t sz
= _cpp_remaining_tokens_num_in_context (context
);
2187 if (index
< (int) sz
)
2188 return _cpp_token_from_context_at (context
, index
);
2190 context
= context
->prev
;
2193 /* We will have to read some new tokens after all (and do so
2194 without invalidating preceding tokens). */
2196 pfile
->keep_tokens
++;
2198 /* For peeked tokens temporarily disable line_change reporting,
2199 until the tokens are parsed for real. */
2200 void (*line_change
) (cpp_reader
*, const cpp_token
*, int)
2201 = pfile
->cb
.line_change
;
2202 pfile
->cb
.line_change
= NULL
;
2206 peektok
= _cpp_lex_token (pfile
);
2207 if (peektok
->type
== CPP_EOF
)
2215 _cpp_backup_tokens_direct (pfile
, count
- index
);
2216 pfile
->keep_tokens
--;
2217 pfile
->cb
.line_change
= line_change
;
2222 /* Allocate a single token that is invalidated at the same time as the
2223 rest of the tokens on the line. Has its line and col set to the
2224 same as the last lexed token, so that diagnostics appear in the
2227 _cpp_temp_token (cpp_reader
*pfile
)
2229 cpp_token
*old
, *result
;
2230 ptrdiff_t sz
= pfile
->cur_run
->limit
- pfile
->cur_token
;
2231 ptrdiff_t la
= (ptrdiff_t) pfile
->lookaheads
;
2233 old
= pfile
->cur_token
- 1;
2234 /* Any pre-existing lookaheads must not be clobbered. */
2239 tokenrun
*next
= next_tokenrun (pfile
->cur_run
);
2242 memmove (next
->base
+ 1, next
->base
,
2243 (la
- sz
) * sizeof (cpp_token
));
2245 next
->base
[0] = pfile
->cur_run
->limit
[-1];
2249 memmove (pfile
->cur_token
+ 1, pfile
->cur_token
,
2250 MIN (la
, sz
- 1) * sizeof (cpp_token
));
2253 if (!sz
&& pfile
->cur_token
== pfile
->cur_run
->limit
)
2255 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2256 pfile
->cur_token
= pfile
->cur_run
->base
;
2259 result
= pfile
->cur_token
++;
2260 result
->src_loc
= old
->src_loc
;
2264 /* Lex a token into RESULT (external interface). Takes care of issues
2265 like directive handling, token lookahead, multiple include
2266 optimization and skipping. */
2268 _cpp_lex_token (cpp_reader
*pfile
)
2274 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
2276 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2277 pfile
->cur_token
= pfile
->cur_run
->base
;
2279 /* We assume that the current token is somewhere in the current
2281 if (pfile
->cur_token
< pfile
->cur_run
->base
2282 || pfile
->cur_token
>= pfile
->cur_run
->limit
)
2285 if (pfile
->lookaheads
)
2287 pfile
->lookaheads
--;
2288 result
= pfile
->cur_token
++;
2291 result
= _cpp_lex_direct (pfile
);
2293 if (result
->flags
& BOL
)
2295 /* Is this a directive. If _cpp_handle_directive returns
2296 false, it is an assembler #. */
2297 if (result
->type
== CPP_HASH
2298 /* 6.10.3 p 11: Directives in a list of macro arguments
2299 gives undefined behavior. This implementation
2300 handles the directive as normal. */
2301 && pfile
->state
.parsing_args
!= 1)
2303 if (_cpp_handle_directive (pfile
, result
->flags
& PREV_WHITE
))
2305 if (pfile
->directive_result
.type
== CPP_PADDING
)
2307 result
= &pfile
->directive_result
;
2310 else if (pfile
->state
.in_deferred_pragma
)
2311 result
= &pfile
->directive_result
;
2313 if (pfile
->cb
.line_change
&& !pfile
->state
.skipping
)
2314 pfile
->cb
.line_change (pfile
, result
, pfile
->state
.parsing_args
);
2317 /* We don't skip tokens in directives. */
2318 if (pfile
->state
.in_directive
|| pfile
->state
.in_deferred_pragma
)
2321 /* Outside a directive, invalidate controlling macros. At file
2322 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
2323 get here and MI optimization works. */
2324 pfile
->mi_valid
= false;
2326 if (!pfile
->state
.skipping
|| result
->type
== CPP_EOF
)
2333 /* Returns true if a fresh line has been loaded. */
2335 _cpp_get_fresh_line (cpp_reader
*pfile
)
2339 /* We can't get a new line until we leave the current directive. */
2340 if (pfile
->state
.in_directive
)
2345 cpp_buffer
*buffer
= pfile
->buffer
;
2347 if (!buffer
->need_line
)
2350 if (buffer
->next_line
< buffer
->rlimit
)
2352 _cpp_clean_line (pfile
);
2356 /* First, get out of parsing arguments state. */
2357 if (pfile
->state
.parsing_args
)
2360 /* End of buffer. Non-empty files should end in a newline. */
2361 if (buffer
->buf
!= buffer
->rlimit
2362 && buffer
->next_line
> buffer
->rlimit
2363 && !buffer
->from_stage3
)
2365 /* Clip to buffer size. */
2366 buffer
->next_line
= buffer
->rlimit
;
2369 return_at_eof
= buffer
->return_at_eof
;
2370 _cpp_pop_buffer (pfile
);
2371 if (pfile
->buffer
== NULL
|| return_at_eof
)
2376 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
2379 result->type = ELSE_TYPE; \
2380 if (*buffer->cur == CHAR) \
2381 buffer->cur++, result->type = THEN_TYPE; \
2385 /* Lex a token into pfile->cur_token, which is also incremented, to
2386 get diagnostics pointing to the correct location.
2388 Does not handle issues such as token lookahead, multiple-include
2389 optimization, directives, skipping etc. This function is only
2390 suitable for use by _cpp_lex_token, and in special cases like
2391 lex_expansion_token which doesn't care for any of these issues.
2393 When meeting a newline, returns CPP_EOF if parsing a directive,
2394 otherwise returns to the start of the token buffer if permissible.
2395 Returns the location of the lexed token. */
2397 _cpp_lex_direct (cpp_reader
*pfile
)
2401 const unsigned char *comment_start
= NULL
;
2402 cpp_token
*result
= pfile
->cur_token
++;
2406 buffer
= pfile
->buffer
;
2407 if (buffer
->need_line
)
2409 if (pfile
->state
.in_deferred_pragma
)
2411 result
->type
= CPP_PRAGMA_EOL
;
2412 pfile
->state
.in_deferred_pragma
= false;
2413 if (!pfile
->state
.pragma_allow_expansion
)
2414 pfile
->state
.prevent_expansion
--;
2417 if (!_cpp_get_fresh_line (pfile
))
2419 result
->type
= CPP_EOF
;
2420 if (!pfile
->state
.in_directive
)
2422 /* Tell the compiler the line number of the EOF token. */
2423 result
->src_loc
= pfile
->line_table
->highest_line
;
2424 result
->flags
= BOL
;
2428 if (buffer
!= pfile
->buffer
)
2429 comment_start
= NULL
;
2430 if (!pfile
->keep_tokens
)
2432 pfile
->cur_run
= &pfile
->base_run
;
2433 result
= pfile
->base_run
.base
;
2434 pfile
->cur_token
= result
+ 1;
2436 result
->flags
= BOL
;
2437 if (pfile
->state
.parsing_args
== 2)
2438 result
->flags
|= PREV_WHITE
;
2440 buffer
= pfile
->buffer
;
2442 result
->src_loc
= pfile
->line_table
->highest_line
;
2445 if (buffer
->cur
>= buffer
->notes
[buffer
->cur_note
].pos
2446 && !pfile
->overlaid_buffer
)
2448 _cpp_process_line_notes (pfile
, false);
2449 result
->src_loc
= pfile
->line_table
->highest_line
;
2453 if (pfile
->forced_token_location_p
)
2454 result
->src_loc
= *pfile
->forced_token_location_p
;
2456 result
->src_loc
= linemap_position_for_column (pfile
->line_table
,
2457 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2461 case ' ': case '\t': case '\f': case '\v': case '\0':
2462 result
->flags
|= PREV_WHITE
;
2463 skip_whitespace (pfile
, c
);
2467 if (buffer
->cur
< buffer
->rlimit
)
2468 CPP_INCREMENT_LINE (pfile
, 0);
2469 buffer
->need_line
= true;
2472 case '0': case '1': case '2': case '3': case '4':
2473 case '5': case '6': case '7': case '8': case '9':
2475 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2476 result
->type
= CPP_NUMBER
;
2477 lex_number (pfile
, &result
->val
.str
, &nst
);
2478 warn_about_normalization (pfile
, result
, &nst
);
2486 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2487 wide strings or raw strings. */
2488 if (c
== 'L' || CPP_OPTION (pfile
, rliterals
)
2489 || (c
!= 'R' && CPP_OPTION (pfile
, uliterals
)))
2491 if ((*buffer
->cur
== '\'' && c
!= 'R')
2492 || *buffer
->cur
== '"'
2493 || (*buffer
->cur
== 'R'
2495 && buffer
->cur
[1] == '"'
2496 && CPP_OPTION (pfile
, rliterals
))
2497 || (*buffer
->cur
== '8'
2499 && ((buffer
->cur
[1] == '"' || (buffer
->cur
[1] == '\''
2500 && CPP_OPTION (pfile
, utf8_char_literals
)))
2501 || (buffer
->cur
[1] == 'R' && buffer
->cur
[2] == '"'
2502 && CPP_OPTION (pfile
, rliterals
)))))
2504 lex_string (pfile
, result
, buffer
->cur
- 1);
2511 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2512 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2513 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2514 case 's': case 't': case 'v': case 'w': case 'x':
2516 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2517 case 'G': case 'H': case 'I': case 'J': case 'K':
2518 case 'M': case 'N': case 'O': case 'P': case 'Q':
2519 case 'S': case 'T': case 'V': case 'W': case 'X':
2521 result
->type
= CPP_NAME
;
2523 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2524 result
->val
.node
.node
= lex_identifier (pfile
, buffer
->cur
- 1, false,
2526 &result
->val
.node
.spelling
);
2527 warn_about_normalization (pfile
, result
, &nst
);
2530 /* Convert named operators to their proper types. */
2531 if (result
->val
.node
.node
->flags
& NODE_OPERATOR
)
2533 result
->flags
|= NAMED_OP
;
2534 result
->type
= (enum cpp_ttype
) result
->val
.node
.node
->directive_index
;
2537 /* Signal FALLTHROUGH comment followed by another token. */
2539 && fallthrough_comment_p (pfile
, comment_start
))
2540 result
->flags
|= PREV_FALLTHROUGH
;
2545 lex_string (pfile
, result
, buffer
->cur
- 1);
2549 /* A potential block or line comment. */
2550 comment_start
= buffer
->cur
;
2555 if (_cpp_skip_block_comment (pfile
))
2556 cpp_error (pfile
, CPP_DL_ERROR
, "unterminated comment");
2558 else if (c
== '/' && ! CPP_OPTION (pfile
, traditional
))
2560 /* Don't warn for system headers. */
2561 if (cpp_in_system_header (pfile
))
2563 /* Warn about comments if pedantically GNUC89, and not
2564 in system headers. */
2565 else if (CPP_OPTION (pfile
, lang
) == CLK_GNUC89
2566 && CPP_PEDANTIC (pfile
)
2567 && ! buffer
->warned_cplusplus_comments
)
2569 cpp_error (pfile
, CPP_DL_PEDWARN
,
2570 "C++ style comments are not allowed in ISO C90");
2571 cpp_error (pfile
, CPP_DL_PEDWARN
,
2572 "(this will be reported only once per input file)");
2573 buffer
->warned_cplusplus_comments
= 1;
2575 /* Or if specifically desired via -Wc90-c99-compat. */
2576 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
) > 0
2577 && ! CPP_OPTION (pfile
, cplusplus
)
2578 && ! buffer
->warned_cplusplus_comments
)
2580 cpp_error (pfile
, CPP_DL_WARNING
,
2581 "C++ style comments are incompatible with C90");
2582 cpp_error (pfile
, CPP_DL_WARNING
,
2583 "(this will be reported only once per input file)");
2584 buffer
->warned_cplusplus_comments
= 1;
2586 /* In C89/C94, C++ style comments are forbidden. */
2587 else if ((CPP_OPTION (pfile
, lang
) == CLK_STDC89
2588 || CPP_OPTION (pfile
, lang
) == CLK_STDC94
))
2590 /* But don't be confused about valid code such as
2591 - // immediately followed by *,
2592 - // in a preprocessing directive,
2593 - // in an #if 0 block. */
2594 if (buffer
->cur
[1] == '*'
2595 || pfile
->state
.in_directive
2596 || pfile
->state
.skipping
)
2598 result
->type
= CPP_DIV
;
2601 else if (! buffer
->warned_cplusplus_comments
)
2603 cpp_error (pfile
, CPP_DL_ERROR
,
2604 "C++ style comments are not allowed in ISO C90");
2605 cpp_error (pfile
, CPP_DL_ERROR
,
2606 "(this will be reported only once per input "
2608 buffer
->warned_cplusplus_comments
= 1;
2611 if (skip_line_comment (pfile
) && CPP_OPTION (pfile
, warn_comments
))
2612 cpp_warning (pfile
, CPP_W_COMMENTS
, "multi-line comment");
2617 result
->type
= CPP_DIV_EQ
;
2622 result
->type
= CPP_DIV
;
2626 if (!pfile
->state
.save_comments
)
2628 result
->flags
|= PREV_WHITE
;
2629 goto update_tokens_line
;
2632 if (fallthrough_comment_p (pfile
, comment_start
))
2633 result
->flags
|= PREV_FALLTHROUGH
;
2635 /* Save the comment as a token in its own right. */
2636 save_comment (pfile
, result
, comment_start
, c
);
2640 if (pfile
->state
.angled_headers
)
2642 lex_string (pfile
, result
, buffer
->cur
- 1);
2643 if (result
->type
!= CPP_LESS
)
2647 result
->type
= CPP_LESS
;
2648 if (*buffer
->cur
== '=')
2649 buffer
->cur
++, result
->type
= CPP_LESS_EQ
;
2650 else if (*buffer
->cur
== '<')
2653 IF_NEXT_IS ('=', CPP_LSHIFT_EQ
, CPP_LSHIFT
);
2655 else if (CPP_OPTION (pfile
, digraphs
))
2657 if (*buffer
->cur
== ':')
2659 /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next
2660 three characters are <:: and the subsequent character
2661 is neither : nor >, the < is treated as a preprocessor
2662 token by itself". */
2663 if (CPP_OPTION (pfile
, cplusplus
)
2664 && CPP_OPTION (pfile
, lang
) != CLK_CXX98
2665 && CPP_OPTION (pfile
, lang
) != CLK_GNUCXX
2666 && buffer
->cur
[1] == ':'
2667 && buffer
->cur
[2] != ':' && buffer
->cur
[2] != '>')
2671 result
->flags
|= DIGRAPH
;
2672 result
->type
= CPP_OPEN_SQUARE
;
2674 else if (*buffer
->cur
== '%')
2677 result
->flags
|= DIGRAPH
;
2678 result
->type
= CPP_OPEN_BRACE
;
2684 result
->type
= CPP_GREATER
;
2685 if (*buffer
->cur
== '=')
2686 buffer
->cur
++, result
->type
= CPP_GREATER_EQ
;
2687 else if (*buffer
->cur
== '>')
2690 IF_NEXT_IS ('=', CPP_RSHIFT_EQ
, CPP_RSHIFT
);
2695 result
->type
= CPP_MOD
;
2696 if (*buffer
->cur
== '=')
2697 buffer
->cur
++, result
->type
= CPP_MOD_EQ
;
2698 else if (CPP_OPTION (pfile
, digraphs
))
2700 if (*buffer
->cur
== ':')
2703 result
->flags
|= DIGRAPH
;
2704 result
->type
= CPP_HASH
;
2705 if (*buffer
->cur
== '%' && buffer
->cur
[1] == ':')
2706 buffer
->cur
+= 2, result
->type
= CPP_PASTE
, result
->val
.token_no
= 0;
2708 else if (*buffer
->cur
== '>')
2711 result
->flags
|= DIGRAPH
;
2712 result
->type
= CPP_CLOSE_BRACE
;
2718 result
->type
= CPP_DOT
;
2719 if (ISDIGIT (*buffer
->cur
))
2721 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2722 result
->type
= CPP_NUMBER
;
2723 lex_number (pfile
, &result
->val
.str
, &nst
);
2724 warn_about_normalization (pfile
, result
, &nst
);
2726 else if (*buffer
->cur
== '.' && buffer
->cur
[1] == '.')
2727 buffer
->cur
+= 2, result
->type
= CPP_ELLIPSIS
;
2728 else if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2729 buffer
->cur
++, result
->type
= CPP_DOT_STAR
;
2733 result
->type
= CPP_PLUS
;
2734 if (*buffer
->cur
== '+')
2735 buffer
->cur
++, result
->type
= CPP_PLUS_PLUS
;
2736 else if (*buffer
->cur
== '=')
2737 buffer
->cur
++, result
->type
= CPP_PLUS_EQ
;
2741 result
->type
= CPP_MINUS
;
2742 if (*buffer
->cur
== '>')
2745 result
->type
= CPP_DEREF
;
2746 if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2747 buffer
->cur
++, result
->type
= CPP_DEREF_STAR
;
2749 else if (*buffer
->cur
== '-')
2750 buffer
->cur
++, result
->type
= CPP_MINUS_MINUS
;
2751 else if (*buffer
->cur
== '=')
2752 buffer
->cur
++, result
->type
= CPP_MINUS_EQ
;
2756 result
->type
= CPP_AND
;
2757 if (*buffer
->cur
== '&')
2758 buffer
->cur
++, result
->type
= CPP_AND_AND
;
2759 else if (*buffer
->cur
== '=')
2760 buffer
->cur
++, result
->type
= CPP_AND_EQ
;
2764 result
->type
= CPP_OR
;
2765 if (*buffer
->cur
== '|')
2766 buffer
->cur
++, result
->type
= CPP_OR_OR
;
2767 else if (*buffer
->cur
== '=')
2768 buffer
->cur
++, result
->type
= CPP_OR_EQ
;
2772 result
->type
= CPP_COLON
;
2773 if (*buffer
->cur
== ':' && CPP_OPTION (pfile
, cplusplus
))
2774 buffer
->cur
++, result
->type
= CPP_SCOPE
;
2775 else if (*buffer
->cur
== '>' && CPP_OPTION (pfile
, digraphs
))
2778 result
->flags
|= DIGRAPH
;
2779 result
->type
= CPP_CLOSE_SQUARE
;
2783 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ
, CPP_MULT
); break;
2784 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ
, CPP_EQ
); break;
2785 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ
, CPP_NOT
); break;
2786 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ
, CPP_XOR
); break;
2787 case '#': IF_NEXT_IS ('#', CPP_PASTE
, CPP_HASH
); result
->val
.token_no
= 0; break;
2789 case '?': result
->type
= CPP_QUERY
; break;
2790 case '~': result
->type
= CPP_COMPL
; break;
2791 case ',': result
->type
= CPP_COMMA
; break;
2792 case '(': result
->type
= CPP_OPEN_PAREN
; break;
2793 case ')': result
->type
= CPP_CLOSE_PAREN
; break;
2794 case '[': result
->type
= CPP_OPEN_SQUARE
; break;
2795 case ']': result
->type
= CPP_CLOSE_SQUARE
; break;
2796 case '{': result
->type
= CPP_OPEN_BRACE
; break;
2797 case '}': result
->type
= CPP_CLOSE_BRACE
; break;
2798 case ';': result
->type
= CPP_SEMICOLON
; break;
2800 /* @ is a punctuator in Objective-C. */
2801 case '@': result
->type
= CPP_ATSIGN
; break;
2806 const uchar
*base
= --buffer
->cur
;
2807 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2809 if (forms_identifier_p (pfile
, true, &nst
))
2811 result
->type
= CPP_NAME
;
2812 result
->val
.node
.node
= lex_identifier (pfile
, base
, true, &nst
,
2813 &result
->val
.node
.spelling
);
2814 warn_about_normalization (pfile
, result
, &nst
);
2822 create_literal (pfile
, result
, buffer
->cur
- 1, 1, CPP_OTHER
);
2826 source_range tok_range
;
2827 tok_range
.m_start
= result
->src_loc
;
2828 if (result
->src_loc
>= RESERVED_LOCATION_COUNT
)
2830 = linemap_position_for_column (pfile
->line_table
,
2831 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2833 tok_range
.m_finish
= tok_range
.m_start
;
2835 result
->src_loc
= COMBINE_LOCATION_DATA (pfile
->line_table
,
2842 /* An upper bound on the number of bytes needed to spell TOKEN.
2843 Does not include preceding whitespace. */
2845 cpp_token_len (const cpp_token
*token
)
2849 switch (TOKEN_SPELL (token
))
2851 default: len
= 6; break;
2852 case SPELL_LITERAL
: len
= token
->val
.str
.len
; break;
2853 case SPELL_IDENT
: len
= NODE_LEN (token
->val
.node
.node
) * 10; break;
2859 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2860 Return the number of bytes read out of NAME. (There are always
2861 10 bytes written to BUFFER.) */
2864 utf8_to_ucn (unsigned char *buffer
, const unsigned char *name
)
2870 unsigned long utf32
;
2872 /* Compute the length of the UTF-8 sequence. */
2873 for (t
= *name
; t
& 0x80; t
<<= 1)
2876 utf32
= *name
& (0x7F >> ucn_len
);
2877 for (ucn_len_c
= 1; ucn_len_c
< ucn_len
; ucn_len_c
++)
2879 utf32
= (utf32
<< 6) | (*++name
& 0x3F);
2881 /* Ill-formed UTF-8. */
2882 if ((*name
& ~0x3F) != 0x80)
2888 for (j
= 7; j
>= 0; j
--)
2889 *buffer
++ = "0123456789abcdef"[(utf32
>> (4 * j
)) & 0xF];
2893 /* Given a token TYPE corresponding to a digraph, return a pointer to
2894 the spelling of the digraph. */
2895 static const unsigned char *
2896 cpp_digraph2name (enum cpp_ttype type
)
2898 return digraph_spellings
[(int) type
- (int) CPP_FIRST_DIGRAPH
];
2901 /* Write the spelling of an identifier IDENT, using UCNs, to BUFFER.
2902 The buffer must already contain the enough space to hold the
2903 token's spelling. Returns a pointer to the character after the
2904 last character written. */
2906 _cpp_spell_ident_ucns (unsigned char *buffer
, cpp_hashnode
*ident
)
2909 const unsigned char *name
= NODE_NAME (ident
);
2911 for (i
= 0; i
< NODE_LEN (ident
); i
++)
2912 if (name
[i
] & ~0x7F)
2914 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2918 *buffer
++ = name
[i
];
2923 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
2924 already contain the enough space to hold the token's spelling.
2925 Returns a pointer to the character after the last character written.
2926 FORSTRING is true if this is to be the spelling after translation
2927 phase 1 (with the original spelling of extended identifiers), false
2928 if extended identifiers should always be written using UCNs (there is
2929 no option for always writing them in the internal UTF-8 form).
2930 FIXME: Would be nice if we didn't need the PFILE argument. */
2932 cpp_spell_token (cpp_reader
*pfile
, const cpp_token
*token
,
2933 unsigned char *buffer
, bool forstring
)
2935 switch (TOKEN_SPELL (token
))
2937 case SPELL_OPERATOR
:
2939 const unsigned char *spelling
;
2942 if (token
->flags
& DIGRAPH
)
2943 spelling
= cpp_digraph2name (token
->type
);
2944 else if (token
->flags
& NAMED_OP
)
2947 spelling
= TOKEN_NAME (token
);
2949 while ((c
= *spelling
++) != '\0')
2958 memcpy (buffer
, NODE_NAME (token
->val
.node
.spelling
),
2959 NODE_LEN (token
->val
.node
.spelling
));
2960 buffer
+= NODE_LEN (token
->val
.node
.spelling
);
2963 buffer
= _cpp_spell_ident_ucns (buffer
, token
->val
.node
.node
);
2967 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
2968 buffer
+= token
->val
.str
.len
;
2972 cpp_error (pfile
, CPP_DL_ICE
,
2973 "unspellable token %s", TOKEN_NAME (token
));
2980 /* Returns TOKEN spelt as a null-terminated string. The string is
2981 freed when the reader is destroyed. Useful for diagnostics. */
2983 cpp_token_as_text (cpp_reader
*pfile
, const cpp_token
*token
)
2985 unsigned int len
= cpp_token_len (token
) + 1;
2986 unsigned char *start
= _cpp_unaligned_alloc (pfile
, len
), *end
;
2988 end
= cpp_spell_token (pfile
, token
, start
, false);
2994 /* Returns a pointer to a string which spells the token defined by
2995 TYPE and FLAGS. Used by C front ends, which really should move to
2996 using cpp_token_as_text. */
2998 cpp_type2name (enum cpp_ttype type
, unsigned char flags
)
3000 if (flags
& DIGRAPH
)
3001 return (const char *) cpp_digraph2name (type
);
3002 else if (flags
& NAMED_OP
)
3003 return cpp_named_operator2name (type
);
3005 return (const char *) token_spellings
[type
].name
;
3008 /* Writes the spelling of token to FP, without any preceding space.
3009 Separated from cpp_spell_token for efficiency - to avoid stdio
3010 double-buffering. */
3012 cpp_output_token (const cpp_token
*token
, FILE *fp
)
3014 switch (TOKEN_SPELL (token
))
3016 case SPELL_OPERATOR
:
3018 const unsigned char *spelling
;
3021 if (token
->flags
& DIGRAPH
)
3022 spelling
= cpp_digraph2name (token
->type
);
3023 else if (token
->flags
& NAMED_OP
)
3026 spelling
= TOKEN_NAME (token
);
3031 while ((c
= *++spelling
) != '\0');
3039 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
3041 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
3042 if (name
[i
] & ~0x7F)
3044 unsigned char buffer
[10];
3045 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
3046 fwrite (buffer
, 1, 10, fp
);
3049 fputc (NODE_NAME (token
->val
.node
.node
)[i
], fp
);
3054 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
3058 /* An error, most probably. */
3063 /* Compare two tokens. */
3065 _cpp_equiv_tokens (const cpp_token
*a
, const cpp_token
*b
)
3067 if (a
->type
== b
->type
&& a
->flags
== b
->flags
)
3068 switch (TOKEN_SPELL (a
))
3070 default: /* Keep compiler happy. */
3071 case SPELL_OPERATOR
:
3072 /* token_no is used to track where multiple consecutive ##
3073 tokens were originally located. */
3074 return (a
->type
!= CPP_PASTE
|| a
->val
.token_no
== b
->val
.token_no
);
3076 return (a
->type
!= CPP_MACRO_ARG
3077 || (a
->val
.macro_arg
.arg_no
== b
->val
.macro_arg
.arg_no
3078 && a
->val
.macro_arg
.spelling
== b
->val
.macro_arg
.spelling
));
3080 return (a
->val
.node
.node
== b
->val
.node
.node
3081 && a
->val
.node
.spelling
== b
->val
.node
.spelling
);
3083 return (a
->val
.str
.len
== b
->val
.str
.len
3084 && !memcmp (a
->val
.str
.text
, b
->val
.str
.text
,
3091 /* Returns nonzero if a space should be inserted to avoid an
3092 accidental token paste for output. For simplicity, it is
3093 conservative, and occasionally advises a space where one is not
3094 needed, e.g. "." and ".2". */
3096 cpp_avoid_paste (cpp_reader
*pfile
, const cpp_token
*token1
,
3097 const cpp_token
*token2
)
3099 enum cpp_ttype a
= token1
->type
, b
= token2
->type
;
3102 if (token1
->flags
& NAMED_OP
)
3104 if (token2
->flags
& NAMED_OP
)
3108 if (token2
->flags
& DIGRAPH
)
3109 c
= digraph_spellings
[(int) b
- (int) CPP_FIRST_DIGRAPH
][0];
3110 else if (token_spellings
[b
].category
== SPELL_OPERATOR
)
3111 c
= token_spellings
[b
].name
[0];
3113 /* Quickly get everything that can paste with an '='. */
3114 if ((int) a
<= (int) CPP_LAST_EQ
&& c
== '=')
3119 case CPP_GREATER
: return c
== '>';
3120 case CPP_LESS
: return c
== '<' || c
== '%' || c
== ':';
3121 case CPP_PLUS
: return c
== '+';
3122 case CPP_MINUS
: return c
== '-' || c
== '>';
3123 case CPP_DIV
: return c
== '/' || c
== '*'; /* Comments. */
3124 case CPP_MOD
: return c
== ':' || c
== '>';
3125 case CPP_AND
: return c
== '&';
3126 case CPP_OR
: return c
== '|';
3127 case CPP_COLON
: return c
== ':' || c
== '>';
3128 case CPP_DEREF
: return c
== '*';
3129 case CPP_DOT
: return c
== '.' || c
== '%' || b
== CPP_NUMBER
;
3130 case CPP_HASH
: return c
== '#' || c
== '%'; /* Digraph form. */
3131 case CPP_NAME
: return ((b
== CPP_NUMBER
3132 && name_p (pfile
, &token2
->val
.str
))
3134 || b
== CPP_CHAR
|| b
== CPP_STRING
); /* L */
3135 case CPP_NUMBER
: return (b
== CPP_NUMBER
|| b
== CPP_NAME
3136 || c
== '.' || c
== '+' || c
== '-');
3138 case CPP_OTHER
: return ((token1
->val
.str
.text
[0] == '\\'
3140 || (CPP_OPTION (pfile
, objc
)
3141 && token1
->val
.str
.text
[0] == '@'
3142 && (b
== CPP_NAME
|| b
== CPP_STRING
)));
3145 case CPP_UTF8STRING
:
3147 case CPP_STRING32
: return (CPP_OPTION (pfile
, user_literals
)
3149 || (TOKEN_SPELL (token2
) == SPELL_LITERAL
3150 && ISIDST (token2
->val
.str
.text
[0]))));
3158 /* Output all the remaining tokens on the current line, and a newline
3159 character, to FP. Leading whitespace is removed. If there are
3160 macros, special token padding is not performed. */
3162 cpp_output_line (cpp_reader
*pfile
, FILE *fp
)
3164 const cpp_token
*token
;
3166 token
= cpp_get_token (pfile
);
3167 while (token
->type
!= CPP_EOF
)
3169 cpp_output_token (token
, fp
);
3170 token
= cpp_get_token (pfile
);
3171 if (token
->flags
& PREV_WHITE
)
3178 /* Return a string representation of all the remaining tokens on the
3179 current line. The result is allocated using xmalloc and must be
3180 freed by the caller. */
3182 cpp_output_line_to_string (cpp_reader
*pfile
, const unsigned char *dir_name
)
3184 const cpp_token
*token
;
3185 unsigned int out
= dir_name
? ustrlen (dir_name
) : 0;
3186 unsigned int alloced
= 120 + out
;
3187 unsigned char *result
= (unsigned char *) xmalloc (alloced
);
3189 /* If DIR_NAME is empty, there are no initial contents. */
3192 sprintf ((char *) result
, "#%s ", dir_name
);
3196 token
= cpp_get_token (pfile
);
3197 while (token
->type
!= CPP_EOF
)
3199 unsigned char *last
;
3200 /* Include room for a possible space and the terminating nul. */
3201 unsigned int len
= cpp_token_len (token
) + 2;
3203 if (out
+ len
> alloced
)
3206 if (out
+ len
> alloced
)
3207 alloced
= out
+ len
;
3208 result
= (unsigned char *) xrealloc (result
, alloced
);
3211 last
= cpp_spell_token (pfile
, token
, &result
[out
], 0);
3212 out
= last
- result
;
3214 token
= cpp_get_token (pfile
);
3215 if (token
->flags
& PREV_WHITE
)
3216 result
[out
++] = ' ';
3223 /* Memory buffers. Changing these three constants can have a dramatic
3224 effect on performance. The values here are reasonable defaults,
3225 but might be tuned. If you adjust them, be sure to test across a
3226 range of uses of cpplib, including heavy nested function-like macro
3227 expansion. Also check the change in peak memory usage (NJAMD is a
3228 good tool for this). */
3229 #define MIN_BUFF_SIZE 8000
3230 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
3231 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
3232 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
3234 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
3235 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
3238 /* Create a new allocation buffer. Place the control block at the end
3239 of the buffer, so that buffer overflows will cause immediate chaos. */
3241 new_buff (size_t len
)
3244 unsigned char *base
;
3246 if (len
< MIN_BUFF_SIZE
)
3247 len
= MIN_BUFF_SIZE
;
3248 len
= CPP_ALIGN (len
);
3250 #ifdef ENABLE_VALGRIND_ANNOTATIONS
3251 /* Valgrind warns about uses of interior pointers, so put _cpp_buff
3253 size_t slen
= CPP_ALIGN2 (sizeof (_cpp_buff
), 2 * DEFAULT_ALIGNMENT
);
3254 base
= XNEWVEC (unsigned char, len
+ slen
);
3255 result
= (_cpp_buff
*) base
;
3258 base
= XNEWVEC (unsigned char, len
+ sizeof (_cpp_buff
));
3259 result
= (_cpp_buff
*) (base
+ len
);
3261 result
->base
= base
;
3263 result
->limit
= base
+ len
;
3264 result
->next
= NULL
;
3268 /* Place a chain of unwanted allocation buffers on the free list. */
3270 _cpp_release_buff (cpp_reader
*pfile
, _cpp_buff
*buff
)
3272 _cpp_buff
*end
= buff
;
3276 end
->next
= pfile
->free_buffs
;
3277 pfile
->free_buffs
= buff
;
3280 /* Return a free buffer of size at least MIN_SIZE. */
3282 _cpp_get_buff (cpp_reader
*pfile
, size_t min_size
)
3284 _cpp_buff
*result
, **p
;
3286 for (p
= &pfile
->free_buffs
;; p
= &(*p
)->next
)
3291 return new_buff (min_size
);
3293 size
= result
->limit
- result
->base
;
3294 /* Return a buffer that's big enough, but don't waste one that's
3296 if (size
>= min_size
&& size
<= BUFF_SIZE_UPPER_BOUND (min_size
))
3301 result
->next
= NULL
;
3302 result
->cur
= result
->base
;
3306 /* Creates a new buffer with enough space to hold the uncommitted
3307 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
3308 the excess bytes to the new buffer. Chains the new buffer after
3309 BUFF, and returns the new buffer. */
3311 _cpp_append_extend_buff (cpp_reader
*pfile
, _cpp_buff
*buff
, size_t min_extra
)
3313 size_t size
= EXTENDED_BUFF_SIZE (buff
, min_extra
);
3314 _cpp_buff
*new_buff
= _cpp_get_buff (pfile
, size
);
3316 buff
->next
= new_buff
;
3317 memcpy (new_buff
->base
, buff
->cur
, BUFF_ROOM (buff
));
3321 /* Creates a new buffer with enough space to hold the uncommitted
3322 remaining bytes of the buffer pointed to by BUFF, and at least
3323 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
3324 Chains the new buffer before the buffer pointed to by BUFF, and
3325 updates the pointer to point to the new buffer. */
3327 _cpp_extend_buff (cpp_reader
*pfile
, _cpp_buff
**pbuff
, size_t min_extra
)
3329 _cpp_buff
*new_buff
, *old_buff
= *pbuff
;
3330 size_t size
= EXTENDED_BUFF_SIZE (old_buff
, min_extra
);
3332 new_buff
= _cpp_get_buff (pfile
, size
);
3333 memcpy (new_buff
->base
, old_buff
->cur
, BUFF_ROOM (old_buff
));
3334 new_buff
->next
= old_buff
;
3338 /* Free a chain of buffers starting at BUFF. */
3340 _cpp_free_buff (_cpp_buff
*buff
)
3344 for (; buff
; buff
= next
)
3347 #ifdef ENABLE_VALGRIND_ANNOTATIONS
3355 /* Allocate permanent, unaligned storage of length LEN. */
3357 _cpp_unaligned_alloc (cpp_reader
*pfile
, size_t len
)
3359 _cpp_buff
*buff
= pfile
->u_buff
;
3360 unsigned char *result
= buff
->cur
;
3362 if (len
> (size_t) (buff
->limit
- result
))
3364 buff
= _cpp_get_buff (pfile
, len
);
3365 buff
->next
= pfile
->u_buff
;
3366 pfile
->u_buff
= buff
;
3370 buff
->cur
= result
+ len
;
3374 /* Allocate permanent, unaligned storage of length LEN from a_buff.
3375 That buffer is used for growing allocations when saving macro
3376 replacement lists in a #define, and when parsing an answer to an
3377 assertion in #assert, #unassert or #if (and therefore possibly
3378 whilst expanding macros). It therefore must not be used by any
3379 code that they might call: specifically the lexer and the guts of
3382 All existing other uses clearly fit this restriction: storing
3383 registered pragmas during initialization. */
3385 _cpp_aligned_alloc (cpp_reader
*pfile
, size_t len
)
3387 _cpp_buff
*buff
= pfile
->a_buff
;
3388 unsigned char *result
= buff
->cur
;
3390 if (len
> (size_t) (buff
->limit
- result
))
3392 buff
= _cpp_get_buff (pfile
, len
);
3393 buff
->next
= pfile
->a_buff
;
3394 pfile
->a_buff
= buff
;
3398 buff
->cur
= result
+ len
;
3402 /* Say which field of TOK is in use. */
3404 enum cpp_token_fld_kind
3405 cpp_token_val_index (const cpp_token
*tok
)
3407 switch (TOKEN_SPELL (tok
))
3410 return CPP_TOKEN_FLD_NODE
;
3412 return CPP_TOKEN_FLD_STR
;
3413 case SPELL_OPERATOR
:
3414 if (tok
->type
== CPP_PASTE
)
3415 return CPP_TOKEN_FLD_TOKEN_NO
;
3417 return CPP_TOKEN_FLD_NONE
;
3419 if (tok
->type
== CPP_MACRO_ARG
)
3420 return CPP_TOKEN_FLD_ARG_NO
;
3421 else if (tok
->type
== CPP_PADDING
)
3422 return CPP_TOKEN_FLD_SOURCE
;
3423 else if (tok
->type
== CPP_PRAGMA
)
3424 return CPP_TOKEN_FLD_PRAGMA
;
3427 return CPP_TOKEN_FLD_NONE
;
3431 /* All tokens lexed in R after calling this function will be forced to have
3432 their source_location the same as the location referenced by P, until
3433 cpp_stop_forcing_token_locations is called for R. */
3436 cpp_force_token_locations (cpp_reader
*r
, source_location
*p
)
3438 r
->forced_token_location_p
= p
;
3441 /* Go back to assigning locations naturally for lexed tokens. */
3444 cpp_stop_forcing_token_locations (cpp_reader
*r
)
3446 r
->forced_token_location_p
= NULL
;