1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000-2016 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
8 This program is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by the
10 Free Software Foundation; either version 3, or (at your option) any
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
37 enum spell_type category
;
38 const unsigned char *name
;
41 static const unsigned char *const digraph_spellings
[] =
42 { UC
"%:", UC
"%:%:", UC
"<:", UC
":>", UC
"<%", UC
"%>" };
44 #define OP(e, s) { SPELL_OPERATOR, UC s },
45 #define TK(e, s) { SPELL_ ## s, UC #e },
46 static const struct token_spelling token_spellings
[N_TTYPES
] = { TTYPE_TABLE
};
50 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
51 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
53 static void add_line_note (cpp_buffer
*, const uchar
*, unsigned int);
54 static int skip_line_comment (cpp_reader
*);
55 static void skip_whitespace (cpp_reader
*, cppchar_t
);
56 static void lex_string (cpp_reader
*, cpp_token
*, const uchar
*);
57 static void save_comment (cpp_reader
*, cpp_token
*, const uchar
*, cppchar_t
);
58 static void store_comment (cpp_reader
*, cpp_token
*);
59 static void create_literal (cpp_reader
*, cpp_token
*, const uchar
*,
60 unsigned int, enum cpp_ttype
);
61 static bool warn_in_comment (cpp_reader
*, _cpp_line_note
*);
62 static int name_p (cpp_reader
*, const cpp_string
*);
63 static tokenrun
*next_tokenrun (tokenrun
*);
65 static _cpp_buff
*new_buff (size_t);
70 Compares, the token TOKEN to the NUL-terminated string STRING.
71 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
73 cpp_ideq (const cpp_token
*token
, const char *string
)
75 if (token
->type
!= CPP_NAME
)
78 return !ustrcmp (NODE_NAME (token
->val
.node
.node
), (const uchar
*) string
);
81 /* Record a note TYPE at byte POS into the current cleaned logical
84 add_line_note (cpp_buffer
*buffer
, const uchar
*pos
, unsigned int type
)
86 if (buffer
->notes_used
== buffer
->notes_cap
)
88 buffer
->notes_cap
= buffer
->notes_cap
* 2 + 200;
89 buffer
->notes
= XRESIZEVEC (_cpp_line_note
, buffer
->notes
,
93 buffer
->notes
[buffer
->notes_used
].pos
= pos
;
94 buffer
->notes
[buffer
->notes_used
].type
= type
;
99 /* Fast path to find line special characters using optimized character
100 scanning algorithms. Anything complicated falls back to the slow
101 path below. Since this loop is very hot it's worth doing these kinds
104 One of the paths through the ifdefs should provide
106 const uchar *search_line_fast (const uchar *s, const uchar *end);
108 Between S and END, search for \n, \r, \\, ?. Return a pointer to
111 Note that the last character of the buffer is *always* a newline,
112 as forced by _cpp_convert_input. This fact can be used to avoid
113 explicitly looking for the end of the buffer. */
115 /* Configure gives us an ifdef test. */
116 #ifndef WORDS_BIGENDIAN
117 #define WORDS_BIGENDIAN 0
120 /* We'd like the largest integer that fits into a register. There's nothing
121 in <stdint.h> that gives us that. For most hosts this is unsigned long,
122 but MS decided on an LLP64 model. Thankfully when building with GCC we
123 can get the "real" word size. */
125 typedef unsigned int word_type
__attribute__((__mode__(__word__
)));
127 typedef unsigned long word_type
;
130 /* The code below is only expecting sizes 4 or 8.
131 Die at compile-time if this expectation is violated. */
132 typedef char check_word_type_size
133 [(sizeof(word_type
) == 8 || sizeof(word_type
) == 4) * 2 - 1];
135 /* Return X with the first N bytes forced to values that won't match one
136 of the interesting characters. Note that NUL is not interesting. */
138 static inline word_type
139 acc_char_mask_misalign (word_type val
, unsigned int n
)
149 /* Return X replicated to all byte positions within WORD_TYPE. */
151 static inline word_type
152 acc_char_replicate (uchar x
)
156 ret
= (x
<< 24) | (x
<< 16) | (x
<< 8) | x
;
157 if (sizeof(word_type
) == 8)
158 ret
= (ret
<< 16 << 16) | ret
;
162 /* Return non-zero if some byte of VAL is (probably) C. */
164 static inline word_type
165 acc_char_cmp (word_type val
, word_type c
)
167 #if defined(__GNUC__) && defined(__alpha__)
168 /* We can get exact results using a compare-bytes instruction.
169 Get (val == c) via (0 >= (val ^ c)). */
170 return __builtin_alpha_cmpbge (0, val
^ c
);
172 word_type magic
= 0x7efefefeU
;
173 if (sizeof(word_type
) == 8)
174 magic
= (magic
<< 16 << 16) | 0xfefefefeU
;
178 return ((val
+ magic
) ^ ~val
) & ~magic
;
182 /* Given the result of acc_char_cmp is non-zero, return the index of
183 the found character. If this was a false positive, return -1. */
186 acc_char_index (word_type cmp ATTRIBUTE_UNUSED
,
187 word_type val ATTRIBUTE_UNUSED
)
189 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
190 /* The cmpbge instruction sets *bits* of the result corresponding to
191 matches in the bytes with no false positives. */
192 return __builtin_ctzl (cmp
);
196 /* ??? It would be nice to force unrolling here,
197 and have all of these constants folded. */
198 for (i
= 0; i
< sizeof(word_type
); ++i
)
202 c
= (val
>> (sizeof(word_type
) - i
- 1) * 8) & 0xff;
204 c
= (val
>> i
* 8) & 0xff;
206 if (c
== '\n' || c
== '\r' || c
== '\\' || c
== '?')
214 /* A version of the fast scanner using bit fiddling techniques.
216 For 32-bit words, one would normally perform 16 comparisons and
217 16 branches. With this algorithm one performs 24 arithmetic
218 operations and one branch. Whether this is faster with a 32-bit
219 word size is going to be somewhat system dependent.
221 For 64-bit words, we eliminate twice the number of comparisons
222 and branches without increasing the number of arithmetic operations.
223 It's almost certainly going to be a win with 64-bit word size. */
225 static const uchar
* search_line_acc_char (const uchar
*, const uchar
*)
229 search_line_acc_char (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
231 const word_type repl_nl
= acc_char_replicate ('\n');
232 const word_type repl_cr
= acc_char_replicate ('\r');
233 const word_type repl_bs
= acc_char_replicate ('\\');
234 const word_type repl_qm
= acc_char_replicate ('?');
236 unsigned int misalign
;
240 /* Align the buffer. Mask out any bytes from before the beginning. */
241 p
= (word_type
*)((uintptr_t)s
& -sizeof(word_type
));
243 misalign
= (uintptr_t)s
& (sizeof(word_type
) - 1);
245 val
= acc_char_mask_misalign (val
, misalign
);
250 t
= acc_char_cmp (val
, repl_nl
);
251 t
|= acc_char_cmp (val
, repl_cr
);
252 t
|= acc_char_cmp (val
, repl_bs
);
253 t
|= acc_char_cmp (val
, repl_qm
);
255 if (__builtin_expect (t
!= 0, 0))
257 int i
= acc_char_index (t
, val
);
259 return (const uchar
*)p
+ i
;
266 /* Disable on Solaris 2/x86 until the following problem can be properly
269 The Solaris 10+ assembler tags objects with the instruction set
270 extensions used, so SSE4.2 executables cannot run on machines that
271 don't support that extension. */
273 #if (GCC_VERSION >= 4005) && (__GNUC__ >= 5 || !defined(__PIC__)) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
275 /* Replicated character data to be shared between implementations.
276 Recall that outside of a context with vector support we can't
277 define compatible vector types, therefore these are all defined
278 in terms of raw characters. */
279 static const char repl_chars
[4][16] __attribute__((aligned(16))) = {
280 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
281 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
282 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
283 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
284 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
285 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
286 { '?', '?', '?', '?', '?', '?', '?', '?',
287 '?', '?', '?', '?', '?', '?', '?', '?' },
290 /* A version of the fast scanner using MMX vectorized byte compare insns.
292 This uses the PMOVMSKB instruction which was introduced with "MMX2",
293 which was packaged into SSE1; it is also present in the AMD MMX
294 extension. Mark the function as using "sse" so that we emit a real
295 "emms" instruction, rather than the 3dNOW "femms" instruction. */
299 __attribute__((__target__("sse")))
301 search_line_mmx (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
303 typedef char v8qi
__attribute__ ((__vector_size__ (8)));
304 typedef int __m64
__attribute__ ((__vector_size__ (8), __may_alias__
));
306 const v8qi repl_nl
= *(const v8qi
*)repl_chars
[0];
307 const v8qi repl_cr
= *(const v8qi
*)repl_chars
[1];
308 const v8qi repl_bs
= *(const v8qi
*)repl_chars
[2];
309 const v8qi repl_qm
= *(const v8qi
*)repl_chars
[3];
311 unsigned int misalign
, found
, mask
;
315 /* Align the source pointer. While MMX doesn't generate unaligned data
316 faults, this allows us to safely scan to the end of the buffer without
317 reading beyond the end of the last page. */
318 misalign
= (uintptr_t)s
& 7;
319 p
= (const v8qi
*)((uintptr_t)s
& -8);
322 /* Create a mask for the bytes that are valid within the first
323 16-byte block. The Idea here is that the AND with the mask
324 within the loop is "free", since we need some AND or TEST
325 insn in order to set the flags for the branch anyway. */
326 mask
= -1u << misalign
;
328 /* Main loop processing 8 bytes at a time. */
336 t
= __builtin_ia32_pcmpeqb(data
, repl_nl
);
337 c
= __builtin_ia32_pcmpeqb(data
, repl_cr
);
338 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
339 c
= __builtin_ia32_pcmpeqb(data
, repl_bs
);
340 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
341 c
= __builtin_ia32_pcmpeqb(data
, repl_qm
);
342 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
343 found
= __builtin_ia32_pmovmskb (t
);
348 __builtin_ia32_emms ();
350 /* FOUND contains 1 in bits for which we matched a relevant
351 character. Conversion to the byte index is trivial. */
352 found
= __builtin_ctz(found
);
353 return (const uchar
*)p
+ found
;
356 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
360 __attribute__((__target__("sse2")))
362 search_line_sse2 (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
364 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
366 const v16qi repl_nl
= *(const v16qi
*)repl_chars
[0];
367 const v16qi repl_cr
= *(const v16qi
*)repl_chars
[1];
368 const v16qi repl_bs
= *(const v16qi
*)repl_chars
[2];
369 const v16qi repl_qm
= *(const v16qi
*)repl_chars
[3];
371 unsigned int misalign
, found
, mask
;
375 /* Align the source pointer. */
376 misalign
= (uintptr_t)s
& 15;
377 p
= (const v16qi
*)((uintptr_t)s
& -16);
380 /* Create a mask for the bytes that are valid within the first
381 16-byte block. The Idea here is that the AND with the mask
382 within the loop is "free", since we need some AND or TEST
383 insn in order to set the flags for the branch anyway. */
384 mask
= -1u << misalign
;
386 /* Main loop processing 16 bytes at a time. */
394 t
= __builtin_ia32_pcmpeqb128(data
, repl_nl
);
395 t
|= __builtin_ia32_pcmpeqb128(data
, repl_cr
);
396 t
|= __builtin_ia32_pcmpeqb128(data
, repl_bs
);
397 t
|= __builtin_ia32_pcmpeqb128(data
, repl_qm
);
398 found
= __builtin_ia32_pmovmskb128 (t
);
403 /* FOUND contains 1 in bits for which we matched a relevant
404 character. Conversion to the byte index is trivial. */
405 found
= __builtin_ctz(found
);
406 return (const uchar
*)p
+ found
;
410 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
414 __attribute__((__target__("sse4.2")))
416 search_line_sse42 (const uchar
*s
, const uchar
*end
)
418 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
419 static const v16qi search
= { '\n', '\r', '?', '\\' };
421 uintptr_t si
= (uintptr_t)s
;
424 /* Check for unaligned input. */
429 if (__builtin_expect (end
- s
< 16, 0)
430 && __builtin_expect ((si
& 0xfff) > 0xff0, 0))
432 /* There are less than 16 bytes left in the buffer, and less
433 than 16 bytes left on the page. Reading 16 bytes at this
434 point might generate a spurious page fault. Defer to the
435 SSE2 implementation, which already handles alignment. */
436 return search_line_sse2 (s
, end
);
439 /* ??? The builtin doesn't understand that the PCMPESTRI read from
440 memory need not be aligned. */
441 sv
= __builtin_ia32_loaddqu ((const char *) s
);
442 index
= __builtin_ia32_pcmpestri128 (search
, 4, sv
, 16, 0);
444 if (__builtin_expect (index
< 16, 0))
447 /* Advance the pointer to an aligned address. We will re-scan a
448 few bytes, but we no longer need care for reading past the
449 end of a page, since we're guaranteed a match. */
450 s
= (const uchar
*)((si
+ 15) & -16);
453 /* Main loop, processing 16 bytes at a time. */
454 #ifdef __GCC_ASM_FLAG_OUTPUTS__
459 /* By using inline assembly instead of the builtin,
460 we can use the result, as well as the flags set. */
461 __asm ("%vpcmpestri\t$0, %2, %3"
462 : "=c"(index
), "=@ccc"(f
)
463 : "m"(*s
), "x"(search
), "a"(4), "d"(16));
471 /* By doing the whole loop in inline assembly,
472 we can make proper use of the flags set. */
473 __asm ( ".balign 16\n"
475 " %vpcmpestri\t$0, (%1), %2\n"
477 : "=&c"(index
), "+r"(s
)
478 : "x"(search
), "a"(4), "d"(16));
486 /* Work around out-dated assemblers without sse4 support. */
487 #define search_line_sse42 search_line_sse2
490 /* Check the CPU capabilities. */
492 #include "../gcc/config/i386/cpuid.h"
494 typedef const uchar
* (*search_line_fast_type
) (const uchar
*, const uchar
*);
495 static search_line_fast_type search_line_fast
;
497 #define HAVE_init_vectorized_lexer 1
499 init_vectorized_lexer (void)
501 unsigned dummy
, ecx
= 0, edx
= 0;
502 search_line_fast_type impl
= search_line_acc_char
;
505 #if defined(__SSE4_2__)
507 #elif defined(__SSE2__)
509 #elif defined(__SSE__)
514 impl
= search_line_sse42
;
515 else if (__get_cpuid (1, &dummy
, &dummy
, &ecx
, &edx
) || minimum
== 2)
517 if (minimum
== 3 || (ecx
& bit_SSE4_2
))
518 impl
= search_line_sse42
;
519 else if (minimum
== 2 || (edx
& bit_SSE2
))
520 impl
= search_line_sse2
;
521 else if (minimum
== 1 || (edx
& bit_SSE
))
522 impl
= search_line_mmx
;
524 else if (__get_cpuid (0x80000001, &dummy
, &dummy
, &dummy
, &edx
))
527 || (edx
& (bit_MMXEXT
| bit_CMOV
)) == (bit_MMXEXT
| bit_CMOV
))
528 impl
= search_line_mmx
;
531 search_line_fast
= impl
;
534 #elif defined(_ARCH_PWR8) && defined(__ALTIVEC__)
536 /* A vection of the fast scanner using AltiVec vectorized byte compares
537 and VSX unaligned loads (when VSX is available). This is otherwise
538 the same as the pre-GCC 5 version. */
540 ATTRIBUTE_NO_SANITIZE_UNDEFINED
542 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
544 typedef __attribute__((altivec(vector
))) unsigned char vc
;
547 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
548 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
551 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
552 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
555 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
556 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
559 '?', '?', '?', '?', '?', '?', '?', '?',
560 '?', '?', '?', '?', '?', '?', '?', '?',
562 const vc zero
= { 0 };
566 /* Main loop processing 16 bytes at a time. */
569 vc m_nl
, m_cr
, m_bs
, m_qm
;
571 data
= *((const vc
*)s
);
574 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
575 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
576 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
577 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
578 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
580 /* T now contains 0xff in bytes for which we matched one of the relevant
581 characters. We want to exit the loop if any byte in T is non-zero.
582 Below is the expansion of vec_any_ne(t, zero). */
584 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
586 /* Restore s to to point to the 16 bytes we just processed. */
590 #define N (sizeof(vc) / sizeof(long))
594 /* Statically assert that N is 2 or 4. */
595 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
597 unsigned long l
, i
= 0;
601 /* Find the first word of T that is non-zero. */
608 s
+= sizeof(unsigned long);
612 s
+= sizeof(unsigned long);
618 s
+= sizeof(unsigned long);
622 /* L now contains 0xff in bytes for which we matched one of the
623 relevant characters. We can find the byte index by finding
624 its bit index and dividing by 8. */
625 #ifdef __BIG_ENDIAN__
626 l
= __builtin_clzl(l
) >> 3;
628 l
= __builtin_ctzl(l
) >> 3;
636 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__) && defined (__BIG_ENDIAN__)
638 /* A vection of the fast scanner using AltiVec vectorized byte compares.
639 This cannot be used for little endian because vec_lvsl/lvsr are
640 deprecated for little endian and the code won't work properly. */
641 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
642 so we can't compile this function without -maltivec on the command line
643 (or implied by some other switch). */
646 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
648 typedef __attribute__((altivec(vector
))) unsigned char vc
;
651 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
652 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
655 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
656 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
659 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
660 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
663 '?', '?', '?', '?', '?', '?', '?', '?',
664 '?', '?', '?', '?', '?', '?', '?', '?',
667 -1, -1, -1, -1, -1, -1, -1, -1,
668 -1, -1, -1, -1, -1, -1, -1, -1,
670 const vc zero
= { 0 };
674 /* Altivec loads automatically mask addresses with -16. This lets us
675 issue the first load as early as possible. */
676 data
= __builtin_vec_ld(0, (const vc
*)s
);
678 /* Discard bytes before the beginning of the buffer. Do this by
679 beginning with all ones and shifting in zeros according to the
680 mis-alignment. The LVSR instruction pulls the exact shift we
681 want from the address. */
682 mask
= __builtin_vec_lvsr(0, s
);
683 mask
= __builtin_vec_perm(zero
, ones
, mask
);
686 /* While altivec loads mask addresses, we still need to align S so
687 that the offset we compute at the end is correct. */
688 s
= (const uchar
*)((uintptr_t)s
& -16);
690 /* Main loop processing 16 bytes at a time. */
694 vc m_nl
, m_cr
, m_bs
, m_qm
;
697 data
= __builtin_vec_ld(0, (const vc
*)s
);
700 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
701 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
702 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
703 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
704 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
706 /* T now contains 0xff in bytes for which we matched one of the relevant
707 characters. We want to exit the loop if any byte in T is non-zero.
708 Below is the expansion of vec_any_ne(t, zero). */
710 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
713 #define N (sizeof(vc) / sizeof(long))
717 /* Statically assert that N is 2 or 4. */
718 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
720 unsigned long l
, i
= 0;
724 /* Find the first word of T that is non-zero. */
731 s
+= sizeof(unsigned long);
735 s
+= sizeof(unsigned long);
741 s
+= sizeof(unsigned long);
745 /* L now contains 0xff in bytes for which we matched one of the
746 relevant characters. We can find the byte index by finding
747 its bit index and dividing by 8. */
748 l
= __builtin_clzl(l
) >> 3;
755 #elif defined (__ARM_NEON)
756 #include "arm_neon.h"
759 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
761 const uint8x16_t repl_nl
= vdupq_n_u8 ('\n');
762 const uint8x16_t repl_cr
= vdupq_n_u8 ('\r');
763 const uint8x16_t repl_bs
= vdupq_n_u8 ('\\');
764 const uint8x16_t repl_qm
= vdupq_n_u8 ('?');
765 const uint8x16_t xmask
= (uint8x16_t
) vdupq_n_u64 (0x8040201008040201ULL
);
767 unsigned int misalign
, found
, mask
;
771 /* Align the source pointer. */
772 misalign
= (uintptr_t)s
& 15;
773 p
= (const uint8_t *)((uintptr_t)s
& -16);
776 /* Create a mask for the bytes that are valid within the first
777 16-byte block. The Idea here is that the AND with the mask
778 within the loop is "free", since we need some AND or TEST
779 insn in order to set the flags for the branch anyway. */
780 mask
= (-1u << misalign
) & 0xffff;
782 /* Main loop, processing 16 bytes at a time. */
790 uint8x16_t t
, u
, v
, w
;
797 t
= vceqq_u8 (data
, repl_nl
);
798 u
= vceqq_u8 (data
, repl_cr
);
799 v
= vorrq_u8 (t
, vceqq_u8 (data
, repl_bs
));
800 w
= vorrq_u8 (u
, vceqq_u8 (data
, repl_qm
));
801 t
= vandq_u8 (vorrq_u8 (v
, w
), xmask
);
802 l
= vpadd_u8 (vget_low_u8 (t
), vget_high_u8 (t
));
806 found
= vget_lane_u32 ((uint32x2_t
) vorr_u64 ((uint64x1_t
) n
,
807 vshr_n_u64 ((uint64x1_t
) n
, 24)), 0);
812 /* FOUND contains 1 in bits for which we matched a relevant
813 character. Conversion to the byte index is trivial. */
814 found
= __builtin_ctz (found
);
815 return (const uchar
*)p
+ found
;
820 /* We only have one accellerated alternative. Use a direct call so that
821 we encourage inlining. */
823 #define search_line_fast search_line_acc_char
827 /* Initialize the lexer if needed. */
830 _cpp_init_lexer (void)
832 #ifdef HAVE_init_vectorized_lexer
833 init_vectorized_lexer ();
837 /* Returns with a logical line that contains no escaped newlines or
838 trigraphs. This is a time-critical inner loop. */
840 _cpp_clean_line (cpp_reader
*pfile
)
846 buffer
= pfile
->buffer
;
847 buffer
->cur_note
= buffer
->notes_used
= 0;
848 buffer
->cur
= buffer
->line_base
= buffer
->next_line
;
849 buffer
->need_line
= false;
850 s
= buffer
->next_line
;
852 if (!buffer
->from_stage3
)
854 const uchar
*pbackslash
= NULL
;
856 /* Fast path. This is the common case of an un-escaped line with
857 no trigraphs. The primary win here is by not writing any
858 data back to memory until we have to. */
861 /* Perform an optimized search for \n, \r, \\, ?. */
862 s
= search_line_fast (s
, buffer
->rlimit
);
867 /* Record the location of the backslash and continue. */
870 else if (__builtin_expect (c
== '?', 0))
872 if (__builtin_expect (s
[1] == '?', false)
873 && _cpp_trigraph_map
[s
[2]])
875 /* Have a trigraph. We may or may not have to convert
876 it. Add a line note regardless, for -Wtrigraphs. */
877 add_line_note (buffer
, s
, s
[2]);
878 if (CPP_OPTION (pfile
, trigraphs
))
880 /* We do, and that means we have to switch to the
883 *d
= _cpp_trigraph_map
[s
[2]];
888 /* Not a trigraph. Continue on fast-path. */
895 /* This must be \r or \n. We're either done, or we'll be forced
896 to write back to the buffer and continue on the slow path. */
899 if (__builtin_expect (s
== buffer
->rlimit
, false))
902 /* DOS line ending? */
903 if (__builtin_expect (c
== '\r', false) && s
[1] == '\n')
906 if (s
== buffer
->rlimit
)
910 if (__builtin_expect (pbackslash
== NULL
, true))
913 /* Check for escaped newline. */
915 while (is_nvspace (p
[-1]))
917 if (p
- 1 != pbackslash
)
920 /* Have an escaped newline; process it and proceed to
922 add_line_note (buffer
, p
- 1, p
!= d
? ' ' : '\\');
924 buffer
->next_line
= p
- 1;
932 if (c
== '\n' || c
== '\r')
934 /* Handle DOS line endings. */
935 if (c
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
937 if (s
== buffer
->rlimit
)
942 while (p
!= buffer
->next_line
&& is_nvspace (p
[-1]))
944 if (p
== buffer
->next_line
|| p
[-1] != '\\')
947 add_line_note (buffer
, p
- 1, p
!= d
? ' ': '\\');
949 buffer
->next_line
= p
- 1;
951 else if (c
== '?' && s
[1] == '?' && _cpp_trigraph_map
[s
[2]])
953 /* Add a note regardless, for the benefit of -Wtrigraphs. */
954 add_line_note (buffer
, d
, s
[2]);
955 if (CPP_OPTION (pfile
, trigraphs
))
957 *d
= _cpp_trigraph_map
[s
[2]];
965 while (*s
!= '\n' && *s
!= '\r')
969 /* Handle DOS line endings. */
970 if (*s
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
976 /* A sentinel note that should never be processed. */
977 add_line_note (buffer
, d
+ 1, '\n');
978 buffer
->next_line
= s
+ 1;
981 /* Return true if the trigraph indicated by NOTE should be warned
982 about in a comment. */
984 warn_in_comment (cpp_reader
*pfile
, _cpp_line_note
*note
)
988 /* Within comments we don't warn about trigraphs, unless the
989 trigraph forms an escaped newline, as that may change
991 if (note
->type
!= '/')
994 /* If -trigraphs, then this was an escaped newline iff the next note
996 if (CPP_OPTION (pfile
, trigraphs
))
997 return note
[1].pos
== note
->pos
;
999 /* Otherwise, see if this forms an escaped newline. */
1001 while (is_nvspace (*p
))
1004 /* There might have been escaped newlines between the trigraph and the
1005 newline we found. Hence the position test. */
1006 return (*p
== '\n' && p
< note
[1].pos
);
1009 /* Process the notes created by add_line_note as far as the current
1012 _cpp_process_line_notes (cpp_reader
*pfile
, int in_comment
)
1014 cpp_buffer
*buffer
= pfile
->buffer
;
1018 _cpp_line_note
*note
= &buffer
->notes
[buffer
->cur_note
];
1021 if (note
->pos
> buffer
->cur
)
1025 col
= CPP_BUF_COLUMN (buffer
, note
->pos
+ 1);
1027 if (note
->type
== '\\' || note
->type
== ' ')
1029 if (note
->type
== ' ' && !in_comment
)
1030 cpp_error_with_line (pfile
, CPP_DL_WARNING
, pfile
->line_table
->highest_line
, col
,
1031 "backslash and newline separated by space");
1033 if (buffer
->next_line
> buffer
->rlimit
)
1035 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
, col
,
1036 "backslash-newline at end of file");
1037 /* Prevent "no newline at end of file" warning. */
1038 buffer
->next_line
= buffer
->rlimit
;
1041 buffer
->line_base
= note
->pos
;
1042 CPP_INCREMENT_LINE (pfile
, 0);
1044 else if (_cpp_trigraph_map
[note
->type
])
1046 if (CPP_OPTION (pfile
, warn_trigraphs
)
1047 && (!in_comment
|| warn_in_comment (pfile
, note
)))
1049 if (CPP_OPTION (pfile
, trigraphs
))
1050 cpp_warning_with_line (pfile
, CPP_W_TRIGRAPHS
,
1051 pfile
->line_table
->highest_line
, col
,
1052 "trigraph ??%c converted to %c",
1054 (int) _cpp_trigraph_map
[note
->type
]);
1057 cpp_warning_with_line
1058 (pfile
, CPP_W_TRIGRAPHS
,
1059 pfile
->line_table
->highest_line
, col
,
1060 "trigraph ??%c ignored, use -trigraphs to enable",
1065 else if (note
->type
== 0)
1066 /* Already processed in lex_raw_string. */;
1072 /* Skip a C-style block comment. We find the end of the comment by
1073 seeing if an asterisk is before every '/' we encounter. Returns
1074 nonzero if comment terminated by EOF, zero otherwise.
1076 Buffer->cur points to the initial asterisk of the comment. */
1078 _cpp_skip_block_comment (cpp_reader
*pfile
)
1080 cpp_buffer
*buffer
= pfile
->buffer
;
1081 const uchar
*cur
= buffer
->cur
;
1090 /* People like decorating comments with '*', so check for '/'
1091 instead for efficiency. */
1099 /* Warn about potential nested comments, but not if the '/'
1100 comes immediately before the true comment delimiter.
1101 Don't bother to get it right across escaped newlines. */
1102 if (CPP_OPTION (pfile
, warn_comments
)
1103 && cur
[0] == '*' && cur
[1] != '/')
1106 cpp_warning_with_line (pfile
, CPP_W_COMMENTS
,
1107 pfile
->line_table
->highest_line
,
1108 CPP_BUF_COL (buffer
),
1109 "\"/*\" within comment");
1115 buffer
->cur
= cur
- 1;
1116 _cpp_process_line_notes (pfile
, true);
1117 if (buffer
->next_line
>= buffer
->rlimit
)
1119 _cpp_clean_line (pfile
);
1121 cols
= buffer
->next_line
- buffer
->line_base
;
1122 CPP_INCREMENT_LINE (pfile
, cols
);
1129 _cpp_process_line_notes (pfile
, true);
1133 /* Skip a C++ line comment, leaving buffer->cur pointing to the
1134 terminating newline. Handles escaped newlines. Returns nonzero
1135 if a multiline comment. */
1137 skip_line_comment (cpp_reader
*pfile
)
1139 cpp_buffer
*buffer
= pfile
->buffer
;
1140 source_location orig_line
= pfile
->line_table
->highest_line
;
1142 while (*buffer
->cur
!= '\n')
1145 _cpp_process_line_notes (pfile
, true);
1146 return orig_line
!= pfile
->line_table
->highest_line
;
1149 /* Skips whitespace, saving the next non-whitespace character. */
1151 skip_whitespace (cpp_reader
*pfile
, cppchar_t c
)
1153 cpp_buffer
*buffer
= pfile
->buffer
;
1154 bool saw_NUL
= false;
1158 /* Horizontal space always OK. */
1159 if (c
== ' ' || c
== '\t')
1161 /* Just \f \v or \0 left. */
1164 else if (pfile
->state
.in_directive
&& CPP_PEDANTIC (pfile
))
1165 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
,
1166 CPP_BUF_COL (buffer
),
1167 "%s in preprocessing directive",
1168 c
== '\f' ? "form feed" : "vertical tab");
1172 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
1173 while (is_nvspace (c
));
1176 cpp_error (pfile
, CPP_DL_WARNING
, "null character(s) ignored");
1181 /* See if the characters of a number token are valid in a name (no
1182 '.', '+' or '-'). */
1184 name_p (cpp_reader
*pfile
, const cpp_string
*string
)
1188 for (i
= 0; i
< string
->len
; i
++)
1189 if (!is_idchar (string
->text
[i
]))
1195 /* After parsing an identifier or other sequence, produce a warning about
1196 sequences not in NFC/NFKC. */
1198 warn_about_normalization (cpp_reader
*pfile
,
1199 const cpp_token
*token
,
1200 const struct normalize_state
*s
)
1202 if (CPP_OPTION (pfile
, warn_normalize
) < NORMALIZE_STATE_RESULT (s
)
1203 && !pfile
->state
.skipping
)
1205 /* Make sure that the token is printed using UCNs, even
1206 if we'd otherwise happily print UTF-8. */
1207 unsigned char *buf
= XNEWVEC (unsigned char, cpp_token_len (token
));
1210 sz
= cpp_spell_token (pfile
, token
, buf
, false) - buf
;
1211 if (NORMALIZE_STATE_RESULT (s
) == normalized_C
)
1212 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1213 "`%.*s' is not in NFKC", (int) sz
, buf
);
1215 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1216 "`%.*s' is not in NFC", (int) sz
, buf
);
1221 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1222 an identifier. FIRST is TRUE if this starts an identifier. */
1224 forms_identifier_p (cpp_reader
*pfile
, int first
,
1225 struct normalize_state
*state
)
1227 cpp_buffer
*buffer
= pfile
->buffer
;
1229 if (*buffer
->cur
== '$')
1231 if (!CPP_OPTION (pfile
, dollars_in_ident
))
1235 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1237 CPP_OPTION (pfile
, warn_dollars
) = 0;
1238 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1244 /* Is this a syntactically valid UCN? */
1245 if (CPP_OPTION (pfile
, extended_identifiers
)
1246 && *buffer
->cur
== '\\'
1247 && (buffer
->cur
[1] == 'u' || buffer
->cur
[1] == 'U'))
1251 if (_cpp_valid_ucn (pfile
, &buffer
->cur
, buffer
->rlimit
, 1 + !first
,
1252 state
, &s
, NULL
, NULL
))
1260 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1261 static cpp_hashnode
*
1262 lex_identifier_intern (cpp_reader
*pfile
, const uchar
*base
)
1264 cpp_hashnode
*result
;
1267 unsigned int hash
= HT_HASHSTEP (0, *base
);
1270 while (ISIDNUM (*cur
))
1272 hash
= HT_HASHSTEP (hash
, *cur
);
1276 hash
= HT_HASHFINISH (hash
, len
);
1277 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1278 base
, len
, hash
, HT_ALLOC
));
1280 /* Rarely, identifiers require diagnostics when lexed. */
1281 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1282 && !pfile
->state
.skipping
, 0))
1284 /* It is allowed to poison the same identifier twice. */
1285 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1286 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1287 NODE_NAME (result
));
1289 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1290 replacement list of a variadic macro. */
1291 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1292 && !pfile
->state
.va_args_ok
)
1294 if (CPP_OPTION (pfile
, cplusplus
))
1295 cpp_error (pfile
, CPP_DL_PEDWARN
,
1296 "__VA_ARGS__ can only appear in the expansion"
1297 " of a C++11 variadic macro");
1299 cpp_error (pfile
, CPP_DL_PEDWARN
,
1300 "__VA_ARGS__ can only appear in the expansion"
1301 " of a C99 variadic macro");
1304 /* For -Wc++-compat, warn about use of C++ named operators. */
1305 if (result
->flags
& NODE_WARN_OPERATOR
)
1306 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1307 "identifier \"%s\" is a special operator name in C++",
1308 NODE_NAME (result
));
1314 /* Get the cpp_hashnode of an identifier specified by NAME in
1315 the current cpp_reader object. If none is found, NULL is returned. */
1317 _cpp_lex_identifier (cpp_reader
*pfile
, const char *name
)
1319 cpp_hashnode
*result
;
1320 result
= lex_identifier_intern (pfile
, (uchar
*) name
);
1324 /* Lex an identifier starting at BUFFER->CUR - 1. */
1325 static cpp_hashnode
*
1326 lex_identifier (cpp_reader
*pfile
, const uchar
*base
, bool starts_ucn
,
1327 struct normalize_state
*nst
, cpp_hashnode
**spelling
)
1329 cpp_hashnode
*result
;
1332 unsigned int hash
= HT_HASHSTEP (0, *base
);
1334 cur
= pfile
->buffer
->cur
;
1337 while (ISIDNUM (*cur
))
1339 hash
= HT_HASHSTEP (hash
, *cur
);
1342 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *(cur
- 1));
1344 pfile
->buffer
->cur
= cur
;
1345 if (starts_ucn
|| forms_identifier_p (pfile
, false, nst
))
1347 /* Slower version for identifiers containing UCNs (or $). */
1349 while (ISIDNUM (*pfile
->buffer
->cur
))
1351 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *pfile
->buffer
->cur
);
1352 pfile
->buffer
->cur
++;
1354 } while (forms_identifier_p (pfile
, false, nst
));
1355 result
= _cpp_interpret_identifier (pfile
, base
,
1356 pfile
->buffer
->cur
- base
);
1357 *spelling
= cpp_lookup (pfile
, base
, pfile
->buffer
->cur
- base
);
1362 hash
= HT_HASHFINISH (hash
, len
);
1364 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1365 base
, len
, hash
, HT_ALLOC
));
1369 /* Rarely, identifiers require diagnostics when lexed. */
1370 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1371 && !pfile
->state
.skipping
, 0))
1373 /* It is allowed to poison the same identifier twice. */
1374 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1375 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1376 NODE_NAME (result
));
1378 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1379 replacement list of a variadic macro. */
1380 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1381 && !pfile
->state
.va_args_ok
)
1383 if (CPP_OPTION (pfile
, cplusplus
))
1384 cpp_error (pfile
, CPP_DL_PEDWARN
,
1385 "__VA_ARGS__ can only appear in the expansion"
1386 " of a C++11 variadic macro");
1388 cpp_error (pfile
, CPP_DL_PEDWARN
,
1389 "__VA_ARGS__ can only appear in the expansion"
1390 " of a C99 variadic macro");
1393 /* For -Wc++-compat, warn about use of C++ named operators. */
1394 if (result
->flags
& NODE_WARN_OPERATOR
)
1395 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1396 "identifier \"%s\" is a special operator name in C++",
1397 NODE_NAME (result
));
1403 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1405 lex_number (cpp_reader
*pfile
, cpp_string
*number
,
1406 struct normalize_state
*nst
)
1412 base
= pfile
->buffer
->cur
- 1;
1415 cur
= pfile
->buffer
->cur
;
1417 /* N.B. ISIDNUM does not include $. */
1418 while (ISIDNUM (*cur
) || *cur
== '.' || DIGIT_SEP (*cur
)
1419 || VALID_SIGN (*cur
, cur
[-1]))
1421 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *cur
);
1424 /* A number can't end with a digit separator. */
1425 while (cur
> pfile
->buffer
->cur
&& DIGIT_SEP (cur
[-1]))
1428 pfile
->buffer
->cur
= cur
;
1430 while (forms_identifier_p (pfile
, false, nst
));
1432 number
->len
= cur
- base
;
1433 dest
= _cpp_unaligned_alloc (pfile
, number
->len
+ 1);
1434 memcpy (dest
, base
, number
->len
);
1435 dest
[number
->len
] = '\0';
1436 number
->text
= dest
;
1439 /* Create a token of type TYPE with a literal spelling. */
1441 create_literal (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1442 unsigned int len
, enum cpp_ttype type
)
1444 uchar
*dest
= _cpp_unaligned_alloc (pfile
, len
+ 1);
1446 memcpy (dest
, base
, len
);
1449 token
->val
.str
.len
= len
;
1450 token
->val
.str
.text
= dest
;
1453 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1454 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1457 bufring_append (cpp_reader
*pfile
, const uchar
*base
, size_t len
,
1458 _cpp_buff
**first_buff_p
, _cpp_buff
**last_buff_p
)
1460 _cpp_buff
*first_buff
= *first_buff_p
;
1461 _cpp_buff
*last_buff
= *last_buff_p
;
1463 if (first_buff
== NULL
)
1464 first_buff
= last_buff
= _cpp_get_buff (pfile
, len
);
1465 else if (len
> BUFF_ROOM (last_buff
))
1467 size_t room
= BUFF_ROOM (last_buff
);
1468 memcpy (BUFF_FRONT (last_buff
), base
, room
);
1469 BUFF_FRONT (last_buff
) += room
;
1472 last_buff
= _cpp_append_extend_buff (pfile
, last_buff
, len
);
1475 memcpy (BUFF_FRONT (last_buff
), base
, len
);
1476 BUFF_FRONT (last_buff
) += len
;
1478 *first_buff_p
= first_buff
;
1479 *last_buff_p
= last_buff
;
1483 /* Returns true if a macro has been defined.
1484 This might not work if compile with -save-temps,
1485 or preprocess separately from compilation. */
1488 is_macro(cpp_reader
*pfile
, const uchar
*base
)
1490 const uchar
*cur
= base
;
1491 if (! ISIDST (*cur
))
1493 unsigned int hash
= HT_HASHSTEP (0, *cur
);
1495 while (ISIDNUM (*cur
))
1497 hash
= HT_HASHSTEP (hash
, *cur
);
1500 hash
= HT_HASHFINISH (hash
, cur
- base
);
1502 cpp_hashnode
*result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1503 base
, cur
- base
, hash
, HT_NO_INSERT
));
1505 return !result
? false : (result
->type
== NT_MACRO
);
1509 /* Lexes a raw string. The stored string contains the spelling, including
1510 double quotes, delimiter string, '(' and ')', any leading
1511 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1512 literal, or CPP_OTHER if it was not properly terminated.
1514 The spelling is NUL-terminated, but it is not guaranteed that this
1515 is the first NUL since embedded NULs are preserved. */
1518 lex_raw_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1521 uchar raw_prefix
[17];
1522 uchar temp_buffer
[18];
1523 const uchar
*orig_base
;
1524 unsigned int raw_prefix_len
= 0, raw_suffix_len
= 0;
1525 enum raw_str_phase
{ RAW_STR_PREFIX
, RAW_STR
, RAW_STR_SUFFIX
};
1526 raw_str_phase phase
= RAW_STR_PREFIX
;
1527 enum cpp_ttype type
;
1528 size_t total_len
= 0;
1529 /* Index into temp_buffer during phases other than RAW_STR,
1530 during RAW_STR phase 17 to tell BUF_APPEND that nothing should
1531 be appended to temp_buffer. */
1532 size_t temp_buffer_len
= 0;
1533 _cpp_buff
*first_buff
= NULL
, *last_buff
= NULL
;
1534 size_t raw_prefix_start
;
1535 _cpp_line_note
*note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1537 type
= (*base
== 'L' ? CPP_WSTRING
:
1538 *base
== 'U' ? CPP_STRING32
:
1539 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1542 #define BUF_APPEND(STR,LEN) \
1544 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1545 &first_buff, &last_buff); \
1546 total_len += (LEN); \
1547 if (__builtin_expect (temp_buffer_len < 17, 0) \
1548 && (const uchar *)(STR) != base \
1551 memcpy (temp_buffer + temp_buffer_len, \
1552 (const uchar *)(STR), (LEN)); \
1553 temp_buffer_len += (LEN); \
1559 raw_prefix_start
= cur
- base
;
1564 /* If we previously performed any trigraph or line splicing
1565 transformations, undo them in between the opening and closing
1567 while (note
->pos
< cur
)
1569 for (; note
->pos
== cur
; ++note
)
1575 /* Restore backslash followed by newline. */
1576 BUF_APPEND (base
, cur
- base
);
1578 BUF_APPEND ("\\", 1);
1580 if (note
->type
== ' ')
1582 /* GNU backslash whitespace newline extension. FIXME
1583 could be any sequence of non-vertical space. When we
1584 can properly restore any such sequence, we should mark
1585 this note as handled so _cpp_process_line_notes
1587 BUF_APPEND (" ", 1);
1590 BUF_APPEND ("\n", 1);
1594 /* Already handled. */
1598 if (_cpp_trigraph_map
[note
->type
])
1600 /* Don't warn about this trigraph in
1601 _cpp_process_line_notes, since trigraphs show up as
1602 trigraphs in raw strings. */
1603 uchar type
= note
->type
;
1606 if (!CPP_OPTION (pfile
, trigraphs
))
1607 /* If we didn't convert the trigraph in the first
1608 place, don't do anything now either. */
1611 BUF_APPEND (base
, cur
- base
);
1613 BUF_APPEND ("??", 2);
1615 /* ??/ followed by newline gets two line notes, one for
1616 the trigraph and one for the backslash/newline. */
1617 if (type
== '/' && note
[1].pos
== cur
)
1619 if (note
[1].type
!= '\\'
1620 && note
[1].type
!= ' ')
1622 BUF_APPEND ("/", 1);
1624 goto after_backslash
;
1628 /* Skip the replacement character. */
1630 BUF_APPEND (&type
, 1);
1641 if (__builtin_expect (temp_buffer_len
< 17, 0))
1642 temp_buffer
[temp_buffer_len
++] = c
;
1645 if (phase
== RAW_STR_PREFIX
)
1647 while (raw_prefix_len
< temp_buffer_len
)
1649 raw_prefix
[raw_prefix_len
] = temp_buffer
[raw_prefix_len
];
1650 switch (raw_prefix
[raw_prefix_len
])
1652 case ' ': case '(': case ')': case '\\': case '\t':
1653 case '\v': case '\f': case '\n': default:
1655 /* Basic source charset except the above chars. */
1656 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1657 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1658 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1659 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1661 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1662 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1663 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1664 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1666 case '0': case '1': case '2': case '3': case '4': case '5':
1667 case '6': case '7': case '8': case '9':
1668 case '_': case '{': case '}': case '#': case '[': case ']':
1669 case '<': case '>': case '%': case ':': case ';': case '.':
1670 case '?': case '*': case '+': case '-': case '/': case '^':
1671 case '&': case '|': case '~': case '!': case '=': case ',':
1672 case '"': case '\'':
1673 if (raw_prefix_len
< 16)
1681 if (raw_prefix
[raw_prefix_len
] != '(')
1683 int col
= CPP_BUF_COLUMN (pfile
->buffer
, cur
) + 1;
1684 if (raw_prefix_len
== 16)
1685 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1686 col
, "raw string delimiter longer "
1687 "than 16 characters");
1688 else if (raw_prefix
[raw_prefix_len
] == '\n')
1689 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1690 col
, "invalid new-line in raw "
1691 "string delimiter");
1693 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1694 col
, "invalid character '%c' in "
1695 "raw string delimiter",
1696 (int) raw_prefix
[raw_prefix_len
]);
1697 pfile
->buffer
->cur
= orig_base
+ raw_prefix_start
- 1;
1698 create_literal (pfile
, token
, orig_base
,
1699 raw_prefix_start
- 1, CPP_OTHER
);
1701 _cpp_release_buff (pfile
, first_buff
);
1704 raw_prefix
[raw_prefix_len
] = '"';
1706 /* Nothing should be appended to temp_buffer during
1708 temp_buffer_len
= 17;
1713 else if (phase
== RAW_STR_SUFFIX
)
1715 while (raw_suffix_len
<= raw_prefix_len
1716 && raw_suffix_len
< temp_buffer_len
1717 && temp_buffer
[raw_suffix_len
] == raw_prefix
[raw_suffix_len
])
1719 if (raw_suffix_len
> raw_prefix_len
)
1721 if (raw_suffix_len
== temp_buffer_len
)
1724 /* Nothing should be appended to temp_buffer during
1726 temp_buffer_len
= 17;
1730 phase
= RAW_STR_SUFFIX
;
1732 temp_buffer_len
= 0;
1736 if (pfile
->state
.in_directive
1737 || (pfile
->state
.parsing_args
1738 && pfile
->buffer
->next_line
>= pfile
->buffer
->rlimit
))
1742 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, 0,
1743 "unterminated raw string");
1747 BUF_APPEND (base
, cur
- base
);
1749 if (pfile
->buffer
->cur
< pfile
->buffer
->rlimit
)
1750 CPP_INCREMENT_LINE (pfile
, 0);
1751 pfile
->buffer
->need_line
= true;
1753 pfile
->buffer
->cur
= cur
-1;
1754 _cpp_process_line_notes (pfile
, false);
1755 if (!_cpp_get_fresh_line (pfile
))
1757 source_location src_loc
= token
->src_loc
;
1758 token
->type
= CPP_EOF
;
1759 /* Tell the compiler the line number of the EOF token. */
1760 token
->src_loc
= pfile
->line_table
->highest_line
;
1762 if (first_buff
!= NULL
)
1763 _cpp_release_buff (pfile
, first_buff
);
1764 cpp_error_with_line (pfile
, CPP_DL_ERROR
, src_loc
, 0,
1765 "unterminated raw string");
1769 cur
= base
= pfile
->buffer
->cur
;
1770 note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1774 if (CPP_OPTION (pfile
, user_literals
))
1776 /* If a string format macro, say from inttypes.h, is placed touching
1777 a string literal it could be parsed as a C++11 user-defined string
1778 literal thus breaking the program.
1779 Try to identify macros with is_macro. A warning is issued. */
1780 if (is_macro (pfile
, cur
))
1782 /* Raise a warning, but do not consume subsequent tokens. */
1783 if (CPP_OPTION (pfile
, warn_literal_suffix
) && !pfile
->state
.skipping
)
1784 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1786 "invalid suffix on literal; C++11 requires "
1787 "a space between literal and string macro");
1789 /* Grab user defined literal suffix. */
1790 else if (ISIDST (*cur
))
1792 type
= cpp_userdef_string_add_type (type
);
1795 while (ISIDNUM (*cur
))
1800 pfile
->buffer
->cur
= cur
;
1801 if (first_buff
== NULL
)
1802 create_literal (pfile
, token
, base
, cur
- base
, type
);
1805 uchar
*dest
= _cpp_unaligned_alloc (pfile
, total_len
+ (cur
- base
) + 1);
1808 token
->val
.str
.len
= total_len
+ (cur
- base
);
1809 token
->val
.str
.text
= dest
;
1810 last_buff
= first_buff
;
1811 while (last_buff
!= NULL
)
1813 memcpy (dest
, last_buff
->base
,
1814 BUFF_FRONT (last_buff
) - last_buff
->base
);
1815 dest
+= BUFF_FRONT (last_buff
) - last_buff
->base
;
1816 last_buff
= last_buff
->next
;
1818 _cpp_release_buff (pfile
, first_buff
);
1819 memcpy (dest
, base
, cur
- base
);
1820 dest
[cur
- base
] = '\0';
1824 /* Lexes a string, character constant, or angle-bracketed header file
1825 name. The stored string contains the spelling, including opening
1826 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1827 'R' modifier. It returns the type of the literal, or CPP_OTHER
1828 if it was not properly terminated, or CPP_LESS for an unterminated
1829 header name which must be relexed as normal tokens.
1831 The spelling is NUL-terminated, but it is not guaranteed that this
1832 is the first NUL since embedded NULs are preserved. */
1834 lex_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
)
1836 bool saw_NUL
= false;
1838 cppchar_t terminator
;
1839 enum cpp_ttype type
;
1842 terminator
= *cur
++;
1843 if (terminator
== 'L' || terminator
== 'U')
1844 terminator
= *cur
++;
1845 else if (terminator
== 'u')
1847 terminator
= *cur
++;
1848 if (terminator
== '8')
1849 terminator
= *cur
++;
1851 if (terminator
== 'R')
1853 lex_raw_string (pfile
, token
, base
, cur
);
1856 if (terminator
== '"')
1857 type
= (*base
== 'L' ? CPP_WSTRING
:
1858 *base
== 'U' ? CPP_STRING32
:
1859 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1861 else if (terminator
== '\'')
1862 type
= (*base
== 'L' ? CPP_WCHAR
:
1863 *base
== 'U' ? CPP_CHAR32
:
1864 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8CHAR
: CPP_CHAR16
)
1867 terminator
= '>', type
= CPP_HEADER_NAME
;
1871 cppchar_t c
= *cur
++;
1873 /* In #include-style directives, terminators are not escapable. */
1874 if (c
== '\\' && !pfile
->state
.angled_headers
&& *cur
!= '\n')
1876 else if (c
== terminator
)
1881 /* Unmatched quotes always yield undefined behavior, but
1882 greedy lexing means that what appears to be an unterminated
1883 header name may actually be a legitimate sequence of tokens. */
1884 if (terminator
== '>')
1886 token
->type
= CPP_LESS
;
1896 if (saw_NUL
&& !pfile
->state
.skipping
)
1897 cpp_error (pfile
, CPP_DL_WARNING
,
1898 "null character(s) preserved in literal");
1900 if (type
== CPP_OTHER
&& CPP_OPTION (pfile
, lang
) != CLK_ASM
)
1901 cpp_error (pfile
, CPP_DL_PEDWARN
, "missing terminating %c character",
1904 if (CPP_OPTION (pfile
, user_literals
))
1906 /* If a string format macro, say from inttypes.h, is placed touching
1907 a string literal it could be parsed as a C++11 user-defined string
1908 literal thus breaking the program.
1909 Try to identify macros with is_macro. A warning is issued. */
1910 if (is_macro (pfile
, cur
))
1912 /* Raise a warning, but do not consume subsequent tokens. */
1913 if (CPP_OPTION (pfile
, warn_literal_suffix
) && !pfile
->state
.skipping
)
1914 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1916 "invalid suffix on literal; C++11 requires "
1917 "a space between literal and string macro");
1919 /* Grab user defined literal suffix. */
1920 else if (ISIDST (*cur
))
1922 type
= cpp_userdef_char_add_type (type
);
1923 type
= cpp_userdef_string_add_type (type
);
1926 while (ISIDNUM (*cur
))
1930 else if (CPP_OPTION (pfile
, cpp_warn_cxx11_compat
)
1931 && is_macro (pfile
, cur
)
1932 && !pfile
->state
.skipping
)
1933 cpp_warning_with_line (pfile
, CPP_W_CXX11_COMPAT
,
1934 token
->src_loc
, 0, "C++11 requires a space "
1935 "between string literal and macro");
1937 pfile
->buffer
->cur
= cur
;
1938 create_literal (pfile
, token
, base
, cur
- base
, type
);
1941 /* Return the comment table. The client may not make any assumption
1942 about the ordering of the table. */
1944 cpp_get_comments (cpp_reader
*pfile
)
1946 return &pfile
->comments
;
1949 /* Append a comment to the end of the comment table. */
1951 store_comment (cpp_reader
*pfile
, cpp_token
*token
)
1955 if (pfile
->comments
.allocated
== 0)
1957 pfile
->comments
.allocated
= 256;
1958 pfile
->comments
.entries
= (cpp_comment
*) xmalloc
1959 (pfile
->comments
.allocated
* sizeof (cpp_comment
));
1962 if (pfile
->comments
.count
== pfile
->comments
.allocated
)
1964 pfile
->comments
.allocated
*= 2;
1965 pfile
->comments
.entries
= (cpp_comment
*) xrealloc
1966 (pfile
->comments
.entries
,
1967 pfile
->comments
.allocated
* sizeof (cpp_comment
));
1970 len
= token
->val
.str
.len
;
1972 /* Copy comment. Note, token may not be NULL terminated. */
1973 pfile
->comments
.entries
[pfile
->comments
.count
].comment
=
1974 (char *) xmalloc (sizeof (char) * (len
+ 1));
1975 memcpy (pfile
->comments
.entries
[pfile
->comments
.count
].comment
,
1976 token
->val
.str
.text
, len
);
1977 pfile
->comments
.entries
[pfile
->comments
.count
].comment
[len
] = '\0';
1979 /* Set source location. */
1980 pfile
->comments
.entries
[pfile
->comments
.count
].sloc
= token
->src_loc
;
1982 /* Increment the count of entries in the comment table. */
1983 pfile
->comments
.count
++;
1986 /* The stored comment includes the comment start and any terminator. */
1988 save_comment (cpp_reader
*pfile
, cpp_token
*token
, const unsigned char *from
,
1991 unsigned char *buffer
;
1992 unsigned int len
, clen
, i
;
1994 len
= pfile
->buffer
->cur
- from
+ 1; /* + 1 for the initial '/'. */
1996 /* C++ comments probably (not definitely) have moved past a new
1997 line, which we don't want to save in the comment. */
1998 if (is_vspace (pfile
->buffer
->cur
[-1]))
2001 /* If we are currently in a directive or in argument parsing, then
2002 we need to store all C++ comments as C comments internally, and
2003 so we need to allocate a little extra space in that case.
2005 Note that the only time we encounter a directive here is
2006 when we are saving comments in a "#define". */
2007 clen
= ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
)
2008 && type
== '/') ? len
+ 2 : len
;
2010 buffer
= _cpp_unaligned_alloc (pfile
, clen
);
2012 token
->type
= CPP_COMMENT
;
2013 token
->val
.str
.len
= clen
;
2014 token
->val
.str
.text
= buffer
;
2017 memcpy (buffer
+ 1, from
, len
- 1);
2019 /* Finish conversion to a C comment, if necessary. */
2020 if ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
) && type
== '/')
2023 buffer
[clen
- 2] = '*';
2024 buffer
[clen
- 1] = '/';
2025 /* As there can be in a C++ comments illegal sequences for C comments
2026 we need to filter them out. */
2027 for (i
= 2; i
< (clen
- 2); i
++)
2028 if (buffer
[i
] == '/' && (buffer
[i
- 1] == '*' || buffer
[i
+ 1] == '*'))
2032 /* Finally store this comment for use by clients of libcpp. */
2033 store_comment (pfile
, token
);
2036 /* Returns true if comment at COMMENT_START is a recognized FALLTHROUGH
2040 fallthrough_comment_p (cpp_reader
*pfile
, const unsigned char *comment_start
)
2042 const unsigned char *from
= comment_start
+ 1;
2043 /* Whole comment contents:
2047 if (*from
== '-' || *from
== '@')
2049 size_t len
= sizeof "fallthrough" - 1;
2050 if ((size_t) (pfile
->buffer
->cur
- from
- 1) < len
)
2052 if (memcmp (from
+ 1, "fallthrough", len
))
2056 if (from
[len
+ 1] != '@')
2062 /* Whole comment contents (regex):
2065 else if (*from
== 'l')
2067 size_t len
= sizeof "int -fallthrough" - 1;
2068 if ((size_t) (pfile
->buffer
->cur
- from
- 1) < len
)
2070 if (memcmp (from
+ 1, "int -fallthrough", len
))
2076 /* Whole comment contents (regex):
2077 [ \t.!]*(ELSE,? |INTENTIONAL(LY)? )?FALL(S | |-)?THR(OUGH|U)[ \t.!]*(-[^\n\r]*)?
2078 [ \t.!]*(Else,? |Intentional(ly)? )?Fall((s | |-)[Tt]|t)hr(ough|u)[ \t.!]*(-[^\n\r]*)?
2079 [ \t.!]*([Ee]lse,? |[Ii]ntentional(ly)? )?fall(s | |-)?thr(ough|u)[ \t.!]*(-[^\n\r]*)?
2083 while (*from
== ' ' || *from
== '\t' || *from
== '.' || *from
== '!')
2085 unsigned char f
= *from
;
2086 bool all_upper
= false;
2087 if (f
== 'E' || f
== 'e')
2089 if ((size_t) (pfile
->buffer
->cur
- from
)
2090 < sizeof "else fallthru" - 1)
2092 if (f
== 'E' && memcmp (from
+ 1, "LSE", sizeof "LSE" - 1) == 0)
2094 else if (memcmp (from
+ 1, "lse", sizeof "lse" - 1))
2096 from
+= sizeof "else" - 1;
2102 if (all_upper
&& *from
== 'f')
2104 if (f
== 'e' && *from
== 'F')
2108 else if (f
== 'I' || f
== 'i')
2110 if ((size_t) (pfile
->buffer
->cur
- from
)
2111 < sizeof "intentional fallthru" - 1)
2113 if (f
== 'I' && memcmp (from
+ 1, "NTENTIONAL",
2114 sizeof "NTENTIONAL" - 1) == 0)
2116 else if (memcmp (from
+ 1, "ntentional",
2117 sizeof "ntentional" - 1))
2119 from
+= sizeof "intentional" - 1;
2123 if (all_upper
&& *from
== 'f')
2128 if (memcmp (from
, "LY F", sizeof "LY F" - 1))
2130 from
+= sizeof "LY " - 1;
2134 if (memcmp (from
, "ly ", sizeof "ly " - 1))
2136 from
+= sizeof "ly " - 1;
2138 if (f
== 'i' && *from
== 'F')
2142 if (f
!= 'F' && f
!= 'f')
2144 if ((size_t) (pfile
->buffer
->cur
- from
) < sizeof "fallthru" - 1)
2146 if (f
== 'F' && memcmp (from
+ 1, "ALL", sizeof "ALL" - 1) == 0)
2150 else if (memcmp (from
+ 1, "all", sizeof "all" - 1))
2152 from
+= sizeof "fall" - 1;
2153 if (*from
== (all_upper
? 'S' : 's') && from
[1] == ' ')
2155 else if (*from
== ' ' || *from
== '-')
2157 else if (*from
!= (all_upper
? 'T' : 't'))
2159 if ((f
== 'f' || *from
!= 'T') && (all_upper
|| *from
!= 't'))
2161 if ((size_t) (pfile
->buffer
->cur
- from
) < sizeof "thru" - 1)
2163 if (memcmp (from
+ 1, all_upper
? "HRU" : "hru", sizeof "hru" - 1))
2165 if ((size_t) (pfile
->buffer
->cur
- from
) < sizeof "through" - 1)
2167 if (memcmp (from
+ 1, all_upper
? "HROUGH" : "hrough",
2168 sizeof "hrough" - 1))
2170 from
+= sizeof "through" - 1;
2173 from
+= sizeof "thru" - 1;
2174 while (*from
== ' ' || *from
== '\t' || *from
== '.' || *from
== '!')
2179 if (*comment_start
== '*')
2183 while (*from
&& *from
!= '*'
2184 && *from
!= '\n' && *from
!= '\r')
2186 if (*from
!= '*' || from
[1] == '/')
2193 while (*from
&& *from
!= '\n' && *from
!= '\r')
2197 /* C block comment. */
2198 if (*comment_start
== '*')
2200 if (*from
!= '*' || from
[1] != '/')
2203 /* C++ line comment. */
2204 else if (*from
!= '\n')
2210 /* Allocate COUNT tokens for RUN. */
2212 _cpp_init_tokenrun (tokenrun
*run
, unsigned int count
)
2214 run
->base
= XNEWVEC (cpp_token
, count
);
2215 run
->limit
= run
->base
+ count
;
2219 /* Returns the next tokenrun, or creates one if there is none. */
2221 next_tokenrun (tokenrun
*run
)
2223 if (run
->next
== NULL
)
2225 run
->next
= XNEW (tokenrun
);
2226 run
->next
->prev
= run
;
2227 _cpp_init_tokenrun (run
->next
, 250);
2233 /* Return the number of not yet processed token in a given
2236 _cpp_remaining_tokens_num_in_context (cpp_context
*context
)
2238 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
2239 return (LAST (context
).token
- FIRST (context
).token
);
2240 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
2241 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
2242 return (LAST (context
).ptoken
- FIRST (context
).ptoken
);
2247 /* Returns the token present at index INDEX in a given context. If
2248 INDEX is zero, the next token to be processed is returned. */
2249 static const cpp_token
*
2250 _cpp_token_from_context_at (cpp_context
*context
, int index
)
2252 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
2253 return &(FIRST (context
).token
[index
]);
2254 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
2255 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
2256 return FIRST (context
).ptoken
[index
];
2261 /* Look ahead in the input stream. */
2263 cpp_peek_token (cpp_reader
*pfile
, int index
)
2265 cpp_context
*context
= pfile
->context
;
2266 const cpp_token
*peektok
;
2269 /* First, scan through any pending cpp_context objects. */
2270 while (context
->prev
)
2272 ptrdiff_t sz
= _cpp_remaining_tokens_num_in_context (context
);
2274 if (index
< (int) sz
)
2275 return _cpp_token_from_context_at (context
, index
);
2277 context
= context
->prev
;
2280 /* We will have to read some new tokens after all (and do so
2281 without invalidating preceding tokens). */
2283 pfile
->keep_tokens
++;
2285 /* For peeked tokens temporarily disable line_change reporting,
2286 until the tokens are parsed for real. */
2287 void (*line_change
) (cpp_reader
*, const cpp_token
*, int)
2288 = pfile
->cb
.line_change
;
2289 pfile
->cb
.line_change
= NULL
;
2293 peektok
= _cpp_lex_token (pfile
);
2294 if (peektok
->type
== CPP_EOF
)
2302 _cpp_backup_tokens_direct (pfile
, count
- index
);
2303 pfile
->keep_tokens
--;
2304 pfile
->cb
.line_change
= line_change
;
2309 /* Allocate a single token that is invalidated at the same time as the
2310 rest of the tokens on the line. Has its line and col set to the
2311 same as the last lexed token, so that diagnostics appear in the
2314 _cpp_temp_token (cpp_reader
*pfile
)
2316 cpp_token
*old
, *result
;
2317 ptrdiff_t sz
= pfile
->cur_run
->limit
- pfile
->cur_token
;
2318 ptrdiff_t la
= (ptrdiff_t) pfile
->lookaheads
;
2320 old
= pfile
->cur_token
- 1;
2321 /* Any pre-existing lookaheads must not be clobbered. */
2326 tokenrun
*next
= next_tokenrun (pfile
->cur_run
);
2329 memmove (next
->base
+ 1, next
->base
,
2330 (la
- sz
) * sizeof (cpp_token
));
2332 next
->base
[0] = pfile
->cur_run
->limit
[-1];
2336 memmove (pfile
->cur_token
+ 1, pfile
->cur_token
,
2337 MIN (la
, sz
- 1) * sizeof (cpp_token
));
2340 if (!sz
&& pfile
->cur_token
== pfile
->cur_run
->limit
)
2342 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2343 pfile
->cur_token
= pfile
->cur_run
->base
;
2346 result
= pfile
->cur_token
++;
2347 result
->src_loc
= old
->src_loc
;
2351 /* Lex a token into RESULT (external interface). Takes care of issues
2352 like directive handling, token lookahead, multiple include
2353 optimization and skipping. */
2355 _cpp_lex_token (cpp_reader
*pfile
)
2361 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
2363 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2364 pfile
->cur_token
= pfile
->cur_run
->base
;
2366 /* We assume that the current token is somewhere in the current
2368 if (pfile
->cur_token
< pfile
->cur_run
->base
2369 || pfile
->cur_token
>= pfile
->cur_run
->limit
)
2372 if (pfile
->lookaheads
)
2374 pfile
->lookaheads
--;
2375 result
= pfile
->cur_token
++;
2378 result
= _cpp_lex_direct (pfile
);
2380 if (result
->flags
& BOL
)
2382 /* Is this a directive. If _cpp_handle_directive returns
2383 false, it is an assembler #. */
2384 if (result
->type
== CPP_HASH
2385 /* 6.10.3 p 11: Directives in a list of macro arguments
2386 gives undefined behavior. This implementation
2387 handles the directive as normal. */
2388 && pfile
->state
.parsing_args
!= 1)
2390 if (_cpp_handle_directive (pfile
, result
->flags
& PREV_WHITE
))
2392 if (pfile
->directive_result
.type
== CPP_PADDING
)
2394 result
= &pfile
->directive_result
;
2397 else if (pfile
->state
.in_deferred_pragma
)
2398 result
= &pfile
->directive_result
;
2400 if (pfile
->cb
.line_change
&& !pfile
->state
.skipping
)
2401 pfile
->cb
.line_change (pfile
, result
, pfile
->state
.parsing_args
);
2404 /* We don't skip tokens in directives. */
2405 if (pfile
->state
.in_directive
|| pfile
->state
.in_deferred_pragma
)
2408 /* Outside a directive, invalidate controlling macros. At file
2409 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
2410 get here and MI optimization works. */
2411 pfile
->mi_valid
= false;
2413 if (!pfile
->state
.skipping
|| result
->type
== CPP_EOF
)
2420 /* Returns true if a fresh line has been loaded. */
2422 _cpp_get_fresh_line (cpp_reader
*pfile
)
2426 /* We can't get a new line until we leave the current directive. */
2427 if (pfile
->state
.in_directive
)
2432 cpp_buffer
*buffer
= pfile
->buffer
;
2434 if (!buffer
->need_line
)
2437 if (buffer
->next_line
< buffer
->rlimit
)
2439 _cpp_clean_line (pfile
);
2443 /* First, get out of parsing arguments state. */
2444 if (pfile
->state
.parsing_args
)
2447 /* End of buffer. Non-empty files should end in a newline. */
2448 if (buffer
->buf
!= buffer
->rlimit
2449 && buffer
->next_line
> buffer
->rlimit
2450 && !buffer
->from_stage3
)
2452 /* Clip to buffer size. */
2453 buffer
->next_line
= buffer
->rlimit
;
2456 return_at_eof
= buffer
->return_at_eof
;
2457 _cpp_pop_buffer (pfile
);
2458 if (pfile
->buffer
== NULL
|| return_at_eof
)
2463 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
2466 result->type = ELSE_TYPE; \
2467 if (*buffer->cur == CHAR) \
2468 buffer->cur++, result->type = THEN_TYPE; \
2472 /* Lex a token into pfile->cur_token, which is also incremented, to
2473 get diagnostics pointing to the correct location.
2475 Does not handle issues such as token lookahead, multiple-include
2476 optimization, directives, skipping etc. This function is only
2477 suitable for use by _cpp_lex_token, and in special cases like
2478 lex_expansion_token which doesn't care for any of these issues.
2480 When meeting a newline, returns CPP_EOF if parsing a directive,
2481 otherwise returns to the start of the token buffer if permissible.
2482 Returns the location of the lexed token. */
2484 _cpp_lex_direct (cpp_reader
*pfile
)
2488 const unsigned char *comment_start
;
2489 bool fallthrough_comment
= false;
2490 cpp_token
*result
= pfile
->cur_token
++;
2494 buffer
= pfile
->buffer
;
2495 if (buffer
->need_line
)
2497 if (pfile
->state
.in_deferred_pragma
)
2499 result
->type
= CPP_PRAGMA_EOL
;
2500 pfile
->state
.in_deferred_pragma
= false;
2501 if (!pfile
->state
.pragma_allow_expansion
)
2502 pfile
->state
.prevent_expansion
--;
2505 if (!_cpp_get_fresh_line (pfile
))
2507 result
->type
= CPP_EOF
;
2508 if (!pfile
->state
.in_directive
)
2510 /* Tell the compiler the line number of the EOF token. */
2511 result
->src_loc
= pfile
->line_table
->highest_line
;
2512 result
->flags
= BOL
;
2516 if (buffer
!= pfile
->buffer
)
2517 fallthrough_comment
= false;
2518 if (!pfile
->keep_tokens
)
2520 pfile
->cur_run
= &pfile
->base_run
;
2521 result
= pfile
->base_run
.base
;
2522 pfile
->cur_token
= result
+ 1;
2524 result
->flags
= BOL
;
2525 if (pfile
->state
.parsing_args
== 2)
2526 result
->flags
|= PREV_WHITE
;
2528 buffer
= pfile
->buffer
;
2530 result
->src_loc
= pfile
->line_table
->highest_line
;
2533 if (buffer
->cur
>= buffer
->notes
[buffer
->cur_note
].pos
2534 && !pfile
->overlaid_buffer
)
2536 _cpp_process_line_notes (pfile
, false);
2537 result
->src_loc
= pfile
->line_table
->highest_line
;
2541 if (pfile
->forced_token_location_p
)
2542 result
->src_loc
= *pfile
->forced_token_location_p
;
2544 result
->src_loc
= linemap_position_for_column (pfile
->line_table
,
2545 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2549 case ' ': case '\t': case '\f': case '\v': case '\0':
2550 result
->flags
|= PREV_WHITE
;
2551 skip_whitespace (pfile
, c
);
2555 if (buffer
->cur
< buffer
->rlimit
)
2556 CPP_INCREMENT_LINE (pfile
, 0);
2557 buffer
->need_line
= true;
2560 case '0': case '1': case '2': case '3': case '4':
2561 case '5': case '6': case '7': case '8': case '9':
2563 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2564 result
->type
= CPP_NUMBER
;
2565 lex_number (pfile
, &result
->val
.str
, &nst
);
2566 warn_about_normalization (pfile
, result
, &nst
);
2574 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2575 wide strings or raw strings. */
2576 if (c
== 'L' || CPP_OPTION (pfile
, rliterals
)
2577 || (c
!= 'R' && CPP_OPTION (pfile
, uliterals
)))
2579 if ((*buffer
->cur
== '\'' && c
!= 'R')
2580 || *buffer
->cur
== '"'
2581 || (*buffer
->cur
== 'R'
2583 && buffer
->cur
[1] == '"'
2584 && CPP_OPTION (pfile
, rliterals
))
2585 || (*buffer
->cur
== '8'
2587 && ((buffer
->cur
[1] == '"' || (buffer
->cur
[1] == '\''
2588 && CPP_OPTION (pfile
, utf8_char_literals
)))
2589 || (buffer
->cur
[1] == 'R' && buffer
->cur
[2] == '"'
2590 && CPP_OPTION (pfile
, rliterals
)))))
2592 lex_string (pfile
, result
, buffer
->cur
- 1);
2599 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2600 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2601 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2602 case 's': case 't': case 'v': case 'w': case 'x':
2604 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2605 case 'G': case 'H': case 'I': case 'J': case 'K':
2606 case 'M': case 'N': case 'O': case 'P': case 'Q':
2607 case 'S': case 'T': case 'V': case 'W': case 'X':
2609 result
->type
= CPP_NAME
;
2611 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2612 result
->val
.node
.node
= lex_identifier (pfile
, buffer
->cur
- 1, false,
2614 &result
->val
.node
.spelling
);
2615 warn_about_normalization (pfile
, result
, &nst
);
2618 /* Convert named operators to their proper types. */
2619 if (result
->val
.node
.node
->flags
& NODE_OPERATOR
)
2621 result
->flags
|= NAMED_OP
;
2622 result
->type
= (enum cpp_ttype
) result
->val
.node
.node
->directive_index
;
2625 /* Signal FALLTHROUGH comment followed by another token. */
2626 if (fallthrough_comment
)
2627 result
->flags
|= PREV_FALLTHROUGH
;
2632 lex_string (pfile
, result
, buffer
->cur
- 1);
2636 /* A potential block or line comment. */
2637 comment_start
= buffer
->cur
;
2642 if (_cpp_skip_block_comment (pfile
))
2643 cpp_error (pfile
, CPP_DL_ERROR
, "unterminated comment");
2645 else if (c
== '/' && ! CPP_OPTION (pfile
, traditional
))
2647 /* Don't warn for system headers. */
2648 if (cpp_in_system_header (pfile
))
2650 /* Warn about comments if pedantically GNUC89, and not
2651 in system headers. */
2652 else if (CPP_OPTION (pfile
, lang
) == CLK_GNUC89
2653 && CPP_PEDANTIC (pfile
)
2654 && ! buffer
->warned_cplusplus_comments
)
2656 cpp_error (pfile
, CPP_DL_PEDWARN
,
2657 "C++ style comments are not allowed in ISO C90");
2658 cpp_error (pfile
, CPP_DL_PEDWARN
,
2659 "(this will be reported only once per input file)");
2660 buffer
->warned_cplusplus_comments
= 1;
2662 /* Or if specifically desired via -Wc90-c99-compat. */
2663 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
) > 0
2664 && ! CPP_OPTION (pfile
, cplusplus
)
2665 && ! buffer
->warned_cplusplus_comments
)
2667 cpp_error (pfile
, CPP_DL_WARNING
,
2668 "C++ style comments are incompatible with C90");
2669 cpp_error (pfile
, CPP_DL_WARNING
,
2670 "(this will be reported only once per input file)");
2671 buffer
->warned_cplusplus_comments
= 1;
2673 /* In C89/C94, C++ style comments are forbidden. */
2674 else if ((CPP_OPTION (pfile
, lang
) == CLK_STDC89
2675 || CPP_OPTION (pfile
, lang
) == CLK_STDC94
))
2677 /* But don't be confused about valid code such as
2678 - // immediately followed by *,
2679 - // in a preprocessing directive,
2680 - // in an #if 0 block. */
2681 if (buffer
->cur
[1] == '*'
2682 || pfile
->state
.in_directive
2683 || pfile
->state
.skipping
)
2685 result
->type
= CPP_DIV
;
2688 else if (! buffer
->warned_cplusplus_comments
)
2690 cpp_error (pfile
, CPP_DL_ERROR
,
2691 "C++ style comments are not allowed in ISO C90");
2692 cpp_error (pfile
, CPP_DL_ERROR
,
2693 "(this will be reported only once per input "
2695 buffer
->warned_cplusplus_comments
= 1;
2698 if (skip_line_comment (pfile
) && CPP_OPTION (pfile
, warn_comments
))
2699 cpp_warning (pfile
, CPP_W_COMMENTS
, "multi-line comment");
2704 result
->type
= CPP_DIV_EQ
;
2709 result
->type
= CPP_DIV
;
2713 if (fallthrough_comment_p (pfile
, comment_start
))
2714 fallthrough_comment
= true;
2716 if (!pfile
->state
.save_comments
)
2718 result
->flags
|= PREV_WHITE
;
2719 goto update_tokens_line
;
2722 if (fallthrough_comment
)
2723 result
->flags
|= PREV_FALLTHROUGH
;
2725 /* Save the comment as a token in its own right. */
2726 save_comment (pfile
, result
, comment_start
, c
);
2730 if (pfile
->state
.angled_headers
)
2732 lex_string (pfile
, result
, buffer
->cur
- 1);
2733 if (result
->type
!= CPP_LESS
)
2737 result
->type
= CPP_LESS
;
2738 if (*buffer
->cur
== '=')
2739 buffer
->cur
++, result
->type
= CPP_LESS_EQ
;
2740 else if (*buffer
->cur
== '<')
2743 IF_NEXT_IS ('=', CPP_LSHIFT_EQ
, CPP_LSHIFT
);
2745 else if (CPP_OPTION (pfile
, digraphs
))
2747 if (*buffer
->cur
== ':')
2749 /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next
2750 three characters are <:: and the subsequent character
2751 is neither : nor >, the < is treated as a preprocessor
2752 token by itself". */
2753 if (CPP_OPTION (pfile
, cplusplus
)
2754 && CPP_OPTION (pfile
, lang
) != CLK_CXX98
2755 && CPP_OPTION (pfile
, lang
) != CLK_GNUCXX
2756 && buffer
->cur
[1] == ':'
2757 && buffer
->cur
[2] != ':' && buffer
->cur
[2] != '>')
2761 result
->flags
|= DIGRAPH
;
2762 result
->type
= CPP_OPEN_SQUARE
;
2764 else if (*buffer
->cur
== '%')
2767 result
->flags
|= DIGRAPH
;
2768 result
->type
= CPP_OPEN_BRACE
;
2774 result
->type
= CPP_GREATER
;
2775 if (*buffer
->cur
== '=')
2776 buffer
->cur
++, result
->type
= CPP_GREATER_EQ
;
2777 else if (*buffer
->cur
== '>')
2780 IF_NEXT_IS ('=', CPP_RSHIFT_EQ
, CPP_RSHIFT
);
2785 result
->type
= CPP_MOD
;
2786 if (*buffer
->cur
== '=')
2787 buffer
->cur
++, result
->type
= CPP_MOD_EQ
;
2788 else if (CPP_OPTION (pfile
, digraphs
))
2790 if (*buffer
->cur
== ':')
2793 result
->flags
|= DIGRAPH
;
2794 result
->type
= CPP_HASH
;
2795 if (*buffer
->cur
== '%' && buffer
->cur
[1] == ':')
2796 buffer
->cur
+= 2, result
->type
= CPP_PASTE
, result
->val
.token_no
= 0;
2798 else if (*buffer
->cur
== '>')
2801 result
->flags
|= DIGRAPH
;
2802 result
->type
= CPP_CLOSE_BRACE
;
2808 result
->type
= CPP_DOT
;
2809 if (ISDIGIT (*buffer
->cur
))
2811 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2812 result
->type
= CPP_NUMBER
;
2813 lex_number (pfile
, &result
->val
.str
, &nst
);
2814 warn_about_normalization (pfile
, result
, &nst
);
2816 else if (*buffer
->cur
== '.' && buffer
->cur
[1] == '.')
2817 buffer
->cur
+= 2, result
->type
= CPP_ELLIPSIS
;
2818 else if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2819 buffer
->cur
++, result
->type
= CPP_DOT_STAR
;
2823 result
->type
= CPP_PLUS
;
2824 if (*buffer
->cur
== '+')
2825 buffer
->cur
++, result
->type
= CPP_PLUS_PLUS
;
2826 else if (*buffer
->cur
== '=')
2827 buffer
->cur
++, result
->type
= CPP_PLUS_EQ
;
2831 result
->type
= CPP_MINUS
;
2832 if (*buffer
->cur
== '>')
2835 result
->type
= CPP_DEREF
;
2836 if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2837 buffer
->cur
++, result
->type
= CPP_DEREF_STAR
;
2839 else if (*buffer
->cur
== '-')
2840 buffer
->cur
++, result
->type
= CPP_MINUS_MINUS
;
2841 else if (*buffer
->cur
== '=')
2842 buffer
->cur
++, result
->type
= CPP_MINUS_EQ
;
2846 result
->type
= CPP_AND
;
2847 if (*buffer
->cur
== '&')
2848 buffer
->cur
++, result
->type
= CPP_AND_AND
;
2849 else if (*buffer
->cur
== '=')
2850 buffer
->cur
++, result
->type
= CPP_AND_EQ
;
2854 result
->type
= CPP_OR
;
2855 if (*buffer
->cur
== '|')
2856 buffer
->cur
++, result
->type
= CPP_OR_OR
;
2857 else if (*buffer
->cur
== '=')
2858 buffer
->cur
++, result
->type
= CPP_OR_EQ
;
2862 result
->type
= CPP_COLON
;
2863 if (*buffer
->cur
== ':' && CPP_OPTION (pfile
, cplusplus
))
2864 buffer
->cur
++, result
->type
= CPP_SCOPE
;
2865 else if (*buffer
->cur
== '>' && CPP_OPTION (pfile
, digraphs
))
2868 result
->flags
|= DIGRAPH
;
2869 result
->type
= CPP_CLOSE_SQUARE
;
2873 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ
, CPP_MULT
); break;
2874 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ
, CPP_EQ
); break;
2875 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ
, CPP_NOT
); break;
2876 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ
, CPP_XOR
); break;
2877 case '#': IF_NEXT_IS ('#', CPP_PASTE
, CPP_HASH
); result
->val
.token_no
= 0; break;
2879 case '?': result
->type
= CPP_QUERY
; break;
2880 case '~': result
->type
= CPP_COMPL
; break;
2881 case ',': result
->type
= CPP_COMMA
; break;
2882 case '(': result
->type
= CPP_OPEN_PAREN
; break;
2883 case ')': result
->type
= CPP_CLOSE_PAREN
; break;
2884 case '[': result
->type
= CPP_OPEN_SQUARE
; break;
2885 case ']': result
->type
= CPP_CLOSE_SQUARE
; break;
2886 case '{': result
->type
= CPP_OPEN_BRACE
; break;
2887 case '}': result
->type
= CPP_CLOSE_BRACE
; break;
2888 case ';': result
->type
= CPP_SEMICOLON
; break;
2890 /* @ is a punctuator in Objective-C. */
2891 case '@': result
->type
= CPP_ATSIGN
; break;
2896 const uchar
*base
= --buffer
->cur
;
2897 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2899 if (forms_identifier_p (pfile
, true, &nst
))
2901 result
->type
= CPP_NAME
;
2902 result
->val
.node
.node
= lex_identifier (pfile
, base
, true, &nst
,
2903 &result
->val
.node
.spelling
);
2904 warn_about_normalization (pfile
, result
, &nst
);
2912 create_literal (pfile
, result
, buffer
->cur
- 1, 1, CPP_OTHER
);
2916 source_range tok_range
;
2917 tok_range
.m_start
= result
->src_loc
;
2918 if (result
->src_loc
>= RESERVED_LOCATION_COUNT
)
2920 = linemap_position_for_column (pfile
->line_table
,
2921 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2923 tok_range
.m_finish
= tok_range
.m_start
;
2925 result
->src_loc
= COMBINE_LOCATION_DATA (pfile
->line_table
,
2932 /* An upper bound on the number of bytes needed to spell TOKEN.
2933 Does not include preceding whitespace. */
2935 cpp_token_len (const cpp_token
*token
)
2939 switch (TOKEN_SPELL (token
))
2941 default: len
= 6; break;
2942 case SPELL_LITERAL
: len
= token
->val
.str
.len
; break;
2943 case SPELL_IDENT
: len
= NODE_LEN (token
->val
.node
.node
) * 10; break;
2949 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2950 Return the number of bytes read out of NAME. (There are always
2951 10 bytes written to BUFFER.) */
2954 utf8_to_ucn (unsigned char *buffer
, const unsigned char *name
)
2960 unsigned long utf32
;
2962 /* Compute the length of the UTF-8 sequence. */
2963 for (t
= *name
; t
& 0x80; t
<<= 1)
2966 utf32
= *name
& (0x7F >> ucn_len
);
2967 for (ucn_len_c
= 1; ucn_len_c
< ucn_len
; ucn_len_c
++)
2969 utf32
= (utf32
<< 6) | (*++name
& 0x3F);
2971 /* Ill-formed UTF-8. */
2972 if ((*name
& ~0x3F) != 0x80)
2978 for (j
= 7; j
>= 0; j
--)
2979 *buffer
++ = "0123456789abcdef"[(utf32
>> (4 * j
)) & 0xF];
2983 /* Given a token TYPE corresponding to a digraph, return a pointer to
2984 the spelling of the digraph. */
2985 static const unsigned char *
2986 cpp_digraph2name (enum cpp_ttype type
)
2988 return digraph_spellings
[(int) type
- (int) CPP_FIRST_DIGRAPH
];
2991 /* Write the spelling of an identifier IDENT, using UCNs, to BUFFER.
2992 The buffer must already contain the enough space to hold the
2993 token's spelling. Returns a pointer to the character after the
2994 last character written. */
2996 _cpp_spell_ident_ucns (unsigned char *buffer
, cpp_hashnode
*ident
)
2999 const unsigned char *name
= NODE_NAME (ident
);
3001 for (i
= 0; i
< NODE_LEN (ident
); i
++)
3002 if (name
[i
] & ~0x7F)
3004 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
3008 *buffer
++ = name
[i
];
3013 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
3014 already contain the enough space to hold the token's spelling.
3015 Returns a pointer to the character after the last character written.
3016 FORSTRING is true if this is to be the spelling after translation
3017 phase 1 (with the original spelling of extended identifiers), false
3018 if extended identifiers should always be written using UCNs (there is
3019 no option for always writing them in the internal UTF-8 form).
3020 FIXME: Would be nice if we didn't need the PFILE argument. */
3022 cpp_spell_token (cpp_reader
*pfile
, const cpp_token
*token
,
3023 unsigned char *buffer
, bool forstring
)
3025 switch (TOKEN_SPELL (token
))
3027 case SPELL_OPERATOR
:
3029 const unsigned char *spelling
;
3032 if (token
->flags
& DIGRAPH
)
3033 spelling
= cpp_digraph2name (token
->type
);
3034 else if (token
->flags
& NAMED_OP
)
3037 spelling
= TOKEN_NAME (token
);
3039 while ((c
= *spelling
++) != '\0')
3048 memcpy (buffer
, NODE_NAME (token
->val
.node
.spelling
),
3049 NODE_LEN (token
->val
.node
.spelling
));
3050 buffer
+= NODE_LEN (token
->val
.node
.spelling
);
3053 buffer
= _cpp_spell_ident_ucns (buffer
, token
->val
.node
.node
);
3057 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
3058 buffer
+= token
->val
.str
.len
;
3062 cpp_error (pfile
, CPP_DL_ICE
,
3063 "unspellable token %s", TOKEN_NAME (token
));
3070 /* Returns TOKEN spelt as a null-terminated string. The string is
3071 freed when the reader is destroyed. Useful for diagnostics. */
3073 cpp_token_as_text (cpp_reader
*pfile
, const cpp_token
*token
)
3075 unsigned int len
= cpp_token_len (token
) + 1;
3076 unsigned char *start
= _cpp_unaligned_alloc (pfile
, len
), *end
;
3078 end
= cpp_spell_token (pfile
, token
, start
, false);
3084 /* Returns a pointer to a string which spells the token defined by
3085 TYPE and FLAGS. Used by C front ends, which really should move to
3086 using cpp_token_as_text. */
3088 cpp_type2name (enum cpp_ttype type
, unsigned char flags
)
3090 if (flags
& DIGRAPH
)
3091 return (const char *) cpp_digraph2name (type
);
3092 else if (flags
& NAMED_OP
)
3093 return cpp_named_operator2name (type
);
3095 return (const char *) token_spellings
[type
].name
;
3098 /* Writes the spelling of token to FP, without any preceding space.
3099 Separated from cpp_spell_token for efficiency - to avoid stdio
3100 double-buffering. */
3102 cpp_output_token (const cpp_token
*token
, FILE *fp
)
3104 switch (TOKEN_SPELL (token
))
3106 case SPELL_OPERATOR
:
3108 const unsigned char *spelling
;
3111 if (token
->flags
& DIGRAPH
)
3112 spelling
= cpp_digraph2name (token
->type
);
3113 else if (token
->flags
& NAMED_OP
)
3116 spelling
= TOKEN_NAME (token
);
3121 while ((c
= *++spelling
) != '\0');
3129 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
3131 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
3132 if (name
[i
] & ~0x7F)
3134 unsigned char buffer
[10];
3135 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
3136 fwrite (buffer
, 1, 10, fp
);
3139 fputc (NODE_NAME (token
->val
.node
.node
)[i
], fp
);
3144 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
3148 /* An error, most probably. */
3153 /* Compare two tokens. */
3155 _cpp_equiv_tokens (const cpp_token
*a
, const cpp_token
*b
)
3157 if (a
->type
== b
->type
&& a
->flags
== b
->flags
)
3158 switch (TOKEN_SPELL (a
))
3160 default: /* Keep compiler happy. */
3161 case SPELL_OPERATOR
:
3162 /* token_no is used to track where multiple consecutive ##
3163 tokens were originally located. */
3164 return (a
->type
!= CPP_PASTE
|| a
->val
.token_no
== b
->val
.token_no
);
3166 return (a
->type
!= CPP_MACRO_ARG
3167 || (a
->val
.macro_arg
.arg_no
== b
->val
.macro_arg
.arg_no
3168 && a
->val
.macro_arg
.spelling
== b
->val
.macro_arg
.spelling
));
3170 return (a
->val
.node
.node
== b
->val
.node
.node
3171 && a
->val
.node
.spelling
== b
->val
.node
.spelling
);
3173 return (a
->val
.str
.len
== b
->val
.str
.len
3174 && !memcmp (a
->val
.str
.text
, b
->val
.str
.text
,
3181 /* Returns nonzero if a space should be inserted to avoid an
3182 accidental token paste for output. For simplicity, it is
3183 conservative, and occasionally advises a space where one is not
3184 needed, e.g. "." and ".2". */
3186 cpp_avoid_paste (cpp_reader
*pfile
, const cpp_token
*token1
,
3187 const cpp_token
*token2
)
3189 enum cpp_ttype a
= token1
->type
, b
= token2
->type
;
3192 if (token1
->flags
& NAMED_OP
)
3194 if (token2
->flags
& NAMED_OP
)
3198 if (token2
->flags
& DIGRAPH
)
3199 c
= digraph_spellings
[(int) b
- (int) CPP_FIRST_DIGRAPH
][0];
3200 else if (token_spellings
[b
].category
== SPELL_OPERATOR
)
3201 c
= token_spellings
[b
].name
[0];
3203 /* Quickly get everything that can paste with an '='. */
3204 if ((int) a
<= (int) CPP_LAST_EQ
&& c
== '=')
3209 case CPP_GREATER
: return c
== '>';
3210 case CPP_LESS
: return c
== '<' || c
== '%' || c
== ':';
3211 case CPP_PLUS
: return c
== '+';
3212 case CPP_MINUS
: return c
== '-' || c
== '>';
3213 case CPP_DIV
: return c
== '/' || c
== '*'; /* Comments. */
3214 case CPP_MOD
: return c
== ':' || c
== '>';
3215 case CPP_AND
: return c
== '&';
3216 case CPP_OR
: return c
== '|';
3217 case CPP_COLON
: return c
== ':' || c
== '>';
3218 case CPP_DEREF
: return c
== '*';
3219 case CPP_DOT
: return c
== '.' || c
== '%' || b
== CPP_NUMBER
;
3220 case CPP_HASH
: return c
== '#' || c
== '%'; /* Digraph form. */
3221 case CPP_NAME
: return ((b
== CPP_NUMBER
3222 && name_p (pfile
, &token2
->val
.str
))
3224 || b
== CPP_CHAR
|| b
== CPP_STRING
); /* L */
3225 case CPP_NUMBER
: return (b
== CPP_NUMBER
|| b
== CPP_NAME
3226 || c
== '.' || c
== '+' || c
== '-');
3228 case CPP_OTHER
: return ((token1
->val
.str
.text
[0] == '\\'
3230 || (CPP_OPTION (pfile
, objc
)
3231 && token1
->val
.str
.text
[0] == '@'
3232 && (b
== CPP_NAME
|| b
== CPP_STRING
)));
3235 case CPP_UTF8STRING
:
3237 case CPP_STRING32
: return (CPP_OPTION (pfile
, user_literals
)
3239 || (TOKEN_SPELL (token2
) == SPELL_LITERAL
3240 && ISIDST (token2
->val
.str
.text
[0]))));
3248 /* Output all the remaining tokens on the current line, and a newline
3249 character, to FP. Leading whitespace is removed. If there are
3250 macros, special token padding is not performed. */
3252 cpp_output_line (cpp_reader
*pfile
, FILE *fp
)
3254 const cpp_token
*token
;
3256 token
= cpp_get_token (pfile
);
3257 while (token
->type
!= CPP_EOF
)
3259 cpp_output_token (token
, fp
);
3260 token
= cpp_get_token (pfile
);
3261 if (token
->flags
& PREV_WHITE
)
3268 /* Return a string representation of all the remaining tokens on the
3269 current line. The result is allocated using xmalloc and must be
3270 freed by the caller. */
3272 cpp_output_line_to_string (cpp_reader
*pfile
, const unsigned char *dir_name
)
3274 const cpp_token
*token
;
3275 unsigned int out
= dir_name
? ustrlen (dir_name
) : 0;
3276 unsigned int alloced
= 120 + out
;
3277 unsigned char *result
= (unsigned char *) xmalloc (alloced
);
3279 /* If DIR_NAME is empty, there are no initial contents. */
3282 sprintf ((char *) result
, "#%s ", dir_name
);
3286 token
= cpp_get_token (pfile
);
3287 while (token
->type
!= CPP_EOF
)
3289 unsigned char *last
;
3290 /* Include room for a possible space and the terminating nul. */
3291 unsigned int len
= cpp_token_len (token
) + 2;
3293 if (out
+ len
> alloced
)
3296 if (out
+ len
> alloced
)
3297 alloced
= out
+ len
;
3298 result
= (unsigned char *) xrealloc (result
, alloced
);
3301 last
= cpp_spell_token (pfile
, token
, &result
[out
], 0);
3302 out
= last
- result
;
3304 token
= cpp_get_token (pfile
);
3305 if (token
->flags
& PREV_WHITE
)
3306 result
[out
++] = ' ';
3313 /* Memory buffers. Changing these three constants can have a dramatic
3314 effect on performance. The values here are reasonable defaults,
3315 but might be tuned. If you adjust them, be sure to test across a
3316 range of uses of cpplib, including heavy nested function-like macro
3317 expansion. Also check the change in peak memory usage (NJAMD is a
3318 good tool for this). */
3319 #define MIN_BUFF_SIZE 8000
3320 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
3321 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
3322 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
3324 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
3325 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
3328 /* Create a new allocation buffer. Place the control block at the end
3329 of the buffer, so that buffer overflows will cause immediate chaos. */
3331 new_buff (size_t len
)
3334 unsigned char *base
;
3336 if (len
< MIN_BUFF_SIZE
)
3337 len
= MIN_BUFF_SIZE
;
3338 len
= CPP_ALIGN (len
);
3340 #ifdef ENABLE_VALGRIND_ANNOTATIONS
3341 /* Valgrind warns about uses of interior pointers, so put _cpp_buff
3343 size_t slen
= CPP_ALIGN2 (sizeof (_cpp_buff
), 2 * DEFAULT_ALIGNMENT
);
3344 base
= XNEWVEC (unsigned char, len
+ slen
);
3345 result
= (_cpp_buff
*) base
;
3348 base
= XNEWVEC (unsigned char, len
+ sizeof (_cpp_buff
));
3349 result
= (_cpp_buff
*) (base
+ len
);
3351 result
->base
= base
;
3353 result
->limit
= base
+ len
;
3354 result
->next
= NULL
;
3358 /* Place a chain of unwanted allocation buffers on the free list. */
3360 _cpp_release_buff (cpp_reader
*pfile
, _cpp_buff
*buff
)
3362 _cpp_buff
*end
= buff
;
3366 end
->next
= pfile
->free_buffs
;
3367 pfile
->free_buffs
= buff
;
3370 /* Return a free buffer of size at least MIN_SIZE. */
3372 _cpp_get_buff (cpp_reader
*pfile
, size_t min_size
)
3374 _cpp_buff
*result
, **p
;
3376 for (p
= &pfile
->free_buffs
;; p
= &(*p
)->next
)
3381 return new_buff (min_size
);
3383 size
= result
->limit
- result
->base
;
3384 /* Return a buffer that's big enough, but don't waste one that's
3386 if (size
>= min_size
&& size
<= BUFF_SIZE_UPPER_BOUND (min_size
))
3391 result
->next
= NULL
;
3392 result
->cur
= result
->base
;
3396 /* Creates a new buffer with enough space to hold the uncommitted
3397 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
3398 the excess bytes to the new buffer. Chains the new buffer after
3399 BUFF, and returns the new buffer. */
3401 _cpp_append_extend_buff (cpp_reader
*pfile
, _cpp_buff
*buff
, size_t min_extra
)
3403 size_t size
= EXTENDED_BUFF_SIZE (buff
, min_extra
);
3404 _cpp_buff
*new_buff
= _cpp_get_buff (pfile
, size
);
3406 buff
->next
= new_buff
;
3407 memcpy (new_buff
->base
, buff
->cur
, BUFF_ROOM (buff
));
3411 /* Creates a new buffer with enough space to hold the uncommitted
3412 remaining bytes of the buffer pointed to by BUFF, and at least
3413 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
3414 Chains the new buffer before the buffer pointed to by BUFF, and
3415 updates the pointer to point to the new buffer. */
3417 _cpp_extend_buff (cpp_reader
*pfile
, _cpp_buff
**pbuff
, size_t min_extra
)
3419 _cpp_buff
*new_buff
, *old_buff
= *pbuff
;
3420 size_t size
= EXTENDED_BUFF_SIZE (old_buff
, min_extra
);
3422 new_buff
= _cpp_get_buff (pfile
, size
);
3423 memcpy (new_buff
->base
, old_buff
->cur
, BUFF_ROOM (old_buff
));
3424 new_buff
->next
= old_buff
;
3428 /* Free a chain of buffers starting at BUFF. */
3430 _cpp_free_buff (_cpp_buff
*buff
)
3434 for (; buff
; buff
= next
)
3437 #ifdef ENABLE_VALGRIND_ANNOTATIONS
3445 /* Allocate permanent, unaligned storage of length LEN. */
3447 _cpp_unaligned_alloc (cpp_reader
*pfile
, size_t len
)
3449 _cpp_buff
*buff
= pfile
->u_buff
;
3450 unsigned char *result
= buff
->cur
;
3452 if (len
> (size_t) (buff
->limit
- result
))
3454 buff
= _cpp_get_buff (pfile
, len
);
3455 buff
->next
= pfile
->u_buff
;
3456 pfile
->u_buff
= buff
;
3460 buff
->cur
= result
+ len
;
3464 /* Allocate permanent, unaligned storage of length LEN from a_buff.
3465 That buffer is used for growing allocations when saving macro
3466 replacement lists in a #define, and when parsing an answer to an
3467 assertion in #assert, #unassert or #if (and therefore possibly
3468 whilst expanding macros). It therefore must not be used by any
3469 code that they might call: specifically the lexer and the guts of
3472 All existing other uses clearly fit this restriction: storing
3473 registered pragmas during initialization. */
3475 _cpp_aligned_alloc (cpp_reader
*pfile
, size_t len
)
3477 _cpp_buff
*buff
= pfile
->a_buff
;
3478 unsigned char *result
= buff
->cur
;
3480 if (len
> (size_t) (buff
->limit
- result
))
3482 buff
= _cpp_get_buff (pfile
, len
);
3483 buff
->next
= pfile
->a_buff
;
3484 pfile
->a_buff
= buff
;
3488 buff
->cur
= result
+ len
;
3492 /* Say which field of TOK is in use. */
3494 enum cpp_token_fld_kind
3495 cpp_token_val_index (const cpp_token
*tok
)
3497 switch (TOKEN_SPELL (tok
))
3500 return CPP_TOKEN_FLD_NODE
;
3502 return CPP_TOKEN_FLD_STR
;
3503 case SPELL_OPERATOR
:
3504 if (tok
->type
== CPP_PASTE
)
3505 return CPP_TOKEN_FLD_TOKEN_NO
;
3507 return CPP_TOKEN_FLD_NONE
;
3509 if (tok
->type
== CPP_MACRO_ARG
)
3510 return CPP_TOKEN_FLD_ARG_NO
;
3511 else if (tok
->type
== CPP_PADDING
)
3512 return CPP_TOKEN_FLD_SOURCE
;
3513 else if (tok
->type
== CPP_PRAGMA
)
3514 return CPP_TOKEN_FLD_PRAGMA
;
3517 return CPP_TOKEN_FLD_NONE
;
3521 /* All tokens lexed in R after calling this function will be forced to have
3522 their source_location the same as the location referenced by P, until
3523 cpp_stop_forcing_token_locations is called for R. */
3526 cpp_force_token_locations (cpp_reader
*r
, source_location
*p
)
3528 r
->forced_token_location_p
= p
;
3531 /* Go back to assigning locations naturally for lexed tokens. */
3534 cpp_stop_forcing_token_locations (cpp_reader
*r
)
3536 r
->forced_token_location_p
= NULL
;