1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000-2014 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
8 This program is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by the
10 Free Software Foundation; either version 3, or (at your option) any
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
37 enum spell_type category
;
38 const unsigned char *name
;
41 static const unsigned char *const digraph_spellings
[] =
42 { UC
"%:", UC
"%:%:", UC
"<:", UC
":>", UC
"<%", UC
"%>" };
44 #define OP(e, s) { SPELL_OPERATOR, UC s },
45 #define TK(e, s) { SPELL_ ## s, UC #e },
46 static const struct token_spelling token_spellings
[N_TTYPES
] = { TTYPE_TABLE
};
50 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
51 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
53 static void add_line_note (cpp_buffer
*, const uchar
*, unsigned int);
54 static int skip_line_comment (cpp_reader
*);
55 static void skip_whitespace (cpp_reader
*, cppchar_t
);
56 static void lex_string (cpp_reader
*, cpp_token
*, const uchar
*);
57 static void save_comment (cpp_reader
*, cpp_token
*, const uchar
*, cppchar_t
);
58 static void store_comment (cpp_reader
*, cpp_token
*);
59 static void create_literal (cpp_reader
*, cpp_token
*, const uchar
*,
60 unsigned int, enum cpp_ttype
);
61 static bool warn_in_comment (cpp_reader
*, _cpp_line_note
*);
62 static int name_p (cpp_reader
*, const cpp_string
*);
63 static tokenrun
*next_tokenrun (tokenrun
*);
65 static _cpp_buff
*new_buff (size_t);
70 Compares, the token TOKEN to the NUL-terminated string STRING.
71 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
73 cpp_ideq (const cpp_token
*token
, const char *string
)
75 if (token
->type
!= CPP_NAME
)
78 return !ustrcmp (NODE_NAME (token
->val
.node
.node
), (const uchar
*) string
);
81 /* Record a note TYPE at byte POS into the current cleaned logical
84 add_line_note (cpp_buffer
*buffer
, const uchar
*pos
, unsigned int type
)
86 if (buffer
->notes_used
== buffer
->notes_cap
)
88 buffer
->notes_cap
= buffer
->notes_cap
* 2 + 200;
89 buffer
->notes
= XRESIZEVEC (_cpp_line_note
, buffer
->notes
,
93 buffer
->notes
[buffer
->notes_used
].pos
= pos
;
94 buffer
->notes
[buffer
->notes_used
].type
= type
;
99 /* Fast path to find line special characters using optimized character
100 scanning algorithms. Anything complicated falls back to the slow
101 path below. Since this loop is very hot it's worth doing these kinds
104 One of the paths through the ifdefs should provide
106 const uchar *search_line_fast (const uchar *s, const uchar *end);
108 Between S and END, search for \n, \r, \\, ?. Return a pointer to
111 Note that the last character of the buffer is *always* a newline,
112 as forced by _cpp_convert_input. This fact can be used to avoid
113 explicitly looking for the end of the buffer. */
115 /* Configure gives us an ifdef test. */
116 #ifndef WORDS_BIGENDIAN
117 #define WORDS_BIGENDIAN 0
120 /* We'd like the largest integer that fits into a register. There's nothing
121 in <stdint.h> that gives us that. For most hosts this is unsigned long,
122 but MS decided on an LLP64 model. Thankfully when building with GCC we
123 can get the "real" word size. */
125 typedef unsigned int word_type
__attribute__((__mode__(__word__
)));
127 typedef unsigned long word_type
;
130 /* The code below is only expecting sizes 4 or 8.
131 Die at compile-time if this expectation is violated. */
132 typedef char check_word_type_size
133 [(sizeof(word_type
) == 8 || sizeof(word_type
) == 4) * 2 - 1];
135 /* Return X with the first N bytes forced to values that won't match one
136 of the interesting characters. Note that NUL is not interesting. */
138 static inline word_type
139 acc_char_mask_misalign (word_type val
, unsigned int n
)
149 /* Return X replicated to all byte positions within WORD_TYPE. */
151 static inline word_type
152 acc_char_replicate (uchar x
)
156 ret
= (x
<< 24) | (x
<< 16) | (x
<< 8) | x
;
157 if (sizeof(word_type
) == 8)
158 ret
= (ret
<< 16 << 16) | ret
;
162 /* Return non-zero if some byte of VAL is (probably) C. */
164 static inline word_type
165 acc_char_cmp (word_type val
, word_type c
)
167 #if defined(__GNUC__) && defined(__alpha__)
168 /* We can get exact results using a compare-bytes instruction.
169 Get (val == c) via (0 >= (val ^ c)). */
170 return __builtin_alpha_cmpbge (0, val
^ c
);
172 word_type magic
= 0x7efefefeU
;
173 if (sizeof(word_type
) == 8)
174 magic
= (magic
<< 16 << 16) | 0xfefefefeU
;
178 return ((val
+ magic
) ^ ~val
) & ~magic
;
182 /* Given the result of acc_char_cmp is non-zero, return the index of
183 the found character. If this was a false positive, return -1. */
186 acc_char_index (word_type cmp ATTRIBUTE_UNUSED
,
187 word_type val ATTRIBUTE_UNUSED
)
189 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
190 /* The cmpbge instruction sets *bits* of the result corresponding to
191 matches in the bytes with no false positives. */
192 return __builtin_ctzl (cmp
);
196 /* ??? It would be nice to force unrolling here,
197 and have all of these constants folded. */
198 for (i
= 0; i
< sizeof(word_type
); ++i
)
202 c
= (val
>> (sizeof(word_type
) - i
- 1) * 8) & 0xff;
204 c
= (val
>> i
* 8) & 0xff;
206 if (c
== '\n' || c
== '\r' || c
== '\\' || c
== '?')
214 /* A version of the fast scanner using bit fiddling techniques.
216 For 32-bit words, one would normally perform 16 comparisons and
217 16 branches. With this algorithm one performs 24 arithmetic
218 operations and one branch. Whether this is faster with a 32-bit
219 word size is going to be somewhat system dependent.
221 For 64-bit words, we eliminate twice the number of comparisons
222 and branches without increasing the number of arithmetic operations.
223 It's almost certainly going to be a win with 64-bit word size. */
225 static const uchar
* search_line_acc_char (const uchar
*, const uchar
*)
229 search_line_acc_char (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
231 const word_type repl_nl
= acc_char_replicate ('\n');
232 const word_type repl_cr
= acc_char_replicate ('\r');
233 const word_type repl_bs
= acc_char_replicate ('\\');
234 const word_type repl_qm
= acc_char_replicate ('?');
236 unsigned int misalign
;
240 /* Align the buffer. Mask out any bytes from before the beginning. */
241 p
= (word_type
*)((uintptr_t)s
& -sizeof(word_type
));
243 misalign
= (uintptr_t)s
& (sizeof(word_type
) - 1);
245 val
= acc_char_mask_misalign (val
, misalign
);
250 t
= acc_char_cmp (val
, repl_nl
);
251 t
|= acc_char_cmp (val
, repl_cr
);
252 t
|= acc_char_cmp (val
, repl_bs
);
253 t
|= acc_char_cmp (val
, repl_qm
);
255 if (__builtin_expect (t
!= 0, 0))
257 int i
= acc_char_index (t
, val
);
259 return (const uchar
*)p
+ i
;
266 /* Disable on Solaris 2/x86 until the following problem can be properly
269 The Solaris 10+ assembler tags objects with the instruction set
270 extensions used, so SSE4.2 executables cannot run on machines that
271 don't support that extension. */
273 #if (GCC_VERSION >= 4005) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
275 /* Replicated character data to be shared between implementations.
276 Recall that outside of a context with vector support we can't
277 define compatible vector types, therefore these are all defined
278 in terms of raw characters. */
279 static const char repl_chars
[4][16] __attribute__((aligned(16))) = {
280 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
281 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
282 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
283 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
284 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
285 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
286 { '?', '?', '?', '?', '?', '?', '?', '?',
287 '?', '?', '?', '?', '?', '?', '?', '?' },
290 /* A version of the fast scanner using MMX vectorized byte compare insns.
292 This uses the PMOVMSKB instruction which was introduced with "MMX2",
293 which was packaged into SSE1; it is also present in the AMD MMX
294 extension. Mark the function as using "sse" so that we emit a real
295 "emms" instruction, rather than the 3dNOW "femms" instruction. */
299 __attribute__((__target__("sse")))
301 search_line_mmx (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
303 typedef char v8qi
__attribute__ ((__vector_size__ (8)));
304 typedef int __m64
__attribute__ ((__vector_size__ (8), __may_alias__
));
306 const v8qi repl_nl
= *(const v8qi
*)repl_chars
[0];
307 const v8qi repl_cr
= *(const v8qi
*)repl_chars
[1];
308 const v8qi repl_bs
= *(const v8qi
*)repl_chars
[2];
309 const v8qi repl_qm
= *(const v8qi
*)repl_chars
[3];
311 unsigned int misalign
, found
, mask
;
315 /* Align the source pointer. While MMX doesn't generate unaligned data
316 faults, this allows us to safely scan to the end of the buffer without
317 reading beyond the end of the last page. */
318 misalign
= (uintptr_t)s
& 7;
319 p
= (const v8qi
*)((uintptr_t)s
& -8);
322 /* Create a mask for the bytes that are valid within the first
323 16-byte block. The Idea here is that the AND with the mask
324 within the loop is "free", since we need some AND or TEST
325 insn in order to set the flags for the branch anyway. */
326 mask
= -1u << misalign
;
328 /* Main loop processing 8 bytes at a time. */
336 t
= __builtin_ia32_pcmpeqb(data
, repl_nl
);
337 c
= __builtin_ia32_pcmpeqb(data
, repl_cr
);
338 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
339 c
= __builtin_ia32_pcmpeqb(data
, repl_bs
);
340 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
341 c
= __builtin_ia32_pcmpeqb(data
, repl_qm
);
342 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
343 found
= __builtin_ia32_pmovmskb (t
);
348 __builtin_ia32_emms ();
350 /* FOUND contains 1 in bits for which we matched a relevant
351 character. Conversion to the byte index is trivial. */
352 found
= __builtin_ctz(found
);
353 return (const uchar
*)p
+ found
;
356 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
360 __attribute__((__target__("sse2")))
362 search_line_sse2 (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
364 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
366 const v16qi repl_nl
= *(const v16qi
*)repl_chars
[0];
367 const v16qi repl_cr
= *(const v16qi
*)repl_chars
[1];
368 const v16qi repl_bs
= *(const v16qi
*)repl_chars
[2];
369 const v16qi repl_qm
= *(const v16qi
*)repl_chars
[3];
371 unsigned int misalign
, found
, mask
;
375 /* Align the source pointer. */
376 misalign
= (uintptr_t)s
& 15;
377 p
= (const v16qi
*)((uintptr_t)s
& -16);
380 /* Create a mask for the bytes that are valid within the first
381 16-byte block. The Idea here is that the AND with the mask
382 within the loop is "free", since we need some AND or TEST
383 insn in order to set the flags for the branch anyway. */
384 mask
= -1u << misalign
;
386 /* Main loop processing 16 bytes at a time. */
394 t
= __builtin_ia32_pcmpeqb128(data
, repl_nl
);
395 t
|= __builtin_ia32_pcmpeqb128(data
, repl_cr
);
396 t
|= __builtin_ia32_pcmpeqb128(data
, repl_bs
);
397 t
|= __builtin_ia32_pcmpeqb128(data
, repl_qm
);
398 found
= __builtin_ia32_pmovmskb128 (t
);
403 /* FOUND contains 1 in bits for which we matched a relevant
404 character. Conversion to the byte index is trivial. */
405 found
= __builtin_ctz(found
);
406 return (const uchar
*)p
+ found
;
410 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
414 __attribute__((__target__("sse4.2")))
416 search_line_sse42 (const uchar
*s
, const uchar
*end
)
418 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
419 static const v16qi search
= { '\n', '\r', '?', '\\' };
421 uintptr_t si
= (uintptr_t)s
;
424 /* Check for unaligned input. */
429 if (__builtin_expect (end
- s
< 16, 0)
430 && __builtin_expect ((si
& 0xfff) > 0xff0, 0))
432 /* There are less than 16 bytes left in the buffer, and less
433 than 16 bytes left on the page. Reading 16 bytes at this
434 point might generate a spurious page fault. Defer to the
435 SSE2 implementation, which already handles alignment. */
436 return search_line_sse2 (s
, end
);
439 /* ??? The builtin doesn't understand that the PCMPESTRI read from
440 memory need not be aligned. */
441 sv
= __builtin_ia32_loaddqu ((const char *) s
);
442 index
= __builtin_ia32_pcmpestri128 (search
, 4, sv
, 16, 0);
444 if (__builtin_expect (index
< 16, 0))
447 /* Advance the pointer to an aligned address. We will re-scan a
448 few bytes, but we no longer need care for reading past the
449 end of a page, since we're guaranteed a match. */
450 s
= (const uchar
*)((si
+ 16) & -16);
453 /* Main loop, processing 16 bytes at a time. By doing the whole loop
454 in inline assembly, we can make proper use of the flags set. */
455 __asm ( "sub $16, %1\n"
458 " %vpcmpestri $0, (%1), %2\n"
460 : "=&c"(index
), "+r"(s
)
461 : "x"(search
), "a"(4), "d"(16));
468 /* Work around out-dated assemblers without sse4 support. */
469 #define search_line_sse42 search_line_sse2
472 /* Check the CPU capabilities. */
474 #if (GCC_VERSION >= 5000)
475 #include "../gcc/config/i386/cpuid.h"
480 typedef const uchar
* (*search_line_fast_type
) (const uchar
*, const uchar
*);
481 static search_line_fast_type search_line_fast
;
483 #define HAVE_init_vectorized_lexer 1
485 init_vectorized_lexer (void)
487 unsigned dummy
, ecx
= 0, edx
= 0;
488 search_line_fast_type impl
= search_line_acc_char
;
491 #if defined(__SSE4_2__)
493 #elif defined(__SSE2__)
495 #elif defined(__SSE__)
500 impl
= search_line_sse42
;
501 else if (__get_cpuid (1, &dummy
, &dummy
, &ecx
, &edx
) || minimum
== 2)
503 if (minimum
== 3 || (ecx
& bit_SSE4_2
))
504 impl
= search_line_sse42
;
505 else if (minimum
== 2 || (edx
& bit_SSE2
))
506 impl
= search_line_sse2
;
507 else if (minimum
== 1 || (edx
& bit_SSE
))
508 impl
= search_line_mmx
;
510 else if (__get_cpuid (0x80000001, &dummy
, &dummy
, &dummy
, &edx
))
513 || (edx
& (bit_MMXEXT
| bit_CMOV
)) == (bit_MMXEXT
| bit_CMOV
))
514 impl
= search_line_mmx
;
517 search_line_fast
= impl
;
520 #elif defined(_ARCH_PWR8) && defined(__ALTIVEC__)
522 /* A vection of the fast scanner using AltiVec vectorized byte compares
523 and VSX unaligned loads (when VSX is available). This is otherwise
524 the same as the pre-GCC 5 version. */
527 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
529 typedef __attribute__((altivec(vector
))) unsigned char vc
;
532 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
533 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
536 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
537 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
540 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
541 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
544 '?', '?', '?', '?', '?', '?', '?', '?',
545 '?', '?', '?', '?', '?', '?', '?', '?',
547 const vc zero
= { 0 };
551 /* Main loop processing 16 bytes at a time. */
554 vc m_nl
, m_cr
, m_bs
, m_qm
;
556 data
= *((const vc
*)s
);
559 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
560 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
561 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
562 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
563 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
565 /* T now contains 0xff in bytes for which we matched one of the relevant
566 characters. We want to exit the loop if any byte in T is non-zero.
567 Below is the expansion of vec_any_ne(t, zero). */
569 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
571 /* Restore s to to point to the 16 bytes we just processed. */
575 #define N (sizeof(vc) / sizeof(long))
579 /* Statically assert that N is 2 or 4. */
580 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
582 unsigned long l
, i
= 0;
586 /* Find the first word of T that is non-zero. */
593 s
+= sizeof(unsigned long);
597 s
+= sizeof(unsigned long);
602 s
+= sizeof(unsigned long);
606 /* L now contains 0xff in bytes for which we matched one of the
607 relevant characters. We can find the byte index by finding
608 its bit index and dividing by 8. */
609 #ifdef __BIG_ENDIAN__
610 l
= __builtin_clzl(l
) >> 3;
612 l
= __builtin_ctzl(l
) >> 3;
620 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__) && defined (__BIG_ENDIAN__)
622 /* A vection of the fast scanner using AltiVec vectorized byte compares.
623 This cannot be used for little endian because vec_lvsl/lvsr are
624 deprecated for little endian and the code won't work properly. */
625 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
626 so we can't compile this function without -maltivec on the command line
627 (or implied by some other switch). */
630 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
632 typedef __attribute__((altivec(vector
))) unsigned char vc
;
635 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
636 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
639 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
640 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
643 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
644 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
647 '?', '?', '?', '?', '?', '?', '?', '?',
648 '?', '?', '?', '?', '?', '?', '?', '?',
651 -1, -1, -1, -1, -1, -1, -1, -1,
652 -1, -1, -1, -1, -1, -1, -1, -1,
654 const vc zero
= { 0 };
658 /* Altivec loads automatically mask addresses with -16. This lets us
659 issue the first load as early as possible. */
660 data
= __builtin_vec_ld(0, (const vc
*)s
);
662 /* Discard bytes before the beginning of the buffer. Do this by
663 beginning with all ones and shifting in zeros according to the
664 mis-alignment. The LVSR instruction pulls the exact shift we
665 want from the address. */
666 mask
= __builtin_vec_lvsr(0, s
);
667 mask
= __builtin_vec_perm(zero
, ones
, mask
);
670 /* While altivec loads mask addresses, we still need to align S so
671 that the offset we compute at the end is correct. */
672 s
= (const uchar
*)((uintptr_t)s
& -16);
674 /* Main loop processing 16 bytes at a time. */
678 vc m_nl
, m_cr
, m_bs
, m_qm
;
681 data
= __builtin_vec_ld(0, (const vc
*)s
);
684 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
685 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
686 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
687 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
688 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
690 /* T now contains 0xff in bytes for which we matched one of the relevant
691 characters. We want to exit the loop if any byte in T is non-zero.
692 Below is the expansion of vec_any_ne(t, zero). */
694 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
697 #define N (sizeof(vc) / sizeof(long))
701 /* Statically assert that N is 2 or 4. */
702 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
704 unsigned long l
, i
= 0;
708 /* Find the first word of T that is non-zero. */
715 s
+= sizeof(unsigned long);
719 s
+= sizeof(unsigned long);
724 s
+= sizeof(unsigned long);
728 /* L now contains 0xff in bytes for which we matched one of the
729 relevant characters. We can find the byte index by finding
730 its bit index and dividing by 8. */
731 l
= __builtin_clzl(l
) >> 3;
738 #elif defined (__ARM_NEON__)
739 #include "arm_neon.h"
742 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
744 const uint8x16_t repl_nl
= vdupq_n_u8 ('\n');
745 const uint8x16_t repl_cr
= vdupq_n_u8 ('\r');
746 const uint8x16_t repl_bs
= vdupq_n_u8 ('\\');
747 const uint8x16_t repl_qm
= vdupq_n_u8 ('?');
748 const uint8x16_t xmask
= (uint8x16_t
) vdupq_n_u64 (0x8040201008040201ULL
);
750 unsigned int misalign
, found
, mask
;
754 /* Align the source pointer. */
755 misalign
= (uintptr_t)s
& 15;
756 p
= (const uint8_t *)((uintptr_t)s
& -16);
759 /* Create a mask for the bytes that are valid within the first
760 16-byte block. The Idea here is that the AND with the mask
761 within the loop is "free", since we need some AND or TEST
762 insn in order to set the flags for the branch anyway. */
763 mask
= (-1u << misalign
) & 0xffff;
765 /* Main loop, processing 16 bytes at a time. */
773 uint8x16_t t
, u
, v
, w
;
780 t
= vceqq_u8 (data
, repl_nl
);
781 u
= vceqq_u8 (data
, repl_cr
);
782 v
= vorrq_u8 (t
, vceqq_u8 (data
, repl_bs
));
783 w
= vorrq_u8 (u
, vceqq_u8 (data
, repl_qm
));
784 t
= vandq_u8 (vorrq_u8 (v
, w
), xmask
);
785 l
= vpadd_u8 (vget_low_u8 (t
), vget_high_u8 (t
));
789 found
= vget_lane_u32 ((uint32x2_t
) vorr_u64 ((uint64x1_t
) n
,
790 vshr_n_u64 ((uint64x1_t
) n
, 24)), 0);
795 /* FOUND contains 1 in bits for which we matched a relevant
796 character. Conversion to the byte index is trivial. */
797 found
= __builtin_ctz (found
);
798 return (const uchar
*)p
+ found
;
803 /* We only have one accellerated alternative. Use a direct call so that
804 we encourage inlining. */
806 #define search_line_fast search_line_acc_char
810 /* Initialize the lexer if needed. */
813 _cpp_init_lexer (void)
815 #ifdef HAVE_init_vectorized_lexer
816 init_vectorized_lexer ();
820 /* Returns with a logical line that contains no escaped newlines or
821 trigraphs. This is a time-critical inner loop. */
823 _cpp_clean_line (cpp_reader
*pfile
)
829 buffer
= pfile
->buffer
;
830 buffer
->cur_note
= buffer
->notes_used
= 0;
831 buffer
->cur
= buffer
->line_base
= buffer
->next_line
;
832 buffer
->need_line
= false;
833 s
= buffer
->next_line
;
835 if (!buffer
->from_stage3
)
837 const uchar
*pbackslash
= NULL
;
839 /* Fast path. This is the common case of an un-escaped line with
840 no trigraphs. The primary win here is by not writing any
841 data back to memory until we have to. */
844 /* Perform an optimized search for \n, \r, \\, ?. */
845 s
= search_line_fast (s
, buffer
->rlimit
);
850 /* Record the location of the backslash and continue. */
853 else if (__builtin_expect (c
== '?', 0))
855 if (__builtin_expect (s
[1] == '?', false)
856 && _cpp_trigraph_map
[s
[2]])
858 /* Have a trigraph. We may or may not have to convert
859 it. Add a line note regardless, for -Wtrigraphs. */
860 add_line_note (buffer
, s
, s
[2]);
861 if (CPP_OPTION (pfile
, trigraphs
))
863 /* We do, and that means we have to switch to the
866 *d
= _cpp_trigraph_map
[s
[2]];
871 /* Not a trigraph. Continue on fast-path. */
878 /* This must be \r or \n. We're either done, or we'll be forced
879 to write back to the buffer and continue on the slow path. */
882 if (__builtin_expect (s
== buffer
->rlimit
, false))
885 /* DOS line ending? */
886 if (__builtin_expect (c
== '\r', false) && s
[1] == '\n')
889 if (s
== buffer
->rlimit
)
893 if (__builtin_expect (pbackslash
== NULL
, true))
896 /* Check for escaped newline. */
898 while (is_nvspace (p
[-1]))
900 if (p
- 1 != pbackslash
)
903 /* Have an escaped newline; process it and proceed to
905 add_line_note (buffer
, p
- 1, p
!= d
? ' ' : '\\');
907 buffer
->next_line
= p
- 1;
915 if (c
== '\n' || c
== '\r')
917 /* Handle DOS line endings. */
918 if (c
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
920 if (s
== buffer
->rlimit
)
925 while (p
!= buffer
->next_line
&& is_nvspace (p
[-1]))
927 if (p
== buffer
->next_line
|| p
[-1] != '\\')
930 add_line_note (buffer
, p
- 1, p
!= d
? ' ': '\\');
932 buffer
->next_line
= p
- 1;
934 else if (c
== '?' && s
[1] == '?' && _cpp_trigraph_map
[s
[2]])
936 /* Add a note regardless, for the benefit of -Wtrigraphs. */
937 add_line_note (buffer
, d
, s
[2]);
938 if (CPP_OPTION (pfile
, trigraphs
))
940 *d
= _cpp_trigraph_map
[s
[2]];
948 while (*s
!= '\n' && *s
!= '\r')
952 /* Handle DOS line endings. */
953 if (*s
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
959 /* A sentinel note that should never be processed. */
960 add_line_note (buffer
, d
+ 1, '\n');
961 buffer
->next_line
= s
+ 1;
964 /* Return true if the trigraph indicated by NOTE should be warned
965 about in a comment. */
967 warn_in_comment (cpp_reader
*pfile
, _cpp_line_note
*note
)
971 /* Within comments we don't warn about trigraphs, unless the
972 trigraph forms an escaped newline, as that may change
974 if (note
->type
!= '/')
977 /* If -trigraphs, then this was an escaped newline iff the next note
979 if (CPP_OPTION (pfile
, trigraphs
))
980 return note
[1].pos
== note
->pos
;
982 /* Otherwise, see if this forms an escaped newline. */
984 while (is_nvspace (*p
))
987 /* There might have been escaped newlines between the trigraph and the
988 newline we found. Hence the position test. */
989 return (*p
== '\n' && p
< note
[1].pos
);
992 /* Process the notes created by add_line_note as far as the current
995 _cpp_process_line_notes (cpp_reader
*pfile
, int in_comment
)
997 cpp_buffer
*buffer
= pfile
->buffer
;
1001 _cpp_line_note
*note
= &buffer
->notes
[buffer
->cur_note
];
1004 if (note
->pos
> buffer
->cur
)
1008 col
= CPP_BUF_COLUMN (buffer
, note
->pos
+ 1);
1010 if (note
->type
== '\\' || note
->type
== ' ')
1012 if (note
->type
== ' ' && !in_comment
)
1013 cpp_error_with_line (pfile
, CPP_DL_WARNING
, pfile
->line_table
->highest_line
, col
,
1014 "backslash and newline separated by space");
1016 if (buffer
->next_line
> buffer
->rlimit
)
1018 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
, col
,
1019 "backslash-newline at end of file");
1020 /* Prevent "no newline at end of file" warning. */
1021 buffer
->next_line
= buffer
->rlimit
;
1024 buffer
->line_base
= note
->pos
;
1025 CPP_INCREMENT_LINE (pfile
, 0);
1027 else if (_cpp_trigraph_map
[note
->type
])
1029 if (CPP_OPTION (pfile
, warn_trigraphs
)
1030 && (!in_comment
|| warn_in_comment (pfile
, note
)))
1032 if (CPP_OPTION (pfile
, trigraphs
))
1033 cpp_warning_with_line (pfile
, CPP_W_TRIGRAPHS
,
1034 pfile
->line_table
->highest_line
, col
,
1035 "trigraph ??%c converted to %c",
1037 (int) _cpp_trigraph_map
[note
->type
]);
1040 cpp_warning_with_line
1041 (pfile
, CPP_W_TRIGRAPHS
,
1042 pfile
->line_table
->highest_line
, col
,
1043 "trigraph ??%c ignored, use -trigraphs to enable",
1048 else if (note
->type
== 0)
1049 /* Already processed in lex_raw_string. */;
1055 /* Skip a C-style block comment. We find the end of the comment by
1056 seeing if an asterisk is before every '/' we encounter. Returns
1057 nonzero if comment terminated by EOF, zero otherwise.
1059 Buffer->cur points to the initial asterisk of the comment. */
1061 _cpp_skip_block_comment (cpp_reader
*pfile
)
1063 cpp_buffer
*buffer
= pfile
->buffer
;
1064 const uchar
*cur
= buffer
->cur
;
1073 /* People like decorating comments with '*', so check for '/'
1074 instead for efficiency. */
1082 /* Warn about potential nested comments, but not if the '/'
1083 comes immediately before the true comment delimiter.
1084 Don't bother to get it right across escaped newlines. */
1085 if (CPP_OPTION (pfile
, warn_comments
)
1086 && cur
[0] == '*' && cur
[1] != '/')
1089 cpp_warning_with_line (pfile
, CPP_W_COMMENTS
,
1090 pfile
->line_table
->highest_line
,
1091 CPP_BUF_COL (buffer
),
1092 "\"/*\" within comment");
1098 buffer
->cur
= cur
- 1;
1099 _cpp_process_line_notes (pfile
, true);
1100 if (buffer
->next_line
>= buffer
->rlimit
)
1102 _cpp_clean_line (pfile
);
1104 cols
= buffer
->next_line
- buffer
->line_base
;
1105 CPP_INCREMENT_LINE (pfile
, cols
);
1112 _cpp_process_line_notes (pfile
, true);
1116 /* Skip a C++ line comment, leaving buffer->cur pointing to the
1117 terminating newline. Handles escaped newlines. Returns nonzero
1118 if a multiline comment. */
1120 skip_line_comment (cpp_reader
*pfile
)
1122 cpp_buffer
*buffer
= pfile
->buffer
;
1123 source_location orig_line
= pfile
->line_table
->highest_line
;
1125 while (*buffer
->cur
!= '\n')
1128 _cpp_process_line_notes (pfile
, true);
1129 return orig_line
!= pfile
->line_table
->highest_line
;
1132 /* Skips whitespace, saving the next non-whitespace character. */
1134 skip_whitespace (cpp_reader
*pfile
, cppchar_t c
)
1136 cpp_buffer
*buffer
= pfile
->buffer
;
1137 bool saw_NUL
= false;
1141 /* Horizontal space always OK. */
1142 if (c
== ' ' || c
== '\t')
1144 /* Just \f \v or \0 left. */
1147 else if (pfile
->state
.in_directive
&& CPP_PEDANTIC (pfile
))
1148 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
,
1149 CPP_BUF_COL (buffer
),
1150 "%s in preprocessing directive",
1151 c
== '\f' ? "form feed" : "vertical tab");
1155 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
1156 while (is_nvspace (c
));
1159 cpp_error (pfile
, CPP_DL_WARNING
, "null character(s) ignored");
1164 /* See if the characters of a number token are valid in a name (no
1165 '.', '+' or '-'). */
1167 name_p (cpp_reader
*pfile
, const cpp_string
*string
)
1171 for (i
= 0; i
< string
->len
; i
++)
1172 if (!is_idchar (string
->text
[i
]))
1178 /* After parsing an identifier or other sequence, produce a warning about
1179 sequences not in NFC/NFKC. */
1181 warn_about_normalization (cpp_reader
*pfile
,
1182 const cpp_token
*token
,
1183 const struct normalize_state
*s
)
1185 if (CPP_OPTION (pfile
, warn_normalize
) < NORMALIZE_STATE_RESULT (s
)
1186 && !pfile
->state
.skipping
)
1188 /* Make sure that the token is printed using UCNs, even
1189 if we'd otherwise happily print UTF-8. */
1190 unsigned char *buf
= XNEWVEC (unsigned char, cpp_token_len (token
));
1193 sz
= cpp_spell_token (pfile
, token
, buf
, false) - buf
;
1194 if (NORMALIZE_STATE_RESULT (s
) == normalized_C
)
1195 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1196 "`%.*s' is not in NFKC", (int) sz
, buf
);
1198 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1199 "`%.*s' is not in NFC", (int) sz
, buf
);
1204 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1205 an identifier. FIRST is TRUE if this starts an identifier. */
1207 forms_identifier_p (cpp_reader
*pfile
, int first
,
1208 struct normalize_state
*state
)
1210 cpp_buffer
*buffer
= pfile
->buffer
;
1212 if (*buffer
->cur
== '$')
1214 if (!CPP_OPTION (pfile
, dollars_in_ident
))
1218 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1220 CPP_OPTION (pfile
, warn_dollars
) = 0;
1221 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1227 /* Is this a syntactically valid UCN? */
1228 if (CPP_OPTION (pfile
, extended_identifiers
)
1229 && *buffer
->cur
== '\\'
1230 && (buffer
->cur
[1] == 'u' || buffer
->cur
[1] == 'U'))
1233 if (_cpp_valid_ucn (pfile
, &buffer
->cur
, buffer
->rlimit
, 1 + !first
,
1242 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1243 static cpp_hashnode
*
1244 lex_identifier_intern (cpp_reader
*pfile
, const uchar
*base
)
1246 cpp_hashnode
*result
;
1249 unsigned int hash
= HT_HASHSTEP (0, *base
);
1252 while (ISIDNUM (*cur
))
1254 hash
= HT_HASHSTEP (hash
, *cur
);
1258 hash
= HT_HASHFINISH (hash
, len
);
1259 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1260 base
, len
, hash
, HT_ALLOC
));
1262 /* Rarely, identifiers require diagnostics when lexed. */
1263 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1264 && !pfile
->state
.skipping
, 0))
1266 /* It is allowed to poison the same identifier twice. */
1267 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1268 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1269 NODE_NAME (result
));
1271 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1272 replacement list of a variadic macro. */
1273 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1274 && !pfile
->state
.va_args_ok
)
1276 if (CPP_OPTION (pfile
, cplusplus
))
1277 cpp_error (pfile
, CPP_DL_PEDWARN
,
1278 "__VA_ARGS__ can only appear in the expansion"
1279 " of a C++11 variadic macro");
1281 cpp_error (pfile
, CPP_DL_PEDWARN
,
1282 "__VA_ARGS__ can only appear in the expansion"
1283 " of a C99 variadic macro");
1286 /* For -Wc++-compat, warn about use of C++ named operators. */
1287 if (result
->flags
& NODE_WARN_OPERATOR
)
1288 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1289 "identifier \"%s\" is a special operator name in C++",
1290 NODE_NAME (result
));
1296 /* Get the cpp_hashnode of an identifier specified by NAME in
1297 the current cpp_reader object. If none is found, NULL is returned. */
1299 _cpp_lex_identifier (cpp_reader
*pfile
, const char *name
)
1301 cpp_hashnode
*result
;
1302 result
= lex_identifier_intern (pfile
, (uchar
*) name
);
1306 /* Lex an identifier starting at BUFFER->CUR - 1. */
1307 static cpp_hashnode
*
1308 lex_identifier (cpp_reader
*pfile
, const uchar
*base
, bool starts_ucn
,
1309 struct normalize_state
*nst
, cpp_hashnode
**spelling
)
1311 cpp_hashnode
*result
;
1314 unsigned int hash
= HT_HASHSTEP (0, *base
);
1316 cur
= pfile
->buffer
->cur
;
1319 while (ISIDNUM (*cur
))
1321 hash
= HT_HASHSTEP (hash
, *cur
);
1324 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *(cur
- 1));
1326 pfile
->buffer
->cur
= cur
;
1327 if (starts_ucn
|| forms_identifier_p (pfile
, false, nst
))
1329 /* Slower version for identifiers containing UCNs (or $). */
1331 while (ISIDNUM (*pfile
->buffer
->cur
))
1333 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *pfile
->buffer
->cur
);
1334 pfile
->buffer
->cur
++;
1336 } while (forms_identifier_p (pfile
, false, nst
));
1337 result
= _cpp_interpret_identifier (pfile
, base
,
1338 pfile
->buffer
->cur
- base
);
1339 *spelling
= cpp_lookup (pfile
, base
, pfile
->buffer
->cur
- base
);
1344 hash
= HT_HASHFINISH (hash
, len
);
1346 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1347 base
, len
, hash
, HT_ALLOC
));
1351 /* Rarely, identifiers require diagnostics when lexed. */
1352 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1353 && !pfile
->state
.skipping
, 0))
1355 /* It is allowed to poison the same identifier twice. */
1356 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1357 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1358 NODE_NAME (result
));
1360 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1361 replacement list of a variadic macro. */
1362 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1363 && !pfile
->state
.va_args_ok
)
1365 if (CPP_OPTION (pfile
, cplusplus
))
1366 cpp_error (pfile
, CPP_DL_PEDWARN
,
1367 "__VA_ARGS__ can only appear in the expansion"
1368 " of a C++11 variadic macro");
1370 cpp_error (pfile
, CPP_DL_PEDWARN
,
1371 "__VA_ARGS__ can only appear in the expansion"
1372 " of a C99 variadic macro");
1375 /* For -Wc++-compat, warn about use of C++ named operators. */
1376 if (result
->flags
& NODE_WARN_OPERATOR
)
1377 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1378 "identifier \"%s\" is a special operator name in C++",
1379 NODE_NAME (result
));
1385 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1387 lex_number (cpp_reader
*pfile
, cpp_string
*number
,
1388 struct normalize_state
*nst
)
1394 base
= pfile
->buffer
->cur
- 1;
1397 cur
= pfile
->buffer
->cur
;
1399 /* N.B. ISIDNUM does not include $. */
1400 while (ISIDNUM (*cur
) || *cur
== '.' || DIGIT_SEP (*cur
)
1401 || VALID_SIGN (*cur
, cur
[-1]))
1403 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *cur
);
1407 pfile
->buffer
->cur
= cur
;
1409 while (forms_identifier_p (pfile
, false, nst
));
1411 number
->len
= cur
- base
;
1412 dest
= _cpp_unaligned_alloc (pfile
, number
->len
+ 1);
1413 memcpy (dest
, base
, number
->len
);
1414 dest
[number
->len
] = '\0';
1415 number
->text
= dest
;
1418 /* Create a token of type TYPE with a literal spelling. */
1420 create_literal (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1421 unsigned int len
, enum cpp_ttype type
)
1423 uchar
*dest
= _cpp_unaligned_alloc (pfile
, len
+ 1);
1425 memcpy (dest
, base
, len
);
1428 token
->val
.str
.len
= len
;
1429 token
->val
.str
.text
= dest
;
1432 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1433 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1436 bufring_append (cpp_reader
*pfile
, const uchar
*base
, size_t len
,
1437 _cpp_buff
**first_buff_p
, _cpp_buff
**last_buff_p
)
1439 _cpp_buff
*first_buff
= *first_buff_p
;
1440 _cpp_buff
*last_buff
= *last_buff_p
;
1442 if (first_buff
== NULL
)
1443 first_buff
= last_buff
= _cpp_get_buff (pfile
, len
);
1444 else if (len
> BUFF_ROOM (last_buff
))
1446 size_t room
= BUFF_ROOM (last_buff
);
1447 memcpy (BUFF_FRONT (last_buff
), base
, room
);
1448 BUFF_FRONT (last_buff
) += room
;
1451 last_buff
= _cpp_append_extend_buff (pfile
, last_buff
, len
);
1454 memcpy (BUFF_FRONT (last_buff
), base
, len
);
1455 BUFF_FRONT (last_buff
) += len
;
1457 *first_buff_p
= first_buff
;
1458 *last_buff_p
= last_buff
;
1462 /* Returns true if a macro has been defined.
1463 This might not work if compile with -save-temps,
1464 or preprocess separately from compilation. */
1467 is_macro(cpp_reader
*pfile
, const uchar
*base
)
1469 const uchar
*cur
= base
;
1470 if (! ISIDST (*cur
))
1472 unsigned int hash
= HT_HASHSTEP (0, *cur
);
1474 while (ISIDNUM (*cur
))
1476 hash
= HT_HASHSTEP (hash
, *cur
);
1479 hash
= HT_HASHFINISH (hash
, cur
- base
);
1481 cpp_hashnode
*result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1482 base
, cur
- base
, hash
, HT_NO_INSERT
));
1484 return !result
? false : (result
->type
== NT_MACRO
);
1488 /* Lexes a raw string. The stored string contains the spelling, including
1489 double quotes, delimiter string, '(' and ')', any leading
1490 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1491 literal, or CPP_OTHER if it was not properly terminated.
1493 The spelling is NUL-terminated, but it is not guaranteed that this
1494 is the first NUL since embedded NULs are preserved. */
1497 lex_raw_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1500 uchar raw_prefix
[17];
1501 uchar temp_buffer
[18];
1502 const uchar
*orig_base
;
1503 unsigned int raw_prefix_len
= 0, raw_suffix_len
= 0;
1504 enum raw_str_phase
{ RAW_STR_PREFIX
, RAW_STR
, RAW_STR_SUFFIX
};
1505 raw_str_phase phase
= RAW_STR_PREFIX
;
1506 enum cpp_ttype type
;
1507 size_t total_len
= 0;
1508 /* Index into temp_buffer during phases other than RAW_STR,
1509 during RAW_STR phase 17 to tell BUF_APPEND that nothing should
1510 be appended to temp_buffer. */
1511 size_t temp_buffer_len
= 0;
1512 _cpp_buff
*first_buff
= NULL
, *last_buff
= NULL
;
1513 size_t raw_prefix_start
;
1514 _cpp_line_note
*note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1516 type
= (*base
== 'L' ? CPP_WSTRING
:
1517 *base
== 'U' ? CPP_STRING32
:
1518 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1521 #define BUF_APPEND(STR,LEN) \
1523 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1524 &first_buff, &last_buff); \
1525 total_len += (LEN); \
1526 if (__builtin_expect (temp_buffer_len < 17, 0) \
1527 && (const uchar *)(STR) != base \
1530 memcpy (temp_buffer + temp_buffer_len, \
1531 (const uchar *)(STR), (LEN)); \
1532 temp_buffer_len += (LEN); \
1538 raw_prefix_start
= cur
- base
;
1543 /* If we previously performed any trigraph or line splicing
1544 transformations, undo them in between the opening and closing
1546 while (note
->pos
< cur
)
1548 for (; note
->pos
== cur
; ++note
)
1554 /* Restore backslash followed by newline. */
1555 BUF_APPEND (base
, cur
- base
);
1557 BUF_APPEND ("\\", 1);
1559 if (note
->type
== ' ')
1561 /* GNU backslash whitespace newline extension. FIXME
1562 could be any sequence of non-vertical space. When we
1563 can properly restore any such sequence, we should mark
1564 this note as handled so _cpp_process_line_notes
1566 BUF_APPEND (" ", 1);
1569 BUF_APPEND ("\n", 1);
1573 /* Already handled. */
1577 if (_cpp_trigraph_map
[note
->type
])
1579 /* Don't warn about this trigraph in
1580 _cpp_process_line_notes, since trigraphs show up as
1581 trigraphs in raw strings. */
1582 uchar type
= note
->type
;
1585 if (!CPP_OPTION (pfile
, trigraphs
))
1586 /* If we didn't convert the trigraph in the first
1587 place, don't do anything now either. */
1590 BUF_APPEND (base
, cur
- base
);
1592 BUF_APPEND ("??", 2);
1594 /* ??/ followed by newline gets two line notes, one for
1595 the trigraph and one for the backslash/newline. */
1596 if (type
== '/' && note
[1].pos
== cur
)
1598 if (note
[1].type
!= '\\'
1599 && note
[1].type
!= ' ')
1601 BUF_APPEND ("/", 1);
1603 goto after_backslash
;
1607 /* Skip the replacement character. */
1609 BUF_APPEND (&type
, 1);
1620 if (__builtin_expect (temp_buffer_len
< 17, 0))
1621 temp_buffer
[temp_buffer_len
++] = c
;
1624 if (phase
== RAW_STR_PREFIX
)
1626 while (raw_prefix_len
< temp_buffer_len
)
1628 raw_prefix
[raw_prefix_len
] = temp_buffer
[raw_prefix_len
];
1629 switch (raw_prefix
[raw_prefix_len
])
1631 case ' ': case '(': case ')': case '\\': case '\t':
1632 case '\v': case '\f': case '\n': default:
1634 /* Basic source charset except the above chars. */
1635 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1636 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1637 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1638 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1640 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1641 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1642 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1643 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1645 case '0': case '1': case '2': case '3': case '4': case '5':
1646 case '6': case '7': case '8': case '9':
1647 case '_': case '{': case '}': case '#': case '[': case ']':
1648 case '<': case '>': case '%': case ':': case ';': case '.':
1649 case '?': case '*': case '+': case '-': case '/': case '^':
1650 case '&': case '|': case '~': case '!': case '=': case ',':
1651 case '"': case '\'':
1652 if (raw_prefix_len
< 16)
1660 if (raw_prefix
[raw_prefix_len
] != '(')
1662 int col
= CPP_BUF_COLUMN (pfile
->buffer
, cur
) + 1;
1663 if (raw_prefix_len
== 16)
1664 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1665 col
, "raw string delimiter longer "
1666 "than 16 characters");
1667 else if (raw_prefix
[raw_prefix_len
] == '\n')
1668 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1669 col
, "invalid new-line in raw "
1670 "string delimiter");
1672 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1673 col
, "invalid character '%c' in "
1674 "raw string delimiter",
1675 (int) raw_prefix
[raw_prefix_len
]);
1676 pfile
->buffer
->cur
= orig_base
+ raw_prefix_start
- 1;
1677 create_literal (pfile
, token
, orig_base
,
1678 raw_prefix_start
- 1, CPP_OTHER
);
1680 _cpp_release_buff (pfile
, first_buff
);
1683 raw_prefix
[raw_prefix_len
] = '"';
1685 /* Nothing should be appended to temp_buffer during
1687 temp_buffer_len
= 17;
1692 else if (phase
== RAW_STR_SUFFIX
)
1694 while (raw_suffix_len
<= raw_prefix_len
1695 && raw_suffix_len
< temp_buffer_len
1696 && temp_buffer
[raw_suffix_len
] == raw_prefix
[raw_suffix_len
])
1698 if (raw_suffix_len
> raw_prefix_len
)
1700 if (raw_suffix_len
== temp_buffer_len
)
1703 /* Nothing should be appended to temp_buffer during
1705 temp_buffer_len
= 17;
1709 phase
= RAW_STR_SUFFIX
;
1711 temp_buffer_len
= 0;
1715 if (pfile
->state
.in_directive
1716 || (pfile
->state
.parsing_args
1717 && pfile
->buffer
->next_line
>= pfile
->buffer
->rlimit
))
1721 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, 0,
1722 "unterminated raw string");
1726 BUF_APPEND (base
, cur
- base
);
1728 if (pfile
->buffer
->cur
< pfile
->buffer
->rlimit
)
1729 CPP_INCREMENT_LINE (pfile
, 0);
1730 pfile
->buffer
->need_line
= true;
1732 pfile
->buffer
->cur
= cur
-1;
1733 _cpp_process_line_notes (pfile
, false);
1734 if (!_cpp_get_fresh_line (pfile
))
1736 source_location src_loc
= token
->src_loc
;
1737 token
->type
= CPP_EOF
;
1738 /* Tell the compiler the line number of the EOF token. */
1739 token
->src_loc
= pfile
->line_table
->highest_line
;
1741 if (first_buff
!= NULL
)
1742 _cpp_release_buff (pfile
, first_buff
);
1743 cpp_error_with_line (pfile
, CPP_DL_ERROR
, src_loc
, 0,
1744 "unterminated raw string");
1748 cur
= base
= pfile
->buffer
->cur
;
1749 note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1753 if (CPP_OPTION (pfile
, user_literals
))
1755 /* If a string format macro, say from inttypes.h, is placed touching
1756 a string literal it could be parsed as a C++11 user-defined string
1757 literal thus breaking the program.
1758 Try to identify macros with is_macro. A warning is issued. */
1759 if (is_macro (pfile
, cur
))
1761 /* Raise a warning, but do not consume subsequent tokens. */
1762 if (CPP_OPTION (pfile
, warn_literal_suffix
) && !pfile
->state
.skipping
)
1763 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1765 "invalid suffix on literal; C++11 requires "
1766 "a space between literal and string macro");
1768 /* Grab user defined literal suffix. */
1769 else if (ISIDST (*cur
))
1771 type
= cpp_userdef_string_add_type (type
);
1774 while (ISIDNUM (*cur
))
1779 pfile
->buffer
->cur
= cur
;
1780 if (first_buff
== NULL
)
1781 create_literal (pfile
, token
, base
, cur
- base
, type
);
1784 uchar
*dest
= _cpp_unaligned_alloc (pfile
, total_len
+ (cur
- base
) + 1);
1787 token
->val
.str
.len
= total_len
+ (cur
- base
);
1788 token
->val
.str
.text
= dest
;
1789 last_buff
= first_buff
;
1790 while (last_buff
!= NULL
)
1792 memcpy (dest
, last_buff
->base
,
1793 BUFF_FRONT (last_buff
) - last_buff
->base
);
1794 dest
+= BUFF_FRONT (last_buff
) - last_buff
->base
;
1795 last_buff
= last_buff
->next
;
1797 _cpp_release_buff (pfile
, first_buff
);
1798 memcpy (dest
, base
, cur
- base
);
1799 dest
[cur
- base
] = '\0';
1803 /* Lexes a string, character constant, or angle-bracketed header file
1804 name. The stored string contains the spelling, including opening
1805 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1806 'R' modifier. It returns the type of the literal, or CPP_OTHER
1807 if it was not properly terminated, or CPP_LESS for an unterminated
1808 header name which must be relexed as normal tokens.
1810 The spelling is NUL-terminated, but it is not guaranteed that this
1811 is the first NUL since embedded NULs are preserved. */
1813 lex_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
)
1815 bool saw_NUL
= false;
1817 cppchar_t terminator
;
1818 enum cpp_ttype type
;
1821 terminator
= *cur
++;
1822 if (terminator
== 'L' || terminator
== 'U')
1823 terminator
= *cur
++;
1824 else if (terminator
== 'u')
1826 terminator
= *cur
++;
1827 if (terminator
== '8')
1828 terminator
= *cur
++;
1830 if (terminator
== 'R')
1832 lex_raw_string (pfile
, token
, base
, cur
);
1835 if (terminator
== '"')
1836 type
= (*base
== 'L' ? CPP_WSTRING
:
1837 *base
== 'U' ? CPP_STRING32
:
1838 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1840 else if (terminator
== '\'')
1841 type
= (*base
== 'L' ? CPP_WCHAR
:
1842 *base
== 'U' ? CPP_CHAR32
:
1843 *base
== 'u' ? CPP_CHAR16
: CPP_CHAR
);
1845 terminator
= '>', type
= CPP_HEADER_NAME
;
1849 cppchar_t c
= *cur
++;
1851 /* In #include-style directives, terminators are not escapable. */
1852 if (c
== '\\' && !pfile
->state
.angled_headers
&& *cur
!= '\n')
1854 else if (c
== terminator
)
1859 /* Unmatched quotes always yield undefined behavior, but
1860 greedy lexing means that what appears to be an unterminated
1861 header name may actually be a legitimate sequence of tokens. */
1862 if (terminator
== '>')
1864 token
->type
= CPP_LESS
;
1874 if (saw_NUL
&& !pfile
->state
.skipping
)
1875 cpp_error (pfile
, CPP_DL_WARNING
,
1876 "null character(s) preserved in literal");
1878 if (type
== CPP_OTHER
&& CPP_OPTION (pfile
, lang
) != CLK_ASM
)
1879 cpp_error (pfile
, CPP_DL_PEDWARN
, "missing terminating %c character",
1882 if (CPP_OPTION (pfile
, user_literals
))
1884 /* If a string format macro, say from inttypes.h, is placed touching
1885 a string literal it could be parsed as a C++11 user-defined string
1886 literal thus breaking the program.
1887 Try to identify macros with is_macro. A warning is issued. */
1888 if (is_macro (pfile
, cur
))
1890 /* Raise a warning, but do not consume subsequent tokens. */
1891 if (CPP_OPTION (pfile
, warn_literal_suffix
) && !pfile
->state
.skipping
)
1892 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1894 "invalid suffix on literal; C++11 requires "
1895 "a space between literal and string macro");
1897 /* Grab user defined literal suffix. */
1898 else if (ISIDST (*cur
))
1900 type
= cpp_userdef_char_add_type (type
);
1901 type
= cpp_userdef_string_add_type (type
);
1904 while (ISIDNUM (*cur
))
1909 pfile
->buffer
->cur
= cur
;
1910 create_literal (pfile
, token
, base
, cur
- base
, type
);
1913 /* Return the comment table. The client may not make any assumption
1914 about the ordering of the table. */
1916 cpp_get_comments (cpp_reader
*pfile
)
1918 return &pfile
->comments
;
1921 /* Append a comment to the end of the comment table. */
1923 store_comment (cpp_reader
*pfile
, cpp_token
*token
)
1927 if (pfile
->comments
.allocated
== 0)
1929 pfile
->comments
.allocated
= 256;
1930 pfile
->comments
.entries
= (cpp_comment
*) xmalloc
1931 (pfile
->comments
.allocated
* sizeof (cpp_comment
));
1934 if (pfile
->comments
.count
== pfile
->comments
.allocated
)
1936 pfile
->comments
.allocated
*= 2;
1937 pfile
->comments
.entries
= (cpp_comment
*) xrealloc
1938 (pfile
->comments
.entries
,
1939 pfile
->comments
.allocated
* sizeof (cpp_comment
));
1942 len
= token
->val
.str
.len
;
1944 /* Copy comment. Note, token may not be NULL terminated. */
1945 pfile
->comments
.entries
[pfile
->comments
.count
].comment
=
1946 (char *) xmalloc (sizeof (char) * (len
+ 1));
1947 memcpy (pfile
->comments
.entries
[pfile
->comments
.count
].comment
,
1948 token
->val
.str
.text
, len
);
1949 pfile
->comments
.entries
[pfile
->comments
.count
].comment
[len
] = '\0';
1951 /* Set source location. */
1952 pfile
->comments
.entries
[pfile
->comments
.count
].sloc
= token
->src_loc
;
1954 /* Increment the count of entries in the comment table. */
1955 pfile
->comments
.count
++;
1958 /* The stored comment includes the comment start and any terminator. */
1960 save_comment (cpp_reader
*pfile
, cpp_token
*token
, const unsigned char *from
,
1963 unsigned char *buffer
;
1964 unsigned int len
, clen
, i
;
1966 len
= pfile
->buffer
->cur
- from
+ 1; /* + 1 for the initial '/'. */
1968 /* C++ comments probably (not definitely) have moved past a new
1969 line, which we don't want to save in the comment. */
1970 if (is_vspace (pfile
->buffer
->cur
[-1]))
1973 /* If we are currently in a directive or in argument parsing, then
1974 we need to store all C++ comments as C comments internally, and
1975 so we need to allocate a little extra space in that case.
1977 Note that the only time we encounter a directive here is
1978 when we are saving comments in a "#define". */
1979 clen
= ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
)
1980 && type
== '/') ? len
+ 2 : len
;
1982 buffer
= _cpp_unaligned_alloc (pfile
, clen
);
1984 token
->type
= CPP_COMMENT
;
1985 token
->val
.str
.len
= clen
;
1986 token
->val
.str
.text
= buffer
;
1989 memcpy (buffer
+ 1, from
, len
- 1);
1991 /* Finish conversion to a C comment, if necessary. */
1992 if ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
) && type
== '/')
1995 buffer
[clen
- 2] = '*';
1996 buffer
[clen
- 1] = '/';
1997 /* As there can be in a C++ comments illegal sequences for C comments
1998 we need to filter them out. */
1999 for (i
= 2; i
< (clen
- 2); i
++)
2000 if (buffer
[i
] == '/' && (buffer
[i
- 1] == '*' || buffer
[i
+ 1] == '*'))
2004 /* Finally store this comment for use by clients of libcpp. */
2005 store_comment (pfile
, token
);
2008 /* Allocate COUNT tokens for RUN. */
2010 _cpp_init_tokenrun (tokenrun
*run
, unsigned int count
)
2012 run
->base
= XNEWVEC (cpp_token
, count
);
2013 run
->limit
= run
->base
+ count
;
2017 /* Returns the next tokenrun, or creates one if there is none. */
2019 next_tokenrun (tokenrun
*run
)
2021 if (run
->next
== NULL
)
2023 run
->next
= XNEW (tokenrun
);
2024 run
->next
->prev
= run
;
2025 _cpp_init_tokenrun (run
->next
, 250);
2031 /* Return the number of not yet processed token in a given
2034 _cpp_remaining_tokens_num_in_context (cpp_context
*context
)
2036 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
2037 return (LAST (context
).token
- FIRST (context
).token
);
2038 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
2039 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
2040 return (LAST (context
).ptoken
- FIRST (context
).ptoken
);
2045 /* Returns the token present at index INDEX in a given context. If
2046 INDEX is zero, the next token to be processed is returned. */
2047 static const cpp_token
*
2048 _cpp_token_from_context_at (cpp_context
*context
, int index
)
2050 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
2051 return &(FIRST (context
).token
[index
]);
2052 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
2053 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
2054 return FIRST (context
).ptoken
[index
];
2059 /* Look ahead in the input stream. */
2061 cpp_peek_token (cpp_reader
*pfile
, int index
)
2063 cpp_context
*context
= pfile
->context
;
2064 const cpp_token
*peektok
;
2067 /* First, scan through any pending cpp_context objects. */
2068 while (context
->prev
)
2070 ptrdiff_t sz
= _cpp_remaining_tokens_num_in_context (context
);
2072 if (index
< (int) sz
)
2073 return _cpp_token_from_context_at (context
, index
);
2075 context
= context
->prev
;
2078 /* We will have to read some new tokens after all (and do so
2079 without invalidating preceding tokens). */
2081 pfile
->keep_tokens
++;
2085 peektok
= _cpp_lex_token (pfile
);
2086 if (peektok
->type
== CPP_EOF
)
2091 _cpp_backup_tokens_direct (pfile
, count
+ 1);
2092 pfile
->keep_tokens
--;
2097 /* Allocate a single token that is invalidated at the same time as the
2098 rest of the tokens on the line. Has its line and col set to the
2099 same as the last lexed token, so that diagnostics appear in the
2102 _cpp_temp_token (cpp_reader
*pfile
)
2104 cpp_token
*old
, *result
;
2105 ptrdiff_t sz
= pfile
->cur_run
->limit
- pfile
->cur_token
;
2106 ptrdiff_t la
= (ptrdiff_t) pfile
->lookaheads
;
2108 old
= pfile
->cur_token
- 1;
2109 /* Any pre-existing lookaheads must not be clobbered. */
2114 tokenrun
*next
= next_tokenrun (pfile
->cur_run
);
2117 memmove (next
->base
+ 1, next
->base
,
2118 (la
- sz
) * sizeof (cpp_token
));
2120 next
->base
[0] = pfile
->cur_run
->limit
[-1];
2124 memmove (pfile
->cur_token
+ 1, pfile
->cur_token
,
2125 MIN (la
, sz
- 1) * sizeof (cpp_token
));
2128 if (!sz
&& pfile
->cur_token
== pfile
->cur_run
->limit
)
2130 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2131 pfile
->cur_token
= pfile
->cur_run
->base
;
2134 result
= pfile
->cur_token
++;
2135 result
->src_loc
= old
->src_loc
;
2139 /* Lex a token into RESULT (external interface). Takes care of issues
2140 like directive handling, token lookahead, multiple include
2141 optimization and skipping. */
2143 _cpp_lex_token (cpp_reader
*pfile
)
2149 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
2151 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2152 pfile
->cur_token
= pfile
->cur_run
->base
;
2154 /* We assume that the current token is somewhere in the current
2156 if (pfile
->cur_token
< pfile
->cur_run
->base
2157 || pfile
->cur_token
>= pfile
->cur_run
->limit
)
2160 if (pfile
->lookaheads
)
2162 pfile
->lookaheads
--;
2163 result
= pfile
->cur_token
++;
2166 result
= _cpp_lex_direct (pfile
);
2168 if (result
->flags
& BOL
)
2170 /* Is this a directive. If _cpp_handle_directive returns
2171 false, it is an assembler #. */
2172 if (result
->type
== CPP_HASH
2173 /* 6.10.3 p 11: Directives in a list of macro arguments
2174 gives undefined behavior. This implementation
2175 handles the directive as normal. */
2176 && pfile
->state
.parsing_args
!= 1)
2178 if (_cpp_handle_directive (pfile
, result
->flags
& PREV_WHITE
))
2180 if (pfile
->directive_result
.type
== CPP_PADDING
)
2182 result
= &pfile
->directive_result
;
2185 else if (pfile
->state
.in_deferred_pragma
)
2186 result
= &pfile
->directive_result
;
2188 if (pfile
->cb
.line_change
&& !pfile
->state
.skipping
)
2189 pfile
->cb
.line_change (pfile
, result
, pfile
->state
.parsing_args
);
2192 /* We don't skip tokens in directives. */
2193 if (pfile
->state
.in_directive
|| pfile
->state
.in_deferred_pragma
)
2196 /* Outside a directive, invalidate controlling macros. At file
2197 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
2198 get here and MI optimization works. */
2199 pfile
->mi_valid
= false;
2201 if (!pfile
->state
.skipping
|| result
->type
== CPP_EOF
)
2208 /* Returns true if a fresh line has been loaded. */
2210 _cpp_get_fresh_line (cpp_reader
*pfile
)
2214 /* We can't get a new line until we leave the current directive. */
2215 if (pfile
->state
.in_directive
)
2220 cpp_buffer
*buffer
= pfile
->buffer
;
2222 if (!buffer
->need_line
)
2225 if (buffer
->next_line
< buffer
->rlimit
)
2227 _cpp_clean_line (pfile
);
2231 /* First, get out of parsing arguments state. */
2232 if (pfile
->state
.parsing_args
)
2235 /* End of buffer. Non-empty files should end in a newline. */
2236 if (buffer
->buf
!= buffer
->rlimit
2237 && buffer
->next_line
> buffer
->rlimit
2238 && !buffer
->from_stage3
)
2240 /* Clip to buffer size. */
2241 buffer
->next_line
= buffer
->rlimit
;
2244 return_at_eof
= buffer
->return_at_eof
;
2245 _cpp_pop_buffer (pfile
);
2246 if (pfile
->buffer
== NULL
|| return_at_eof
)
2251 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
2254 result->type = ELSE_TYPE; \
2255 if (*buffer->cur == CHAR) \
2256 buffer->cur++, result->type = THEN_TYPE; \
2260 /* Lex a token into pfile->cur_token, which is also incremented, to
2261 get diagnostics pointing to the correct location.
2263 Does not handle issues such as token lookahead, multiple-include
2264 optimization, directives, skipping etc. This function is only
2265 suitable for use by _cpp_lex_token, and in special cases like
2266 lex_expansion_token which doesn't care for any of these issues.
2268 When meeting a newline, returns CPP_EOF if parsing a directive,
2269 otherwise returns to the start of the token buffer if permissible.
2270 Returns the location of the lexed token. */
2272 _cpp_lex_direct (cpp_reader
*pfile
)
2276 const unsigned char *comment_start
;
2277 cpp_token
*result
= pfile
->cur_token
++;
2281 buffer
= pfile
->buffer
;
2282 if (buffer
->need_line
)
2284 if (pfile
->state
.in_deferred_pragma
)
2286 result
->type
= CPP_PRAGMA_EOL
;
2287 pfile
->state
.in_deferred_pragma
= false;
2288 if (!pfile
->state
.pragma_allow_expansion
)
2289 pfile
->state
.prevent_expansion
--;
2292 if (!_cpp_get_fresh_line (pfile
))
2294 result
->type
= CPP_EOF
;
2295 if (!pfile
->state
.in_directive
)
2297 /* Tell the compiler the line number of the EOF token. */
2298 result
->src_loc
= pfile
->line_table
->highest_line
;
2299 result
->flags
= BOL
;
2303 if (!pfile
->keep_tokens
)
2305 pfile
->cur_run
= &pfile
->base_run
;
2306 result
= pfile
->base_run
.base
;
2307 pfile
->cur_token
= result
+ 1;
2309 result
->flags
= BOL
;
2310 if (pfile
->state
.parsing_args
== 2)
2311 result
->flags
|= PREV_WHITE
;
2313 buffer
= pfile
->buffer
;
2315 result
->src_loc
= pfile
->line_table
->highest_line
;
2318 if (buffer
->cur
>= buffer
->notes
[buffer
->cur_note
].pos
2319 && !pfile
->overlaid_buffer
)
2321 _cpp_process_line_notes (pfile
, false);
2322 result
->src_loc
= pfile
->line_table
->highest_line
;
2326 if (pfile
->forced_token_location_p
)
2327 result
->src_loc
= *pfile
->forced_token_location_p
;
2329 result
->src_loc
= linemap_position_for_column (pfile
->line_table
,
2330 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2334 case ' ': case '\t': case '\f': case '\v': case '\0':
2335 result
->flags
|= PREV_WHITE
;
2336 skip_whitespace (pfile
, c
);
2340 if (buffer
->cur
< buffer
->rlimit
)
2341 CPP_INCREMENT_LINE (pfile
, 0);
2342 buffer
->need_line
= true;
2345 case '0': case '1': case '2': case '3': case '4':
2346 case '5': case '6': case '7': case '8': case '9':
2348 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2349 result
->type
= CPP_NUMBER
;
2350 lex_number (pfile
, &result
->val
.str
, &nst
);
2351 warn_about_normalization (pfile
, result
, &nst
);
2359 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2360 wide strings or raw strings. */
2361 if (c
== 'L' || CPP_OPTION (pfile
, rliterals
)
2362 || (c
!= 'R' && CPP_OPTION (pfile
, uliterals
)))
2364 if ((*buffer
->cur
== '\'' && c
!= 'R')
2365 || *buffer
->cur
== '"'
2366 || (*buffer
->cur
== 'R'
2368 && buffer
->cur
[1] == '"'
2369 && CPP_OPTION (pfile
, rliterals
))
2370 || (*buffer
->cur
== '8'
2372 && (buffer
->cur
[1] == '"'
2373 || (buffer
->cur
[1] == 'R' && buffer
->cur
[2] == '"'
2374 && CPP_OPTION (pfile
, rliterals
)))))
2376 lex_string (pfile
, result
, buffer
->cur
- 1);
2383 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2384 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2385 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2386 case 's': case 't': case 'v': case 'w': case 'x':
2388 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2389 case 'G': case 'H': case 'I': case 'J': case 'K':
2390 case 'M': case 'N': case 'O': case 'P': case 'Q':
2391 case 'S': case 'T': case 'V': case 'W': case 'X':
2393 result
->type
= CPP_NAME
;
2395 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2396 result
->val
.node
.node
= lex_identifier (pfile
, buffer
->cur
- 1, false,
2398 &result
->val
.node
.spelling
);
2399 warn_about_normalization (pfile
, result
, &nst
);
2402 /* Convert named operators to their proper types. */
2403 if (result
->val
.node
.node
->flags
& NODE_OPERATOR
)
2405 result
->flags
|= NAMED_OP
;
2406 result
->type
= (enum cpp_ttype
) result
->val
.node
.node
->directive_index
;
2412 lex_string (pfile
, result
, buffer
->cur
- 1);
2416 /* A potential block or line comment. */
2417 comment_start
= buffer
->cur
;
2422 if (_cpp_skip_block_comment (pfile
))
2423 cpp_error (pfile
, CPP_DL_ERROR
, "unterminated comment");
2425 else if (c
== '/' && ! CPP_OPTION (pfile
, traditional
))
2427 /* Don't warn for system headers. */
2428 if (cpp_in_system_header (pfile
))
2430 /* Warn about comments if pedantically GNUC89, and not
2431 in system headers. */
2432 else if (CPP_OPTION (pfile
, lang
) == CLK_GNUC89
2433 && CPP_PEDANTIC (pfile
)
2434 && ! buffer
->warned_cplusplus_comments
)
2436 cpp_error (pfile
, CPP_DL_PEDWARN
,
2437 "C++ style comments are not allowed in ISO C90");
2438 cpp_error (pfile
, CPP_DL_PEDWARN
,
2439 "(this will be reported only once per input file)");
2440 buffer
->warned_cplusplus_comments
= 1;
2442 /* Or if specifically desired via -Wc90-c99-compat. */
2443 else if (CPP_OPTION (pfile
, cpp_warn_c90_c99_compat
) > 0
2444 && ! CPP_OPTION (pfile
, cplusplus
)
2445 && ! buffer
->warned_cplusplus_comments
)
2447 cpp_error (pfile
, CPP_DL_WARNING
,
2448 "C++ style comments are incompatible with C90");
2449 cpp_error (pfile
, CPP_DL_WARNING
,
2450 "(this will be reported only once per input file)");
2451 buffer
->warned_cplusplus_comments
= 1;
2453 /* In C89/C94, C++ style comments are forbidden. */
2454 else if ((CPP_OPTION (pfile
, lang
) == CLK_STDC89
2455 || CPP_OPTION (pfile
, lang
) == CLK_STDC94
))
2457 /* But don't be confused about valid code such as
2458 - // immediately followed by *,
2459 - // in a preprocessing directive,
2460 - // in an #if 0 block. */
2461 if (buffer
->cur
[1] == '*'
2462 || pfile
->state
.in_directive
2463 || pfile
->state
.skipping
)
2465 result
->type
= CPP_DIV
;
2468 else if (! buffer
->warned_cplusplus_comments
)
2470 cpp_error (pfile
, CPP_DL_ERROR
,
2471 "C++ style comments are not allowed in ISO C90");
2472 cpp_error (pfile
, CPP_DL_ERROR
,
2473 "(this will be reported only once per input "
2475 buffer
->warned_cplusplus_comments
= 1;
2478 if (skip_line_comment (pfile
) && CPP_OPTION (pfile
, warn_comments
))
2479 cpp_warning (pfile
, CPP_W_COMMENTS
, "multi-line comment");
2484 result
->type
= CPP_DIV_EQ
;
2489 result
->type
= CPP_DIV
;
2493 if (!pfile
->state
.save_comments
)
2495 result
->flags
|= PREV_WHITE
;
2496 goto update_tokens_line
;
2499 /* Save the comment as a token in its own right. */
2500 save_comment (pfile
, result
, comment_start
, c
);
2504 if (pfile
->state
.angled_headers
)
2506 lex_string (pfile
, result
, buffer
->cur
- 1);
2507 if (result
->type
!= CPP_LESS
)
2511 result
->type
= CPP_LESS
;
2512 if (*buffer
->cur
== '=')
2513 buffer
->cur
++, result
->type
= CPP_LESS_EQ
;
2514 else if (*buffer
->cur
== '<')
2517 IF_NEXT_IS ('=', CPP_LSHIFT_EQ
, CPP_LSHIFT
);
2519 else if (CPP_OPTION (pfile
, digraphs
))
2521 if (*buffer
->cur
== ':')
2523 /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next
2524 three characters are <:: and the subsequent character
2525 is neither : nor >, the < is treated as a preprocessor
2526 token by itself". */
2527 if (CPP_OPTION (pfile
, cplusplus
)
2528 && CPP_OPTION (pfile
, lang
) != CLK_CXX98
2529 && CPP_OPTION (pfile
, lang
) != CLK_GNUCXX
2530 && buffer
->cur
[1] == ':'
2531 && buffer
->cur
[2] != ':' && buffer
->cur
[2] != '>')
2535 result
->flags
|= DIGRAPH
;
2536 result
->type
= CPP_OPEN_SQUARE
;
2538 else if (*buffer
->cur
== '%')
2541 result
->flags
|= DIGRAPH
;
2542 result
->type
= CPP_OPEN_BRACE
;
2548 result
->type
= CPP_GREATER
;
2549 if (*buffer
->cur
== '=')
2550 buffer
->cur
++, result
->type
= CPP_GREATER_EQ
;
2551 else if (*buffer
->cur
== '>')
2554 IF_NEXT_IS ('=', CPP_RSHIFT_EQ
, CPP_RSHIFT
);
2559 result
->type
= CPP_MOD
;
2560 if (*buffer
->cur
== '=')
2561 buffer
->cur
++, result
->type
= CPP_MOD_EQ
;
2562 else if (CPP_OPTION (pfile
, digraphs
))
2564 if (*buffer
->cur
== ':')
2567 result
->flags
|= DIGRAPH
;
2568 result
->type
= CPP_HASH
;
2569 if (*buffer
->cur
== '%' && buffer
->cur
[1] == ':')
2570 buffer
->cur
+= 2, result
->type
= CPP_PASTE
, result
->val
.token_no
= 0;
2572 else if (*buffer
->cur
== '>')
2575 result
->flags
|= DIGRAPH
;
2576 result
->type
= CPP_CLOSE_BRACE
;
2582 result
->type
= CPP_DOT
;
2583 if (ISDIGIT (*buffer
->cur
))
2585 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2586 result
->type
= CPP_NUMBER
;
2587 lex_number (pfile
, &result
->val
.str
, &nst
);
2588 warn_about_normalization (pfile
, result
, &nst
);
2590 else if (*buffer
->cur
== '.' && buffer
->cur
[1] == '.')
2591 buffer
->cur
+= 2, result
->type
= CPP_ELLIPSIS
;
2592 else if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2593 buffer
->cur
++, result
->type
= CPP_DOT_STAR
;
2597 result
->type
= CPP_PLUS
;
2598 if (*buffer
->cur
== '+')
2599 buffer
->cur
++, result
->type
= CPP_PLUS_PLUS
;
2600 else if (*buffer
->cur
== '=')
2601 buffer
->cur
++, result
->type
= CPP_PLUS_EQ
;
2605 result
->type
= CPP_MINUS
;
2606 if (*buffer
->cur
== '>')
2609 result
->type
= CPP_DEREF
;
2610 if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2611 buffer
->cur
++, result
->type
= CPP_DEREF_STAR
;
2613 else if (*buffer
->cur
== '-')
2614 buffer
->cur
++, result
->type
= CPP_MINUS_MINUS
;
2615 else if (*buffer
->cur
== '=')
2616 buffer
->cur
++, result
->type
= CPP_MINUS_EQ
;
2620 result
->type
= CPP_AND
;
2621 if (*buffer
->cur
== '&')
2622 buffer
->cur
++, result
->type
= CPP_AND_AND
;
2623 else if (*buffer
->cur
== '=')
2624 buffer
->cur
++, result
->type
= CPP_AND_EQ
;
2628 result
->type
= CPP_OR
;
2629 if (*buffer
->cur
== '|')
2630 buffer
->cur
++, result
->type
= CPP_OR_OR
;
2631 else if (*buffer
->cur
== '=')
2632 buffer
->cur
++, result
->type
= CPP_OR_EQ
;
2636 result
->type
= CPP_COLON
;
2637 if (*buffer
->cur
== ':' && CPP_OPTION (pfile
, cplusplus
))
2638 buffer
->cur
++, result
->type
= CPP_SCOPE
;
2639 else if (*buffer
->cur
== '>' && CPP_OPTION (pfile
, digraphs
))
2642 result
->flags
|= DIGRAPH
;
2643 result
->type
= CPP_CLOSE_SQUARE
;
2647 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ
, CPP_MULT
); break;
2648 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ
, CPP_EQ
); break;
2649 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ
, CPP_NOT
); break;
2650 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ
, CPP_XOR
); break;
2651 case '#': IF_NEXT_IS ('#', CPP_PASTE
, CPP_HASH
); result
->val
.token_no
= 0; break;
2653 case '?': result
->type
= CPP_QUERY
; break;
2654 case '~': result
->type
= CPP_COMPL
; break;
2655 case ',': result
->type
= CPP_COMMA
; break;
2656 case '(': result
->type
= CPP_OPEN_PAREN
; break;
2657 case ')': result
->type
= CPP_CLOSE_PAREN
; break;
2658 case '[': result
->type
= CPP_OPEN_SQUARE
; break;
2659 case ']': result
->type
= CPP_CLOSE_SQUARE
; break;
2660 case '{': result
->type
= CPP_OPEN_BRACE
; break;
2661 case '}': result
->type
= CPP_CLOSE_BRACE
; break;
2662 case ';': result
->type
= CPP_SEMICOLON
; break;
2664 /* @ is a punctuator in Objective-C. */
2665 case '@': result
->type
= CPP_ATSIGN
; break;
2670 const uchar
*base
= --buffer
->cur
;
2671 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2673 if (forms_identifier_p (pfile
, true, &nst
))
2675 result
->type
= CPP_NAME
;
2676 result
->val
.node
.node
= lex_identifier (pfile
, base
, true, &nst
,
2677 &result
->val
.node
.spelling
);
2678 warn_about_normalization (pfile
, result
, &nst
);
2685 create_literal (pfile
, result
, buffer
->cur
- 1, 1, CPP_OTHER
);
2692 /* An upper bound on the number of bytes needed to spell TOKEN.
2693 Does not include preceding whitespace. */
2695 cpp_token_len (const cpp_token
*token
)
2699 switch (TOKEN_SPELL (token
))
2701 default: len
= 6; break;
2702 case SPELL_LITERAL
: len
= token
->val
.str
.len
; break;
2703 case SPELL_IDENT
: len
= NODE_LEN (token
->val
.node
.node
) * 10; break;
2709 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2710 Return the number of bytes read out of NAME. (There are always
2711 10 bytes written to BUFFER.) */
2714 utf8_to_ucn (unsigned char *buffer
, const unsigned char *name
)
2720 unsigned long utf32
;
2722 /* Compute the length of the UTF-8 sequence. */
2723 for (t
= *name
; t
& 0x80; t
<<= 1)
2726 utf32
= *name
& (0x7F >> ucn_len
);
2727 for (ucn_len_c
= 1; ucn_len_c
< ucn_len
; ucn_len_c
++)
2729 utf32
= (utf32
<< 6) | (*++name
& 0x3F);
2731 /* Ill-formed UTF-8. */
2732 if ((*name
& ~0x3F) != 0x80)
2738 for (j
= 7; j
>= 0; j
--)
2739 *buffer
++ = "0123456789abcdef"[(utf32
>> (4 * j
)) & 0xF];
2743 /* Given a token TYPE corresponding to a digraph, return a pointer to
2744 the spelling of the digraph. */
2745 static const unsigned char *
2746 cpp_digraph2name (enum cpp_ttype type
)
2748 return digraph_spellings
[(int) type
- (int) CPP_FIRST_DIGRAPH
];
2751 /* Write the spelling of an identifier IDENT, using UCNs, to BUFFER.
2752 The buffer must already contain the enough space to hold the
2753 token's spelling. Returns a pointer to the character after the
2754 last character written. */
2756 _cpp_spell_ident_ucns (unsigned char *buffer
, cpp_hashnode
*ident
)
2759 const unsigned char *name
= NODE_NAME (ident
);
2761 for (i
= 0; i
< NODE_LEN (ident
); i
++)
2762 if (name
[i
] & ~0x7F)
2764 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2768 *buffer
++ = name
[i
];
2773 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
2774 already contain the enough space to hold the token's spelling.
2775 Returns a pointer to the character after the last character written.
2776 FORSTRING is true if this is to be the spelling after translation
2777 phase 1 (with the original spelling of extended identifiers), false
2778 if extended identifiers should always be written using UCNs (there is
2779 no option for always writing them in the internal UTF-8 form).
2780 FIXME: Would be nice if we didn't need the PFILE argument. */
2782 cpp_spell_token (cpp_reader
*pfile
, const cpp_token
*token
,
2783 unsigned char *buffer
, bool forstring
)
2785 switch (TOKEN_SPELL (token
))
2787 case SPELL_OPERATOR
:
2789 const unsigned char *spelling
;
2792 if (token
->flags
& DIGRAPH
)
2793 spelling
= cpp_digraph2name (token
->type
);
2794 else if (token
->flags
& NAMED_OP
)
2797 spelling
= TOKEN_NAME (token
);
2799 while ((c
= *spelling
++) != '\0')
2808 memcpy (buffer
, NODE_NAME (token
->val
.node
.spelling
),
2809 NODE_LEN (token
->val
.node
.spelling
));
2810 buffer
+= NODE_LEN (token
->val
.node
.spelling
);
2813 buffer
= _cpp_spell_ident_ucns (buffer
, token
->val
.node
.node
);
2817 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
2818 buffer
+= token
->val
.str
.len
;
2822 cpp_error (pfile
, CPP_DL_ICE
,
2823 "unspellable token %s", TOKEN_NAME (token
));
2830 /* Returns TOKEN spelt as a null-terminated string. The string is
2831 freed when the reader is destroyed. Useful for diagnostics. */
2833 cpp_token_as_text (cpp_reader
*pfile
, const cpp_token
*token
)
2835 unsigned int len
= cpp_token_len (token
) + 1;
2836 unsigned char *start
= _cpp_unaligned_alloc (pfile
, len
), *end
;
2838 end
= cpp_spell_token (pfile
, token
, start
, false);
2844 /* Returns a pointer to a string which spells the token defined by
2845 TYPE and FLAGS. Used by C front ends, which really should move to
2846 using cpp_token_as_text. */
2848 cpp_type2name (enum cpp_ttype type
, unsigned char flags
)
2850 if (flags
& DIGRAPH
)
2851 return (const char *) cpp_digraph2name (type
);
2852 else if (flags
& NAMED_OP
)
2853 return cpp_named_operator2name (type
);
2855 return (const char *) token_spellings
[type
].name
;
2858 /* Writes the spelling of token to FP, without any preceding space.
2859 Separated from cpp_spell_token for efficiency - to avoid stdio
2860 double-buffering. */
2862 cpp_output_token (const cpp_token
*token
, FILE *fp
)
2864 switch (TOKEN_SPELL (token
))
2866 case SPELL_OPERATOR
:
2868 const unsigned char *spelling
;
2871 if (token
->flags
& DIGRAPH
)
2872 spelling
= cpp_digraph2name (token
->type
);
2873 else if (token
->flags
& NAMED_OP
)
2876 spelling
= TOKEN_NAME (token
);
2881 while ((c
= *++spelling
) != '\0');
2889 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
2891 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
2892 if (name
[i
] & ~0x7F)
2894 unsigned char buffer
[10];
2895 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2896 fwrite (buffer
, 1, 10, fp
);
2899 fputc (NODE_NAME (token
->val
.node
.node
)[i
], fp
);
2904 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
2908 /* An error, most probably. */
2913 /* Compare two tokens. */
2915 _cpp_equiv_tokens (const cpp_token
*a
, const cpp_token
*b
)
2917 if (a
->type
== b
->type
&& a
->flags
== b
->flags
)
2918 switch (TOKEN_SPELL (a
))
2920 default: /* Keep compiler happy. */
2921 case SPELL_OPERATOR
:
2922 /* token_no is used to track where multiple consecutive ##
2923 tokens were originally located. */
2924 return (a
->type
!= CPP_PASTE
|| a
->val
.token_no
== b
->val
.token_no
);
2926 return (a
->type
!= CPP_MACRO_ARG
2927 || (a
->val
.macro_arg
.arg_no
== b
->val
.macro_arg
.arg_no
2928 && a
->val
.macro_arg
.spelling
== b
->val
.macro_arg
.spelling
));
2930 return (a
->val
.node
.node
== b
->val
.node
.node
2931 && a
->val
.node
.spelling
== b
->val
.node
.spelling
);
2933 return (a
->val
.str
.len
== b
->val
.str
.len
2934 && !memcmp (a
->val
.str
.text
, b
->val
.str
.text
,
2941 /* Returns nonzero if a space should be inserted to avoid an
2942 accidental token paste for output. For simplicity, it is
2943 conservative, and occasionally advises a space where one is not
2944 needed, e.g. "." and ".2". */
2946 cpp_avoid_paste (cpp_reader
*pfile
, const cpp_token
*token1
,
2947 const cpp_token
*token2
)
2949 enum cpp_ttype a
= token1
->type
, b
= token2
->type
;
2952 if (token1
->flags
& NAMED_OP
)
2954 if (token2
->flags
& NAMED_OP
)
2958 if (token2
->flags
& DIGRAPH
)
2959 c
= digraph_spellings
[(int) b
- (int) CPP_FIRST_DIGRAPH
][0];
2960 else if (token_spellings
[b
].category
== SPELL_OPERATOR
)
2961 c
= token_spellings
[b
].name
[0];
2963 /* Quickly get everything that can paste with an '='. */
2964 if ((int) a
<= (int) CPP_LAST_EQ
&& c
== '=')
2969 case CPP_GREATER
: return c
== '>';
2970 case CPP_LESS
: return c
== '<' || c
== '%' || c
== ':';
2971 case CPP_PLUS
: return c
== '+';
2972 case CPP_MINUS
: return c
== '-' || c
== '>';
2973 case CPP_DIV
: return c
== '/' || c
== '*'; /* Comments. */
2974 case CPP_MOD
: return c
== ':' || c
== '>';
2975 case CPP_AND
: return c
== '&';
2976 case CPP_OR
: return c
== '|';
2977 case CPP_COLON
: return c
== ':' || c
== '>';
2978 case CPP_DEREF
: return c
== '*';
2979 case CPP_DOT
: return c
== '.' || c
== '%' || b
== CPP_NUMBER
;
2980 case CPP_HASH
: return c
== '#' || c
== '%'; /* Digraph form. */
2981 case CPP_NAME
: return ((b
== CPP_NUMBER
2982 && name_p (pfile
, &token2
->val
.str
))
2984 || b
== CPP_CHAR
|| b
== CPP_STRING
); /* L */
2985 case CPP_NUMBER
: return (b
== CPP_NUMBER
|| b
== CPP_NAME
2986 || c
== '.' || c
== '+' || c
== '-');
2988 case CPP_OTHER
: return ((token1
->val
.str
.text
[0] == '\\'
2990 || (CPP_OPTION (pfile
, objc
)
2991 && token1
->val
.str
.text
[0] == '@'
2992 && (b
== CPP_NAME
|| b
== CPP_STRING
)));
2995 case CPP_UTF8STRING
:
2997 case CPP_STRING32
: return (CPP_OPTION (pfile
, user_literals
)
2999 || (TOKEN_SPELL (token2
) == SPELL_LITERAL
3000 && ISIDST (token2
->val
.str
.text
[0]))));
3008 /* Output all the remaining tokens on the current line, and a newline
3009 character, to FP. Leading whitespace is removed. If there are
3010 macros, special token padding is not performed. */
3012 cpp_output_line (cpp_reader
*pfile
, FILE *fp
)
3014 const cpp_token
*token
;
3016 token
= cpp_get_token (pfile
);
3017 while (token
->type
!= CPP_EOF
)
3019 cpp_output_token (token
, fp
);
3020 token
= cpp_get_token (pfile
);
3021 if (token
->flags
& PREV_WHITE
)
3028 /* Return a string representation of all the remaining tokens on the
3029 current line. The result is allocated using xmalloc and must be
3030 freed by the caller. */
3032 cpp_output_line_to_string (cpp_reader
*pfile
, const unsigned char *dir_name
)
3034 const cpp_token
*token
;
3035 unsigned int out
= dir_name
? ustrlen (dir_name
) : 0;
3036 unsigned int alloced
= 120 + out
;
3037 unsigned char *result
= (unsigned char *) xmalloc (alloced
);
3039 /* If DIR_NAME is empty, there are no initial contents. */
3042 sprintf ((char *) result
, "#%s ", dir_name
);
3046 token
= cpp_get_token (pfile
);
3047 while (token
->type
!= CPP_EOF
)
3049 unsigned char *last
;
3050 /* Include room for a possible space and the terminating nul. */
3051 unsigned int len
= cpp_token_len (token
) + 2;
3053 if (out
+ len
> alloced
)
3056 if (out
+ len
> alloced
)
3057 alloced
= out
+ len
;
3058 result
= (unsigned char *) xrealloc (result
, alloced
);
3061 last
= cpp_spell_token (pfile
, token
, &result
[out
], 0);
3062 out
= last
- result
;
3064 token
= cpp_get_token (pfile
);
3065 if (token
->flags
& PREV_WHITE
)
3066 result
[out
++] = ' ';
3073 /* Memory buffers. Changing these three constants can have a dramatic
3074 effect on performance. The values here are reasonable defaults,
3075 but might be tuned. If you adjust them, be sure to test across a
3076 range of uses of cpplib, including heavy nested function-like macro
3077 expansion. Also check the change in peak memory usage (NJAMD is a
3078 good tool for this). */
3079 #define MIN_BUFF_SIZE 8000
3080 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
3081 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
3082 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
3084 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
3085 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
3088 /* Create a new allocation buffer. Place the control block at the end
3089 of the buffer, so that buffer overflows will cause immediate chaos. */
3091 new_buff (size_t len
)
3094 unsigned char *base
;
3096 if (len
< MIN_BUFF_SIZE
)
3097 len
= MIN_BUFF_SIZE
;
3098 len
= CPP_ALIGN (len
);
3100 #ifdef ENABLE_VALGRIND_CHECKING
3101 /* Valgrind warns about uses of interior pointers, so put _cpp_buff
3103 size_t slen
= CPP_ALIGN2 (sizeof (_cpp_buff
), 2 * DEFAULT_ALIGNMENT
);
3104 base
= XNEWVEC (unsigned char, len
+ slen
);
3105 result
= (_cpp_buff
*) base
;
3108 base
= XNEWVEC (unsigned char, len
+ sizeof (_cpp_buff
));
3109 result
= (_cpp_buff
*) (base
+ len
);
3111 result
->base
= base
;
3113 result
->limit
= base
+ len
;
3114 result
->next
= NULL
;
3118 /* Place a chain of unwanted allocation buffers on the free list. */
3120 _cpp_release_buff (cpp_reader
*pfile
, _cpp_buff
*buff
)
3122 _cpp_buff
*end
= buff
;
3126 end
->next
= pfile
->free_buffs
;
3127 pfile
->free_buffs
= buff
;
3130 /* Return a free buffer of size at least MIN_SIZE. */
3132 _cpp_get_buff (cpp_reader
*pfile
, size_t min_size
)
3134 _cpp_buff
*result
, **p
;
3136 for (p
= &pfile
->free_buffs
;; p
= &(*p
)->next
)
3141 return new_buff (min_size
);
3143 size
= result
->limit
- result
->base
;
3144 /* Return a buffer that's big enough, but don't waste one that's
3146 if (size
>= min_size
&& size
<= BUFF_SIZE_UPPER_BOUND (min_size
))
3151 result
->next
= NULL
;
3152 result
->cur
= result
->base
;
3156 /* Creates a new buffer with enough space to hold the uncommitted
3157 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
3158 the excess bytes to the new buffer. Chains the new buffer after
3159 BUFF, and returns the new buffer. */
3161 _cpp_append_extend_buff (cpp_reader
*pfile
, _cpp_buff
*buff
, size_t min_extra
)
3163 size_t size
= EXTENDED_BUFF_SIZE (buff
, min_extra
);
3164 _cpp_buff
*new_buff
= _cpp_get_buff (pfile
, size
);
3166 buff
->next
= new_buff
;
3167 memcpy (new_buff
->base
, buff
->cur
, BUFF_ROOM (buff
));
3171 /* Creates a new buffer with enough space to hold the uncommitted
3172 remaining bytes of the buffer pointed to by BUFF, and at least
3173 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
3174 Chains the new buffer before the buffer pointed to by BUFF, and
3175 updates the pointer to point to the new buffer. */
3177 _cpp_extend_buff (cpp_reader
*pfile
, _cpp_buff
**pbuff
, size_t min_extra
)
3179 _cpp_buff
*new_buff
, *old_buff
= *pbuff
;
3180 size_t size
= EXTENDED_BUFF_SIZE (old_buff
, min_extra
);
3182 new_buff
= _cpp_get_buff (pfile
, size
);
3183 memcpy (new_buff
->base
, old_buff
->cur
, BUFF_ROOM (old_buff
));
3184 new_buff
->next
= old_buff
;
3188 /* Free a chain of buffers starting at BUFF. */
3190 _cpp_free_buff (_cpp_buff
*buff
)
3194 for (; buff
; buff
= next
)
3197 #ifdef ENABLE_VALGRIND_CHECKING
3205 /* Allocate permanent, unaligned storage of length LEN. */
3207 _cpp_unaligned_alloc (cpp_reader
*pfile
, size_t len
)
3209 _cpp_buff
*buff
= pfile
->u_buff
;
3210 unsigned char *result
= buff
->cur
;
3212 if (len
> (size_t) (buff
->limit
- result
))
3214 buff
= _cpp_get_buff (pfile
, len
);
3215 buff
->next
= pfile
->u_buff
;
3216 pfile
->u_buff
= buff
;
3220 buff
->cur
= result
+ len
;
3224 /* Allocate permanent, unaligned storage of length LEN from a_buff.
3225 That buffer is used for growing allocations when saving macro
3226 replacement lists in a #define, and when parsing an answer to an
3227 assertion in #assert, #unassert or #if (and therefore possibly
3228 whilst expanding macros). It therefore must not be used by any
3229 code that they might call: specifically the lexer and the guts of
3232 All existing other uses clearly fit this restriction: storing
3233 registered pragmas during initialization. */
3235 _cpp_aligned_alloc (cpp_reader
*pfile
, size_t len
)
3237 _cpp_buff
*buff
= pfile
->a_buff
;
3238 unsigned char *result
= buff
->cur
;
3240 if (len
> (size_t) (buff
->limit
- result
))
3242 buff
= _cpp_get_buff (pfile
, len
);
3243 buff
->next
= pfile
->a_buff
;
3244 pfile
->a_buff
= buff
;
3248 buff
->cur
= result
+ len
;
3252 /* Say which field of TOK is in use. */
3254 enum cpp_token_fld_kind
3255 cpp_token_val_index (const cpp_token
*tok
)
3257 switch (TOKEN_SPELL (tok
))
3260 return CPP_TOKEN_FLD_NODE
;
3262 return CPP_TOKEN_FLD_STR
;
3263 case SPELL_OPERATOR
:
3264 if (tok
->type
== CPP_PASTE
)
3265 return CPP_TOKEN_FLD_TOKEN_NO
;
3267 return CPP_TOKEN_FLD_NONE
;
3269 if (tok
->type
== CPP_MACRO_ARG
)
3270 return CPP_TOKEN_FLD_ARG_NO
;
3271 else if (tok
->type
== CPP_PADDING
)
3272 return CPP_TOKEN_FLD_SOURCE
;
3273 else if (tok
->type
== CPP_PRAGMA
)
3274 return CPP_TOKEN_FLD_PRAGMA
;
3275 /* else fall through */
3277 return CPP_TOKEN_FLD_NONE
;
3281 /* All tokens lexed in R after calling this function will be forced to have
3282 their source_location the same as the location referenced by P, until
3283 cpp_stop_forcing_token_locations is called for R. */
3286 cpp_force_token_locations (cpp_reader
*r
, source_location
*p
)
3288 r
->forced_token_location_p
= p
;
3291 /* Go back to assigning locations naturally for lexed tokens. */
3294 cpp_stop_forcing_token_locations (cpp_reader
*r
)
3296 r
->forced_token_location_p
= NULL
;