1 /* CPP Library - lexical analysis.
2 Copyright (C) 2000-2013 Free Software Foundation, Inc.
3 Contributed by Per Bothner, 1994-95.
4 Based on CCCP program by Paul Rubin, June 1986
5 Adapted to ANSI C, Richard Stallman, Jan 1987
6 Broken out to separate file, Zack Weinberg, Mar 2000
8 This program is free software; you can redistribute it and/or modify it
9 under the terms of the GNU General Public License as published by the
10 Free Software Foundation; either version 3, or (at your option) any
13 This program is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with this program; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
37 enum spell_type category
;
38 const unsigned char *name
;
41 static const unsigned char *const digraph_spellings
[] =
42 { UC
"%:", UC
"%:%:", UC
"<:", UC
":>", UC
"<%", UC
"%>" };
44 #define OP(e, s) { SPELL_OPERATOR, UC s },
45 #define TK(e, s) { SPELL_ ## s, UC #e },
46 static const struct token_spelling token_spellings
[N_TTYPES
] = { TTYPE_TABLE
};
50 #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)
51 #define TOKEN_NAME(token) (token_spellings[(token)->type].name)
53 static void add_line_note (cpp_buffer
*, const uchar
*, unsigned int);
54 static int skip_line_comment (cpp_reader
*);
55 static void skip_whitespace (cpp_reader
*, cppchar_t
);
56 static void lex_string (cpp_reader
*, cpp_token
*, const uchar
*);
57 static void save_comment (cpp_reader
*, cpp_token
*, const uchar
*, cppchar_t
);
58 static void store_comment (cpp_reader
*, cpp_token
*);
59 static void create_literal (cpp_reader
*, cpp_token
*, const uchar
*,
60 unsigned int, enum cpp_ttype
);
61 static bool warn_in_comment (cpp_reader
*, _cpp_line_note
*);
62 static int name_p (cpp_reader
*, const cpp_string
*);
63 static tokenrun
*next_tokenrun (tokenrun
*);
65 static _cpp_buff
*new_buff (size_t);
70 Compares, the token TOKEN to the NUL-terminated string STRING.
71 TOKEN must be a CPP_NAME. Returns 1 for equal, 0 for unequal. */
73 cpp_ideq (const cpp_token
*token
, const char *string
)
75 if (token
->type
!= CPP_NAME
)
78 return !ustrcmp (NODE_NAME (token
->val
.node
.node
), (const uchar
*) string
);
81 /* Record a note TYPE at byte POS into the current cleaned logical
84 add_line_note (cpp_buffer
*buffer
, const uchar
*pos
, unsigned int type
)
86 if (buffer
->notes_used
== buffer
->notes_cap
)
88 buffer
->notes_cap
= buffer
->notes_cap
* 2 + 200;
89 buffer
->notes
= XRESIZEVEC (_cpp_line_note
, buffer
->notes
,
93 buffer
->notes
[buffer
->notes_used
].pos
= pos
;
94 buffer
->notes
[buffer
->notes_used
].type
= type
;
99 /* Fast path to find line special characters using optimized character
100 scanning algorithms. Anything complicated falls back to the slow
101 path below. Since this loop is very hot it's worth doing these kinds
104 One of the paths through the ifdefs should provide
106 const uchar *search_line_fast (const uchar *s, const uchar *end);
108 Between S and END, search for \n, \r, \\, ?. Return a pointer to
111 Note that the last character of the buffer is *always* a newline,
112 as forced by _cpp_convert_input. This fact can be used to avoid
113 explicitly looking for the end of the buffer. */
115 /* Configure gives us an ifdef test. */
116 #ifndef WORDS_BIGENDIAN
117 #define WORDS_BIGENDIAN 0
120 /* We'd like the largest integer that fits into a register. There's nothing
121 in <stdint.h> that gives us that. For most hosts this is unsigned long,
122 but MS decided on an LLP64 model. Thankfully when building with GCC we
123 can get the "real" word size. */
125 typedef unsigned int word_type
__attribute__((__mode__(__word__
)));
127 typedef unsigned long word_type
;
130 /* The code below is only expecting sizes 4 or 8.
131 Die at compile-time if this expectation is violated. */
132 typedef char check_word_type_size
133 [(sizeof(word_type
) == 8 || sizeof(word_type
) == 4) * 2 - 1];
135 /* Return X with the first N bytes forced to values that won't match one
136 of the interesting characters. Note that NUL is not interesting. */
138 static inline word_type
139 acc_char_mask_misalign (word_type val
, unsigned int n
)
149 /* Return X replicated to all byte positions within WORD_TYPE. */
151 static inline word_type
152 acc_char_replicate (uchar x
)
156 ret
= (x
<< 24) | (x
<< 16) | (x
<< 8) | x
;
157 if (sizeof(word_type
) == 8)
158 ret
= (ret
<< 16 << 16) | ret
;
162 /* Return non-zero if some byte of VAL is (probably) C. */
164 static inline word_type
165 acc_char_cmp (word_type val
, word_type c
)
167 #if defined(__GNUC__) && defined(__alpha__)
168 /* We can get exact results using a compare-bytes instruction.
169 Get (val == c) via (0 >= (val ^ c)). */
170 return __builtin_alpha_cmpbge (0, val
^ c
);
172 word_type magic
= 0x7efefefeU
;
173 if (sizeof(word_type
) == 8)
174 magic
= (magic
<< 16 << 16) | 0xfefefefeU
;
178 return ((val
+ magic
) ^ ~val
) & ~magic
;
182 /* Given the result of acc_char_cmp is non-zero, return the index of
183 the found character. If this was a false positive, return -1. */
186 acc_char_index (word_type cmp ATTRIBUTE_UNUSED
,
187 word_type val ATTRIBUTE_UNUSED
)
189 #if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN
190 /* The cmpbge instruction sets *bits* of the result corresponding to
191 matches in the bytes with no false positives. */
192 return __builtin_ctzl (cmp
);
196 /* ??? It would be nice to force unrolling here,
197 and have all of these constants folded. */
198 for (i
= 0; i
< sizeof(word_type
); ++i
)
202 c
= (val
>> (sizeof(word_type
) - i
- 1) * 8) & 0xff;
204 c
= (val
>> i
* 8) & 0xff;
206 if (c
== '\n' || c
== '\r' || c
== '\\' || c
== '?')
214 /* A version of the fast scanner using bit fiddling techniques.
216 For 32-bit words, one would normally perform 16 comparisons and
217 16 branches. With this algorithm one performs 24 arithmetic
218 operations and one branch. Whether this is faster with a 32-bit
219 word size is going to be somewhat system dependent.
221 For 64-bit words, we eliminate twice the number of comparisons
222 and branches without increasing the number of arithmetic operations.
223 It's almost certainly going to be a win with 64-bit word size. */
225 static const uchar
* search_line_acc_char (const uchar
*, const uchar
*)
229 search_line_acc_char (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
231 const word_type repl_nl
= acc_char_replicate ('\n');
232 const word_type repl_cr
= acc_char_replicate ('\r');
233 const word_type repl_bs
= acc_char_replicate ('\\');
234 const word_type repl_qm
= acc_char_replicate ('?');
236 unsigned int misalign
;
240 /* Align the buffer. Mask out any bytes from before the beginning. */
241 p
= (word_type
*)((uintptr_t)s
& -sizeof(word_type
));
243 misalign
= (uintptr_t)s
& (sizeof(word_type
) - 1);
245 val
= acc_char_mask_misalign (val
, misalign
);
250 t
= acc_char_cmp (val
, repl_nl
);
251 t
|= acc_char_cmp (val
, repl_cr
);
252 t
|= acc_char_cmp (val
, repl_bs
);
253 t
|= acc_char_cmp (val
, repl_qm
);
255 if (__builtin_expect (t
!= 0, 0))
257 int i
= acc_char_index (t
, val
);
259 return (const uchar
*)p
+ i
;
266 /* Disable on Solaris 2/x86 until the following problems can be properly
269 The Solaris 9 assembler cannot assemble SSE4.2 insns.
270 Before Solaris 9 Update 6, SSE insns cannot be executed.
271 The Solaris 10+ assembler tags objects with the instruction set
272 extensions used, so SSE4.2 executables cannot run on machines that
273 don't support that extension. */
275 #if (GCC_VERSION >= 4005) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__))
277 /* Replicated character data to be shared between implementations.
278 Recall that outside of a context with vector support we can't
279 define compatible vector types, therefore these are all defined
280 in terms of raw characters. */
281 static const char repl_chars
[4][16] __attribute__((aligned(16))) = {
282 { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
283 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },
284 { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
285 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },
286 { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
287 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },
288 { '?', '?', '?', '?', '?', '?', '?', '?',
289 '?', '?', '?', '?', '?', '?', '?', '?' },
292 /* A version of the fast scanner using MMX vectorized byte compare insns.
294 This uses the PMOVMSKB instruction which was introduced with "MMX2",
295 which was packaged into SSE1; it is also present in the AMD MMX
296 extension. Mark the function as using "sse" so that we emit a real
297 "emms" instruction, rather than the 3dNOW "femms" instruction. */
301 __attribute__((__target__("sse")))
303 search_line_mmx (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
305 typedef char v8qi
__attribute__ ((__vector_size__ (8)));
306 typedef int __m64
__attribute__ ((__vector_size__ (8), __may_alias__
));
308 const v8qi repl_nl
= *(const v8qi
*)repl_chars
[0];
309 const v8qi repl_cr
= *(const v8qi
*)repl_chars
[1];
310 const v8qi repl_bs
= *(const v8qi
*)repl_chars
[2];
311 const v8qi repl_qm
= *(const v8qi
*)repl_chars
[3];
313 unsigned int misalign
, found
, mask
;
317 /* Align the source pointer. While MMX doesn't generate unaligned data
318 faults, this allows us to safely scan to the end of the buffer without
319 reading beyond the end of the last page. */
320 misalign
= (uintptr_t)s
& 7;
321 p
= (const v8qi
*)((uintptr_t)s
& -8);
324 /* Create a mask for the bytes that are valid within the first
325 16-byte block. The Idea here is that the AND with the mask
326 within the loop is "free", since we need some AND or TEST
327 insn in order to set the flags for the branch anyway. */
328 mask
= -1u << misalign
;
330 /* Main loop processing 8 bytes at a time. */
338 t
= __builtin_ia32_pcmpeqb(data
, repl_nl
);
339 c
= __builtin_ia32_pcmpeqb(data
, repl_cr
);
340 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
341 c
= __builtin_ia32_pcmpeqb(data
, repl_bs
);
342 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
343 c
= __builtin_ia32_pcmpeqb(data
, repl_qm
);
344 t
= (v8qi
) __builtin_ia32_por ((__m64
)t
, (__m64
)c
);
345 found
= __builtin_ia32_pmovmskb (t
);
350 __builtin_ia32_emms ();
352 /* FOUND contains 1 in bits for which we matched a relevant
353 character. Conversion to the byte index is trivial. */
354 found
= __builtin_ctz(found
);
355 return (const uchar
*)p
+ found
;
358 /* A version of the fast scanner using SSE2 vectorized byte compare insns. */
362 __attribute__((__target__("sse2")))
364 search_line_sse2 (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
366 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
368 const v16qi repl_nl
= *(const v16qi
*)repl_chars
[0];
369 const v16qi repl_cr
= *(const v16qi
*)repl_chars
[1];
370 const v16qi repl_bs
= *(const v16qi
*)repl_chars
[2];
371 const v16qi repl_qm
= *(const v16qi
*)repl_chars
[3];
373 unsigned int misalign
, found
, mask
;
377 /* Align the source pointer. */
378 misalign
= (uintptr_t)s
& 15;
379 p
= (const v16qi
*)((uintptr_t)s
& -16);
382 /* Create a mask for the bytes that are valid within the first
383 16-byte block. The Idea here is that the AND with the mask
384 within the loop is "free", since we need some AND or TEST
385 insn in order to set the flags for the branch anyway. */
386 mask
= -1u << misalign
;
388 /* Main loop processing 16 bytes at a time. */
396 t
= __builtin_ia32_pcmpeqb128(data
, repl_nl
);
397 t
|= __builtin_ia32_pcmpeqb128(data
, repl_cr
);
398 t
|= __builtin_ia32_pcmpeqb128(data
, repl_bs
);
399 t
|= __builtin_ia32_pcmpeqb128(data
, repl_qm
);
400 found
= __builtin_ia32_pmovmskb128 (t
);
405 /* FOUND contains 1 in bits for which we matched a relevant
406 character. Conversion to the byte index is trivial. */
407 found
= __builtin_ctz(found
);
408 return (const uchar
*)p
+ found
;
412 /* A version of the fast scanner using SSE 4.2 vectorized string insns. */
416 __attribute__((__target__("sse4.2")))
418 search_line_sse42 (const uchar
*s
, const uchar
*end
)
420 typedef char v16qi
__attribute__ ((__vector_size__ (16)));
421 static const v16qi search
= { '\n', '\r', '?', '\\' };
423 uintptr_t si
= (uintptr_t)s
;
426 /* Check for unaligned input. */
431 if (__builtin_expect (end
- s
< 16, 0)
432 && __builtin_expect ((si
& 0xfff) > 0xff0, 0))
434 /* There are less than 16 bytes left in the buffer, and less
435 than 16 bytes left on the page. Reading 16 bytes at this
436 point might generate a spurious page fault. Defer to the
437 SSE2 implementation, which already handles alignment. */
438 return search_line_sse2 (s
, end
);
441 /* ??? The builtin doesn't understand that the PCMPESTRI read from
442 memory need not be aligned. */
443 sv
= __builtin_ia32_loaddqu ((const char *) s
);
444 index
= __builtin_ia32_pcmpestri128 (search
, 4, sv
, 16, 0);
446 if (__builtin_expect (index
< 16, 0))
449 /* Advance the pointer to an aligned address. We will re-scan a
450 few bytes, but we no longer need care for reading past the
451 end of a page, since we're guaranteed a match. */
452 s
= (const uchar
*)((si
+ 16) & -16);
455 /* Main loop, processing 16 bytes at a time. By doing the whole loop
456 in inline assembly, we can make proper use of the flags set. */
457 __asm ( "sub $16, %1\n"
460 " %vpcmpestri $0, (%1), %2\n"
462 : "=&c"(index
), "+r"(s
)
463 : "x"(search
), "a"(4), "d"(16));
470 /* Work around out-dated assemblers without sse4 support. */
471 #define search_line_sse42 search_line_sse2
474 /* Check the CPU capabilities. */
476 #include "../gcc/config/i386/cpuid.h"
478 typedef const uchar
* (*search_line_fast_type
) (const uchar
*, const uchar
*);
479 static search_line_fast_type search_line_fast
;
481 #define HAVE_init_vectorized_lexer 1
483 init_vectorized_lexer (void)
485 unsigned dummy
, ecx
= 0, edx
= 0;
486 search_line_fast_type impl
= search_line_acc_char
;
489 #if defined(__SSE4_2__)
491 #elif defined(__SSE2__)
493 #elif defined(__SSE__)
498 impl
= search_line_sse42
;
499 else if (__get_cpuid (1, &dummy
, &dummy
, &ecx
, &edx
) || minimum
== 2)
501 if (minimum
== 3 || (ecx
& bit_SSE4_2
))
502 impl
= search_line_sse42
;
503 else if (minimum
== 2 || (edx
& bit_SSE2
))
504 impl
= search_line_sse2
;
505 else if (minimum
== 1 || (edx
& bit_SSE
))
506 impl
= search_line_mmx
;
508 else if (__get_cpuid (0x80000001, &dummy
, &dummy
, &dummy
, &edx
))
511 || (edx
& (bit_MMXEXT
| bit_CMOV
)) == (bit_MMXEXT
| bit_CMOV
))
512 impl
= search_line_mmx
;
515 search_line_fast
= impl
;
518 #elif (GCC_VERSION >= 4005) && defined(__ALTIVEC__)
520 /* A vection of the fast scanner using AltiVec vectorized byte compares. */
521 /* ??? Unfortunately, attribute(target("altivec")) is not yet supported,
522 so we can't compile this function without -maltivec on the command line
523 (or implied by some other switch). */
526 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
528 typedef __attribute__((altivec(vector
))) unsigned char vc
;
531 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',
532 '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'
535 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',
536 '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'
539 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',
540 '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'
543 '?', '?', '?', '?', '?', '?', '?', '?',
544 '?', '?', '?', '?', '?', '?', '?', '?',
547 -1, -1, -1, -1, -1, -1, -1, -1,
548 -1, -1, -1, -1, -1, -1, -1, -1,
550 const vc zero
= { 0 };
554 /* Altivec loads automatically mask addresses with -16. This lets us
555 issue the first load as early as possible. */
556 data
= __builtin_vec_ld(0, (const vc
*)s
);
558 /* Discard bytes before the beginning of the buffer. Do this by
559 beginning with all ones and shifting in zeros according to the
560 mis-alignment. The LVSR instruction pulls the exact shift we
561 want from the address. */
562 mask
= __builtin_vec_lvsr(0, s
);
563 mask
= __builtin_vec_perm(zero
, ones
, mask
);
566 /* While altivec loads mask addresses, we still need to align S so
567 that the offset we compute at the end is correct. */
568 s
= (const uchar
*)((uintptr_t)s
& -16);
570 /* Main loop processing 16 bytes at a time. */
574 vc m_nl
, m_cr
, m_bs
, m_qm
;
577 data
= __builtin_vec_ld(0, (const vc
*)s
);
580 m_nl
= (vc
) __builtin_vec_cmpeq(data
, repl_nl
);
581 m_cr
= (vc
) __builtin_vec_cmpeq(data
, repl_cr
);
582 m_bs
= (vc
) __builtin_vec_cmpeq(data
, repl_bs
);
583 m_qm
= (vc
) __builtin_vec_cmpeq(data
, repl_qm
);
584 t
= (m_nl
| m_cr
) | (m_bs
| m_qm
);
586 /* T now contains 0xff in bytes for which we matched one of the relevant
587 characters. We want to exit the loop if any byte in T is non-zero.
588 Below is the expansion of vec_any_ne(t, zero). */
590 while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t
, zero
));
593 #define N (sizeof(vc) / sizeof(long))
597 /* Statically assert that N is 2 or 4. */
598 unsigned long l
[(N
== 2 || N
== 4) ? N
: -1];
600 unsigned long l
, i
= 0;
604 /* Find the first word of T that is non-zero. */
611 s
+= sizeof(unsigned long);
615 s
+= sizeof(unsigned long);
620 s
+= sizeof(unsigned long);
624 /* L now contains 0xff in bytes for which we matched one of the
625 relevant characters. We can find the byte index by finding
626 its bit index and dividing by 8. */
627 l
= __builtin_clzl(l
) >> 3;
634 #elif defined (__ARM_NEON__)
635 #include "arm_neon.h"
638 search_line_fast (const uchar
*s
, const uchar
*end ATTRIBUTE_UNUSED
)
640 const uint8x16_t repl_nl
= vdupq_n_u8 ('\n');
641 const uint8x16_t repl_cr
= vdupq_n_u8 ('\r');
642 const uint8x16_t repl_bs
= vdupq_n_u8 ('\\');
643 const uint8x16_t repl_qm
= vdupq_n_u8 ('?');
644 const uint8x16_t xmask
= (uint8x16_t
) vdupq_n_u64 (0x8040201008040201ULL
);
646 unsigned int misalign
, found
, mask
;
650 /* Align the source pointer. */
651 misalign
= (uintptr_t)s
& 15;
652 p
= (const uint8_t *)((uintptr_t)s
& -16);
655 /* Create a mask for the bytes that are valid within the first
656 16-byte block. The Idea here is that the AND with the mask
657 within the loop is "free", since we need some AND or TEST
658 insn in order to set the flags for the branch anyway. */
659 mask
= (-1u << misalign
) & 0xffff;
661 /* Main loop, processing 16 bytes at a time. */
669 uint8x16_t t
, u
, v
, w
;
676 t
= vceqq_u8 (data
, repl_nl
);
677 u
= vceqq_u8 (data
, repl_cr
);
678 v
= vorrq_u8 (t
, vceqq_u8 (data
, repl_bs
));
679 w
= vorrq_u8 (u
, vceqq_u8 (data
, repl_qm
));
680 t
= vandq_u8 (vorrq_u8 (v
, w
), xmask
);
681 l
= vpadd_u8 (vget_low_u8 (t
), vget_high_u8 (t
));
685 found
= vget_lane_u32 ((uint32x2_t
) vorr_u64 ((uint64x1_t
) n
,
686 vshr_n_u64 ((uint64x1_t
) n
, 24)), 0);
691 /* FOUND contains 1 in bits for which we matched a relevant
692 character. Conversion to the byte index is trivial. */
693 found
= __builtin_ctz (found
);
694 return (const uchar
*)p
+ found
;
699 /* We only have one accellerated alternative. Use a direct call so that
700 we encourage inlining. */
702 #define search_line_fast search_line_acc_char
706 /* Initialize the lexer if needed. */
709 _cpp_init_lexer (void)
711 #ifdef HAVE_init_vectorized_lexer
712 init_vectorized_lexer ();
716 /* Returns with a logical line that contains no escaped newlines or
717 trigraphs. This is a time-critical inner loop. */
719 _cpp_clean_line (cpp_reader
*pfile
)
725 buffer
= pfile
->buffer
;
726 buffer
->cur_note
= buffer
->notes_used
= 0;
727 buffer
->cur
= buffer
->line_base
= buffer
->next_line
;
728 buffer
->need_line
= false;
729 s
= buffer
->next_line
;
731 if (!buffer
->from_stage3
)
733 const uchar
*pbackslash
= NULL
;
735 /* Fast path. This is the common case of an un-escaped line with
736 no trigraphs. The primary win here is by not writing any
737 data back to memory until we have to. */
740 /* Perform an optimized search for \n, \r, \\, ?. */
741 s
= search_line_fast (s
, buffer
->rlimit
);
746 /* Record the location of the backslash and continue. */
749 else if (__builtin_expect (c
== '?', 0))
751 if (__builtin_expect (s
[1] == '?', false)
752 && _cpp_trigraph_map
[s
[2]])
754 /* Have a trigraph. We may or may not have to convert
755 it. Add a line note regardless, for -Wtrigraphs. */
756 add_line_note (buffer
, s
, s
[2]);
757 if (CPP_OPTION (pfile
, trigraphs
))
759 /* We do, and that means we have to switch to the
762 *d
= _cpp_trigraph_map
[s
[2]];
767 /* Not a trigraph. Continue on fast-path. */
774 /* This must be \r or \n. We're either done, or we'll be forced
775 to write back to the buffer and continue on the slow path. */
778 if (__builtin_expect (s
== buffer
->rlimit
, false))
781 /* DOS line ending? */
782 if (__builtin_expect (c
== '\r', false) && s
[1] == '\n')
785 if (s
== buffer
->rlimit
)
789 if (__builtin_expect (pbackslash
== NULL
, true))
792 /* Check for escaped newline. */
794 while (is_nvspace (p
[-1]))
796 if (p
- 1 != pbackslash
)
799 /* Have an escaped newline; process it and proceed to
801 add_line_note (buffer
, p
- 1, p
!= d
? ' ' : '\\');
803 buffer
->next_line
= p
- 1;
811 if (c
== '\n' || c
== '\r')
813 /* Handle DOS line endings. */
814 if (c
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
816 if (s
== buffer
->rlimit
)
821 while (p
!= buffer
->next_line
&& is_nvspace (p
[-1]))
823 if (p
== buffer
->next_line
|| p
[-1] != '\\')
826 add_line_note (buffer
, p
- 1, p
!= d
? ' ': '\\');
828 buffer
->next_line
= p
- 1;
830 else if (c
== '?' && s
[1] == '?' && _cpp_trigraph_map
[s
[2]])
832 /* Add a note regardless, for the benefit of -Wtrigraphs. */
833 add_line_note (buffer
, d
, s
[2]);
834 if (CPP_OPTION (pfile
, trigraphs
))
836 *d
= _cpp_trigraph_map
[s
[2]];
844 while (*s
!= '\n' && *s
!= '\r')
848 /* Handle DOS line endings. */
849 if (*s
== '\r' && s
!= buffer
->rlimit
&& s
[1] == '\n')
855 /* A sentinel note that should never be processed. */
856 add_line_note (buffer
, d
+ 1, '\n');
857 buffer
->next_line
= s
+ 1;
860 /* Return true if the trigraph indicated by NOTE should be warned
861 about in a comment. */
863 warn_in_comment (cpp_reader
*pfile
, _cpp_line_note
*note
)
867 /* Within comments we don't warn about trigraphs, unless the
868 trigraph forms an escaped newline, as that may change
870 if (note
->type
!= '/')
873 /* If -trigraphs, then this was an escaped newline iff the next note
875 if (CPP_OPTION (pfile
, trigraphs
))
876 return note
[1].pos
== note
->pos
;
878 /* Otherwise, see if this forms an escaped newline. */
880 while (is_nvspace (*p
))
883 /* There might have been escaped newlines between the trigraph and the
884 newline we found. Hence the position test. */
885 return (*p
== '\n' && p
< note
[1].pos
);
888 /* Process the notes created by add_line_note as far as the current
891 _cpp_process_line_notes (cpp_reader
*pfile
, int in_comment
)
893 cpp_buffer
*buffer
= pfile
->buffer
;
897 _cpp_line_note
*note
= &buffer
->notes
[buffer
->cur_note
];
900 if (note
->pos
> buffer
->cur
)
904 col
= CPP_BUF_COLUMN (buffer
, note
->pos
+ 1);
906 if (note
->type
== '\\' || note
->type
== ' ')
908 if (note
->type
== ' ' && !in_comment
)
909 cpp_error_with_line (pfile
, CPP_DL_WARNING
, pfile
->line_table
->highest_line
, col
,
910 "backslash and newline separated by space");
912 if (buffer
->next_line
> buffer
->rlimit
)
914 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
, col
,
915 "backslash-newline at end of file");
916 /* Prevent "no newline at end of file" warning. */
917 buffer
->next_line
= buffer
->rlimit
;
920 buffer
->line_base
= note
->pos
;
921 CPP_INCREMENT_LINE (pfile
, 0);
923 else if (_cpp_trigraph_map
[note
->type
])
925 if (CPP_OPTION (pfile
, warn_trigraphs
)
926 && (!in_comment
|| warn_in_comment (pfile
, note
)))
928 if (CPP_OPTION (pfile
, trigraphs
))
929 cpp_warning_with_line (pfile
, CPP_W_TRIGRAPHS
,
930 pfile
->line_table
->highest_line
, col
,
931 "trigraph ??%c converted to %c",
933 (int) _cpp_trigraph_map
[note
->type
]);
936 cpp_warning_with_line
937 (pfile
, CPP_W_TRIGRAPHS
,
938 pfile
->line_table
->highest_line
, col
,
939 "trigraph ??%c ignored, use -trigraphs to enable",
944 else if (note
->type
== 0)
945 /* Already processed in lex_raw_string. */;
951 /* Skip a C-style block comment. We find the end of the comment by
952 seeing if an asterisk is before every '/' we encounter. Returns
953 nonzero if comment terminated by EOF, zero otherwise.
955 Buffer->cur points to the initial asterisk of the comment. */
957 _cpp_skip_block_comment (cpp_reader
*pfile
)
959 cpp_buffer
*buffer
= pfile
->buffer
;
960 const uchar
*cur
= buffer
->cur
;
969 /* People like decorating comments with '*', so check for '/'
970 instead for efficiency. */
978 /* Warn about potential nested comments, but not if the '/'
979 comes immediately before the true comment delimiter.
980 Don't bother to get it right across escaped newlines. */
981 if (CPP_OPTION (pfile
, warn_comments
)
982 && cur
[0] == '*' && cur
[1] != '/')
985 cpp_warning_with_line (pfile
, CPP_W_COMMENTS
,
986 pfile
->line_table
->highest_line
,
987 CPP_BUF_COL (buffer
),
988 "\"/*\" within comment");
994 buffer
->cur
= cur
- 1;
995 _cpp_process_line_notes (pfile
, true);
996 if (buffer
->next_line
>= buffer
->rlimit
)
998 _cpp_clean_line (pfile
);
1000 cols
= buffer
->next_line
- buffer
->line_base
;
1001 CPP_INCREMENT_LINE (pfile
, cols
);
1008 _cpp_process_line_notes (pfile
, true);
1012 /* Skip a C++ line comment, leaving buffer->cur pointing to the
1013 terminating newline. Handles escaped newlines. Returns nonzero
1014 if a multiline comment. */
1016 skip_line_comment (cpp_reader
*pfile
)
1018 cpp_buffer
*buffer
= pfile
->buffer
;
1019 source_location orig_line
= pfile
->line_table
->highest_line
;
1021 while (*buffer
->cur
!= '\n')
1024 _cpp_process_line_notes (pfile
, true);
1025 return orig_line
!= pfile
->line_table
->highest_line
;
1028 /* Skips whitespace, saving the next non-whitespace character. */
1030 skip_whitespace (cpp_reader
*pfile
, cppchar_t c
)
1032 cpp_buffer
*buffer
= pfile
->buffer
;
1033 bool saw_NUL
= false;
1037 /* Horizontal space always OK. */
1038 if (c
== ' ' || c
== '\t')
1040 /* Just \f \v or \0 left. */
1043 else if (pfile
->state
.in_directive
&& CPP_PEDANTIC (pfile
))
1044 cpp_error_with_line (pfile
, CPP_DL_PEDWARN
, pfile
->line_table
->highest_line
,
1045 CPP_BUF_COL (buffer
),
1046 "%s in preprocessing directive",
1047 c
== '\f' ? "form feed" : "vertical tab");
1051 /* We only want non-vertical space, i.e. ' ' \t \f \v \0. */
1052 while (is_nvspace (c
));
1055 cpp_error (pfile
, CPP_DL_WARNING
, "null character(s) ignored");
1060 /* See if the characters of a number token are valid in a name (no
1061 '.', '+' or '-'). */
1063 name_p (cpp_reader
*pfile
, const cpp_string
*string
)
1067 for (i
= 0; i
< string
->len
; i
++)
1068 if (!is_idchar (string
->text
[i
]))
1074 /* After parsing an identifier or other sequence, produce a warning about
1075 sequences not in NFC/NFKC. */
1077 warn_about_normalization (cpp_reader
*pfile
,
1078 const cpp_token
*token
,
1079 const struct normalize_state
*s
)
1081 if (CPP_OPTION (pfile
, warn_normalize
) < NORMALIZE_STATE_RESULT (s
)
1082 && !pfile
->state
.skipping
)
1084 /* Make sure that the token is printed using UCNs, even
1085 if we'd otherwise happily print UTF-8. */
1086 unsigned char *buf
= XNEWVEC (unsigned char, cpp_token_len (token
));
1089 sz
= cpp_spell_token (pfile
, token
, buf
, false) - buf
;
1090 if (NORMALIZE_STATE_RESULT (s
) == normalized_C
)
1091 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1092 "`%.*s' is not in NFKC", (int) sz
, buf
);
1094 cpp_warning_with_line (pfile
, CPP_W_NORMALIZE
, token
->src_loc
, 0,
1095 "`%.*s' is not in NFC", (int) sz
, buf
);
1100 /* Returns TRUE if the sequence starting at buffer->cur is invalid in
1101 an identifier. FIRST is TRUE if this starts an identifier. */
1103 forms_identifier_p (cpp_reader
*pfile
, int first
,
1104 struct normalize_state
*state
)
1106 cpp_buffer
*buffer
= pfile
->buffer
;
1108 if (*buffer
->cur
== '$')
1110 if (!CPP_OPTION (pfile
, dollars_in_ident
))
1114 if (CPP_OPTION (pfile
, warn_dollars
) && !pfile
->state
.skipping
)
1116 CPP_OPTION (pfile
, warn_dollars
) = 0;
1117 cpp_error (pfile
, CPP_DL_PEDWARN
, "'$' in identifier or number");
1123 /* Is this a syntactically valid UCN? */
1124 if (CPP_OPTION (pfile
, extended_identifiers
)
1125 && *buffer
->cur
== '\\'
1126 && (buffer
->cur
[1] == 'u' || buffer
->cur
[1] == 'U'))
1129 if (_cpp_valid_ucn (pfile
, &buffer
->cur
, buffer
->rlimit
, 1 + !first
,
1138 /* Helper function to get the cpp_hashnode of the identifier BASE. */
1139 static cpp_hashnode
*
1140 lex_identifier_intern (cpp_reader
*pfile
, const uchar
*base
)
1142 cpp_hashnode
*result
;
1145 unsigned int hash
= HT_HASHSTEP (0, *base
);
1148 while (ISIDNUM (*cur
))
1150 hash
= HT_HASHSTEP (hash
, *cur
);
1154 hash
= HT_HASHFINISH (hash
, len
);
1155 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1156 base
, len
, hash
, HT_ALLOC
));
1158 /* Rarely, identifiers require diagnostics when lexed. */
1159 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1160 && !pfile
->state
.skipping
, 0))
1162 /* It is allowed to poison the same identifier twice. */
1163 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1164 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1165 NODE_NAME (result
));
1167 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1168 replacement list of a variadic macro. */
1169 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1170 && !pfile
->state
.va_args_ok
)
1171 cpp_error (pfile
, CPP_DL_PEDWARN
,
1172 "__VA_ARGS__ can only appear in the expansion"
1173 " of a C99 variadic macro");
1175 /* For -Wc++-compat, warn about use of C++ named operators. */
1176 if (result
->flags
& NODE_WARN_OPERATOR
)
1177 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1178 "identifier \"%s\" is a special operator name in C++",
1179 NODE_NAME (result
));
1185 /* Get the cpp_hashnode of an identifier specified by NAME in
1186 the current cpp_reader object. If none is found, NULL is returned. */
1188 _cpp_lex_identifier (cpp_reader
*pfile
, const char *name
)
1190 cpp_hashnode
*result
;
1191 result
= lex_identifier_intern (pfile
, (uchar
*) name
);
1195 /* Lex an identifier starting at BUFFER->CUR - 1. */
1196 static cpp_hashnode
*
1197 lex_identifier (cpp_reader
*pfile
, const uchar
*base
, bool starts_ucn
,
1198 struct normalize_state
*nst
)
1200 cpp_hashnode
*result
;
1203 unsigned int hash
= HT_HASHSTEP (0, *base
);
1205 cur
= pfile
->buffer
->cur
;
1208 while (ISIDNUM (*cur
))
1210 hash
= HT_HASHSTEP (hash
, *cur
);
1213 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *(cur
- 1));
1215 pfile
->buffer
->cur
= cur
;
1216 if (starts_ucn
|| forms_identifier_p (pfile
, false, nst
))
1218 /* Slower version for identifiers containing UCNs (or $). */
1220 while (ISIDNUM (*pfile
->buffer
->cur
))
1222 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *pfile
->buffer
->cur
);
1223 pfile
->buffer
->cur
++;
1225 } while (forms_identifier_p (pfile
, false, nst
));
1226 result
= _cpp_interpret_identifier (pfile
, base
,
1227 pfile
->buffer
->cur
- base
);
1232 hash
= HT_HASHFINISH (hash
, len
);
1234 result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1235 base
, len
, hash
, HT_ALLOC
));
1238 /* Rarely, identifiers require diagnostics when lexed. */
1239 if (__builtin_expect ((result
->flags
& NODE_DIAGNOSTIC
)
1240 && !pfile
->state
.skipping
, 0))
1242 /* It is allowed to poison the same identifier twice. */
1243 if ((result
->flags
& NODE_POISONED
) && !pfile
->state
.poisoned_ok
)
1244 cpp_error (pfile
, CPP_DL_ERROR
, "attempt to use poisoned \"%s\"",
1245 NODE_NAME (result
));
1247 /* Constraint 6.10.3.5: __VA_ARGS__ should only appear in the
1248 replacement list of a variadic macro. */
1249 if (result
== pfile
->spec_nodes
.n__VA_ARGS__
1250 && !pfile
->state
.va_args_ok
)
1251 cpp_error (pfile
, CPP_DL_PEDWARN
,
1252 "__VA_ARGS__ can only appear in the expansion"
1253 " of a C99 variadic macro");
1255 /* For -Wc++-compat, warn about use of C++ named operators. */
1256 if (result
->flags
& NODE_WARN_OPERATOR
)
1257 cpp_warning (pfile
, CPP_W_CXX_OPERATOR_NAMES
,
1258 "identifier \"%s\" is a special operator name in C++",
1259 NODE_NAME (result
));
1265 /* Lex a number to NUMBER starting at BUFFER->CUR - 1. */
1267 lex_number (cpp_reader
*pfile
, cpp_string
*number
,
1268 struct normalize_state
*nst
)
1274 base
= pfile
->buffer
->cur
- 1;
1277 cur
= pfile
->buffer
->cur
;
1279 /* N.B. ISIDNUM does not include $. */
1280 while (ISIDNUM (*cur
) || *cur
== '.' || DIGIT_SEP (*cur
)
1281 || VALID_SIGN (*cur
, cur
[-1]))
1283 NORMALIZE_STATE_UPDATE_IDNUM (nst
, *cur
);
1287 pfile
->buffer
->cur
= cur
;
1289 while (forms_identifier_p (pfile
, false, nst
));
1291 number
->len
= cur
- base
;
1292 dest
= _cpp_unaligned_alloc (pfile
, number
->len
+ 1);
1293 memcpy (dest
, base
, number
->len
);
1294 dest
[number
->len
] = '\0';
1295 number
->text
= dest
;
1298 /* Create a token of type TYPE with a literal spelling. */
1300 create_literal (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1301 unsigned int len
, enum cpp_ttype type
)
1303 uchar
*dest
= _cpp_unaligned_alloc (pfile
, len
+ 1);
1305 memcpy (dest
, base
, len
);
1308 token
->val
.str
.len
= len
;
1309 token
->val
.str
.text
= dest
;
1312 /* Subroutine of lex_raw_string: Append LEN chars from BASE to the buffer
1313 sequence from *FIRST_BUFF_P to LAST_BUFF_P. */
1316 bufring_append (cpp_reader
*pfile
, const uchar
*base
, size_t len
,
1317 _cpp_buff
**first_buff_p
, _cpp_buff
**last_buff_p
)
1319 _cpp_buff
*first_buff
= *first_buff_p
;
1320 _cpp_buff
*last_buff
= *last_buff_p
;
1322 if (first_buff
== NULL
)
1323 first_buff
= last_buff
= _cpp_get_buff (pfile
, len
);
1324 else if (len
> BUFF_ROOM (last_buff
))
1326 size_t room
= BUFF_ROOM (last_buff
);
1327 memcpy (BUFF_FRONT (last_buff
), base
, room
);
1328 BUFF_FRONT (last_buff
) += room
;
1331 last_buff
= _cpp_append_extend_buff (pfile
, last_buff
, len
);
1334 memcpy (BUFF_FRONT (last_buff
), base
, len
);
1335 BUFF_FRONT (last_buff
) += len
;
1337 *first_buff_p
= first_buff
;
1338 *last_buff_p
= last_buff
;
1342 /* Returns true if a macro has been defined.
1343 This might not work if compile with -save-temps,
1344 or preprocess separately from compilation. */
1347 is_macro(cpp_reader
*pfile
, const uchar
*base
)
1349 const uchar
*cur
= base
;
1350 if (! ISIDST (*cur
))
1352 unsigned int hash
= HT_HASHSTEP (0, *cur
);
1354 while (ISIDNUM (*cur
))
1356 hash
= HT_HASHSTEP (hash
, *cur
);
1359 hash
= HT_HASHFINISH (hash
, cur
- base
);
1361 cpp_hashnode
*result
= CPP_HASHNODE (ht_lookup_with_hash (pfile
->hash_table
,
1362 base
, cur
- base
, hash
, HT_NO_INSERT
));
1364 return !result
? false : (result
->type
== NT_MACRO
);
1368 /* Lexes a raw string. The stored string contains the spelling, including
1369 double quotes, delimiter string, '(' and ')', any leading
1370 'L', 'u', 'U' or 'u8' and 'R' modifier. It returns the type of the
1371 literal, or CPP_OTHER if it was not properly terminated.
1373 The spelling is NUL-terminated, but it is not guaranteed that this
1374 is the first NUL since embedded NULs are preserved. */
1377 lex_raw_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
,
1380 uchar raw_prefix
[17];
1381 uchar temp_buffer
[18];
1382 const uchar
*orig_base
;
1383 unsigned int raw_prefix_len
= 0, raw_suffix_len
= 0;
1384 enum raw_str_phase
{ RAW_STR_PREFIX
, RAW_STR
, RAW_STR_SUFFIX
};
1385 raw_str_phase phase
= RAW_STR_PREFIX
;
1386 enum cpp_ttype type
;
1387 size_t total_len
= 0;
1388 /* Index into temp_buffer during phases other than RAW_STR,
1389 during RAW_STR phase 17 to tell BUF_APPEND that nothing should
1390 be appended to temp_buffer. */
1391 size_t temp_buffer_len
= 0;
1392 _cpp_buff
*first_buff
= NULL
, *last_buff
= NULL
;
1393 size_t raw_prefix_start
;
1394 _cpp_line_note
*note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1396 type
= (*base
== 'L' ? CPP_WSTRING
:
1397 *base
== 'U' ? CPP_STRING32
:
1398 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1401 #define BUF_APPEND(STR,LEN) \
1403 bufring_append (pfile, (const uchar *)(STR), (LEN), \
1404 &first_buff, &last_buff); \
1405 total_len += (LEN); \
1406 if (__builtin_expect (temp_buffer_len < 17, 0) \
1407 && (const uchar *)(STR) != base \
1410 memcpy (temp_buffer + temp_buffer_len, \
1411 (const uchar *)(STR), (LEN)); \
1412 temp_buffer_len += (LEN); \
1418 raw_prefix_start
= cur
- base
;
1423 /* If we previously performed any trigraph or line splicing
1424 transformations, undo them in between the opening and closing
1426 while (note
->pos
< cur
)
1428 for (; note
->pos
== cur
; ++note
)
1434 /* Restore backslash followed by newline. */
1435 BUF_APPEND (base
, cur
- base
);
1437 BUF_APPEND ("\\", 1);
1439 if (note
->type
== ' ')
1441 /* GNU backslash whitespace newline extension. FIXME
1442 could be any sequence of non-vertical space. When we
1443 can properly restore any such sequence, we should mark
1444 this note as handled so _cpp_process_line_notes
1446 BUF_APPEND (" ", 1);
1449 BUF_APPEND ("\n", 1);
1453 /* Already handled. */
1457 if (_cpp_trigraph_map
[note
->type
])
1459 /* Don't warn about this trigraph in
1460 _cpp_process_line_notes, since trigraphs show up as
1461 trigraphs in raw strings. */
1462 uchar type
= note
->type
;
1465 if (!CPP_OPTION (pfile
, trigraphs
))
1466 /* If we didn't convert the trigraph in the first
1467 place, don't do anything now either. */
1470 BUF_APPEND (base
, cur
- base
);
1472 BUF_APPEND ("??", 2);
1474 /* ??/ followed by newline gets two line notes, one for
1475 the trigraph and one for the backslash/newline. */
1476 if (type
== '/' && note
[1].pos
== cur
)
1478 if (note
[1].type
!= '\\'
1479 && note
[1].type
!= ' ')
1481 BUF_APPEND ("/", 1);
1483 goto after_backslash
;
1487 /* Skip the replacement character. */
1489 BUF_APPEND (&type
, 1);
1500 if (__builtin_expect (temp_buffer_len
< 17, 0))
1501 temp_buffer
[temp_buffer_len
++] = c
;
1504 if (phase
== RAW_STR_PREFIX
)
1506 while (raw_prefix_len
< temp_buffer_len
)
1508 raw_prefix
[raw_prefix_len
] = temp_buffer
[raw_prefix_len
];
1509 switch (raw_prefix
[raw_prefix_len
])
1511 case ' ': case '(': case ')': case '\\': case '\t':
1512 case '\v': case '\f': case '\n': default:
1514 /* Basic source charset except the above chars. */
1515 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
1516 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
1517 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
1518 case 's': case 't': case 'u': case 'v': case 'w': case 'x':
1520 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
1521 case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
1522 case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
1523 case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
1525 case '0': case '1': case '2': case '3': case '4': case '5':
1526 case '6': case '7': case '8': case '9':
1527 case '_': case '{': case '}': case '#': case '[': case ']':
1528 case '<': case '>': case '%': case ':': case ';': case '.':
1529 case '?': case '*': case '+': case '-': case '/': case '^':
1530 case '&': case '|': case '~': case '!': case '=': case ',':
1531 case '"': case '\'':
1532 if (raw_prefix_len
< 16)
1540 if (raw_prefix
[raw_prefix_len
] != '(')
1542 int col
= CPP_BUF_COLUMN (pfile
->buffer
, cur
) + 1;
1543 if (raw_prefix_len
== 16)
1544 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1545 col
, "raw string delimiter longer "
1546 "than 16 characters");
1547 else if (raw_prefix
[raw_prefix_len
] == '\n')
1548 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1549 col
, "invalid new-line in raw "
1550 "string delimiter");
1552 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
,
1553 col
, "invalid character '%c' in "
1554 "raw string delimiter",
1555 (int) raw_prefix
[raw_prefix_len
]);
1556 pfile
->buffer
->cur
= orig_base
+ raw_prefix_start
- 1;
1557 create_literal (pfile
, token
, orig_base
,
1558 raw_prefix_start
- 1, CPP_OTHER
);
1560 _cpp_release_buff (pfile
, first_buff
);
1563 raw_prefix
[raw_prefix_len
] = '"';
1565 /* Nothing should be appended to temp_buffer during
1567 temp_buffer_len
= 17;
1572 else if (phase
== RAW_STR_SUFFIX
)
1574 while (raw_suffix_len
<= raw_prefix_len
1575 && raw_suffix_len
< temp_buffer_len
1576 && temp_buffer
[raw_suffix_len
] == raw_prefix
[raw_suffix_len
])
1578 if (raw_suffix_len
> raw_prefix_len
)
1580 if (raw_suffix_len
== temp_buffer_len
)
1583 /* Nothing should be appended to temp_buffer during
1585 temp_buffer_len
= 17;
1589 phase
= RAW_STR_SUFFIX
;
1591 temp_buffer_len
= 0;
1595 if (pfile
->state
.in_directive
1596 || (pfile
->state
.parsing_args
1597 && pfile
->buffer
->next_line
>= pfile
->buffer
->rlimit
))
1601 cpp_error_with_line (pfile
, CPP_DL_ERROR
, token
->src_loc
, 0,
1602 "unterminated raw string");
1606 BUF_APPEND (base
, cur
- base
);
1608 if (pfile
->buffer
->cur
< pfile
->buffer
->rlimit
)
1609 CPP_INCREMENT_LINE (pfile
, 0);
1610 pfile
->buffer
->need_line
= true;
1612 pfile
->buffer
->cur
= cur
-1;
1613 _cpp_process_line_notes (pfile
, false);
1614 if (!_cpp_get_fresh_line (pfile
))
1616 source_location src_loc
= token
->src_loc
;
1617 token
->type
= CPP_EOF
;
1618 /* Tell the compiler the line number of the EOF token. */
1619 token
->src_loc
= pfile
->line_table
->highest_line
;
1621 if (first_buff
!= NULL
)
1622 _cpp_release_buff (pfile
, first_buff
);
1623 cpp_error_with_line (pfile
, CPP_DL_ERROR
, src_loc
, 0,
1624 "unterminated raw string");
1628 cur
= base
= pfile
->buffer
->cur
;
1629 note
= &pfile
->buffer
->notes
[pfile
->buffer
->cur_note
];
1633 if (CPP_OPTION (pfile
, user_literals
))
1635 /* If a string format macro, say from inttypes.h, is placed touching
1636 a string literal it could be parsed as a C++11 user-defined string
1637 literal thus breaking the program.
1638 Try to identify macros with is_macro. A warning is issued. */
1639 if (is_macro (pfile
, cur
))
1641 /* Raise a warning, but do not consume subsequent tokens. */
1642 if (CPP_OPTION (pfile
, warn_literal_suffix
))
1643 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1645 "invalid suffix on literal; C++11 requires "
1646 "a space between literal and string macro");
1648 /* Grab user defined literal suffix. */
1649 else if (ISIDST (*cur
))
1651 type
= cpp_userdef_string_add_type (type
);
1654 while (ISIDNUM (*cur
))
1659 pfile
->buffer
->cur
= cur
;
1660 if (first_buff
== NULL
)
1661 create_literal (pfile
, token
, base
, cur
- base
, type
);
1664 uchar
*dest
= _cpp_unaligned_alloc (pfile
, total_len
+ (cur
- base
) + 1);
1667 token
->val
.str
.len
= total_len
+ (cur
- base
);
1668 token
->val
.str
.text
= dest
;
1669 last_buff
= first_buff
;
1670 while (last_buff
!= NULL
)
1672 memcpy (dest
, last_buff
->base
,
1673 BUFF_FRONT (last_buff
) - last_buff
->base
);
1674 dest
+= BUFF_FRONT (last_buff
) - last_buff
->base
;
1675 last_buff
= last_buff
->next
;
1677 _cpp_release_buff (pfile
, first_buff
);
1678 memcpy (dest
, base
, cur
- base
);
1679 dest
[cur
- base
] = '\0';
1683 /* Lexes a string, character constant, or angle-bracketed header file
1684 name. The stored string contains the spelling, including opening
1685 quote and any leading 'L', 'u', 'U' or 'u8' and optional
1686 'R' modifier. It returns the type of the literal, or CPP_OTHER
1687 if it was not properly terminated, or CPP_LESS for an unterminated
1688 header name which must be relexed as normal tokens.
1690 The spelling is NUL-terminated, but it is not guaranteed that this
1691 is the first NUL since embedded NULs are preserved. */
1693 lex_string (cpp_reader
*pfile
, cpp_token
*token
, const uchar
*base
)
1695 bool saw_NUL
= false;
1697 cppchar_t terminator
;
1698 enum cpp_ttype type
;
1701 terminator
= *cur
++;
1702 if (terminator
== 'L' || terminator
== 'U')
1703 terminator
= *cur
++;
1704 else if (terminator
== 'u')
1706 terminator
= *cur
++;
1707 if (terminator
== '8')
1708 terminator
= *cur
++;
1710 if (terminator
== 'R')
1712 lex_raw_string (pfile
, token
, base
, cur
);
1715 if (terminator
== '"')
1716 type
= (*base
== 'L' ? CPP_WSTRING
:
1717 *base
== 'U' ? CPP_STRING32
:
1718 *base
== 'u' ? (base
[1] == '8' ? CPP_UTF8STRING
: CPP_STRING16
)
1720 else if (terminator
== '\'')
1721 type
= (*base
== 'L' ? CPP_WCHAR
:
1722 *base
== 'U' ? CPP_CHAR32
:
1723 *base
== 'u' ? CPP_CHAR16
: CPP_CHAR
);
1725 terminator
= '>', type
= CPP_HEADER_NAME
;
1729 cppchar_t c
= *cur
++;
1731 /* In #include-style directives, terminators are not escapable. */
1732 if (c
== '\\' && !pfile
->state
.angled_headers
&& *cur
!= '\n')
1734 else if (c
== terminator
)
1739 /* Unmatched quotes always yield undefined behavior, but
1740 greedy lexing means that what appears to be an unterminated
1741 header name may actually be a legitimate sequence of tokens. */
1742 if (terminator
== '>')
1744 token
->type
= CPP_LESS
;
1754 if (saw_NUL
&& !pfile
->state
.skipping
)
1755 cpp_error (pfile
, CPP_DL_WARNING
,
1756 "null character(s) preserved in literal");
1758 if (type
== CPP_OTHER
&& CPP_OPTION (pfile
, lang
) != CLK_ASM
)
1759 cpp_error (pfile
, CPP_DL_PEDWARN
, "missing terminating %c character",
1762 if (CPP_OPTION (pfile
, user_literals
))
1764 /* If a string format macro, say from inttypes.h, is placed touching
1765 a string literal it could be parsed as a C++11 user-defined string
1766 literal thus breaking the program.
1767 Try to identify macros with is_macro. A warning is issued. */
1768 if (is_macro (pfile
, cur
))
1770 /* Raise a warning, but do not consume subsequent tokens. */
1771 if (CPP_OPTION (pfile
, warn_literal_suffix
))
1772 cpp_warning_with_line (pfile
, CPP_W_LITERAL_SUFFIX
,
1774 "invalid suffix on literal; C++11 requires "
1775 "a space between literal and string macro");
1777 /* Grab user defined literal suffix. */
1778 else if (ISIDST (*cur
))
1780 type
= cpp_userdef_char_add_type (type
);
1781 type
= cpp_userdef_string_add_type (type
);
1784 while (ISIDNUM (*cur
))
1789 pfile
->buffer
->cur
= cur
;
1790 create_literal (pfile
, token
, base
, cur
- base
, type
);
1793 /* Return the comment table. The client may not make any assumption
1794 about the ordering of the table. */
1796 cpp_get_comments (cpp_reader
*pfile
)
1798 return &pfile
->comments
;
1801 /* Append a comment to the end of the comment table. */
1803 store_comment (cpp_reader
*pfile
, cpp_token
*token
)
1807 if (pfile
->comments
.allocated
== 0)
1809 pfile
->comments
.allocated
= 256;
1810 pfile
->comments
.entries
= (cpp_comment
*) xmalloc
1811 (pfile
->comments
.allocated
* sizeof (cpp_comment
));
1814 if (pfile
->comments
.count
== pfile
->comments
.allocated
)
1816 pfile
->comments
.allocated
*= 2;
1817 pfile
->comments
.entries
= (cpp_comment
*) xrealloc
1818 (pfile
->comments
.entries
,
1819 pfile
->comments
.allocated
* sizeof (cpp_comment
));
1822 len
= token
->val
.str
.len
;
1824 /* Copy comment. Note, token may not be NULL terminated. */
1825 pfile
->comments
.entries
[pfile
->comments
.count
].comment
=
1826 (char *) xmalloc (sizeof (char) * (len
+ 1));
1827 memcpy (pfile
->comments
.entries
[pfile
->comments
.count
].comment
,
1828 token
->val
.str
.text
, len
);
1829 pfile
->comments
.entries
[pfile
->comments
.count
].comment
[len
] = '\0';
1831 /* Set source location. */
1832 pfile
->comments
.entries
[pfile
->comments
.count
].sloc
= token
->src_loc
;
1834 /* Increment the count of entries in the comment table. */
1835 pfile
->comments
.count
++;
1838 /* The stored comment includes the comment start and any terminator. */
1840 save_comment (cpp_reader
*pfile
, cpp_token
*token
, const unsigned char *from
,
1843 unsigned char *buffer
;
1844 unsigned int len
, clen
, i
;
1846 len
= pfile
->buffer
->cur
- from
+ 1; /* + 1 for the initial '/'. */
1848 /* C++ comments probably (not definitely) have moved past a new
1849 line, which we don't want to save in the comment. */
1850 if (is_vspace (pfile
->buffer
->cur
[-1]))
1853 /* If we are currently in a directive or in argument parsing, then
1854 we need to store all C++ comments as C comments internally, and
1855 so we need to allocate a little extra space in that case.
1857 Note that the only time we encounter a directive here is
1858 when we are saving comments in a "#define". */
1859 clen
= ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
)
1860 && type
== '/') ? len
+ 2 : len
;
1862 buffer
= _cpp_unaligned_alloc (pfile
, clen
);
1864 token
->type
= CPP_COMMENT
;
1865 token
->val
.str
.len
= clen
;
1866 token
->val
.str
.text
= buffer
;
1869 memcpy (buffer
+ 1, from
, len
- 1);
1871 /* Finish conversion to a C comment, if necessary. */
1872 if ((pfile
->state
.in_directive
|| pfile
->state
.parsing_args
) && type
== '/')
1875 buffer
[clen
- 2] = '*';
1876 buffer
[clen
- 1] = '/';
1877 /* As there can be in a C++ comments illegal sequences for C comments
1878 we need to filter them out. */
1879 for (i
= 2; i
< (clen
- 2); i
++)
1880 if (buffer
[i
] == '/' && (buffer
[i
- 1] == '*' || buffer
[i
+ 1] == '*'))
1884 /* Finally store this comment for use by clients of libcpp. */
1885 store_comment (pfile
, token
);
1888 /* Allocate COUNT tokens for RUN. */
1890 _cpp_init_tokenrun (tokenrun
*run
, unsigned int count
)
1892 run
->base
= XNEWVEC (cpp_token
, count
);
1893 run
->limit
= run
->base
+ count
;
1897 /* Returns the next tokenrun, or creates one if there is none. */
1899 next_tokenrun (tokenrun
*run
)
1901 if (run
->next
== NULL
)
1903 run
->next
= XNEW (tokenrun
);
1904 run
->next
->prev
= run
;
1905 _cpp_init_tokenrun (run
->next
, 250);
1911 /* Return the number of not yet processed token in a given
1914 _cpp_remaining_tokens_num_in_context (cpp_context
*context
)
1916 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
1917 return (LAST (context
).token
- FIRST (context
).token
);
1918 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
1919 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
1920 return (LAST (context
).ptoken
- FIRST (context
).ptoken
);
1925 /* Returns the token present at index INDEX in a given context. If
1926 INDEX is zero, the next token to be processed is returned. */
1927 static const cpp_token
*
1928 _cpp_token_from_context_at (cpp_context
*context
, int index
)
1930 if (context
->tokens_kind
== TOKENS_KIND_DIRECT
)
1931 return &(FIRST (context
).token
[index
]);
1932 else if (context
->tokens_kind
== TOKENS_KIND_INDIRECT
1933 || context
->tokens_kind
== TOKENS_KIND_EXTENDED
)
1934 return FIRST (context
).ptoken
[index
];
1939 /* Look ahead in the input stream. */
1941 cpp_peek_token (cpp_reader
*pfile
, int index
)
1943 cpp_context
*context
= pfile
->context
;
1944 const cpp_token
*peektok
;
1947 /* First, scan through any pending cpp_context objects. */
1948 while (context
->prev
)
1950 ptrdiff_t sz
= _cpp_remaining_tokens_num_in_context (context
);
1952 if (index
< (int) sz
)
1953 return _cpp_token_from_context_at (context
, index
);
1955 context
= context
->prev
;
1958 /* We will have to read some new tokens after all (and do so
1959 without invalidating preceding tokens). */
1961 pfile
->keep_tokens
++;
1965 peektok
= _cpp_lex_token (pfile
);
1966 if (peektok
->type
== CPP_EOF
)
1971 _cpp_backup_tokens_direct (pfile
, count
+ 1);
1972 pfile
->keep_tokens
--;
1977 /* Allocate a single token that is invalidated at the same time as the
1978 rest of the tokens on the line. Has its line and col set to the
1979 same as the last lexed token, so that diagnostics appear in the
1982 _cpp_temp_token (cpp_reader
*pfile
)
1984 cpp_token
*old
, *result
;
1985 ptrdiff_t sz
= pfile
->cur_run
->limit
- pfile
->cur_token
;
1986 ptrdiff_t la
= (ptrdiff_t) pfile
->lookaheads
;
1988 old
= pfile
->cur_token
- 1;
1989 /* Any pre-existing lookaheads must not be clobbered. */
1994 tokenrun
*next
= next_tokenrun (pfile
->cur_run
);
1997 memmove (next
->base
+ 1, next
->base
,
1998 (la
- sz
) * sizeof (cpp_token
));
2000 next
->base
[0] = pfile
->cur_run
->limit
[-1];
2004 memmove (pfile
->cur_token
+ 1, pfile
->cur_token
,
2005 MIN (la
, sz
- 1) * sizeof (cpp_token
));
2008 if (!sz
&& pfile
->cur_token
== pfile
->cur_run
->limit
)
2010 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2011 pfile
->cur_token
= pfile
->cur_run
->base
;
2014 result
= pfile
->cur_token
++;
2015 result
->src_loc
= old
->src_loc
;
2019 /* Lex a token into RESULT (external interface). Takes care of issues
2020 like directive handling, token lookahead, multiple include
2021 optimization and skipping. */
2023 _cpp_lex_token (cpp_reader
*pfile
)
2029 if (pfile
->cur_token
== pfile
->cur_run
->limit
)
2031 pfile
->cur_run
= next_tokenrun (pfile
->cur_run
);
2032 pfile
->cur_token
= pfile
->cur_run
->base
;
2034 /* We assume that the current token is somewhere in the current
2036 if (pfile
->cur_token
< pfile
->cur_run
->base
2037 || pfile
->cur_token
>= pfile
->cur_run
->limit
)
2040 if (pfile
->lookaheads
)
2042 pfile
->lookaheads
--;
2043 result
= pfile
->cur_token
++;
2046 result
= _cpp_lex_direct (pfile
);
2048 if (result
->flags
& BOL
)
2050 /* Is this a directive. If _cpp_handle_directive returns
2051 false, it is an assembler #. */
2052 if (result
->type
== CPP_HASH
2053 /* 6.10.3 p 11: Directives in a list of macro arguments
2054 gives undefined behavior. This implementation
2055 handles the directive as normal. */
2056 && pfile
->state
.parsing_args
!= 1)
2058 if (_cpp_handle_directive (pfile
, result
->flags
& PREV_WHITE
))
2060 if (pfile
->directive_result
.type
== CPP_PADDING
)
2062 result
= &pfile
->directive_result
;
2065 else if (pfile
->state
.in_deferred_pragma
)
2066 result
= &pfile
->directive_result
;
2068 if (pfile
->cb
.line_change
&& !pfile
->state
.skipping
)
2069 pfile
->cb
.line_change (pfile
, result
, pfile
->state
.parsing_args
);
2072 /* We don't skip tokens in directives. */
2073 if (pfile
->state
.in_directive
|| pfile
->state
.in_deferred_pragma
)
2076 /* Outside a directive, invalidate controlling macros. At file
2077 EOF, _cpp_lex_direct takes care of popping the buffer, so we never
2078 get here and MI optimization works. */
2079 pfile
->mi_valid
= false;
2081 if (!pfile
->state
.skipping
|| result
->type
== CPP_EOF
)
2088 /* Returns true if a fresh line has been loaded. */
2090 _cpp_get_fresh_line (cpp_reader
*pfile
)
2094 /* We can't get a new line until we leave the current directive. */
2095 if (pfile
->state
.in_directive
)
2100 cpp_buffer
*buffer
= pfile
->buffer
;
2102 if (!buffer
->need_line
)
2105 if (buffer
->next_line
< buffer
->rlimit
)
2107 _cpp_clean_line (pfile
);
2111 /* First, get out of parsing arguments state. */
2112 if (pfile
->state
.parsing_args
)
2115 /* End of buffer. Non-empty files should end in a newline. */
2116 if (buffer
->buf
!= buffer
->rlimit
2117 && buffer
->next_line
> buffer
->rlimit
2118 && !buffer
->from_stage3
)
2120 /* Clip to buffer size. */
2121 buffer
->next_line
= buffer
->rlimit
;
2124 return_at_eof
= buffer
->return_at_eof
;
2125 _cpp_pop_buffer (pfile
);
2126 if (pfile
->buffer
== NULL
|| return_at_eof
)
2131 #define IF_NEXT_IS(CHAR, THEN_TYPE, ELSE_TYPE) \
2134 result->type = ELSE_TYPE; \
2135 if (*buffer->cur == CHAR) \
2136 buffer->cur++, result->type = THEN_TYPE; \
2140 /* Lex a token into pfile->cur_token, which is also incremented, to
2141 get diagnostics pointing to the correct location.
2143 Does not handle issues such as token lookahead, multiple-include
2144 optimization, directives, skipping etc. This function is only
2145 suitable for use by _cpp_lex_token, and in special cases like
2146 lex_expansion_token which doesn't care for any of these issues.
2148 When meeting a newline, returns CPP_EOF if parsing a directive,
2149 otherwise returns to the start of the token buffer if permissible.
2150 Returns the location of the lexed token. */
2152 _cpp_lex_direct (cpp_reader
*pfile
)
2156 const unsigned char *comment_start
;
2157 cpp_token
*result
= pfile
->cur_token
++;
2161 buffer
= pfile
->buffer
;
2162 if (buffer
->need_line
)
2164 if (pfile
->state
.in_deferred_pragma
)
2166 result
->type
= CPP_PRAGMA_EOL
;
2167 pfile
->state
.in_deferred_pragma
= false;
2168 if (!pfile
->state
.pragma_allow_expansion
)
2169 pfile
->state
.prevent_expansion
--;
2172 if (!_cpp_get_fresh_line (pfile
))
2174 result
->type
= CPP_EOF
;
2175 if (!pfile
->state
.in_directive
)
2177 /* Tell the compiler the line number of the EOF token. */
2178 result
->src_loc
= pfile
->line_table
->highest_line
;
2179 result
->flags
= BOL
;
2183 if (!pfile
->keep_tokens
)
2185 pfile
->cur_run
= &pfile
->base_run
;
2186 result
= pfile
->base_run
.base
;
2187 pfile
->cur_token
= result
+ 1;
2189 result
->flags
= BOL
;
2190 if (pfile
->state
.parsing_args
== 2)
2191 result
->flags
|= PREV_WHITE
;
2193 buffer
= pfile
->buffer
;
2195 result
->src_loc
= pfile
->line_table
->highest_line
;
2198 if (buffer
->cur
>= buffer
->notes
[buffer
->cur_note
].pos
2199 && !pfile
->overlaid_buffer
)
2201 _cpp_process_line_notes (pfile
, false);
2202 result
->src_loc
= pfile
->line_table
->highest_line
;
2206 if (pfile
->forced_token_location_p
)
2207 result
->src_loc
= *pfile
->forced_token_location_p
;
2209 result
->src_loc
= linemap_position_for_column (pfile
->line_table
,
2210 CPP_BUF_COLUMN (buffer
, buffer
->cur
));
2214 case ' ': case '\t': case '\f': case '\v': case '\0':
2215 result
->flags
|= PREV_WHITE
;
2216 skip_whitespace (pfile
, c
);
2220 if (buffer
->cur
< buffer
->rlimit
)
2221 CPP_INCREMENT_LINE (pfile
, 0);
2222 buffer
->need_line
= true;
2225 case '0': case '1': case '2': case '3': case '4':
2226 case '5': case '6': case '7': case '8': case '9':
2228 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2229 result
->type
= CPP_NUMBER
;
2230 lex_number (pfile
, &result
->val
.str
, &nst
);
2231 warn_about_normalization (pfile
, result
, &nst
);
2239 /* 'L', 'u', 'U', 'u8' or 'R' may introduce wide characters,
2240 wide strings or raw strings. */
2241 if (c
== 'L' || CPP_OPTION (pfile
, rliterals
)
2242 || (c
!= 'R' && CPP_OPTION (pfile
, uliterals
)))
2244 if ((*buffer
->cur
== '\'' && c
!= 'R')
2245 || *buffer
->cur
== '"'
2246 || (*buffer
->cur
== 'R'
2248 && buffer
->cur
[1] == '"'
2249 && CPP_OPTION (pfile
, rliterals
))
2250 || (*buffer
->cur
== '8'
2252 && (buffer
->cur
[1] == '"'
2253 || (buffer
->cur
[1] == 'R' && buffer
->cur
[2] == '"'
2254 && CPP_OPTION (pfile
, rliterals
)))))
2256 lex_string (pfile
, result
, buffer
->cur
- 1);
2263 case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
2264 case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
2265 case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
2266 case 's': case 't': case 'v': case 'w': case 'x':
2268 case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
2269 case 'G': case 'H': case 'I': case 'J': case 'K':
2270 case 'M': case 'N': case 'O': case 'P': case 'Q':
2271 case 'S': case 'T': case 'V': case 'W': case 'X':
2273 result
->type
= CPP_NAME
;
2275 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2276 result
->val
.node
.node
= lex_identifier (pfile
, buffer
->cur
- 1, false,
2278 warn_about_normalization (pfile
, result
, &nst
);
2281 /* Convert named operators to their proper types. */
2282 if (result
->val
.node
.node
->flags
& NODE_OPERATOR
)
2284 result
->flags
|= NAMED_OP
;
2285 result
->type
= (enum cpp_ttype
) result
->val
.node
.node
->directive_index
;
2291 lex_string (pfile
, result
, buffer
->cur
- 1);
2295 /* A potential block or line comment. */
2296 comment_start
= buffer
->cur
;
2301 if (_cpp_skip_block_comment (pfile
))
2302 cpp_error (pfile
, CPP_DL_ERROR
, "unterminated comment");
2304 else if (c
== '/' && (CPP_OPTION (pfile
, cplusplus_comments
)
2305 || cpp_in_system_header (pfile
)))
2307 /* Warn about comments only if pedantically GNUC89, and not
2308 in system headers. */
2309 if (CPP_OPTION (pfile
, lang
) == CLK_GNUC89
&& CPP_PEDANTIC (pfile
)
2310 && ! buffer
->warned_cplusplus_comments
)
2312 cpp_error (pfile
, CPP_DL_PEDWARN
,
2313 "C++ style comments are not allowed in ISO C90");
2314 cpp_error (pfile
, CPP_DL_PEDWARN
,
2315 "(this will be reported only once per input file)");
2316 buffer
->warned_cplusplus_comments
= 1;
2319 if (skip_line_comment (pfile
) && CPP_OPTION (pfile
, warn_comments
))
2320 cpp_warning (pfile
, CPP_W_COMMENTS
, "multi-line comment");
2325 result
->type
= CPP_DIV_EQ
;
2330 result
->type
= CPP_DIV
;
2334 if (!pfile
->state
.save_comments
)
2336 result
->flags
|= PREV_WHITE
;
2337 goto update_tokens_line
;
2340 /* Save the comment as a token in its own right. */
2341 save_comment (pfile
, result
, comment_start
, c
);
2345 if (pfile
->state
.angled_headers
)
2347 lex_string (pfile
, result
, buffer
->cur
- 1);
2348 if (result
->type
!= CPP_LESS
)
2352 result
->type
= CPP_LESS
;
2353 if (*buffer
->cur
== '=')
2354 buffer
->cur
++, result
->type
= CPP_LESS_EQ
;
2355 else if (*buffer
->cur
== '<')
2358 IF_NEXT_IS ('=', CPP_LSHIFT_EQ
, CPP_LSHIFT
);
2360 else if (CPP_OPTION (pfile
, digraphs
))
2362 if (*buffer
->cur
== ':')
2364 /* C++11 [2.5/3 lex.pptoken], "Otherwise, if the next
2365 three characters are <:: and the subsequent character
2366 is neither : nor >, the < is treated as a preprocessor
2367 token by itself". */
2368 if (CPP_OPTION (pfile
, cplusplus
)
2369 && CPP_OPTION (pfile
, lang
) != CLK_CXX98
2370 && CPP_OPTION (pfile
, lang
) != CLK_GNUCXX
2371 && buffer
->cur
[1] == ':'
2372 && buffer
->cur
[2] != ':' && buffer
->cur
[2] != '>')
2376 result
->flags
|= DIGRAPH
;
2377 result
->type
= CPP_OPEN_SQUARE
;
2379 else if (*buffer
->cur
== '%')
2382 result
->flags
|= DIGRAPH
;
2383 result
->type
= CPP_OPEN_BRACE
;
2389 result
->type
= CPP_GREATER
;
2390 if (*buffer
->cur
== '=')
2391 buffer
->cur
++, result
->type
= CPP_GREATER_EQ
;
2392 else if (*buffer
->cur
== '>')
2395 IF_NEXT_IS ('=', CPP_RSHIFT_EQ
, CPP_RSHIFT
);
2400 result
->type
= CPP_MOD
;
2401 if (*buffer
->cur
== '=')
2402 buffer
->cur
++, result
->type
= CPP_MOD_EQ
;
2403 else if (CPP_OPTION (pfile
, digraphs
))
2405 if (*buffer
->cur
== ':')
2408 result
->flags
|= DIGRAPH
;
2409 result
->type
= CPP_HASH
;
2410 if (*buffer
->cur
== '%' && buffer
->cur
[1] == ':')
2411 buffer
->cur
+= 2, result
->type
= CPP_PASTE
, result
->val
.token_no
= 0;
2413 else if (*buffer
->cur
== '>')
2416 result
->flags
|= DIGRAPH
;
2417 result
->type
= CPP_CLOSE_BRACE
;
2423 result
->type
= CPP_DOT
;
2424 if (ISDIGIT (*buffer
->cur
))
2426 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2427 result
->type
= CPP_NUMBER
;
2428 lex_number (pfile
, &result
->val
.str
, &nst
);
2429 warn_about_normalization (pfile
, result
, &nst
);
2431 else if (*buffer
->cur
== '.' && buffer
->cur
[1] == '.')
2432 buffer
->cur
+= 2, result
->type
= CPP_ELLIPSIS
;
2433 else if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2434 buffer
->cur
++, result
->type
= CPP_DOT_STAR
;
2438 result
->type
= CPP_PLUS
;
2439 if (*buffer
->cur
== '+')
2440 buffer
->cur
++, result
->type
= CPP_PLUS_PLUS
;
2441 else if (*buffer
->cur
== '=')
2442 buffer
->cur
++, result
->type
= CPP_PLUS_EQ
;
2446 result
->type
= CPP_MINUS
;
2447 if (*buffer
->cur
== '>')
2450 result
->type
= CPP_DEREF
;
2451 if (*buffer
->cur
== '*' && CPP_OPTION (pfile
, cplusplus
))
2452 buffer
->cur
++, result
->type
= CPP_DEREF_STAR
;
2454 else if (*buffer
->cur
== '-')
2455 buffer
->cur
++, result
->type
= CPP_MINUS_MINUS
;
2456 else if (*buffer
->cur
== '=')
2457 buffer
->cur
++, result
->type
= CPP_MINUS_EQ
;
2461 result
->type
= CPP_AND
;
2462 if (*buffer
->cur
== '&')
2463 buffer
->cur
++, result
->type
= CPP_AND_AND
;
2464 else if (*buffer
->cur
== '=')
2465 buffer
->cur
++, result
->type
= CPP_AND_EQ
;
2469 result
->type
= CPP_OR
;
2470 if (*buffer
->cur
== '|')
2471 buffer
->cur
++, result
->type
= CPP_OR_OR
;
2472 else if (*buffer
->cur
== '=')
2473 buffer
->cur
++, result
->type
= CPP_OR_EQ
;
2477 result
->type
= CPP_COLON
;
2478 if (*buffer
->cur
== ':' && CPP_OPTION (pfile
, cplusplus
))
2479 buffer
->cur
++, result
->type
= CPP_SCOPE
;
2480 else if (*buffer
->cur
== '>' && CPP_OPTION (pfile
, digraphs
))
2483 result
->flags
|= DIGRAPH
;
2484 result
->type
= CPP_CLOSE_SQUARE
;
2488 case '*': IF_NEXT_IS ('=', CPP_MULT_EQ
, CPP_MULT
); break;
2489 case '=': IF_NEXT_IS ('=', CPP_EQ_EQ
, CPP_EQ
); break;
2490 case '!': IF_NEXT_IS ('=', CPP_NOT_EQ
, CPP_NOT
); break;
2491 case '^': IF_NEXT_IS ('=', CPP_XOR_EQ
, CPP_XOR
); break;
2492 case '#': IF_NEXT_IS ('#', CPP_PASTE
, CPP_HASH
); result
->val
.token_no
= 0; break;
2494 case '?': result
->type
= CPP_QUERY
; break;
2495 case '~': result
->type
= CPP_COMPL
; break;
2496 case ',': result
->type
= CPP_COMMA
; break;
2497 case '(': result
->type
= CPP_OPEN_PAREN
; break;
2498 case ')': result
->type
= CPP_CLOSE_PAREN
; break;
2499 case '[': result
->type
= CPP_OPEN_SQUARE
; break;
2500 case ']': result
->type
= CPP_CLOSE_SQUARE
; break;
2501 case '{': result
->type
= CPP_OPEN_BRACE
; break;
2502 case '}': result
->type
= CPP_CLOSE_BRACE
; break;
2503 case ';': result
->type
= CPP_SEMICOLON
; break;
2505 /* @ is a punctuator in Objective-C. */
2506 case '@': result
->type
= CPP_ATSIGN
; break;
2511 const uchar
*base
= --buffer
->cur
;
2512 struct normalize_state nst
= INITIAL_NORMALIZE_STATE
;
2514 if (forms_identifier_p (pfile
, true, &nst
))
2516 result
->type
= CPP_NAME
;
2517 result
->val
.node
.node
= lex_identifier (pfile
, base
, true, &nst
);
2518 warn_about_normalization (pfile
, result
, &nst
);
2525 create_literal (pfile
, result
, buffer
->cur
- 1, 1, CPP_OTHER
);
2532 /* An upper bound on the number of bytes needed to spell TOKEN.
2533 Does not include preceding whitespace. */
2535 cpp_token_len (const cpp_token
*token
)
2539 switch (TOKEN_SPELL (token
))
2541 default: len
= 6; break;
2542 case SPELL_LITERAL
: len
= token
->val
.str
.len
; break;
2543 case SPELL_IDENT
: len
= NODE_LEN (token
->val
.node
.node
) * 10; break;
2549 /* Parse UTF-8 out of NAMEP and place a \U escape in BUFFER.
2550 Return the number of bytes read out of NAME. (There are always
2551 10 bytes written to BUFFER.) */
2554 utf8_to_ucn (unsigned char *buffer
, const unsigned char *name
)
2560 unsigned long utf32
;
2562 /* Compute the length of the UTF-8 sequence. */
2563 for (t
= *name
; t
& 0x80; t
<<= 1)
2566 utf32
= *name
& (0x7F >> ucn_len
);
2567 for (ucn_len_c
= 1; ucn_len_c
< ucn_len
; ucn_len_c
++)
2569 utf32
= (utf32
<< 6) | (*++name
& 0x3F);
2571 /* Ill-formed UTF-8. */
2572 if ((*name
& ~0x3F) != 0x80)
2578 for (j
= 7; j
>= 0; j
--)
2579 *buffer
++ = "0123456789abcdef"[(utf32
>> (4 * j
)) & 0xF];
2583 /* Given a token TYPE corresponding to a digraph, return a pointer to
2584 the spelling of the digraph. */
2585 static const unsigned char *
2586 cpp_digraph2name (enum cpp_ttype type
)
2588 return digraph_spellings
[(int) type
- (int) CPP_FIRST_DIGRAPH
];
2591 /* Write the spelling of a token TOKEN to BUFFER. The buffer must
2592 already contain the enough space to hold the token's spelling.
2593 Returns a pointer to the character after the last character written.
2594 FORSTRING is true if this is to be the spelling after translation
2595 phase 1 (this is different for UCNs).
2596 FIXME: Would be nice if we didn't need the PFILE argument. */
2598 cpp_spell_token (cpp_reader
*pfile
, const cpp_token
*token
,
2599 unsigned char *buffer
, bool forstring
)
2601 switch (TOKEN_SPELL (token
))
2603 case SPELL_OPERATOR
:
2605 const unsigned char *spelling
;
2608 if (token
->flags
& DIGRAPH
)
2609 spelling
= cpp_digraph2name (token
->type
);
2610 else if (token
->flags
& NAMED_OP
)
2613 spelling
= TOKEN_NAME (token
);
2615 while ((c
= *spelling
++) != '\0')
2624 memcpy (buffer
, NODE_NAME (token
->val
.node
.node
),
2625 NODE_LEN (token
->val
.node
.node
));
2626 buffer
+= NODE_LEN (token
->val
.node
.node
);
2631 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
2633 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
2634 if (name
[i
] & ~0x7F)
2636 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2640 *buffer
++ = NODE_NAME (token
->val
.node
.node
)[i
];
2645 memcpy (buffer
, token
->val
.str
.text
, token
->val
.str
.len
);
2646 buffer
+= token
->val
.str
.len
;
2650 cpp_error (pfile
, CPP_DL_ICE
,
2651 "unspellable token %s", TOKEN_NAME (token
));
2658 /* Returns TOKEN spelt as a null-terminated string. The string is
2659 freed when the reader is destroyed. Useful for diagnostics. */
2661 cpp_token_as_text (cpp_reader
*pfile
, const cpp_token
*token
)
2663 unsigned int len
= cpp_token_len (token
) + 1;
2664 unsigned char *start
= _cpp_unaligned_alloc (pfile
, len
), *end
;
2666 end
= cpp_spell_token (pfile
, token
, start
, false);
2672 /* Returns a pointer to a string which spells the token defined by
2673 TYPE and FLAGS. Used by C front ends, which really should move to
2674 using cpp_token_as_text. */
2676 cpp_type2name (enum cpp_ttype type
, unsigned char flags
)
2678 if (flags
& DIGRAPH
)
2679 return (const char *) cpp_digraph2name (type
);
2680 else if (flags
& NAMED_OP
)
2681 return cpp_named_operator2name (type
);
2683 return (const char *) token_spellings
[type
].name
;
2686 /* Writes the spelling of token to FP, without any preceding space.
2687 Separated from cpp_spell_token for efficiency - to avoid stdio
2688 double-buffering. */
2690 cpp_output_token (const cpp_token
*token
, FILE *fp
)
2692 switch (TOKEN_SPELL (token
))
2694 case SPELL_OPERATOR
:
2696 const unsigned char *spelling
;
2699 if (token
->flags
& DIGRAPH
)
2700 spelling
= cpp_digraph2name (token
->type
);
2701 else if (token
->flags
& NAMED_OP
)
2704 spelling
= TOKEN_NAME (token
);
2709 while ((c
= *++spelling
) != '\0');
2717 const unsigned char * name
= NODE_NAME (token
->val
.node
.node
);
2719 for (i
= 0; i
< NODE_LEN (token
->val
.node
.node
); i
++)
2720 if (name
[i
] & ~0x7F)
2722 unsigned char buffer
[10];
2723 i
+= utf8_to_ucn (buffer
, name
+ i
) - 1;
2724 fwrite (buffer
, 1, 10, fp
);
2727 fputc (NODE_NAME (token
->val
.node
.node
)[i
], fp
);
2732 fwrite (token
->val
.str
.text
, 1, token
->val
.str
.len
, fp
);
2736 /* An error, most probably. */
2741 /* Compare two tokens. */
2743 _cpp_equiv_tokens (const cpp_token
*a
, const cpp_token
*b
)
2745 if (a
->type
== b
->type
&& a
->flags
== b
->flags
)
2746 switch (TOKEN_SPELL (a
))
2748 default: /* Keep compiler happy. */
2749 case SPELL_OPERATOR
:
2750 /* token_no is used to track where multiple consecutive ##
2751 tokens were originally located. */
2752 return (a
->type
!= CPP_PASTE
|| a
->val
.token_no
== b
->val
.token_no
);
2754 return (a
->type
!= CPP_MACRO_ARG
2755 || a
->val
.macro_arg
.arg_no
== b
->val
.macro_arg
.arg_no
);
2757 return a
->val
.node
.node
== b
->val
.node
.node
;
2759 return (a
->val
.str
.len
== b
->val
.str
.len
2760 && !memcmp (a
->val
.str
.text
, b
->val
.str
.text
,
2767 /* Returns nonzero if a space should be inserted to avoid an
2768 accidental token paste for output. For simplicity, it is
2769 conservative, and occasionally advises a space where one is not
2770 needed, e.g. "." and ".2". */
2772 cpp_avoid_paste (cpp_reader
*pfile
, const cpp_token
*token1
,
2773 const cpp_token
*token2
)
2775 enum cpp_ttype a
= token1
->type
, b
= token2
->type
;
2778 if (token1
->flags
& NAMED_OP
)
2780 if (token2
->flags
& NAMED_OP
)
2784 if (token2
->flags
& DIGRAPH
)
2785 c
= digraph_spellings
[(int) b
- (int) CPP_FIRST_DIGRAPH
][0];
2786 else if (token_spellings
[b
].category
== SPELL_OPERATOR
)
2787 c
= token_spellings
[b
].name
[0];
2789 /* Quickly get everything that can paste with an '='. */
2790 if ((int) a
<= (int) CPP_LAST_EQ
&& c
== '=')
2795 case CPP_GREATER
: return c
== '>';
2796 case CPP_LESS
: return c
== '<' || c
== '%' || c
== ':';
2797 case CPP_PLUS
: return c
== '+';
2798 case CPP_MINUS
: return c
== '-' || c
== '>';
2799 case CPP_DIV
: return c
== '/' || c
== '*'; /* Comments. */
2800 case CPP_MOD
: return c
== ':' || c
== '>';
2801 case CPP_AND
: return c
== '&';
2802 case CPP_OR
: return c
== '|';
2803 case CPP_COLON
: return c
== ':' || c
== '>';
2804 case CPP_DEREF
: return c
== '*';
2805 case CPP_DOT
: return c
== '.' || c
== '%' || b
== CPP_NUMBER
;
2806 case CPP_HASH
: return c
== '#' || c
== '%'; /* Digraph form. */
2807 case CPP_NAME
: return ((b
== CPP_NUMBER
2808 && name_p (pfile
, &token2
->val
.str
))
2810 || b
== CPP_CHAR
|| b
== CPP_STRING
); /* L */
2811 case CPP_NUMBER
: return (b
== CPP_NUMBER
|| b
== CPP_NAME
2812 || c
== '.' || c
== '+' || c
== '-');
2814 case CPP_OTHER
: return ((token1
->val
.str
.text
[0] == '\\'
2816 || (CPP_OPTION (pfile
, objc
)
2817 && token1
->val
.str
.text
[0] == '@'
2818 && (b
== CPP_NAME
|| b
== CPP_STRING
)));
2821 case CPP_UTF8STRING
:
2823 case CPP_STRING32
: return (CPP_OPTION (pfile
, user_literals
)
2825 || (TOKEN_SPELL (token2
) == SPELL_LITERAL
2826 && ISIDST (token2
->val
.str
.text
[0]))));
2834 /* Output all the remaining tokens on the current line, and a newline
2835 character, to FP. Leading whitespace is removed. If there are
2836 macros, special token padding is not performed. */
2838 cpp_output_line (cpp_reader
*pfile
, FILE *fp
)
2840 const cpp_token
*token
;
2842 token
= cpp_get_token (pfile
);
2843 while (token
->type
!= CPP_EOF
)
2845 cpp_output_token (token
, fp
);
2846 token
= cpp_get_token (pfile
);
2847 if (token
->flags
& PREV_WHITE
)
2854 /* Return a string representation of all the remaining tokens on the
2855 current line. The result is allocated using xmalloc and must be
2856 freed by the caller. */
2858 cpp_output_line_to_string (cpp_reader
*pfile
, const unsigned char *dir_name
)
2860 const cpp_token
*token
;
2861 unsigned int out
= dir_name
? ustrlen (dir_name
) : 0;
2862 unsigned int alloced
= 120 + out
;
2863 unsigned char *result
= (unsigned char *) xmalloc (alloced
);
2865 /* If DIR_NAME is empty, there are no initial contents. */
2868 sprintf ((char *) result
, "#%s ", dir_name
);
2872 token
= cpp_get_token (pfile
);
2873 while (token
->type
!= CPP_EOF
)
2875 unsigned char *last
;
2876 /* Include room for a possible space and the terminating nul. */
2877 unsigned int len
= cpp_token_len (token
) + 2;
2879 if (out
+ len
> alloced
)
2882 if (out
+ len
> alloced
)
2883 alloced
= out
+ len
;
2884 result
= (unsigned char *) xrealloc (result
, alloced
);
2887 last
= cpp_spell_token (pfile
, token
, &result
[out
], 0);
2888 out
= last
- result
;
2890 token
= cpp_get_token (pfile
);
2891 if (token
->flags
& PREV_WHITE
)
2892 result
[out
++] = ' ';
2899 /* Memory buffers. Changing these three constants can have a dramatic
2900 effect on performance. The values here are reasonable defaults,
2901 but might be tuned. If you adjust them, be sure to test across a
2902 range of uses of cpplib, including heavy nested function-like macro
2903 expansion. Also check the change in peak memory usage (NJAMD is a
2904 good tool for this). */
2905 #define MIN_BUFF_SIZE 8000
2906 #define BUFF_SIZE_UPPER_BOUND(MIN_SIZE) (MIN_BUFF_SIZE + (MIN_SIZE) * 3 / 2)
2907 #define EXTENDED_BUFF_SIZE(BUFF, MIN_EXTRA) \
2908 (MIN_EXTRA + ((BUFF)->limit - (BUFF)->cur) * 2)
2910 #if MIN_BUFF_SIZE > BUFF_SIZE_UPPER_BOUND (0)
2911 #error BUFF_SIZE_UPPER_BOUND must be at least as large as MIN_BUFF_SIZE!
2914 /* Create a new allocation buffer. Place the control block at the end
2915 of the buffer, so that buffer overflows will cause immediate chaos. */
2917 new_buff (size_t len
)
2920 unsigned char *base
;
2922 if (len
< MIN_BUFF_SIZE
)
2923 len
= MIN_BUFF_SIZE
;
2924 len
= CPP_ALIGN (len
);
2926 #ifdef ENABLE_VALGRIND_CHECKING
2927 /* Valgrind warns about uses of interior pointers, so put _cpp_buff
2929 size_t slen
= CPP_ALIGN2 (sizeof (_cpp_buff
), 2 * DEFAULT_ALIGNMENT
);
2930 base
= XNEWVEC (unsigned char, len
+ slen
);
2931 result
= (_cpp_buff
*) base
;
2934 base
= XNEWVEC (unsigned char, len
+ sizeof (_cpp_buff
));
2935 result
= (_cpp_buff
*) (base
+ len
);
2937 result
->base
= base
;
2939 result
->limit
= base
+ len
;
2940 result
->next
= NULL
;
2944 /* Place a chain of unwanted allocation buffers on the free list. */
2946 _cpp_release_buff (cpp_reader
*pfile
, _cpp_buff
*buff
)
2948 _cpp_buff
*end
= buff
;
2952 end
->next
= pfile
->free_buffs
;
2953 pfile
->free_buffs
= buff
;
2956 /* Return a free buffer of size at least MIN_SIZE. */
2958 _cpp_get_buff (cpp_reader
*pfile
, size_t min_size
)
2960 _cpp_buff
*result
, **p
;
2962 for (p
= &pfile
->free_buffs
;; p
= &(*p
)->next
)
2967 return new_buff (min_size
);
2969 size
= result
->limit
- result
->base
;
2970 /* Return a buffer that's big enough, but don't waste one that's
2972 if (size
>= min_size
&& size
<= BUFF_SIZE_UPPER_BOUND (min_size
))
2977 result
->next
= NULL
;
2978 result
->cur
= result
->base
;
2982 /* Creates a new buffer with enough space to hold the uncommitted
2983 remaining bytes of BUFF, and at least MIN_EXTRA more bytes. Copies
2984 the excess bytes to the new buffer. Chains the new buffer after
2985 BUFF, and returns the new buffer. */
2987 _cpp_append_extend_buff (cpp_reader
*pfile
, _cpp_buff
*buff
, size_t min_extra
)
2989 size_t size
= EXTENDED_BUFF_SIZE (buff
, min_extra
);
2990 _cpp_buff
*new_buff
= _cpp_get_buff (pfile
, size
);
2992 buff
->next
= new_buff
;
2993 memcpy (new_buff
->base
, buff
->cur
, BUFF_ROOM (buff
));
2997 /* Creates a new buffer with enough space to hold the uncommitted
2998 remaining bytes of the buffer pointed to by BUFF, and at least
2999 MIN_EXTRA more bytes. Copies the excess bytes to the new buffer.
3000 Chains the new buffer before the buffer pointed to by BUFF, and
3001 updates the pointer to point to the new buffer. */
3003 _cpp_extend_buff (cpp_reader
*pfile
, _cpp_buff
**pbuff
, size_t min_extra
)
3005 _cpp_buff
*new_buff
, *old_buff
= *pbuff
;
3006 size_t size
= EXTENDED_BUFF_SIZE (old_buff
, min_extra
);
3008 new_buff
= _cpp_get_buff (pfile
, size
);
3009 memcpy (new_buff
->base
, old_buff
->cur
, BUFF_ROOM (old_buff
));
3010 new_buff
->next
= old_buff
;
3014 /* Free a chain of buffers starting at BUFF. */
3016 _cpp_free_buff (_cpp_buff
*buff
)
3020 for (; buff
; buff
= next
)
3023 #ifdef ENABLE_VALGRIND_CHECKING
3031 /* Allocate permanent, unaligned storage of length LEN. */
3033 _cpp_unaligned_alloc (cpp_reader
*pfile
, size_t len
)
3035 _cpp_buff
*buff
= pfile
->u_buff
;
3036 unsigned char *result
= buff
->cur
;
3038 if (len
> (size_t) (buff
->limit
- result
))
3040 buff
= _cpp_get_buff (pfile
, len
);
3041 buff
->next
= pfile
->u_buff
;
3042 pfile
->u_buff
= buff
;
3046 buff
->cur
= result
+ len
;
3050 /* Allocate permanent, unaligned storage of length LEN from a_buff.
3051 That buffer is used for growing allocations when saving macro
3052 replacement lists in a #define, and when parsing an answer to an
3053 assertion in #assert, #unassert or #if (and therefore possibly
3054 whilst expanding macros). It therefore must not be used by any
3055 code that they might call: specifically the lexer and the guts of
3058 All existing other uses clearly fit this restriction: storing
3059 registered pragmas during initialization. */
3061 _cpp_aligned_alloc (cpp_reader
*pfile
, size_t len
)
3063 _cpp_buff
*buff
= pfile
->a_buff
;
3064 unsigned char *result
= buff
->cur
;
3066 if (len
> (size_t) (buff
->limit
- result
))
3068 buff
= _cpp_get_buff (pfile
, len
);
3069 buff
->next
= pfile
->a_buff
;
3070 pfile
->a_buff
= buff
;
3074 buff
->cur
= result
+ len
;
3078 /* Say which field of TOK is in use. */
3080 enum cpp_token_fld_kind
3081 cpp_token_val_index (const cpp_token
*tok
)
3083 switch (TOKEN_SPELL (tok
))
3086 return CPP_TOKEN_FLD_NODE
;
3088 return CPP_TOKEN_FLD_STR
;
3089 case SPELL_OPERATOR
:
3090 if (tok
->type
== CPP_PASTE
)
3091 return CPP_TOKEN_FLD_TOKEN_NO
;
3093 return CPP_TOKEN_FLD_NONE
;
3095 if (tok
->type
== CPP_MACRO_ARG
)
3096 return CPP_TOKEN_FLD_ARG_NO
;
3097 else if (tok
->type
== CPP_PADDING
)
3098 return CPP_TOKEN_FLD_SOURCE
;
3099 else if (tok
->type
== CPP_PRAGMA
)
3100 return CPP_TOKEN_FLD_PRAGMA
;
3101 /* else fall through */
3103 return CPP_TOKEN_FLD_NONE
;
3107 /* All tokens lexed in R after calling this function will be forced to have
3108 their source_location the same as the location referenced by P, until
3109 cpp_stop_forcing_token_locations is called for R. */
3112 cpp_force_token_locations (cpp_reader
*r
, source_location
*p
)
3114 r
->forced_token_location_p
= p
;
3117 /* Go back to assigning locations naturally for lexed tokens. */
3120 cpp_stop_forcing_token_locations (cpp_reader
*r
)
3122 r
->forced_token_location_p
= NULL
;