2 * Wrc preprocessor lexical analysis
4 * Copyright 1999-2000 Bertho A. Stultiens (BS)
6 * 24-Apr-2000 BS - Started from scratch to restructure everything
7 * and reintegrate the source into the wine-tree.
8 * 04-Jan-2000 BS - Added comments about the lexicographical
9 * grammar to give some insight in the complexity.
10 * 28-Dec-1999 BS - Eliminated backing-up of the flexer by running
11 * `flex -b' on the source. This results in some
12 * weirdo extra rules, but a much faster scanner.
13 * 23-Dec-1999 BS - Started this file
15 *-------------------------------------------------------------------------
16 * The preprocessor's lexographical grammar (approximately):
18 * pp := {ws} # {ws} if {ws} {expr} {ws} \n
19 * | {ws} # {ws} ifdef {ws} {id} {ws} \n
20 * | {ws} # {ws} ifndef {ws} {id} {ws} \n
21 * | {ws} # {ws} elif {ws} {expr} {ws} \n
22 * | {ws} # {ws} else {ws} \n
23 * | {ws} # {ws} endif {ws} \n
24 * | {ws} # {ws} include {ws} < {anytext} > \n
25 * | {ws} # {ws} include {ws} " {anytext} " \n
26 * | {ws} # {ws} define {ws} {anytext} \n
27 * | {ws} # {ws} define( {arglist} ) {ws} {expansion} \n
28 * | {ws} # {ws} pragma {ws} {anytext} \n
29 * | {ws} # {ws} ident {ws} {anytext} \n
30 * | {ws} # {ws} error {ws} {anytext} \n
31 * | {ws} # {ws} warning {ws} {anytext} \n
32 * | {ws} # {ws} line {ws} " {anytext} " {number} \n
33 * | {ws} # {ws} {number} " {anytext} " {number} [{number} [{number}]] \n
38 * expr := {expr} [+-*%^/|&] {expr}
39 * | {expr} {logor|logand} {expr}
41 * | {expr} ? {expr} : {expr}
47 * id := [a-zA-Z_][a-zA-Z0-9_]*
49 * anytext := [^\n]* (see note)
54 * | {arglist} , {id} ...
59 * | {anytext} ## {anytext}
63 * Note: "anytext" is not always "[^\n]*". This is because the
64 * trailing context must be considered as well.
66 * The only certain assumption for the preprocessor to make is that
67 * directives start at the beginning of the line, followed by a '#'
68 * and end with a newline.
69 * Any directive may be suffixed with a line-continuation. Also
70 * classical comment / *...* / (note: no comments within comments,
71 * therefore spaces) is considered to be a line-continuation
72 * (according to gcc and egcs AFAIK, ANSI is a bit vague).
73 * Comments have not been added to the above grammer for simplicity
74 * reasons. However, it is allowed to enter comment anywhere within
75 * the directives as long as they do not interfere with the context.
76 * All comments are considered to be deletable whitespace (both
77 * classical form "/ *...* /" and C++ form "//...\n").
79 * All recursive scans, except for macro-expansion, are done by the
80 * parser, whereas the simple state transitions of non-recursive
81 * directives are done in the scanner. This results in the many
82 * exclusive start-conditions of the scanner.
84 * Macro expansions are slightly more difficult because they have to
85 * prescan the arguments. Parameter substitution is literal if the
86 * substitution is # or ## (either side). This enables new identifiers
87 * to be created (see 'info cpp' node Macro|Pitfalls|Prescan for more
90 * FIXME: Variable macro parameters is recognized, but not yet
91 * expanded. I have to reread the ANSI standard on the subject (yes,
94 * The following special defines are supported:
95 * __FILE__ -> "thissource.c"
97 * __DATE__ -> "May 1 2000"
98 * __TIME__ -> "23:59:59"
99 * These macros expand, as expected, into their ANSI defined values.
101 * The same include prevention is implemented as gcc and egcs does.
102 * This results in faster processing because we do not read the text
103 * at all. Some wine-sources attempt to include the same file 4 or 5
104 * times. This strategy also saves a lot blank output-lines, which in
105 * its turn improves the real resource scanner/parser.
110 * Special flex options and exclusive scanner start-conditions
113 %option never-interactive
136 cident [a-zA-Z_][0-9a-zA-Z_]*
137 ul [uUlL]|[uUlL][lL]|[lL][uU]|[lL][lL][uU]|[uU][lL][lL]|[lL][uU][lL]
155 * Make sure that we are running an appropriate version of flex.
157 #if !defined(YY_FLEX_MAJOR_VERSION) || (1000 * YY_FLEX_MAJOR_VERSION + YY_FLEX_MINOR_VERSION < 2005)
158 #error Must use flex version 2.5.1 or higher (yy_scan_* routines are required).
161 #define YY_USE_PROTOS
163 #define YY_READ_BUF_SIZE 65536 /* So we read most of a file at once */
165 #define yy_current_state() YY_START
166 #define yy_pp_state(x) yy_pop_state(); yy_push_state(x)
169 * Always update the current character position within a line
171 #define YY_USER_ACTION char_number+=ppleng;
174 * Buffer management for includes and expansions
176 #define MAXBUFFERSTACK 128 /* Nesting more than 128 includes or macro expansion textss is insane */
178 typedef struct bufferstackentry {
179 YY_BUFFER_STATE bufferstate; /* Buffer to switch back to */
180 pp_entry_t *define; /* Points to expanding define or NULL if handling includes */
181 int line_number; /* Line that we were handling */
182 int char_number; /* The current position on that line */
183 char *filename; /* Filename that we were handling */
184 int if_depth; /* How many #if:s deep to check matching #endif:s */
185 int ncontinuations; /* Remember the continuation state */
186 int should_pop; /* Set if we must pop the start-state on EOF */
187 /* Include management */
190 char *include_filename;
193 } bufferstackentry_t;
195 #define ALLOCBLOCKSIZE (1 << 10) /* Allocate these chunks at a time for string-buffers */
198 * Macro expansion nesting
199 * We need the stack to handle expansions while scanning
200 * a macro's arguments. The TOS must always be the macro
201 * that receives the current expansion from the scanner.
203 #define MAXMACEXPSTACK 128 /* Nesting more than 128 macro expansions is insane */
205 typedef struct macexpstackentry {
206 pp_entry_t *ppp; /* This macro we are scanning */
207 char **args; /* With these arguments */
208 char **ppargs; /* Resulting in these preprocessed arguments */
209 int *nnls; /* Number of newlines per argument */
210 int nargs; /* And this many arguments scanned */
211 int parentheses; /* Nesting level of () */
212 int curargsize; /* Current scanning argument's size */
213 int curargalloc; /* Current scanning argument's block allocated */
214 char *curarg; /* Current scanning argument's content */
215 } macexpstackentry_t;
217 #define MACROPARENTHESES() (top_macro()->parentheses)
222 static void newline(int);
223 static int make_number(int radix, YYSTYPE *val, char *str, int len);
224 static void put_buffer(char *s, int len);
225 /* Buffer management */
226 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop);
227 static bufferstackentry_t *pop_buffer(void);
228 /* String functions */
229 static void new_string(void);
230 static void add_string(char *str, int len);
231 static char *get_string(void);
232 static void put_string(void);
233 static int string_start(void);
234 /* Macro functions */
235 static void push_macro(pp_entry_t *ppp);
236 static macexpstackentry_t *top_macro(void);
237 static macexpstackentry_t *pop_macro(void);
238 static void free_macro(macexpstackentry_t *mep);
239 static void add_text_to_macro(char *text, int len);
240 static void macro_add_arg(int last);
241 static void macro_add_expansion(void);
243 static void expand_special(pp_entry_t *ppp);
244 static void expand_define(pp_entry_t *ppp);
245 static void expand_macro(macexpstackentry_t *mep);
250 static int ncontinuations;
252 static int strbuf_idx = 0;
253 static int strbuf_alloc = 0;
254 static char *strbuffer = NULL;
255 static int str_startline;
257 static macexpstackentry_t *macexpstack[MAXMACEXPSTACK];
258 static int macexpstackidx = 0;
260 static bufferstackentry_t bufferstack[MAXBUFFERSTACK];
261 static int bufferstackidx = 0;
267 * Trace the include files to prevent double reading.
268 * This save 20..30% of processing time for most stuff
269 * that uses complex includes.
271 * -1 Don't track or seen junk
272 * 0 New include, waiting for "#ifndef __xxx_h"
273 * 1 Seen #ifndef, waiting for "#define __xxx_h ..."
274 * 2 Seen #endif, waiting for EOF
276 int include_state = -1;
277 char *include_ppp = NULL; /* The define to be set from the #ifndef */
278 int include_ifdepth = 0; /* The level of ifs at the #ifdef */
279 int seen_junk = 0; /* Set when junk is seen */
280 includelogicentry_t *includelogiclist = NULL;
285 **************************************************************************
286 * The scanner starts here
287 **************************************************************************
292 * Catch line-continuations.
293 * Note: Gcc keeps the line-continuations in, for example, strings
294 * intact. However, I prefer to remove them all so that the next
295 * scanner will not need to reduce the continuation state.
300 * Detect the leading # of a preprocessor directive.
302 <INITIAL,pp_ignore>^{ws}*# seen_junk++; yy_push_state(pp_pp);
305 * Scan for the preprocessor directives
307 <pp_pp>{ws}*include{ws}* if(yy_top_state() != pp_ignore) {yy_pp_state(pp_inc); return tINCLUDE;} else {yy_pp_state(pp_eol);}
308 <pp_pp>{ws}*define{ws}* yy_pp_state(yy_current_state() != pp_ignore ? pp_def : pp_eol);
309 <pp_pp>{ws}*error{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tERROR;
310 <pp_pp>{ws}*warning{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tWARNING;
311 <pp_pp>{ws}*pragma{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tPRAGMA;
312 <pp_pp>{ws}*ident{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tPPIDENT;
313 <pp_pp>{ws}*undef{ws}* if(yy_top_state() != pp_ignore) {yy_pp_state(pp_ifd); return tUNDEF;} else {yy_pp_state(pp_eol);}
314 <pp_pp>{ws}*ifdef{ws}* yy_pp_state(pp_ifd); return tIFDEF;
315 <pp_pp>{ws}*ifndef{ws}* seen_junk--; yy_pp_state(pp_ifd); return tIFNDEF;
316 <pp_pp>{ws}*if{ws}* yy_pp_state(pp_if); return tIF;
317 <pp_pp>{ws}*elif{ws}* yy_pp_state(pp_if); return tELIF;
318 <pp_pp>{ws}*else{ws}* return tELSE;
319 <pp_pp>{ws}*endif{ws}* return tENDIF;
320 <pp_pp>{ws}*line{ws}* if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tLINE;} else {yy_pp_state(pp_eol);}
321 <pp_pp>{ws}+ if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tGCCLINE;} else {yy_pp_state(pp_eol);}
322 <pp_pp>{ws}*[a-z]+ pperror("Invalid preprocessor token '%s'", pptext);
323 <pp_pp>\n newline(1); yy_pop_state(); return tNL; /* This could be the null-token */
324 <pp_pp>. return *pptext;
327 * Handle #include and #line
329 <pp_line>[0-9]+ return make_number(10, &pplval, pptext, ppleng);
330 <pp_inc>\< new_string(); add_string(pptext, ppleng); yy_push_state(pp_iqs);
331 <pp_inc,pp_line>\" new_string(); add_string(pptext, ppleng); yy_push_state(pp_dqs);
332 <pp_inc,pp_line>{ws}+ ;
333 <pp_inc,pp_line>\n newline(1); yy_pop_state(); return tNL;
334 <pp_inc,pp_line>. pperror(yy_current_state() == pp_inc ? "Trailing junk in #include" : "Trailing junk in #line");
337 * Ignore all input when a false clause is parsed
339 <pp_ignore>[^#/\\\n]+ ;
340 <pp_ignore>\n newline(1);
344 * Handle #if and #elif.
345 * These require conditionals to be evaluated, but we do not
346 * want to jam the scanner normally when we see these tokens.
347 * Note: tIDENT is handled below.
350 <pp_if>0[0-7]*{ul}? return make_number(8, &pplval, pptext, ppleng);
351 <pp_if>0[0-7]*[8-9]+{ul}? pperror("Invalid octal digit");
352 <pp_if>[1-9][0-9]*{ul}? return make_number(10, &pplval, pptext, ppleng);
353 <pp_if>0[xX][0-9a-fA-F]+{ul}? return make_number(16, &pplval, pptext, ppleng);
354 <pp_if>0[xX] pperror("Invalid hex number");
355 <pp_if>defined yy_push_state(pp_defined); return tDEFINED;
356 <pp_if>"<<" return tLSHIFT;
357 <pp_if>">>" return tRSHIFT;
358 <pp_if>"&&" return tLOGAND;
359 <pp_if>"||" return tLOGOR;
360 <pp_if>"==" return tEQ;
361 <pp_if>"!=" return tNE;
362 <pp_if>"<=" return tLTE;
363 <pp_if>">=" return tGTE;
364 <pp_if>\n newline(1); yy_pop_state(); return tNL;
366 <pp_if>\' new_string(); add_string(pptext, ppleng); yy_push_state(pp_sqs);
367 <pp_if>\" pperror("String constants not allowed in conditionals");
368 <pp_if>. return *pptext;
371 * Handle #ifdef, #ifndef and #undef
372 * to get only an untranslated/unexpanded identifier
374 <pp_ifd>{cident} pplval.cptr = xstrdup(pptext); return tIDENT;
376 <pp_ifd>\n newline(1); yy_pop_state(); return tNL;
377 <pp_ifd>. pperror("Identifier expected");
380 * Handle the special 'defined' keyword.
381 * This is necessary to get the identifier prior to any
384 <pp_defined>{cident} yy_pop_state(); pplval.cptr = xstrdup(pptext); return tIDENT;
386 <pp_defined>(\()|(\)) return *pptext;
387 <pp_defined>(\\.)|(\n)|(.) pperror("Identifier expected");
390 * Handle #error, #warning, #pragma and #ident.
391 * Pass everything literally to the parser, which
392 * will act appropriately.
393 * Comments are stripped from the literal text.
395 <pp_eol>[^/\\\n]+ if(yy_top_state() != pp_ignore) { pplval.cptr = xstrdup(pptext); return tLITERAL; }
396 <pp_eol>\/[^/\\\n*]* if(yy_top_state() != pp_ignore) { pplval.cptr = xstrdup(pptext); return tLITERAL; }
397 <pp_eol>(\\)|(\/[^/*]) if(yy_top_state() != pp_ignore) { pplval.cptr = xstrdup(pptext); return tLITERAL; }
398 <pp_eol>\n newline(1); yy_pop_state(); if(yy_current_state() != pp_ignore) { return tNL; }
401 * Handle left side of #define
403 <pp_def>{cident}\( pplval.cptr = xstrdup(pptext); pplval.cptr[ppleng-1] = '\0'; yy_pp_state(pp_macro); return tMACRO;
404 <pp_def>{cident} pplval.cptr = xstrdup(pptext); yy_pp_state(pp_define); return tDEFINE;
406 <pp_def>(\n)|(.) perror("Identifier expected");
409 * Scan the substitution of a define
411 <pp_define>[^'"/\\\n]+ pplval.cptr = xstrdup(pptext); return tLITERAL;
412 <pp_define>(\\)|(\/[^/*]) pplval.cptr = xstrdup(pptext); return tLITERAL;
413 <pp_define>\\\n{ws}+ newline(0); pplval.cptr = xstrdup(" "); return tLITERAL;
414 <pp_define>\n newline(1); yy_pop_state(); return tNL;
415 <pp_define>\' new_string(); add_string(pptext, ppleng); yy_push_state(pp_sqs);
416 <pp_define>\" new_string(); add_string(pptext, ppleng); yy_push_state(pp_dqs);
419 * Scan the definition macro arguments
421 <pp_macro>\){ws}* yy_pp_state(pp_mbody); return tMACROEND;
423 <pp_macro>{cident} pplval.cptr = xstrdup(pptext); return tIDENT;
424 <pp_macro>, return ',';
425 <pp_macro>"..." return tELIPSIS;
426 <pp_macro>(\n)|(.)|(\.\.?) pperror("Argument identifier expected");
429 * Scan the substitution of a macro
431 <pp_mbody>[^a-zA-Z0-9'"#/\\\n]+ pplval.cptr = xstrdup(pptext); return tLITERAL;
432 <pp_mbody>{cident} pplval.cptr = xstrdup(pptext); return tIDENT;
433 <pp_mbody>\#\# return tCONCAT;
434 <pp_mbody>\# return tSTRINGIZE;
435 <pp_mbody>[0-9][^'"#/\\\n]* pplval.cptr = xstrdup(pptext); return tLITERAL;
436 <pp_mbody>(\\)|(\/[^/*'"#\\\n]*) pplval.cptr = xstrdup(pptext); return tLITERAL;
437 <pp_mbody>\\\n{ws}+ newline(0); pplval.cptr = xstrdup(" "); return tLITERAL;
438 <pp_mbody>\n newline(1); yy_pop_state(); return tNL;
439 <pp_mbody>\' new_string(); add_string(pptext, ppleng); yy_push_state(pp_sqs);
440 <pp_mbody>\" new_string(); add_string(pptext, ppleng); yy_push_state(pp_dqs);
443 * Macro expansion text scanning.
444 * This state is active just after the identifier is scanned
445 * that triggers an expansion. We *must* delete the leading
446 * whitespace before we can start scanning for arguments.
448 * If we do not see a '(' as next trailing token, then we have
449 * a false alarm. We just continue with a nose-bleed...
451 <pp_macign>{ws}*/\( yy_pp_state(pp_macscan);
453 if(yy_top_state() != pp_macscan)
456 <pp_macign>{ws}*\\\n newline(0);
457 <pp_macign>{ws}+|{ws}*\\|. {
458 macexpstackentry_t *mac = pop_macro();
460 put_buffer(mac->ppp->ident, strlen(mac->ppp->ident));
461 put_buffer(pptext, ppleng);
466 * Macro expansion argument text scanning.
467 * This state is active when a macro's arguments are being read for expansion.
470 if(++MACROPARENTHESES() > 1)
471 add_text_to_macro(pptext, ppleng);
474 if(--MACROPARENTHESES() == 0)
480 add_text_to_macro(pptext, ppleng);
483 if(MACROPARENTHESES() > 1)
484 add_text_to_macro(pptext, ppleng);
488 <pp_macscan>\" new_string(); add_string(pptext, ppleng); yy_push_state(pp_dqs);
489 <pp_macscan>\' new_string(); add_string(pptext, ppleng); yy_push_state(pp_sqs);
490 <pp_macscan>"/*" yy_push_state(pp_comment); add_text_to_macro(" ", 1);
491 <pp_macscan>\n line_number++; char_number = 1; add_text_to_macro(pptext, ppleng);
492 <pp_macscan>([^/(),\\\n"']+)|(\/[^/*(),\\\n'"]*)|(.) add_text_to_macro(pptext, ppleng);
495 * Comment handling (almost all start-conditions)
497 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_defined,pp_def,pp_define,pp_macro,pp_mbody>"/*" yy_push_state(pp_comment);
498 <pp_comment>[^*\n]*|"*"+[^*/\n]* ;
499 <pp_comment>\n newline(0);
500 <pp_comment>"*"+"/" yy_pop_state();
503 * Remove C++ style comment (almost all start-conditions)
505 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,pp_macscan>"//"[^\n]* {
506 if(pptext[ppleng-1] == '\\')
507 ppwarning("C++ style comment ends with an escaped newline (escape ignored)");
511 * Single, double and <> quoted constants
513 <INITIAL,pp_macexp>\" seen_junk++; new_string(); add_string(pptext, ppleng); yy_push_state(pp_dqs);
514 <INITIAL,pp_macexp>\' seen_junk++; new_string(); add_string(pptext, ppleng); yy_push_state(pp_sqs);
515 <pp_dqs>[^"\\\n]+ add_string(pptext, ppleng);
517 add_string(pptext, ppleng);
519 switch(yy_current_state())
526 pplval.cptr = get_string();
532 <pp_sqs>[^'\\\n]+ add_string(pptext, ppleng);
534 add_string(pptext, ppleng);
536 switch(yy_current_state())
541 pplval.cptr = get_string();
547 <pp_iqs>[^\>\\\n]+ add_string(pptext, ppleng);
549 add_string(pptext, ppleng);
551 pplval.cptr = get_string();
554 <pp_iqs,pp_dqs,pp_sqs>\\. add_string(pptext, ppleng);
555 <pp_iqs,pp_dqs,pp_sqs>\n {
557 add_string(pptext, ppleng);
558 ppwarning("Newline in string constant encounterd (started line %d)", string_start());
562 * Identifier scanning
564 <INITIAL,pp_if,pp_inc,pp_macexp>{cident} {
567 if(!(ppp = pplookup(pptext)))
569 if(yy_current_state() == pp_inc)
570 pperror("Expected include filename");
572 if(yy_current_state() == pp_if)
574 pplval.cptr = xstrdup(pptext);
578 put_buffer(pptext, ppleng);
580 else if(!ppp->expanding)
591 yy_push_state(pp_macign);
595 internal_error(__FILE__, __LINE__, "Invalid define type %d\n", ppp->type);
601 * Everything else that needs to be passed and
602 * newline and continuation handling
604 <INITIAL,pp_macexp>[^a-zA-Z_#'"/\\\n \r\t\f\v]+|(\/|\\)[^a-zA-Z_/*'"\\\n \r\t\v\f]* seen_junk++; put_buffer(pptext, ppleng);
605 <INITIAL,pp_macexp>{ws}+ put_buffer(pptext, ppleng);
606 <INITIAL>\n newline(1);
609 * Special catcher for macro argmument expansion to prevent
610 * newlines to propagate to the output or admin.
612 <pp_macexp>(\n)|(.) put_buffer(pptext, ppleng);
615 * This is a 'catch-all' rule to discover errors in the scanner
616 * in an orderly manner.
618 <*>. seen_junk++; ppwarning("Unmatched text '%c' (0x%02x); please report\n", isprint(*pptext) ? *pptext : ' ', *pptext);
621 YY_BUFFER_STATE b = YY_CURRENT_BUFFER;
622 bufferstackentry_t *bep = pop_buffer();
624 if((!bep && get_if_depth()) || (bep && get_if_depth() != bep->if_depth))
625 ppwarning("Unmatched #if/#endif at end of file");
629 if(YY_START != INITIAL)
630 pperror("Unexpected end of file during preprocessing");
633 else if(bep->should_pop == 2)
635 macexpstackentry_t *mac;
644 **************************************************************************
646 **************************************************************************
658 *-------------------------------------------------------------------------
659 * Output newlines or set them as continuations
660 *-------------------------------------------------------------------------
662 static void newline(int dowrite)
669 for(;ncontinuations; ncontinuations--)
676 *-------------------------------------------------------------------------
677 * Make a number out of an any-base and suffixed string
679 * Possible number extensions:
682 * - "LL" long long int
684 * - "UL" unsigned long int
685 * - "ULL" unsigned long long int
686 * - "LU" unsigned long int
687 * - "LLU" unsigned long long int
691 * The sizes of resulting 'int' and 'long' are compiler specific.
692 * I depend on sizeof(int) > 2 here (although a relatively safe
694 * Long longs are not yet implemented because this is very compiler
695 * specific and I don't want to think too much about the problems.
697 *-------------------------------------------------------------------------
699 static int make_number(int radix, YYSTYPE *val, char *str, int len)
707 ext[2] = toupper(str[len-1]);
708 ext[1] = len > 1 ? toupper(str[len-2]) : ' ';
709 ext[0] = len > 2 ? toupper(str[len-3]) : ' ';
711 if(!strcmp(ext, "LUL"))
712 pperror("Invalid constant suffix");
713 else if(!strcmp(ext, "LLU") || !strcmp(ext, "ULL"))
718 else if(!strcmp(ext+1, "LU") || !strcmp(ext+1, "UL"))
723 else if(!strcmp(ext+1, "LL"))
727 else if(!strcmp(ext+2, "L"))
731 else if(!strcmp(ext+2, "U"))
737 internal_error(__FILE__, __LINE__, "long long constants not implemented yet");
741 val->ulong = strtoul(str, NULL, radix);
744 else if(!is_u && is_l)
746 val->slong = strtol(str, NULL, radix);
749 else if(is_u && !is_l)
751 val->uint = (unsigned int)strtoul(str, NULL, radix);
752 if(!win32 && val->uint > 65535)
754 pperror("Constant overflow");
759 /* Else it must be an int... */
760 val->sint = (int)strtol(str, NULL, radix);
761 if(!win32 && (val->sint < -32768 || val->sint > 32768))
764 * Note: test must be > 32768 because unary minus
765 * is handled as an expression! This can result in
766 * failure and must be checked in the parser.
768 pperror("Constant overflow");
775 *-------------------------------------------------------------------------
776 * Macro and define expansion support
778 * FIXME: Variable macro arguments.
779 *-------------------------------------------------------------------------
781 static void expand_special(pp_entry_t *ppp)
784 static char *buf = NULL;
786 assert(ppp->type == def_special);
788 if(!strcmp(ppp->ident, "__LINE__"))
790 dbgtext = "def_special(__LINE__)";
791 buf = xrealloc(buf, 32);
792 sprintf(buf, "%d", line_number);
794 else if(!strcmp(ppp->ident, "__FILE__"))
796 dbgtext = "def_special(__FILE__)";
797 buf = xrealloc(buf, strlen(input_name) + 3);
798 sprintf(buf, "\"%s\"", input_name);
800 else if(!strcmp(ppp->ident, "__DATE__"))
802 dbgtext = "def_special(__DATE__)";
803 buf = xrealloc(buf, 32);
804 strftime(buf, 32, "\"%b %d %Y\"", localtime(&now));
806 else if(!strcmp(ppp->ident, "__TIME__"))
808 dbgtext = "def_special(__TIME__)";
809 buf = xrealloc(buf, 32);
810 strftime(buf, 32, "\"%H:%M:%S\"", localtime(&now));
813 internal_error(__FILE__, __LINE__, "Special macro '%s' not found...\n", ppp->ident);
815 if(debuglevel & DEBUGLEVEL_PPLEX)
816 fprintf(stderr, "expand_special(%d): %s:%d: '%s' -> '%s'\n",
825 push_buffer(ppp, NULL, NULL, 0);
830 static void expand_define(pp_entry_t *ppp)
832 assert(ppp->type == def_define);
834 if(debuglevel & DEBUGLEVEL_PPLEX)
835 fprintf(stderr, "expand_define(%d): %s:%d: '%s' -> '%s'\n",
841 if(ppp->subst.text && ppp->subst.text[0])
843 push_buffer(ppp, NULL, NULL, 0);
844 yy_scan_string(ppp->subst.text);
848 static int curdef_idx = 0;
849 static int curdef_alloc = 0;
850 static char *curdef_text = NULL;
852 static void add_text(char *str, int len)
856 if(curdef_idx >= curdef_alloc || curdef_alloc - curdef_idx < len)
858 curdef_alloc += (len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1);
859 curdef_text = xrealloc(curdef_text, curdef_alloc * sizeof(curdef_text[0]));
860 if(curdef_alloc > 65536)
861 ppwarning("Reallocating macro-expansion buffer larger than 64kB");
863 memcpy(&curdef_text[curdef_idx], str, len);
867 static mtext_t *add_expand_text(mtext_t *mtp, macexpstackentry_t *mep, int *nnl)
880 if(debuglevel & DEBUGLEVEL_PPLEX)
881 fprintf(stderr, "add_expand_text: exp_text: '%s'\n", mtp->subst.text);
882 add_text(mtp->subst.text, strlen(mtp->subst.text));
886 if(debuglevel & DEBUGLEVEL_PPLEX)
887 fprintf(stderr, "add_expand_text: exp_stringize(%d): '%s'\n",
889 mep->args[mtp->subst.argidx]);
890 cptr = mep->args[mtp->subst.argidx];
894 if(*cptr == '"' || *cptr == '\\')
903 if(debuglevel & DEBUGLEVEL_PPLEX)
904 fprintf(stderr, "add_expand_text: exp_concat\n");
905 /* Remove trailing whitespace from current expansion text */
908 if(isspace(curdef_text[curdef_idx-1]))
913 /* tag current position and recursively expand the next part */
915 mtp = add_expand_text(mtp->next, mep, nnl);
917 /* Now get rid of the leading space of the expansion */
918 cptr = &curdef_text[tag];
919 n = curdef_idx - tag;
930 if(cptr != &curdef_text[tag])
932 memmove(&curdef_text[tag], cptr, n);
933 curdef_idx -= (curdef_idx - tag) - n;
938 if((mtp->next && mtp->next->type == exp_concat) || (mtp->prev && mtp->prev->type == exp_concat))
939 exp = mep->args[mtp->subst.argidx];
941 exp = mep->ppargs[mtp->subst.argidx];
944 add_text(exp, strlen(exp));
945 *nnl -= mep->nnls[mtp->subst.argidx];
946 cptr = strchr(exp, '\n');
950 cptr = strchr(cptr+1, '\n');
952 mep->nnls[mtp->subst.argidx] = 0;
954 if(debuglevel & DEBUGLEVEL_PPLEX)
955 fprintf(stderr, "add_expand_text: exp_subst(%d): '%s'\n", mtp->subst.argidx, exp);
959 internal_error(__FILE__, __LINE__, "Invalid expansion type (%d) in macro expansion\n", mtp->type);
964 static void expand_macro(macexpstackentry_t *mep)
970 pp_entry_t *ppp = mep->ppp;
971 int nargs = mep->nargs;
973 assert(ppp->type == def_macro);
974 assert(ppp->expanding == 0);
976 if((ppp->nargs >= 0 && nargs != ppp->nargs) || (ppp->nargs < 0 && nargs < -ppp->nargs))
977 pperror("Too %s macro arguments (%d)", nargs < abs(ppp->nargs) ? "few" : "many", nargs);
979 for(n = 0; n < nargs; n++)
982 if(debuglevel & DEBUGLEVEL_PPLEX)
983 fprintf(stderr, "expand_macro(%d): %s:%d: '%s'(%d,%d) -> ...\n",
993 for(mtp = ppp->subst.mtext; mtp; mtp = mtp->next)
995 if(!(mtp = add_expand_text(mtp, mep, &nnl)))
999 for(n = 0; n < nnl; n++)
1002 /* To make sure there is room and termination (see below) */
1005 /* Strip trailing whitespace from expansion */
1006 for(k = curdef_idx, cptr = &curdef_text[curdef_idx-1]; k > 0; k--, cptr--)
1013 * We must add *one* whitespace to make sure that there
1014 * is a token-seperation after the expansion.
1020 /* Strip leading whitespace from expansion */
1021 for(n = 0, cptr = curdef_text; n < k; n++, cptr++)
1029 if(debuglevel & DEBUGLEVEL_PPLEX)
1030 fprintf(stderr, "expand_text: '%s'\n", curdef_text + n);
1031 push_buffer(ppp, NULL, NULL, 0);
1032 /*yy_scan_bytes(curdef_text + n, k - n);*/
1033 yy_scan_string(curdef_text + n);
1038 *-------------------------------------------------------------------------
1039 * String collection routines
1040 *-------------------------------------------------------------------------
1042 static void new_string(void)
1046 ppwarning("new_string: strbuf_idx != 0");
1049 str_startline = line_number;
1052 static void add_string(char *str, int len)
1056 if(strbuf_idx >= strbuf_alloc || strbuf_alloc - strbuf_idx < len)
1058 strbuf_alloc += (len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1);
1059 strbuffer = xrealloc(strbuffer, strbuf_alloc * sizeof(strbuffer[0]));
1060 if(strbuf_alloc > 65536)
1061 ppwarning("Reallocating string buffer larger than 64kB");
1063 memcpy(&strbuffer[strbuf_idx], str, len);
1067 static char *get_string(void)
1069 char *str = (char *)xmalloc(strbuf_idx + 1);
1070 memcpy(str, strbuffer, strbuf_idx);
1071 str[strbuf_idx] = '\0';
1078 static void put_string(void)
1080 put_buffer(strbuffer, strbuf_idx);
1086 static int string_start(void)
1088 return str_startline;
1093 *-------------------------------------------------------------------------
1095 *-------------------------------------------------------------------------
1097 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop)
1100 printf("push_buffer(%d): %p %p %p %d\n", bufferstackidx, ppp, filename, incname, pop);
1101 if(bufferstackidx >= MAXBUFFERSTACK)
1102 internal_error(__FILE__, __LINE__, "Buffer stack overflow");
1104 memset(&bufferstack[bufferstackidx], 0, sizeof(bufferstack[0]));
1105 bufferstack[bufferstackidx].bufferstate = YY_CURRENT_BUFFER;
1106 bufferstack[bufferstackidx].define = ppp;
1107 bufferstack[bufferstackidx].line_number = line_number;
1108 bufferstack[bufferstackidx].char_number = char_number;
1109 bufferstack[bufferstackidx].if_depth = get_if_depth();
1110 bufferstack[bufferstackidx].should_pop = pop;
1111 bufferstack[bufferstackidx].filename = input_name;
1112 bufferstack[bufferstackidx].ncontinuations = ncontinuations;
1113 bufferstack[bufferstackidx].include_state = include_state;
1114 bufferstack[bufferstackidx].include_ppp = include_ppp;
1115 bufferstack[bufferstackidx].include_filename = incname;
1116 bufferstack[bufferstackidx].include_ifdepth = include_ifdepth;
1117 bufferstack[bufferstackidx].seen_junk = seen_junk;
1123 /* These will track the pperror to the correct file and line */
1126 input_name = filename;
1130 internal_error(__FILE__, __LINE__, "Pushing buffer without knowing where to go to");
1134 static bufferstackentry_t *pop_buffer(void)
1136 if(bufferstackidx < 0)
1137 internal_error(__FILE__, __LINE__, "Bufferstack underflow?");
1139 if(bufferstackidx == 0)
1144 if(bufferstack[bufferstackidx].define)
1145 bufferstack[bufferstackidx].define->expanding = 0;
1148 line_number = bufferstack[bufferstackidx].line_number;
1149 char_number = bufferstack[bufferstackidx].char_number;
1150 input_name = bufferstack[bufferstackidx].filename;
1151 ncontinuations = bufferstack[bufferstackidx].ncontinuations;
1152 if(!bufferstack[bufferstackidx].should_pop)
1155 fprintf(ppout, "# %d \"%s\" 2\n", line_number, input_name);
1157 /* We have EOF, check the include logic */
1158 if(include_state == 2 && !seen_junk && include_ppp)
1160 pp_entry_t *ppp = pplookup(include_ppp);
1163 includelogicentry_t *iep = xmalloc(sizeof(includelogicentry_t));
1166 iep->filename = bufferstack[bufferstackidx].include_filename;
1167 iep->next = includelogiclist;
1169 iep->next->prev = iep;
1170 includelogiclist = iep;
1171 if(debuglevel & DEBUGLEVEL_PPMSG)
1172 fprintf(stderr, "pop_buffer: %s:%d: includelogic added, include_ppp='%s', file='%s'\n", input_name, line_number, include_ppp, iep->filename);
1174 else if(bufferstack[bufferstackidx].include_filename)
1175 free(bufferstack[bufferstackidx].include_filename);
1179 include_state = bufferstack[bufferstackidx].include_state;
1180 include_ppp = bufferstack[bufferstackidx].include_ppp;
1181 include_ifdepth = bufferstack[bufferstackidx].include_ifdepth;
1182 seen_junk = bufferstack[bufferstackidx].seen_junk;
1187 printf("pop_buffer(%d): %p %p (%d, %d, %d) %p %d\n",
1189 bufferstack[bufferstackidx].bufferstate,
1190 bufferstack[bufferstackidx].define,
1191 bufferstack[bufferstackidx].line_number,
1192 bufferstack[bufferstackidx].char_number,
1193 bufferstack[bufferstackidx].if_depth,
1194 bufferstack[bufferstackidx].filename,
1195 bufferstack[bufferstackidx].should_pop);
1197 pp_switch_to_buffer(bufferstack[bufferstackidx].bufferstate);
1199 if(bufferstack[bufferstackidx].should_pop)
1201 if(yy_current_state() == pp_macexp)
1202 macro_add_expansion();
1204 internal_error(__FILE__, __LINE__, "Pop buffer and state without macro expansion state");
1208 return &bufferstack[bufferstackidx];
1213 *-------------------------------------------------------------------------
1214 * Macro nestng support
1215 *-------------------------------------------------------------------------
1217 static void push_macro(pp_entry_t *ppp)
1219 if(macexpstackidx >= MAXMACEXPSTACK)
1220 pperror("Too many nested macros");
1222 macexpstack[macexpstackidx] = xmalloc(sizeof(macexpstack[0][0]));
1224 macexpstack[macexpstackidx]->ppp = ppp;
1228 static macexpstackentry_t *top_macro(void)
1230 return macexpstackidx > 0 ? macexpstack[macexpstackidx-1] : NULL;
1233 static macexpstackentry_t *pop_macro(void)
1235 if(macexpstackidx <= 0)
1236 internal_error(__FILE__, __LINE__, "Macro expansion stack underflow\n");
1237 return macexpstack[--macexpstackidx];
1240 static void free_macro(macexpstackentry_t *mep)
1244 for(i = 0; i < mep->nargs; i++)
1255 static void add_text_to_macro(char *text, int len)
1257 macexpstackentry_t *mep = top_macro();
1259 assert(mep->ppp->expanding == 0);
1261 if(mep->curargalloc - mep->curargsize <= len)
1263 mep->curargalloc += ALLOCBLOCKSIZE;
1264 mep->curarg = xrealloc(mep->curarg, mep->curargalloc * sizeof(mep->curarg[0]));
1266 memcpy(mep->curarg + mep->curargsize, text, len+1); /* +1 for '\0' */
1267 mep->curargsize += len;
1270 static void macro_add_arg(int last)
1274 macexpstackentry_t *mep = top_macro();
1276 assert(mep->ppp->expanding == 0);
1278 mep->args = xrealloc(mep->args, (mep->nargs+1) * sizeof(mep->args[0]));
1279 mep->ppargs = xrealloc(mep->ppargs, (mep->nargs+1) * sizeof(mep->ppargs[0]));
1280 mep->nnls = xrealloc(mep->nnls, (mep->nargs+1) * sizeof(mep->nnls[0]));
1281 mep->args[mep->nargs] = xstrdup(mep->curarg ? mep->curarg : "");
1282 cptr = mep->args[mep->nargs]-1;
1283 while((cptr = strchr(cptr+1, '\n')))
1287 mep->nnls[mep->nargs] = nnl;
1290 mep->curargalloc = mep->curargsize = 0;
1293 if(debuglevel & DEBUGLEVEL_PPLEX)
1294 fprintf(stderr, "macro_add_arg: %s:%d: %d -> '%s'\n",
1298 mep->args[mep->nargs-1]);
1300 /* Each macro argument must be expanded to cope with stingize */
1301 if(last || mep->args[mep->nargs-1][0])
1303 yy_push_state(pp_macexp);
1304 push_buffer(NULL, NULL, NULL, last ? 2 : 1);
1305 yy_scan_string(mep->args[mep->nargs-1]);
1306 /*mep->bufferstackidx = bufferstackidx; But not nested! */
1310 static void macro_add_expansion(void)
1312 macexpstackentry_t *mep = top_macro();
1314 assert(mep->ppp->expanding == 0);
1316 mep->ppargs[mep->nargs-1] = xstrdup(mep->curarg ? mep->curarg : "");
1318 mep->curargalloc = mep->curargsize = 0;
1321 if(debuglevel & DEBUGLEVEL_PPLEX)
1322 fprintf(stderr, "macro_add_expansion: %s:%d: %d -> '%s'\n",
1326 mep->ppargs[mep->nargs-1]);
1331 *-------------------------------------------------------------------------
1333 *-------------------------------------------------------------------------
1335 static void put_buffer(char *s, int len)
1338 add_text_to_macro(s, len);
1340 fwrite(s, 1, len, ppout);
1345 *-------------------------------------------------------------------------
1346 * Include management
1347 *-------------------------------------------------------------------------
1349 void do_include(char *fname, int type)
1353 includelogicentry_t *iep;
1355 for(iep = includelogiclist; iep; iep = iep->next)
1357 if(!strcmp(iep->filename, fname))
1360 * We are done. The file was included before.
1361 * If the define was deleted, then this entry would have
1371 pperror("Empty include filename");
1373 /* Undo the effect of the quotation */
1376 if((ppin = open_include(fname+1, type, &newpath)) == NULL)
1377 pperror("Unable to open include file %s", fname+1);
1379 fname[n-1] = *fname; /* Redo the quotes */
1380 push_buffer(NULL, newpath, fname, 0);
1384 if(debuglevel & DEBUGLEVEL_PPMSG)
1385 fprintf(stderr, "do_include: %s:%d: include_state=%d, include_ppp='%s', include_ifdepth=%d\n", input_name, line_number, include_state, include_ppp, include_ifdepth);
1386 pp_switch_to_buffer(pp_create_buffer(ppin, YY_BUF_SIZE));
1388 fprintf(ppout, "# 1 \"%s\" 1%s\n", newpath, type ? "" : " 3");
1392 *-------------------------------------------------------------------------
1393 * Push/pop preprocessor ignore state when processing conditionals
1395 *-------------------------------------------------------------------------
1397 void push_ignore_state(void)
1399 yy_push_state(pp_ignore);
1402 void pop_ignore_state(void)