2 * Wrc preprocessor lexical analysis
4 * Copyright 1999-2000 Bertho A. Stultiens (BS)
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
21 * 24-Apr-2000 BS - Started from scratch to restructure everything
22 * and reintegrate the source into the wine-tree.
23 * 04-Jan-2000 BS - Added comments about the lexicographical
24 * grammar to give some insight in the complexity.
25 * 28-Dec-1999 BS - Eliminated backing-up of the flexer by running
26 * `flex -b' on the source. This results in some
27 * weirdo extra rules, but a much faster scanner.
28 * 23-Dec-1999 BS - Started this file
30 *-------------------------------------------------------------------------
31 * The preprocessor's lexographical grammar (approximately):
33 * pp := {ws} # {ws} if {ws} {expr} {ws} \n
34 * | {ws} # {ws} ifdef {ws} {id} {ws} \n
35 * | {ws} # {ws} ifndef {ws} {id} {ws} \n
36 * | {ws} # {ws} elif {ws} {expr} {ws} \n
37 * | {ws} # {ws} else {ws} \n
38 * | {ws} # {ws} endif {ws} \n
39 * | {ws} # {ws} include {ws} < {anytext} > \n
40 * | {ws} # {ws} include {ws} " {anytext} " \n
41 * | {ws} # {ws} define {ws} {anytext} \n
42 * | {ws} # {ws} define( {arglist} ) {ws} {expansion} \n
43 * | {ws} # {ws} pragma {ws} {anytext} \n
44 * | {ws} # {ws} ident {ws} {anytext} \n
45 * | {ws} # {ws} error {ws} {anytext} \n
46 * | {ws} # {ws} warning {ws} {anytext} \n
47 * | {ws} # {ws} line {ws} " {anytext} " {number} \n
48 * | {ws} # {ws} {number} " {anytext} " {number} [ {number} [{number}] ] \n
53 * expr := {expr} [+-*%^/|&] {expr}
54 * | {expr} {logor|logand} {expr}
56 * | {expr} ? {expr} : {expr}
62 * id := [a-zA-Z_][a-zA-Z0-9_]*
64 * anytext := [^\n]* (see note)
69 * | {arglist} , {id} ...
74 * | {anytext} ## {anytext}
78 * Note: "anytext" is not always "[^\n]*". This is because the
79 * trailing context must be considered as well.
81 * The only certain assumption for the preprocessor to make is that
82 * directives start at the beginning of the line, followed by a '#'
83 * and end with a newline.
84 * Any directive may be suffixed with a line-continuation. Also
85 * classical comment / *...* / (note: no comments within comments,
86 * therefore spaces) is considered to be a line-continuation
87 * (according to gcc and egcs AFAIK, ANSI is a bit vague).
88 * Comments have not been added to the above grammar for simplicity
89 * reasons. However, it is allowed to enter comment anywhere within
90 * the directives as long as they do not interfere with the context.
91 * All comments are considered to be deletable whitespace (both
92 * classical form "/ *...* /" and C++ form "//...\n").
94 * All recursive scans, except for macro-expansion, are done by the
95 * parser, whereas the simple state transitions of non-recursive
96 * directives are done in the scanner. This results in the many
97 * exclusive start-conditions of the scanner.
99 * Macro expansions are slightly more difficult because they have to
100 * prescan the arguments. Parameter substitution is literal if the
101 * substitution is # or ## (either side). This enables new identifiers
102 * to be created (see 'info cpp' node Macro|Pitfalls|Prescan for more
105 * FIXME: Variable macro parameters is recognized, but not yet
106 * expanded. I have to reread the ANSI standard on the subject (yes,
109 * The following special defines are supported:
110 * __FILE__ -> "thissource.c"
112 * __DATE__ -> "May 1 2000"
113 * __TIME__ -> "23:59:59"
114 * These macros expand, as expected, into their ANSI defined values.
116 * The same include prevention is implemented as gcc and egcs does.
117 * This results in faster processing because we do not read the text
118 * at all. Some wine-sources attempt to include the same file 4 or 5
119 * times. This strategy also saves a lot blank output-lines, which in
120 * its turn improves the real resource scanner/parser.
125 * Special flex options and exclusive scanner start-conditions
128 %option 8bit never-interactive
130 %option prefix="ppy_"
155 cident [a-zA-Z_][0-9a-zA-Z_]*
156 ul [uUlL]|[uUlL][lL]|[lL][uU]|[lL][lL][uU]|[uU][lL][lL]|[lL][uU][lL]
166 #ifndef HAVE_UNISTD_H
167 #define YY_NO_UNISTD_H
170 #include "wpp_private.h"
174 * Make sure that we are running an appropriate version of flex.
176 #if !defined(YY_FLEX_MAJOR_VERSION) || (1000 * YY_FLEX_MAJOR_VERSION + YY_FLEX_MINOR_VERSION < 2005)
177 #error Must use flex version 2.5.1 or higher (yy_scan_* routines are required).
180 #define YY_READ_BUF_SIZE 65536 /* So we read most of a file at once */
182 #define yy_current_state() YY_START
183 #define yy_pp_state(x) yy_pop_state(); yy_push_state(x)
186 * Always update the current character position within a line
188 #define YY_USER_ACTION pp_status.char_number+=ppy_leng;
191 * Buffer management for includes and expansions
193 #define MAXBUFFERSTACK 128 /* Nesting more than 128 includes or macro expansion textss is insane */
195 typedef struct bufferstackentry {
196 YY_BUFFER_STATE bufferstate; /* Buffer to switch back to */
197 pp_entry_t *define; /* Points to expanding define or NULL if handling includes */
198 int line_number; /* Line that we were handling */
199 int char_number; /* The current position on that line */
200 const char *filename; /* Filename that we were handling */
201 int if_depth; /* How many #if:s deep to check matching #endif:s */
202 int ncontinuations; /* Remember the continuation state */
203 int should_pop; /* Set if we must pop the start-state on EOF */
204 /* Include management */
205 include_state_t incl;
206 char *include_filename;
207 } bufferstackentry_t;
209 #define ALLOCBLOCKSIZE (1 << 10) /* Allocate these chunks at a time for string-buffers */
212 * Macro expansion nesting
213 * We need the stack to handle expansions while scanning
214 * a macro's arguments. The TOS must always be the macro
215 * that receives the current expansion from the scanner.
217 #define MAXMACEXPSTACK 128 /* Nesting more than 128 macro expansions is insane */
219 typedef struct macexpstackentry {
220 pp_entry_t *ppp; /* This macro we are scanning */
221 char **args; /* With these arguments */
222 char **ppargs; /* Resulting in these preprocessed arguments */
223 int *nnls; /* Number of newlines per argument */
224 int nargs; /* And this many arguments scanned */
225 int parentheses; /* Nesting level of () */
226 int curargsize; /* Current scanning argument's size */
227 int curargalloc; /* Current scanning argument's block allocated */
228 char *curarg; /* Current scanning argument's content */
229 } macexpstackentry_t;
231 #define MACROPARENTHESES() (top_macro()->parentheses)
236 static void newline(int);
237 static int make_number(int radix, YYSTYPE *val, const char *str, int len);
238 static void put_buffer(const char *s, int len);
239 /* Buffer management */
240 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop);
241 static bufferstackentry_t *pop_buffer(void);
242 /* String functions */
243 static void new_string(void);
244 static void add_string(const char *str, int len);
245 static char *get_string(void);
246 static void put_string(void);
247 static int string_start(void);
248 /* Macro functions */
249 static void push_macro(pp_entry_t *ppp);
250 static macexpstackentry_t *top_macro(void);
251 static macexpstackentry_t *pop_macro(void);
252 static void free_macro(macexpstackentry_t *mep);
253 static void add_text_to_macro(const char *text, int len);
254 static void macro_add_arg(int last);
255 static void macro_add_expansion(void);
257 static void expand_special(pp_entry_t *ppp);
258 static void expand_define(pp_entry_t *ppp);
259 static void expand_macro(macexpstackentry_t *mep);
264 static int ncontinuations;
266 static int strbuf_idx = 0;
267 static int strbuf_alloc = 0;
268 static char *strbuffer = NULL;
269 static int str_startline;
271 static macexpstackentry_t *macexpstack[MAXMACEXPSTACK];
272 static int macexpstackidx = 0;
274 static bufferstackentry_t bufferstack[MAXBUFFERSTACK];
275 static int bufferstackidx = 0;
280 include_state_t pp_incl_state =
288 includelogicentry_t *pp_includelogiclist = NULL;
293 **************************************************************************
294 * The scanner starts here
295 **************************************************************************
300 * Catch line-continuations.
301 * Note: Gcc keeps the line-continuations in, for example, strings
302 * intact. However, I prefer to remove them all so that the next
303 * scanner will not need to reduce the continuation state.
305 * <*>\\\n newline(0);
309 * Detect the leading # of a preprocessor directive.
311 <INITIAL,pp_ignore>^{ws}*# pp_incl_state.seen_junk++; yy_push_state(pp_pp);
314 * Scan for the preprocessor directives
316 <pp_pp>{ws}*include{ws}* if(yy_top_state() != pp_ignore) {yy_pp_state(pp_inc); return tINCLUDE;} else {yy_pp_state(pp_eol);}
317 <pp_pp>{ws}*define{ws}* yy_pp_state(yy_current_state() != pp_ignore ? pp_def : pp_eol);
318 <pp_pp>{ws}*error{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tERROR;
319 <pp_pp>{ws}*warning{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tWARNING;
320 <pp_pp>{ws}*pragma{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tPRAGMA;
321 <pp_pp>{ws}*ident{ws}* yy_pp_state(pp_eol); if(yy_top_state() != pp_ignore) return tPPIDENT;
322 <pp_pp>{ws}*undef{ws}* if(yy_top_state() != pp_ignore) {yy_pp_state(pp_ifd); return tUNDEF;} else {yy_pp_state(pp_eol);}
323 <pp_pp>{ws}*ifdef{ws}* yy_pp_state(pp_ifd); return tIFDEF;
324 <pp_pp>{ws}*ifndef{ws}* pp_incl_state.seen_junk--; yy_pp_state(pp_ifd); return tIFNDEF;
325 <pp_pp>{ws}*if{ws}* yy_pp_state(pp_if); return tIF;
326 <pp_pp>{ws}*elif{ws}* yy_pp_state(pp_if); return tELIF;
327 <pp_pp>{ws}*else{ws}* yy_pp_state(pp_endif); return tELSE;
328 <pp_pp>{ws}*endif{ws}* yy_pp_state(pp_endif); return tENDIF;
329 <pp_pp>{ws}*line{ws}* if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tLINE;} else {yy_pp_state(pp_eol);}
330 <pp_pp>{ws}+ if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tGCCLINE;} else {yy_pp_state(pp_eol);}
331 <pp_pp>{ws}*[a-z]+ ppy_error("Invalid preprocessor token '%s'", ppy_text);
332 <pp_pp>\r?\n newline(1); yy_pop_state(); return tNL; /* This could be the null-token */
333 <pp_pp>\\\r?\n newline(0);
334 <pp_pp>\\\r? ppy_error("Preprocessor junk '%s'", ppy_text);
335 <pp_pp>. return *ppy_text;
338 * Handle #include and #line
340 <pp_line>[0-9]+ return make_number(10, &ppy_lval, ppy_text, ppy_leng);
341 <pp_inc>\< new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_iqs);
342 <pp_inc,pp_line>\" new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
343 <pp_inc,pp_line>{ws}+ ;
344 <pp_inc,pp_line>\n newline(1); yy_pop_state(); return tNL;
345 <pp_inc,pp_line>\\\r?\n newline(0);
346 <pp_inc,pp_line>(\\\r?)|(.) ppy_error(yy_current_state() == pp_inc ? "Trailing junk in #include" : "Trailing junk in #line");
349 * Ignore all input when a false clause is parsed
351 <pp_ignore>[^#/\\\n]+ ;
352 <pp_ignore>\n newline(1);
353 <pp_ignore>\\\r?\n newline(0);
354 <pp_ignore>(\\\r?)|(.) ;
357 * Handle #if and #elif.
358 * These require conditionals to be evaluated, but we do not
359 * want to jam the scanner normally when we see these tokens.
360 * Note: tIDENT is handled below.
363 <pp_if>0[0-7]*{ul}? return make_number(8, &ppy_lval, ppy_text, ppy_leng);
364 <pp_if>0[0-7]*[8-9]+{ul}? ppy_error("Invalid octal digit");
365 <pp_if>[1-9][0-9]*{ul}? return make_number(10, &ppy_lval, ppy_text, ppy_leng);
366 <pp_if>0[xX][0-9a-fA-F]+{ul}? return make_number(16, &ppy_lval, ppy_text, ppy_leng);
367 <pp_if>0[xX] ppy_error("Invalid hex number");
368 <pp_if>defined yy_push_state(pp_defined); return tDEFINED;
369 <pp_if>"<<" return tLSHIFT;
370 <pp_if>">>" return tRSHIFT;
371 <pp_if>"&&" return tLOGAND;
372 <pp_if>"||" return tLOGOR;
373 <pp_if>"==" return tEQ;
374 <pp_if>"!=" return tNE;
375 <pp_if>"<=" return tLTE;
376 <pp_if>">=" return tGTE;
377 <pp_if>\n newline(1); yy_pop_state(); return tNL;
378 <pp_if>\\\r?\n newline(0);
379 <pp_if>\\\r? ppy_error("Junk in conditional expression");
381 <pp_if>\' new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
382 <pp_if>\" ppy_error("String constants not allowed in conditionals");
383 <pp_if>. return *ppy_text;
386 * Handle #ifdef, #ifndef and #undef
387 * to get only an untranslated/unexpanded identifier
389 <pp_ifd>{cident} ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
391 <pp_ifd>\n newline(1); yy_pop_state(); return tNL;
392 <pp_ifd>\\\r?\n newline(0);
393 <pp_ifd>(\\\r?)|(.) ppy_error("Identifier expected");
396 * Handle #else and #endif.
399 <pp_endif>\n newline(1); yy_pop_state(); return tNL;
400 <pp_endif>\\\r?\n newline(0);
401 <pp_endif>. ppy_error("Garbage after #else or #endif.");
404 * Handle the special 'defined' keyword.
405 * This is necessary to get the identifier prior to any
408 <pp_defined>{cident} yy_pop_state(); ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
410 <pp_defined>(\()|(\)) return *ppy_text;
411 <pp_defined>\\\r?\n newline(0);
412 <pp_defined>(\\.)|(\n)|(.) ppy_error("Identifier expected");
415 * Handle #error, #warning, #pragma and #ident.
416 * Pass everything literally to the parser, which
417 * will act appropriately.
418 * Comments are stripped from the literal text.
420 <pp_eol>[^/\\\n]+ if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
421 <pp_eol>\/[^/\\\n*]* if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
422 <pp_eol>(\\\r?)|(\/[^/*]) if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
423 <pp_eol>\n newline(1); yy_pop_state(); if(yy_current_state() != pp_ignore) { return tNL; }
424 <pp_eol>\\\r?\n newline(0);
427 * Handle left side of #define
429 <pp_def>{cident}\( ppy_lval.cptr = pp_xstrdup(ppy_text); ppy_lval.cptr[ppy_leng-1] = '\0'; yy_pp_state(pp_macro); return tMACRO;
430 <pp_def>{cident} ppy_lval.cptr = pp_xstrdup(ppy_text); yy_pp_state(pp_define); return tDEFINE;
432 <pp_def>\\\r?\n newline(0);
433 <pp_def>(\\\r?)|(\n)|(.) perror("Identifier expected");
436 * Scan the substitution of a define
438 <pp_define>[^'"/\\\n]+ ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
439 <pp_define>(\\\r?)|(\/[^/*]) ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
440 <pp_define>\\\r?\n{ws}+ newline(0); ppy_lval.cptr = pp_xstrdup(" "); return tLITERAL;
441 <pp_define>\\\r?\n newline(0);
442 <pp_define>\n newline(1); yy_pop_state(); return tNL;
443 <pp_define>\' new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
444 <pp_define>\" new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
447 * Scan the definition macro arguments
449 <pp_macro>\){ws}* yy_pp_state(pp_mbody); return tMACROEND;
451 <pp_macro>{cident} ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
452 <pp_macro>, return ',';
453 <pp_macro>"..." return tELIPSIS;
454 <pp_macro>(\\\r?)|(\n)|(.)|(\.\.?) ppy_error("Argument identifier expected");
455 <pp_macro>\\\r?\n newline(0);
458 * Scan the substitution of a macro
460 <pp_mbody>[^a-zA-Z0-9'"#/\\\n]+ ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
461 <pp_mbody>{cident} ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
462 <pp_mbody>\#\# return tCONCAT;
463 <pp_mbody>\# return tSTRINGIZE;
464 <pp_mbody>[0-9][^'"#/\\\n]* ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
465 <pp_mbody>(\\\r?)|(\/[^/*'"#\\\n]*) ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
466 <pp_mbody>\\\r?\n{ws}+ newline(0); ppy_lval.cptr = pp_xstrdup(" "); return tLITERAL;
467 <pp_mbody>\\\r?\n newline(0);
468 <pp_mbody>\n newline(1); yy_pop_state(); return tNL;
469 <pp_mbody>\' new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
470 <pp_mbody>\" new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
473 * Macro expansion text scanning.
474 * This state is active just after the identifier is scanned
475 * that triggers an expansion. We *must* delete the leading
476 * whitespace before we can start scanning for arguments.
478 * If we do not see a '(' as next trailing token, then we have
479 * a false alarm. We just continue with a nose-bleed...
481 <pp_macign>{ws}*/\( yy_pp_state(pp_macscan);
483 if(yy_top_state() != pp_macscan)
486 <pp_macign>{ws}*\\\r?\n newline(0);
487 <pp_macign>{ws}+|{ws}*\\\r?|. {
488 macexpstackentry_t *mac = pop_macro();
490 put_buffer(mac->ppp->ident, strlen(mac->ppp->ident));
491 put_buffer(ppy_text, ppy_leng);
496 * Macro expansion argument text scanning.
497 * This state is active when a macro's arguments are being read for expansion.
500 if(++MACROPARENTHESES() > 1)
501 add_text_to_macro(ppy_text, ppy_leng);
504 if(--MACROPARENTHESES() == 0)
510 add_text_to_macro(ppy_text, ppy_leng);
513 if(MACROPARENTHESES() > 1)
514 add_text_to_macro(ppy_text, ppy_leng);
518 <pp_macscan>\" new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
519 <pp_macscan>\' new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
520 <pp_macscan>"/*" yy_push_state(pp_comment); add_text_to_macro(" ", 1);
521 <pp_macscan>\n pp_status.line_number++; pp_status.char_number = 1; add_text_to_macro(ppy_text, ppy_leng);
522 <pp_macscan>([^/(),\\\n"']+)|(\/[^/*(),\\\n'"]*)|(\\\r?)|(.) add_text_to_macro(ppy_text, ppy_leng);
523 <pp_macscan>\\\r?\n newline(0);
526 * Comment handling (almost all start-conditions)
528 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_endif,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,RCINCL>"/*" yy_push_state(pp_comment);
529 <pp_comment>[^*\n]*|"*"+[^*/\n]* ;
530 <pp_comment>\n newline(0);
531 <pp_comment>"*"+"/" yy_pop_state();
534 * Remove C++ style comment (almost all start-conditions)
536 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_endif,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,pp_macscan,RCINCL>"//"[^\n]* {
537 if(ppy_text[ppy_leng-1] == '\\')
538 ppy_warning("C++ style comment ends with an escaped newline (escape ignored)");
542 * Single, double and <> quoted constants
544 <INITIAL,pp_macexp>\" pp_incl_state.seen_junk++; new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
545 <INITIAL,pp_macexp>\' pp_incl_state.seen_junk++; new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
546 <pp_dqs>[^"\\\n]+ add_string(ppy_text, ppy_leng);
548 add_string(ppy_text, ppy_leng);
550 switch(yy_current_state())
557 if (yy_current_state()==RCINCL) yy_pop_state();
558 ppy_lval.cptr = get_string();
561 ppy_lval.cptr = get_string();
567 <pp_sqs>[^'\\\n]+ add_string(ppy_text, ppy_leng);
569 add_string(ppy_text, ppy_leng);
571 switch(yy_current_state())
576 ppy_lval.cptr = get_string();
582 <pp_iqs>[^\>\\\n]+ add_string(ppy_text, ppy_leng);
584 add_string(ppy_text, ppy_leng);
586 ppy_lval.cptr = get_string();
591 * This is tricky; we need to remove the line-continuation
592 * from preprocessor strings, but OTOH retain them in all
593 * other strings. This is because the resource grammar is
594 * even more braindead than initially analysed and line-
595 * continuations in strings introduce, sigh, newlines in
596 * the output. There goes the concept of non-breaking, non-
597 * spacing whitespace.
599 switch(yy_top_state())
609 add_string(ppy_text, ppy_leng);
613 <pp_iqs,pp_dqs,pp_sqs>\\. add_string(ppy_text, ppy_leng);
614 <pp_iqs,pp_dqs,pp_sqs>\n {
616 add_string(ppy_text, ppy_leng);
617 ppy_warning("Newline in string constant encounterd (started line %d)", string_start());
621 * Identifier scanning
623 <INITIAL,pp_if,pp_inc,pp_macexp>{cident} {
625 pp_incl_state.seen_junk++;
626 if(!(ppp = pplookup(ppy_text)))
628 if(yy_current_state() == pp_inc)
629 ppy_error("Expected include filename");
631 if(yy_current_state() == pp_if)
633 ppy_lval.cptr = pp_xstrdup(ppy_text);
637 if((yy_current_state()==INITIAL) && (strcasecmp(ppy_text,"RCINCLUDE")==0)){
638 yy_push_state(RCINCL);
641 else put_buffer(ppy_text, ppy_leng);
644 else if(!ppp->expanding)
655 yy_push_state(pp_macign);
659 pp_internal_error(__FILE__, __LINE__, "Invalid define type %d\n", ppp->type);
662 else put_buffer(ppy_text, ppy_leng);
666 * Everything else that needs to be passed and
667 * newline and continuation handling
669 <INITIAL,pp_macexp>[^a-zA-Z_#'"/\\\n \r\t\f\v]+|(\/|\\)[^a-zA-Z_/*'"\\\n \r\t\v\f]* pp_incl_state.seen_junk++; put_buffer(ppy_text, ppy_leng);
670 <INITIAL,pp_macexp>{ws}+ put_buffer(ppy_text, ppy_leng);
671 <INITIAL>\n newline(1);
672 <INITIAL>\\\r?\n newline(0);
673 <INITIAL>\\\r? pp_incl_state.seen_junk++; put_buffer(ppy_text, ppy_leng);
676 * Special catcher for macro argmument expansion to prevent
677 * newlines to propagate to the output or admin.
679 <pp_macexp>(\n)|(.)|(\\\r?(\n|.)) put_buffer(ppy_text, ppy_leng);
681 <RCINCL>[A-Za-z0-9_\.\\/]+ {
682 ppy_lval.cptr=pp_xstrdup(ppy_text);
684 return tRCINCLUDEPATH;
690 new_string(); add_string(ppy_text,ppy_leng);yy_push_state(pp_dqs);
694 * This is a 'catch-all' rule to discover errors in the scanner
695 * in an orderly manner.
697 <*>. pp_incl_state.seen_junk++; ppy_warning("Unmatched text '%c' (0x%02x); please report\n", isprint(*ppy_text & 0xff) ? *ppy_text : ' ', *ppy_text);
700 YY_BUFFER_STATE b = YY_CURRENT_BUFFER;
701 bufferstackentry_t *bep = pop_buffer();
703 if((!bep && pp_get_if_depth()) || (bep && pp_get_if_depth() != bep->if_depth))
704 ppy_warning("Unmatched #if/#endif at end of file");
708 if(YY_START != INITIAL)
709 ppy_error("Unexpected end of file during preprocessing");
712 else if(bep->should_pop == 2)
714 macexpstackentry_t *mac;
718 ppy__delete_buffer(b);
723 **************************************************************************
725 **************************************************************************
737 *-------------------------------------------------------------------------
738 * Output newlines or set them as continuations
740 * Input: -1 - Don't count this one, but update local position (see pp_dqs)
741 * 0 - Line-continuation seen and cache output
742 * 1 - Newline seen and flush output
743 *-------------------------------------------------------------------------
745 static void newline(int dowrite)
747 pp_status.line_number++;
748 pp_status.char_number = 1;
756 for(;ncontinuations; ncontinuations--)
763 *-------------------------------------------------------------------------
764 * Make a number out of an any-base and suffixed string
766 * Possible number extensions:
769 * - "LL" long long int
771 * - "UL" unsigned long int
772 * - "ULL" unsigned long long int
773 * - "LU" unsigned long int
774 * - "LLU" unsigned long long int
778 * The sizes of resulting 'int' and 'long' are compiler specific.
779 * I depend on sizeof(int) > 2 here (although a relatively safe
781 * Long longs are not yet implemented because this is very compiler
782 * specific and I don't want to think too much about the problems.
784 *-------------------------------------------------------------------------
786 static int make_number(int radix, YYSTYPE *val, const char *str, int len)
794 ext[2] = toupper(str[len-1]);
795 ext[1] = len > 1 ? toupper(str[len-2]) : ' ';
796 ext[0] = len > 2 ? toupper(str[len-3]) : ' ';
798 if(!strcmp(ext, "LUL"))
799 ppy_error("Invalid constant suffix");
800 else if(!strcmp(ext, "LLU") || !strcmp(ext, "ULL"))
805 else if(!strcmp(ext+1, "LU") || !strcmp(ext+1, "UL"))
810 else if(!strcmp(ext+1, "LL"))
814 else if(!strcmp(ext+2, "L"))
818 else if(!strcmp(ext+2, "U"))
825 /* Assume as in the declaration of wrc_ull_t and wrc_sll_t */
826 #ifdef HAVE_LONG_LONG
829 val->ull = strtoull(str, NULL, radix);
834 val->sll = strtoll(str, NULL, radix);
838 pp_internal_error(__FILE__, __LINE__, "long long constants not supported on this platform");
841 else if(is_u && is_l)
843 val->ulong = strtoul(str, NULL, radix);
846 else if(!is_u && is_l)
848 val->slong = strtol(str, NULL, radix);
851 else if(is_u && !is_l)
853 val->uint = (unsigned int)strtoul(str, NULL, radix);
857 /* Else it must be an int... */
858 val->sint = (int)strtol(str, NULL, radix);
864 *-------------------------------------------------------------------------
865 * Macro and define expansion support
867 * FIXME: Variable macro arguments.
868 *-------------------------------------------------------------------------
870 static void expand_special(pp_entry_t *ppp)
872 const char *dbgtext = "?";
873 static char *buf = NULL;
875 assert(ppp->type == def_special);
877 if(!strcmp(ppp->ident, "__LINE__"))
879 dbgtext = "def_special(__LINE__)";
880 buf = pp_xrealloc(buf, 32);
881 sprintf(buf, "%d", pp_status.line_number);
883 else if(!strcmp(ppp->ident, "__FILE__"))
885 dbgtext = "def_special(__FILE__)";
886 buf = pp_xrealloc(buf, strlen(pp_status.input) + 3);
887 sprintf(buf, "\"%s\"", pp_status.input);
890 pp_internal_error(__FILE__, __LINE__, "Special macro '%s' not found...\n", ppp->ident);
893 fprintf(stderr, "expand_special(%d): %s:%d: '%s' -> '%s'\n",
896 pp_status.line_number,
902 push_buffer(ppp, NULL, NULL, 0);
907 static void expand_define(pp_entry_t *ppp)
909 assert(ppp->type == def_define);
912 fprintf(stderr, "expand_define(%d): %s:%d: '%s' -> '%s'\n",
915 pp_status.line_number,
918 if(ppp->subst.text && ppp->subst.text[0])
920 push_buffer(ppp, NULL, NULL, 0);
921 yy_scan_string(ppp->subst.text);
925 static int curdef_idx = 0;
926 static int curdef_alloc = 0;
927 static char *curdef_text = NULL;
929 static void add_text(const char *str, int len)
933 if(curdef_idx >= curdef_alloc || curdef_alloc - curdef_idx < len)
935 curdef_alloc += (len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1);
936 curdef_text = pp_xrealloc(curdef_text, curdef_alloc * sizeof(curdef_text[0]));
937 if(curdef_alloc > 65536)
938 ppy_warning("Reallocating macro-expansion buffer larger than 64kB");
940 memcpy(&curdef_text[curdef_idx], str, len);
944 static mtext_t *add_expand_text(mtext_t *mtp, macexpstackentry_t *mep, int *nnl)
958 fprintf(stderr, "add_expand_text: exp_text: '%s'\n", mtp->subst.text);
959 add_text(mtp->subst.text, strlen(mtp->subst.text));
964 fprintf(stderr, "add_expand_text: exp_stringize(%d): '%s'\n",
966 mep->args[mtp->subst.argidx]);
967 cptr = mep->args[mtp->subst.argidx];
971 if(*cptr == '"' || *cptr == '\\')
981 fprintf(stderr, "add_expand_text: exp_concat\n");
982 /* Remove trailing whitespace from current expansion text */
985 if(isspace(curdef_text[curdef_idx-1] & 0xff))
990 /* tag current position and recursively expand the next part */
992 mtp = add_expand_text(mtp->next, mep, nnl);
994 /* Now get rid of the leading space of the expansion */
995 cptr = &curdef_text[tag];
996 n = curdef_idx - tag;
999 if(isspace(*cptr & 0xff))
1007 if(cptr != &curdef_text[tag])
1009 memmove(&curdef_text[tag], cptr, n);
1010 curdef_idx -= (curdef_idx - tag) - n;
1015 if((mtp->next && mtp->next->type == exp_concat) || (mtp->prev && mtp->prev->type == exp_concat))
1016 exp = mep->args[mtp->subst.argidx];
1018 exp = mep->ppargs[mtp->subst.argidx];
1021 add_text(exp, strlen(exp));
1022 *nnl -= mep->nnls[mtp->subst.argidx];
1023 cptr = strchr(exp, '\n');
1027 cptr = strchr(cptr+1, '\n');
1029 mep->nnls[mtp->subst.argidx] = 0;
1032 fprintf(stderr, "add_expand_text: exp_subst(%d): '%s'\n", mtp->subst.argidx, exp);
1036 pp_internal_error(__FILE__, __LINE__, "Invalid expansion type (%d) in macro expansion\n", mtp->type);
1041 static void expand_macro(macexpstackentry_t *mep)
1047 pp_entry_t *ppp = mep->ppp;
1048 int nargs = mep->nargs;
1050 assert(ppp->type == def_macro);
1051 assert(ppp->expanding == 0);
1053 if((ppp->nargs >= 0 && nargs != ppp->nargs) || (ppp->nargs < 0 && nargs < -ppp->nargs))
1054 ppy_error("Too %s macro arguments (%d)", nargs < abs(ppp->nargs) ? "few" : "many", nargs);
1056 for(n = 0; n < nargs; n++)
1057 nnl += mep->nnls[n];
1060 fprintf(stderr, "expand_macro(%d): %s:%d: '%s'(%d,%d) -> ...\n",
1063 pp_status.line_number,
1070 for(mtp = ppp->subst.mtext; mtp; mtp = mtp->next)
1072 if(!(mtp = add_expand_text(mtp, mep, &nnl)))
1076 for(n = 0; n < nnl; n++)
1079 /* To make sure there is room and termination (see below) */
1082 /* Strip trailing whitespace from expansion */
1083 for(k = curdef_idx, cptr = &curdef_text[curdef_idx-1]; k > 0; k--, cptr--)
1085 if(!isspace(*cptr & 0xff))
1090 * We must add *one* whitespace to make sure that there
1091 * is a token-separation after the expansion.
1097 /* Strip leading whitespace from expansion */
1098 for(n = 0, cptr = curdef_text; n < k; n++, cptr++)
1100 if(!isspace(*cptr & 0xff))
1107 fprintf(stderr, "expand_text: '%s'\n", curdef_text + n);
1108 push_buffer(ppp, NULL, NULL, 0);
1109 /*yy_scan_bytes(curdef_text + n, k - n);*/
1110 yy_scan_string(curdef_text + n);
1115 *-------------------------------------------------------------------------
1116 * String collection routines
1117 *-------------------------------------------------------------------------
1119 static void new_string(void)
1123 ppy_warning("new_string: strbuf_idx != 0");
1126 str_startline = pp_status.line_number;
1129 static void add_string(const char *str, int len)
1133 if(strbuf_idx >= strbuf_alloc || strbuf_alloc - strbuf_idx < len)
1135 strbuf_alloc += (len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1);
1136 strbuffer = pp_xrealloc(strbuffer, strbuf_alloc * sizeof(strbuffer[0]));
1137 if(strbuf_alloc > 65536)
1138 ppy_warning("Reallocating string buffer larger than 64kB");
1140 memcpy(&strbuffer[strbuf_idx], str, len);
1144 static char *get_string(void)
1146 char *str = pp_xmalloc(strbuf_idx + 1);
1147 memcpy(str, strbuffer, strbuf_idx);
1148 str[strbuf_idx] = '\0';
1155 static void put_string(void)
1157 put_buffer(strbuffer, strbuf_idx);
1163 static int string_start(void)
1165 return str_startline;
1170 *-------------------------------------------------------------------------
1172 *-------------------------------------------------------------------------
1174 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop)
1177 printf("push_buffer(%d): %p %p %p %d\n", bufferstackidx, ppp, filename, incname, pop);
1178 if(bufferstackidx >= MAXBUFFERSTACK)
1179 pp_internal_error(__FILE__, __LINE__, "Buffer stack overflow");
1181 memset(&bufferstack[bufferstackidx], 0, sizeof(bufferstack[0]));
1182 bufferstack[bufferstackidx].bufferstate = YY_CURRENT_BUFFER;
1183 bufferstack[bufferstackidx].define = ppp;
1184 bufferstack[bufferstackidx].line_number = pp_status.line_number;
1185 bufferstack[bufferstackidx].char_number = pp_status.char_number;
1186 bufferstack[bufferstackidx].if_depth = pp_get_if_depth();
1187 bufferstack[bufferstackidx].should_pop = pop;
1188 bufferstack[bufferstackidx].filename = pp_status.input;
1189 bufferstack[bufferstackidx].ncontinuations = ncontinuations;
1190 bufferstack[bufferstackidx].incl = pp_incl_state;
1191 bufferstack[bufferstackidx].include_filename = incname;
1197 /* These will track the ppy_error to the correct file and line */
1198 pp_status.line_number = 1;
1199 pp_status.char_number = 1;
1200 pp_status.input = filename;
1204 pp_internal_error(__FILE__, __LINE__, "Pushing buffer without knowing where to go to");
1208 static bufferstackentry_t *pop_buffer(void)
1210 if(bufferstackidx < 0)
1211 pp_internal_error(__FILE__, __LINE__, "Bufferstack underflow?");
1213 if(bufferstackidx == 0)
1218 if(bufferstack[bufferstackidx].define)
1219 bufferstack[bufferstackidx].define->expanding = 0;
1222 pp_status.line_number = bufferstack[bufferstackidx].line_number;
1223 pp_status.char_number = bufferstack[bufferstackidx].char_number;
1224 pp_status.input = bufferstack[bufferstackidx].filename;
1225 ncontinuations = bufferstack[bufferstackidx].ncontinuations;
1226 if(!bufferstack[bufferstackidx].should_pop)
1229 fprintf(ppy_out, "# %d \"%s\" 2\n", pp_status.line_number, pp_status.input);
1231 /* We have EOF, check the include logic */
1232 if(pp_incl_state.state == 2 && !pp_incl_state.seen_junk && pp_incl_state.ppp)
1234 pp_entry_t *ppp = pplookup(pp_incl_state.ppp);
1237 includelogicentry_t *iep = pp_xmalloc(sizeof(includelogicentry_t));
1240 iep->filename = bufferstack[bufferstackidx].include_filename;
1242 iep->next = pp_includelogiclist;
1244 iep->next->prev = iep;
1245 pp_includelogiclist = iep;
1247 fprintf(stderr, "pop_buffer: %s:%d: includelogic added, include_ppp='%s', file='%s'\n", pp_status.input, pp_status.line_number, pp_incl_state.ppp, iep->filename);
1250 free(bufferstack[bufferstackidx].include_filename);
1252 free(pp_incl_state.ppp);
1253 pp_incl_state = bufferstack[bufferstackidx].incl;
1259 printf("pop_buffer(%d): %p %p (%d, %d, %d) %p %d\n",
1261 bufferstack[bufferstackidx].bufferstate,
1262 bufferstack[bufferstackidx].define,
1263 bufferstack[bufferstackidx].line_number,
1264 bufferstack[bufferstackidx].char_number,
1265 bufferstack[bufferstackidx].if_depth,
1266 bufferstack[bufferstackidx].filename,
1267 bufferstack[bufferstackidx].should_pop);
1269 ppy__switch_to_buffer(bufferstack[bufferstackidx].bufferstate);
1271 if(bufferstack[bufferstackidx].should_pop)
1273 if(yy_current_state() == pp_macexp)
1274 macro_add_expansion();
1276 pp_internal_error(__FILE__, __LINE__, "Pop buffer and state without macro expansion state");
1280 return &bufferstack[bufferstackidx];
1285 *-------------------------------------------------------------------------
1286 * Macro nestng support
1287 *-------------------------------------------------------------------------
1289 static void push_macro(pp_entry_t *ppp)
1291 if(macexpstackidx >= MAXMACEXPSTACK)
1292 ppy_error("Too many nested macros");
1294 macexpstack[macexpstackidx] = pp_xmalloc(sizeof(macexpstack[0][0]));
1295 memset( macexpstack[macexpstackidx], 0, sizeof(macexpstack[0][0]));
1296 macexpstack[macexpstackidx]->ppp = ppp;
1300 static macexpstackentry_t *top_macro(void)
1302 return macexpstackidx > 0 ? macexpstack[macexpstackidx-1] : NULL;
1305 static macexpstackentry_t *pop_macro(void)
1307 if(macexpstackidx <= 0)
1308 pp_internal_error(__FILE__, __LINE__, "Macro expansion stack underflow\n");
1309 return macexpstack[--macexpstackidx];
1312 static void free_macro(macexpstackentry_t *mep)
1316 for(i = 0; i < mep->nargs; i++)
1324 static void add_text_to_macro(const char *text, int len)
1326 macexpstackentry_t *mep = top_macro();
1328 assert(mep->ppp->expanding == 0);
1330 if(mep->curargalloc - mep->curargsize <= len+1) /* +1 for '\0' */
1332 mep->curargalloc += (ALLOCBLOCKSIZE > len+1) ? ALLOCBLOCKSIZE : len+1;
1333 mep->curarg = pp_xrealloc(mep->curarg, mep->curargalloc * sizeof(mep->curarg[0]));
1335 memcpy(mep->curarg + mep->curargsize, text, len);
1336 mep->curargsize += len;
1337 mep->curarg[mep->curargsize] = '\0';
1340 static void macro_add_arg(int last)
1344 macexpstackentry_t *mep = top_macro();
1346 assert(mep->ppp->expanding == 0);
1348 mep->args = pp_xrealloc(mep->args, (mep->nargs+1) * sizeof(mep->args[0]));
1349 mep->ppargs = pp_xrealloc(mep->ppargs, (mep->nargs+1) * sizeof(mep->ppargs[0]));
1350 mep->nnls = pp_xrealloc(mep->nnls, (mep->nargs+1) * sizeof(mep->nnls[0]));
1351 mep->args[mep->nargs] = pp_xstrdup(mep->curarg ? mep->curarg : "");
1352 cptr = mep->args[mep->nargs]-1;
1353 while((cptr = strchr(cptr+1, '\n')))
1357 mep->nnls[mep->nargs] = nnl;
1360 mep->curargalloc = mep->curargsize = 0;
1364 fprintf(stderr, "macro_add_arg: %s:%d: %d -> '%s'\n",
1366 pp_status.line_number,
1368 mep->args[mep->nargs-1]);
1370 /* Each macro argument must be expanded to cope with stingize */
1371 if(last || mep->args[mep->nargs-1][0])
1373 yy_push_state(pp_macexp);
1374 push_buffer(NULL, NULL, NULL, last ? 2 : 1);
1375 yy_scan_string(mep->args[mep->nargs-1]);
1376 /*mep->bufferstackidx = bufferstackidx; But not nested! */
1380 static void macro_add_expansion(void)
1382 macexpstackentry_t *mep = top_macro();
1384 assert(mep->ppp->expanding == 0);
1386 mep->ppargs[mep->nargs-1] = pp_xstrdup(mep->curarg ? mep->curarg : "");
1388 mep->curargalloc = mep->curargsize = 0;
1392 fprintf(stderr, "macro_add_expansion: %s:%d: %d -> '%s'\n",
1394 pp_status.line_number,
1396 mep->ppargs[mep->nargs-1]);
1401 *-------------------------------------------------------------------------
1403 *-------------------------------------------------------------------------
1405 static void put_buffer(const char *s, int len)
1408 add_text_to_macro(s, len);
1410 fwrite(s, 1, len, ppy_out);
1415 *-------------------------------------------------------------------------
1416 * Include management
1417 *-------------------------------------------------------------------------
1419 void pp_do_include(char *fname, int type)
1423 includelogicentry_t *iep;
1425 for(iep = pp_includelogiclist; iep; iep = iep->next)
1427 if(!strcmp(iep->filename, fname))
1430 * We are done. The file was included before.
1431 * If the define was deleted, then this entry would have
1441 ppy_error("Empty include filename");
1443 /* Undo the effect of the quotation */
1446 if((ppy_in = pp_open_include(fname+1, type ? pp_status.input : NULL, &newpath)) == NULL)
1447 ppy_error("Unable to open include file %s", fname+1);
1449 fname[n-1] = *fname; /* Redo the quotes */
1450 push_buffer(NULL, newpath, fname, 0);
1451 pp_incl_state.seen_junk = 0;
1452 pp_incl_state.state = 0;
1453 pp_incl_state.ppp = NULL;
1456 fprintf(stderr, "pp_do_include: %s:%d: include_state=%d, include_ppp='%s', include_ifdepth=%d\n",
1457 pp_status.input, pp_status.line_number, pp_incl_state.state, pp_incl_state.ppp, pp_incl_state.ifdepth);
1458 ppy__switch_to_buffer(ppy__create_buffer(ppy_in, YY_BUF_SIZE));
1460 fprintf(ppy_out, "# 1 \"%s\" 1%s\n", newpath, type ? "" : " 3");
1464 *-------------------------------------------------------------------------
1465 * Push/pop preprocessor ignore state when processing conditionals
1467 *-------------------------------------------------------------------------
1469 void pp_push_ignore_state(void)
1471 yy_push_state(pp_ignore);
1474 void pp_pop_ignore_state(void)