2 * Do C preprocessing, based on a token list gathered by
5 * This may not be the smartest preprocessor on the planet.
7 * Copyright (C) 2003 Linus Torvalds, all rights reserved.
25 static int true_nesting
= 0;
26 static int false_nesting
= 0;
27 static struct token
*unmatched_if
= NULL
;
28 static int elif_ignore
[MAXNEST
];
29 #define if_nesting (true_nesting + false_nesting)
32 * This is stupid - the tokenizer already guarantees unique
33 * identifiers, so we should just compare identifier pointers
35 static int match_string_ident(struct ident
*ident
, const char *str
)
37 return !str
[ident
->len
] && !memcmp(str
, ident
->name
, ident
->len
);
40 static struct token
*alloc_token(struct token
*dup
)
42 struct token
*token
= __alloc_token(0);
44 token
->stream
= dup
->stream
;
45 token
->line
= dup
->line
;
46 token
->pos
= dup
->pos
;
47 token
->whitespace
= 1;
51 static const char *show_token_sequence(struct token
*token
);
53 /* Head is one-before-list, and last is one-past-list */
54 static struct token
*for_each_ident(struct token
*head
, struct token
*(*action
)(struct token
*head
, struct token
*))
57 struct token
*next
= head
->next
;
59 /* Did we hit the end of the current expansion? */
63 if (next
->type
== TOKEN_IDENT
)
64 next
= action(head
, next
);
71 static struct token
*is_defined(struct token
*head
, struct token
*token
, struct token
*next
)
73 char *string
[] = { "0", "1" };
74 char *defined
= string
[lookup_symbol(token
->ident
, NS_PREPROCESSOR
) != NULL
];
75 struct token
*newtoken
= alloc_token(token
);
77 newtoken
->type
= TOKEN_INTEGER
;
78 newtoken
->integer
= defined
;
79 newtoken
->next
= next
;
80 head
->next
= newtoken
;
85 struct token
*defined_one_symbol(struct token
*head
, struct token
*next
)
87 if (match_string_ident(next
->ident
, "defined")) {
88 struct token
*token
= next
->next
;
89 struct token
*past
= token
->next
;
91 if (match_op(token
, '(')) {
94 if (!match_op(past
, ')'))
98 if (token
->type
== TOKEN_IDENT
)
99 return is_defined(head
, token
, past
);
104 static struct token
*expand_defined(struct token
*head
)
106 return for_each_ident(head
, defined_one_symbol
);
109 /* Expand symbol 'sym' between 'head->next' and 'head->next->next' */
110 static struct token
*expand(struct token
*, struct symbol
*);
112 struct token
*expand_one_symbol(struct token
*head
, struct token
*token
)
114 struct symbol
*sym
= lookup_symbol(token
->ident
, NS_PREPROCESSOR
);
115 if (sym
&& !sym
->busy
) {
116 if (sym
->arglist
&& !match_op(token
->next
, '('))
118 return expand(head
, sym
);
123 static struct token
*expand_list(struct token
*head
)
125 return for_each_ident(head
, expand_one_symbol
);
128 static struct token
*find_argument_end(struct token
*start
)
132 while (!eof_token(start
)) {
133 struct token
*next
= start
->next
;
134 if (match_op(next
, '('))
136 else if (match_op(next
, ')')) {
138 start
->next
= &eof_token_entry
;
141 } else if (!nesting
&& match_op(next
, ','))
142 next
->special
= SPECIAL_ARG_SEPARATOR
;
148 static struct token
*dup_token(struct token
*token
, struct token
*pos
, int newline
)
150 struct token
*alloc
= alloc_token(pos
);
151 alloc
->type
= token
->type
;
152 alloc
->line
= pos
->line
;
153 alloc
->newline
= newline
;
154 alloc
->integer
= token
->integer
;
158 static void insert(struct token
*token
, struct token
*prev
)
160 token
->next
= &eof_token_entry
;
164 static struct token
* replace(struct token
*token
, struct token
*prev
, struct token
*list
)
166 int newline
= token
->newline
;
168 prev
->next
= &eof_token_entry
;
169 while (!eof_token(list
) && !match_op(list
, SPECIAL_ARG_SEPARATOR
)) {
170 struct token
*newtok
= dup_token(list
, token
, newline
);
172 insert(newtok
, prev
);
179 static struct token
*get_argument(int nr
, struct token
*args
)
183 while (!eof_token(args
)) {
184 if (match_op(args
, SPECIAL_ARG_SEPARATOR
))
193 static struct token
*stringify(struct token
*token
, struct token
*arg
)
195 const char *s
= show_token_sequence(arg
);
196 int size
= strlen(s
)+1;
197 struct token
*newtoken
= alloc_token(token
);
198 struct string
*string
= __alloc_string(size
);
200 newtoken
->newline
= token
->newline
;
201 memcpy(string
->data
, s
, size
);
202 string
->length
= size
;
203 newtoken
->type
= TOKEN_STRING
;
204 newtoken
->string
= string
;
205 newtoken
->next
= &eof_token_entry
;
209 static int arg_number(struct token
*arglist
, struct ident
*ident
)
213 while (!eof_token(arglist
)) {
214 if (arglist
->ident
== ident
)
217 arglist
= arglist
->next
;
222 static struct token empty_arg_token
= { .type
= TOKEN_EOF
};
224 static struct token
*expand_one_arg(struct token
*head
, struct token
*token
,
225 struct token
*arglist
, struct token
*arguments
)
227 int nr
= arg_number(arglist
, token
->ident
);
228 struct token
*orig_head
= head
;
231 struct token
*arg
= get_argument(nr
, arguments
);
232 struct token
*last
= token
->next
;
233 token
->next
= &eof_token_entry
;
236 * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
237 * then the 'x' goes away too.
239 if (match_op(head
, SPECIAL_HASHHASH
) && eof_token(arg
)) {
240 arg
= &empty_arg_token
;
241 empty_arg_token
.next
= &eof_token_entry
;
244 head
= replace(token
, head
, arg
);
245 if (!match_op(orig_head
, SPECIAL_HASHHASH
) && !match_op(last
, SPECIAL_HASHHASH
) && !match_op(orig_head
, '#'))
246 head
= expand_list(orig_head
);
253 static void expand_arguments(struct token
*token
, struct token
*head
,
254 struct token
*arguments
, struct token
*arglist
)
257 struct token
*next
= head
->next
;
259 /* Did we hit the end of the current expansion? */
263 if (match_op(next
, '#')) {
264 struct token
*nextnext
= next
->next
;
265 int nr
= arg_number(arglist
, nextnext
->ident
);
266 if (nextnext
!= head
&& nr
>= 0 && nextnext
->type
== TOKEN_IDENT
) {
267 struct token
* last
= nextnext
->next
;
268 struct token
*newtoken
= stringify(nextnext
, get_argument(nr
, arguments
));
269 replace(nextnext
, head
, newtoken
);
272 warn(next
, "'#' operation is not followed by argument name");
275 if (next
->type
== TOKEN_IDENT
)
276 next
= expand_one_arg(head
, next
, arglist
, arguments
);
283 * Possibly valid combinations:
284 * - anything + 'empty_arg_token' is empty.
285 * - ident + ident - combine (==ident)
286 * - ident + number - combine (==ident)
287 * - number + number - combine (==number)
288 * - number + ident - combine (==number)
289 * - string + string - leave as is, C will combine them anyway
290 * others cause an error and leave the two tokens as separate tokens.
292 static struct token
*hashhash(struct token
*head
, struct token
*first
, struct token
*second
)
294 static char buffer
[512], *p
;
295 struct token
*newtoken
;
296 static const char *src
;
299 first
->next
= second
;
302 * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
303 * then the 'x' goes away too.
305 * See expand_one_arg.
307 if (second
->type
== TOKEN_EOF
) {
308 head
->next
= second
->next
;
313 switch (first
->type
) {
315 len
= strlen(first
->integer
);
316 src
= first
->integer
;
319 len
= first
->ident
->len
;
320 src
= first
->ident
->name
;
328 switch (second
->type
) {
330 len
= strlen(second
->integer
);
331 src
= second
->integer
;
334 len
= second
->ident
->len
;
335 src
= second
->ident
->name
;
344 newtoken
= alloc_token(first
);
345 head
->next
= newtoken
;
346 newtoken
->type
= first
->type
;
347 switch (newtoken
->type
) {
349 newtoken
->ident
= built_in_ident(buffer
);
352 newtoken
->integer
= __alloc_bytes(p
- buffer
);
353 memcpy(newtoken
->integer
, buffer
, p
- buffer
);
359 static void retokenize(struct token
*head
)
361 struct token
* next
= head
->next
;
362 struct token
* nextnext
= next
->next
;
363 struct token
* nextnextnext
= nextnext
->next
;
365 if (eof_token(next
) || eof_token(nextnext
))
369 if (eof_token(nextnextnext
))
372 if (match_op(nextnext
, SPECIAL_HASHHASH
)) {
373 struct token
*newtoken
= hashhash(head
, next
, nextnextnext
);
376 nextnext
= nextnextnext
->next
;
377 nextnextnext
= nextnext
->next
;
379 newtoken
->next
= nextnext
;
380 if (!eof_token(nextnext
))
387 nextnext
= nextnext
->next
;
388 nextnextnext
= nextnextnext
->next
;
392 static struct token
*expand(struct token
*head
, struct symbol
*sym
)
394 struct token
*arguments
, *token
, *last
;
402 arguments
= last
->next
;
403 last
= find_argument_end(last
);
405 token
->next
= &eof_token_entry
;
407 /* Replace the token with the token expansion */
408 replace(token
, head
, sym
->expansion
);
410 /* Then, replace all the arguments with their expansions */
412 expand_arguments(token
, head
, arguments
, sym
->arglist
);
414 /* Re-tokenize the sequence if any ## token exists.. */
417 /* Finally, expand the expansion itself .. */
418 head
= expand_list(head
);
420 /* Put the rest of the stuff in place again */
426 static const char *token_name_sequence(struct token
*token
, int endop
, struct token
*start
)
429 static char buffer
[256];
433 while (!eof_token(token
) && !match_op(token
, endop
)) {
435 const char *val
= token
->string
->data
;
436 if (token
->type
!= TOKEN_STRING
)
437 val
= show_token(token
);
439 memcpy(ptr
, val
, len
);
444 if (endop
&& !match_op(token
, endop
))
445 warn(start
, "expected '>' at end of filename");
449 static void do_include(struct token
*head
, struct token
*token
, const char *filename
)
451 int endlen
= strlen(filename
) + 1;
452 char **pptr
= includepath
, *path
;
454 while ((path
= *pptr
++) != NULL
) {
455 int fd
, len
= strlen(path
);
456 static char fullname
[PATH_MAX
];
458 memcpy(fullname
, path
, len
);
459 memcpy(fullname
+len
, filename
, endlen
);
460 fd
= open(fullname
, O_RDONLY
);
462 char * streamname
= __alloc_bytes(len
+ endlen
);
463 memcpy(streamname
, fullname
, len
+ endlen
);
464 head
->next
= tokenize(streamname
, fd
, head
->next
);
469 warn(token
, "unable to open '%s'", filename
);
472 static int handle_include(struct stream
*stream
, struct token
*head
, struct token
*token
)
474 const char *filename
;
478 if (stream
->constant
== -1)
479 stream
->constant
= 0;
484 if (!match_op(next
, '<')) {
490 filename
= token_name_sequence(token
, expect
, token
);
491 do_include(head
, token
, filename
);
495 static int token_list_different(struct token
*list1
, struct token
*list2
)
500 if (!list1
|| !list2
)
502 if (list1
->type
!= list2
->type
)
510 static int handle_define(struct stream
*stream
, struct token
*head
, struct token
*token
)
512 struct token
*arglist
, *expansion
;
513 struct token
*left
= token
->next
;
517 if (left
->type
!= TOKEN_IDENT
) {
518 warn(head
, "expected identifier to 'define'");
526 expansion
= left
->next
;
527 if (!expansion
->whitespace
&& match_op(expansion
, '(')) {
529 while (!eof_token(expansion
)) {
530 struct token
*next
= expansion
->next
;
531 if (match_op(next
, ')')) {
532 // Terminate the arglist
533 expansion
->next
= &eof_token_entry
;
534 expansion
= next
->next
;
537 if (match_op(next
, ','))
538 expansion
->next
= next
->next
;
541 arglist
= arglist
->next
;
544 sym
= lookup_symbol(name
, NS_PREPROCESSOR
);
546 if (token_list_different(sym
->expansion
, expansion
) ||
547 token_list_different(sym
->arglist
, arglist
)) {
548 warn(left
, "preprocessor token redefined");
549 warn(sym
->token
, "this was the original definition");
553 sym
= alloc_symbol(left
, SYM_NONE
);
554 bind_symbol(sym
, name
, NS_PREPROCESSOR
);
556 sym
->expansion
= expansion
;
557 sym
->arglist
= arglist
;
561 static int handle_undef(struct stream
*stream
, struct token
*head
, struct token
*token
)
563 struct token
*left
= token
->next
;
566 if (left
->type
!= TOKEN_IDENT
) {
567 warn(head
, "expected identifier to 'undef'");
572 sym
= &left
->ident
->symbols
;
574 struct symbol
*t
= *sym
;
575 if (t
->namespace == NS_PREPROCESSOR
) {
584 static int preprocessor_if(struct token
*token
, int true)
587 unmatched_if
= token
;
588 elif_ignore
[if_nesting
] = false_nesting
|| true;
589 if (false_nesting
|| !true) {
597 static int token_defined(struct token
*token
)
599 if (token
->type
== TOKEN_IDENT
)
600 return lookup_symbol(token
->ident
, NS_PREPROCESSOR
) != NULL
;
602 warn(token
, "expected identifier for #if[n]def");
606 static int handle_ifdef(struct stream
*stream
, struct token
*head
, struct token
*token
)
608 return preprocessor_if(token
, token_defined(token
->next
));
611 static int handle_ifndef(struct stream
*stream
, struct token
*head
, struct token
*token
)
613 struct token
*next
= token
->next
;
614 if (stream
->constant
== -1) {
616 if (next
->type
== TOKEN_IDENT
) {
617 if (!stream
->protect
|| stream
->protect
== next
->ident
) {
619 stream
->protect
= next
->ident
;
620 stream
->nesting
= if_nesting
+1;
623 stream
->constant
= newconstant
;
625 return preprocessor_if(token
, !token_defined(next
));
628 static unsigned long long get_int_value(const char *str
)
630 unsigned long long value
= 0;
631 unsigned int base
= 10, digit
;
635 base
= 18; // the -= 2 for the octal case will
636 str
++; // skip the 'x'
639 str
++; // skip the 'o' or 'x/X'
640 base
-= 2; // the fall-through will make this 8
642 while ((digit
= hexval(*str
)) < base
) {
643 value
= value
* base
+ digit
;
649 static long long primary_value(struct token
*token
)
651 switch (token
->type
) {
653 return get_int_value(token
->integer
);
655 error(token
, "bad constant expression");
659 long long get_expression_value(struct expression
*expr
)
661 long long left
, middle
, right
;
663 switch (expr
->type
) {
665 return primary_value(expr
->token
);
667 struct symbol
*sym
= expr
->symbol
;
668 if (!sym
|| !sym
->ctype
.base_type
|| sym
->ctype
.base_type
->type
!= SYM_ENUM
) {
669 warn(expr
->token
, "undefined identifier in constant expression");
675 #define OP(x,y) case x: return left y right;
677 left
= get_expression_value(expr
->left
);
678 if (!left
&& expr
->op
== SPECIAL_LOGICAL_AND
)
680 if (left
&& expr
->op
== SPECIAL_LOGICAL_OR
)
682 right
= get_expression_value(expr
->right
);
684 OP('+',+); OP('-',-); OP('*',*); OP('/',/);
685 OP('%',%); OP('<',<); OP('>',>);
686 OP('&',&);OP('|',|);OP('^',^);
687 OP(SPECIAL_EQUAL
,==); OP(SPECIAL_NOTEQUAL
,!=);
688 OP(SPECIAL_LTE
,<=); OP(SPECIAL_LEFTSHIFT
,<<);
689 OP(SPECIAL_RIGHTSHIFT
,>>); OP(SPECIAL_GTE
,>=);
690 OP(SPECIAL_LOGICAL_AND
,&&);OP(SPECIAL_LOGICAL_OR
,||);
695 #define OP(x,y) case x: return y left;
697 left
= get_expression_value(expr
->unop
);
699 OP('+', +); OP('-', -); OP('!', !); OP('~', ~); OP('(', );
703 case EXPR_CONDITIONAL
:
704 left
= get_expression_value(expr
->conditional
);
705 if (!expr
->cond_true
)
708 middle
= get_expression_value(expr
->cond_true
);
709 right
= get_expression_value(expr
->cond_false
);
710 return left
? middle
: right
;
712 error(expr
->token
, "bad constant expression");
716 extern struct token
*conditional_expression(struct token
*token
, struct expression
**tree
);
718 static int expression_value(struct token
*head
)
720 struct expression
*expr
;
724 expand_defined(head
);
726 token
= conditional_expression(head
->next
, &expr
);
727 if (!eof_token(token
))
728 warn(token
, "garbage at end: %s", show_token_sequence(token
));
729 value
= get_expression_value(expr
);
733 static int handle_if(struct stream
*stream
, struct token
*head
, struct token
*token
)
737 value
= expression_value(token
);
738 return preprocessor_if(token
, value
);
741 static int handle_elif(struct stream
* stream
, struct token
*head
, struct token
*token
)
743 if (stream
->nesting
== if_nesting
)
744 stream
->constant
= 0;
746 /* If this whole if-thing is if'ed out, an elif cannot help */
747 if (elif_ignore
[if_nesting
-1])
749 if (expression_value(token
)) {
752 elif_ignore
[if_nesting
-1] = 1;
761 warn(token
, "unmatched '#elif'");
765 static int handle_else(struct stream
*stream
, struct token
*head
, struct token
*token
)
767 if (stream
->nesting
== if_nesting
)
768 stream
->constant
= 0;
770 /* If this whole if-thing is if'ed out, an else cannot help */
771 if (elif_ignore
[if_nesting
-1])
775 elif_ignore
[if_nesting
-1] = 1;
783 warn(token
, "unmatched #else");
787 static int handle_endif(struct stream
*stream
, struct token
*head
, struct token
*token
)
789 if (stream
->constant
== -2 && stream
->nesting
== if_nesting
)
790 stream
->constant
= -1;
800 warn(token
, "unmatched #endif");
804 static const char *show_token_sequence(struct token
*token
)
806 static char buffer
[256];
812 while (!eof_token(token
)) {
813 const char *val
= show_token(token
);
814 int len
= strlen(val
);
817 memcpy(ptr
, val
, len
);
820 whitespace
= token
->whitespace
;
827 static int handle_warning(struct stream
*stream
, struct token
*head
, struct token
*token
)
831 warn(token
, "%s", show_token_sequence(token
->next
));
835 static int handle_error(struct stream
*stream
, struct token
*head
, struct token
*token
)
839 error(token
, "%s", show_token_sequence(token
->next
));
843 static int handle_preprocessor_command(struct stream
*stream
, struct token
*head
, struct ident
*ident
, struct token
*token
)
848 int (*handler
)(struct stream
*, struct token
*, struct token
*);
850 { "define", handle_define
},
851 { "undef", handle_undef
},
852 { "ifdef", handle_ifdef
},
853 { "ifndef", handle_ifndef
},
854 { "else", handle_else
},
855 { "endif", handle_endif
},
857 { "elif", handle_elif
},
858 { "warning", handle_warning
},
859 { "error", handle_error
},
860 { "include", handle_include
},
863 for (i
= 0; i
< (sizeof (handlers
) / sizeof (handlers
[0])); i
++) {
864 if (match_string_ident(ident
, handlers
[i
].name
))
865 return handlers
[i
].handler(stream
, head
, token
);
870 static void handle_preprocessor_line(struct stream
*stream
, struct token
* head
, struct token
*token
)
875 if (token
->type
== TOKEN_IDENT
)
876 if (handle_preprocessor_command(stream
, head
, token
->ident
, token
))
878 warn(token
, "unrecognized preprocessor line '%s'", show_token_sequence(token
));
881 static void preprocessor_line(struct stream
*stream
, struct token
* head
)
883 struct token
*start
= head
->next
, *next
;
884 struct token
**tp
= &start
->next
;
893 *tp
= &eof_token_entry
;
894 handle_preprocessor_line(stream
, head
, start
->next
);
897 static void do_preprocess(struct token
*head
)
900 struct token
*next
= head
->next
;
901 struct stream
*stream
= input_streams
+ next
->stream
;
903 if (next
->newline
&& match_op(next
, '#')) {
904 preprocessor_line(stream
, head
);
909 head
->next
= next
->next
;
913 switch (next
->type
) {
914 case TOKEN_STREAMEND
:
915 if (stream
->constant
== -1 && stream
->protect
) {
916 stream
->constant
= 1;
919 case TOKEN_STREAMBEGIN
:
920 head
->next
= next
->next
;
924 next
= expand_one_symbol(head
, next
);
928 * Any token expansion (even if it ended up being an
929 * empty expansion) in this stream implies it can't
932 stream
->constant
= 0;
936 } while (!eof_token(head
));
939 struct token
* preprocess(struct token
*token
)
941 struct token header
= { 0, };
944 do_preprocess(&header
);
946 warn(unmatched_if
, "unmatched preprocessor conditional");
948 // Drop all expressions from pre-processing, they're not used any more.
949 clear_expression_alloc();