2 * Do C preprocessing, based on a token list gathered by
5 * This may not be the smartest preprocessor on the planet.
7 * Copyright (C) 2003 Transmeta Corp.
10 * Licensed under the Open Software License version 1.1
22 #include "pre-process.h"
27 #include "expression.h"
30 int preprocessing
= 0;
32 #define MAX_NEST (256)
33 static int true_nesting
= 0;
34 static int false_nesting
= 0;
35 static struct token
*unmatched_if
= NULL
;
36 static char elif_ignore
[MAX_NEST
];
37 #define if_nesting (true_nesting + false_nesting)
39 #define INCLUDEPATHS 32
40 const char *includepath
[INCLUDEPATHS
+1] = {
44 const char *sys_includepath
[] = {
50 const char *gcc_includepath
[] = {
57 * This is stupid - the tokenizer already guarantees unique
58 * identifiers, so we should just compare identifier pointers
60 int match_string_ident(struct ident
*ident
, const char *str
)
62 return !str
[ident
->len
] && !memcmp(str
, ident
->name
, ident
->len
);
65 static struct token
*alloc_token(struct position
*pos
)
67 struct token
*token
= __alloc_token(0);
69 token
->pos
.stream
= pos
->stream
;
70 token
->pos
.line
= pos
->line
;
71 token
->pos
.pos
= pos
->pos
;
72 token
->pos
.whitespace
= 1;
76 static const char *show_token_sequence(struct token
*token
);
78 /* Head is one-before-list, and last is one-past-list */
79 static struct token
*for_each_ident(struct token
*parent
, struct token
*head
, struct token
*(*action
)(struct token
*parent
, struct token
*head
, struct token
*))
82 struct token
*next
= head
->next
;
84 /* Did we hit the end of the current expansion? */
88 if (token_type(next
) == TOKEN_IDENT
)
89 next
= action(parent
, head
, next
);
96 static struct token
*is_defined(struct token
*head
, struct token
*token
, struct token
*next
)
98 char *string
[] = { "0", "1" };
99 char *defined
= string
[lookup_symbol(token
->ident
, NS_PREPROCESSOR
) != NULL
];
100 struct token
*newtoken
= alloc_token(&token
->pos
);
102 token_type(newtoken
) = TOKEN_INTEGER
;
103 newtoken
->integer
= defined
;
104 newtoken
->next
= next
;
105 head
->next
= newtoken
;
110 struct token
*defined_one_symbol(struct token
*head
, struct token
*next
)
112 struct token
*token
= next
->next
;
113 struct token
*past
= token
->next
;
115 if (match_op(token
, '(')) {
118 if (!match_op(past
, ')'))
122 if (token_type(token
) == TOKEN_IDENT
)
123 return is_defined(head
, token
, past
);
127 struct token variable_argument
= { .next
= &eof_token_entry
};
129 /* Expand symbol 'sym' between 'head->next' and 'head->next->next' */
130 static struct token
*expand(struct token
*, struct token
*, struct symbol
*);
132 static void replace_with_string(struct token
*token
, const char *str
)
134 int size
= strlen(str
) + 1;
135 struct string
*s
= __alloc_string(size
);
138 memcpy(s
->data
, str
, size
);
139 token_type(token
) = TOKEN_STRING
;
143 static void replace_with_integer(struct token
*token
, unsigned int val
)
145 char *buf
= __alloc_bytes(10);
146 sprintf(buf
, "%d", val
);
147 token_type(token
) = TOKEN_INTEGER
;
148 token
->integer
= buf
;
151 struct token
*expand_one_symbol(struct token
*parent
, struct token
*head
, struct token
*token
)
156 /* Avoid recursive expansion */
158 while ((x
= x
->parent
) != NULL
) {
159 if (parent
&& x
->ident
== parent
->ident
)
161 if (x
->ident
== token
->ident
)
165 sym
= lookup_symbol(token
->ident
, NS_PREPROCESSOR
);
167 if (sym
->arglist
&& !match_op(token
->next
, '('))
169 return expand(token
, head
, sym
);
171 if (token
->ident
== &__LINE___ident
) {
172 replace_with_integer(token
, token
->pos
.line
);
173 } else if (token
->ident
== &__FILE___ident
) {
174 replace_with_string(token
, (input_streams
+ token
->pos
.stream
)->name
);
175 } else if (token
->ident
== &defined_ident
) {
176 return defined_one_symbol(head
, token
);
181 static struct token
*expand_list(struct token
*parent
, struct token
*head
)
183 return for_each_ident(parent
, head
, expand_one_symbol
);
186 static struct token
*find_argument_end(struct token
*start
, struct token
*arglist
)
190 while (!eof_token(start
)) {
191 struct token
*next
= start
->next
;
192 if (match_op(next
, '('))
194 else if (match_op(next
, ')')) {
196 start
->next
= &eof_token_entry
;
199 } else if (!nesting
&& match_op(next
, ',') && arglist
->next
!= &variable_argument
200 && !match_op(arglist
, SPECIAL_ELLIPSIS
)) {
201 next
->special
= SPECIAL_ARG_SEPARATOR
;
202 arglist
= arglist
->next
;
209 static struct token
*dup_token(struct token
*token
, struct position
*streampos
, struct position
*pos
)
211 struct token
*alloc
= alloc_token(streampos
);
212 token_type(alloc
) = token_type(token
);
213 alloc
->pos
.newline
= pos
->newline
;
214 alloc
->pos
.whitespace
= pos
->whitespace
;
215 alloc
->integer
= token
->integer
;
219 static void insert(struct token
*token
, struct token
*prev
)
221 token
->next
= prev
->next
;
225 static struct token
* replace(struct token
*parent
, struct token
*token
, struct token
*prev
, struct token
*list
)
227 struct position
*pos
= &token
->pos
;
229 prev
->next
= token
->next
;
230 while (!eof_token(list
) && !match_op(list
, SPECIAL_ARG_SEPARATOR
)) {
231 struct token
*newtok
= dup_token(list
, &token
->pos
, pos
);
232 newtok
->parent
= parent
;
233 insert(newtok
, prev
);
241 static struct token
*get_argument(int nr
, struct token
*args
)
245 while (!eof_token(args
)) {
246 if (match_op(args
, SPECIAL_ARG_SEPARATOR
))
255 static struct token
*stringify(struct token
*token
, struct token
*arg
)
257 const char *s
= show_token_sequence(arg
);
258 int size
= strlen(s
)+1;
259 struct token
*newtoken
= alloc_token(&token
->pos
);
260 struct string
*string
= __alloc_string(size
);
262 newtoken
->pos
.newline
= token
->pos
.newline
;
263 memcpy(string
->data
, s
, size
);
264 string
->length
= size
;
265 token_type(newtoken
) = TOKEN_STRING
;
266 newtoken
->string
= string
;
267 newtoken
->next
= &eof_token_entry
;
271 static int arg_number(struct token
*arglist
, struct ident
*ident
)
275 while (!eof_token(arglist
)) {
276 if (match_op(arglist
, SPECIAL_ELLIPSIS
) && ident
== &__VA_ARGS___ident
)
278 if (arglist
->ident
== ident
)
281 arglist
= arglist
->next
;
286 static struct token empty_arg_token
= { .pos
= { .type
= TOKEN_EOF
} };
288 static struct token
*expand_one_arg(struct token
*parent
, struct token
*head
, struct token
*token
,
289 struct token
*arglist
, struct token
*arguments
)
291 int nr
= arg_number(arglist
, token
->ident
);
292 struct token
*orig_head
= head
;
295 struct token
*arg
= get_argument(nr
, arguments
);
296 struct token
*last
= token
->next
;
297 token
->next
= &eof_token_entry
;
300 * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
301 * then the 'x' goes away too.
303 if (match_op(head
, SPECIAL_HASHHASH
) && eof_token(arg
)) {
304 arg
= &empty_arg_token
;
305 empty_arg_token
.next
= &eof_token_entry
;
308 head
= replace(NULL
, token
, head
, arg
);
309 if (!match_op(orig_head
, SPECIAL_HASHHASH
) && !match_op(last
, SPECIAL_HASHHASH
) && !match_op(orig_head
, '#'))
310 head
= expand_list(parent
, orig_head
);
317 static void expand_arguments(struct token
*parent
,
318 struct token
*token
, struct token
*head
,
319 struct token
*arguments
, struct token
*arglist
)
322 struct token
*next
= head
->next
;
324 /* Did we hit the end of the current expansion? */
328 if (match_op(next
, '#')) {
329 struct token
*nextnext
= next
->next
;
330 int nr
= arg_number(arglist
, nextnext
->ident
);
331 if (nextnext
!= head
&& nr
>= 0 && token_type(nextnext
) == TOKEN_IDENT
) {
332 struct token
*newtoken
= stringify(nextnext
, get_argument(nr
, arguments
));
333 replace(NULL
, nextnext
, head
, newtoken
);
336 warn(next
->pos
, "'#' operation is not followed by argument name");
339 if (token_type(next
) == TOKEN_IDENT
)
340 next
= expand_one_arg(parent
, head
, next
, arglist
, arguments
);
347 * Possibly valid combinations:
348 * - anything + 'empty_arg_token' is empty.
349 * - ident + ident - combine (==ident)
350 * - ident + number - combine (==ident)
351 * - number + number - combine (==number)
352 * - number + ident - combine (==number)
353 * - string + string - leave as is, C will combine them anyway
354 * others cause an error and leave the two tokens as separate tokens.
356 static struct token
*hashhash(struct token
*head
, struct token
*first
, struct token
*second
)
358 static char buffer
[512], *p
;
359 struct token
*newtoken
;
360 static const char *src
;
363 first
->next
= second
;
366 * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
367 * then the 'x' goes away too.
369 * See expand_one_arg.
371 if (token_type(second
) == TOKEN_EOF
) {
372 head
->next
= second
->next
;
377 switch (token_type(first
)) {
379 len
= strlen(first
->integer
);
380 src
= first
->integer
;
383 len
= first
->ident
->len
;
384 src
= first
->ident
->name
;
392 switch (token_type(second
)) {
394 len
= strlen(second
->integer
);
395 src
= second
->integer
;
398 len
= second
->ident
->len
;
399 src
= second
->ident
->name
;
408 newtoken
= alloc_token(&first
->pos
);
409 head
->next
= newtoken
;
410 token_type(newtoken
) = token_type(first
);
411 switch (token_type(newtoken
)) {
413 newtoken
->ident
= built_in_ident(buffer
);
416 newtoken
->integer
= __alloc_bytes(p
- buffer
);
417 memcpy(newtoken
->integer
, buffer
, p
- buffer
);
423 static void retokenize(struct token
*head
)
425 struct token
* next
= head
->next
;
426 struct token
* nextnext
= next
->next
;
427 struct token
* nextnextnext
= nextnext
->next
;
429 if (eof_token(next
) || eof_token(nextnext
))
433 if (eof_token(nextnextnext
))
436 if (match_op(nextnext
, SPECIAL_HASHHASH
)) {
437 struct token
*newtoken
= hashhash(head
, next
, nextnextnext
);
440 nextnext
= nextnextnext
->next
;
441 nextnextnext
= nextnext
->next
;
443 newtoken
->next
= nextnext
;
444 if (!eof_token(nextnext
))
451 nextnext
= nextnext
->next
;
452 nextnextnext
= nextnextnext
->next
;
456 static struct token
*expand(struct token
*parent
, struct token
*head
, struct symbol
*sym
)
458 struct token
*arguments
, *token
, *last
;
465 arguments
= last
->next
;
466 last
= find_argument_end(last
, sym
->arglist
);
468 token
->next
= &eof_token_entry
;
470 /* Replace the token with the token expansion */
471 replace(parent
, token
, head
, sym
->expansion
);
473 /* Then, replace all the arguments with their expansions */
475 expand_arguments(parent
, token
, head
, arguments
, sym
->arglist
);
477 /* Re-tokenize the sequence if any ## token exists.. */
481 while (!eof_token(token
->next
))
487 static const char *token_name_sequence(struct token
*token
, int endop
, struct token
*start
)
490 static char buffer
[256];
494 while (!eof_token(token
) && !match_op(token
, endop
)) {
496 const char *val
= token
->string
->data
;
497 if (token_type(token
) != TOKEN_STRING
)
498 val
= show_token(token
);
500 memcpy(ptr
, val
, len
);
505 if (endop
&& !match_op(token
, endop
))
506 warn(start
->pos
, "expected '>' at end of filename");
510 static int try_include(const char *path
, int plen
, const char *filename
, int flen
, struct token
*head
)
513 static char fullname
[PATH_MAX
];
515 memcpy(fullname
, path
, plen
);
516 if (plen
&& path
[plen
-1] != '/') {
517 fullname
[plen
] = '/';
520 memcpy(fullname
+plen
, filename
, flen
);
521 fd
= open(fullname
, O_RDONLY
);
523 char * streamname
= __alloc_bytes(plen
+ flen
);
524 memcpy(streamname
, fullname
, plen
+ flen
);
525 head
->next
= tokenize(streamname
, fd
, head
->next
);
532 static int do_include_path(const char **pptr
, struct token
*head
, struct token
*token
, const char *filename
, int flen
)
536 while ((path
= *pptr
++) != NULL
) {
537 if (!try_include(path
, strlen(path
), filename
, flen
, head
))
545 static void do_include(int local
, struct stream
*stream
, struct token
*head
, struct token
*token
, const char *filename
)
547 int flen
= strlen(filename
) + 1;
549 /* Same directory as current stream? */
556 slash
= strrchr(path
, '/');
557 plen
= slash
? slash
- path
: 0;
559 if (try_include(path
, plen
, filename
, flen
, head
))
563 /* Check the standard include paths.. */
564 if (do_include_path(includepath
, head
, token
, filename
, flen
))
566 if (do_include_path(sys_includepath
, head
, token
, filename
, flen
))
568 if (do_include_path(gcc_includepath
, head
, token
, filename
, flen
))
571 error(token
->pos
, "unable to open '%s'", filename
);
574 static int handle_include(struct stream
*stream
, struct token
*head
, struct token
*token
)
576 const char *filename
;
580 if (stream
->constant
== -1)
581 stream
->constant
= 0;
586 if (!match_op(next
, '<')) {
587 expand_list(NULL
, token
);
592 filename
= token_name_sequence(token
, expect
, token
);
593 do_include(!expect
, stream
, head
, token
, filename
);
597 static int token_list_different(struct token
*list1
, struct token
*list2
)
602 if (!list1
|| !list2
)
604 if (token_type(list1
) != token_type(list2
))
612 static int handle_define(struct stream
*stream
, struct token
*head
, struct token
*token
)
614 struct token
*arglist
, *expansion
;
615 struct token
*left
= token
->next
;
619 if (token_type(left
) != TOKEN_IDENT
) {
620 warn(head
->pos
, "expected identifier to 'define'");
628 expansion
= left
->next
;
629 if (!expansion
->pos
.whitespace
&& match_op(expansion
, '(')) {
631 while (!eof_token(expansion
)) {
632 struct token
*next
= expansion
->next
;
633 if (token_type(expansion
) == TOKEN_IDENT
) {
634 if (expansion
->ident
== &__VA_ARGS___ident
)
635 warn(expansion
->pos
, "__VA_ARGS__ can only appear in the expansion of a C99 variadic macro");
636 if (match_op(next
, SPECIAL_ELLIPSIS
)) {
637 expansion
->next
= &variable_argument
;
642 if (match_op(next
, ')')) {
643 // Terminate the arglist
644 expansion
->next
= &eof_token_entry
;
645 expansion
= next
->next
;
648 if (match_op(next
, ','))
649 expansion
->next
= next
->next
;
652 arglist
= arglist
->next
;
655 sym
= lookup_symbol(name
, NS_PREPROCESSOR
);
657 if (token_list_different(sym
->expansion
, expansion
) ||
658 token_list_different(sym
->arglist
, arglist
)) {
659 warn(left
->pos
, "preprocessor token %.*s redefined",
660 name
->len
, name
->name
);
661 info(sym
->pos
, "this was the original definition");
665 sym
= alloc_symbol(left
->pos
, SYM_NODE
);
666 bind_symbol(sym
, name
, NS_PREPROCESSOR
);
668 sym
->expansion
= expansion
;
669 sym
->arglist
= arglist
;
673 static int handle_undef(struct stream
*stream
, struct token
*head
, struct token
*token
)
675 struct token
*left
= token
->next
;
678 if (token_type(left
) != TOKEN_IDENT
) {
679 warn(head
->pos
, "expected identifier to 'undef'");
684 sym
= &left
->ident
->symbols
;
686 struct symbol
*t
= *sym
;
687 if (t
->namespace == NS_PREPROCESSOR
) {
696 static int preprocessor_if(struct token
*token
, int true)
699 unmatched_if
= token
;
700 if (if_nesting
>= MAX_NEST
)
701 error(token
->pos
, "Maximum preprocessor conditional level exhausted");
702 elif_ignore
[if_nesting
] = false_nesting
|| true;
703 if (false_nesting
|| !true) {
711 static int token_defined(struct token
*token
)
713 if (token_type(token
) == TOKEN_IDENT
)
714 return lookup_symbol(token
->ident
, NS_PREPROCESSOR
) != NULL
;
716 warn(token
->pos
, "expected identifier for #if[n]def");
720 static int handle_ifdef(struct stream
*stream
, struct token
*head
, struct token
*token
)
722 return preprocessor_if(token
, token_defined(token
->next
));
725 static int handle_ifndef(struct stream
*stream
, struct token
*head
, struct token
*token
)
727 struct token
*next
= token
->next
;
728 if (stream
->constant
== -1) {
730 if (token_type(next
) == TOKEN_IDENT
) {
731 if (!stream
->protect
|| stream
->protect
== next
->ident
) {
733 stream
->protect
= next
->ident
;
734 stream
->nesting
= if_nesting
+1;
737 stream
->constant
= newconstant
;
739 return preprocessor_if(token
, !token_defined(next
));
742 static int expression_value(struct token
*head
)
744 struct expression
*expr
;
748 expand_list(NULL
, head
);
749 token
= constant_expression(head
->next
, &expr
);
750 if (!eof_token(token
))
751 warn(token
->pos
, "garbage at end: %s", show_token_sequence(token
));
752 value
= get_expression_value(expr
);
756 static int handle_if(struct stream
*stream
, struct token
*head
, struct token
*token
)
760 value
= expression_value(token
);
761 return preprocessor_if(token
, value
);
764 static int handle_elif(struct stream
* stream
, struct token
*head
, struct token
*token
)
766 if (stream
->nesting
== if_nesting
)
767 stream
->constant
= 0;
769 /* If this whole if-thing is if'ed out, an elif cannot help */
770 if (elif_ignore
[if_nesting
-1])
772 if (expression_value(token
)) {
775 elif_ignore
[if_nesting
-1] = 1;
784 warn(token
->pos
, "unmatched '#elif'");
788 static int handle_else(struct stream
*stream
, struct token
*head
, struct token
*token
)
790 if (stream
->nesting
== if_nesting
)
791 stream
->constant
= 0;
793 /* If this whole if-thing is if'ed out, an else cannot help */
794 if (elif_ignore
[if_nesting
-1])
798 elif_ignore
[if_nesting
-1] = 1;
806 warn(token
->pos
, "unmatched #else");
810 static int handle_endif(struct stream
*stream
, struct token
*head
, struct token
*token
)
812 if (stream
->constant
== -2 && stream
->nesting
== if_nesting
)
813 stream
->constant
= -1;
823 warn(token
->pos
, "unmatched #endif");
827 static const char *show_token_sequence(struct token
*token
)
829 static char buffer
[1024];
835 while (!eof_token(token
) && !match_op(token
, SPECIAL_ARG_SEPARATOR
)) {
836 const char *val
= show_token(token
);
837 int len
= strlen(val
);
839 if (ptr
+ whitespace
+ len
> buffer
+ sizeof(buffer
)) {
840 warn(token
->pos
, "too long token expansion");
846 memcpy(ptr
, val
, len
);
849 whitespace
= token
->pos
.whitespace
;
855 static int handle_warning(struct stream
*stream
, struct token
*head
, struct token
*token
)
859 warn(token
->pos
, "%s", show_token_sequence(token
->next
));
863 static int handle_error(struct stream
*stream
, struct token
*head
, struct token
*token
)
867 warn(token
->pos
, "%s", show_token_sequence(token
->next
));
871 static int handle_nostdinc(struct stream
*stream
, struct token
*head
, struct token
*token
)
875 includepath
[0] = NULL
;
879 static void add_path_entry(struct token
*token
, const char *path
)
883 for (i
= 0; i
< INCLUDEPATHS
; i
++) {
884 if (!includepath
[i
]) {
885 includepath
[i
] = path
;
886 includepath
[i
+1] = NULL
;
890 warn(token
->pos
, "too many include path entries");
893 static int handle_add_include(struct stream
*stream
, struct token
*head
, struct token
*token
)
897 if (eof_token(token
))
899 if (token_type(token
) != TOKEN_STRING
) {
900 warn(token
->pos
, "expected path string");
903 add_path_entry(token
, token
->string
->data
);
908 * We replace "#pragma xxx" with "__pragma__" in the token
909 * stream. Just as an example.
911 * We'll just #define that away for now, but the theory here
912 * is that we can use this to insert arbitrary token sequences
913 * to turn the pragma's into internal front-end sequences for
914 * when we actually start caring about them.
916 * So eventually this will turn into some kind of extended
917 * __attribute__() like thing, except called __pragma__(xxx).
919 static int handle_pragma(struct stream
*stream
, struct token
*head
, struct token
*token
)
921 struct token
*next
= head
->next
;
923 token
->ident
= &pragma_ident
;
924 token
->pos
.newline
= 1;
925 token
->pos
.whitespace
= 1;
932 static int handle_preprocessor_command(struct stream
*stream
, struct token
*head
, struct ident
*ident
, struct token
*token
)
937 int (*handler
)(struct stream
*, struct token
*, struct token
*);
939 { "define", handle_define
},
940 { "undef", handle_undef
},
941 { "ifdef", handle_ifdef
},
942 { "ifndef", handle_ifndef
},
943 { "else", handle_else
},
944 { "endif", handle_endif
},
946 { "elif", handle_elif
},
947 { "warning", handle_warning
},
948 { "error", handle_error
},
949 { "include", handle_include
},
950 { "pragma", handle_pragma
},
952 // our internal preprocessor tokens
953 { "nostdinc", handle_nostdinc
},
954 { "add_include", handle_add_include
},
957 for (i
= 0; i
< (sizeof (handlers
) / sizeof (handlers
[0])); i
++) {
958 if (match_string_ident(ident
, handlers
[i
].name
))
959 return handlers
[i
].handler(stream
, head
, token
);
964 static void handle_preprocessor_line(struct stream
*stream
, struct token
* head
, struct token
*token
)
969 if (token_type(token
) == TOKEN_IDENT
)
970 if (handle_preprocessor_command(stream
, head
, token
->ident
, token
))
972 warn(token
->pos
, "unrecognized preprocessor line '%s'", show_token_sequence(token
));
975 static void preprocessor_line(struct stream
*stream
, struct token
* head
)
977 struct token
*start
= head
->next
, *next
;
978 struct token
**tp
= &start
->next
;
982 if (next
->pos
.newline
)
987 *tp
= &eof_token_entry
;
988 handle_preprocessor_line(stream
, head
, start
->next
);
991 static void do_preprocess(struct token
*head
)
994 struct token
*next
= head
->next
;
995 struct stream
*stream
= input_streams
+ next
->pos
.stream
;
997 if (next
->pos
.newline
&& match_op(next
, '#')) {
998 preprocessor_line(stream
, head
);
1002 if (false_nesting
) {
1003 head
->next
= next
->next
;
1007 switch (token_type(next
)) {
1008 case TOKEN_STREAMEND
:
1009 if (stream
->constant
== -1 && stream
->protect
) {
1010 stream
->constant
= 1;
1013 case TOKEN_STREAMBEGIN
:
1014 head
->next
= next
->next
;
1018 next
= expand_one_symbol(next
, head
, next
);
1022 * Any token expansion (even if it ended up being an
1023 * empty expansion) in this stream implies it can't
1026 stream
->constant
= 0;
1030 } while (!eof_token(head
));
1033 struct token
* preprocess(struct token
*token
)
1035 struct token header
= { };
1038 header
.next
= token
;
1039 do_preprocess(&header
);
1041 warn(unmatched_if
->pos
, "unmatched preprocessor conditional");
1043 // Drop all expressions from pre-processing, they're not used any more.
1044 clear_expression_alloc();