pre-process.c

   1 /*
   2  * Do C preprocessing, based on a token list gathered by
   3  * the tokenizer.
   4  *
   5  * This may not be the smartest preprocessor on the planet.
   6  *
   7  * Copyright (C) 2003 Transmeta Corp.
   8  *
   9  *  Licensed under the Open Software License version 1.1
  10  */
  11 #include <stdio.h>
  12 #include <stdlib.h>
  13 #include <stdarg.h>
  14 #include <stddef.h>
  15 #include <string.h>
  16 #include <ctype.h>
  17 #include <unistd.h>
  18 #include <fcntl.h>
  19 #include <limits.h>
  20
  21 #include "lib.h"
  22 #include "parse.h"
  23 #include "token.h"
  24 #include "symbol.h"
  25 #include "expression.h"
  26
  27 int preprocessing = 0;
  28
  29 #define MAXNEST (16)
  30 static int true_nesting = 0;
  31 static int false_nesting = 0;
  32 static struct token *unmatched_if = NULL;
  33 static int elif_ignore[MAXNEST];
  34 #define if_nesting (true_nesting + false_nesting)
  35
  36 #define INCLUDEPATHS 32
  37 const char *includepath[INCLUDEPATHS+1] = {
  38         NULL
  39 };
  40
  41 const char *sys_includepath[] = {
  42         "/usr/include",
  43         "/usr/local/include",
  44         NULL,
  45 };
  46
  47 const char *gcc_includepath[] = {
  48         "/usr/lib/gcc-lib/i386-redhat-linux/3.2.1/include",
  49         "/usr/lib/gcc-lib/i386-redhat-linux/3.2.2/include",
  50         NULL
  51 };
  52
  53
  54 /*
  55  * This is stupid - the tokenizer already guarantees unique
  56  * identifiers, so we should just compare identifier pointers
  57  */
  58 int match_string_ident(struct ident *ident, const char *str)
  59 {
  60         return !str[ident->len] && !memcmp(str, ident->name, ident->len);
  61 }
  62
  63 static struct token *alloc_token(struct position *pos)
  64 {
  65         struct token *token = __alloc_token(0);
  66
  67         token->pos.stream = pos->stream;
  68         token->pos.line = pos->line;
  69         token->pos.pos = pos->pos;
  70         token->pos.whitespace = 1;
  71         return token;
  72 }
  73
  74 static const char *show_token_sequence(struct token *token);
  75
  76 /* Head is one-before-list, and last is one-past-list */
  77 static struct token *for_each_ident(struct token *parent, struct token *head, struct token *(*action)(struct token *parent, struct token *head, struct token *))
  78 {
  79         for (;;) {
  80                 struct token *next = head->next;
  81
  82                 /* Did we hit the end of the current expansion? */
  83                 if (eof_token(next))
  84                         break;
  85
  86                 if (token_type(next) == TOKEN_IDENT)
  87                         next = action(parent, head, next);
  88
  89                 head = next;
  90         }
  91         return head;
  92 }
  93
  94 static struct token *is_defined(struct token *head, struct token *token, struct token *next)
  95 {
  96         char *string[] = { "0", "1" };
  97         char *defined = string[lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL];
  98         struct token *newtoken = alloc_token(&token->pos);
  99
 100         token_type(newtoken) = TOKEN_INTEGER;
 101         newtoken->integer = defined;
 102         newtoken->next = next;
 103         head->next = newtoken;
 104         return next;
 105 }
 106
 107
 108 struct token *defined_one_symbol(struct token *head, struct token *next)
 109 {
 110         struct token *token = next->next;
 111         struct token *past = token->next;
 112
 113         if (match_op(token, '(')) {
 114                 token = past;
 115                 past = token->next;
 116                 if (!match_op(past, ')'))
 117                         return next;
 118                 past = past->next;
 119         }
 120         if (token_type(token) == TOKEN_IDENT)
 121                 return is_defined(head, token, past);
 122         return next;
 123 }
 124
 125 /* Expand symbol 'sym' between 'head->next' and 'head->next->next' */
 126 static struct token *expand(struct token *, struct token *, struct symbol *);
 127
 128 static void replace_with_string(struct token *token, const char *str)
 129 {
 130         int size = strlen(str) + 1;
 131         struct string *s = __alloc_string(size);
 132
 133         s->length = size;
 134         memcpy(s->data, str, size);
 135         token_type(token) = TOKEN_STRING;
 136         token->string = s;
 137 }
 138
 139 static void replace_with_integer(struct token *token, unsigned int val)
 140 {
 141         char *buf = __alloc_bytes(10);
 142         sprintf(buf, "%d", val);
 143         token_type(token) = TOKEN_INTEGER;
 144         token->integer = buf;
 145 }
 146
 147 struct token *expand_one_symbol(struct token *parent, struct token *head, struct token *token)
 148 {
 149         struct symbol *sym;
 150         struct token *x;
 151
 152         /* Avoid recursive expansion */
 153         x = token;
 154         while ((x = x->parent) != NULL) {
 155                 if (x->ident == parent->ident)
 156                         return token;
 157         }
 158
 159         sym = lookup_symbol(token->ident, NS_PREPROCESSOR);
 160         if (sym) {
 161                 if (sym->arglist && !match_op(token->next, '('))
 162                         return token;
 163                 return expand(parent, head, sym);
 164         }
 165         if (!memcmp(token->ident->name, "__LINE__", 9)) {
 166                 replace_with_integer(token, token->pos.line);
 167         } else if (!memcmp(token->ident->name, "__FILE__", 9)) {
 168                 replace_with_string(token, (input_streams + token->pos.stream)->name);
 169         } else if (!memcmp(token->ident->name, "defined", 8)) {
 170                 return defined_one_symbol(head, token);
 171         }
 172         return token;
 173 }
 174
 175 static struct token *expand_list(struct token *parent, struct token *head)
 176 {
 177         return for_each_ident(parent, head, expand_one_symbol);
 178 }
 179
 180 static struct token *find_argument_end(struct token *start)
 181 {
 182         int nesting = 0;
 183
 184         while (!eof_token(start)) {
 185                 struct token *next = start->next;
 186                 if (match_op(next, '('))
 187                         nesting++;
 188                 else if (match_op(next, ')')) {
 189                         if (--nesting < 0) {
 190                                 start->next = &eof_token_entry;
 191                                 return next->next;
 192                         }
 193                 } else if (!nesting && match_op(next, ','))
 194                         next->special = SPECIAL_ARG_SEPARATOR;
 195                 start = next;
 196         }
 197         return start;
 198 }
 199
 200 static struct token *dup_token(struct token *token, struct position *pos, int newline)
 201 {
 202         struct token *alloc = alloc_token(pos);
 203         token_type(alloc) = token_type(token);
 204         alloc->pos.line = pos->line;
 205         alloc->pos.newline = newline;
 206         alloc->integer = token->integer;
 207         return alloc;
 208 }
 209
 210 static void insert(struct token *token, struct token *prev)
 211 {
 212         token->next = prev->next;
 213         prev->next = token;
 214 }
 215
 216 static struct token * replace(struct token *parent, struct token *token, struct token *prev, struct token *list)
 217 {
 218         int newline = token->pos.newline;
 219
 220         prev->next = token->next;
 221         while (!eof_token(list) && !match_op(list, SPECIAL_ARG_SEPARATOR)) {
 222                 struct token *newtok = dup_token(list, &token->pos, newline);
 223                 newtok->parent = parent;
 224                 newline = 0;
 225                 insert(newtok, prev);
 226                 prev = newtok;
 227                 list = list->next;
 228         }
 229         return prev;
 230 }
 231
 232 static struct token *get_argument(int nr, struct token *args)
 233 {
 234         if (!nr)
 235                 return args;
 236         while (!eof_token(args)) {
 237                 if (match_op(args, SPECIAL_ARG_SEPARATOR))
 238                         if (!--nr)
 239                                 return args->next;
 240                 args = args->next;
 241         }
 242
 243         return args;
 244 }
 245
 246 static struct token *stringify(struct token *token, struct token *arg)
 247 {
 248         const char *s = show_token_sequence(arg);
 249         int size = strlen(s)+1;
 250         struct token *newtoken = alloc_token(&token->pos);
 251         struct string *string = __alloc_string(size);
 252
 253         newtoken->pos.newline = token->pos.newline;
 254         memcpy(string->data, s, size);
 255         string->length = size;
 256         token_type(newtoken) = TOKEN_STRING;
 257         newtoken->string = string;
 258         newtoken->next = &eof_token_entry;
 259         return newtoken;
 260 }
 261
 262 static int arg_number(struct token *arglist, struct ident *ident)
 263 {
 264         int nr = 0;
 265
 266         while (!eof_token(arglist)) {
 267                 if (arglist->ident == ident)
 268                         return nr;
 269                 nr++;
 270                 arglist = arglist->next;
 271         }
 272         return -1;
 273 }
 274
 275 static struct token empty_arg_token = { .pos = { .type = TOKEN_EOF } };
 276
 277 static struct token *expand_one_arg(struct token *parent, struct token *head, struct token *token,
 278                 struct token *arglist, struct token *arguments)
 279 {
 280         int nr = arg_number(arglist, token->ident);
 281         struct token *orig_head = head;
 282
 283         if (nr >= 0) {
 284                 struct token *arg = get_argument(nr, arguments);
 285                 struct token *last = token->next;
 286                 token->next = &eof_token_entry;
 287
 288                 /*
 289                  * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
 290                  * then the 'x' goes away too.
 291                  */
 292                 if (match_op(head, SPECIAL_HASHHASH) && eof_token(arg)) {
 293                         arg = &empty_arg_token;
 294                         empty_arg_token.next = &eof_token_entry;
 295                 }
 296
 297                 head = replace(parent, token, head, arg);
 298                 if (!match_op(orig_head, SPECIAL_HASHHASH) && !match_op(last, SPECIAL_HASHHASH) && !match_op(orig_head, '#'))
 299                         head = expand_list(parent, orig_head);
 300                 head->next = last;
 301                 return head;
 302         }
 303         return token;
 304 }
 305
 306 static void expand_arguments(struct token *parent,
 307         struct token *token, struct token *head,
 308         struct token *arguments, struct token *arglist)
 309 {
 310         for (;;) {
 311                 struct token *next = head->next;
 312
 313                 /* Did we hit the end of the current expansion? */
 314                 if (eof_token(next))
 315                         break;
 316
 317                 if (match_op(next, '#')) {
 318                         struct token *nextnext = next->next;
 319                         int nr = arg_number(arglist, nextnext->ident);
 320                         if (nextnext != head && nr >= 0 && token_type(nextnext) == TOKEN_IDENT) {
 321                                 struct token *newtoken = stringify(nextnext, get_argument(nr, arguments));
 322                                 replace(parent, nextnext, head, newtoken);
 323                                 continue;
 324                         }
 325                         warn(next->pos, "'#' operation is not followed by argument name");
 326                 }
 327
 328                 if (token_type(next) == TOKEN_IDENT)
 329                         next = expand_one_arg(parent, head, next, arglist, arguments);
 330
 331                 head = next;
 332         }
 333 }
 334
 335 /*
 336  * Possibly valid combinations:
 337  *  - anything + 'empty_arg_token' is empty.
 338  *  - ident + ident - combine (==ident)
 339  *  - ident + number - combine (==ident)
 340  *  - number + number - combine (==number)
 341  *  - number + ident - combine (==number)
 342  *  - string + string - leave as is, C will combine them anyway
 343  * others cause an error and leave the two tokens as separate tokens.
 344  */
 345 static struct token *hashhash(struct token *head, struct token *first, struct token *second)
 346 {
 347         static char buffer[512], *p;
 348         struct token *newtoken;
 349         static const char *src;
 350         int len;
 351
 352         first->next = second;
 353
 354         /*
 355          * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
 356          * then the 'x' goes away too.
 357          *
 358          * See expand_one_arg.
 359          */
 360         if (token_type(second) == TOKEN_EOF) {
 361                 head->next = second->next;
 362                 return head;
 363         }
 364
 365         p = buffer;
 366         switch (token_type(first)) {
 367         case TOKEN_INTEGER:
 368                 len = strlen(first->integer);
 369                 src = first->integer;
 370                 break;
 371         case TOKEN_IDENT:
 372                 len = first->ident->len;
 373                 src = first->ident->name;
 374                 break;
 375         default:
 376                 return second;
 377         }
 378         memcpy(p, src, len);
 379         p += len;
 380
 381         switch (token_type(second)) {
 382         case TOKEN_INTEGER:
 383                 len = strlen(second->integer);
 384                 src = second->integer;
 385                 break;
 386         case TOKEN_IDENT:
 387                 len = second->ident->len;
 388                 src = second->ident->name;
 389                 break;
 390         default:
 391                 return second;
 392         }
 393         memcpy(p, src, len);
 394         p += len;
 395         *p++ = 0;
 396
 397         newtoken = alloc_token(&first->pos);
 398         head->next = newtoken;
 399         token_type(newtoken) = token_type(first);
 400         switch (token_type(newtoken)) {
 401         case TOKEN_IDENT:
 402                 newtoken->ident = built_in_ident(buffer);
 403                 break;
 404         case TOKEN_INTEGER:
 405                 newtoken->integer = __alloc_bytes(p - buffer);
 406                 memcpy(newtoken->integer, buffer, p - buffer);
 407                 break;
 408         }
 409         return newtoken;
 410 }
 411
 412 static void retokenize(struct token *head)
 413 {
 414         struct token * next = head->next;
 415         struct token * nextnext = next->next;
 416         struct token * nextnextnext = nextnext->next;
 417
 418         if (eof_token(next) || eof_token(nextnext))
 419                 return;
 420
 421         for (;;) {
 422                 if (eof_token(nextnextnext))
 423                         break;
 424
 425                 if (match_op(nextnext, SPECIAL_HASHHASH)) {
 426                         struct token *newtoken = hashhash(head, next, nextnextnext);
 427
 428                         next = newtoken;
 429                         nextnext = nextnextnext->next;
 430                         nextnextnext = nextnext->next;
 431
 432                         newtoken->next = nextnext;
 433                         if (!eof_token(nextnext))
 434                                 continue;
 435                         break;
 436                 }
 437
 438                 head = next;
 439                 next = nextnext;
 440                 nextnext = nextnext->next;
 441                 nextnextnext = nextnextnext->next;
 442         }
 443 }
 444
 445 static struct token *expand(struct token *parent, struct token *head, struct symbol *sym)
 446 {
 447         struct token *arguments, *token, *last;
 448
 449         token = head->next;
 450         last = token->next;
 451
 452         arguments = NULL;
 453         if (sym->arglist) {
 454                 arguments = last->next;
 455                 last = find_argument_end(last);
 456         }
 457         token->next = &eof_token_entry;
 458
 459         /* Replace the token with the token expansion */
 460         replace(parent, token, head, sym->expansion);
 461
 462         /* Then, replace all the arguments with their expansions */
 463         if (arguments)
 464                 expand_arguments(parent, token, head, arguments, sym->arglist);
 465
 466         /* Re-tokenize the sequence if any ## token exists.. */
 467         retokenize(head);
 468
 469         token = head;
 470         while (!eof_token(token->next))
 471                 token = token->next;
 472         token->next = last;
 473         return head;
 474 }
 475
 476 static const char *token_name_sequence(struct token *token, int endop, struct token *start)
 477 {
 478         struct token *last;
 479         static char buffer[256];
 480         char *ptr = buffer;
 481
 482         last = token;
 483         while (!eof_token(token) && !match_op(token, endop)) {
 484                 int len;
 485                 const char *val = token->string->data;
 486                 if (token_type(token) != TOKEN_STRING)
 487                         val = show_token(token);
 488                 len = strlen(val);
 489                 memcpy(ptr, val, len);
 490                 ptr += len;
 491                 token = token->next;
 492         }
 493         *ptr = 0;
 494         if (endop && !match_op(token, endop))
 495                 warn(start->pos, "expected '>' at end of filename");
 496         return buffer;
 497 }
 498
 499 static int try_include(const char *path, int plen, const char *filename, int flen, struct token *head)
 500 {
 501         int fd;
 502         static char fullname[PATH_MAX];
 503
 504         memcpy(fullname, path, plen);
 505         if (plen && path[plen-1] != '/') {
 506                 fullname[plen] = '/';
 507                 plen++;
 508         }
 509         memcpy(fullname+plen, filename, flen);
 510         fd = open(fullname, O_RDONLY);
 511         if (fd >= 0) {
 512                 char * streamname = __alloc_bytes(plen + flen);
 513                 memcpy(streamname, fullname, plen + flen);
 514                 head->next = tokenize(streamname, fd, head->next);
 515                 close(fd);
 516                 return 1;
 517         }
 518         return 0;
 519 }
 520
 521 static int do_include_path(const char **pptr, struct token *head, struct token *token, const char *filename, int flen)
 522 {
 523         const char *path;
 524
 525         while ((path = *pptr++) != NULL) {
 526                 if (!try_include(path, strlen(path), filename, flen, head))
 527                         continue;
 528                 return 1;
 529         }
 530         return 0;
 531 }
 532
 533
 534 static void do_include(struct stream *stream, struct token *head, struct token *token, const char *filename)
 535 {
 536         const char *path;
 537         char *slash;
 538         int flen = strlen(filename) + 1;
 539
 540         /* Check the standard include paths.. */
 541         if (do_include_path(includepath, head, token, filename, flen))
 542                 return;
 543         if (do_include_path(sys_includepath, head, token, filename, flen))
 544                 return;
 545         if (do_include_path(gcc_includepath, head, token, filename, flen))
 546                 return;
 547
 548         /* Check same directory as current stream.. */
 549         path = stream->name;
 550         slash = strrchr(path, '/');
 551         if (slash) {
 552                 if (try_include(path, slash-path, filename, flen, head))
 553                         return;
 554         }
 555
 556         error(token->pos, "unable to open '%s'", filename);
 557 }
 558
 559 static int handle_include(struct stream *stream, struct token *head, struct token *token)
 560 {
 561         const char *filename;
 562         struct token *next;
 563         int expect;
 564
 565         if (stream->constant == -1)
 566                 stream->constant = 0;
 567         if (false_nesting)
 568                 return 1;
 569         next = token->next;
 570         expect = '>';
 571         if (!match_op(next, '<')) {
 572                 expand_list(NULL, token);
 573                 expect = 0;
 574                 next = token;
 575         }
 576         token = next->next;
 577         filename = token_name_sequence(token, expect, token);
 578         do_include(stream, head, token, filename);
 579         return 1;
 580 }
 581
 582 static int token_list_different(struct token *list1, struct token *list2)
 583 {
 584         for (;;) {
 585                 if (list1 == list2)
 586                         return 0;
 587                 if (!list1 || !list2)
 588                         return 1;
 589                 if (token_type(list1) != token_type(list2))
 590                         return 1;
 591                 list1 = list1->next;
 592                 list2 = list2->next;
 593         }
 594 }
 595
 596
 597 static int handle_define(struct stream *stream, struct token *head, struct token *token)
 598 {
 599         struct token *arglist, *expansion;
 600         struct token *left = token->next;
 601         struct symbol *sym;
 602         struct ident *name;
 603
 604         if (token_type(left) != TOKEN_IDENT) {
 605                 warn(head->pos, "expected identifier to 'define'");
 606                 return 0;
 607         }
 608         if (false_nesting)
 609                 return 1;
 610         name = left->ident;
 611
 612         arglist = NULL;
 613         expansion = left->next;
 614         if (!expansion->pos.whitespace && match_op(expansion, '(')) {
 615                 arglist = expansion;
 616                 while (!eof_token(expansion)) {
 617                         struct token *next = expansion->next;
 618                         if (match_op(next, ')')) {
 619                                 // Terminate the arglist
 620                                 expansion->next = &eof_token_entry;
 621                                 expansion = next->next;
 622                                 break;
 623                         }
 624                         if (match_op(next, ','))
 625                                 expansion->next = next->next;
 626                         expansion = next;
 627                 }
 628                 arglist = arglist->next;
 629         }
 630
 631         sym = lookup_symbol(name, NS_PREPROCESSOR);
 632         if (sym) {
 633                 if (token_list_different(sym->expansion, expansion) ||
 634                     token_list_different(sym->arglist, arglist)) {
 635                         warn(left->pos, "preprocessor token redefined");
 636                         warn(sym->pos, "this was the original definition");
 637                 }
 638                 return 1;
 639         }
 640         sym = alloc_symbol(left->pos, SYM_NODE);
 641         bind_symbol(sym, name, NS_PREPROCESSOR);
 642
 643         sym->expansion = expansion;
 644         sym->arglist = arglist;
 645         return 1;
 646 }
 647
 648 static int handle_undef(struct stream *stream, struct token *head, struct token *token)
 649 {
 650         struct token *left = token->next;
 651         struct symbol **sym;
 652
 653         if (token_type(left) != TOKEN_IDENT) {
 654                 warn(head->pos, "expected identifier to 'undef'");
 655                 return 0;
 656         }
 657         if (false_nesting)
 658                 return 1;
 659         sym = &left->ident->symbols;
 660         while (*sym) {
 661                 struct symbol *t = *sym;
 662                 if (t->namespace == NS_PREPROCESSOR) {
 663                         *sym = t->next_id;
 664                         return 1;
 665                 }
 666                 sym = &t->next_id;
 667         }
 668         return 1;
 669 }
 670
 671 static int preprocessor_if(struct token *token, int true)
 672 {
 673         if (if_nesting == 0)
 674                 unmatched_if = token;
 675         elif_ignore[if_nesting] = false_nesting || true;
 676         if (false_nesting || !true) {
 677                 false_nesting++;
 678                 return 1;
 679         }
 680         true_nesting++;
 681         return 1;
 682 }
 683
 684 static int token_defined(struct token *token)
 685 {
 686         if (token_type(token) == TOKEN_IDENT)
 687                 return lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL;
 688
 689         warn(token->pos, "expected identifier for #if[n]def");
 690         return 0;
 691 }
 692
 693 static int handle_ifdef(struct stream *stream, struct token *head, struct token *token)
 694 {
 695         return preprocessor_if(token, token_defined(token->next));
 696 }
 697
 698 static int handle_ifndef(struct stream *stream, struct token *head, struct token *token)
 699 {
 700         struct token *next = token->next;
 701         if (stream->constant == -1) {
 702                 int newconstant = 0;
 703                 if (token_type(next) == TOKEN_IDENT) {
 704                         if (!stream->protect || stream->protect == next->ident) {
 705                                 newconstant = -2;
 706                                 stream->protect = next->ident;
 707                                 stream->nesting = if_nesting+1;
 708                         }
 709                 }
 710                 stream->constant = newconstant;
 711         }
 712         return preprocessor_if(token, !token_defined(next));
 713 }
 714
 715 static int expression_value(struct token *head)
 716 {
 717         struct expression *expr;
 718         struct token *token;
 719         long long value;
 720
 721         expand_list(NULL, head);
 722         token = constant_expression(head->next, &expr);
 723         if (!eof_token(token))
 724                 warn(token->pos, "garbage at end: %s", show_token_sequence(token));
 725         value = get_expression_value(expr);
 726         return value != 0;
 727 }
 728
 729 static int handle_if(struct stream *stream, struct token *head, struct token *token)
 730 {
 731         int value = 0;
 732         if (!false_nesting)
 733                 value = expression_value(token);
 734         return preprocessor_if(token, value);
 735 }
 736
 737 static int handle_elif(struct stream * stream, struct token *head, struct token *token)
 738 {
 739         if (stream->nesting == if_nesting)
 740                 stream->constant = 0;
 741         if (false_nesting) {
 742                 /* If this whole if-thing is if'ed out, an elif cannot help */
 743                 if (elif_ignore[if_nesting-1])
 744                         return 1;
 745                 if (expression_value(token)) {
 746                         false_nesting--;
 747                         true_nesting++;
 748                         elif_ignore[if_nesting-1] = 1;
 749                 }
 750                 return 1;
 751         }
 752         if (true_nesting) {
 753                 false_nesting = 1;
 754                 true_nesting--;
 755                 return 1;
 756         }
 757         warn(token->pos, "unmatched '#elif'");
 758         return 1;
 759 }
 760
 761 static int handle_else(struct stream *stream, struct token *head, struct token *token)
 762 {
 763         if (stream->nesting == if_nesting)
 764                 stream->constant = 0;
 765         if (false_nesting) {
 766                 /* If this whole if-thing is if'ed out, an else cannot help */
 767                 if (elif_ignore[if_nesting-1])
 768                         return 1;
 769                 false_nesting--;
 770                 true_nesting++;
 771                 elif_ignore[if_nesting-1] = 1;
 772                 return 1;
 773         }
 774         if (true_nesting) {
 775                 true_nesting--;
 776                 false_nesting = 1;
 777                 return 1;
 778         }
 779         warn(token->pos, "unmatched #else");
 780         return 1;
 781 }
 782
 783 static int handle_endif(struct stream *stream, struct token *head, struct token *token)
 784 {
 785         if (stream->constant == -2 && stream->nesting == if_nesting)
 786                 stream->constant = -1;
 787
 788         if (false_nesting) {
 789                 false_nesting--;
 790                 return 1;
 791         }
 792         if (true_nesting) {
 793                 true_nesting--;
 794                 return 1;
 795         }
 796         warn(token->pos, "unmatched #endif");
 797         return 1;
 798 }
 799
 800 static const char *show_token_sequence(struct token *token)
 801 {
 802         static char buffer[256];
 803         char *ptr = buffer;
 804         int whitespace = 0;
 805
 806         if (!token)
 807                 return "<none>";
 808         while (!eof_token(token) && !match_op(token, SPECIAL_ARG_SEPARATOR)) {
 809                 const char *val = show_token(token);
 810                 int len = strlen(val);
 811                 if (whitespace)
 812                         *ptr++ = ' ';
 813                 memcpy(ptr, val, len);
 814                 ptr += len;
 815                 token = token->next;
 816                 whitespace = token->pos.whitespace;
 817         }
 818         *ptr = 0;
 819         return buffer;
 820 }
 821
 822 static int handle_warning(struct stream *stream, struct token *head, struct token *token)
 823 {
 824         if (false_nesting)
 825                 return 1;
 826         warn(token->pos, "%s", show_token_sequence(token->next));
 827         return 1;
 828 }
 829
 830 static int handle_error(struct stream *stream, struct token *head, struct token *token)
 831 {
 832         if (false_nesting)
 833                 return 1;
 834         warn(token->pos, "%s", show_token_sequence(token->next));
 835         return 1;
 836 }
 837
 838 static int handle_nostdinc(struct stream *stream, struct token *head, struct token *token)
 839 {
 840         if (false_nesting)
 841                 return 1;
 842         includepath[0] = NULL;
 843         return 1;
 844 }
 845
 846 static void add_path_entry(struct token *token, const char *path)
 847 {
 848         int i;
 849
 850         for (i = 0; i < INCLUDEPATHS; i++) {
 851                 if (!includepath[i]) {
 852                         includepath[i] = path;
 853                         includepath[i+1] = NULL;
 854                         return;
 855                 }
 856         }
 857         warn(token->pos, "too many include path entries");
 858 }
 859
 860 static int handle_add_include(struct stream *stream, struct token *head, struct token *token)
 861 {
 862         for (;;) {
 863                 token = token->next;
 864                 if (eof_token(token))
 865                         return 1;
 866                 if (token_type(token) != TOKEN_STRING) {
 867                         warn(token->pos, "expected path string");
 868                         return 1;
 869                 }
 870                 add_path_entry(token, token->string->data);
 871         }
 872 }
 873
 874 static int handle_preprocessor_command(struct stream *stream, struct token *head, struct ident *ident, struct token *token)
 875 {
 876         int i;
 877         static struct {
 878                 const char *name;
 879                 int (*handler)(struct stream *, struct token *, struct token *);
 880         } handlers[] = {
 881                 { "define",     handle_define },
 882                 { "undef",      handle_undef },
 883                 { "ifdef",      handle_ifdef },
 884                 { "ifndef",     handle_ifndef },
 885                 { "else",       handle_else },
 886                 { "endif",      handle_endif },
 887                 { "if",         handle_if },
 888                 { "elif",       handle_elif },
 889                 { "warning",    handle_warning },
 890                 { "error",      handle_error },
 891                 { "include",    handle_include },
 892
 893                 // our internal preprocessor tokens
 894                 { "nostdinc",   handle_nostdinc },
 895                 { "add_include", handle_add_include },
 896         };
 897
 898         for (i = 0; i < (sizeof (handlers) / sizeof (handlers[0])); i++) {
 899                 if (match_string_ident(ident, handlers[i].name))
 900                         return handlers[i].handler(stream, head, token);
 901         }
 902         return 0;
 903 }
 904
 905 static void handle_preprocessor_line(struct stream *stream, struct token * head, struct token *token)
 906 {
 907         if (!token)
 908                 return;
 909
 910         if (token_type(token) == TOKEN_IDENT)
 911                 if (handle_preprocessor_command(stream, head, token->ident, token))
 912                         return;
 913         warn(token->pos, "unrecognized preprocessor line '%s'", show_token_sequence(token));
 914 }
 915
 916 static void preprocessor_line(struct stream *stream, struct token * head)
 917 {
 918         struct token *start = head->next, *next;
 919         struct token **tp = &start->next;
 920
 921         for (;;) {
 922                 next = *tp;
 923                 if (next->pos.newline)
 924                         break;
 925                 tp = &next->next;
 926         }
 927         head->next = next;
 928         *tp = &eof_token_entry;
 929         handle_preprocessor_line(stream, head, start->next);
 930 }
 931
 932 static void do_preprocess(struct token *head)
 933 {
 934         do {
 935                 struct token *next = head->next;
 936                 struct stream *stream = input_streams + next->pos.stream;
 937
 938                 if (next->pos.newline && match_op(next, '#')) {
 939                         preprocessor_line(stream, head);
 940                         continue;
 941                 }
 942
 943                 if (false_nesting) {
 944                         head->next = next->next;
 945                         continue;
 946                 }
 947
 948                 switch (token_type(next)) {
 949                 case TOKEN_STREAMEND:
 950                         if (stream->constant == -1 && stream->protect) {
 951                                 stream->constant = 1;
 952                         }
 953                         /* fallthrough */
 954                 case TOKEN_STREAMBEGIN:
 955                         head->next = next->next;
 956                         continue;
 957
 958                 case TOKEN_IDENT:
 959                         next = expand_one_symbol(next, head, next);
 960                         /* fallthrough */
 961                 default:
 962                         /*
 963                          * Any token expansion (even if it ended up being an
 964                          * empty expansion) in this stream implies it can't
 965                          * be constant.
 966                          */
 967                         stream->constant = 0;
 968                 }
 969
 970                 head = next;
 971         } while (!eof_token(head));
 972 }
 973
 974 struct token * preprocess(struct token *token)
 975 {
 976         struct token header = { };
 977
 978         preprocessing = 1;
 979         header.next = token;
 980         do_preprocess(&header);
 981         if (if_nesting)
 982                 warn(unmatched_if->pos, "unmatched preprocessor conditional");
 983
 984         // Drop all expressions from pre-processing, they're not used any more.
 985         clear_expression_alloc();
 986         preprocessing = 0;
 987
 988         return header.next;
 989 }