pre-process.c

   1 /*
   2  * Do C preprocessing, based on a token list gathered by
   3  * the tokenizer.
   4  *
   5  * This may not be the smartest preprocessor on the planet.
   6  *
   7  * Copyright (C) 2003 Linus Torvalds, all rights reserved.
   8  */
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <stdarg.h>
  12 #include <stddef.h>
  13 #include <string.h>
  14 #include <ctype.h>
  15 #include <unistd.h>
  16 #include <fcntl.h>
  17 #include <limits.h>
  18
  19 #include "lib.h"
  20 #include "parse.h"
  21 #include "token.h"
  22 #include "symbol.h"
  23
  24 #define MAXNEST (16)
  25 static int true_nesting = 0;
  26 static int false_nesting = 0;
  27 static struct token *unmatched_if = NULL;
  28 static int elif_ignore[MAXNEST];
  29 #define if_nesting (true_nesting + false_nesting)
  30
  31 /*
  32  * This is stupid - the tokenizer already guarantees unique
  33  * identifiers, so we should just compare identifier pointers
  34  */
  35 static int match_string_ident(struct ident *ident, const char *str)
  36 {
  37         return !str[ident->len] && !memcmp(str, ident->name, ident->len);
  38 }
  39
  40 static struct token *alloc_token(struct token *dup)
  41 {
  42         struct token *token = __alloc_token(0);
  43
  44         token->stream = dup->stream;
  45         token->line = dup->line;
  46         token->pos = dup->pos;
  47         token->whitespace = 1;
  48         return token;
  49 }
  50
  51 static const char *show_token_sequence(struct token *token);
  52
  53 /* Head is one-before-list, and last is one-past-list */
  54 static struct token *for_each_ident(struct token *head, struct token *(*action)(struct token *head, struct token *))
  55 {
  56         for (;;) {
  57                 struct token *next = head->next;
  58
  59                 /* Did we hit the end of the current expansion? */
  60                 if (eof_token(next))
  61                         break;
  62
  63                 if (next->type == TOKEN_IDENT)
  64                         next = action(head, next);
  65
  66                 head = next;
  67         }
  68         return head;
  69 }
  70
  71 static struct token *is_defined(struct token *head, struct token *token, struct token *next)
  72 {
  73         char *string[] = { "0", "1" };
  74         char *defined = string[lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL];
  75         struct token *newtoken = alloc_token(token);
  76
  77         newtoken->type = TOKEN_INTEGER;
  78         newtoken->integer = defined;
  79         newtoken->next = next;
  80         head->next = newtoken;
  81         return next;
  82 }
  83
  84
  85 struct token *defined_one_symbol(struct token *head, struct token *next)
  86 {
  87         if (match_string_ident(next->ident, "defined")) {
  88                 struct token *token = next->next;
  89                 struct token *past = token->next;
  90
  91                 if (match_op(token, '(')) {
  92                         token = past;
  93                         past = token->next;
  94                         if (!match_op(past, ')'))
  95                                 return next;
  96                         past = past->next;
  97                 }
  98                 if (token->type == TOKEN_IDENT)
  99                         return is_defined(head, token, past);
 100         }
 101         return next;
 102 }
 103
 104 static struct token *expand_defined(struct token *head)
 105 {
 106         return for_each_ident(head, defined_one_symbol);
 107 }
 108
 109 /* Expand symbol 'sym' between 'head->next' and 'head->next->next' */
 110 static struct token *expand(struct token *, struct symbol *);
 111
 112 struct token *expand_one_symbol(struct token *head, struct token *token)
 113 {
 114         struct symbol *sym = lookup_symbol(token->ident, NS_PREPROCESSOR);
 115         if (sym && !sym->busy) {
 116                 if (sym->arglist && !match_op(token->next, '('))
 117                         return token;
 118                 return expand(head, sym);
 119         }
 120         return token;
 121 }
 122
 123 static struct token *expand_list(struct token *head)
 124 {
 125         return for_each_ident(head, expand_one_symbol);
 126 }
 127
 128 static struct token *find_argument_end(struct token *start)
 129 {
 130         int nesting = 0;
 131
 132         while (!eof_token(start)) {
 133                 struct token *next = start->next;
 134                 if (match_op(next, '('))
 135                         nesting++;
 136                 else if (match_op(next, ')')) {
 137                         if (--nesting < 0) {
 138                                 start->next = &eof_token_entry;
 139                                 return next->next;
 140                         }
 141                 } else if (!nesting && match_op(next, ','))
 142                         next->special = SPECIAL_ARG_SEPARATOR;
 143                 start = next;
 144         }
 145         return start;
 146 }
 147
 148 static struct token *dup_token(struct token *token, struct token *pos, int newline)
 149 {
 150         struct token *alloc = alloc_token(pos);
 151         alloc->type = token->type;
 152         alloc->line = pos->line;
 153         alloc->newline = newline;
 154         alloc->integer = token->integer;
 155         return alloc;
 156 }
 157
 158 static void insert(struct token *token, struct token *prev)
 159 {
 160         token->next = &eof_token_entry;
 161         prev->next = token;
 162 }
 163
 164 static struct token * replace(struct token *token, struct token *prev, struct token *list)
 165 {
 166         int newline = token->newline;
 167
 168         prev->next = &eof_token_entry;
 169         while (!eof_token(list) && !match_op(list, SPECIAL_ARG_SEPARATOR)) {
 170                 struct token *newtok = dup_token(list, token, newline);
 171                 newline = 0;
 172                 insert(newtok, prev);
 173                 prev = newtok;
 174                 list = list->next;
 175         }
 176         return prev;
 177 }
 178
 179 static struct token *get_argument(int nr, struct token *args)
 180 {
 181         if (!nr)
 182                 return args;
 183         while (!eof_token(args)) {
 184                 if (match_op(args, SPECIAL_ARG_SEPARATOR))
 185                         if (!--nr)
 186                                 return args->next;
 187                 args = args->next;
 188         }
 189
 190         return args;
 191 }
 192
 193 static struct token *stringify(struct token *token, struct token *arg)
 194 {
 195         const char *s = show_token_sequence(arg);
 196         int size = strlen(s)+1;
 197         struct token *newtoken = alloc_token(token);
 198         struct string *string = __alloc_string(size);
 199
 200         newtoken->newline = token->newline;
 201         memcpy(string->data, s, size);
 202         string->length = size;
 203         newtoken->type = TOKEN_STRING;
 204         newtoken->string = string;
 205         newtoken->next = &eof_token_entry;
 206         return newtoken;
 207 }
 208
 209 static int arg_number(struct token *arglist, struct ident *ident)
 210 {
 211         int nr = 0;
 212
 213         while (!eof_token(arglist)) {
 214                 if (arglist->ident == ident)
 215                         return nr;
 216                 nr++;
 217                 arglist = arglist->next;
 218         }
 219         return -1;
 220 }
 221
 222 static struct token empty_arg_token = { .type = TOKEN_EOF };
 223
 224 static struct token *expand_one_arg(struct token *head, struct token *token,
 225                 struct token *arglist, struct token *arguments)
 226 {
 227         int nr = arg_number(arglist, token->ident);
 228         struct token *orig_head = head;
 229
 230         if (nr >= 0) {
 231                 struct token *arg = get_argument(nr, arguments);
 232                 struct token *last = token->next;
 233                 token->next = &eof_token_entry;
 234
 235                 /*
 236                  * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
 237                  * then the 'x' goes away too.
 238                  */
 239                 if (match_op(head, SPECIAL_HASHHASH) && eof_token(arg)) {
 240                         arg = &empty_arg_token;
 241                         empty_arg_token.next = &eof_token_entry;
 242                 }
 243
 244                 head = replace(token, head, arg);
 245                 if (!match_op(orig_head, SPECIAL_HASHHASH) && !match_op(last, SPECIAL_HASHHASH) && !match_op(orig_head, '#'))
 246                         head = expand_list(orig_head);
 247                 head->next = last;
 248                 return head;
 249         }
 250         return token;
 251 }
 252
 253 static void expand_arguments(struct token *token, struct token *head,
 254         struct token *arguments, struct token *arglist)
 255 {
 256         for (;;) {
 257                 struct token *next = head->next;
 258
 259                 /* Did we hit the end of the current expansion? */
 260                 if (eof_token(next))
 261                         break;
 262
 263                 if (match_op(next, '#')) {
 264                         struct token *nextnext = next->next;
 265                         int nr = arg_number(arglist, nextnext->ident);
 266                         if (nextnext != head && nr >= 0 && nextnext->type == TOKEN_IDENT) {
 267                                 struct token * last = nextnext->next;
 268                                 struct token *newtoken = stringify(nextnext, get_argument(nr, arguments));
 269                                 replace(nextnext, head, newtoken);
 270                                 continue;
 271                         }
 272                         warn(next, "'#' operation is not followed by argument name");
 273                 }
 274
 275                 if (next->type == TOKEN_IDENT)
 276                         next = expand_one_arg(head, next, arglist, arguments);
 277
 278                 head = next;
 279         }
 280 }
 281
 282 /*
 283  * Possibly valid combinations:
 284  *  - anything + 'empty_arg_token' is empty.
 285  *  - ident + ident - combine (==ident)
 286  *  - ident + number - combine (==ident)
 287  *  - number + number - combine (==number)
 288  *  - number + ident - combine (==number)
 289  *  - string + string - leave as is, C will combine them anyway
 290  * others cause an error and leave the two tokens as separate tokens.
 291  */
 292 static struct token *hashhash(struct token *head, struct token *first, struct token *second)
 293 {
 294         static char buffer[512], *p;
 295         struct token *newtoken;
 296         static const char *src;
 297         int len;
 298
 299         first->next = second;
 300
 301         /*
 302          * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
 303          * then the 'x' goes away too.
 304          *
 305          * See expand_one_arg.
 306          */
 307         if (second->type == TOKEN_EOF) {
 308                 head->next = second->next;
 309                 return head;
 310         }
 311
 312         p = buffer;
 313         switch (first->type) {
 314         case TOKEN_INTEGER:
 315                 len = strlen(first->integer);
 316                 src = first->integer;
 317                 break;
 318         case TOKEN_IDENT:
 319                 len = first->ident->len;
 320                 src = first->ident->name;
 321                 break;
 322         default:
 323                 return second;
 324         }
 325         memcpy(p, src, len);
 326         p += len;
 327
 328         switch (second->type) {
 329         case TOKEN_INTEGER:
 330                 len = strlen(second->integer);
 331                 src = second->integer;
 332                 break;
 333         case TOKEN_IDENT:
 334                 len = second->ident->len;
 335                 src = second->ident->name;
 336                 break;
 337         default:
 338                 return second;
 339         }
 340         memcpy(p, src, len);
 341         p += len;
 342         *p++ = 0;
 343
 344         newtoken = alloc_token(first);
 345         head->next = newtoken;
 346         newtoken->type = first->type;
 347         switch (newtoken->type) {
 348         case TOKEN_IDENT:
 349                 newtoken->ident = built_in_ident(buffer);
 350                 break;
 351         case TOKEN_INTEGER:
 352                 newtoken->integer = __alloc_bytes(p - buffer);
 353                 memcpy(newtoken->integer, buffer, p - buffer);
 354                 break;
 355         }
 356         return newtoken;
 357 }
 358
 359 static void retokenize(struct token *head)
 360 {
 361         struct token * next = head->next;
 362         struct token * nextnext = next->next;
 363         struct token * nextnextnext = nextnext->next;
 364
 365         if (eof_token(next) || eof_token(nextnext))
 366                 return;
 367
 368         for (;;) {
 369                 if (eof_token(nextnextnext))
 370                         break;
 371
 372                 if (match_op(nextnext, SPECIAL_HASHHASH)) {
 373                         struct token *newtoken = hashhash(head, next, nextnextnext);
 374
 375                         next = newtoken;
 376                         nextnext = nextnextnext->next;
 377                         nextnextnext = nextnext->next;
 378
 379                         newtoken->next = nextnext;
 380                         if (!eof_token(nextnext))
 381                                 continue;
 382                         break;
 383                 }
 384
 385                 head = next;
 386                 next = nextnext;
 387                 nextnext = nextnext->next;
 388                 nextnextnext = nextnextnext->next;
 389         }
 390 }
 391
 392 static struct token *expand(struct token *head, struct symbol *sym)
 393 {
 394         struct token *arguments, *token, *last;
 395
 396         sym->busy++;
 397         token = head->next;
 398         last = token->next;
 399
 400         arguments = NULL;
 401         if (sym->arglist) {
 402                 arguments = last->next;
 403                 last = find_argument_end(last);
 404         }
 405         token->next = &eof_token_entry;
 406
 407         /* Replace the token with the token expansion */
 408         replace(token, head, sym->expansion);
 409
 410         /* Then, replace all the arguments with their expansions */
 411         if (arguments)
 412                 expand_arguments(token, head, arguments, sym->arglist);
 413
 414         /* Re-tokenize the sequence if any ## token exists.. */
 415         retokenize(head);
 416
 417         /* Finally, expand the expansion itself .. */
 418         head = expand_list(head);
 419
 420         /* Put the rest of the stuff in place again */
 421         head->next = last;
 422         sym->busy--;
 423         return head;
 424 }
 425
 426 static const char *token_name_sequence(struct token *token, int endop, struct token *start)
 427 {
 428         struct token *last;
 429         static char buffer[256];
 430         char *ptr = buffer;
 431
 432         last = token;
 433         while (!eof_token(token) && !match_op(token, endop)) {
 434                 int len;
 435                 const char *val = token->string->data;
 436                 if (token->type != TOKEN_STRING)
 437                         val = show_token(token);
 438                 len = strlen(val);
 439                 memcpy(ptr, val, len);
 440                 ptr += len;
 441                 token = token->next;
 442         }
 443         *ptr = 0;
 444         if (endop && !match_op(token, endop))
 445                 warn(start, "expected '>' at end of filename");
 446         return buffer;
 447 }
 448
 449 static void do_include(struct token *head, struct token *token, const char *filename)
 450 {
 451         int endlen = strlen(filename) + 1;
 452         char **pptr = includepath, *path;
 453
 454         while ((path = *pptr++) != NULL) {
 455                 int fd, len = strlen(path);
 456                 static char fullname[PATH_MAX];
 457
 458                 memcpy(fullname, path, len);
 459                 memcpy(fullname+len, filename, endlen);
 460                 fd = open(fullname, O_RDONLY);
 461                 if (fd >= 0) {
 462                         char * streamname = __alloc_bytes(len + endlen);
 463                         memcpy(streamname, fullname, len + endlen);
 464                         head->next = tokenize(streamname, fd, head->next);
 465                         close(fd);
 466                         return;
 467                 }
 468         }
 469         warn(token, "unable to open '%s'", filename);
 470 }
 471
 472 static int handle_include(struct stream *stream, struct token *head, struct token *token)
 473 {
 474         const char *filename;
 475         struct token *next;
 476         int expect;
 477
 478         if (stream->constant == -1)
 479                 stream->constant = 0;
 480         if (false_nesting)
 481                 return 1;
 482         next = token->next;
 483         expect = '>';
 484         if (!match_op(next, '<')) {
 485                 expand_list(token);
 486                 expect = 0;
 487                 next = token;
 488         }
 489         token = next->next;
 490         filename = token_name_sequence(token, expect, token);
 491         do_include(head, token, filename);
 492         return 1;
 493 }
 494
 495 static int token_list_different(struct token *list1, struct token *list2)
 496 {
 497         for (;;) {
 498                 if (list1 == list2)
 499                         return 0;
 500                 if (!list1 || !list2)
 501                         return 1;
 502                 if (list1->type != list2->type)
 503                         return 1;
 504                 list1 = list1->next;
 505                 list2 = list2->next;
 506         }
 507 }
 508
 509
 510 static int handle_define(struct stream *stream, struct token *head, struct token *token)
 511 {
 512         struct token *arglist, *expansion;
 513         struct token *left = token->next;
 514         struct symbol *sym;
 515         struct ident *name;
 516
 517         if (left->type != TOKEN_IDENT) {
 518                 warn(head, "expected identifier to 'define'");
 519                 return 0;
 520         }
 521         if (false_nesting)
 522                 return 1;
 523         name = left->ident;
 524
 525         arglist = NULL;
 526         expansion = left->next;
 527         if (!expansion->whitespace && match_op(expansion, '(')) {
 528                 arglist = expansion;
 529                 while (!eof_token(expansion)) {
 530                         struct token *next = expansion->next;
 531                         if (match_op(next, ')')) {
 532                                 // Terminate the arglist
 533                                 expansion->next = &eof_token_entry;
 534                                 expansion = next->next;
 535                                 break;
 536                         }
 537                         if (match_op(next, ','))
 538                                 expansion->next = next->next;
 539                         expansion = next;
 540                 }
 541                 arglist = arglist->next;
 542         }
 543
 544         sym = lookup_symbol(name, NS_PREPROCESSOR);
 545         if (sym) {
 546                 if (token_list_different(sym->expansion, expansion) ||
 547                     token_list_different(sym->arglist, arglist)) {
 548                         warn(left, "preprocessor token redefined");
 549                         warn(sym->token, "this was the original definition");
 550                 }
 551                 return 1;
 552         }
 553         sym = alloc_symbol(left, SYM_NONE);
 554         bind_symbol(sym, name, NS_PREPROCESSOR);
 555
 556         sym->expansion = expansion;
 557         sym->arglist = arglist;
 558         return 1;
 559 }
 560
 561 static int handle_undef(struct stream *stream, struct token *head, struct token *token)
 562 {
 563         struct token *left = token->next;
 564         struct symbol **sym;
 565
 566         if (left->type != TOKEN_IDENT) {
 567                 warn(head, "expected identifier to 'undef'");
 568                 return 0;
 569         }
 570         if (false_nesting)
 571                 return 1;
 572         sym = &left->ident->symbols;
 573         while (*sym) {
 574                 struct symbol *t = *sym;
 575                 if (t->namespace == NS_PREPROCESSOR) {
 576                         *sym = t->next_id;
 577                         return 1;
 578                 }
 579                 sym = &t->next_id;
 580         }
 581         return 1;
 582 }
 583
 584 static int preprocessor_if(struct token *token, int true)
 585 {
 586         if (if_nesting == 0)
 587                 unmatched_if = token;
 588         elif_ignore[if_nesting] = false_nesting || true;
 589         if (false_nesting || !true) {
 590                 false_nesting++;
 591                 return 1;
 592         }
 593         true_nesting++;
 594         return 1;
 595 }
 596
 597 static int token_defined(struct token *token)
 598 {
 599         if (token->type == TOKEN_IDENT)
 600                 return lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL;
 601
 602         warn(token, "expected identifier for #if[n]def");
 603         return 0;
 604 }
 605
 606 static int handle_ifdef(struct stream *stream, struct token *head, struct token *token)
 607 {
 608         return preprocessor_if(token, token_defined(token->next));
 609 }
 610
 611 static int handle_ifndef(struct stream *stream, struct token *head, struct token *token)
 612 {
 613         struct token *next = token->next;
 614         if (stream->constant == -1) {
 615                 int newconstant = 0;
 616                 if (next->type == TOKEN_IDENT) {
 617                         if (!stream->protect || stream->protect == next->ident) {
 618                                 newconstant = -2;
 619                                 stream->protect = next->ident;
 620                                 stream->nesting = if_nesting+1;
 621                         }
 622                 }
 623                 stream->constant = newconstant;
 624         }
 625         return preprocessor_if(token, !token_defined(next));
 626 }
 627
 628 static unsigned long long get_int_value(const char *str)
 629 {
 630         unsigned long long value = 0;
 631         unsigned int base = 10, digit;
 632
 633         switch (str[0]) {
 634         case 'x':
 635                 base = 18;      // the -= 2 for the octal case will
 636                 str++;          // skip the 'x'
 637         /* fallthrough */
 638         case 'o':
 639                 str++;          // skip the 'o' or 'x/X'
 640                 base -= 2;      // the fall-through will make this 8
 641         }
 642         while ((digit = hexval(*str)) < base) {
 643                 value = value * base + digit;
 644                 str++;
 645         }
 646         return value;
 647 }
 648
 649 static long long primary_value(struct token *token)
 650 {
 651         switch (token->type) {
 652         case TOKEN_INTEGER:
 653                 return get_int_value(token->integer);
 654         }
 655         error(token, "bad constant expression");
 656         return 0;
 657 }
 658
 659 long long get_expression_value(struct expression *expr)
 660 {
 661         long long left, middle, right;
 662
 663         switch (expr->type) {
 664         case EXPR_CONSTANT:
 665                 return primary_value(expr->token);
 666         case EXPR_SYMBOL: {
 667                 struct symbol *sym = expr->symbol;
 668                 if (!sym || !sym->ctype.base_type || sym->ctype.base_type->type != SYM_ENUM) {
 669                         warn(expr->token, "undefined identifier in constant expression");
 670                         return 0;
 671                 }
 672                 return sym->value;
 673         }
 674
 675 #define OP(x,y) case x: return left y right;
 676         case EXPR_BINOP:
 677                 left = get_expression_value(expr->left);
 678                 if (!left && expr->op == SPECIAL_LOGICAL_AND)
 679                         return 0;
 680                 if (left && expr->op == SPECIAL_LOGICAL_OR)
 681                         return 1;
 682                 right = get_expression_value(expr->right);
 683                 switch (expr->op) {
 684                         OP('+',+); OP('-',-); OP('*',*); OP('/',/);
 685                         OP('%',%); OP('<',<); OP('>',>);
 686                         OP('&',&);OP('|',|);OP('^',^);
 687                         OP(SPECIAL_EQUAL,==); OP(SPECIAL_NOTEQUAL,!=);
 688                         OP(SPECIAL_LTE,<=); OP(SPECIAL_LEFTSHIFT,<<);
 689                         OP(SPECIAL_RIGHTSHIFT,>>); OP(SPECIAL_GTE,>=);
 690                         OP(SPECIAL_LOGICAL_AND,&&);OP(SPECIAL_LOGICAL_OR,||);
 691                 }
 692                 break;
 693
 694 #undef OP
 695 #define OP(x,y) case x: return y left;
 696         case EXPR_PREOP:
 697                 left = get_expression_value(expr->unop);
 698                 switch (expr->op) {
 699                         OP('+', +); OP('-', -); OP('!', !); OP('~', ~); OP('(', );
 700                 }
 701                 break;
 702
 703         case EXPR_CONDITIONAL:
 704                 left = get_expression_value(expr->conditional);
 705                 if (!expr->cond_true)
 706                         middle = left;
 707                 else
 708                         middle = get_expression_value(expr->cond_true);
 709                 right = get_expression_value(expr->cond_false);
 710                 return left ? middle : right;
 711         }
 712         error(expr->token, "bad constant expression");
 713         return 0;
 714 }
 715
 716 extern struct token *conditional_expression(struct token *token, struct expression **tree);
 717
 718 static int expression_value(struct token *head)
 719 {
 720         struct expression *expr;
 721         struct token *token;
 722         long long value;
 723
 724         expand_defined(head);
 725         expand_list(head);
 726         token = conditional_expression(head->next, &expr);
 727         if (!eof_token(token))
 728                 warn(token, "garbage at end: %s", show_token_sequence(token));
 729         value = get_expression_value(expr);
 730         return value != 0;
 731 }
 732
 733 static int handle_if(struct stream *stream, struct token *head, struct token *token)
 734 {
 735         int value = 0;
 736         if (!false_nesting)
 737                 value = expression_value(token);
 738         return preprocessor_if(token, value);
 739 }
 740
 741 static int handle_elif(struct stream * stream, struct token *head, struct token *token)
 742 {
 743         if (stream->nesting == if_nesting)
 744                 stream->constant = 0;
 745         if (false_nesting) {
 746                 /* If this whole if-thing is if'ed out, an elif cannot help */
 747                 if (elif_ignore[if_nesting-1])
 748                         return 1;
 749                 if (expression_value(token)) {
 750                         false_nesting--;
 751                         true_nesting++;
 752                         elif_ignore[if_nesting-1] = 1;
 753                 }
 754                 return 1;
 755         }
 756         if (true_nesting) {
 757                 false_nesting = 1;
 758                 true_nesting--;
 759                 return 1;
 760         }
 761         warn(token, "unmatched '#elif'");
 762         return 1;
 763 }
 764
 765 static int handle_else(struct stream *stream, struct token *head, struct token *token)
 766 {
 767         if (stream->nesting == if_nesting)
 768                 stream->constant = 0;
 769         if (false_nesting) {
 770                 /* If this whole if-thing is if'ed out, an else cannot help */
 771                 if (elif_ignore[if_nesting-1])
 772                         return 1;
 773                 false_nesting--;
 774                 true_nesting++;
 775                 elif_ignore[if_nesting-1] = 1;
 776                 return 1;
 777         }
 778         if (true_nesting) {
 779                 true_nesting--;
 780                 false_nesting = 1;
 781                 return 1;
 782         }
 783         warn(token, "unmatched #else");
 784         return 1;
 785 }
 786
 787 static int handle_endif(struct stream *stream, struct token *head, struct token *token)
 788 {
 789         if (stream->constant == -2 && stream->nesting == if_nesting)
 790                 stream->constant = -1;
 791
 792         if (false_nesting) {
 793                 false_nesting--;
 794                 return 1;
 795         }
 796         if (true_nesting) {
 797                 true_nesting--;
 798                 return 1;
 799         }
 800         warn(token, "unmatched #endif");
 801         return 1;
 802 }
 803
 804 static const char *show_token_sequence(struct token *token)
 805 {
 806         static char buffer[256];
 807         char *ptr = buffer;
 808         int whitespace = 0;
 809
 810         if (!token)
 811                 return "<none>";
 812         while (!eof_token(token)) {
 813                 const char *val = show_token(token);
 814                 int len = strlen(val);
 815                 if (whitespace)
 816                         *ptr++ = ' ';
 817                 memcpy(ptr, val, len);
 818                 ptr += len;
 819                 token = token->next;
 820                 whitespace = token->whitespace;
 821         }
 822         *ptr++ = 0;
 823         *ptr = 0;
 824         return buffer;
 825 }
 826
 827 static int handle_warning(struct stream *stream, struct token *head, struct token *token)
 828 {
 829         if (false_nesting)
 830                 return 1;
 831         warn(token, "%s", show_token_sequence(token->next));
 832         return 1;
 833 }
 834
 835 static int handle_error(struct stream *stream, struct token *head, struct token *token)
 836 {
 837         if (false_nesting)
 838                 return 1;
 839         error(token, "%s", show_token_sequence(token->next));
 840         return 1;
 841 }
 842
 843 static int handle_preprocessor_command(struct stream *stream, struct token *head, struct ident *ident, struct token *token)
 844 {
 845         int i;
 846         static struct {
 847                 const char *name;
 848                 int (*handler)(struct stream *, struct token *, struct token *);
 849         } handlers[] = {
 850                 { "define",     handle_define },
 851                 { "undef",      handle_undef },
 852                 { "ifdef",      handle_ifdef },
 853                 { "ifndef",     handle_ifndef },
 854                 { "else",       handle_else },
 855                 { "endif",      handle_endif },
 856                 { "if",         handle_if },
 857                 { "elif",       handle_elif },
 858                 { "warning",    handle_warning },
 859                 { "error",      handle_error },
 860                 { "include",    handle_include },
 861         };
 862
 863         for (i = 0; i < (sizeof (handlers) / sizeof (handlers[0])); i++) {
 864                 if (match_string_ident(ident, handlers[i].name))
 865                         return handlers[i].handler(stream, head, token);
 866         }
 867         return 0;
 868 }
 869
 870 static void handle_preprocessor_line(struct stream *stream, struct token * head, struct token *token)
 871 {
 872         if (!token)
 873                 return;
 874
 875         if (token->type == TOKEN_IDENT)
 876                 if (handle_preprocessor_command(stream, head, token->ident, token))
 877                         return;
 878         warn(token, "unrecognized preprocessor line '%s'", show_token_sequence(token));
 879 }
 880
 881 static void preprocessor_line(struct stream *stream, struct token * head)
 882 {
 883         struct token *start = head->next, *next;
 884         struct token **tp = &start->next;
 885
 886         for (;;) {
 887                 next = *tp;
 888                 if (next->newline)
 889                         break;
 890                 tp = &next->next;
 891         }
 892         head->next = next;
 893         *tp = &eof_token_entry;
 894         handle_preprocessor_line(stream, head, start->next);
 895 }
 896
 897 static void do_preprocess(struct token *head)
 898 {
 899         do {
 900                 struct token *next = head->next;
 901                 struct stream *stream = input_streams + next->stream;
 902
 903                 if (next->newline && match_op(next, '#')) {
 904                         preprocessor_line(stream, head);
 905                         continue;
 906                 }
 907
 908                 if (false_nesting) {
 909                         head->next = next->next;
 910                         continue;
 911                 }
 912
 913                 switch (next->type) {
 914                 case TOKEN_STREAMEND:
 915                         if (stream->constant == -1 && stream->protect) {
 916                                 stream->constant = 1;
 917                         }
 918                         /* fallthrough */
 919                 case TOKEN_STREAMBEGIN:
 920                         head->next = next->next;
 921                         continue;
 922
 923                 case TOKEN_IDENT:
 924                         next = expand_one_symbol(head, next);
 925                         /* fallthrough */
 926                 default:
 927                         /*
 928                          * Any token expansion (even if it ended up being an
 929                          * empty expansion) in this stream implies it can't
 930                          * be constant.
 931                          */
 932                         stream->constant = 0;
 933                 }
 934
 935                 head = next;
 936         } while (!eof_token(head));
 937 }
 938
 939 struct token * preprocess(struct token *token)
 940 {
 941         struct token header = { 0, };
 942
 943         header.next = token;
 944         do_preprocess(&header);
 945         if (if_nesting)
 946                 warn(unmatched_if, "unmatched preprocessor conditional");
 947
 948         // Drop all expressions from pre-processing, they're not used any more.
 949         clear_expression_alloc();
 950
 951         return header.next;
 952 }