pre-process.c

   1 /*
   2  * Do C preprocessing, based on a token list gathered by
   3  * the tokenizer.
   4  *
   5  * This may not be the smartest preprocessor on the planet.
   6  *
   7  * Copyright (C) 2003 Transmeta Corp.
   8  *
   9  *  Licensed under the Open Software License version 1.1
  10  */
  11 #include <stdio.h>
  12 #include <stdlib.h>
  13 #include <stdarg.h>
  14 #include <stddef.h>
  15 #include <string.h>
  16 #include <ctype.h>
  17 #include <unistd.h>
  18 #include <fcntl.h>
  19 #include <limits.h>
  20
  21 #include "lib.h"
  22 #include "parse.h"
  23 #include "token.h"
  24 #include "symbol.h"
  25 #include "expression.h"
  26
  27 int preprocessing = 0;
  28
  29 #define MAXNEST (16)
  30 static int true_nesting = 0;
  31 static int false_nesting = 0;
  32 static struct token *unmatched_if = NULL;
  33 static int elif_ignore[MAXNEST];
  34 #define if_nesting (true_nesting + false_nesting)
  35
  36 #define INCLUDEPATHS 32
  37 const char *includepath[INCLUDEPATHS+1] = {
  38         NULL
  39 };
  40
  41 const char *sys_includepath[] = {
  42         "/usr/include",
  43         "/usr/local/include",
  44         NULL,
  45 };
  46
  47 const char *gcc_includepath[] = {
  48         "/usr/lib/gcc-lib/i386-redhat-linux/3.2.1/include",
  49         "/usr/lib/gcc-lib/i386-redhat-linux/3.2.2/include",
  50         NULL
  51 };
  52
  53
  54 /*
  55  * This is stupid - the tokenizer already guarantees unique
  56  * identifiers, so we should just compare identifier pointers
  57  */
  58 int match_string_ident(struct ident *ident, const char *str)
  59 {
  60         return !str[ident->len] && !memcmp(str, ident->name, ident->len);
  61 }
  62
  63 static struct token *alloc_token(struct position *pos)
  64 {
  65         struct token *token = __alloc_token(0);
  66
  67         token->pos.stream = pos->stream;
  68         token->pos.line = pos->line;
  69         token->pos.pos = pos->pos;
  70         token->pos.whitespace = 1;
  71         return token;
  72 }
  73
  74 static const char *show_token_sequence(struct token *token);
  75
  76 /* Head is one-before-list, and last is one-past-list */
  77 static struct token *for_each_ident(struct token *parent, struct token *head, struct token *(*action)(struct token *parent, struct token *head, struct token *))
  78 {
  79         for (;;) {
  80                 struct token *next = head->next;
  81
  82                 /* Did we hit the end of the current expansion? */
  83                 if (eof_token(next))
  84                         break;
  85
  86                 if (token_type(next) == TOKEN_IDENT)
  87                         next = action(parent, head, next);
  88
  89                 head = next;
  90         }
  91         return head;
  92 }
  93
  94 static struct token *is_defined(struct token *head, struct token *token, struct token *next)
  95 {
  96         char *string[] = { "0", "1" };
  97         char *defined = string[lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL];
  98         struct token *newtoken = alloc_token(&token->pos);
  99
 100         token_type(newtoken) = TOKEN_INTEGER;
 101         newtoken->integer = defined;
 102         newtoken->next = next;
 103         head->next = newtoken;
 104         return next;
 105 }
 106
 107
 108 struct token *defined_one_symbol(struct token *head, struct token *next)
 109 {
 110         struct token *token = next->next;
 111         struct token *past = token->next;
 112
 113         if (match_op(token, '(')) {
 114                 token = past;
 115                 past = token->next;
 116                 if (!match_op(past, ')'))
 117                         return next;
 118                 past = past->next;
 119         }
 120         if (token_type(token) == TOKEN_IDENT)
 121                 return is_defined(head, token, past);
 122         return next;
 123 }
 124
 125 /* Expand symbol 'sym' between 'head->next' and 'head->next->next' */
 126 static struct token *expand(struct token *, struct token *, struct symbol *);
 127
 128 static void replace_with_string(struct token *token, const char *str)
 129 {
 130         int size = strlen(str) + 1;
 131         struct string *s = __alloc_string(size);
 132
 133         s->length = size;
 134         memcpy(s->data, str, size);
 135         token_type(token) = TOKEN_STRING;
 136         token->string = s;
 137 }
 138
 139 static void replace_with_integer(struct token *token, unsigned int val)
 140 {
 141         char *buf = __alloc_bytes(10);
 142         sprintf(buf, "%d", val);
 143         token_type(token) = TOKEN_INTEGER;
 144         token->integer = buf;
 145 }
 146
 147 struct token *expand_one_symbol(struct token *parent, struct token *head, struct token *token)
 148 {
 149         struct symbol *sym;
 150         struct token *x;
 151
 152         /* Avoid recursive expansion */
 153         x = token;
 154         while ((x = x->parent) != NULL) {
 155                 if (x->ident == parent->ident)
 156                         return token;
 157         }
 158
 159         sym = lookup_symbol(token->ident, NS_PREPROCESSOR);
 160         if (sym) {
 161                 if (sym->arglist && !match_op(token->next, '('))
 162                         return token;
 163                 return expand(parent, head, sym);
 164         }
 165         if (!memcmp(token->ident->name, "__LINE__", 9)) {
 166                 replace_with_integer(token, token->pos.line);
 167         } else if (!memcmp(token->ident->name, "__FILE__", 9)) {
 168                 replace_with_string(token, (input_streams + token->pos.stream)->name);
 169         } else if (!memcmp(token->ident->name, "defined", 8)) {
 170                 return defined_one_symbol(head, token);
 171         }
 172         return token;
 173 }
 174
 175 static struct token *expand_list(struct token *parent, struct token *head)
 176 {
 177         return for_each_ident(parent, head, expand_one_symbol);
 178 }
 179
 180 static struct token *find_argument_end(struct token *start)
 181 {
 182         int nesting = 0;
 183
 184         while (!eof_token(start)) {
 185                 struct token *next = start->next;
 186                 if (match_op(next, '('))
 187                         nesting++;
 188                 else if (match_op(next, ')')) {
 189                         if (--nesting < 0) {
 190                                 start->next = &eof_token_entry;
 191                                 return next->next;
 192                         }
 193                 } else if (!nesting && match_op(next, ','))
 194                         next->special = SPECIAL_ARG_SEPARATOR;
 195                 start = next;
 196         }
 197         return start;
 198 }
 199
 200 static struct token *dup_token(struct token *token, struct position *pos, int newline)
 201 {
 202         struct token *alloc = alloc_token(pos);
 203         token_type(alloc) = token_type(token);
 204         alloc->pos.line = pos->line;
 205         alloc->pos.newline = newline;
 206         alloc->integer = token->integer;
 207         return alloc;
 208 }
 209
 210 static void insert(struct token *token, struct token *prev)
 211 {
 212         token->next = prev->next;
 213         prev->next = token;
 214 }
 215
 216 static struct token * replace(struct token *parent, struct token *token, struct token *prev, struct token *list)
 217 {
 218         int newline = token->pos.newline;
 219
 220         prev->next = token->next;
 221         while (!eof_token(list) && !match_op(list, SPECIAL_ARG_SEPARATOR)) {
 222                 struct token *newtok = dup_token(list, &token->pos, newline);
 223                 newtok->parent = parent;
 224                 newline = 0;
 225                 insert(newtok, prev);
 226                 prev = newtok;
 227                 list = list->next;
 228         }
 229         return prev;
 230 }
 231
 232 static struct token *get_argument(int nr, struct token *args)
 233 {
 234         if (!nr)
 235                 return args;
 236         while (!eof_token(args)) {
 237                 if (match_op(args, SPECIAL_ARG_SEPARATOR))
 238                         if (!--nr)
 239                                 return args->next;
 240                 args = args->next;
 241         }
 242
 243         return args;
 244 }
 245
 246 static struct token *stringify(struct token *token, struct token *arg)
 247 {
 248         const char *s = show_token_sequence(arg);
 249         int size = strlen(s)+1;
 250         struct token *newtoken = alloc_token(&token->pos);
 251         struct string *string = __alloc_string(size);
 252
 253         newtoken->pos.newline = token->pos.newline;
 254         memcpy(string->data, s, size);
 255         string->length = size;
 256         token_type(newtoken) = TOKEN_STRING;
 257         newtoken->string = string;
 258         newtoken->next = &eof_token_entry;
 259         return newtoken;
 260 }
 261
 262 static int arg_number(struct token *arglist, struct ident *ident)
 263 {
 264         int nr = 0;
 265
 266         while (!eof_token(arglist)) {
 267                 if (arglist->ident == ident)
 268                         return nr;
 269                 nr++;
 270                 arglist = arglist->next;
 271         }
 272         return -1;
 273 }
 274
 275 static struct token empty_arg_token = { .pos = { .type = TOKEN_EOF } };
 276
 277 static struct token *expand_one_arg(struct token *parent, struct token *head, struct token *token,
 278                 struct token *arglist, struct token *arguments)
 279 {
 280         int nr = arg_number(arglist, token->ident);
 281         struct token *orig_head = head;
 282
 283         if (nr >= 0) {
 284                 struct token *arg = get_argument(nr, arguments);
 285                 struct token *last = token->next;
 286                 token->next = &eof_token_entry;
 287
 288                 /*
 289                  * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
 290                  * then the 'x' goes away too.
 291                  */
 292                 if (match_op(head, SPECIAL_HASHHASH) && eof_token(arg)) {
 293                         arg = &empty_arg_token;
 294                         empty_arg_token.next = &eof_token_entry;
 295                 }
 296
 297                 head = replace(parent, token, head, arg);
 298                 if (!match_op(orig_head, SPECIAL_HASHHASH) && !match_op(last, SPECIAL_HASHHASH) && !match_op(orig_head, '#'))
 299                         head = expand_list(parent, orig_head);
 300                 head->next = last;
 301                 return head;
 302         }
 303         return token;
 304 }
 305
 306 static void expand_arguments(struct token *parent,
 307         struct token *token, struct token *head,
 308         struct token *arguments, struct token *arglist)
 309 {
 310         for (;;) {
 311                 struct token *next = head->next;
 312
 313                 /* Did we hit the end of the current expansion? */
 314                 if (eof_token(next))
 315                         break;
 316
 317                 if (match_op(next, '#')) {
 318                         struct token *nextnext = next->next;
 319                         int nr = arg_number(arglist, nextnext->ident);
 320                         if (nextnext != head && nr >= 0 && token_type(nextnext) == TOKEN_IDENT) {
 321                                 struct token *newtoken = stringify(nextnext, get_argument(nr, arguments));
 322                                 replace(parent, nextnext, head, newtoken);
 323                                 continue;
 324                         }
 325                         warn(next->pos, "'#' operation is not followed by argument name");
 326                 }
 327
 328                 if (token_type(next) == TOKEN_IDENT)
 329                         next = expand_one_arg(parent, head, next, arglist, arguments);
 330
 331                 head = next;
 332         }
 333 }
 334
 335 /*
 336  * Possibly valid combinations:
 337  *  - anything + 'empty_arg_token' is empty.
 338  *  - ident + ident - combine (==ident)
 339  *  - ident + number - combine (==ident)
 340  *  - number + number - combine (==number)
 341  *  - number + ident - combine (==number)
 342  *  - string + string - leave as is, C will combine them anyway
 343  * others cause an error and leave the two tokens as separate tokens.
 344  */
 345 static struct token *hashhash(struct token *head, struct token *first, struct token *second)
 346 {
 347         static char buffer[512], *p;
 348         struct token *newtoken;
 349         static const char *src;
 350         int len;
 351
 352         first->next = second;
 353
 354         /*
 355          * Special case for gcc 'x ## arg' semantics: if 'arg' is empty
 356          * then the 'x' goes away too.
 357          *
 358          * See expand_one_arg.
 359          */
 360         if (token_type(second) == TOKEN_EOF) {
 361                 head->next = second->next;
 362                 return head;
 363         }
 364
 365         p = buffer;
 366         switch (token_type(first)) {
 367         case TOKEN_INTEGER:
 368                 len = strlen(first->integer);
 369                 src = first->integer;
 370                 break;
 371         case TOKEN_IDENT:
 372                 len = first->ident->len;
 373                 src = first->ident->name;
 374                 break;
 375         default:
 376                 return second;
 377         }
 378         memcpy(p, src, len);
 379         p += len;
 380
 381         switch (token_type(second)) {
 382         case TOKEN_INTEGER:
 383                 len = strlen(second->integer);
 384                 src = second->integer;
 385                 break;
 386         case TOKEN_IDENT:
 387                 len = second->ident->len;
 388                 src = second->ident->name;
 389                 break;
 390         default:
 391                 return second;
 392         }
 393         memcpy(p, src, len);
 394         p += len;
 395         *p++ = 0;
 396
 397         newtoken = alloc_token(&first->pos);
 398         head->next = newtoken;
 399         token_type(newtoken) = token_type(first);
 400         switch (token_type(newtoken)) {
 401         case TOKEN_IDENT:
 402                 newtoken->ident = built_in_ident(buffer);
 403                 break;
 404         case TOKEN_INTEGER:
 405                 newtoken->integer = __alloc_bytes(p - buffer);
 406                 memcpy(newtoken->integer, buffer, p - buffer);
 407                 break;
 408         }
 409         return newtoken;
 410 }
 411
 412 static void retokenize(struct token *head)
 413 {
 414         struct token * next = head->next;
 415         struct token * nextnext = next->next;
 416         struct token * nextnextnext = nextnext->next;
 417
 418         if (eof_token(next) || eof_token(nextnext))
 419                 return;
 420
 421         for (;;) {
 422                 if (eof_token(nextnextnext))
 423                         break;
 424
 425                 if (match_op(nextnext, SPECIAL_HASHHASH)) {
 426                         struct token *newtoken = hashhash(head, next, nextnextnext);
 427
 428                         next = newtoken;
 429                         nextnext = nextnextnext->next;
 430                         nextnextnext = nextnext->next;
 431
 432                         newtoken->next = nextnext;
 433                         if (!eof_token(nextnext))
 434                                 continue;
 435                         break;
 436                 }
 437
 438                 head = next;
 439                 next = nextnext;
 440                 nextnext = nextnext->next;
 441                 nextnextnext = nextnextnext->next;
 442         }
 443 }
 444
 445 static struct token *expand(struct token *parent, struct token *head, struct symbol *sym)
 446 {
 447         struct token *arguments, *token, *last;
 448
 449         token = head->next;
 450         last = token->next;
 451
 452         arguments = NULL;
 453         if (sym->arglist) {
 454                 arguments = last->next;
 455                 last = find_argument_end(last);
 456         }
 457         token->next = &eof_token_entry;
 458
 459         /* Replace the token with the token expansion */
 460         replace(parent, token, head, sym->expansion);
 461
 462         /* Then, replace all the arguments with their expansions */
 463         if (arguments)
 464                 expand_arguments(parent, token, head, arguments, sym->arglist);
 465
 466         /* Re-tokenize the sequence if any ## token exists.. */
 467         retokenize(head);
 468
 469         token = head;
 470         while (!eof_token(token->next))
 471                 token = token->next;
 472         token->next = last;
 473         return head;
 474 }
 475
 476 static const char *token_name_sequence(struct token *token, int endop, struct token *start)
 477 {
 478         struct token *last;
 479         static char buffer[256];
 480         char *ptr = buffer;
 481
 482         last = token;
 483         while (!eof_token(token) && !match_op(token, endop)) {
 484                 int len;
 485                 const char *val = token->string->data;
 486                 if (token_type(token) != TOKEN_STRING)
 487                         val = show_token(token);
 488                 len = strlen(val);
 489                 memcpy(ptr, val, len);
 490                 ptr += len;
 491                 token = token->next;
 492         }
 493         *ptr = 0;
 494         if (endop && !match_op(token, endop))
 495                 warn(start->pos, "expected '>' at end of filename");
 496         return buffer;
 497 }
 498
 499 static int try_include(const char *path, int plen, const char *filename, int flen, struct token *head)
 500 {
 501         int fd;
 502         static char fullname[PATH_MAX];
 503
 504         memcpy(fullname, path, plen);
 505         if (plen && path[plen-1] != '/') {
 506                 fullname[plen] = '/';
 507                 plen++;
 508         }
 509         memcpy(fullname+plen, filename, flen);
 510         fd = open(fullname, O_RDONLY);
 511         if (fd >= 0) {
 512                 char * streamname = __alloc_bytes(plen + flen);
 513                 memcpy(streamname, fullname, plen + flen);
 514                 head->next = tokenize(streamname, fd, head->next);
 515                 close(fd);
 516                 return 1;
 517         }
 518         return 0;
 519 }
 520
 521 static int do_include_path(const char **pptr, struct token *head, struct token *token, const char *filename, int flen)
 522 {
 523         const char *path;
 524
 525         while ((path = *pptr++) != NULL) {
 526                 if (!try_include(path, strlen(path), filename, flen, head))
 527                         continue;
 528                 return 1;
 529         }
 530         return 0;
 531 }
 532
 533
 534 static void do_include(struct stream *stream, struct token *head, struct token *token, const char *filename)
 535 {
 536         const char *path;
 537         char *slash;
 538         int flen = strlen(filename) + 1;
 539
 540         /* Check the standard include paths.. */
 541         if (do_include_path(includepath, head, token, filename, flen))
 542                 return;
 543         if (do_include_path(sys_includepath, head, token, filename, flen))
 544                 return;
 545         if (do_include_path(gcc_includepath, head, token, filename, flen))
 546                 return;
 547
 548         /* Check same directory as current stream.. */
 549         path = stream->name;
 550         slash = strrchr(path, '/');
 551         if (slash) {
 552                 if (try_include(path, slash-path, filename, flen, head))
 553                         return;
 554         }
 555
 556         /* Check current directory */
 557         if (try_include("", 0, filename, flen, head))
 558                 return;
 559
 560         error(token->pos, "unable to open '%s'", filename);
 561 }
 562
 563 static int handle_include(struct stream *stream, struct token *head, struct token *token)
 564 {
 565         const char *filename;
 566         struct token *next;
 567         int expect;
 568
 569         if (stream->constant == -1)
 570                 stream->constant = 0;
 571         if (false_nesting)
 572                 return 1;
 573         next = token->next;
 574         expect = '>';
 575         if (!match_op(next, '<')) {
 576                 expand_list(NULL, token);
 577                 expect = 0;
 578                 next = token;
 579         }
 580         token = next->next;
 581         filename = token_name_sequence(token, expect, token);
 582         do_include(stream, head, token, filename);
 583         return 1;
 584 }
 585
 586 static int token_list_different(struct token *list1, struct token *list2)
 587 {
 588         for (;;) {
 589                 if (list1 == list2)
 590                         return 0;
 591                 if (!list1 || !list2)
 592                         return 1;
 593                 if (token_type(list1) != token_type(list2))
 594                         return 1;
 595                 list1 = list1->next;
 596                 list2 = list2->next;
 597         }
 598 }
 599
 600
 601 static int handle_define(struct stream *stream, struct token *head, struct token *token)
 602 {
 603         struct token *arglist, *expansion;
 604         struct token *left = token->next;
 605         struct symbol *sym;
 606         struct ident *name;
 607
 608         if (token_type(left) != TOKEN_IDENT) {
 609                 warn(head->pos, "expected identifier to 'define'");
 610                 return 0;
 611         }
 612         if (false_nesting)
 613                 return 1;
 614         name = left->ident;
 615
 616         arglist = NULL;
 617         expansion = left->next;
 618         if (!expansion->pos.whitespace && match_op(expansion, '(')) {
 619                 arglist = expansion;
 620                 while (!eof_token(expansion)) {
 621                         struct token *next = expansion->next;
 622                         if (match_op(next, ')')) {
 623                                 // Terminate the arglist
 624                                 expansion->next = &eof_token_entry;
 625                                 expansion = next->next;
 626                                 break;
 627                         }
 628                         if (match_op(next, ','))
 629                                 expansion->next = next->next;
 630                         expansion = next;
 631                 }
 632                 arglist = arglist->next;
 633         }
 634
 635         sym = lookup_symbol(name, NS_PREPROCESSOR);
 636         if (sym) {
 637                 if (token_list_different(sym->expansion, expansion) ||
 638                     token_list_different(sym->arglist, arglist)) {
 639                         warn(left->pos, "preprocessor token redefined");
 640                         warn(sym->pos, "this was the original definition");
 641                 }
 642                 return 1;
 643         }
 644         sym = alloc_symbol(left->pos, SYM_NODE);
 645         bind_symbol(sym, name, NS_PREPROCESSOR);
 646
 647         sym->expansion = expansion;
 648         sym->arglist = arglist;
 649         return 1;
 650 }
 651
 652 static int handle_undef(struct stream *stream, struct token *head, struct token *token)
 653 {
 654         struct token *left = token->next;
 655         struct symbol **sym;
 656
 657         if (token_type(left) != TOKEN_IDENT) {
 658                 warn(head->pos, "expected identifier to 'undef'");
 659                 return 0;
 660         }
 661         if (false_nesting)
 662                 return 1;
 663         sym = &left->ident->symbols;
 664         while (*sym) {
 665                 struct symbol *t = *sym;
 666                 if (t->namespace == NS_PREPROCESSOR) {
 667                         *sym = t->next_id;
 668                         return 1;
 669                 }
 670                 sym = &t->next_id;
 671         }
 672         return 1;
 673 }
 674
 675 static int preprocessor_if(struct token *token, int true)
 676 {
 677         if (if_nesting == 0)
 678                 unmatched_if = token;
 679         elif_ignore[if_nesting] = false_nesting || true;
 680         if (false_nesting || !true) {
 681                 false_nesting++;
 682                 return 1;
 683         }
 684         true_nesting++;
 685         return 1;
 686 }
 687
 688 static int token_defined(struct token *token)
 689 {
 690         if (token_type(token) == TOKEN_IDENT)
 691                 return lookup_symbol(token->ident, NS_PREPROCESSOR) != NULL;
 692
 693         warn(token->pos, "expected identifier for #if[n]def");
 694         return 0;
 695 }
 696
 697 static int handle_ifdef(struct stream *stream, struct token *head, struct token *token)
 698 {
 699         return preprocessor_if(token, token_defined(token->next));
 700 }
 701
 702 static int handle_ifndef(struct stream *stream, struct token *head, struct token *token)
 703 {
 704         struct token *next = token->next;
 705         if (stream->constant == -1) {
 706                 int newconstant = 0;
 707                 if (token_type(next) == TOKEN_IDENT) {
 708                         if (!stream->protect || stream->protect == next->ident) {
 709                                 newconstant = -2;
 710                                 stream->protect = next->ident;
 711                                 stream->nesting = if_nesting+1;
 712                         }
 713                 }
 714                 stream->constant = newconstant;
 715         }
 716         return preprocessor_if(token, !token_defined(next));
 717 }
 718
 719 static int expression_value(struct token *head)
 720 {
 721         struct expression *expr;
 722         struct token *token;
 723         long long value;
 724
 725         expand_list(NULL, head);
 726         token = constant_expression(head->next, &expr);
 727         if (!eof_token(token))
 728                 warn(token->pos, "garbage at end: %s", show_token_sequence(token));
 729         value = get_expression_value(expr);
 730         return value != 0;
 731 }
 732
 733 static int handle_if(struct stream *stream, struct token *head, struct token *token)
 734 {
 735         int value = 0;
 736         if (!false_nesting)
 737                 value = expression_value(token);
 738         return preprocessor_if(token, value);
 739 }
 740
 741 static int handle_elif(struct stream * stream, struct token *head, struct token *token)
 742 {
 743         if (stream->nesting == if_nesting)
 744                 stream->constant = 0;
 745         if (false_nesting) {
 746                 /* If this whole if-thing is if'ed out, an elif cannot help */
 747                 if (elif_ignore[if_nesting-1])
 748                         return 1;
 749                 if (expression_value(token)) {
 750                         false_nesting--;
 751                         true_nesting++;
 752                         elif_ignore[if_nesting-1] = 1;
 753                 }
 754                 return 1;
 755         }
 756         if (true_nesting) {
 757                 false_nesting = 1;
 758                 true_nesting--;
 759                 return 1;
 760         }
 761         warn(token->pos, "unmatched '#elif'");
 762         return 1;
 763 }
 764
 765 static int handle_else(struct stream *stream, struct token *head, struct token *token)
 766 {
 767         if (stream->nesting == if_nesting)
 768                 stream->constant = 0;
 769         if (false_nesting) {
 770                 /* If this whole if-thing is if'ed out, an else cannot help */
 771                 if (elif_ignore[if_nesting-1])
 772                         return 1;
 773                 false_nesting--;
 774                 true_nesting++;
 775                 elif_ignore[if_nesting-1] = 1;
 776                 return 1;
 777         }
 778         if (true_nesting) {
 779                 true_nesting--;
 780                 false_nesting = 1;
 781                 return 1;
 782         }
 783         warn(token->pos, "unmatched #else");
 784         return 1;
 785 }
 786
 787 static int handle_endif(struct stream *stream, struct token *head, struct token *token)
 788 {
 789         if (stream->constant == -2 && stream->nesting == if_nesting)
 790                 stream->constant = -1;
 791
 792         if (false_nesting) {
 793                 false_nesting--;
 794                 return 1;
 795         }
 796         if (true_nesting) {
 797                 true_nesting--;
 798                 return 1;
 799         }
 800         warn(token->pos, "unmatched #endif");
 801         return 1;
 802 }
 803
 804 static const char *show_token_sequence(struct token *token)
 805 {
 806         static char buffer[256];
 807         char *ptr = buffer;
 808         int whitespace = 0;
 809
 810         if (!token)
 811                 return "<none>";
 812         while (!eof_token(token) && !match_op(token, SPECIAL_ARG_SEPARATOR)) {
 813                 const char *val = show_token(token);
 814                 int len = strlen(val);
 815                 if (whitespace)
 816                         *ptr++ = ' ';
 817                 memcpy(ptr, val, len);
 818                 ptr += len;
 819                 token = token->next;
 820                 whitespace = token->pos.whitespace;
 821         }
 822         *ptr = 0;
 823         return buffer;
 824 }
 825
 826 static int handle_warning(struct stream *stream, struct token *head, struct token *token)
 827 {
 828         if (false_nesting)
 829                 return 1;
 830         warn(token->pos, "%s", show_token_sequence(token->next));
 831         return 1;
 832 }
 833
 834 static int handle_error(struct stream *stream, struct token *head, struct token *token)
 835 {
 836         if (false_nesting)
 837                 return 1;
 838         warn(token->pos, "%s", show_token_sequence(token->next));
 839         return 1;
 840 }
 841
 842 static int handle_nostdinc(struct stream *stream, struct token *head, struct token *token)
 843 {
 844         if (false_nesting)
 845                 return 1;
 846         includepath[0] = NULL;
 847         return 1;
 848 }
 849
 850 static void add_path_entry(struct token *token, const char *path)
 851 {
 852         int i;
 853
 854         for (i = 0; i < INCLUDEPATHS; i++) {
 855                 if (!includepath[i]) {
 856                         includepath[i] = path;
 857                         includepath[i+1] = NULL;
 858                         return;
 859                 }
 860         }
 861         warn(token->pos, "too many include path entries");
 862 }
 863
 864 static int handle_add_include(struct stream *stream, struct token *head, struct token *token)
 865 {
 866         for (;;) {
 867                 token = token->next;
 868                 if (eof_token(token))
 869                         return 1;
 870                 if (token_type(token) != TOKEN_STRING) {
 871                         warn(token->pos, "expected path string");
 872                         return 1;
 873                 }
 874                 add_path_entry(token, token->string->data);
 875         }
 876 }
 877
 878 static int handle_preprocessor_command(struct stream *stream, struct token *head, struct ident *ident, struct token *token)
 879 {
 880         int i;
 881         static struct {
 882                 const char *name;
 883                 int (*handler)(struct stream *, struct token *, struct token *);
 884         } handlers[] = {
 885                 { "define",     handle_define },
 886                 { "undef",      handle_undef },
 887                 { "ifdef",      handle_ifdef },
 888                 { "ifndef",     handle_ifndef },
 889                 { "else",       handle_else },
 890                 { "endif",      handle_endif },
 891                 { "if",         handle_if },
 892                 { "elif",       handle_elif },
 893                 { "warning",    handle_warning },
 894                 { "error",      handle_error },
 895                 { "include",    handle_include },
 896
 897                 // our internal preprocessor tokens
 898                 { "nostdinc",   handle_nostdinc },
 899                 { "add_include", handle_add_include },
 900         };
 901
 902         for (i = 0; i < (sizeof (handlers) / sizeof (handlers[0])); i++) {
 903                 if (match_string_ident(ident, handlers[i].name))
 904                         return handlers[i].handler(stream, head, token);
 905         }
 906         return 0;
 907 }
 908
 909 static void handle_preprocessor_line(struct stream *stream, struct token * head, struct token *token)
 910 {
 911         if (!token)
 912                 return;
 913
 914         if (token_type(token) == TOKEN_IDENT)
 915                 if (handle_preprocessor_command(stream, head, token->ident, token))
 916                         return;
 917         warn(token->pos, "unrecognized preprocessor line '%s'", show_token_sequence(token));
 918 }
 919
 920 static void preprocessor_line(struct stream *stream, struct token * head)
 921 {
 922         struct token *start = head->next, *next;
 923         struct token **tp = &start->next;
 924
 925         for (;;) {
 926                 next = *tp;
 927                 if (next->pos.newline)
 928                         break;
 929                 tp = &next->next;
 930         }
 931         head->next = next;
 932         *tp = &eof_token_entry;
 933         handle_preprocessor_line(stream, head, start->next);
 934 }
 935
 936 static void do_preprocess(struct token *head)
 937 {
 938         do {
 939                 struct token *next = head->next;
 940                 struct stream *stream = input_streams + next->pos.stream;
 941
 942                 if (next->pos.newline && match_op(next, '#')) {
 943                         preprocessor_line(stream, head);
 944                         continue;
 945                 }
 946
 947                 if (false_nesting) {
 948                         head->next = next->next;
 949                         continue;
 950                 }
 951
 952                 switch (token_type(next)) {
 953                 case TOKEN_STREAMEND:
 954                         if (stream->constant == -1 && stream->protect) {
 955                                 stream->constant = 1;
 956                         }
 957                         /* fallthrough */
 958                 case TOKEN_STREAMBEGIN:
 959                         head->next = next->next;
 960                         continue;
 961
 962                 case TOKEN_IDENT:
 963                         next = expand_one_symbol(next, head, next);
 964                         /* fallthrough */
 965                 default:
 966                         /*
 967                          * Any token expansion (even if it ended up being an
 968                          * empty expansion) in this stream implies it can't
 969                          * be constant.
 970                          */
 971                         stream->constant = 0;
 972                 }
 973
 974                 head = next;
 975         } while (!eof_token(head));
 976 }
 977
 978 struct token * preprocess(struct token *token)
 979 {
 980         struct token header = { };
 981
 982         preprocessing = 1;
 983         header.next = token;
 984         do_preprocess(&header);
 985         if (if_nesting)
 986                 warn(unmatched_if->pos, "unmatched preprocessor conditional");
 987
 988         // Drop all expressions from pre-processing, they're not used any more.
 989         clear_expression_alloc();
 990         preprocessing = 0;
 991
 992         return header.next;
 993 }