tccpp.c

   1 /*
   2  *  TCC - Tiny C Compiler
   3  *
   4  *  Copyright (c) 2001-2004 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include "tcc.h"
  22
  23 /********************************************************/
  24 /* global variables */
  25
  26 ST_DATA int tok_flags;
  27 /* additional informations about token */
  28 #define TOK_FLAG_BOL   0x0001 /* beginning of line before */
  29 #define TOK_FLAG_BOF   0x0002 /* beginning of file before */
  30 #define TOK_FLAG_ENDIF 0x0004 /* a endif was found matching starting #ifdef */
  31 #define TOK_FLAG_EOF   0x0008 /* end of file */
  32
  33 ST_DATA int parse_flags;
  34 #define PARSE_FLAG_PREPROCESS 0x0001 /* activate preprocessing */
  35 #define PARSE_FLAG_TOK_NUM    0x0002 /* return numbers instead of TOK_PPNUM */
  36 #define PARSE_FLAG_LINEFEED   0x0004 /* line feed is returned as a
  37                                         token. line feed is also
  38                                         returned at eof */
  39 #define PARSE_FLAG_ASM_COMMENTS 0x0008 /* '#' can be used for line comment */
  40 #define PARSE_FLAG_SPACES     0x0010 /* next() returns space tokens (for -E) */
  41
  42 ST_DATA struct BufferedFile *file;
  43 ST_DATA int ch, tok;
  44 ST_DATA CValue tokc;
  45 ST_DATA const int *macro_ptr;
  46 ST_DATA CString tokcstr; /* current parsed string, if any */
  47
  48 /* display benchmark infos */
  49 ST_DATA int total_lines;
  50 ST_DATA int total_bytes;
  51 ST_DATA int tok_ident;
  52 ST_DATA TokenSym **table_ident;
  53
  54 /* ------------------------------------------------------------------------- */
  55
  56 static int *macro_ptr_allocated;
  57 static const int *unget_saved_macro_ptr;
  58 static int unget_saved_buffer[TOK_MAX_SIZE + 1];
  59 static int unget_buffer_enabled;
  60 static TokenSym *hash_ident[TOK_HASH_SIZE];
  61 static char token_buf[STRING_MAX_SIZE + 1];
  62 /* true if isid(c) || isnum(c) */
  63 static unsigned char isidnum_table[256-CH_EOF];
  64
  65 static const char tcc_keywords[] =
  66 #define DEF(id, str) str "\0"
  67 #include "tcctok.h"
  68 #undef DEF
  69 ;
  70
  71 /* WARNING: the content of this string encodes token numbers */
  72 static const unsigned char tok_two_chars[] =
  73     "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
  74     "-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
  75
  76 struct macro_level {
  77     struct macro_level *prev;
  78     const int *p;
  79 };
  80
  81 ST_FUNC void next_nomacro(void);
  82 static void next_nomacro_spc(void);
  83 static void macro_subst(
  84     TokenString *tok_str,
  85     Sym **nested_list,
  86     const int *macro_str,
  87     struct macro_level **can_read_stream
  88     );
  89
  90 ST_FUNC void skip(int c)
  91 {
  92     if (tok != c)
  93         tcc_error("'%c' expected (got \"%s\")", c, get_tok_str(tok, &tokc));
  94     next();
  95 }
  96
  97 ST_FUNC void expect(const char *msg)
  98 {
  99     tcc_error("%s expected", msg);
 100 }
 101
 102 /* ------------------------------------------------------------------------- */
 103 /* CString handling */
 104 static void cstr_realloc(CString *cstr, int new_size)
 105 {
 106     int size;
 107     void *data;
 108
 109     size = cstr->size_allocated;
 110     if (size == 0)
 111         size = 8; /* no need to allocate a too small first string */
 112     while (size < new_size)
 113         size = size * 2;
 114     data = tcc_realloc(cstr->data_allocated, size);
 115     if (!data)
 116         tcc_error("memory full");
 117     cstr->data_allocated = data;
 118     cstr->size_allocated = size;
 119     cstr->data = data;
 120 }
 121
 122 /* add a byte */
 123 PUB_FUNC void cstr_ccat(CString *cstr, int ch)
 124 {
 125     int size;
 126     size = cstr->size + 1;
 127     if (size > cstr->size_allocated)
 128         cstr_realloc(cstr, size);
 129     ((unsigned char *)cstr->data)[size - 1] = ch;
 130     cstr->size = size;
 131 }
 132
 133 PUB_FUNC void cstr_cat(CString *cstr, const char *str)
 134 {
 135     int c;
 136     for(;;) {
 137         c = *str;
 138         if (c == '\0')
 139             break;
 140         cstr_ccat(cstr, c);
 141         str++;
 142     }
 143 }
 144
 145 /* add a wide char */
 146 PUB_FUNC void cstr_wccat(CString *cstr, int ch)
 147 {
 148     int size;
 149     size = cstr->size + sizeof(nwchar_t);
 150     if (size > cstr->size_allocated)
 151         cstr_realloc(cstr, size);
 152     *(nwchar_t *)(((unsigned char *)cstr->data) + size - sizeof(nwchar_t)) = ch;
 153     cstr->size = size;
 154 }
 155
 156 PUB_FUNC void cstr_new(CString *cstr)
 157 {
 158     memset(cstr, 0, sizeof(CString));
 159 }
 160
 161 /* free string and reset it to NULL */
 162 PUB_FUNC void cstr_free(CString *cstr)
 163 {
 164     tcc_free(cstr->data_allocated);
 165     cstr_new(cstr);
 166 }
 167
 168 /* XXX: unicode ? */
 169 static void add_char(CString *cstr, int c)
 170 {
 171     if (c == '\'' || c == '\"' || c == '\\') {
 172         /* XXX: could be more precise if char or string */
 173         cstr_ccat(cstr, '\\');
 174     }
 175     if (c >= 32 && c <= 126) {
 176         cstr_ccat(cstr, c);
 177     } else {
 178         cstr_ccat(cstr, '\\');
 179         if (c == '\n') {
 180             cstr_ccat(cstr, 'n');
 181         } else {
 182             cstr_ccat(cstr, '0' + ((c >> 6) & 7));
 183             cstr_ccat(cstr, '0' + ((c >> 3) & 7));
 184             cstr_ccat(cstr, '0' + (c & 7));
 185         }
 186     }
 187 }
 188
 189 /* ------------------------------------------------------------------------- */
 190 /* allocate a new token */
 191 static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
 192 {
 193     TokenSym *ts, **ptable;
 194     int i;
 195
 196     if (tok_ident >= SYM_FIRST_ANOM)
 197         tcc_error("memory full");
 198
 199     /* expand token table if needed */
 200     i = tok_ident - TOK_IDENT;
 201     if ((i % TOK_ALLOC_INCR) == 0) {
 202         ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
 203         if (!ptable)
 204             tcc_error("memory full");
 205         table_ident = ptable;
 206     }
 207
 208     ts = tcc_malloc(sizeof(TokenSym) + len);
 209     table_ident[i] = ts;
 210     ts->tok = tok_ident++;
 211     ts->sym_define = NULL;
 212     ts->sym_label = NULL;
 213     ts->sym_struct = NULL;
 214     ts->sym_identifier = NULL;
 215     ts->len = len;
 216     ts->hash_next = NULL;
 217     memcpy(ts->str, str, len);
 218     ts->str[len] = '\0';
 219     *pts = ts;
 220     return ts;
 221 }
 222
 223 #define TOK_HASH_INIT 1
 224 #define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))
 225
 226 /* find a token and add it if not found */
 227 ST_FUNC TokenSym *tok_alloc(const char *str, int len)
 228 {
 229     TokenSym *ts, **pts;
 230     int i;
 231     unsigned int h;
 232
 233     h = TOK_HASH_INIT;
 234     for(i=0;i<len;i++)
 235         h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
 236     h &= (TOK_HASH_SIZE - 1);
 237
 238     pts = &hash_ident[h];
 239     for(;;) {
 240         ts = *pts;
 241         if (!ts)
 242             break;
 243         if (ts->len == len && !memcmp(ts->str, str, len))
 244             return ts;
 245         pts = &(ts->hash_next);
 246     }
 247     return tok_alloc_new(pts, str, len);
 248 }
 249
 250 /* XXX: buffer overflow */
 251 /* XXX: float tokens */
 252 ST_FUNC char *get_tok_str(int v, CValue *cv)
 253 {
 254     static char buf[STRING_MAX_SIZE + 1];
 255     static CString cstr_buf;
 256     CString *cstr;
 257     char *p;
 258     int i, len;
 259
 260     /* NOTE: to go faster, we give a fixed buffer for small strings */
 261     cstr_reset(&cstr_buf);
 262     cstr_buf.data = buf;
 263     cstr_buf.size_allocated = sizeof(buf);
 264     p = buf;
 265
 266     switch(v) {
 267     case TOK_CINT:
 268     case TOK_CUINT:
 269         /* XXX: not quite exact, but only useful for testing */
 270         sprintf(p, "%u", cv->ui);
 271         break;
 272     case TOK_CLLONG:
 273     case TOK_CULLONG:
 274         /* XXX: not quite exact, but only useful for testing  */
 275 #ifdef _WIN32
 276         sprintf(p, "%u", (unsigned)cv->ull);
 277 #else
 278         sprintf(p, "%Lu", cv->ull);
 279 #endif
 280         break;
 281     case TOK_LCHAR:
 282         cstr_ccat(&cstr_buf, 'L');
 283     case TOK_CCHAR:
 284         cstr_ccat(&cstr_buf, '\'');
 285         add_char(&cstr_buf, cv->i);
 286         cstr_ccat(&cstr_buf, '\'');
 287         cstr_ccat(&cstr_buf, '\0');
 288         break;
 289     case TOK_PPNUM:
 290         cstr = cv->cstr;
 291         len = cstr->size - 1;
 292         for(i=0;i<len;i++)
 293             add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
 294         cstr_ccat(&cstr_buf, '\0');
 295         break;
 296     case TOK_LSTR:
 297         cstr_ccat(&cstr_buf, 'L');
 298     case TOK_STR:
 299         cstr = cv->cstr;
 300         cstr_ccat(&cstr_buf, '\"');
 301         if (v == TOK_STR) {
 302             len = cstr->size - 1;
 303             for(i=0;i<len;i++)
 304                 add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
 305         } else {
 306             len = (cstr->size / sizeof(nwchar_t)) - 1;
 307             for(i=0;i<len;i++)
 308                 add_char(&cstr_buf, ((nwchar_t *)cstr->data)[i]);
 309         }
 310         cstr_ccat(&cstr_buf, '\"');
 311         cstr_ccat(&cstr_buf, '\0');
 312         break;
 313     case TOK_LT:
 314         v = '<';
 315         goto addv;
 316     case TOK_GT:
 317         v = '>';
 318         goto addv;
 319     case TOK_DOTS:
 320         return strcpy(p, "...");
 321     case TOK_A_SHL:
 322         return strcpy(p, "<<=");
 323     case TOK_A_SAR:
 324         return strcpy(p, ">>=");
 325     default:
 326         if (v < TOK_IDENT) {
 327             /* search in two bytes table */
 328             const unsigned char *q = tok_two_chars;
 329             while (*q) {
 330                 if (q[2] == v) {
 331                     *p++ = q[0];
 332                     *p++ = q[1];
 333                     *p = '\0';
 334                     return buf;
 335                 }
 336                 q += 3;
 337             }
 338         addv:
 339             *p++ = v;
 340             *p = '\0';
 341         } else if (v < tok_ident) {
 342             return table_ident[v - TOK_IDENT]->str;
 343         } else if (v >= SYM_FIRST_ANOM) {
 344             /* special name for anonymous symbol */
 345             sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
 346         } else {
 347             /* should never happen */
 348             return NULL;
 349         }
 350         break;
 351     }
 352     return cstr_buf.data;
 353 }
 354
 355 /* fill input buffer and peek next char */
 356 static int tcc_peekc_slow(BufferedFile *bf)
 357 {
 358     int len;
 359     /* only tries to read if really end of buffer */
 360     if (bf->buf_ptr >= bf->buf_end) {
 361         if (bf->fd != -1) {
 362 #if defined(PARSE_DEBUG)
 363             len = 8;
 364 #else
 365             len = IO_BUF_SIZE;
 366 #endif
 367             len = read(bf->fd, bf->buffer, len);
 368             if (len < 0)
 369                 len = 0;
 370         } else {
 371             len = 0;
 372         }
 373         total_bytes += len;
 374         bf->buf_ptr = bf->buffer;
 375         bf->buf_end = bf->buffer + len;
 376         *bf->buf_end = CH_EOB;
 377     }
 378     if (bf->buf_ptr < bf->buf_end) {
 379         return bf->buf_ptr[0];
 380     } else {
 381         bf->buf_ptr = bf->buf_end;
 382         return CH_EOF;
 383     }
 384 }
 385
 386 /* return the current character, handling end of block if necessary
 387    (but not stray) */
 388 ST_FUNC int handle_eob(void)
 389 {
 390     return tcc_peekc_slow(file);
 391 }
 392
 393 /* read next char from current input file and handle end of input buffer */
 394 ST_INLN void inp(void)
 395 {
 396     ch = *(++(file->buf_ptr));
 397     /* end of buffer/file handling */
 398     if (ch == CH_EOB)
 399         ch = handle_eob();
 400 }
 401
 402 /* handle '\[\r]\n' */
 403 static int handle_stray_noerror(void)
 404 {
 405     while (ch == '\\') {
 406         inp();
 407         if (ch == '\n') {
 408             file->line_num++;
 409             inp();
 410         } else if (ch == '\r') {
 411             inp();
 412             if (ch != '\n')
 413                 goto fail;
 414             file->line_num++;
 415             inp();
 416         } else {
 417         fail:
 418             return 1;
 419         }
 420     }
 421     return 0;
 422 }
 423
 424 static void handle_stray(void)
 425 {
 426     if (handle_stray_noerror())
 427         tcc_error("stray '\\' in program");
 428 }
 429
 430 /* skip the stray and handle the \\n case. Output an error if
 431    incorrect char after the stray */
 432 static int handle_stray1(uint8_t *p)
 433 {
 434     int c;
 435
 436     if (p >= file->buf_end) {
 437         file->buf_ptr = p;
 438         c = handle_eob();
 439         p = file->buf_ptr;
 440         if (c == '\\')
 441             goto parse_stray;
 442     } else {
 443     parse_stray:
 444         file->buf_ptr = p;
 445         ch = *p;
 446         handle_stray();
 447         p = file->buf_ptr;
 448         c = *p;
 449     }
 450     return c;
 451 }
 452
 453 /* handle just the EOB case, but not stray */
 454 #define PEEKC_EOB(c, p)\
 455 {\
 456     p++;\
 457     c = *p;\
 458     if (c == '\\') {\
 459         file->buf_ptr = p;\
 460         c = handle_eob();\
 461         p = file->buf_ptr;\
 462     }\
 463 }
 464
 465 /* handle the complicated stray case */
 466 #define PEEKC(c, p)\
 467 {\
 468     p++;\
 469     c = *p;\
 470     if (c == '\\') {\
 471         c = handle_stray1(p);\
 472         p = file->buf_ptr;\
 473     }\
 474 }
 475
 476 /* input with '\[\r]\n' handling. Note that this function cannot
 477    handle other characters after '\', so you cannot call it inside
 478    strings or comments */
 479 ST_FUNC void minp(void)
 480 {
 481     inp();
 482     if (ch == '\\')
 483         handle_stray();
 484 }
 485
 486
 487 /* single line C++ comments */
 488 static uint8_t *parse_line_comment(uint8_t *p)
 489 {
 490     int c;
 491
 492     p++;
 493     for(;;) {
 494         c = *p;
 495     redo:
 496         if (c == '\n' || c == CH_EOF) {
 497             break;
 498         } else if (c == '\\') {
 499             file->buf_ptr = p;
 500             c = handle_eob();
 501             p = file->buf_ptr;
 502             if (c == '\\') {
 503                 PEEKC_EOB(c, p);
 504                 if (c == '\n') {
 505                     file->line_num++;
 506                     PEEKC_EOB(c, p);
 507                 } else if (c == '\r') {
 508                     PEEKC_EOB(c, p);
 509                     if (c == '\n') {
 510                         file->line_num++;
 511                         PEEKC_EOB(c, p);
 512                     }
 513                 }
 514             } else {
 515                 goto redo;
 516             }
 517         } else {
 518             p++;
 519         }
 520     }
 521     return p;
 522 }
 523
 524 /* C comments */
 525 ST_FUNC uint8_t *parse_comment(uint8_t *p)
 526 {
 527     int c;
 528
 529     p++;
 530     for(;;) {
 531         /* fast skip loop */
 532         for(;;) {
 533             c = *p;
 534             if (c == '\n' || c == '*' || c == '\\')
 535                 break;
 536             p++;
 537             c = *p;
 538             if (c == '\n' || c == '*' || c == '\\')
 539                 break;
 540             p++;
 541         }
 542         /* now we can handle all the cases */
 543         if (c == '\n') {
 544             file->line_num++;
 545             p++;
 546         } else if (c == '*') {
 547             p++;
 548             for(;;) {
 549                 c = *p;
 550                 if (c == '*') {
 551                     p++;
 552                 } else if (c == '/') {
 553                     goto end_of_comment;
 554                 } else if (c == '\\') {
 555                     file->buf_ptr = p;
 556                     c = handle_eob();
 557                     p = file->buf_ptr;
 558                     if (c == '\\') {
 559                         /* skip '\[\r]\n', otherwise just skip the stray */
 560                         while (c == '\\') {
 561                             PEEKC_EOB(c, p);
 562                             if (c == '\n') {
 563                                 file->line_num++;
 564                                 PEEKC_EOB(c, p);
 565                             } else if (c == '\r') {
 566                                 PEEKC_EOB(c, p);
 567                                 if (c == '\n') {
 568                                     file->line_num++;
 569                                     PEEKC_EOB(c, p);
 570                                 }
 571                             } else {
 572                                 goto after_star;
 573                             }
 574                         }
 575                     }
 576                 } else {
 577                     break;
 578                 }
 579             }
 580         after_star: ;
 581         } else {
 582             /* stray, eob or eof */
 583             file->buf_ptr = p;
 584             c = handle_eob();
 585             p = file->buf_ptr;
 586             if (c == CH_EOF) {
 587                 tcc_error("unexpected end of file in comment");
 588             } else if (c == '\\') {
 589                 p++;
 590             }
 591         }
 592     }
 593  end_of_comment:
 594     p++;
 595     return p;
 596 }
 597
 598 #define cinp minp
 599
 600 static inline void skip_spaces(void)
 601 {
 602     while (is_space(ch))
 603         cinp();
 604 }
 605
 606 static inline int check_space(int t, int *spc)
 607 {
 608     if (is_space(t)) {
 609         if (*spc)
 610             return 1;
 611         *spc = 1;
 612     } else
 613         *spc = 0;
 614     return 0;
 615 }
 616
 617 /* parse a string without interpreting escapes */
 618 static uint8_t *parse_pp_string(uint8_t *p,
 619                                 int sep, CString *str)
 620 {
 621     int c;
 622     p++;
 623     for(;;) {
 624         c = *p;
 625         if (c == sep) {
 626             break;
 627         } else if (c == '\\') {
 628             file->buf_ptr = p;
 629             c = handle_eob();
 630             p = file->buf_ptr;
 631             if (c == CH_EOF) {
 632             unterminated_string:
 633                 /* XXX: indicate line number of start of string */
 634                 tcc_error("missing terminating %c character", sep);
 635             } else if (c == '\\') {
 636                 /* escape : just skip \[\r]\n */
 637                 PEEKC_EOB(c, p);
 638                 if (c == '\n') {
 639                     file->line_num++;
 640                     p++;
 641                 } else if (c == '\r') {
 642                     PEEKC_EOB(c, p);
 643                     if (c != '\n')
 644                         expect("'\n' after '\r'");
 645                     file->line_num++;
 646                     p++;
 647                 } else if (c == CH_EOF) {
 648                     goto unterminated_string;
 649                 } else {
 650                     if (str) {
 651                         cstr_ccat(str, '\\');
 652                         cstr_ccat(str, c);
 653                     }
 654                     p++;
 655                 }
 656             }
 657         } else if (c == '\n') {
 658             file->line_num++;
 659             goto add_char;
 660         } else if (c == '\r') {
 661             PEEKC_EOB(c, p);
 662             if (c != '\n') {
 663                 if (str)
 664                     cstr_ccat(str, '\r');
 665             } else {
 666                 file->line_num++;
 667                 goto add_char;
 668             }
 669         } else {
 670         add_char:
 671             if (str)
 672                 cstr_ccat(str, c);
 673             p++;
 674         }
 675     }
 676     p++;
 677     return p;
 678 }
 679
 680 /* skip block of text until #else, #elif or #endif. skip also pairs of
 681    #if/#endif */
 682 static void preprocess_skip(void)
 683 {
 684     int a, start_of_line, c, in_warn_or_error;
 685     uint8_t *p;
 686
 687     p = file->buf_ptr;
 688     a = 0;
 689 redo_start:
 690     start_of_line = 1;
 691     in_warn_or_error = 0;
 692     for(;;) {
 693     redo_no_start:
 694         c = *p;
 695         switch(c) {
 696         case ' ':
 697         case '\t':
 698         case '\f':
 699         case '\v':
 700         case '\r':
 701             p++;
 702             goto redo_no_start;
 703         case '\n':
 704             file->line_num++;
 705             p++;
 706             goto redo_start;
 707         case '\\':
 708             file->buf_ptr = p;
 709             c = handle_eob();
 710             if (c == CH_EOF) {
 711                 expect("#endif");
 712             } else if (c == '\\') {
 713                 ch = file->buf_ptr[0];
 714                 handle_stray_noerror();
 715             }
 716             p = file->buf_ptr;
 717             goto redo_no_start;
 718         /* skip strings */
 719         case '\"':
 720         case '\'':
 721             if (in_warn_or_error)
 722                 goto _default;
 723             p = parse_pp_string(p, c, NULL);
 724             break;
 725         /* skip comments */
 726         case '/':
 727             if (in_warn_or_error)
 728                 goto _default;
 729             file->buf_ptr = p;
 730             ch = *p;
 731             minp();
 732             p = file->buf_ptr;
 733             if (ch == '*') {
 734                 p = parse_comment(p);
 735             } else if (ch == '/') {
 736                 p = parse_line_comment(p);
 737             }
 738             break;
 739         case '#':
 740             p++;
 741             if (start_of_line) {
 742                 file->buf_ptr = p;
 743                 next_nomacro();
 744                 p = file->buf_ptr;
 745                 if (a == 0 &&
 746                     (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
 747                     goto the_end;
 748                 if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
 749                     a++;
 750                 else if (tok == TOK_ENDIF)
 751                     a--;
 752                 else if( tok == TOK_ERROR || tok == TOK_WARNING)
 753                     in_warn_or_error = 1;
 754                 else if (tok == TOK_LINEFEED)
 755                     goto redo_start;
 756             }
 757             break;
 758 _default:
 759         default:
 760             p++;
 761             break;
 762         }
 763         start_of_line = 0;
 764     }
 765  the_end: ;
 766     file->buf_ptr = p;
 767 }
 768
 769 /* ParseState handling */
 770
 771 /* XXX: currently, no include file info is stored. Thus, we cannot display
 772    accurate messages if the function or data definition spans multiple
 773    files */
 774
 775 /* save current parse state in 's' */
 776 ST_FUNC void save_parse_state(ParseState *s)
 777 {
 778     s->line_num = file->line_num;
 779     s->macro_ptr = macro_ptr;
 780     s->tok = tok;
 781     s->tokc = tokc;
 782 }
 783
 784 /* restore parse state from 's' */
 785 ST_FUNC void restore_parse_state(ParseState *s)
 786 {
 787     file->line_num = s->line_num;
 788     macro_ptr = s->macro_ptr;
 789     tok = s->tok;
 790     tokc = s->tokc;
 791 }
 792
 793 /* return the number of additional 'ints' necessary to store the
 794    token */
 795 static inline int tok_ext_size(int t)
 796 {
 797     switch(t) {
 798         /* 4 bytes */
 799     case TOK_CINT:
 800     case TOK_CUINT:
 801     case TOK_CCHAR:
 802     case TOK_LCHAR:
 803     case TOK_CFLOAT:
 804     case TOK_LINENUM:
 805         return 1;
 806     case TOK_STR:
 807     case TOK_LSTR:
 808     case TOK_PPNUM:
 809         tcc_error("unsupported token");
 810         return 1;
 811     case TOK_CDOUBLE:
 812     case TOK_CLLONG:
 813     case TOK_CULLONG:
 814         return 2;
 815     case TOK_CLDOUBLE:
 816         return LDOUBLE_SIZE / 4;
 817     default:
 818         return 0;
 819     }
 820 }
 821
 822 /* token string handling */
 823
 824 ST_INLN void tok_str_new(TokenString *s)
 825 {
 826     s->str = NULL;
 827     s->len = 0;
 828     s->allocated_len = 0;
 829     s->last_line_num = -1;
 830 }
 831
 832 ST_FUNC void tok_str_free(int *str)
 833 {
 834     tcc_free(str);
 835 }
 836
 837 static int *tok_str_realloc(TokenString *s)
 838 {
 839     int *str, len;
 840
 841     if (s->allocated_len == 0) {
 842         len = 8;
 843     } else {
 844         len = s->allocated_len * 2;
 845     }
 846     str = tcc_realloc(s->str, len * sizeof(int));
 847     if (!str)
 848         tcc_error("memory full");
 849     s->allocated_len = len;
 850     s->str = str;
 851     return str;
 852 }
 853
 854 ST_FUNC void tok_str_add(TokenString *s, int t)
 855 {
 856     int len, *str;
 857
 858     len = s->len;
 859     str = s->str;
 860     if (len >= s->allocated_len)
 861         str = tok_str_realloc(s);
 862     str[len++] = t;
 863     s->len = len;
 864 }
 865
 866 static void tok_str_add2(TokenString *s, int t, CValue *cv)
 867 {
 868     int len, *str;
 869
 870     len = s->len;
 871     str = s->str;
 872
 873     /* allocate space for worst case */
 874     if (len + TOK_MAX_SIZE > s->allocated_len)
 875         str = tok_str_realloc(s);
 876     str[len++] = t;
 877     switch(t) {
 878     case TOK_CINT:
 879     case TOK_CUINT:
 880     case TOK_CCHAR:
 881     case TOK_LCHAR:
 882     case TOK_CFLOAT:
 883     case TOK_LINENUM:
 884         str[len++] = cv->tab[0];
 885         break;
 886     case TOK_PPNUM:
 887     case TOK_STR:
 888     case TOK_LSTR:
 889         {
 890             int nb_words;
 891             CString *cstr;
 892
 893             nb_words = (sizeof(CString) + cv->cstr->size + 3) >> 2;
 894             while ((len + nb_words) > s->allocated_len)
 895                 str = tok_str_realloc(s);
 896             cstr = (CString *)(str + len);
 897             cstr->data = NULL;
 898             cstr->size = cv->cstr->size;
 899             cstr->data_allocated = NULL;
 900             cstr->size_allocated = cstr->size;
 901             memcpy((char *)cstr + sizeof(CString),
 902                    cv->cstr->data, cstr->size);
 903             len += nb_words;
 904         }
 905         break;
 906     case TOK_CDOUBLE:
 907     case TOK_CLLONG:
 908     case TOK_CULLONG:
 909 #if LDOUBLE_SIZE == 8
 910     case TOK_CLDOUBLE:
 911 #endif
 912         str[len++] = cv->tab[0];
 913         str[len++] = cv->tab[1];
 914         break;
 915 #if LDOUBLE_SIZE == 12
 916     case TOK_CLDOUBLE:
 917         str[len++] = cv->tab[0];
 918         str[len++] = cv->tab[1];
 919         str[len++] = cv->tab[2];
 920 #elif LDOUBLE_SIZE == 16
 921     case TOK_CLDOUBLE:
 922         str[len++] = cv->tab[0];
 923         str[len++] = cv->tab[1];
 924         str[len++] = cv->tab[2];
 925         str[len++] = cv->tab[3];
 926 #elif LDOUBLE_SIZE != 8
 927 #error add long double size support
 928 #endif
 929         break;
 930     default:
 931         break;
 932     }
 933     s->len = len;
 934 }
 935
 936 /* add the current parse token in token string 's' */
 937 ST_FUNC void tok_str_add_tok(TokenString *s)
 938 {
 939     CValue cval;
 940
 941     /* save line number info */
 942     if (file->line_num != s->last_line_num) {
 943         s->last_line_num = file->line_num;
 944         cval.i = s->last_line_num;
 945         tok_str_add2(s, TOK_LINENUM, &cval);
 946     }
 947     tok_str_add2(s, tok, &tokc);
 948 }
 949
 950 /* get a token from an integer array and increment pointer
 951    accordingly. we code it as a macro to avoid pointer aliasing. */
 952 static inline void TOK_GET(int *t, const int **pp, CValue *cv)
 953 {
 954     const int *p = *pp;
 955     int n, *tab;
 956
 957     tab = cv->tab;
 958     switch(*t = *p++) {
 959     case TOK_CINT:
 960     case TOK_CUINT:
 961     case TOK_CCHAR:
 962     case TOK_LCHAR:
 963     case TOK_CFLOAT:
 964     case TOK_LINENUM:
 965         tab[0] = *p++;
 966         break;
 967     case TOK_STR:
 968     case TOK_LSTR:
 969     case TOK_PPNUM:
 970         cv->cstr = (CString *)p;
 971         cv->cstr->data = (char *)p + sizeof(CString);
 972         p += (sizeof(CString) + cv->cstr->size + 3) >> 2;
 973         break;
 974     case TOK_CDOUBLE:
 975     case TOK_CLLONG:
 976     case TOK_CULLONG:
 977         n = 2;
 978         goto copy;
 979     case TOK_CLDOUBLE:
 980 #if LDOUBLE_SIZE == 16
 981         n = 4;
 982 #elif LDOUBLE_SIZE == 12
 983         n = 3;
 984 #elif LDOUBLE_SIZE == 8
 985         n = 2;
 986 #else
 987 # error add long double size support
 988 #endif
 989     copy:
 990         do
 991             *tab++ = *p++;
 992         while (--n);
 993         break;
 994     default:
 995         break;
 996     }
 997     *pp = p;
 998 }
 999
1000 static int macro_is_equal(const int *a, const int *b)
1001 {
1002     char buf[STRING_MAX_SIZE + 1];
1003     CValue cv;
1004     int t;
1005     while (*a && *b) {
1006         TOK_GET(&t, &a, &cv);
1007         pstrcpy(buf, sizeof buf, get_tok_str(t, &cv));
1008         TOK_GET(&t, &b, &cv);
1009         if (strcmp(buf, get_tok_str(t, &cv)))
1010             return 0;
1011     }
1012     return !(*a || *b);
1013 }
1014
1015 /* defines handling */
1016 ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
1017 {
1018     Sym *s;
1019
1020     s = define_find(v);
1021     if (s && !macro_is_equal(s->d, str))
1022         tcc_warning("%s redefined", get_tok_str(v, NULL));
1023
1024     s = sym_push2(&define_stack, v, macro_type, 0);
1025     s->d = str;
1026     s->next = first_arg;
1027     table_ident[v - TOK_IDENT]->sym_define = s;
1028 }
1029
1030 /* undefined a define symbol. Its name is just set to zero */
1031 ST_FUNC void define_undef(Sym *s)
1032 {
1033     int v;
1034     v = s->v;
1035     if (v >= TOK_IDENT && v < tok_ident)
1036         table_ident[v - TOK_IDENT]->sym_define = NULL;
1037     s->v = 0;
1038 }
1039
1040 ST_INLN Sym *define_find(int v)
1041 {
1042     v -= TOK_IDENT;
1043     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1044         return NULL;
1045     return table_ident[v]->sym_define;
1046 }
1047
1048 /* free define stack until top reaches 'b' */
1049 ST_FUNC void free_defines(Sym *b)
1050 {
1051     Sym *top, *top1;
1052     int v;
1053
1054     top = define_stack;
1055     while (top != b) {
1056         top1 = top->prev;
1057         /* do not free args or predefined defines */
1058         if (top->d)
1059             tok_str_free(top->d);
1060         v = top->v;
1061         if (v >= TOK_IDENT && v < tok_ident)
1062             table_ident[v - TOK_IDENT]->sym_define = NULL;
1063         sym_free(top);
1064         top = top1;
1065     }
1066     define_stack = b;
1067 }
1068
1069 /* label lookup */
1070 ST_FUNC Sym *label_find(int v)
1071 {
1072     v -= TOK_IDENT;
1073     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1074         return NULL;
1075     return table_ident[v]->sym_label;
1076 }
1077
1078 ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
1079 {
1080     Sym *s, **ps;
1081     s = sym_push2(ptop, v, 0, 0);
1082     s->r = flags;
1083     ps = &table_ident[v - TOK_IDENT]->sym_label;
1084     if (ptop == &global_label_stack) {
1085         /* modify the top most local identifier, so that
1086            sym_identifier will point to 's' when popped */
1087         while (*ps != NULL)
1088             ps = &(*ps)->prev_tok;
1089     }
1090     s->prev_tok = *ps;
1091     *ps = s;
1092     return s;
1093 }
1094
1095 /* pop labels until element last is reached. Look if any labels are
1096    undefined. Define symbols if '&&label' was used. */
1097 ST_FUNC void label_pop(Sym **ptop, Sym *slast)
1098 {
1099     Sym *s, *s1;
1100     for(s = *ptop; s != slast; s = s1) {
1101         s1 = s->prev;
1102         if (s->r == LABEL_DECLARED) {
1103             tcc_warning("label '%s' declared but not used", get_tok_str(s->v, NULL));
1104         } else if (s->r == LABEL_FORWARD) {
1105                 tcc_error("label '%s' used but not defined",
1106                       get_tok_str(s->v, NULL));
1107         } else {
1108             if (s->c) {
1109                 /* define corresponding symbol. A size of
1110                    1 is put. */
1111                 put_extern_sym(s, cur_text_section, s->jnext, 1);
1112             }
1113         }
1114         /* remove label */
1115         table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
1116         sym_free(s);
1117     }
1118     *ptop = slast;
1119 }
1120
1121 /* eval an expression for #if/#elif */
1122 static int expr_preprocess(void)
1123 {
1124     int c, t;
1125     TokenString str;
1126
1127     tok_str_new(&str);
1128     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1129         next(); /* do macro subst */
1130         if (tok == TOK_DEFINED) {
1131             next_nomacro();
1132             t = tok;
1133             if (t == '(')
1134                 next_nomacro();
1135             c = define_find(tok) != 0;
1136             if (t == '(')
1137                 next_nomacro();
1138             tok = TOK_CINT;
1139             tokc.i = c;
1140         } else if (tok >= TOK_IDENT) {
1141             /* if undefined macro */
1142             tok = TOK_CINT;
1143             tokc.i = 0;
1144         }
1145         tok_str_add_tok(&str);
1146     }
1147     tok_str_add(&str, -1); /* simulate end of file */
1148     tok_str_add(&str, 0);
1149     /* now evaluate C constant expression */
1150     macro_ptr = str.str;
1151     next();
1152     c = expr_const();
1153     macro_ptr = NULL;
1154     tok_str_free(str.str);
1155     return c != 0;
1156 }
1157
1158 #if defined(PARSE_DEBUG) || defined(PP_DEBUG)
1159 static void tok_print(int *str)
1160 {
1161     int t;
1162     CValue cval;
1163
1164     printf("<");
1165     while (1) {
1166         TOK_GET(&t, &str, &cval);
1167         if (!t)
1168             break;
1169         printf("%s", get_tok_str(t, &cval));
1170     }
1171     printf(">\n");
1172 }
1173 #endif
1174
1175 /* parse after #define */
1176 ST_FUNC void parse_define(void)
1177 {
1178     Sym *s, *first, **ps;
1179     int v, t, varg, is_vaargs, spc;
1180     TokenString str;
1181
1182     v = tok;
1183     if (v < TOK_IDENT)
1184         tcc_error("invalid macro name '%s'", get_tok_str(tok, &tokc));
1185     /* XXX: should check if same macro (ANSI) */
1186     first = NULL;
1187     t = MACRO_OBJ;
1188     /* '(' must be just after macro definition for MACRO_FUNC */
1189     next_nomacro_spc();
1190     if (tok == '(') {
1191         next_nomacro();
1192         ps = &first;
1193         while (tok != ')') {
1194             varg = tok;
1195             next_nomacro();
1196             is_vaargs = 0;
1197             if (varg == TOK_DOTS) {
1198                 varg = TOK___VA_ARGS__;
1199                 is_vaargs = 1;
1200             } else if (tok == TOK_DOTS && gnu_ext) {
1201                 is_vaargs = 1;
1202                 next_nomacro();
1203             }
1204             if (varg < TOK_IDENT)
1205                 tcc_error("badly punctuated parameter list");
1206             s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
1207             *ps = s;
1208             ps = &s->next;
1209             if (tok != ',')
1210                 break;
1211             next_nomacro();
1212         }
1213         if (tok == ')')
1214             next_nomacro_spc();
1215         t = MACRO_FUNC;
1216     }
1217     tok_str_new(&str);
1218     spc = 2;
1219     /* EOF testing necessary for '-D' handling */
1220     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1221         /* remove spaces around ## and after '#' */
1222         if (TOK_TWOSHARPS == tok) {
1223             if (1 == spc)
1224                 --str.len;
1225             spc = 2;
1226         } else if ('#' == tok) {
1227             spc = 2;
1228         } else if (check_space(tok, &spc)) {
1229             goto skip;
1230         }
1231         tok_str_add2(&str, tok, &tokc);
1232     skip:
1233         next_nomacro_spc();
1234     }
1235     if (spc == 1)
1236         --str.len; /* remove trailing space */
1237     tok_str_add(&str, 0);
1238 #ifdef PP_DEBUG
1239     printf("define %s %d: ", get_tok_str(v, NULL), t);
1240     tok_print(str.str);
1241 #endif
1242     define_push(v, t, str.str, first);
1243 }
1244
1245 static inline int hash_cached_include(int type, const char *filename)
1246 {
1247     const unsigned char *s;
1248     unsigned int h;
1249
1250     h = TOK_HASH_INIT;
1251     h = TOK_HASH_FUNC(h, type);
1252     s = filename;
1253     while (*s) {
1254         h = TOK_HASH_FUNC(h, *s);
1255         s++;
1256     }
1257     h &= (CACHED_INCLUDES_HASH_SIZE - 1);
1258     return h;
1259 }
1260
1261 /* XXX: use a token or a hash table to accelerate matching ? */
1262 static CachedInclude *search_cached_include(TCCState *s1,
1263                                             int type, const char *filename)
1264 {
1265     CachedInclude *e;
1266     int i, h;
1267     h = hash_cached_include(type, filename);
1268     i = s1->cached_includes_hash[h];
1269     for(;;) {
1270         if (i == 0)
1271             break;
1272         e = s1->cached_includes[i - 1];
1273         if (e->type == type && !PATHCMP(e->filename, filename))
1274             return e;
1275         i = e->hash_next;
1276     }
1277     return NULL;
1278 }
1279
1280 static inline void add_cached_include(TCCState *s1, int type,
1281                                       const char *filename, int ifndef_macro)
1282 {
1283     CachedInclude *e;
1284     int h;
1285
1286     if (search_cached_include(s1, type, filename))
1287         return;
1288 #ifdef INC_DEBUG
1289     printf("adding cached '%s' %s\n", filename, get_tok_str(ifndef_macro, NULL));
1290 #endif
1291     e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
1292     if (!e)
1293         return;
1294     e->type = type;
1295     strcpy(e->filename, filename);
1296     e->ifndef_macro = ifndef_macro;
1297     dynarray_add((void ***)&s1->cached_includes, &s1->nb_cached_includes, e);
1298     /* add in hash table */
1299     h = hash_cached_include(type, filename);
1300     e->hash_next = s1->cached_includes_hash[h];
1301     s1->cached_includes_hash[h] = s1->nb_cached_includes;
1302 }
1303
1304 static void pragma_parse(TCCState *s1)
1305 {
1306     int val;
1307
1308     next();
1309     if (tok == TOK_pack) {
1310         /*
1311           This may be:
1312           #pragma pack(1) // set
1313           #pragma pack() // reset to default
1314           #pragma pack(push,1) // push & set
1315           #pragma pack(pop) // restore previous
1316         */
1317         next();
1318         skip('(');
1319         if (tok == TOK_ASM_pop) {
1320             next();
1321             if (s1->pack_stack_ptr <= s1->pack_stack) {
1322             stk_error:
1323                 tcc_error("out of pack stack");
1324             }
1325             s1->pack_stack_ptr--;
1326         } else {
1327             val = 0;
1328             if (tok != ')') {
1329                 if (tok == TOK_ASM_push) {
1330                     next();
1331                     if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
1332                         goto stk_error;
1333                     s1->pack_stack_ptr++;
1334                     skip(',');
1335                 }
1336                 if (tok != TOK_CINT) {
1337                 pack_error:
1338                     tcc_error("invalid pack pragma");
1339                 }
1340                 val = tokc.i;
1341                 if (val < 1 || val > 16 || (val & (val - 1)) != 0)
1342                     goto pack_error;
1343                 next();
1344             }
1345             *s1->pack_stack_ptr = val;
1346             skip(')');
1347         }
1348     }
1349 }
1350
1351 /* is_bof is true if first non space token at beginning of file */
1352 ST_FUNC void preprocess(int is_bof)
1353 {
1354     TCCState *s1 = tcc_state;
1355     int i, c, n, saved_parse_flags;
1356     char buf[1024], *q;
1357     Sym *s;
1358
1359     saved_parse_flags = parse_flags;
1360     parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM |
1361         PARSE_FLAG_LINEFEED;
1362     next_nomacro();
1363  redo:
1364     switch(tok) {
1365     case TOK_DEFINE:
1366         next_nomacro();
1367         parse_define();
1368         break;
1369     case TOK_UNDEF:
1370         next_nomacro();
1371         s = define_find(tok);
1372         /* undefine symbol by putting an invalid name */
1373         if (s)
1374             define_undef(s);
1375         break;
1376     case TOK_INCLUDE:
1377     case TOK_INCLUDE_NEXT:
1378         ch = file->buf_ptr[0];
1379         /* XXX: incorrect if comments : use next_nomacro with a special mode */
1380         skip_spaces();
1381         if (ch == '<') {
1382             c = '>';
1383             goto read_name;
1384         } else if (ch == '\"') {
1385             c = ch;
1386         read_name:
1387             inp();
1388             q = buf;
1389             while (ch != c && ch != '\n' && ch != CH_EOF) {
1390                 if ((q - buf) < sizeof(buf) - 1)
1391                     *q++ = ch;
1392                 if (ch == '\\') {
1393                     if (handle_stray_noerror() == 0)
1394                         --q;
1395                 } else
1396                     inp();
1397             }
1398             *q = '\0';
1399             minp();
1400 #if 0
1401             /* eat all spaces and comments after include */
1402             /* XXX: slightly incorrect */
1403             while (ch1 != '\n' && ch1 != CH_EOF)
1404                 inp();
1405 #endif
1406         } else {
1407             /* computed #include : either we have only strings or
1408                we have anything enclosed in '<>' */
1409             next();
1410             buf[0] = '\0';
1411             if (tok == TOK_STR) {
1412                 while (tok != TOK_LINEFEED) {
1413                     if (tok != TOK_STR) {
1414                     include_syntax:
1415                         tcc_error("'#include' expects \"FILENAME\" or <FILENAME>");
1416                     }
1417                     pstrcat(buf, sizeof(buf), (char *)tokc.cstr->data);
1418                     next();
1419                 }
1420                 c = '\"';
1421             } else {
1422                 int len;
1423                 while (tok != TOK_LINEFEED) {
1424                     pstrcat(buf, sizeof(buf), get_tok_str(tok, &tokc));
1425                     next();
1426                 }
1427                 len = strlen(buf);
1428                 /* check syntax and remove '<>' */
1429                 if (len < 2 || buf[0] != '<' || buf[len - 1] != '>')
1430                     goto include_syntax;
1431                 memmove(buf, buf + 1, len - 2);
1432                 buf[len - 2] = '\0';
1433                 c = '>';
1434             }
1435         }
1436
1437         if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
1438             tcc_error("#include recursion too deep");
1439
1440         n = s1->nb_include_paths + s1->nb_sysinclude_paths;
1441         for (i = -2; i < n; ++i) {
1442             char buf1[sizeof file->filename];
1443             CachedInclude *e;
1444             const char *path;
1445             int size, fd;
1446
1447             if (i == -2) {
1448                 /* check absolute include path */
1449                 if (!IS_ABSPATH(buf))
1450                     continue;
1451                 buf1[0] = 0;
1452
1453             } else if (i == -1) {
1454                 /* search in current dir if "header.h" */
1455                 if (c != '\"')
1456                     continue;
1457                 size = tcc_basename(file->filename) - file->filename;
1458                 memcpy(buf1, file->filename, size);
1459                 buf1[size] = '\0';
1460
1461             } else {
1462                 /* search in all the include paths */
1463                 if (i < s1->nb_include_paths)
1464                     path = s1->include_paths[i];
1465                 else
1466                     path = s1->sysinclude_paths[i - s1->nb_include_paths];
1467                 pstrcpy(buf1, sizeof(buf1), path);
1468                 pstrcat(buf1, sizeof(buf1), "/");
1469             }
1470
1471             pstrcat(buf1, sizeof(buf1), buf);
1472
1473             e = search_cached_include(s1, c, buf1);
1474             if (e && define_find(e->ifndef_macro)) {
1475                 /* no need to parse the include because the 'ifndef macro'
1476                    is defined */
1477 #ifdef INC_DEBUG
1478                 printf("%s: skipping %s\n", file->filename, buf);
1479 #endif
1480                 fd = 0;
1481             } else {
1482                 fd = tcc_open(s1, buf1);
1483                 if (fd < 0)
1484                     continue;
1485             }
1486
1487             if (tok == TOK_INCLUDE_NEXT) {
1488                 tok = TOK_INCLUDE;
1489                 if (fd)
1490                     tcc_close();
1491                 continue;
1492             }
1493
1494             if (0 == fd)
1495                 goto include_done;
1496
1497 #ifdef INC_DEBUG
1498             printf("%s: including %s\n", file->filename, buf1);
1499 #endif
1500             /* update target deps */
1501             dynarray_add((void ***)&s1->target_deps, &s1->nb_target_deps,
1502                     tcc_strdup(buf1));
1503            /* XXX: fix current line init */
1504            /* push current file in stack */
1505             *s1->include_stack_ptr++ = file->prev;
1506             file->inc_type = c;
1507             pstrcpy(file->inc_filename, sizeof(file->inc_filename), buf1);
1508             /* add include file debug info */
1509             if (s1->do_debug)
1510                 put_stabs(file->filename, N_BINCL, 0, 0, 0);
1511             tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
1512             ch = file->buf_ptr[0];
1513             goto the_end;
1514         }
1515         tcc_error("include file '%s' not found", buf);
1516 include_done:
1517         break;
1518     case TOK_IFNDEF:
1519         c = 1;
1520         goto do_ifdef;
1521     case TOK_IF:
1522         c = expr_preprocess();
1523         goto do_if;
1524     case TOK_IFDEF:
1525         c = 0;
1526     do_ifdef:
1527         next_nomacro();
1528         if (tok < TOK_IDENT)
1529             tcc_error("invalid argument for '#if%sdef'", c ? "n" : "");
1530         if (is_bof) {
1531             if (c) {
1532 #ifdef INC_DEBUG
1533                 printf("#ifndef %s\n", get_tok_str(tok, NULL));
1534 #endif
1535                 file->ifndef_macro = tok;
1536             }
1537         }
1538         c = (define_find(tok) != 0) ^ c;
1539     do_if:
1540         if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
1541             tcc_error("memory full");
1542         *s1->ifdef_stack_ptr++ = c;
1543         goto test_skip;
1544     case TOK_ELSE:
1545         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1546             tcc_error("#else without matching #if");
1547         if (s1->ifdef_stack_ptr[-1] & 2)
1548             tcc_error("#else after #else");
1549         c = (s1->ifdef_stack_ptr[-1] ^= 3);
1550         goto test_else;
1551     case TOK_ELIF:
1552         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1553             tcc_error("#elif without matching #if");
1554         c = s1->ifdef_stack_ptr[-1];
1555         if (c > 1)
1556             tcc_error("#elif after #else");
1557         /* last #if/#elif expression was true: we skip */
1558         if (c == 1)
1559             goto skip;
1560         c = expr_preprocess();
1561         s1->ifdef_stack_ptr[-1] = c;
1562     test_else:
1563         if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1)
1564             file->ifndef_macro = 0;
1565     test_skip:
1566         if (!(c & 1)) {
1567         skip:
1568             preprocess_skip();
1569             is_bof = 0;
1570             goto redo;
1571         }
1572         break;
1573     case TOK_ENDIF:
1574         if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
1575             tcc_error("#endif without matching #if");
1576         s1->ifdef_stack_ptr--;
1577         /* '#ifndef macro' was at the start of file. Now we check if
1578            an '#endif' is exactly at the end of file */
1579         if (file->ifndef_macro &&
1580             s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
1581             file->ifndef_macro_saved = file->ifndef_macro;
1582             /* need to set to zero to avoid false matches if another
1583                #ifndef at middle of file */
1584             file->ifndef_macro = 0;
1585             while (tok != TOK_LINEFEED)
1586                 next_nomacro();
1587             tok_flags |= TOK_FLAG_ENDIF;
1588             goto the_end;
1589         }
1590         break;
1591     case TOK_LINE:
1592         next();
1593         if (tok != TOK_CINT)
1594             tcc_error("#line");
1595         file->line_num = tokc.i - 1; /* the line number will be incremented after */
1596         next();
1597         if (tok != TOK_LINEFEED) {
1598             if (tok != TOK_STR)
1599                 tcc_error("#line");
1600             pstrcpy(file->filename, sizeof(file->filename),
1601                     (char *)tokc.cstr->data);
1602         }
1603         break;
1604     case TOK_ERROR:
1605     case TOK_WARNING:
1606         c = tok;
1607         ch = file->buf_ptr[0];
1608         skip_spaces();
1609         q = buf;
1610         while (ch != '\n' && ch != CH_EOF) {
1611             if ((q - buf) < sizeof(buf) - 1)
1612                 *q++ = ch;
1613             if (ch == '\\') {
1614                 if (handle_stray_noerror() == 0)
1615                     --q;
1616             } else
1617                 inp();
1618         }
1619         *q = '\0';
1620         if (c == TOK_ERROR)
1621             tcc_error("#error %s", buf);
1622         else
1623             tcc_warning("#warning %s", buf);
1624         break;
1625     case TOK_PRAGMA:
1626         pragma_parse(s1);
1627         break;
1628     default:
1629         if (tok == TOK_LINEFEED || tok == '!' || tok == TOK_PPNUM) {
1630             /* '!' is ignored to allow C scripts. numbers are ignored
1631                to emulate cpp behaviour */
1632         } else {
1633             if (!(saved_parse_flags & PARSE_FLAG_ASM_COMMENTS))
1634                 tcc_warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
1635             else {
1636                 /* this is a gas line comment in an 'S' file. */
1637                 file->buf_ptr = parse_line_comment(file->buf_ptr);
1638                 goto the_end;
1639             }
1640         }
1641         break;
1642     }
1643     /* ignore other preprocess commands or #! for C scripts */
1644     while (tok != TOK_LINEFEED)
1645         next_nomacro();
1646  the_end:
1647     parse_flags = saved_parse_flags;
1648 }
1649
1650 /* evaluate escape codes in a string. */
1651 static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
1652 {
1653     int c, n;
1654     const uint8_t *p;
1655
1656     p = buf;
1657     for(;;) {
1658         c = *p;
1659         if (c == '\0')
1660             break;
1661         if (c == '\\') {
1662             p++;
1663             /* escape */
1664             c = *p;
1665             switch(c) {
1666             case '0': case '1': case '2': case '3':
1667             case '4': case '5': case '6': case '7':
1668                 /* at most three octal digits */
1669                 n = c - '0';
1670                 p++;
1671                 c = *p;
1672                 if (isoct(c)) {
1673                     n = n * 8 + c - '0';
1674                     p++;
1675                     c = *p;
1676                     if (isoct(c)) {
1677                         n = n * 8 + c - '0';
1678                         p++;
1679                     }
1680                 }
1681                 c = n;
1682                 goto add_char_nonext;
1683             case 'x':
1684             case 'u':
1685             case 'U':
1686                 p++;
1687                 n = 0;
1688                 for(;;) {
1689                     c = *p;
1690                     if (c >= 'a' && c <= 'f')
1691                         c = c - 'a' + 10;
1692                     else if (c >= 'A' && c <= 'F')
1693                         c = c - 'A' + 10;
1694                     else if (isnum(c))
1695                         c = c - '0';
1696                     else
1697                         break;
1698                     n = n * 16 + c;
1699                     p++;
1700                 }
1701                 c = n;
1702                 goto add_char_nonext;
1703             case 'a':
1704                 c = '\a';
1705                 break;
1706             case 'b':
1707                 c = '\b';
1708                 break;
1709             case 'f':
1710                 c = '\f';
1711                 break;
1712             case 'n':
1713                 c = '\n';
1714                 break;
1715             case 'r':
1716                 c = '\r';
1717                 break;
1718             case 't':
1719                 c = '\t';
1720                 break;
1721             case 'v':
1722                 c = '\v';
1723                 break;
1724             case 'e':
1725                 if (!gnu_ext)
1726                     goto invalid_escape;
1727                 c = 27;
1728                 break;
1729             case '\'':
1730             case '\"':
1731             case '\\':
1732             case '?':
1733                 break;
1734             default:
1735             invalid_escape:
1736                 if (c >= '!' && c <= '~')
1737                     tcc_warning("unknown escape sequence: \'\\%c\'", c);
1738                 else
1739                     tcc_warning("unknown escape sequence: \'\\x%x\'", c);
1740                 break;
1741             }
1742         }
1743         p++;
1744     add_char_nonext:
1745         if (!is_long)
1746             cstr_ccat(outstr, c);
1747         else
1748             cstr_wccat(outstr, c);
1749     }
1750     /* add a trailing '\0' */
1751     if (!is_long)
1752         cstr_ccat(outstr, '\0');
1753     else
1754         cstr_wccat(outstr, '\0');
1755 }
1756
1757 /* we use 64 bit numbers */
1758 #define BN_SIZE 2
1759
1760 /* bn = (bn << shift) | or_val */
1761 static void bn_lshift(unsigned int *bn, int shift, int or_val)
1762 {
1763     int i;
1764     unsigned int v;
1765     for(i=0;i<BN_SIZE;i++) {
1766         v = bn[i];
1767         bn[i] = (v << shift) | or_val;
1768         or_val = v >> (32 - shift);
1769     }
1770 }
1771
1772 static void bn_zero(unsigned int *bn)
1773 {
1774     int i;
1775     for(i=0;i<BN_SIZE;i++) {
1776         bn[i] = 0;
1777     }
1778 }
1779
1780 /* parse number in null terminated string 'p' and return it in the
1781    current token */
1782 static void parse_number(const char *p)
1783 {
1784     int b, t, shift, frac_bits, s, exp_val, ch;
1785     char *q;
1786     unsigned int bn[BN_SIZE];
1787     double d;
1788
1789     /* number */
1790     q = token_buf;
1791     ch = *p++;
1792     t = ch;
1793     ch = *p++;
1794     *q++ = t;
1795     b = 10;
1796     if (t == '.') {
1797         goto float_frac_parse;
1798     } else if (t == '0') {
1799         if (ch == 'x' || ch == 'X') {
1800             q--;
1801             ch = *p++;
1802             b = 16;
1803         } else if (tcc_ext && (ch == 'b' || ch == 'B')) {
1804             q--;
1805             ch = *p++;
1806             b = 2;
1807         }
1808     }
1809     /* parse all digits. cannot check octal numbers at this stage
1810        because of floating point constants */
1811     while (1) {
1812         if (ch >= 'a' && ch <= 'f')
1813             t = ch - 'a' + 10;
1814         else if (ch >= 'A' && ch <= 'F')
1815             t = ch - 'A' + 10;
1816         else if (isnum(ch))
1817             t = ch - '0';
1818         else
1819             break;
1820         if (t >= b)
1821             break;
1822         if (q >= token_buf + STRING_MAX_SIZE) {
1823         num_too_long:
1824             tcc_error("number too long");
1825         }
1826         *q++ = ch;
1827         ch = *p++;
1828     }
1829     if (ch == '.' ||
1830         ((ch == 'e' || ch == 'E') && b == 10) ||
1831         ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
1832         if (b != 10) {
1833             /* NOTE: strtox should support that for hexa numbers, but
1834                non ISOC99 libcs do not support it, so we prefer to do
1835                it by hand */
1836             /* hexadecimal or binary floats */
1837             /* XXX: handle overflows */
1838             *q = '\0';
1839             if (b == 16)
1840                 shift = 4;
1841             else
1842                 shift = 2;
1843             bn_zero(bn);
1844             q = token_buf;
1845             while (1) {
1846                 t = *q++;
1847                 if (t == '\0') {
1848                     break;
1849                 } else if (t >= 'a') {
1850                     t = t - 'a' + 10;
1851                 } else if (t >= 'A') {
1852                     t = t - 'A' + 10;
1853                 } else {
1854                     t = t - '0';
1855                 }
1856                 bn_lshift(bn, shift, t);
1857             }
1858             frac_bits = 0;
1859             if (ch == '.') {
1860                 ch = *p++;
1861                 while (1) {
1862                     t = ch;
1863                     if (t >= 'a' && t <= 'f') {
1864                         t = t - 'a' + 10;
1865                     } else if (t >= 'A' && t <= 'F') {
1866                         t = t - 'A' + 10;
1867                     } else if (t >= '0' && t <= '9') {
1868                         t = t - '0';
1869                     } else {
1870                         break;
1871                     }
1872                     if (t >= b)
1873                         tcc_error("invalid digit");
1874                     bn_lshift(bn, shift, t);
1875                     frac_bits += shift;
1876                     ch = *p++;
1877                 }
1878             }
1879             if (ch != 'p' && ch != 'P')
1880                 expect("exponent");
1881             ch = *p++;
1882             s = 1;
1883             exp_val = 0;
1884             if (ch == '+') {
1885                 ch = *p++;
1886             } else if (ch == '-') {
1887                 s = -1;
1888                 ch = *p++;
1889             }
1890             if (ch < '0' || ch > '9')
1891                 expect("exponent digits");
1892             while (ch >= '0' && ch <= '9') {
1893                 exp_val = exp_val * 10 + ch - '0';
1894                 ch = *p++;
1895             }
1896             exp_val = exp_val * s;
1897
1898             /* now we can generate the number */
1899             /* XXX: should patch directly float number */
1900             d = (double)bn[1] * 4294967296.0 + (double)bn[0];
1901             d = ldexp(d, exp_val - frac_bits);
1902             t = toup(ch);
1903             if (t == 'F') {
1904                 ch = *p++;
1905                 tok = TOK_CFLOAT;
1906                 /* float : should handle overflow */
1907                 tokc.f = (float)d;
1908             } else if (t == 'L') {
1909                 ch = *p++;
1910 #ifdef TCC_TARGET_PE
1911                 tok = TOK_CDOUBLE;
1912                 tokc.d = d;
1913 #else
1914                 tok = TOK_CLDOUBLE;
1915                 /* XXX: not large enough */
1916                 tokc.ld = (long double)d;
1917 #endif
1918             } else {
1919                 tok = TOK_CDOUBLE;
1920                 tokc.d = d;
1921             }
1922         } else {
1923             /* decimal floats */
1924             if (ch == '.') {
1925                 if (q >= token_buf + STRING_MAX_SIZE)
1926                     goto num_too_long;
1927                 *q++ = ch;
1928                 ch = *p++;
1929             float_frac_parse:
1930                 while (ch >= '0' && ch <= '9') {
1931                     if (q >= token_buf + STRING_MAX_SIZE)
1932                         goto num_too_long;
1933                     *q++ = ch;
1934                     ch = *p++;
1935                 }
1936             }
1937             if (ch == 'e' || ch == 'E') {
1938                 if (q >= token_buf + STRING_MAX_SIZE)
1939                     goto num_too_long;
1940                 *q++ = ch;
1941                 ch = *p++;
1942                 if (ch == '-' || ch == '+') {
1943                     if (q >= token_buf + STRING_MAX_SIZE)
1944                         goto num_too_long;
1945                     *q++ = ch;
1946                     ch = *p++;
1947                 }
1948                 if (ch < '0' || ch > '9')
1949                     expect("exponent digits");
1950                 while (ch >= '0' && ch <= '9') {
1951                     if (q >= token_buf + STRING_MAX_SIZE)
1952                         goto num_too_long;
1953                     *q++ = ch;
1954                     ch = *p++;
1955                 }
1956             }
1957             *q = '\0';
1958             t = toup(ch);
1959             errno = 0;
1960             if (t == 'F') {
1961                 ch = *p++;
1962                 tok = TOK_CFLOAT;
1963                 tokc.f = strtof(token_buf, NULL);
1964             } else if (t == 'L') {
1965                 ch = *p++;
1966 #ifdef TCC_TARGET_PE
1967                 tok = TOK_CDOUBLE;
1968                 tokc.d = strtod(token_buf, NULL);
1969 #else
1970                 tok = TOK_CLDOUBLE;
1971                 tokc.ld = strtold(token_buf, NULL);
1972 #endif
1973             } else {
1974                 tok = TOK_CDOUBLE;
1975                 tokc.d = strtod(token_buf, NULL);
1976             }
1977         }
1978     } else {
1979         unsigned long long n, n1;
1980         int lcount, ucount;
1981
1982         /* integer number */
1983         *q = '\0';
1984         q = token_buf;
1985         if (b == 10 && *q == '0') {
1986             b = 8;
1987             q++;
1988         }
1989         n = 0;
1990         while(1) {
1991             t = *q++;
1992             /* no need for checks except for base 10 / 8 errors */
1993             if (t == '\0') {
1994                 break;
1995             } else if (t >= 'a') {
1996                 t = t - 'a' + 10;
1997             } else if (t >= 'A') {
1998                 t = t - 'A' + 10;
1999             } else {
2000                 t = t - '0';
2001                 if (t >= b)
2002                     tcc_error("invalid digit");
2003             }
2004             n1 = n;
2005             n = n * b + t;
2006             /* detect overflow */
2007             /* XXX: this test is not reliable */
2008             if (n < n1)
2009                 tcc_error("integer constant overflow");
2010         }
2011
2012         /* XXX: not exactly ANSI compliant */
2013         if ((n & 0xffffffff00000000LL) != 0) {
2014             if ((n >> 63) != 0)
2015                 tok = TOK_CULLONG;
2016             else
2017                 tok = TOK_CLLONG;
2018         } else if (n > 0x7fffffff) {
2019             tok = TOK_CUINT;
2020         } else {
2021             tok = TOK_CINT;
2022         }
2023         lcount = 0;
2024         ucount = 0;
2025         for(;;) {
2026             t = toup(ch);
2027             if (t == 'L') {
2028                 if (lcount >= 2)
2029                     tcc_error("three 'l's in integer constant");
2030                 lcount++;
2031 #if !defined TCC_TARGET_X86_64 || defined TCC_TARGET_PE
2032                 if (lcount == 2) {
2033 #endif
2034                     if (tok == TOK_CINT)
2035                         tok = TOK_CLLONG;
2036                     else if (tok == TOK_CUINT)
2037                         tok = TOK_CULLONG;
2038 #if !defined TCC_TARGET_X86_64 || defined TCC_TARGET_PE
2039                 }
2040 #endif
2041                 ch = *p++;
2042             } else if (t == 'U') {
2043                 if (ucount >= 1)
2044                     tcc_error("two 'u's in integer constant");
2045                 ucount++;
2046                 if (tok == TOK_CINT)
2047                     tok = TOK_CUINT;
2048                 else if (tok == TOK_CLLONG)
2049                     tok = TOK_CULLONG;
2050                 ch = *p++;
2051             } else {
2052                 break;
2053             }
2054         }
2055         if (tok == TOK_CINT || tok == TOK_CUINT)
2056             tokc.ui = n;
2057         else
2058             tokc.ull = n;
2059     }
2060     if (ch)
2061         tcc_error("invalid number\n");
2062 }
2063
2064
2065 #define PARSE2(c1, tok1, c2, tok2)              \
2066     case c1:                                    \
2067         PEEKC(c, p);                            \
2068         if (c == c2) {                          \
2069             p++;                                \
2070             tok = tok2;                         \
2071         } else {                                \
2072             tok = tok1;                         \
2073         }                                       \
2074         break;
2075
2076 /* return next token without macro substitution */
2077 static inline void next_nomacro1(void)
2078 {
2079     int t, c, is_long;
2080     TokenSym *ts;
2081     uint8_t *p, *p1;
2082     unsigned int h;
2083
2084     p = file->buf_ptr;
2085  redo_no_start:
2086     c = *p;
2087     switch(c) {
2088     case ' ':
2089     case '\t':
2090         tok = c;
2091         p++;
2092         goto keep_tok_flags;
2093     case '\f':
2094     case '\v':
2095     case '\r':
2096         p++;
2097         goto redo_no_start;
2098     case '\\':
2099         /* first look if it is in fact an end of buffer */
2100         if (p >= file->buf_end) {
2101             file->buf_ptr = p;
2102             handle_eob();
2103             p = file->buf_ptr;
2104             if (p >= file->buf_end)
2105                 goto parse_eof;
2106             else
2107                 goto redo_no_start;
2108         } else {
2109             file->buf_ptr = p;
2110             ch = *p;
2111             handle_stray();
2112             p = file->buf_ptr;
2113             goto redo_no_start;
2114         }
2115     parse_eof:
2116         {
2117             TCCState *s1 = tcc_state;
2118             if ((parse_flags & PARSE_FLAG_LINEFEED)
2119                 && !(tok_flags & TOK_FLAG_EOF)) {
2120                 tok_flags |= TOK_FLAG_EOF;
2121                 tok = TOK_LINEFEED;
2122                 goto keep_tok_flags;
2123             } else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
2124                 tok = TOK_EOF;
2125             } else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
2126                 tcc_error("missing #endif");
2127             } else if (s1->include_stack_ptr == s1->include_stack) {
2128                 /* no include left : end of file. */
2129                 tok = TOK_EOF;
2130             } else {
2131                 tok_flags &= ~TOK_FLAG_EOF;
2132                 /* pop include file */
2133
2134                 /* test if previous '#endif' was after a #ifdef at
2135                    start of file */
2136                 if (tok_flags & TOK_FLAG_ENDIF) {
2137 #ifdef INC_DEBUG
2138                     printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
2139 #endif
2140                     add_cached_include(s1, file->inc_type, file->inc_filename,
2141                                        file->ifndef_macro_saved);
2142                 }
2143
2144                 /* add end of include file debug info */
2145                 if (tcc_state->do_debug) {
2146                     put_stabd(N_EINCL, 0, 0);
2147                 }
2148                 /* pop include stack */
2149                 tcc_close();
2150                 s1->include_stack_ptr--;
2151                 p = file->buf_ptr;
2152                 goto redo_no_start;
2153             }
2154         }
2155         break;
2156
2157     case '\n':
2158         file->line_num++;
2159         tok_flags |= TOK_FLAG_BOL;
2160         p++;
2161 maybe_newline:
2162         if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
2163             goto redo_no_start;
2164         tok = TOK_LINEFEED;
2165         goto keep_tok_flags;
2166
2167     case '#':
2168         /* XXX: simplify */
2169         PEEKC(c, p);
2170         if ((tok_flags & TOK_FLAG_BOL) &&
2171             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2172             file->buf_ptr = p;
2173             preprocess(tok_flags & TOK_FLAG_BOF);
2174             p = file->buf_ptr;
2175             goto maybe_newline;
2176         } else {
2177             if (c == '#') {
2178                 p++;
2179                 tok = TOK_TWOSHARPS;
2180             } else {
2181                 if (parse_flags & PARSE_FLAG_ASM_COMMENTS) {
2182                     p = parse_line_comment(p - 1);
2183                     goto redo_no_start;
2184                 } else {
2185                     tok = '#';
2186                 }
2187             }
2188         }
2189         break;
2190
2191     case 'a': case 'b': case 'c': case 'd':
2192     case 'e': case 'f': case 'g': case 'h':
2193     case 'i': case 'j': case 'k': case 'l':
2194     case 'm': case 'n': case 'o': case 'p':
2195     case 'q': case 'r': case 's': case 't':
2196     case 'u': case 'v': case 'w': case 'x':
2197     case 'y': case 'z':
2198     case 'A': case 'B': case 'C': case 'D':
2199     case 'E': case 'F': case 'G': case 'H':
2200     case 'I': case 'J': case 'K':
2201     case 'M': case 'N': case 'O': case 'P':
2202     case 'Q': case 'R': case 'S': case 'T':
2203     case 'U': case 'V': case 'W': case 'X':
2204     case 'Y': case 'Z':
2205     case '_':
2206     parse_ident_fast:
2207         p1 = p;
2208         h = TOK_HASH_INIT;
2209         h = TOK_HASH_FUNC(h, c);
2210         p++;
2211         for(;;) {
2212             c = *p;
2213             if (!isidnum_table[c-CH_EOF])
2214                 break;
2215             h = TOK_HASH_FUNC(h, c);
2216             p++;
2217         }
2218         if (c != '\\') {
2219             TokenSym **pts;
2220             int len;
2221
2222             /* fast case : no stray found, so we have the full token
2223                and we have already hashed it */
2224             len = p - p1;
2225             h &= (TOK_HASH_SIZE - 1);
2226             pts = &hash_ident[h];
2227             for(;;) {
2228                 ts = *pts;
2229                 if (!ts)
2230                     break;
2231                 if (ts->len == len && !memcmp(ts->str, p1, len))
2232                     goto token_found;
2233                 pts = &(ts->hash_next);
2234             }
2235             ts = tok_alloc_new(pts, p1, len);
2236         token_found: ;
2237         } else {
2238             /* slower case */
2239             cstr_reset(&tokcstr);
2240
2241             while (p1 < p) {
2242                 cstr_ccat(&tokcstr, *p1);
2243                 p1++;
2244             }
2245             p--;
2246             PEEKC(c, p);
2247         parse_ident_slow:
2248             while (isidnum_table[c-CH_EOF]) {
2249                 cstr_ccat(&tokcstr, c);
2250                 PEEKC(c, p);
2251             }
2252             ts = tok_alloc(tokcstr.data, tokcstr.size);
2253         }
2254         tok = ts->tok;
2255         break;
2256     case 'L':
2257         t = p[1];
2258         if (t != '\\' && t != '\'' && t != '\"') {
2259             /* fast case */
2260             goto parse_ident_fast;
2261         } else {
2262             PEEKC(c, p);
2263             if (c == '\'' || c == '\"') {
2264                 is_long = 1;
2265                 goto str_const;
2266             } else {
2267                 cstr_reset(&tokcstr);
2268                 cstr_ccat(&tokcstr, 'L');
2269                 goto parse_ident_slow;
2270             }
2271         }
2272         break;
2273     case '0': case '1': case '2': case '3':
2274     case '4': case '5': case '6': case '7':
2275     case '8': case '9':
2276
2277         cstr_reset(&tokcstr);
2278         /* after the first digit, accept digits, alpha, '.' or sign if
2279            prefixed by 'eEpP' */
2280     parse_num:
2281         for(;;) {
2282             t = c;
2283             cstr_ccat(&tokcstr, c);
2284             PEEKC(c, p);
2285             if (!(isnum(c) || isid(c) || c == '.' ||
2286                   ((c == '+' || c == '-') &&
2287                    (t == 'e' || t == 'E' || t == 'p' || t == 'P'))))
2288                 break;
2289         }
2290         /* We add a trailing '\0' to ease parsing */
2291         cstr_ccat(&tokcstr, '\0');
2292         tokc.cstr = &tokcstr;
2293         tok = TOK_PPNUM;
2294         break;
2295     case '.':
2296         /* special dot handling because it can also start a number */
2297         PEEKC(c, p);
2298         if (isnum(c)) {
2299             cstr_reset(&tokcstr);
2300             cstr_ccat(&tokcstr, '.');
2301             goto parse_num;
2302         } else if (c == '.') {
2303             PEEKC(c, p);
2304             if (c != '.')
2305                 expect("'.'");
2306             PEEKC(c, p);
2307             tok = TOK_DOTS;
2308         } else {
2309             tok = '.';
2310         }
2311         break;
2312     case '\'':
2313     case '\"':
2314         is_long = 0;
2315     str_const:
2316         {
2317             CString str;
2318             int sep;
2319
2320             sep = c;
2321
2322             /* parse the string */
2323             cstr_new(&str);
2324             p = parse_pp_string(p, sep, &str);
2325             cstr_ccat(&str, '\0');
2326
2327             /* eval the escape (should be done as TOK_PPNUM) */
2328             cstr_reset(&tokcstr);
2329             parse_escape_string(&tokcstr, str.data, is_long);
2330             cstr_free(&str);
2331
2332             if (sep == '\'') {
2333                 int char_size;
2334                 /* XXX: make it portable */
2335                 if (!is_long)
2336                     char_size = 1;
2337                 else
2338                     char_size = sizeof(nwchar_t);
2339                 if (tokcstr.size <= char_size)
2340                     tcc_error("empty character constant");
2341                 if (tokcstr.size > 2 * char_size)
2342                     tcc_warning("multi-character character constant");
2343                 if (!is_long) {
2344                     tokc.i = *(int8_t *)tokcstr.data;
2345                     tok = TOK_CCHAR;
2346                 } else {
2347                     tokc.i = *(nwchar_t *)tokcstr.data;
2348                     tok = TOK_LCHAR;
2349                 }
2350             } else {
2351                 tokc.cstr = &tokcstr;
2352                 if (!is_long)
2353                     tok = TOK_STR;
2354                 else
2355                     tok = TOK_LSTR;
2356             }
2357         }
2358         break;
2359
2360     case '<':
2361         PEEKC(c, p);
2362         if (c == '=') {
2363             p++;
2364             tok = TOK_LE;
2365         } else if (c == '<') {
2366             PEEKC(c, p);
2367             if (c == '=') {
2368                 p++;
2369                 tok = TOK_A_SHL;
2370             } else {
2371                 tok = TOK_SHL;
2372             }
2373         } else {
2374             tok = TOK_LT;
2375         }
2376         break;
2377
2378     case '>':
2379         PEEKC(c, p);
2380         if (c == '=') {
2381             p++;
2382             tok = TOK_GE;
2383         } else if (c == '>') {
2384             PEEKC(c, p);
2385             if (c == '=') {
2386                 p++;
2387                 tok = TOK_A_SAR;
2388             } else {
2389                 tok = TOK_SAR;
2390             }
2391         } else {
2392             tok = TOK_GT;
2393         }
2394         break;
2395
2396     case '&':
2397         PEEKC(c, p);
2398         if (c == '&') {
2399             p++;
2400             tok = TOK_LAND;
2401         } else if (c == '=') {
2402             p++;
2403             tok = TOK_A_AND;
2404         } else {
2405             tok = '&';
2406         }
2407         break;
2408
2409     case '|':
2410         PEEKC(c, p);
2411         if (c == '|') {
2412             p++;
2413             tok = TOK_LOR;
2414         } else if (c == '=') {
2415             p++;
2416             tok = TOK_A_OR;
2417         } else {
2418             tok = '|';
2419         }
2420         break;
2421
2422     case '+':
2423         PEEKC(c, p);
2424         if (c == '+') {
2425             p++;
2426             tok = TOK_INC;
2427         } else if (c == '=') {
2428             p++;
2429             tok = TOK_A_ADD;
2430         } else {
2431             tok = '+';
2432         }
2433         break;
2434
2435     case '-':
2436         PEEKC(c, p);
2437         if (c == '-') {
2438             p++;
2439             tok = TOK_DEC;
2440         } else if (c == '=') {
2441             p++;
2442             tok = TOK_A_SUB;
2443         } else if (c == '>') {
2444             p++;
2445             tok = TOK_ARROW;
2446         } else {
2447             tok = '-';
2448         }
2449         break;
2450
2451     PARSE2('!', '!', '=', TOK_NE)
2452     PARSE2('=', '=', '=', TOK_EQ)
2453     PARSE2('*', '*', '=', TOK_A_MUL)
2454     PARSE2('%', '%', '=', TOK_A_MOD)
2455     PARSE2('^', '^', '=', TOK_A_XOR)
2456
2457         /* comments or operator */
2458     case '/':
2459         PEEKC(c, p);
2460         if (c == '*') {
2461             p = parse_comment(p);
2462             /* comments replaced by a blank */
2463             tok = ' ';
2464             goto keep_tok_flags;
2465         } else if (c == '/') {
2466             p = parse_line_comment(p);
2467             tok = ' ';
2468             goto keep_tok_flags;
2469         } else if (c == '=') {
2470             p++;
2471             tok = TOK_A_DIV;
2472         } else {
2473             tok = '/';
2474         }
2475         break;
2476
2477         /* simple tokens */
2478     case '(':
2479     case ')':
2480     case '[':
2481     case ']':
2482     case '{':
2483     case '}':
2484     case ',':
2485     case ';':
2486     case ':':
2487     case '?':
2488     case '~':
2489     case '$': /* only used in assembler */
2490     case '@': /* dito */
2491         tok = c;
2492         p++;
2493         break;
2494     default:
2495         tcc_error("unrecognized character \\x%02x", c);
2496         break;
2497     }
2498     tok_flags = 0;
2499 keep_tok_flags:
2500     file->buf_ptr = p;
2501 #if defined(PARSE_DEBUG)
2502     printf("token = %s\n", get_tok_str(tok, &tokc));
2503 #endif
2504 }
2505
2506 /* return next token without macro substitution. Can read input from
2507    macro_ptr buffer */
2508 static void next_nomacro_spc(void)
2509 {
2510     if (macro_ptr) {
2511     redo:
2512         tok = *macro_ptr;
2513         if (tok) {
2514             TOK_GET(&tok, &macro_ptr, &tokc);
2515             if (tok == TOK_LINENUM) {
2516                 file->line_num = tokc.i;
2517                 goto redo;
2518             }
2519         }
2520     } else {
2521         next_nomacro1();
2522     }
2523 }
2524
2525 ST_FUNC void next_nomacro(void)
2526 {
2527     do {
2528         next_nomacro_spc();
2529     } while (is_space(tok));
2530 }
2531
2532 /* substitute args in macro_str and return allocated string */
2533 static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
2534 {
2535     int last_tok, t, spc;
2536     const int *st;
2537     Sym *s;
2538     CValue cval;
2539     TokenString str;
2540     CString cstr;
2541
2542     tok_str_new(&str);
2543     last_tok = 0;
2544     while(1) {
2545         TOK_GET(&t, &macro_str, &cval);
2546         if (!t)
2547             break;
2548         if (t == '#') {
2549             /* stringize */
2550             TOK_GET(&t, &macro_str, &cval);
2551             if (!t)
2552                 break;
2553             s = sym_find2(args, t);
2554             if (s) {
2555                 cstr_new(&cstr);
2556                 st = s->d;
2557                 spc = 0;
2558                 while (*st) {
2559                     TOK_GET(&t, &st, &cval);
2560                     if (!check_space(t, &spc))
2561                         cstr_cat(&cstr, get_tok_str(t, &cval));
2562                 }
2563                 cstr.size -= spc;
2564                 cstr_ccat(&cstr, '\0');
2565 #ifdef PP_DEBUG
2566                 printf("stringize: %s\n", (char *)cstr.data);
2567 #endif
2568                 /* add string */
2569                 cval.cstr = &cstr;
2570                 tok_str_add2(&str, TOK_STR, &cval);
2571                 cstr_free(&cstr);
2572             } else {
2573                 tok_str_add2(&str, t, &cval);
2574             }
2575         } else if (t >= TOK_IDENT) {
2576             s = sym_find2(args, t);
2577             if (s) {
2578                 st = s->d;
2579                 /* if '##' is present before or after, no arg substitution */
2580                 if (*macro_str == TOK_TWOSHARPS || last_tok == TOK_TWOSHARPS) {
2581                     /* special case for var arg macros : ## eats the
2582                        ',' if empty VA_ARGS variable. */
2583                     /* XXX: test of the ',' is not 100%
2584                        reliable. should fix it to avoid security
2585                        problems */
2586                     if (gnu_ext && s->type.t &&
2587                         last_tok == TOK_TWOSHARPS &&
2588                         str.len >= 2 && str.str[str.len - 2] == ',') {
2589                         if (*st == 0) {
2590                             /* suppress ',' '##' */
2591                             str.len -= 2;
2592                         } else {
2593                             /* suppress '##' and add variable */
2594                             str.len--;
2595                             goto add_var;
2596                         }
2597                     } else {
2598                         int t1;
2599                     add_var:
2600                         for(;;) {
2601                             TOK_GET(&t1, &st, &cval);
2602                             if (!t1)
2603                                 break;
2604                             tok_str_add2(&str, t1, &cval);
2605                         }
2606                     }
2607                 } else {
2608                     /* NOTE: the stream cannot be read when macro
2609                        substituing an argument */
2610                     macro_subst(&str, nested_list, st, NULL);
2611                 }
2612             } else {
2613                 tok_str_add(&str, t);
2614             }
2615         } else {
2616             tok_str_add2(&str, t, &cval);
2617         }
2618         last_tok = t;
2619     }
2620     tok_str_add(&str, 0);
2621     return str.str;
2622 }
2623
2624 static char const ab_month_name[12][4] =
2625 {
2626     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
2627     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
2628 };
2629
2630 /* do macro substitution of current token with macro 's' and add
2631    result to (tok_str,tok_len). 'nested_list' is the list of all
2632    macros we got inside to avoid recursing. Return non zero if no
2633    substitution needs to be done */
2634 static int macro_subst_tok(TokenString *tok_str,
2635                            Sym **nested_list, Sym *s, struct macro_level **can_read_stream)
2636 {
2637     Sym *args, *sa, *sa1;
2638     int mstr_allocated, parlevel, *mstr, t, t1, spc;
2639     const int *p;
2640     TokenString str;
2641     char *cstrval;
2642     CValue cval;
2643     CString cstr;
2644     char buf[32];
2645
2646     /* if symbol is a macro, prepare substitution */
2647     /* special macros */
2648     if (tok == TOK___LINE__) {
2649         snprintf(buf, sizeof(buf), "%d", file->line_num);
2650         cstrval = buf;
2651         t1 = TOK_PPNUM;
2652         goto add_cstr1;
2653     } else if (tok == TOK___FILE__) {
2654         cstrval = file->filename;
2655         goto add_cstr;
2656     } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
2657         time_t ti;
2658         struct tm *tm;
2659
2660         time(&ti);
2661         tm = localtime(&ti);
2662         if (tok == TOK___DATE__) {
2663             snprintf(buf, sizeof(buf), "%s %2d %d",
2664                      ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
2665         } else {
2666             snprintf(buf, sizeof(buf), "%02d:%02d:%02d",
2667                      tm->tm_hour, tm->tm_min, tm->tm_sec);
2668         }
2669         cstrval = buf;
2670     add_cstr:
2671         t1 = TOK_STR;
2672     add_cstr1:
2673         cstr_new(&cstr);
2674         cstr_cat(&cstr, cstrval);
2675         cstr_ccat(&cstr, '\0');
2676         cval.cstr = &cstr;
2677         tok_str_add2(tok_str, t1, &cval);
2678         cstr_free(&cstr);
2679     } else {
2680         mstr = s->d;
2681         mstr_allocated = 0;
2682         if (s->type.t == MACRO_FUNC) {
2683             /* NOTE: we do not use next_nomacro to avoid eating the
2684                next token. XXX: find better solution */
2685         redo:
2686             if (macro_ptr) {
2687                 p = macro_ptr;
2688                 while (is_space(t = *p) || TOK_LINEFEED == t)
2689                     ++p;
2690                 if (t == 0 && can_read_stream) {
2691                     /* end of macro stream: we must look at the token
2692                        after in the file */
2693                     struct macro_level *ml = *can_read_stream;
2694                     macro_ptr = NULL;
2695                     if (ml)
2696                     {
2697                         macro_ptr = ml->p;
2698                         ml->p = NULL;
2699                         *can_read_stream = ml -> prev;
2700                     }
2701                     /* also, end of scope for nested defined symbol */
2702                     (*nested_list)->v = -1;
2703                     goto redo;
2704                 }
2705             } else {
2706                 /* XXX: incorrect with comments */
2707                 ch = file->buf_ptr[0];
2708                 while (is_space(ch) || ch == '\n')
2709                     cinp();
2710                 t = ch;
2711             }
2712             if (t != '(') /* no macro subst */
2713                 return -1;
2714
2715             /* argument macro */
2716             next_nomacro();
2717             next_nomacro();
2718             args = NULL;
2719             sa = s->next;
2720             /* NOTE: empty args are allowed, except if no args */
2721             for(;;) {
2722                 /* handle '()' case */
2723                 if (!args && !sa && tok == ')')
2724                     break;
2725                 if (!sa)
2726                     tcc_error("macro '%s' used with too many args",
2727                           get_tok_str(s->v, 0));
2728                 tok_str_new(&str);
2729                 parlevel = spc = 0;
2730                 /* NOTE: non zero sa->t indicates VA_ARGS */
2731                 while ((parlevel > 0 ||
2732                         (tok != ')' &&
2733                          (tok != ',' || sa->type.t))) &&
2734                        tok != -1) {
2735                     if (tok == '(')
2736                         parlevel++;
2737                     else if (tok == ')')
2738                         parlevel--;
2739                     if (tok == TOK_LINEFEED)
2740                         tok = ' ';
2741                     if (!check_space(tok, &spc))
2742                         tok_str_add2(&str, tok, &tokc);
2743                     next_nomacro_spc();
2744                 }
2745                 str.len -= spc;
2746                 tok_str_add(&str, 0);
2747                 sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
2748                 sa1->d = str.str;
2749                 sa = sa->next;
2750                 if (tok == ')') {
2751                     /* special case for gcc var args: add an empty
2752                        var arg argument if it is omitted */
2753                     if (sa && sa->type.t && gnu_ext)
2754                         continue;
2755                     else
2756                         break;
2757                 }
2758                 if (tok != ',')
2759                     expect(",");
2760                 next_nomacro();
2761             }
2762             if (sa) {
2763                 tcc_error("macro '%s' used with too few args",
2764                       get_tok_str(s->v, 0));
2765             }
2766
2767             /* now subst each arg */
2768             mstr = macro_arg_subst(nested_list, mstr, args);
2769             /* free memory */
2770             sa = args;
2771             while (sa) {
2772                 sa1 = sa->prev;
2773                 tok_str_free(sa->d);
2774                 sym_free(sa);
2775                 sa = sa1;
2776             }
2777             mstr_allocated = 1;
2778         }
2779         sym_push2(nested_list, s->v, 0, 0);
2780         macro_subst(tok_str, nested_list, mstr, can_read_stream);
2781         /* pop nested defined symbol */
2782         sa1 = *nested_list;
2783         *nested_list = sa1->prev;
2784         sym_free(sa1);
2785         if (mstr_allocated)
2786             tok_str_free(mstr);
2787     }
2788     return 0;
2789 }
2790
2791 /* handle the '##' operator. Return NULL if no '##' seen. Otherwise
2792    return the resulting string (which must be freed). */
2793 static inline int *macro_twosharps(const int *macro_str)
2794 {
2795     const int *ptr;
2796     int t;
2797     CValue cval;
2798     TokenString macro_str1;
2799     CString cstr;
2800     int n, start_of_nosubsts;
2801
2802     /* we search the first '##' */
2803     for(ptr = macro_str;;) {
2804         TOK_GET(&t, &ptr, &cval);
2805         if (t == TOK_TWOSHARPS)
2806             break;
2807         /* nothing more to do if end of string */
2808         if (t == 0)
2809             return NULL;
2810     }
2811
2812     /* we saw '##', so we need more processing to handle it */
2813     start_of_nosubsts = -1;
2814     tok_str_new(&macro_str1);
2815     for(ptr = macro_str;;) {
2816         TOK_GET(&tok, &ptr, &tokc);
2817         if (tok == 0)
2818             break;
2819         if (tok == TOK_TWOSHARPS)
2820             continue;
2821         if (tok == TOK_NOSUBST && start_of_nosubsts < 0)
2822             start_of_nosubsts = macro_str1.len;
2823         while (*ptr == TOK_TWOSHARPS) {
2824             /* given 'a##b', remove nosubsts preceding 'a' */
2825             if (start_of_nosubsts >= 0)
2826                 macro_str1.len = start_of_nosubsts;
2827             /* given 'a##b', skip '##' */
2828             t = *++ptr;
2829             /* given 'a##b', remove nosubsts preceding 'b' */
2830             while (t == TOK_NOSUBST)
2831                 t = *++ptr;
2832
2833             if (t && t != TOK_TWOSHARPS) {
2834                 TOK_GET(&t, &ptr, &cval);
2835
2836                 /* We concatenate the two tokens */
2837                 cstr_new(&cstr);
2838                 cstr_cat(&cstr, get_tok_str(tok, &tokc));
2839                 n = cstr.size;
2840                 cstr_cat(&cstr, get_tok_str(t, &cval));
2841                 cstr_ccat(&cstr, '\0');
2842
2843                 tcc_open_bf(tcc_state, "<paste>", cstr.size);
2844                 memcpy(file->buffer, cstr.data, cstr.size);
2845                 for (;;) {
2846                     next_nomacro1();
2847                     if (0 == *file->buf_ptr)
2848                         break;
2849                     tok_str_add2(&macro_str1, tok, &tokc);
2850                     tcc_warning("pasting \"%.*s\" and \"%s\" does not give a valid preprocessing token",
2851                         n, cstr.data, (char*)cstr.data + n);
2852                 }
2853                 tcc_close();
2854                 cstr_reset(&cstr);
2855             }
2856         }
2857         if (tok != TOK_NOSUBST)
2858             start_of_nosubsts = -1;
2859         tok_str_add2(&macro_str1, tok, &tokc);
2860     }
2861     tok_str_add(&macro_str1, 0);
2862     return macro_str1.str;
2863 }
2864
2865
2866 /* do macro substitution of macro_str and add result to
2867    (tok_str,tok_len). 'nested_list' is the list of all macros we got
2868    inside to avoid recursing. */
2869 static void macro_subst(TokenString *tok_str, Sym **nested_list,
2870                         const int *macro_str, struct macro_level ** can_read_stream)
2871 {
2872     Sym *s;
2873     int *macro_str1;
2874     const int *ptr;
2875     int t, ret, spc;
2876     CValue cval;
2877     struct macro_level ml;
2878     int force_blank;
2879
2880     /* first scan for '##' operator handling */
2881     ptr = macro_str;
2882     macro_str1 = macro_twosharps(ptr);
2883
2884     if (macro_str1)
2885         ptr = macro_str1;
2886     spc = 0;
2887     force_blank = 0;
2888
2889     while (1) {
2890         /* NOTE: ptr == NULL can only happen if tokens are read from
2891            file stream due to a macro function call */
2892         if (ptr == NULL)
2893             break;
2894         TOK_GET(&t, &ptr, &cval);
2895         if (t == 0)
2896             break;
2897         if (t == TOK_NOSUBST) {
2898             /* following token has already been subst'd. just copy it on */
2899             tok_str_add2(tok_str, TOK_NOSUBST, NULL);
2900             TOK_GET(&t, &ptr, &cval);
2901             goto no_subst;
2902         }
2903         s = define_find(t);
2904         if (s != NULL) {
2905             /* if nested substitution, do nothing */
2906             if (sym_find2(*nested_list, t)) {
2907                 /* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
2908                 tok_str_add2(tok_str, TOK_NOSUBST, NULL);
2909                 goto no_subst;
2910             }
2911             ml.p = macro_ptr;
2912             if (can_read_stream)
2913                 ml.prev = *can_read_stream, *can_read_stream = &ml;
2914             macro_ptr = (int *)ptr;
2915             tok = t;
2916             ret = macro_subst_tok(tok_str, nested_list, s, can_read_stream);
2917             ptr = (int *)macro_ptr;
2918             macro_ptr = ml.p;
2919             if (can_read_stream && *can_read_stream == &ml)
2920                 *can_read_stream = ml.prev;
2921             if (ret != 0)
2922                 goto no_subst;
2923             if (parse_flags & PARSE_FLAG_SPACES)
2924                 force_blank = 1;
2925         } else {
2926         no_subst:
2927             if (force_blank) {
2928                 tok_str_add(tok_str, ' ');
2929                 spc = 1;
2930                 force_blank = 0;
2931             }
2932             if (!check_space(t, &spc))
2933                 tok_str_add2(tok_str, t, &cval);
2934         }
2935     }
2936     if (macro_str1)
2937         tok_str_free(macro_str1);
2938 }
2939
2940 /* return next token with macro substitution */
2941 ST_FUNC void next(void)
2942 {
2943     Sym *nested_list, *s;
2944     TokenString str;
2945     struct macro_level *ml;
2946
2947  redo:
2948     if (parse_flags & PARSE_FLAG_SPACES)
2949         next_nomacro_spc();
2950     else
2951         next_nomacro();
2952     if (!macro_ptr) {
2953         /* if not reading from macro substituted string, then try
2954            to substitute macros */
2955         if (tok >= TOK_IDENT &&
2956             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2957             s = define_find(tok);
2958             if (s) {
2959                 /* we have a macro: we try to substitute */
2960                 tok_str_new(&str);
2961                 nested_list = NULL;
2962                 ml = NULL;
2963                 if (macro_subst_tok(&str, &nested_list, s, &ml) == 0) {
2964                     /* substitution done, NOTE: maybe empty */
2965                     tok_str_add(&str, 0);
2966                     macro_ptr = str.str;
2967                     macro_ptr_allocated = str.str;
2968                     goto redo;
2969                 }
2970             }
2971         }
2972     } else {
2973         if (tok == 0) {
2974             /* end of macro or end of unget buffer */
2975             if (unget_buffer_enabled) {
2976                 macro_ptr = unget_saved_macro_ptr;
2977                 unget_buffer_enabled = 0;
2978             } else {
2979                 /* end of macro string: free it */
2980                 tok_str_free(macro_ptr_allocated);
2981                 macro_ptr_allocated = NULL;
2982                 macro_ptr = NULL;
2983             }
2984             goto redo;
2985         } else if (tok == TOK_NOSUBST) {
2986             /* discard preprocessor's nosubst markers */
2987             goto redo;
2988         }
2989     }
2990
2991     /* convert preprocessor tokens into C tokens */
2992     if (tok == TOK_PPNUM &&
2993         (parse_flags & PARSE_FLAG_TOK_NUM)) {
2994         parse_number((char *)tokc.cstr->data);
2995     }
2996 }
2997
2998 /* push back current token and set current token to 'last_tok'. Only
2999    identifier case handled for labels. */
3000 ST_INLN void unget_tok(int last_tok)
3001 {
3002     int i, n;
3003     int *q;
3004     unget_saved_macro_ptr = macro_ptr;
3005     unget_buffer_enabled = 1;
3006     q = unget_saved_buffer;
3007     macro_ptr = q;
3008     *q++ = tok;
3009     n = tok_ext_size(tok) - 1;
3010     for(i=0;i<n;i++)
3011         *q++ = tokc.tab[i];
3012     *q = 0; /* end of token string */
3013     tok = last_tok;
3014 }
3015
3016
3017 /* better than nothing, but needs extension to handle '-E' option
3018    correctly too */
3019 ST_FUNC void preprocess_init(TCCState *s1)
3020 {
3021     s1->include_stack_ptr = s1->include_stack;
3022     /* XXX: move that before to avoid having to initialize
3023        file->ifdef_stack_ptr ? */
3024     s1->ifdef_stack_ptr = s1->ifdef_stack;
3025     file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
3026
3027     /* XXX: not ANSI compliant: bound checking says error */
3028     vtop = vstack - 1;
3029     s1->pack_stack[0] = 0;
3030     s1->pack_stack_ptr = s1->pack_stack;
3031 }
3032
3033 ST_FUNC void preprocess_new()
3034 {
3035     int i, c;
3036     const char *p, *r;
3037
3038     /* init isid table */
3039     for(i=CH_EOF;i<256;i++)
3040         isidnum_table[i-CH_EOF] = isid(i) || isnum(i);
3041
3042     /* add all tokens */
3043     table_ident = NULL;
3044     memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
3045
3046     tok_ident = TOK_IDENT;
3047     p = tcc_keywords;
3048     while (*p) {
3049         r = p;
3050         for(;;) {
3051             c = *r++;
3052             if (c == '\0')
3053                 break;
3054         }
3055         tok_alloc(p, r - p - 1);
3056         p = r;
3057     }
3058 }
3059
3060 /* Preprocess the current file */
3061 ST_FUNC int tcc_preprocess(TCCState *s1)
3062 {
3063     Sym *define_start;
3064
3065     BufferedFile *file_ref, **iptr, **iptr_new;
3066     int token_seen, line_ref, d;
3067     const char *s;
3068
3069     preprocess_init(s1);
3070     define_start = define_stack;
3071     ch = file->buf_ptr[0];
3072     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
3073     parse_flags = PARSE_FLAG_ASM_COMMENTS | PARSE_FLAG_PREPROCESS |
3074         PARSE_FLAG_LINEFEED | PARSE_FLAG_SPACES;
3075     token_seen = 0;
3076     line_ref = 0;
3077     file_ref = NULL;
3078     iptr = s1->include_stack_ptr;
3079
3080     for (;;) {
3081         next();
3082         if (tok == TOK_EOF) {
3083             break;
3084         } else if (file != file_ref) {
3085             goto print_line;
3086         } else if (tok == TOK_LINEFEED) {
3087             if (!token_seen)
3088                 continue;
3089             ++line_ref;
3090             token_seen = 0;
3091         } else if (!token_seen) {
3092             d = file->line_num - line_ref;
3093             if (file != file_ref || d < 0 || d >= 8) {
3094 print_line:
3095                 iptr_new = s1->include_stack_ptr;
3096                 s = iptr_new > iptr ? " 1"
3097                   : iptr_new < iptr ? " 2"
3098                   : iptr_new > s1->include_stack ? " 3"
3099                   : ""
3100                   ;
3101                 iptr = iptr_new;
3102                 fprintf(s1->outfile, "# %d \"%s\"%s\n", file->line_num, file->filename, s);
3103             } else {
3104                 while (d)
3105                     fputs("\n", s1->outfile), --d;
3106             }
3107             line_ref = (file_ref = file)->line_num;
3108             token_seen = tok != TOK_LINEFEED;
3109             if (!token_seen)
3110                 continue;
3111         }
3112         fputs(get_tok_str(tok, &tokc), s1->outfile);
3113     }
3114     free_defines(define_start);
3115     return 0;
3116 }