tccpp.c

   1 /*
   2  *  TCC - Tiny C Compiler
   3  *
   4  *  Copyright (c) 2001-2004 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21
  22 static const char tcc_keywords[] =
  23 #define DEF(id, str) str "\0"
  24 #include "tcctok.h"
  25 #undef DEF
  26 ;
  27
  28 /* WARNING: the content of this string encodes token numbers */
  29 static char tok_two_chars[] = "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
  30
  31 /* true if isid(c) || isnum(c) */
  32 static unsigned char isidnum_table[256-CH_EOF];
  33
  34
  35 struct macro_level {
  36     struct macro_level *prev;
  37     int *p;
  38 };
  39
  40 static void next_nomacro(void);
  41 static void next_nomacro_spc(void);
  42 static void macro_subst(TokenString *tok_str, Sym **nested_list,
  43                         const int *macro_str, struct macro_level **can_read_stream);
  44
  45
  46 /* allocate a new token */
  47 static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
  48 {
  49     TokenSym *ts, **ptable;
  50     int i;
  51
  52     if (tok_ident >= SYM_FIRST_ANOM)
  53         error("memory full");
  54
  55     /* expand token table if needed */
  56     i = tok_ident - TOK_IDENT;
  57     if ((i % TOK_ALLOC_INCR) == 0) {
  58         ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
  59         if (!ptable)
  60             error("memory full");
  61         table_ident = ptable;
  62     }
  63
  64     ts = tcc_malloc(sizeof(TokenSym) + len);
  65     table_ident[i] = ts;
  66     ts->tok = tok_ident++;
  67     ts->sym_define = NULL;
  68     ts->sym_label = NULL;
  69     ts->sym_struct = NULL;
  70     ts->sym_identifier = NULL;
  71     ts->len = len;
  72     ts->hash_next = NULL;
  73     memcpy(ts->str, str, len);
  74     ts->str[len] = '\0';
  75     *pts = ts;
  76     return ts;
  77 }
  78
  79 #define TOK_HASH_INIT 1
  80 #define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))
  81
  82 /* find a token and add it if not found */
  83 static TokenSym *tok_alloc(const char *str, int len)
  84 {
  85     TokenSym *ts, **pts;
  86     int i;
  87     unsigned int h;
  88
  89     h = TOK_HASH_INIT;
  90     for(i=0;i<len;i++)
  91         h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
  92     h &= (TOK_HASH_SIZE - 1);
  93
  94     pts = &hash_ident[h];
  95     for(;;) {
  96         ts = *pts;
  97         if (!ts)
  98             break;
  99         if (ts->len == len && !memcmp(ts->str, str, len))
 100             return ts;
 101         pts = &(ts->hash_next);
 102     }
 103     return tok_alloc_new(pts, str, len);
 104 }
 105
 106 /* XXX: buffer overflow */
 107 /* XXX: float tokens */
 108 char *get_tok_str(int v, CValue *cv)
 109 {
 110     static char buf[STRING_MAX_SIZE + 1];
 111     static CString cstr_buf;
 112     CString *cstr;
 113     unsigned char *q;
 114     char *p;
 115     int i, len;
 116
 117     /* NOTE: to go faster, we give a fixed buffer for small strings */
 118     cstr_reset(&cstr_buf);
 119     cstr_buf.data = buf;
 120     cstr_buf.size_allocated = sizeof(buf);
 121     p = buf;
 122
 123     switch(v) {
 124     case TOK_CINT:
 125     case TOK_CUINT:
 126         /* XXX: not quite exact, but only useful for testing */
 127         sprintf(p, "%u", cv->ui);
 128         break;
 129     case TOK_CLLONG:
 130     case TOK_CULLONG:
 131         /* XXX: not quite exact, but only useful for testing  */
 132 #ifdef _WIN32
 133         sprintf(p, "%u", (unsigned)cv->ull);
 134 #else
 135         sprintf(p, "%Lu", cv->ull);
 136 #endif
 137         break;
 138     case TOK_LCHAR:
 139         cstr_ccat(&cstr_buf, 'L');
 140     case TOK_CCHAR:
 141         cstr_ccat(&cstr_buf, '\'');
 142         add_char(&cstr_buf, cv->i);
 143         cstr_ccat(&cstr_buf, '\'');
 144         cstr_ccat(&cstr_buf, '\0');
 145         break;
 146     case TOK_PPNUM:
 147         cstr = cv->cstr;
 148         len = cstr->size - 1;
 149         for(i=0;i<len;i++)
 150             add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
 151         cstr_ccat(&cstr_buf, '\0');
 152         break;
 153     case TOK_LSTR:
 154         cstr_ccat(&cstr_buf, 'L');
 155     case TOK_STR:
 156         cstr = cv->cstr;
 157         cstr_ccat(&cstr_buf, '\"');
 158         if (v == TOK_STR) {
 159             len = cstr->size - 1;
 160             for(i=0;i<len;i++)
 161                 add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
 162         } else {
 163             len = (cstr->size / sizeof(nwchar_t)) - 1;
 164             for(i=0;i<len;i++)
 165                 add_char(&cstr_buf, ((nwchar_t *)cstr->data)[i]);
 166         }
 167         cstr_ccat(&cstr_buf, '\"');
 168         cstr_ccat(&cstr_buf, '\0');
 169         break;
 170     case TOK_LT:
 171         v = '<';
 172         goto addv;
 173     case TOK_GT:
 174         v = '>';
 175         goto addv;
 176     case TOK_DOTS:
 177         return strcpy(p, "...");
 178     case TOK_A_SHL:
 179         return strcpy(p, "<<=");
 180     case TOK_A_SAR:
 181         return strcpy(p, ">>=");
 182     default:
 183         if (v < TOK_IDENT) {
 184             /* search in two bytes table */
 185             q = tok_two_chars;
 186             while (*q) {
 187                 if (q[2] == v) {
 188                     *p++ = q[0];
 189                     *p++ = q[1];
 190                     *p = '\0';
 191                     return buf;
 192                 }
 193                 q += 3;
 194             }
 195         addv:
 196             *p++ = v;
 197             *p = '\0';
 198         } else if (v < tok_ident) {
 199             return table_ident[v - TOK_IDENT]->str;
 200         } else if (v >= SYM_FIRST_ANOM) {
 201             /* special name for anonymous symbol */
 202             sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
 203         } else {
 204             /* should never happen */
 205             return NULL;
 206         }
 207         break;
 208     }
 209     return cstr_buf.data;
 210 }
 211
 212 /* fill input buffer and peek next char */
 213 static int tcc_peekc_slow(BufferedFile *bf)
 214 {
 215     int len;
 216     /* only tries to read if really end of buffer */
 217     if (bf->buf_ptr >= bf->buf_end) {
 218         if (bf->fd != -1) {
 219 #if defined(PARSE_DEBUG)
 220             len = 8;
 221 #else
 222             len = IO_BUF_SIZE;
 223 #endif
 224             len = read(bf->fd, bf->buffer, len);
 225             if (len < 0)
 226                 len = 0;
 227         } else {
 228             len = 0;
 229         }
 230         total_bytes += len;
 231         bf->buf_ptr = bf->buffer;
 232         bf->buf_end = bf->buffer + len;
 233         *bf->buf_end = CH_EOB;
 234     }
 235     if (bf->buf_ptr < bf->buf_end) {
 236         return bf->buf_ptr[0];
 237     } else {
 238         bf->buf_ptr = bf->buf_end;
 239         return CH_EOF;
 240     }
 241 }
 242
 243 /* return the current character, handling end of block if necessary
 244    (but not stray) */
 245 static int handle_eob(void)
 246 {
 247     return tcc_peekc_slow(file);
 248 }
 249
 250 /* read next char from current input file and handle end of input buffer */
 251 static inline void inp(void)
 252 {
 253     ch = *(++(file->buf_ptr));
 254     /* end of buffer/file handling */
 255     if (ch == CH_EOB)
 256         ch = handle_eob();
 257 }
 258
 259 /* handle '\[\r]\n' */
 260 static int handle_stray_noerror(void)
 261 {
 262     while (ch == '\\') {
 263         inp();
 264         if (ch == '\n') {
 265             file->line_num++;
 266             inp();
 267         } else if (ch == '\r') {
 268             inp();
 269             if (ch != '\n')
 270                 goto fail;
 271             file->line_num++;
 272             inp();
 273         } else {
 274         fail:
 275             return 1;
 276         }
 277     }
 278     return 0;
 279 }
 280
 281 static void handle_stray(void)
 282 {
 283     if (handle_stray_noerror())
 284         error("stray '\\' in program");
 285 }
 286
 287 /* skip the stray and handle the \\n case. Output an error if
 288    incorrect char after the stray */
 289 static int handle_stray1(uint8_t *p)
 290 {
 291     int c;
 292
 293     if (p >= file->buf_end) {
 294         file->buf_ptr = p;
 295         c = handle_eob();
 296         p = file->buf_ptr;
 297         if (c == '\\')
 298             goto parse_stray;
 299     } else {
 300     parse_stray:
 301         file->buf_ptr = p;
 302         ch = *p;
 303         handle_stray();
 304         p = file->buf_ptr;
 305         c = *p;
 306     }
 307     return c;
 308 }
 309
 310 /* handle just the EOB case, but not stray */
 311 #define PEEKC_EOB(c, p)\
 312 {\
 313     p++;\
 314     c = *p;\
 315     if (c == '\\') {\
 316         file->buf_ptr = p;\
 317         c = handle_eob();\
 318         p = file->buf_ptr;\
 319     }\
 320 }
 321
 322 /* handle the complicated stray case */
 323 #define PEEKC(c, p)\
 324 {\
 325     p++;\
 326     c = *p;\
 327     if (c == '\\') {\
 328         c = handle_stray1(p);\
 329         p = file->buf_ptr;\
 330     }\
 331 }
 332
 333 /* input with '\[\r]\n' handling. Note that this function cannot
 334    handle other characters after '\', so you cannot call it inside
 335    strings or comments */
 336 static void minp(void)
 337 {
 338     inp();
 339     if (ch == '\\')
 340         handle_stray();
 341 }
 342
 343
 344 /* single line C++ comments */
 345 static uint8_t *parse_line_comment(uint8_t *p)
 346 {
 347     int c;
 348
 349     p++;
 350     for(;;) {
 351         c = *p;
 352     redo:
 353         if (c == '\n' || c == CH_EOF) {
 354             break;
 355         } else if (c == '\\') {
 356             file->buf_ptr = p;
 357             c = handle_eob();
 358             p = file->buf_ptr;
 359             if (c == '\\') {
 360                 PEEKC_EOB(c, p);
 361                 if (c == '\n') {
 362                     file->line_num++;
 363                     PEEKC_EOB(c, p);
 364                 } else if (c == '\r') {
 365                     PEEKC_EOB(c, p);
 366                     if (c == '\n') {
 367                         file->line_num++;
 368                         PEEKC_EOB(c, p);
 369                     }
 370                 }
 371             } else {
 372                 goto redo;
 373             }
 374         } else {
 375             p++;
 376         }
 377     }
 378     return p;
 379 }
 380
 381 /* C comments */
 382 static uint8_t *parse_comment(uint8_t *p)
 383 {
 384     int c;
 385
 386     p++;
 387     for(;;) {
 388         /* fast skip loop */
 389         for(;;) {
 390             c = *p;
 391             if (c == '\n' || c == '*' || c == '\\')
 392                 break;
 393             p++;
 394             c = *p;
 395             if (c == '\n' || c == '*' || c == '\\')
 396                 break;
 397             p++;
 398         }
 399         /* now we can handle all the cases */
 400         if (c == '\n') {
 401             file->line_num++;
 402             p++;
 403         } else if (c == '*') {
 404             p++;
 405             for(;;) {
 406                 c = *p;
 407                 if (c == '*') {
 408                     p++;
 409                 } else if (c == '/') {
 410                     goto end_of_comment;
 411                 } else if (c == '\\') {
 412                     file->buf_ptr = p;
 413                     c = handle_eob();
 414                     p = file->buf_ptr;
 415                     if (c == '\\') {
 416                         /* skip '\[\r]\n', otherwise just skip the stray */
 417                         while (c == '\\') {
 418                             PEEKC_EOB(c, p);
 419                             if (c == '\n') {
 420                                 file->line_num++;
 421                                 PEEKC_EOB(c, p);
 422                             } else if (c == '\r') {
 423                                 PEEKC_EOB(c, p);
 424                                 if (c == '\n') {
 425                                     file->line_num++;
 426                                     PEEKC_EOB(c, p);
 427                                 }
 428                             } else {
 429                                 goto after_star;
 430                             }
 431                         }
 432                     }
 433                 } else {
 434                     break;
 435                 }
 436             }
 437         after_star: ;
 438         } else {
 439             /* stray, eob or eof */
 440             file->buf_ptr = p;
 441             c = handle_eob();
 442             p = file->buf_ptr;
 443             if (c == CH_EOF) {
 444                 error("unexpected end of file in comment");
 445             } else if (c == '\\') {
 446                 p++;
 447             }
 448         }
 449     }
 450  end_of_comment:
 451     p++;
 452     return p;
 453 }
 454
 455 #define cinp minp
 456
 457 static inline void skip_spaces(void)
 458 {
 459     while (is_space(ch))
 460         cinp();
 461 }
 462
 463 static inline int check_space(int t, int *spc)
 464 {
 465     if (is_space(t)) {
 466         if (*spc)
 467             return 1;
 468         *spc = 1;
 469     } else
 470         *spc = 0;
 471     return 0;
 472 }
 473
 474 /* parse a string without interpreting escapes */
 475 static uint8_t *parse_pp_string(uint8_t *p,
 476                                 int sep, CString *str)
 477 {
 478     int c;
 479     p++;
 480     for(;;) {
 481         c = *p;
 482         if (c == sep) {
 483             break;
 484         } else if (c == '\\') {
 485             file->buf_ptr = p;
 486             c = handle_eob();
 487             p = file->buf_ptr;
 488             if (c == CH_EOF) {
 489             unterminated_string:
 490                 /* XXX: indicate line number of start of string */
 491                 error("missing terminating %c character", sep);
 492             } else if (c == '\\') {
 493                 /* escape : just skip \[\r]\n */
 494                 PEEKC_EOB(c, p);
 495                 if (c == '\n') {
 496                     file->line_num++;
 497                     p++;
 498                 } else if (c == '\r') {
 499                     PEEKC_EOB(c, p);
 500                     if (c != '\n')
 501                         expect("'\n' after '\r'");
 502                     file->line_num++;
 503                     p++;
 504                 } else if (c == CH_EOF) {
 505                     goto unterminated_string;
 506                 } else {
 507                     if (str) {
 508                         cstr_ccat(str, '\\');
 509                         cstr_ccat(str, c);
 510                     }
 511                     p++;
 512                 }
 513             }
 514         } else if (c == '\n') {
 515             file->line_num++;
 516             goto add_char;
 517         } else if (c == '\r') {
 518             PEEKC_EOB(c, p);
 519             if (c != '\n') {
 520                 if (str)
 521                     cstr_ccat(str, '\r');
 522             } else {
 523                 file->line_num++;
 524                 goto add_char;
 525             }
 526         } else {
 527         add_char:
 528             if (str)
 529                 cstr_ccat(str, c);
 530             p++;
 531         }
 532     }
 533     p++;
 534     return p;
 535 }
 536
 537 /* skip block of text until #else, #elif or #endif. skip also pairs of
 538    #if/#endif */
 539 void preprocess_skip(void)
 540 {
 541     int a, start_of_line, c, in_warn_or_error;
 542     uint8_t *p;
 543
 544     p = file->buf_ptr;
 545     a = 0;
 546 redo_start:
 547     start_of_line = 1;
 548     in_warn_or_error = 0;
 549     for(;;) {
 550     redo_no_start:
 551         c = *p;
 552         switch(c) {
 553         case ' ':
 554         case '\t':
 555         case '\f':
 556         case '\v':
 557         case '\r':
 558             p++;
 559             goto redo_no_start;
 560         case '\n':
 561             file->line_num++;
 562             p++;
 563             goto redo_start;
 564         case '\\':
 565             file->buf_ptr = p;
 566             c = handle_eob();
 567             if (c == CH_EOF) {
 568                 expect("#endif");
 569             } else if (c == '\\') {
 570                 ch = file->buf_ptr[0];
 571                 handle_stray_noerror();
 572             }
 573             p = file->buf_ptr;
 574             goto redo_no_start;
 575         /* skip strings */
 576         case '\"':
 577         case '\'':
 578             if (in_warn_or_error)
 579                 goto _default;
 580             p = parse_pp_string(p, c, NULL);
 581             break;
 582         /* skip comments */
 583         case '/':
 584             if (in_warn_or_error)
 585                 goto _default;
 586             file->buf_ptr = p;
 587             ch = *p;
 588             minp();
 589             p = file->buf_ptr;
 590             if (ch == '*') {
 591                 p = parse_comment(p);
 592             } else if (ch == '/') {
 593                 p = parse_line_comment(p);
 594             }
 595             break;
 596         case '#':
 597             p++;
 598             if (start_of_line) {
 599                 file->buf_ptr = p;
 600                 next_nomacro();
 601                 p = file->buf_ptr;
 602                 if (a == 0 &&
 603                     (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
 604                     goto the_end;
 605                 if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
 606                     a++;
 607                 else if (tok == TOK_ENDIF)
 608                     a--;
 609                 else if( tok == TOK_ERROR || tok == TOK_WARNING)
 610                     in_warn_or_error = 1;
 611             }
 612             break;
 613 _default:
 614         default:
 615             p++;
 616             break;
 617         }
 618         start_of_line = 0;
 619     }
 620  the_end: ;
 621     file->buf_ptr = p;
 622 }
 623
 624 /* ParseState handling */
 625
 626 /* XXX: currently, no include file info is stored. Thus, we cannot display
 627    accurate messages if the function or data definition spans multiple
 628    files */
 629
 630 /* save current parse state in 's' */
 631 void save_parse_state(ParseState *s)
 632 {
 633     s->line_num = file->line_num;
 634     s->macro_ptr = macro_ptr;
 635     s->tok = tok;
 636     s->tokc = tokc;
 637 }
 638
 639 /* restore parse state from 's' */
 640 void restore_parse_state(ParseState *s)
 641 {
 642     file->line_num = s->line_num;
 643     macro_ptr = s->macro_ptr;
 644     tok = s->tok;
 645     tokc = s->tokc;
 646 }
 647
 648 /* return the number of additional 'ints' necessary to store the
 649    token */
 650 static inline int tok_ext_size(int t)
 651 {
 652     switch(t) {
 653         /* 4 bytes */
 654     case TOK_CINT:
 655     case TOK_CUINT:
 656     case TOK_CCHAR:
 657     case TOK_LCHAR:
 658     case TOK_CFLOAT:
 659     case TOK_LINENUM:
 660         return 1;
 661     case TOK_STR:
 662     case TOK_LSTR:
 663     case TOK_PPNUM:
 664         error("unsupported token");
 665         return 1;
 666     case TOK_CDOUBLE:
 667     case TOK_CLLONG:
 668     case TOK_CULLONG:
 669         return 2;
 670     case TOK_CLDOUBLE:
 671         return LDOUBLE_SIZE / 4;
 672     default:
 673         return 0;
 674     }
 675 }
 676
 677 /* token string handling */
 678
 679 static inline void tok_str_new(TokenString *s)
 680 {
 681     s->str = NULL;
 682     s->len = 0;
 683     s->allocated_len = 0;
 684     s->last_line_num = -1;
 685 }
 686
 687 static void tok_str_free(int *str)
 688 {
 689     tcc_free(str);
 690 }
 691
 692 static int *tok_str_realloc(TokenString *s)
 693 {
 694     int *str, len;
 695
 696     if (s->allocated_len == 0) {
 697         len = 8;
 698     } else {
 699         len = s->allocated_len * 2;
 700     }
 701     str = tcc_realloc(s->str, len * sizeof(int));
 702     if (!str)
 703         error("memory full");
 704     s->allocated_len = len;
 705     s->str = str;
 706     return str;
 707 }
 708
 709 static void tok_str_add(TokenString *s, int t)
 710 {
 711     int len, *str;
 712
 713     len = s->len;
 714     str = s->str;
 715     if (len >= s->allocated_len)
 716         str = tok_str_realloc(s);
 717     str[len++] = t;
 718     s->len = len;
 719 }
 720
 721 static void tok_str_add2(TokenString *s, int t, CValue *cv)
 722 {
 723     int len, *str;
 724
 725     len = s->len;
 726     str = s->str;
 727
 728     /* allocate space for worst case */
 729     if (len + TOK_MAX_SIZE > s->allocated_len)
 730         str = tok_str_realloc(s);
 731     str[len++] = t;
 732     switch(t) {
 733     case TOK_CINT:
 734     case TOK_CUINT:
 735     case TOK_CCHAR:
 736     case TOK_LCHAR:
 737     case TOK_CFLOAT:
 738     case TOK_LINENUM:
 739         str[len++] = cv->tab[0];
 740         break;
 741     case TOK_PPNUM:
 742     case TOK_STR:
 743     case TOK_LSTR:
 744         {
 745             int nb_words;
 746             CString *cstr;
 747
 748             nb_words = (sizeof(CString) + cv->cstr->size + 3) >> 2;
 749             while ((len + nb_words) > s->allocated_len)
 750                 str = tok_str_realloc(s);
 751             cstr = (CString *)(str + len);
 752             cstr->data = NULL;
 753             cstr->size = cv->cstr->size;
 754             cstr->data_allocated = NULL;
 755             cstr->size_allocated = cstr->size;
 756             memcpy((char *)cstr + sizeof(CString),
 757                    cv->cstr->data, cstr->size);
 758             len += nb_words;
 759         }
 760         break;
 761     case TOK_CDOUBLE:
 762     case TOK_CLLONG:
 763     case TOK_CULLONG:
 764 #if LDOUBLE_SIZE == 8
 765     case TOK_CLDOUBLE:
 766 #endif
 767         str[len++] = cv->tab[0];
 768         str[len++] = cv->tab[1];
 769         break;
 770 #if LDOUBLE_SIZE == 12
 771     case TOK_CLDOUBLE:
 772         str[len++] = cv->tab[0];
 773         str[len++] = cv->tab[1];
 774         str[len++] = cv->tab[2];
 775 #elif LDOUBLE_SIZE == 16
 776     case TOK_CLDOUBLE:
 777         str[len++] = cv->tab[0];
 778         str[len++] = cv->tab[1];
 779         str[len++] = cv->tab[2];
 780         str[len++] = cv->tab[3];
 781 #elif LDOUBLE_SIZE != 8
 782 #error add long double size support
 783 #endif
 784         break;
 785     default:
 786         break;
 787     }
 788     s->len = len;
 789 }
 790
 791 /* add the current parse token in token string 's' */
 792 static void tok_str_add_tok(TokenString *s)
 793 {
 794     CValue cval;
 795
 796     /* save line number info */
 797     if (file->line_num != s->last_line_num) {
 798         s->last_line_num = file->line_num;
 799         cval.i = s->last_line_num;
 800         tok_str_add2(s, TOK_LINENUM, &cval);
 801     }
 802     tok_str_add2(s, tok, &tokc);
 803 }
 804
 805 #if LDOUBLE_SIZE == 16
 806 #define LDOUBLE_GET(p, cv)                      \
 807         cv.tab[0] = p[0];                       \
 808         cv.tab[1] = p[1];                       \
 809         cv.tab[2] = p[2];                       \
 810         cv.tab[3] = p[3];
 811 #elif LDOUBLE_SIZE == 12
 812 #define LDOUBLE_GET(p, cv)                      \
 813         cv.tab[0] = p[0];                       \
 814         cv.tab[1] = p[1];                       \
 815         cv.tab[2] = p[2];
 816 #elif LDOUBLE_SIZE == 8
 817 #define LDOUBLE_GET(p, cv)                      \
 818         cv.tab[0] = p[0];                       \
 819         cv.tab[1] = p[1];
 820 #else
 821 #error add long double size support
 822 #endif
 823
 824
 825 /* get a token from an integer array and increment pointer
 826    accordingly. we code it as a macro to avoid pointer aliasing. */
 827 #define TOK_GET(t, p, cv)                       \
 828 {                                               \
 829     t = *p++;                                   \
 830     switch(t) {                                 \
 831     case TOK_CINT:                              \
 832     case TOK_CUINT:                             \
 833     case TOK_CCHAR:                             \
 834     case TOK_LCHAR:                             \
 835     case TOK_CFLOAT:                            \
 836     case TOK_LINENUM:                           \
 837         cv.tab[0] = *p++;                       \
 838         break;                                  \
 839     case TOK_STR:                               \
 840     case TOK_LSTR:                              \
 841     case TOK_PPNUM:                             \
 842         cv.cstr = (CString *)p;                 \
 843         cv.cstr->data = (char *)p + sizeof(CString);\
 844         p += (sizeof(CString) + cv.cstr->size + 3) >> 2;\
 845         break;                                  \
 846     case TOK_CDOUBLE:                           \
 847     case TOK_CLLONG:                            \
 848     case TOK_CULLONG:                           \
 849         cv.tab[0] = p[0];                       \
 850         cv.tab[1] = p[1];                       \
 851         p += 2;                                 \
 852         break;                                  \
 853     case TOK_CLDOUBLE:                          \
 854         LDOUBLE_GET(p, cv);                     \
 855         p += LDOUBLE_SIZE / 4;                  \
 856         break;                                  \
 857     default:                                    \
 858         break;                                  \
 859     }                                           \
 860 }
 861
 862 /* defines handling */
 863 static inline void define_push(int v, int macro_type, int *str, Sym *first_arg)
 864 {
 865     Sym *s;
 866
 867     s = sym_push2(&define_stack, v, macro_type, 0);
 868     s->d = str;
 869     s->next = first_arg;
 870     table_ident[v - TOK_IDENT]->sym_define = s;
 871 }
 872
 873 /* undefined a define symbol. Its name is just set to zero */
 874 static void define_undef(Sym *s)
 875 {
 876     int v;
 877     v = s->v;
 878     if (v >= TOK_IDENT && v < tok_ident)
 879         table_ident[v - TOK_IDENT]->sym_define = NULL;
 880     s->v = 0;
 881 }
 882
 883 static inline Sym *define_find(int v)
 884 {
 885     v -= TOK_IDENT;
 886     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
 887         return NULL;
 888     return table_ident[v]->sym_define;
 889 }
 890
 891 /* free define stack until top reaches 'b' */
 892 static void free_defines(Sym *b)
 893 {
 894     Sym *top, *top1;
 895     int v;
 896
 897     top = define_stack;
 898     while (top != b) {
 899         top1 = top->prev;
 900         /* do not free args or predefined defines */
 901         if (top->d)
 902             tok_str_free(top->d);
 903         v = top->v;
 904         if (v >= TOK_IDENT && v < tok_ident)
 905             table_ident[v - TOK_IDENT]->sym_define = NULL;
 906         sym_free(top);
 907         top = top1;
 908     }
 909     define_stack = b;
 910 }
 911
 912 /* label lookup */
 913 static Sym *label_find(int v)
 914 {
 915     v -= TOK_IDENT;
 916     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
 917         return NULL;
 918     return table_ident[v]->sym_label;
 919 }
 920
 921 static Sym *label_push(Sym **ptop, int v, int flags)
 922 {
 923     Sym *s, **ps;
 924     s = sym_push2(ptop, v, 0, 0);
 925     s->r = flags;
 926     ps = &table_ident[v - TOK_IDENT]->sym_label;
 927     if (ptop == &global_label_stack) {
 928         /* modify the top most local identifier, so that
 929            sym_identifier will point to 's' when popped */
 930         while (*ps != NULL)
 931             ps = &(*ps)->prev_tok;
 932     }
 933     s->prev_tok = *ps;
 934     *ps = s;
 935     return s;
 936 }
 937
 938 /* pop labels until element last is reached. Look if any labels are
 939    undefined. Define symbols if '&&label' was used. */
 940 static void label_pop(Sym **ptop, Sym *slast)
 941 {
 942     Sym *s, *s1;
 943     for(s = *ptop; s != slast; s = s1) {
 944         s1 = s->prev;
 945         if (s->r == LABEL_DECLARED) {
 946             warning("label '%s' declared but not used", get_tok_str(s->v, NULL));
 947         } else if (s->r == LABEL_FORWARD) {
 948                 error("label '%s' used but not defined",
 949                       get_tok_str(s->v, NULL));
 950         } else {
 951             if (s->c) {
 952                 /* define corresponding symbol. A size of
 953                    1 is put. */
 954                 put_extern_sym(s, cur_text_section, s->jnext, 1);
 955             }
 956         }
 957         /* remove label */
 958         table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
 959         sym_free(s);
 960     }
 961     *ptop = slast;
 962 }
 963
 964 /* eval an expression for #if/#elif */
 965 static int expr_preprocess(void)
 966 {
 967     int c, t;
 968     TokenString str;
 969
 970     tok_str_new(&str);
 971     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
 972         next(); /* do macro subst */
 973         if (tok == TOK_DEFINED) {
 974             next_nomacro();
 975             t = tok;
 976             if (t == '(')
 977                 next_nomacro();
 978             c = define_find(tok) != 0;
 979             if (t == '(')
 980                 next_nomacro();
 981             tok = TOK_CINT;
 982             tokc.i = c;
 983         } else if (tok >= TOK_IDENT) {
 984             /* if undefined macro */
 985             tok = TOK_CINT;
 986             tokc.i = 0;
 987         }
 988         tok_str_add_tok(&str);
 989     }
 990     tok_str_add(&str, -1); /* simulate end of file */
 991     tok_str_add(&str, 0);
 992     /* now evaluate C constant expression */
 993     macro_ptr = str.str;
 994     next();
 995     c = expr_const();
 996     macro_ptr = NULL;
 997     tok_str_free(str.str);
 998     return c != 0;
 999 }
1000
1001 #if defined(PARSE_DEBUG) || defined(PP_DEBUG)
1002 static void tok_print(int *str)
1003 {
1004     int t;
1005     CValue cval;
1006
1007     printf("<");
1008     while (1) {
1009         TOK_GET(t, str, cval);
1010         if (!t)
1011             break;
1012         printf("%s", get_tok_str(t, &cval));
1013     }
1014     printf(">\n");
1015 }
1016 #endif
1017
1018 /* parse after #define */
1019 static void parse_define(void)
1020 {
1021     Sym *s, *first, **ps;
1022     int v, t, varg, is_vaargs, spc;
1023     TokenString str;
1024
1025     v = tok;
1026     if (v < TOK_IDENT)
1027         error("invalid macro name '%s'", get_tok_str(tok, &tokc));
1028     /* XXX: should check if same macro (ANSI) */
1029     first = NULL;
1030     t = MACRO_OBJ;
1031     /* '(' must be just after macro definition for MACRO_FUNC */
1032     next_nomacro_spc();
1033     if (tok == '(') {
1034         next_nomacro();
1035         ps = &first;
1036         while (tok != ')') {
1037             varg = tok;
1038             next_nomacro();
1039             is_vaargs = 0;
1040             if (varg == TOK_DOTS) {
1041                 varg = TOK___VA_ARGS__;
1042                 is_vaargs = 1;
1043             } else if (tok == TOK_DOTS && gnu_ext) {
1044                 is_vaargs = 1;
1045                 next_nomacro();
1046             }
1047             if (varg < TOK_IDENT)
1048                 error("badly punctuated parameter list");
1049             s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
1050             *ps = s;
1051             ps = &s->next;
1052             if (tok != ',')
1053                 break;
1054             next_nomacro();
1055         }
1056         if (tok == ')')
1057             next_nomacro_spc();
1058         t = MACRO_FUNC;
1059     }
1060     tok_str_new(&str);
1061     spc = 2;
1062     /* EOF testing necessary for '-D' handling */
1063     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1064         /* remove spaces around ## and after '#' */
1065         if (TOK_TWOSHARPS == tok) {
1066             if (1 == spc)
1067                 --str.len;
1068             spc = 2;
1069         } else if ('#' == tok) {
1070             spc = 2;
1071         } else if (check_space(tok, &spc)) {
1072             goto skip;
1073         }
1074         tok_str_add2(&str, tok, &tokc);
1075     skip:
1076         next_nomacro_spc();
1077     }
1078     if (spc == 1)
1079         --str.len; /* remove trailing space */
1080     tok_str_add(&str, 0);
1081 #ifdef PP_DEBUG
1082     printf("define %s %d: ", get_tok_str(v, NULL), t);
1083     tok_print(str.str);
1084 #endif
1085     define_push(v, t, str.str, first);
1086 }
1087
1088 static inline int hash_cached_include(int type, const char *filename)
1089 {
1090     const unsigned char *s;
1091     unsigned int h;
1092
1093     h = TOK_HASH_INIT;
1094     h = TOK_HASH_FUNC(h, type);
1095     s = filename;
1096     while (*s) {
1097         h = TOK_HASH_FUNC(h, *s);
1098         s++;
1099     }
1100     h &= (CACHED_INCLUDES_HASH_SIZE - 1);
1101     return h;
1102 }
1103
1104 /* XXX: use a token or a hash table to accelerate matching ? */
1105 static CachedInclude *search_cached_include(TCCState *s1,
1106                                             int type, const char *filename)
1107 {
1108     CachedInclude *e;
1109     int i, h;
1110     h = hash_cached_include(type, filename);
1111     i = s1->cached_includes_hash[h];
1112     for(;;) {
1113         if (i == 0)
1114             break;
1115         e = s1->cached_includes[i - 1];
1116         if (e->type == type && !PATHCMP(e->filename, filename))
1117             return e;
1118         i = e->hash_next;
1119     }
1120     return NULL;
1121 }
1122
1123 static inline void add_cached_include(TCCState *s1, int type,
1124                                       const char *filename, int ifndef_macro)
1125 {
1126     CachedInclude *e;
1127     int h;
1128
1129     if (search_cached_include(s1, type, filename))
1130         return;
1131 #ifdef INC_DEBUG
1132     printf("adding cached '%s' %s\n", filename, get_tok_str(ifndef_macro, NULL));
1133 #endif
1134     e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
1135     if (!e)
1136         return;
1137     e->type = type;
1138     strcpy(e->filename, filename);
1139     e->ifndef_macro = ifndef_macro;
1140     dynarray_add((void ***)&s1->cached_includes, &s1->nb_cached_includes, e);
1141     /* add in hash table */
1142     h = hash_cached_include(type, filename);
1143     e->hash_next = s1->cached_includes_hash[h];
1144     s1->cached_includes_hash[h] = s1->nb_cached_includes;
1145 }
1146
1147 static void pragma_parse(TCCState *s1)
1148 {
1149     int val;
1150
1151     next();
1152     if (tok == TOK_pack) {
1153         /*
1154           This may be:
1155           #pragma pack(1) // set
1156           #pragma pack() // reset to default
1157           #pragma pack(push,1) // push & set
1158           #pragma pack(pop) // restore previous
1159         */
1160         next();
1161         skip('(');
1162         if (tok == TOK_ASM_pop) {
1163             next();
1164             if (s1->pack_stack_ptr <= s1->pack_stack) {
1165             stk_error:
1166                 error("out of pack stack");
1167             }
1168             s1->pack_stack_ptr--;
1169         } else {
1170             val = 0;
1171             if (tok != ')') {
1172                 if (tok == TOK_ASM_push) {
1173                     next();
1174                     if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
1175                         goto stk_error;
1176                     s1->pack_stack_ptr++;
1177                     skip(',');
1178                 }
1179                 if (tok != TOK_CINT) {
1180                 pack_error:
1181                     error("invalid pack pragma");
1182                 }
1183                 val = tokc.i;
1184                 if (val < 1 || val > 16 || (val & (val - 1)) != 0)
1185                     goto pack_error;
1186                 next();
1187             }
1188             *s1->pack_stack_ptr = val;
1189             skip(')');
1190         }
1191     }
1192 }
1193
1194 /* is_bof is true if first non space token at beginning of file */
1195 static void preprocess(int is_bof)
1196 {
1197     TCCState *s1 = tcc_state;
1198     int i, c, n, saved_parse_flags;
1199     char buf[1024], *q;
1200     Sym *s;
1201
1202     saved_parse_flags = parse_flags;
1203     parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM |
1204         PARSE_FLAG_LINEFEED;
1205     next_nomacro();
1206  redo:
1207     switch(tok) {
1208     case TOK_DEFINE:
1209         next_nomacro();
1210         parse_define();
1211         break;
1212     case TOK_UNDEF:
1213         next_nomacro();
1214         s = define_find(tok);
1215         /* undefine symbol by putting an invalid name */
1216         if (s)
1217             define_undef(s);
1218         break;
1219     case TOK_INCLUDE:
1220     case TOK_INCLUDE_NEXT:
1221         ch = file->buf_ptr[0];
1222         /* XXX: incorrect if comments : use next_nomacro with a special mode */
1223         skip_spaces();
1224         if (ch == '<') {
1225             c = '>';
1226             goto read_name;
1227         } else if (ch == '\"') {
1228             c = ch;
1229         read_name:
1230             inp();
1231             q = buf;
1232             while (ch != c && ch != '\n' && ch != CH_EOF) {
1233                 if ((q - buf) < sizeof(buf) - 1)
1234                     *q++ = ch;
1235                 if (ch == '\\') {
1236                     if (handle_stray_noerror() == 0)
1237                         --q;
1238                 } else
1239                     inp();
1240             }
1241             *q = '\0';
1242             minp();
1243 #if 0
1244             /* eat all spaces and comments after include */
1245             /* XXX: slightly incorrect */
1246             while (ch1 != '\n' && ch1 != CH_EOF)
1247                 inp();
1248 #endif
1249         } else {
1250             /* computed #include : either we have only strings or
1251                we have anything enclosed in '<>' */
1252             next();
1253             buf[0] = '\0';
1254             if (tok == TOK_STR) {
1255                 while (tok != TOK_LINEFEED) {
1256                     if (tok != TOK_STR) {
1257                     include_syntax:
1258                         error("'#include' expects \"FILENAME\" or <FILENAME>");
1259                     }
1260                     pstrcat(buf, sizeof(buf), (char *)tokc.cstr->data);
1261                     next();
1262                 }
1263                 c = '\"';
1264             } else {
1265                 int len;
1266                 while (tok != TOK_LINEFEED) {
1267                     pstrcat(buf, sizeof(buf), get_tok_str(tok, &tokc));
1268                     next();
1269                 }
1270                 len = strlen(buf);
1271                 /* check syntax and remove '<>' */
1272                 if (len < 2 || buf[0] != '<' || buf[len - 1] != '>')
1273                     goto include_syntax;
1274                 memmove(buf, buf + 1, len - 2);
1275                 buf[len - 2] = '\0';
1276                 c = '>';
1277             }
1278         }
1279
1280         if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
1281             error("#include recursion too deep");
1282
1283         n = s1->nb_include_paths + s1->nb_sysinclude_paths;
1284         for (i = -2; i < n; ++i) {
1285             char buf1[sizeof file->filename];
1286             BufferedFile *f;
1287             CachedInclude *e;
1288             const char *path;
1289             int size;
1290
1291             if (i == -2) {
1292                 /* check absolute include path */
1293                 if (!IS_ABSPATH(buf))
1294                     continue;
1295                 buf1[0] = 0;
1296
1297             } else if (i == -1) {
1298                 /* search in current dir if "header.h" */
1299                 if (c != '\"')
1300                     continue;
1301                 size = tcc_basename(file->filename) - file->filename;
1302                 memcpy(buf1, file->filename, size);
1303                 buf1[size] = '\0';
1304
1305             } else {
1306                 /* search in all the include paths */
1307                 if (i < s1->nb_include_paths)
1308                     path = s1->include_paths[i];
1309                 else
1310                     path = s1->sysinclude_paths[i - s1->nb_include_paths];
1311                 pstrcpy(buf1, sizeof(buf1), path);
1312                 pstrcat(buf1, sizeof(buf1), "/");
1313             }
1314
1315             pstrcat(buf1, sizeof(buf1), buf);
1316
1317             e = search_cached_include(s1, c, buf1);
1318             if (e && define_find(e->ifndef_macro)) {
1319                 /* no need to parse the include because the 'ifndef macro'
1320                    is defined */
1321 #ifdef INC_DEBUG
1322                 printf("%s: skipping %s\n", file->filename, buf);
1323 #endif
1324                 f = NULL;
1325             }  else {
1326                 f = tcc_open(s1, buf1);
1327                 if (!f)
1328                     continue;
1329             }
1330
1331             if (tok == TOK_INCLUDE_NEXT) {
1332                 tok = TOK_INCLUDE;
1333                 if (f)
1334                     tcc_close(f);
1335                 continue;
1336             }
1337
1338             if (!f)
1339                 goto include_done;
1340
1341 #ifdef INC_DEBUG
1342             printf("%s: including %s\n", file->filename, buf1);
1343 #endif
1344
1345            /* XXX: fix current line init */
1346            /* push current file in stack */
1347             *s1->include_stack_ptr++ = file;
1348             f->inc_type = c;
1349             pstrcpy(f->inc_filename, sizeof(f->inc_filename), buf1);
1350             file = f;
1351             /* add include file debug info */
1352             if (tcc_state->do_debug) {
1353                 put_stabs(file->filename, N_BINCL, 0, 0, 0);
1354             }
1355             tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
1356             ch = file->buf_ptr[0];
1357             goto the_end;
1358         }
1359         error("include file '%s' not found", buf);
1360 include_done:
1361         break;
1362     case TOK_IFNDEF:
1363         c = 1;
1364         goto do_ifdef;
1365     case TOK_IF:
1366         c = expr_preprocess();
1367         goto do_if;
1368     case TOK_IFDEF:
1369         c = 0;
1370     do_ifdef:
1371         next_nomacro();
1372         if (tok < TOK_IDENT)
1373             error("invalid argument for '#if%sdef'", c ? "n" : "");
1374         if (is_bof) {
1375             if (c) {
1376 #ifdef INC_DEBUG
1377                 printf("#ifndef %s\n", get_tok_str(tok, NULL));
1378 #endif
1379                 file->ifndef_macro = tok;
1380             }
1381         }
1382         c = (define_find(tok) != 0) ^ c;
1383     do_if:
1384         if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
1385             error("memory full");
1386         *s1->ifdef_stack_ptr++ = c;
1387         goto test_skip;
1388     case TOK_ELSE:
1389         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1390             error("#else without matching #if");
1391         if (s1->ifdef_stack_ptr[-1] & 2)
1392             error("#else after #else");
1393         c = (s1->ifdef_stack_ptr[-1] ^= 3);
1394         goto test_skip;
1395     case TOK_ELIF:
1396         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1397             error("#elif without matching #if");
1398         c = s1->ifdef_stack_ptr[-1];
1399         if (c > 1)
1400             error("#elif after #else");
1401         /* last #if/#elif expression was true: we skip */
1402         if (c == 1)
1403             goto skip;
1404         c = expr_preprocess();
1405         s1->ifdef_stack_ptr[-1] = c;
1406     test_skip:
1407         if (!(c & 1)) {
1408         skip:
1409             preprocess_skip();
1410             is_bof = 0;
1411             goto redo;
1412         }
1413         break;
1414     case TOK_ENDIF:
1415         if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
1416             error("#endif without matching #if");
1417         s1->ifdef_stack_ptr--;
1418         /* '#ifndef macro' was at the start of file. Now we check if
1419            an '#endif' is exactly at the end of file */
1420         if (file->ifndef_macro &&
1421             s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
1422             file->ifndef_macro_saved = file->ifndef_macro;
1423             /* need to set to zero to avoid false matches if another
1424                #ifndef at middle of file */
1425             file->ifndef_macro = 0;
1426             while (tok != TOK_LINEFEED)
1427                 next_nomacro();
1428             tok_flags |= TOK_FLAG_ENDIF;
1429             goto the_end;
1430         }
1431         break;
1432     case TOK_LINE:
1433         next();
1434         if (tok != TOK_CINT)
1435             error("#line");
1436         file->line_num = tokc.i - 1; /* the line number will be incremented after */
1437         next();
1438         if (tok != TOK_LINEFEED) {
1439             if (tok != TOK_STR)
1440                 error("#line");
1441             pstrcpy(file->filename, sizeof(file->filename),
1442                     (char *)tokc.cstr->data);
1443         }
1444         break;
1445     case TOK_ERROR:
1446     case TOK_WARNING:
1447         c = tok;
1448         ch = file->buf_ptr[0];
1449         skip_spaces();
1450         q = buf;
1451         while (ch != '\n' && ch != CH_EOF) {
1452             if ((q - buf) < sizeof(buf) - 1)
1453                 *q++ = ch;
1454             if (ch == '\\') {
1455                 if (handle_stray_noerror() == 0)
1456                     --q;
1457             } else
1458                 inp();
1459         }
1460         *q = '\0';
1461         if (c == TOK_ERROR)
1462             error("#error %s", buf);
1463         else
1464             warning("#warning %s", buf);
1465         break;
1466     case TOK_PRAGMA:
1467         pragma_parse(s1);
1468         break;
1469     default:
1470         if (tok == TOK_LINEFEED || tok == '!' || tok == TOK_CINT) {
1471             /* '!' is ignored to allow C scripts. numbers are ignored
1472                to emulate cpp behaviour */
1473         } else {
1474             if (!(saved_parse_flags & PARSE_FLAG_ASM_COMMENTS))
1475                 warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
1476         }
1477         break;
1478     }
1479     /* ignore other preprocess commands or #! for C scripts */
1480     while (tok != TOK_LINEFEED)
1481         next_nomacro();
1482  the_end:
1483     parse_flags = saved_parse_flags;
1484 }
1485
1486 /* evaluate escape codes in a string. */
1487 static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
1488 {
1489     int c, n;
1490     const uint8_t *p;
1491
1492     p = buf;
1493     for(;;) {
1494         c = *p;
1495         if (c == '\0')
1496             break;
1497         if (c == '\\') {
1498             p++;
1499             /* escape */
1500             c = *p;
1501             switch(c) {
1502             case '0': case '1': case '2': case '3':
1503             case '4': case '5': case '6': case '7':
1504                 /* at most three octal digits */
1505                 n = c - '0';
1506                 p++;
1507                 c = *p;
1508                 if (isoct(c)) {
1509                     n = n * 8 + c - '0';
1510                     p++;
1511                     c = *p;
1512                     if (isoct(c)) {
1513                         n = n * 8 + c - '0';
1514                         p++;
1515                     }
1516                 }
1517                 c = n;
1518                 goto add_char_nonext;
1519             case 'x':
1520             case 'u':
1521             case 'U':
1522                 p++;
1523                 n = 0;
1524                 for(;;) {
1525                     c = *p;
1526                     if (c >= 'a' && c <= 'f')
1527                         c = c - 'a' + 10;
1528                     else if (c >= 'A' && c <= 'F')
1529                         c = c - 'A' + 10;
1530                     else if (isnum(c))
1531                         c = c - '0';
1532                     else
1533                         break;
1534                     n = n * 16 + c;
1535                     p++;
1536                 }
1537                 c = n;
1538                 goto add_char_nonext;
1539             case 'a':
1540                 c = '\a';
1541                 break;
1542             case 'b':
1543                 c = '\b';
1544                 break;
1545             case 'f':
1546                 c = '\f';
1547                 break;
1548             case 'n':
1549                 c = '\n';
1550                 break;
1551             case 'r':
1552                 c = '\r';
1553                 break;
1554             case 't':
1555                 c = '\t';
1556                 break;
1557             case 'v':
1558                 c = '\v';
1559                 break;
1560             case 'e':
1561                 if (!gnu_ext)
1562                     goto invalid_escape;
1563                 c = 27;
1564                 break;
1565             case '\'':
1566             case '\"':
1567             case '\\':
1568             case '?':
1569                 break;
1570             default:
1571             invalid_escape:
1572                 if (c >= '!' && c <= '~')
1573                     warning("unknown escape sequence: \'\\%c\'", c);
1574                 else
1575                     warning("unknown escape sequence: \'\\x%x\'", c);
1576                 break;
1577             }
1578         }
1579         p++;
1580     add_char_nonext:
1581         if (!is_long)
1582             cstr_ccat(outstr, c);
1583         else
1584             cstr_wccat(outstr, c);
1585     }
1586     /* add a trailing '\0' */
1587     if (!is_long)
1588         cstr_ccat(outstr, '\0');
1589     else
1590         cstr_wccat(outstr, '\0');
1591 }
1592
1593 /* we use 64 bit numbers */
1594 #define BN_SIZE 2
1595
1596 /* bn = (bn << shift) | or_val */
1597 void bn_lshift(unsigned int *bn, int shift, int or_val)
1598 {
1599     int i;
1600     unsigned int v;
1601     for(i=0;i<BN_SIZE;i++) {
1602         v = bn[i];
1603         bn[i] = (v << shift) | or_val;
1604         or_val = v >> (32 - shift);
1605     }
1606 }
1607
1608 void bn_zero(unsigned int *bn)
1609 {
1610     int i;
1611     for(i=0;i<BN_SIZE;i++) {
1612         bn[i] = 0;
1613     }
1614 }
1615
1616 /* parse number in null terminated string 'p' and return it in the
1617    current token */
1618 void parse_number(const char *p)
1619 {
1620     int b, t, shift, frac_bits, s, exp_val, ch;
1621     char *q;
1622     unsigned int bn[BN_SIZE];
1623     double d;
1624
1625     /* number */
1626     q = token_buf;
1627     ch = *p++;
1628     t = ch;
1629     ch = *p++;
1630     *q++ = t;
1631     b = 10;
1632     if (t == '.') {
1633         goto float_frac_parse;
1634     } else if (t == '0') {
1635         if (ch == 'x' || ch == 'X') {
1636             q--;
1637             ch = *p++;
1638             b = 16;
1639         } else if (tcc_ext && (ch == 'b' || ch == 'B')) {
1640             q--;
1641             ch = *p++;
1642             b = 2;
1643         }
1644     }
1645     /* parse all digits. cannot check octal numbers at this stage
1646        because of floating point constants */
1647     while (1) {
1648         if (ch >= 'a' && ch <= 'f')
1649             t = ch - 'a' + 10;
1650         else if (ch >= 'A' && ch <= 'F')
1651             t = ch - 'A' + 10;
1652         else if (isnum(ch))
1653             t = ch - '0';
1654         else
1655             break;
1656         if (t >= b)
1657             break;
1658         if (q >= token_buf + STRING_MAX_SIZE) {
1659         num_too_long:
1660             error("number too long");
1661         }
1662         *q++ = ch;
1663         ch = *p++;
1664     }
1665     if (ch == '.' ||
1666         ((ch == 'e' || ch == 'E') && b == 10) ||
1667         ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
1668         if (b != 10) {
1669             /* NOTE: strtox should support that for hexa numbers, but
1670                non ISOC99 libcs do not support it, so we prefer to do
1671                it by hand */
1672             /* hexadecimal or binary floats */
1673             /* XXX: handle overflows */
1674             *q = '\0';
1675             if (b == 16)
1676                 shift = 4;
1677             else
1678                 shift = 2;
1679             bn_zero(bn);
1680             q = token_buf;
1681             while (1) {
1682                 t = *q++;
1683                 if (t == '\0') {
1684                     break;
1685                 } else if (t >= 'a') {
1686                     t = t - 'a' + 10;
1687                 } else if (t >= 'A') {
1688                     t = t - 'A' + 10;
1689                 } else {
1690                     t = t - '0';
1691                 }
1692                 bn_lshift(bn, shift, t);
1693             }
1694             frac_bits = 0;
1695             if (ch == '.') {
1696                 ch = *p++;
1697                 while (1) {
1698                     t = ch;
1699                     if (t >= 'a' && t <= 'f') {
1700                         t = t - 'a' + 10;
1701                     } else if (t >= 'A' && t <= 'F') {
1702                         t = t - 'A' + 10;
1703                     } else if (t >= '0' && t <= '9') {
1704                         t = t - '0';
1705                     } else {
1706                         break;
1707                     }
1708                     if (t >= b)
1709                         error("invalid digit");
1710                     bn_lshift(bn, shift, t);
1711                     frac_bits += shift;
1712                     ch = *p++;
1713                 }
1714             }
1715             if (ch != 'p' && ch != 'P')
1716                 expect("exponent");
1717             ch = *p++;
1718             s = 1;
1719             exp_val = 0;
1720             if (ch == '+') {
1721                 ch = *p++;
1722             } else if (ch == '-') {
1723                 s = -1;
1724                 ch = *p++;
1725             }
1726             if (ch < '0' || ch > '9')
1727                 expect("exponent digits");
1728             while (ch >= '0' && ch <= '9') {
1729                 exp_val = exp_val * 10 + ch - '0';
1730                 ch = *p++;
1731             }
1732             exp_val = exp_val * s;
1733
1734             /* now we can generate the number */
1735             /* XXX: should patch directly float number */
1736             d = (double)bn[1] * 4294967296.0 + (double)bn[0];
1737             d = ldexp(d, exp_val - frac_bits);
1738             t = toup(ch);
1739             if (t == 'F') {
1740                 ch = *p++;
1741                 tok = TOK_CFLOAT;
1742                 /* float : should handle overflow */
1743                 tokc.f = (float)d;
1744             } else if (t == 'L') {
1745                 ch = *p++;
1746                 tok = TOK_CLDOUBLE;
1747                 /* XXX: not large enough */
1748                 tokc.ld = (long double)d;
1749             } else {
1750                 tok = TOK_CDOUBLE;
1751                 tokc.d = d;
1752             }
1753         } else {
1754             /* decimal floats */
1755             if (ch == '.') {
1756                 if (q >= token_buf + STRING_MAX_SIZE)
1757                     goto num_too_long;
1758                 *q++ = ch;
1759                 ch = *p++;
1760             float_frac_parse:
1761                 while (ch >= '0' && ch <= '9') {
1762                     if (q >= token_buf + STRING_MAX_SIZE)
1763                         goto num_too_long;
1764                     *q++ = ch;
1765                     ch = *p++;
1766                 }
1767             }
1768             if (ch == 'e' || ch == 'E') {
1769                 if (q >= token_buf + STRING_MAX_SIZE)
1770                     goto num_too_long;
1771                 *q++ = ch;
1772                 ch = *p++;
1773                 if (ch == '-' || ch == '+') {
1774                     if (q >= token_buf + STRING_MAX_SIZE)
1775                         goto num_too_long;
1776                     *q++ = ch;
1777                     ch = *p++;
1778                 }
1779                 if (ch < '0' || ch > '9')
1780                     expect("exponent digits");
1781                 while (ch >= '0' && ch <= '9') {
1782                     if (q >= token_buf + STRING_MAX_SIZE)
1783                         goto num_too_long;
1784                     *q++ = ch;
1785                     ch = *p++;
1786                 }
1787             }
1788             *q = '\0';
1789             t = toup(ch);
1790             errno = 0;
1791             if (t == 'F') {
1792                 ch = *p++;
1793                 tok = TOK_CFLOAT;
1794                 tokc.f = strtof(token_buf, NULL);
1795             } else if (t == 'L') {
1796                 ch = *p++;
1797                 tok = TOK_CLDOUBLE;
1798                 tokc.ld = strtold(token_buf, NULL);
1799             } else {
1800                 tok = TOK_CDOUBLE;
1801                 tokc.d = strtod(token_buf, NULL);
1802             }
1803         }
1804     } else {
1805         unsigned long long n, n1;
1806         int lcount, ucount;
1807
1808         /* integer number */
1809         *q = '\0';
1810         q = token_buf;
1811         if (b == 10 && *q == '0') {
1812             b = 8;
1813             q++;
1814         }
1815         n = 0;
1816         while(1) {
1817             t = *q++;
1818             /* no need for checks except for base 10 / 8 errors */
1819             if (t == '\0') {
1820                 break;
1821             } else if (t >= 'a') {
1822                 t = t - 'a' + 10;
1823             } else if (t >= 'A') {
1824                 t = t - 'A' + 10;
1825             } else {
1826                 t = t - '0';
1827                 if (t >= b)
1828                     error("invalid digit");
1829             }
1830             n1 = n;
1831             n = n * b + t;
1832             /* detect overflow */
1833             /* XXX: this test is not reliable */
1834             if (n < n1)
1835                 error("integer constant overflow");
1836         }
1837
1838         /* XXX: not exactly ANSI compliant */
1839         if ((n & 0xffffffff00000000LL) != 0) {
1840             if ((n >> 63) != 0)
1841                 tok = TOK_CULLONG;
1842             else
1843                 tok = TOK_CLLONG;
1844         } else if (n > 0x7fffffff) {
1845             tok = TOK_CUINT;
1846         } else {
1847             tok = TOK_CINT;
1848         }
1849         lcount = 0;
1850         ucount = 0;
1851         for(;;) {
1852             t = toup(ch);
1853             if (t == 'L') {
1854                 if (lcount >= 2)
1855                     error("three 'l's in integer constant");
1856                 lcount++;
1857                 if (lcount == 2) {
1858                     if (tok == TOK_CINT)
1859                         tok = TOK_CLLONG;
1860                     else if (tok == TOK_CUINT)
1861                         tok = TOK_CULLONG;
1862                 }
1863                 ch = *p++;
1864             } else if (t == 'U') {
1865                 if (ucount >= 1)
1866                     error("two 'u's in integer constant");
1867                 ucount++;
1868                 if (tok == TOK_CINT)
1869                     tok = TOK_CUINT;
1870                 else if (tok == TOK_CLLONG)
1871                     tok = TOK_CULLONG;
1872                 ch = *p++;
1873             } else {
1874                 break;
1875             }
1876         }
1877         if (tok == TOK_CINT || tok == TOK_CUINT)
1878             tokc.ui = n;
1879         else
1880             tokc.ull = n;
1881     }
1882     if (ch)
1883         error("invalid number\n");
1884 }
1885
1886
1887 #define PARSE2(c1, tok1, c2, tok2)              \
1888     case c1:                                    \
1889         PEEKC(c, p);                            \
1890         if (c == c2) {                          \
1891             p++;                                \
1892             tok = tok2;                         \
1893         } else {                                \
1894             tok = tok1;                         \
1895         }                                       \
1896         break;
1897
1898 /* return next token without macro substitution */
1899 static inline void next_nomacro1(void)
1900 {
1901     int t, c, is_long;
1902     TokenSym *ts;
1903     uint8_t *p, *p1;
1904     unsigned int h;
1905
1906     p = file->buf_ptr;
1907  redo_no_start:
1908     c = *p;
1909     switch(c) {
1910     case ' ':
1911     case '\t':
1912         tok = c;
1913         p++;
1914         goto keep_tok_flags;
1915     case '\f':
1916     case '\v':
1917     case '\r':
1918         p++;
1919         goto redo_no_start;
1920     case '\\':
1921         /* first look if it is in fact an end of buffer */
1922         if (p >= file->buf_end) {
1923             file->buf_ptr = p;
1924             handle_eob();
1925             p = file->buf_ptr;
1926             if (p >= file->buf_end)
1927                 goto parse_eof;
1928             else
1929                 goto redo_no_start;
1930         } else {
1931             file->buf_ptr = p;
1932             ch = *p;
1933             handle_stray();
1934             p = file->buf_ptr;
1935             goto redo_no_start;
1936         }
1937     parse_eof:
1938         {
1939             TCCState *s1 = tcc_state;
1940             if ((parse_flags & PARSE_FLAG_LINEFEED)
1941                 && !(tok_flags & TOK_FLAG_EOF)) {
1942                 tok_flags |= TOK_FLAG_EOF;
1943                 tok = TOK_LINEFEED;
1944                 goto keep_tok_flags;
1945             } else if (s1->include_stack_ptr == s1->include_stack ||
1946                        !(parse_flags & PARSE_FLAG_PREPROCESS)) {
1947                 /* no include left : end of file. */
1948                 tok = TOK_EOF;
1949             } else {
1950                 tok_flags &= ~TOK_FLAG_EOF;
1951                 /* pop include file */
1952
1953                 /* test if previous '#endif' was after a #ifdef at
1954                    start of file */
1955                 if (tok_flags & TOK_FLAG_ENDIF) {
1956 #ifdef INC_DEBUG
1957                     printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
1958 #endif
1959                     add_cached_include(s1, file->inc_type, file->inc_filename,
1960                                        file->ifndef_macro_saved);
1961                 }
1962
1963                 /* add end of include file debug info */
1964                 if (tcc_state->do_debug) {
1965                     put_stabd(N_EINCL, 0, 0);
1966                 }
1967                 /* pop include stack */
1968                 tcc_close(file);
1969                 s1->include_stack_ptr--;
1970                 file = *s1->include_stack_ptr;
1971                 p = file->buf_ptr;
1972                 goto redo_no_start;
1973             }
1974         }
1975         break;
1976
1977     case '\n':
1978         file->line_num++;
1979         tok_flags |= TOK_FLAG_BOL;
1980         p++;
1981 maybe_newline:
1982         if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
1983             goto redo_no_start;
1984         tok = TOK_LINEFEED;
1985         goto keep_tok_flags;
1986
1987     case '#':
1988         /* XXX: simplify */
1989         PEEKC(c, p);
1990         if ((tok_flags & TOK_FLAG_BOL) &&
1991             (parse_flags & PARSE_FLAG_PREPROCESS)) {
1992             file->buf_ptr = p;
1993             preprocess(tok_flags & TOK_FLAG_BOF);
1994             p = file->buf_ptr;
1995             goto maybe_newline;
1996         } else {
1997             if (c == '#') {
1998                 p++;
1999                 tok = TOK_TWOSHARPS;
2000             } else {
2001                 if (parse_flags & PARSE_FLAG_ASM_COMMENTS) {
2002                     p = parse_line_comment(p - 1);
2003                     goto redo_no_start;
2004                 } else {
2005                     tok = '#';
2006                 }
2007             }
2008         }
2009         break;
2010
2011     case 'a': case 'b': case 'c': case 'd':
2012     case 'e': case 'f': case 'g': case 'h':
2013     case 'i': case 'j': case 'k': case 'l':
2014     case 'm': case 'n': case 'o': case 'p':
2015     case 'q': case 'r': case 's': case 't':
2016     case 'u': case 'v': case 'w': case 'x':
2017     case 'y': case 'z':
2018     case 'A': case 'B': case 'C': case 'D':
2019     case 'E': case 'F': case 'G': case 'H':
2020     case 'I': case 'J': case 'K':
2021     case 'M': case 'N': case 'O': case 'P':
2022     case 'Q': case 'R': case 'S': case 'T':
2023     case 'U': case 'V': case 'W': case 'X':
2024     case 'Y': case 'Z':
2025     case '_':
2026     parse_ident_fast:
2027         p1 = p;
2028         h = TOK_HASH_INIT;
2029         h = TOK_HASH_FUNC(h, c);
2030         p++;
2031         for(;;) {
2032             c = *p;
2033             if (!isidnum_table[c-CH_EOF])
2034                 break;
2035             h = TOK_HASH_FUNC(h, c);
2036             p++;
2037         }
2038         if (c != '\\') {
2039             TokenSym **pts;
2040             int len;
2041
2042             /* fast case : no stray found, so we have the full token
2043                and we have already hashed it */
2044             len = p - p1;
2045             h &= (TOK_HASH_SIZE - 1);
2046             pts = &hash_ident[h];
2047             for(;;) {
2048                 ts = *pts;
2049                 if (!ts)
2050                     break;
2051                 if (ts->len == len && !memcmp(ts->str, p1, len))
2052                     goto token_found;
2053                 pts = &(ts->hash_next);
2054             }
2055             ts = tok_alloc_new(pts, p1, len);
2056         token_found: ;
2057         } else {
2058             /* slower case */
2059             cstr_reset(&tokcstr);
2060
2061             while (p1 < p) {
2062                 cstr_ccat(&tokcstr, *p1);
2063                 p1++;
2064             }
2065             p--;
2066             PEEKC(c, p);
2067         parse_ident_slow:
2068             while (isidnum_table[c-CH_EOF]) {
2069                 cstr_ccat(&tokcstr, c);
2070                 PEEKC(c, p);
2071             }
2072             ts = tok_alloc(tokcstr.data, tokcstr.size);
2073         }
2074         tok = ts->tok;
2075         break;
2076     case 'L':
2077         t = p[1];
2078         if (t != '\\' && t != '\'' && t != '\"') {
2079             /* fast case */
2080             goto parse_ident_fast;
2081         } else {
2082             PEEKC(c, p);
2083             if (c == '\'' || c == '\"') {
2084                 is_long = 1;
2085                 goto str_const;
2086             } else {
2087                 cstr_reset(&tokcstr);
2088                 cstr_ccat(&tokcstr, 'L');
2089                 goto parse_ident_slow;
2090             }
2091         }
2092         break;
2093     case '0': case '1': case '2': case '3':
2094     case '4': case '5': case '6': case '7':
2095     case '8': case '9':
2096
2097         cstr_reset(&tokcstr);
2098         /* after the first digit, accept digits, alpha, '.' or sign if
2099            prefixed by 'eEpP' */
2100     parse_num:
2101         for(;;) {
2102             t = c;
2103             cstr_ccat(&tokcstr, c);
2104             PEEKC(c, p);
2105             if (!(isnum(c) || isid(c) || c == '.' ||
2106                   ((c == '+' || c == '-') &&
2107                    (t == 'e' || t == 'E' || t == 'p' || t == 'P'))))
2108                 break;
2109         }
2110         /* We add a trailing '\0' to ease parsing */
2111         cstr_ccat(&tokcstr, '\0');
2112         tokc.cstr = &tokcstr;
2113         tok = TOK_PPNUM;
2114         break;
2115     case '.':
2116         /* special dot handling because it can also start a number */
2117         PEEKC(c, p);
2118         if (isnum(c)) {
2119             cstr_reset(&tokcstr);
2120             cstr_ccat(&tokcstr, '.');
2121             goto parse_num;
2122         } else if (c == '.') {
2123             PEEKC(c, p);
2124             if (c != '.')
2125                 expect("'.'");
2126             PEEKC(c, p);
2127             tok = TOK_DOTS;
2128         } else {
2129             tok = '.';
2130         }
2131         break;
2132     case '\'':
2133     case '\"':
2134         is_long = 0;
2135     str_const:
2136         {
2137             CString str;
2138             int sep;
2139
2140             sep = c;
2141
2142             /* parse the string */
2143             cstr_new(&str);
2144             p = parse_pp_string(p, sep, &str);
2145             cstr_ccat(&str, '\0');
2146
2147             /* eval the escape (should be done as TOK_PPNUM) */
2148             cstr_reset(&tokcstr);
2149             parse_escape_string(&tokcstr, str.data, is_long);
2150             cstr_free(&str);
2151
2152             if (sep == '\'') {
2153                 int char_size;
2154                 /* XXX: make it portable */
2155                 if (!is_long)
2156                     char_size = 1;
2157                 else
2158                     char_size = sizeof(nwchar_t);
2159                 if (tokcstr.size <= char_size)
2160                     error("empty character constant");
2161                 if (tokcstr.size > 2 * char_size)
2162                     warning("multi-character character constant");
2163                 if (!is_long) {
2164                     tokc.i = *(int8_t *)tokcstr.data;
2165                     tok = TOK_CCHAR;
2166                 } else {
2167                     tokc.i = *(nwchar_t *)tokcstr.data;
2168                     tok = TOK_LCHAR;
2169                 }
2170             } else {
2171                 tokc.cstr = &tokcstr;
2172                 if (!is_long)
2173                     tok = TOK_STR;
2174                 else
2175                     tok = TOK_LSTR;
2176             }
2177         }
2178         break;
2179
2180     case '<':
2181         PEEKC(c, p);
2182         if (c == '=') {
2183             p++;
2184             tok = TOK_LE;
2185         } else if (c == '<') {
2186             PEEKC(c, p);
2187             if (c == '=') {
2188                 p++;
2189                 tok = TOK_A_SHL;
2190             } else {
2191                 tok = TOK_SHL;
2192             }
2193         } else {
2194             tok = TOK_LT;
2195         }
2196         break;
2197
2198     case '>':
2199         PEEKC(c, p);
2200         if (c == '=') {
2201             p++;
2202             tok = TOK_GE;
2203         } else if (c == '>') {
2204             PEEKC(c, p);
2205             if (c == '=') {
2206                 p++;
2207                 tok = TOK_A_SAR;
2208             } else {
2209                 tok = TOK_SAR;
2210             }
2211         } else {
2212             tok = TOK_GT;
2213         }
2214         break;
2215
2216     case '&':
2217         PEEKC(c, p);
2218         if (c == '&') {
2219             p++;
2220             tok = TOK_LAND;
2221         } else if (c == '=') {
2222             p++;
2223             tok = TOK_A_AND;
2224         } else {
2225             tok = '&';
2226         }
2227         break;
2228
2229     case '|':
2230         PEEKC(c, p);
2231         if (c == '|') {
2232             p++;
2233             tok = TOK_LOR;
2234         } else if (c == '=') {
2235             p++;
2236             tok = TOK_A_OR;
2237         } else {
2238             tok = '|';
2239         }
2240         break;
2241
2242     case '+':
2243         PEEKC(c, p);
2244         if (c == '+') {
2245             p++;
2246             tok = TOK_INC;
2247         } else if (c == '=') {
2248             p++;
2249             tok = TOK_A_ADD;
2250         } else {
2251             tok = '+';
2252         }
2253         break;
2254
2255     case '-':
2256         PEEKC(c, p);
2257         if (c == '-') {
2258             p++;
2259             tok = TOK_DEC;
2260         } else if (c == '=') {
2261             p++;
2262             tok = TOK_A_SUB;
2263         } else if (c == '>') {
2264             p++;
2265             tok = TOK_ARROW;
2266         } else {
2267             tok = '-';
2268         }
2269         break;
2270
2271     PARSE2('!', '!', '=', TOK_NE)
2272     PARSE2('=', '=', '=', TOK_EQ)
2273     PARSE2('*', '*', '=', TOK_A_MUL)
2274     PARSE2('%', '%', '=', TOK_A_MOD)
2275     PARSE2('^', '^', '=', TOK_A_XOR)
2276
2277         /* comments or operator */
2278     case '/':
2279         PEEKC(c, p);
2280         if (c == '*') {
2281             p = parse_comment(p);
2282             goto redo_no_start;
2283         } else if (c == '/') {
2284             p = parse_line_comment(p);
2285             goto redo_no_start;
2286         } else if (c == '=') {
2287             p++;
2288             tok = TOK_A_DIV;
2289         } else {
2290             tok = '/';
2291         }
2292         break;
2293
2294         /* simple tokens */
2295     case '(':
2296     case ')':
2297     case '[':
2298     case ']':
2299     case '{':
2300     case '}':
2301     case ',':
2302     case ';':
2303     case ':':
2304     case '?':
2305     case '~':
2306     case '$': /* only used in assembler */
2307     case '@': /* dito */
2308         tok = c;
2309         p++;
2310         break;
2311     default:
2312         error("unrecognized character \\x%02x", c);
2313         break;
2314     }
2315     tok_flags = 0;
2316 keep_tok_flags:
2317     file->buf_ptr = p;
2318 #if defined(PARSE_DEBUG)
2319     printf("token = %s\n", get_tok_str(tok, &tokc));
2320 #endif
2321 }
2322
2323 /* return next token without macro substitution. Can read input from
2324    macro_ptr buffer */
2325 static void next_nomacro_spc(void)
2326 {
2327     if (macro_ptr) {
2328     redo:
2329         tok = *macro_ptr;
2330         if (tok) {
2331             TOK_GET(tok, macro_ptr, tokc);
2332             if (tok == TOK_LINENUM) {
2333                 file->line_num = tokc.i;
2334                 goto redo;
2335             }
2336         }
2337     } else {
2338         next_nomacro1();
2339     }
2340 }
2341
2342 static void next_nomacro(void)
2343 {
2344     do {
2345         next_nomacro_spc();
2346     } while (is_space(tok));
2347 }
2348
2349 /* substitute args in macro_str and return allocated string */
2350 static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args)
2351 {
2352     int *st, last_tok, t, spc;
2353     Sym *s;
2354     CValue cval;
2355     TokenString str;
2356     CString cstr;
2357
2358     tok_str_new(&str);
2359     last_tok = 0;
2360     while(1) {
2361         TOK_GET(t, macro_str, cval);
2362         if (!t)
2363             break;
2364         if (t == '#') {
2365             /* stringize */
2366             TOK_GET(t, macro_str, cval);
2367             if (!t)
2368                 break;
2369             s = sym_find2(args, t);
2370             if (s) {
2371                 cstr_new(&cstr);
2372                 st = s->d;
2373                 spc = 0;
2374                 while (*st) {
2375                     TOK_GET(t, st, cval);
2376                     if (!check_space(t, &spc))
2377                         cstr_cat(&cstr, get_tok_str(t, &cval));
2378                 }
2379                 cstr.size -= spc;
2380                 cstr_ccat(&cstr, '\0');
2381 #ifdef PP_DEBUG
2382                 printf("stringize: %s\n", (char *)cstr.data);
2383 #endif
2384                 /* add string */
2385                 cval.cstr = &cstr;
2386                 tok_str_add2(&str, TOK_STR, &cval);
2387                 cstr_free(&cstr);
2388             } else {
2389                 tok_str_add2(&str, t, &cval);
2390             }
2391         } else if (t >= TOK_IDENT) {
2392             s = sym_find2(args, t);
2393             if (s) {
2394                 st = s->d;
2395                 /* if '##' is present before or after, no arg substitution */
2396                 if (*macro_str == TOK_TWOSHARPS || last_tok == TOK_TWOSHARPS) {
2397                     /* special case for var arg macros : ## eats the
2398                        ',' if empty VA_ARGS variable. */
2399                     /* XXX: test of the ',' is not 100%
2400                        reliable. should fix it to avoid security
2401                        problems */
2402                     if (gnu_ext && s->type.t &&
2403                         last_tok == TOK_TWOSHARPS &&
2404                         str.len >= 2 && str.str[str.len - 2] == ',') {
2405                         if (*st == 0) {
2406                             /* suppress ',' '##' */
2407                             str.len -= 2;
2408                         } else {
2409                             /* suppress '##' and add variable */
2410                             str.len--;
2411                             goto add_var;
2412                         }
2413                     } else {
2414                         int t1;
2415                     add_var:
2416                         for(;;) {
2417                             TOK_GET(t1, st, cval);
2418                             if (!t1)
2419                                 break;
2420                             tok_str_add2(&str, t1, &cval);
2421                         }
2422                     }
2423                 } else {
2424                     /* NOTE: the stream cannot be read when macro
2425                        substituing an argument */
2426                     macro_subst(&str, nested_list, st, NULL);
2427                 }
2428             } else {
2429                 tok_str_add(&str, t);
2430             }
2431         } else {
2432             tok_str_add2(&str, t, &cval);
2433         }
2434         last_tok = t;
2435     }
2436     tok_str_add(&str, 0);
2437     return str.str;
2438 }
2439
2440 static char const ab_month_name[12][4] =
2441 {
2442     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
2443     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
2444 };
2445
2446 /* do macro substitution of current token with macro 's' and add
2447    result to (tok_str,tok_len). 'nested_list' is the list of all
2448    macros we got inside to avoid recursing. Return non zero if no
2449    substitution needs to be done */
2450 static int macro_subst_tok(TokenString *tok_str,
2451                            Sym **nested_list, Sym *s, struct macro_level **can_read_stream)
2452 {
2453     Sym *args, *sa, *sa1;
2454     int mstr_allocated, parlevel, *mstr, t, t1, *p, spc;
2455     TokenString str;
2456     char *cstrval;
2457     CValue cval;
2458     CString cstr;
2459     char buf[32];
2460
2461     /* if symbol is a macro, prepare substitution */
2462     /* special macros */
2463     if (tok == TOK___LINE__) {
2464         snprintf(buf, sizeof(buf), "%d", file->line_num);
2465         cstrval = buf;
2466         t1 = TOK_PPNUM;
2467         goto add_cstr1;
2468     } else if (tok == TOK___FILE__) {
2469         cstrval = file->filename;
2470         goto add_cstr;
2471     } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
2472         time_t ti;
2473         struct tm *tm;
2474
2475         time(&ti);
2476         tm = localtime(&ti);
2477         if (tok == TOK___DATE__) {
2478             snprintf(buf, sizeof(buf), "%s %2d %d",
2479                      ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
2480         } else {
2481             snprintf(buf, sizeof(buf), "%02d:%02d:%02d",
2482                      tm->tm_hour, tm->tm_min, tm->tm_sec);
2483         }
2484         cstrval = buf;
2485     add_cstr:
2486         t1 = TOK_STR;
2487     add_cstr1:
2488         cstr_new(&cstr);
2489         cstr_cat(&cstr, cstrval);
2490         cstr_ccat(&cstr, '\0');
2491         cval.cstr = &cstr;
2492         tok_str_add2(tok_str, t1, &cval);
2493         cstr_free(&cstr);
2494     } else {
2495         mstr = s->d;
2496         mstr_allocated = 0;
2497         if (s->type.t == MACRO_FUNC) {
2498             /* NOTE: we do not use next_nomacro to avoid eating the
2499                next token. XXX: find better solution */
2500         redo:
2501             if (macro_ptr) {
2502                 p = macro_ptr;
2503                 while (is_space(t = *p) || TOK_LINEFEED == t)
2504                     ++p;
2505                 if (t == 0 && can_read_stream) {
2506                     /* end of macro stream: we must look at the token
2507                        after in the file */
2508                     struct macro_level *ml = *can_read_stream;
2509                     macro_ptr = NULL;
2510                     if (ml)
2511                     {
2512                         macro_ptr = ml->p;
2513                         ml->p = NULL;
2514                         *can_read_stream = ml -> prev;
2515                     }
2516                     goto redo;
2517                 }
2518             } else {
2519                 /* XXX: incorrect with comments */
2520                 ch = file->buf_ptr[0];
2521                 while (is_space(ch) || ch == '\n')
2522                     cinp();
2523                 t = ch;
2524             }
2525             if (t != '(') /* no macro subst */
2526                 return -1;
2527
2528             /* argument macro */
2529             next_nomacro();
2530             next_nomacro();
2531             args = NULL;
2532             sa = s->next;
2533             /* NOTE: empty args are allowed, except if no args */
2534             for(;;) {
2535                 /* handle '()' case */
2536                 if (!args && !sa && tok == ')')
2537                     break;
2538                 if (!sa)
2539                     error("macro '%s' used with too many args",
2540                           get_tok_str(s->v, 0));
2541                 tok_str_new(&str);
2542                 parlevel = spc = 0;
2543                 /* NOTE: non zero sa->t indicates VA_ARGS */
2544                 while ((parlevel > 0 ||
2545                         (tok != ')' &&
2546                          (tok != ',' || sa->type.t))) &&
2547                        tok != -1) {
2548                     if (tok == '(')
2549                         parlevel++;
2550                     else if (tok == ')')
2551                         parlevel--;
2552                     if (tok == TOK_LINEFEED)
2553                         tok = ' ';
2554                     if (!check_space(tok, &spc))
2555                         tok_str_add2(&str, tok, &tokc);
2556                     next_nomacro_spc();
2557                 }
2558                 str.len -= spc;
2559                 tok_str_add(&str, 0);
2560                 sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
2561                 sa1->d = str.str;
2562                 sa = sa->next;
2563                 if (tok == ')') {
2564                     /* special case for gcc var args: add an empty
2565                        var arg argument if it is omitted */
2566                     if (sa && sa->type.t && gnu_ext)
2567                         continue;
2568                     else
2569                         break;
2570                 }
2571                 if (tok != ',')
2572                     expect(",");
2573                 next_nomacro();
2574             }
2575             if (sa) {
2576                 error("macro '%s' used with too few args",
2577                       get_tok_str(s->v, 0));
2578             }
2579
2580             /* now subst each arg */
2581             mstr = macro_arg_subst(nested_list, mstr, args);
2582             /* free memory */
2583             sa = args;
2584             while (sa) {
2585                 sa1 = sa->prev;
2586                 tok_str_free(sa->d);
2587                 sym_free(sa);
2588                 sa = sa1;
2589             }
2590             mstr_allocated = 1;
2591         }
2592         sym_push2(nested_list, s->v, 0, 0);
2593         macro_subst(tok_str, nested_list, mstr, can_read_stream);
2594         /* pop nested defined symbol */
2595         sa1 = *nested_list;
2596         *nested_list = sa1->prev;
2597         sym_free(sa1);
2598         if (mstr_allocated)
2599             tok_str_free(mstr);
2600     }
2601     return 0;
2602 }
2603
2604 /* handle the '##' operator. Return NULL if no '##' seen. Otherwise
2605    return the resulting string (which must be freed). */
2606 static inline int *macro_twosharps(const int *macro_str)
2607 {
2608     const int *ptr;
2609     int t;
2610     CValue cval;
2611     TokenString macro_str1;
2612     CString cstr;
2613     char *p;
2614     int n;
2615
2616     /* we search the first '##' */
2617     for(ptr = macro_str;;) {
2618         TOK_GET(t, ptr, cval);
2619         if (t == TOK_TWOSHARPS)
2620             break;
2621         /* nothing more to do if end of string */
2622         if (t == 0)
2623             return NULL;
2624     }
2625
2626     /* we saw '##', so we need more processing to handle it */
2627     tok_str_new(&macro_str1);
2628     for(ptr = macro_str;;) {
2629         TOK_GET(tok, ptr, tokc);
2630         if (tok == 0)
2631             break;
2632         if (tok == TOK_TWOSHARPS)
2633             continue;
2634         while (*ptr == TOK_TWOSHARPS) {
2635             t = *++ptr;
2636             if (t && t != TOK_TWOSHARPS) {
2637                 TOK_GET(t, ptr, cval);
2638
2639                 /* We concatenate the two tokens */
2640                 cstr_new(&cstr);
2641                 cstr_cat(&cstr, get_tok_str(tok, &tokc));
2642                 n = cstr.size;
2643                 cstr_cat(&cstr, get_tok_str(t, &cval));
2644                 cstr_ccat(&cstr, '\0');
2645
2646                 p = file->buf_ptr;
2647                 file->buf_ptr = cstr.data;
2648                 for (;;) {
2649                     next_nomacro1();
2650                     if (0 == *file->buf_ptr)
2651                         break;
2652                     tok_str_add2(&macro_str1, tok, &tokc);
2653
2654                     warning("pasting \"%.*s\" and \"%s\" does not give a valid preprocessing token",
2655                         n, cstr.data, (char*)cstr.data + n);
2656                 }
2657                 file->buf_ptr = p;
2658                 cstr_reset(&cstr);
2659             }
2660         }
2661         tok_str_add2(&macro_str1, tok, &tokc);
2662     }
2663     tok_str_add(&macro_str1, 0);
2664     return macro_str1.str;
2665 }
2666
2667
2668 /* do macro substitution of macro_str and add result to
2669    (tok_str,tok_len). 'nested_list' is the list of all macros we got
2670    inside to avoid recursing. */
2671 static void macro_subst(TokenString *tok_str, Sym **nested_list,
2672                         const int *macro_str, struct macro_level ** can_read_stream)
2673 {
2674     Sym *s;
2675     int *macro_str1;
2676     const int *ptr;
2677     int t, ret, spc;
2678     CValue cval;
2679     struct macro_level ml;
2680
2681     /* first scan for '##' operator handling */
2682     ptr = macro_str;
2683     macro_str1 = macro_twosharps(ptr);
2684     if (macro_str1)
2685         ptr = macro_str1;
2686     spc = 0;
2687     while (1) {
2688         /* NOTE: ptr == NULL can only happen if tokens are read from
2689            file stream due to a macro function call */
2690         if (ptr == NULL)
2691             break;
2692         TOK_GET(t, ptr, cval);
2693         if (t == 0)
2694             break;
2695         s = define_find(t);
2696         if (s != NULL) {
2697             /* if nested substitution, do nothing */
2698             if (sym_find2(*nested_list, t))
2699                 goto no_subst;
2700             ml.p = macro_ptr;
2701             if (can_read_stream)
2702                 ml.prev = *can_read_stream, *can_read_stream = &ml;
2703             macro_ptr = (int *)ptr;
2704             tok = t;
2705             ret = macro_subst_tok(tok_str, nested_list, s, can_read_stream);
2706             ptr = (int *)macro_ptr;
2707             macro_ptr = ml.p;
2708             if (can_read_stream && *can_read_stream == &ml)
2709                 *can_read_stream = ml.prev;
2710             if (ret != 0)
2711                 goto no_subst;
2712         } else {
2713         no_subst:
2714             if (!check_space(t, &spc))
2715                 tok_str_add2(tok_str, t, &cval);
2716         }
2717     }
2718     if (macro_str1)
2719         tok_str_free(macro_str1);
2720 }
2721
2722 /* return next token with macro substitution */
2723 static void next(void)
2724 {
2725     Sym *nested_list, *s;
2726     TokenString str;
2727     struct macro_level *ml;
2728
2729  redo:
2730     if (parse_flags & PARSE_FLAG_SPACES)
2731         next_nomacro_spc();
2732     else
2733         next_nomacro();
2734     if (!macro_ptr) {
2735         /* if not reading from macro substituted string, then try
2736            to substitute macros */
2737         if (tok >= TOK_IDENT &&
2738             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2739             s = define_find(tok);
2740             if (s) {
2741                 /* we have a macro: we try to substitute */
2742                 tok_str_new(&str);
2743                 nested_list = NULL;
2744                 ml = NULL;
2745                 if (macro_subst_tok(&str, &nested_list, s, &ml) == 0) {
2746                     /* substitution done, NOTE: maybe empty */
2747                     tok_str_add(&str, 0);
2748                     macro_ptr = str.str;
2749                     macro_ptr_allocated = str.str;
2750                     goto redo;
2751                 }
2752             }
2753         }
2754     } else {
2755         if (tok == 0) {
2756             /* end of macro or end of unget buffer */
2757             if (unget_buffer_enabled) {
2758                 macro_ptr = unget_saved_macro_ptr;
2759                 unget_buffer_enabled = 0;
2760             } else {
2761                 /* end of macro string: free it */
2762                 tok_str_free(macro_ptr_allocated);
2763                 macro_ptr = NULL;
2764             }
2765             goto redo;
2766         }
2767     }
2768
2769     /* convert preprocessor tokens into C tokens */
2770     if (tok == TOK_PPNUM &&
2771         (parse_flags & PARSE_FLAG_TOK_NUM)) {
2772         parse_number((char *)tokc.cstr->data);
2773     }
2774 }
2775
2776 /* push back current token and set current token to 'last_tok'. Only
2777    identifier case handled for labels. */
2778 static inline void unget_tok(int last_tok)
2779 {
2780     int i, n;
2781     int *q;
2782     unget_saved_macro_ptr = macro_ptr;
2783     unget_buffer_enabled = 1;
2784     q = unget_saved_buffer;
2785     macro_ptr = q;
2786     *q++ = tok;
2787     n = tok_ext_size(tok) - 1;
2788     for(i=0;i<n;i++)
2789         *q++ = tokc.tab[i];
2790     *q = 0; /* end of token string */
2791     tok = last_tok;
2792 }
2793
2794
2795 /* better than nothing, but needs extension to handle '-E' option
2796    correctly too */
2797 static void preprocess_init(TCCState *s1)
2798 {
2799     s1->include_stack_ptr = s1->include_stack;
2800     /* XXX: move that before to avoid having to initialize
2801        file->ifdef_stack_ptr ? */
2802     s1->ifdef_stack_ptr = s1->ifdef_stack;
2803     file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
2804
2805     /* XXX: not ANSI compliant: bound checking says error */
2806     vtop = vstack - 1;
2807     s1->pack_stack[0] = 0;
2808     s1->pack_stack_ptr = s1->pack_stack;
2809 }
2810
2811 void preprocess_new()
2812 {
2813     int i, c;
2814     const char *p, *r;
2815     TokenSym *ts;
2816
2817     /* init isid table */
2818     for(i=CH_EOF;i<256;i++)
2819         isidnum_table[i-CH_EOF] = isid(i) || isnum(i);
2820
2821     /* add all tokens */
2822     table_ident = NULL;
2823     memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
2824
2825     tok_ident = TOK_IDENT;
2826     p = tcc_keywords;
2827     while (*p) {
2828         r = p;
2829         for(;;) {
2830             c = *r++;
2831             if (c == '\0')
2832                 break;
2833         }
2834         ts = tok_alloc(p, r - p - 1);
2835         p = r;
2836     }
2837 }
2838
2839 /* Preprocess the current file */
2840 static int tcc_preprocess(TCCState *s1)
2841 {
2842     Sym *define_start;
2843     BufferedFile *file_ref, **iptr, **iptr_new;
2844     int token_seen, line_ref, d;
2845     const char *s;
2846
2847     preprocess_init(s1);
2848     define_start = define_stack;
2849     ch = file->buf_ptr[0];
2850     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
2851     parse_flags = PARSE_FLAG_ASM_COMMENTS | PARSE_FLAG_PREPROCESS |
2852         PARSE_FLAG_LINEFEED | PARSE_FLAG_SPACES;
2853     token_seen = 0;
2854     line_ref = 0;
2855     file_ref = NULL;
2856
2857     iptr = s1->include_stack_ptr;
2858     for (;;) {
2859         next();
2860         if (tok == TOK_EOF) {
2861             break;
2862         } else if (file != file_ref) {
2863             goto print_line;
2864         } else if (tok == TOK_LINEFEED) {
2865             if (!token_seen)
2866                 continue;
2867             ++line_ref;
2868             token_seen = 0;
2869         } else if (!token_seen) {
2870             d = file->line_num - line_ref;
2871             if (file != file_ref || d < 0 || d >= 8) {
2872 print_line:
2873                 iptr_new = s1->include_stack_ptr;
2874                 s = iptr_new > iptr ? " 1"
2875                   : iptr_new < iptr ? " 2"
2876                   : iptr_new > s1->include_stack ? " 3"
2877                   : ""
2878                   ;
2879                 iptr = iptr_new;
2880                 fprintf(s1->outfile, "# %d \"%s\"%s\n", file->line_num, file->filename, s);
2881             } else {
2882                 while (d)
2883                     fputs("\n", s1->outfile), --d;
2884             }
2885             line_ref = (file_ref = file)->line_num;
2886             token_seen = tok != TOK_LINEFEED;
2887             if (!token_seen)
2888                 continue;
2889         }
2890         fputs(get_tok_str(tok, &tokc), s1->outfile);
2891     }
2892     free_defines(define_start);
2893     return 0;
2894 }
2895