tccpp.c

   1 /*
   2  *  TCC - Tiny C Compiler
   3  *
   4  *  Copyright (c) 2001-2004 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #define USING_GLOBALS
  22 #include "tcc.h"
  23
  24 /********************************************************/
  25 /* global variables */
  26
  27 ST_DATA int tok_flags;
  28 ST_DATA int parse_flags;
  29
  30 ST_DATA struct BufferedFile *file;
  31 ST_DATA int ch, tok;
  32 ST_DATA CValue tokc;
  33 ST_DATA const int *macro_ptr;
  34 ST_DATA CString tokcstr; /* current parsed string, if any */
  35
  36 /* display benchmark infos */
  37 ST_DATA int tok_ident;
  38 ST_DATA TokenSym **table_ident;
  39
  40 /* ------------------------------------------------------------------------- */
  41
  42 static TokenSym *hash_ident[TOK_HASH_SIZE];
  43 static char token_buf[STRING_MAX_SIZE + 1];
  44 static CString cstr_buf;
  45 static CString macro_equal_buf;
  46 static TokenString tokstr_buf;
  47 static unsigned char isidnum_table[256 - CH_EOF];
  48 static int pp_debug_tok, pp_debug_symv;
  49 static int pp_once;
  50 static int pp_expr;
  51 static int pp_counter;
  52 static void tok_print(const char *msg, const int *str);
  53
  54 static struct TinyAlloc *toksym_alloc;
  55 static struct TinyAlloc *tokstr_alloc;
  56
  57 static TokenString *macro_stack;
  58
  59 static const char tcc_keywords[] =
  60 #define DEF(id, str) str "\0"
  61 #include "tcctok.h"
  62 #undef DEF
  63 ;
  64
  65 /* WARNING: the content of this string encodes token numbers */
  66 static const unsigned char tok_two_chars[] =
  67 /* outdated -- gr
  68     "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
  69     "-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
  70 */{
  71     '<','=', TOK_LE,
  72     '>','=', TOK_GE,
  73     '!','=', TOK_NE,
  74     '&','&', TOK_LAND,
  75     '|','|', TOK_LOR,
  76     '+','+', TOK_INC,
  77     '-','-', TOK_DEC,
  78     '=','=', TOK_EQ,
  79     '<','<', TOK_SHL,
  80     '>','>', TOK_SAR,
  81     '+','=', TOK_A_ADD,
  82     '-','=', TOK_A_SUB,
  83     '*','=', TOK_A_MUL,
  84     '/','=', TOK_A_DIV,
  85     '%','=', TOK_A_MOD,
  86     '&','=', TOK_A_AND,
  87     '^','=', TOK_A_XOR,
  88     '|','=', TOK_A_OR,
  89     '-','>', TOK_ARROW,
  90     '.','.', TOK_TWODOTS,
  91     '#','#', TOK_TWOSHARPS,
  92     0
  93 };
  94
  95 static void next_nomacro_spc(void);
  96
  97 ST_FUNC void skip(int c)
  98 {
  99     if (tok != c)
 100         tcc_error("'%c' expected (got \"%s\")", c, get_tok_str(tok, &tokc));
 101     next();
 102 }
 103
 104 ST_FUNC void expect(const char *msg)
 105 {
 106     tcc_error("%s expected", msg);
 107 }
 108
 109 /* ------------------------------------------------------------------------- */
 110 /* Custom allocator for tiny objects */
 111
 112 #ifndef __BOUNDS_CHECKING_ON
 113 #define USE_TAL
 114 #endif
 115
 116 #ifndef USE_TAL
 117 #define tal_free(al, p) tcc_free(p)
 118 #define tal_realloc(al, p, size) tcc_realloc(p, size)
 119 #define tal_new(a,b,c)
 120 #define tal_delete(a)
 121 #else
 122 #if !defined(MEM_DEBUG)
 123 #define tal_free(al, p) tal_free_impl(al, p)
 124 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size)
 125 #define TAL_DEBUG_PARAMS
 126 #else
 127 #define TAL_DEBUG 1
 128 //#define TAL_INFO 1 /* collect and dump allocators stats */
 129 #define tal_free(al, p) tal_free_impl(al, p, __FILE__, __LINE__)
 130 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size, __FILE__, __LINE__)
 131 #define TAL_DEBUG_PARAMS , const char *file, int line
 132 #define TAL_DEBUG_FILE_LEN 40
 133 #endif
 134
 135 #define TOKSYM_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenSym in table_ident */
 136 #define TOKSTR_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenString instances */
 137 #define CSTR_TAL_SIZE       (256 * 1024) /* allocator for tiny CString instances */
 138 #define TOKSYM_TAL_LIMIT    256 /* prefer unique limits to distinguish allocators debug msgs */
 139 #define TOKSTR_TAL_LIMIT    128 /* 32 * sizeof(int) */
 140 #define CSTR_TAL_LIMIT      1024
 141
 142 typedef struct TinyAlloc {
 143     unsigned  limit;
 144     unsigned  size;
 145     uint8_t *buffer;
 146     uint8_t *p;
 147     unsigned  nb_allocs;
 148     struct TinyAlloc *next, *top;
 149 #ifdef TAL_INFO
 150     unsigned  nb_peak;
 151     unsigned  nb_total;
 152     unsigned  nb_missed;
 153     uint8_t *peak_p;
 154 #endif
 155 } TinyAlloc;
 156
 157 typedef struct tal_header_t {
 158     unsigned  size;
 159 #ifdef TAL_DEBUG
 160     int     line_num; /* negative line_num used for double free check */
 161     char    file_name[TAL_DEBUG_FILE_LEN + 1];
 162 #endif
 163 } tal_header_t;
 164
 165 /* ------------------------------------------------------------------------- */
 166
 167 static TinyAlloc *tal_new(TinyAlloc **pal, unsigned limit, unsigned size)
 168 {
 169     TinyAlloc *al = tcc_mallocz(sizeof(TinyAlloc));
 170     al->p = al->buffer = tcc_malloc(size);
 171     al->limit = limit;
 172     al->size = size;
 173     if (pal) *pal = al;
 174     return al;
 175 }
 176
 177 static void tal_delete(TinyAlloc *al)
 178 {
 179     TinyAlloc *next;
 180
 181 tail_call:
 182     if (!al)
 183         return;
 184 #ifdef TAL_INFO
 185     fprintf(stderr, "limit=%5d, size=%5g MB, nb_peak=%6d, nb_total=%8d, nb_missed=%6d, usage=%5.1f%%\n",
 186             al->limit, al->size / 1024.0 / 1024.0, al->nb_peak, al->nb_total, al->nb_missed,
 187             (al->peak_p - al->buffer) * 100.0 / al->size);
 188 #endif
 189 #ifdef TAL_DEBUG
 190     if (al->nb_allocs > 0) {
 191         uint8_t *p;
 192         fprintf(stderr, "TAL_DEBUG: memory leak %d chunk(s) (limit= %d)\n",
 193                 al->nb_allocs, al->limit);
 194         p = al->buffer;
 195         while (p < al->p) {
 196             tal_header_t *header = (tal_header_t *)p;
 197             if (header->line_num > 0) {
 198                 fprintf(stderr, "%s:%d: chunk of %d bytes leaked\n",
 199                         header->file_name, header->line_num, header->size);
 200             }
 201             p += header->size + sizeof(tal_header_t);
 202         }
 203 #if MEM_DEBUG-0 == 2
 204         exit(2);
 205 #endif
 206     }
 207 #endif
 208     next = al->next;
 209     tcc_free(al->buffer);
 210     tcc_free(al);
 211     al = next;
 212     goto tail_call;
 213 }
 214
 215 static void tal_free_impl(TinyAlloc *al, void *p TAL_DEBUG_PARAMS)
 216 {
 217     if (!p)
 218         return;
 219 tail_call:
 220     if (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size) {
 221 #ifdef TAL_DEBUG
 222         tal_header_t *header = (((tal_header_t *)p) - 1);
 223         if (header->line_num < 0) {
 224             fprintf(stderr, "%s:%d: TAL_DEBUG: double frees chunk from\n",
 225                     file, line);
 226             fprintf(stderr, "%s:%d: %d bytes\n",
 227                     header->file_name, (int)-header->line_num, (int)header->size);
 228         } else
 229             header->line_num = -header->line_num;
 230 #endif
 231         al->nb_allocs--;
 232         if (!al->nb_allocs)
 233             al->p = al->buffer;
 234     } else if (al->next) {
 235         al = al->next;
 236         goto tail_call;
 237     }
 238     else
 239         tcc_free(p);
 240 }
 241
 242 static void *tal_realloc_impl(TinyAlloc **pal, void *p, unsigned size TAL_DEBUG_PARAMS)
 243 {
 244     tal_header_t *header;
 245     void *ret;
 246     int is_own;
 247     unsigned adj_size = (size + 3) & -4;
 248     TinyAlloc *al = *pal;
 249
 250 tail_call:
 251     is_own = (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size);
 252     if ((!p || is_own) && size <= al->limit) {
 253         if (al->p + adj_size + sizeof(tal_header_t) < al->buffer + al->size) {
 254             header = (tal_header_t *)al->p;
 255             header->size = adj_size;
 256 #ifdef TAL_DEBUG
 257             { int ofs = strlen(file) - TAL_DEBUG_FILE_LEN;
 258             strncpy(header->file_name, file + (ofs > 0 ? ofs : 0), TAL_DEBUG_FILE_LEN);
 259             header->file_name[TAL_DEBUG_FILE_LEN] = 0;
 260             header->line_num = line; }
 261 #endif
 262             ret = al->p + sizeof(tal_header_t);
 263             al->p += adj_size + sizeof(tal_header_t);
 264             if (is_own) {
 265                 header = (((tal_header_t *)p) - 1);
 266                 memcpy(ret, p, header->size);
 267 #ifdef TAL_DEBUG
 268                 header->line_num = -header->line_num;
 269 #endif
 270             } else {
 271                 al->nb_allocs++;
 272             }
 273 #ifdef TAL_INFO
 274             if (al->nb_peak < al->nb_allocs)
 275                 al->nb_peak = al->nb_allocs;
 276             if (al->peak_p < al->p)
 277                 al->peak_p = al->p;
 278             al->nb_total++;
 279 #endif
 280             return ret;
 281         } else if (is_own) {
 282             al->nb_allocs--;
 283             ret = tal_realloc(*pal, 0, size);
 284             header = (((tal_header_t *)p) - 1);
 285             memcpy(ret, p, header->size);
 286 #ifdef TAL_DEBUG
 287             header->line_num = -header->line_num;
 288 #endif
 289             return ret;
 290         }
 291         if (al->next) {
 292             al = al->next;
 293         } else {
 294             TinyAlloc *bottom = al, *next = al->top ? al->top : al;
 295
 296             al = tal_new(pal, next->limit, next->size * 2);
 297             al->next = next;
 298             bottom->top = al;
 299         }
 300         goto tail_call;
 301     }
 302     if (is_own) {
 303         al->nb_allocs--;
 304         ret = tcc_malloc(size);
 305         header = (((tal_header_t *)p) - 1);
 306         memcpy(ret, p, header->size);
 307 #ifdef TAL_DEBUG
 308         header->line_num = -header->line_num;
 309 #endif
 310     } else if (al->next) {
 311         al = al->next;
 312         goto tail_call;
 313     } else
 314         ret = tcc_realloc(p, size);
 315 #ifdef TAL_INFO
 316     al->nb_missed++;
 317 #endif
 318     return ret;
 319 }
 320
 321 #endif /* USE_TAL */
 322
 323 /* ------------------------------------------------------------------------- */
 324 /* CString handling */
 325 static void cstr_realloc(CString *cstr, int new_size)
 326 {
 327     int size;
 328
 329     size = cstr->size_allocated;
 330     if (size < 8)
 331         size = 8; /* no need to allocate a too small first string */
 332     while (size < new_size)
 333         size = size * 2;
 334     cstr->data = tcc_realloc(cstr->data, size);
 335     cstr->size_allocated = size;
 336 }
 337
 338 /* add a byte */
 339 ST_INLN void cstr_ccat(CString *cstr, int ch)
 340 {
 341     int size;
 342     size = cstr->size + 1;
 343     if (size > cstr->size_allocated)
 344         cstr_realloc(cstr, size);
 345     ((unsigned char *)cstr->data)[size - 1] = ch;
 346     cstr->size = size;
 347 }
 348
 349 ST_FUNC void cstr_cat(CString *cstr, const char *str, int len)
 350 {
 351     int size;
 352     if (len <= 0)
 353         len = strlen(str) + 1 + len;
 354     size = cstr->size + len;
 355     if (size > cstr->size_allocated)
 356         cstr_realloc(cstr, size);
 357     memmove(((unsigned char *)cstr->data) + cstr->size, str, len);
 358     cstr->size = size;
 359 }
 360
 361 /* add a wide char */
 362 ST_FUNC void cstr_wccat(CString *cstr, int ch)
 363 {
 364     int size;
 365     size = cstr->size + sizeof(nwchar_t);
 366     if (size > cstr->size_allocated)
 367         cstr_realloc(cstr, size);
 368     *(nwchar_t *)(((unsigned char *)cstr->data) + size - sizeof(nwchar_t)) = ch;
 369     cstr->size = size;
 370 }
 371
 372 ST_FUNC void cstr_new(CString *cstr)
 373 {
 374     memset(cstr, 0, sizeof(CString));
 375 }
 376
 377 /* free string and reset it to NULL */
 378 ST_FUNC void cstr_free(CString *cstr)
 379 {
 380     tcc_free(cstr->data);
 381     cstr_new(cstr);
 382 }
 383
 384 /* reset string to empty */
 385 ST_FUNC void cstr_reset(CString *cstr)
 386 {
 387     cstr->size = 0;
 388 }
 389
 390 ST_FUNC int cstr_printf(CString *cstr, const char *fmt, ...)
 391 {
 392     va_list v;
 393     int len, size;
 394
 395     va_start(v, fmt);
 396     len = vsnprintf(NULL, 0, fmt, v);
 397     va_end(v);
 398     size = cstr->size + len + 1;
 399     if (size > cstr->size_allocated)
 400         cstr_realloc(cstr, size);
 401     va_start(v, fmt);
 402     vsnprintf((char*)cstr->data + cstr->size, size, fmt, v);
 403     va_end(v);
 404     cstr->size += len;
 405     return len;
 406 }
 407
 408 /* XXX: unicode ? */
 409 static void add_char(CString *cstr, int c)
 410 {
 411     if (c == '\'' || c == '\"' || c == '\\') {
 412         /* XXX: could be more precise if char or string */
 413         cstr_ccat(cstr, '\\');
 414     }
 415     if (c >= 32 && c <= 126) {
 416         cstr_ccat(cstr, c);
 417     } else {
 418         cstr_ccat(cstr, '\\');
 419         if (c == '\n') {
 420             cstr_ccat(cstr, 'n');
 421         } else {
 422             cstr_ccat(cstr, '0' + ((c >> 6) & 7));
 423             cstr_ccat(cstr, '0' + ((c >> 3) & 7));
 424             cstr_ccat(cstr, '0' + (c & 7));
 425         }
 426     }
 427 }
 428
 429 /* ------------------------------------------------------------------------- */
 430 /* allocate a new token */
 431 static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
 432 {
 433     TokenSym *ts, **ptable;
 434     int i;
 435
 436     if (tok_ident >= SYM_FIRST_ANOM)
 437         tcc_error("memory full (symbols)");
 438
 439     /* expand token table if needed */
 440     i = tok_ident - TOK_IDENT;
 441     if ((i % TOK_ALLOC_INCR) == 0) {
 442         ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
 443         table_ident = ptable;
 444     }
 445
 446     ts = tal_realloc(toksym_alloc, 0, sizeof(TokenSym) + len);
 447     table_ident[i] = ts;
 448     ts->tok = tok_ident++;
 449     ts->sym_define = NULL;
 450     ts->sym_label = NULL;
 451     ts->sym_struct = NULL;
 452     ts->sym_identifier = NULL;
 453     ts->len = len;
 454     ts->hash_next = NULL;
 455     memcpy(ts->str, str, len);
 456     ts->str[len] = '\0';
 457     *pts = ts;
 458     return ts;
 459 }
 460
 461 #define TOK_HASH_INIT 1
 462 #define TOK_HASH_FUNC(h, c) ((h) + ((h) << 5) + ((h) >> 27) + (c))
 463
 464
 465 /* find a token and add it if not found */
 466 ST_FUNC TokenSym *tok_alloc(const char *str, int len)
 467 {
 468     TokenSym *ts, **pts;
 469     int i;
 470     unsigned int h;
 471
 472     h = TOK_HASH_INIT;
 473     for(i=0;i<len;i++)
 474         h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
 475     h &= (TOK_HASH_SIZE - 1);
 476
 477     pts = &hash_ident[h];
 478     for(;;) {
 479         ts = *pts;
 480         if (!ts)
 481             break;
 482         if (ts->len == len && !memcmp(ts->str, str, len))
 483             return ts;
 484         pts = &(ts->hash_next);
 485     }
 486     return tok_alloc_new(pts, str, len);
 487 }
 488
 489 /* XXX: buffer overflow */
 490 /* XXX: float tokens */
 491 ST_FUNC const char *get_tok_str(int v, CValue *cv)
 492 {
 493     char *p;
 494     int i, len;
 495
 496     cstr_reset(&cstr_buf);
 497     p = cstr_buf.data;
 498
 499     switch(v) {
 500     case TOK_CINT:
 501     case TOK_CUINT:
 502     case TOK_CLONG:
 503     case TOK_CULONG:
 504     case TOK_CLLONG:
 505     case TOK_CULLONG:
 506         /* XXX: not quite exact, but only useful for testing  */
 507 #ifdef _WIN32
 508         sprintf(p, "%u", (unsigned)cv->i);
 509 #else
 510         sprintf(p, "%llu", (unsigned long long)cv->i);
 511 #endif
 512         break;
 513     case TOK_LCHAR:
 514         cstr_ccat(&cstr_buf, 'L');
 515     case TOK_CCHAR:
 516         cstr_ccat(&cstr_buf, '\'');
 517         add_char(&cstr_buf, cv->i);
 518         cstr_ccat(&cstr_buf, '\'');
 519         cstr_ccat(&cstr_buf, '\0');
 520         break;
 521     case TOK_PPNUM:
 522     case TOK_PPSTR:
 523         return (char*)cv->str.data;
 524     case TOK_LSTR:
 525         cstr_ccat(&cstr_buf, 'L');
 526     case TOK_STR:
 527         cstr_ccat(&cstr_buf, '\"');
 528         if (v == TOK_STR) {
 529             len = cv->str.size - 1;
 530             for(i=0;i<len;i++)
 531                 add_char(&cstr_buf, ((unsigned char *)cv->str.data)[i]);
 532         } else {
 533             len = (cv->str.size / sizeof(nwchar_t)) - 1;
 534             for(i=0;i<len;i++)
 535                 add_char(&cstr_buf, ((nwchar_t *)cv->str.data)[i]);
 536         }
 537         cstr_ccat(&cstr_buf, '\"');
 538         cstr_ccat(&cstr_buf, '\0');
 539         break;
 540
 541     case TOK_CFLOAT:
 542         cstr_cat(&cstr_buf, "<float>", 0);
 543         break;
 544     case TOK_CDOUBLE:
 545         cstr_cat(&cstr_buf, "<double>", 0);
 546         break;
 547     case TOK_CLDOUBLE:
 548         cstr_cat(&cstr_buf, "<long double>", 0);
 549         break;
 550     case TOK_LINENUM:
 551         cstr_cat(&cstr_buf, "<linenumber>", 0);
 552         break;
 553
 554     /* above tokens have value, the ones below don't */
 555     case TOK_LT:
 556         v = '<';
 557         goto addv;
 558     case TOK_GT:
 559         v = '>';
 560         goto addv;
 561     case TOK_DOTS:
 562         return strcpy(p, "...");
 563     case TOK_A_SHL:
 564         return strcpy(p, "<<=");
 565     case TOK_A_SAR:
 566         return strcpy(p, ">>=");
 567     case TOK_EOF:
 568         return strcpy(p, "<eof>");
 569     default:
 570         if (v < TOK_IDENT) {
 571             /* search in two bytes table */
 572             const unsigned char *q = tok_two_chars;
 573             while (*q) {
 574                 if (q[2] == v) {
 575                     *p++ = q[0];
 576                     *p++ = q[1];
 577                     *p = '\0';
 578                     return cstr_buf.data;
 579                 }
 580                 q += 3;
 581             }
 582         if (v >= 127) {
 583             sprintf(cstr_buf.data, "<%02x>", v);
 584             return cstr_buf.data;
 585         }
 586         addv:
 587             *p++ = v;
 588             *p = '\0';
 589         } else if (v < tok_ident) {
 590             return table_ident[v - TOK_IDENT]->str;
 591         } else if (v >= SYM_FIRST_ANOM) {
 592             /* special name for anonymous symbol */
 593             sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
 594         } else {
 595             /* should never happen */
 596             return NULL;
 597         }
 598         break;
 599     }
 600     return cstr_buf.data;
 601 }
 602
 603 /* return the current character, handling end of block if necessary
 604    (but not stray) */
 605 static int handle_eob(void)
 606 {
 607     BufferedFile *bf = file;
 608     int len;
 609
 610     /* only tries to read if really end of buffer */
 611     if (bf->buf_ptr >= bf->buf_end) {
 612         if (bf->fd >= 0) {
 613 #if defined(PARSE_DEBUG)
 614             len = 1;
 615 #else
 616             len = IO_BUF_SIZE;
 617 #endif
 618             len = read(bf->fd, bf->buffer, len);
 619             if (len < 0)
 620                 len = 0;
 621         } else {
 622             len = 0;
 623         }
 624         total_bytes += len;
 625         bf->buf_ptr = bf->buffer;
 626         bf->buf_end = bf->buffer + len;
 627         *bf->buf_end = CH_EOB;
 628     }
 629     if (bf->buf_ptr < bf->buf_end) {
 630         return bf->buf_ptr[0];
 631     } else {
 632         bf->buf_ptr = bf->buf_end;
 633         return CH_EOF;
 634     }
 635 }
 636
 637 /* read next char from current input file and handle end of input buffer */
 638 static inline void inp(void)
 639 {
 640     ch = *(++(file->buf_ptr));
 641     /* end of buffer/file handling */
 642     if (ch == CH_EOB)
 643         ch = handle_eob();
 644 }
 645
 646 /* handle '\[\r]\n' */
 647 static int handle_stray_noerror(void)
 648 {
 649     while (ch == '\\') {
 650         inp();
 651         if (ch == '\n') {
 652             file->line_num++;
 653             inp();
 654         } else if (ch == '\r') {
 655             inp();
 656             if (ch != '\n')
 657                 goto fail;
 658             file->line_num++;
 659             inp();
 660         } else {
 661         fail:
 662             return 1;
 663         }
 664     }
 665     return 0;
 666 }
 667
 668 static void handle_stray(void)
 669 {
 670     if (handle_stray_noerror())
 671         tcc_error("stray '\\' in program");
 672 }
 673
 674 /* skip the stray and handle the \\n case. Output an error if
 675    incorrect char after the stray */
 676 static int handle_stray1(uint8_t *p)
 677 {
 678     int c;
 679
 680     file->buf_ptr = p;
 681     if (p >= file->buf_end) {
 682         c = handle_eob();
 683         if (c != '\\')
 684             return c;
 685         p = file->buf_ptr;
 686     }
 687     ch = *p;
 688     if (handle_stray_noerror()) {
 689         if (!(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
 690             tcc_error("stray '\\' in program");
 691         *--file->buf_ptr = '\\';
 692     }
 693     p = file->buf_ptr;
 694     c = *p;
 695     return c;
 696 }
 697
 698 /* handle just the EOB case, but not stray */
 699 #define PEEKC_EOB(c, p)\
 700 {\
 701     p++;\
 702     c = *p;\
 703     if (c == '\\') {\
 704         file->buf_ptr = p;\
 705         c = handle_eob();\
 706         p = file->buf_ptr;\
 707     }\
 708 }
 709
 710 /* handle the complicated stray case */
 711 #define PEEKC(c, p)\
 712 {\
 713     p++;\
 714     c = *p;\
 715     if (c == '\\') {\
 716         c = handle_stray1(p);\
 717         p = file->buf_ptr;\
 718     }\
 719 }
 720
 721 /* input with '\[\r]\n' handling. Note that this function cannot
 722    handle other characters after '\', so you cannot call it inside
 723    strings or comments */
 724 static void minp(void)
 725 {
 726     inp();
 727     if (ch == '\\')
 728         handle_stray();
 729 }
 730
 731 /* single line C++ comments */
 732 static uint8_t *parse_line_comment(uint8_t *p)
 733 {
 734     int c;
 735
 736     p++;
 737     for(;;) {
 738         c = *p;
 739     redo:
 740         if (c == '\n' || c == CH_EOF) {
 741             break;
 742         } else if (c == '\\') {
 743             file->buf_ptr = p;
 744             c = handle_eob();
 745             p = file->buf_ptr;
 746             if (c == '\\') {
 747                 PEEKC_EOB(c, p);
 748                 if (c == '\n') {
 749                     file->line_num++;
 750                     PEEKC_EOB(c, p);
 751                 } else if (c == '\r') {
 752                     PEEKC_EOB(c, p);
 753                     if (c == '\n') {
 754                         file->line_num++;
 755                         PEEKC_EOB(c, p);
 756                     }
 757                 }
 758             } else {
 759                 goto redo;
 760             }
 761         } else {
 762             p++;
 763         }
 764     }
 765     return p;
 766 }
 767
 768 /* C comments */
 769 static uint8_t *parse_comment(uint8_t *p)
 770 {
 771     int c;
 772
 773     p++;
 774     for(;;) {
 775         /* fast skip loop */
 776         for(;;) {
 777             c = *p;
 778             if (c == '\n' || c == '*' || c == '\\')
 779                 break;
 780             p++;
 781             c = *p;
 782             if (c == '\n' || c == '*' || c == '\\')
 783                 break;
 784             p++;
 785         }
 786         /* now we can handle all the cases */
 787         if (c == '\n') {
 788             file->line_num++;
 789             p++;
 790         } else if (c == '*') {
 791             p++;
 792             for(;;) {
 793                 c = *p;
 794                 if (c == '*') {
 795                     p++;
 796                 } else if (c == '/') {
 797                     goto end_of_comment;
 798                 } else if (c == '\\') {
 799                     file->buf_ptr = p;
 800                     c = handle_eob();
 801                     p = file->buf_ptr;
 802                     if (c == CH_EOF)
 803                         tcc_error("unexpected end of file in comment");
 804                     if (c == '\\') {
 805                         /* skip '\[\r]\n', otherwise just skip the stray */
 806                         while (c == '\\') {
 807                             PEEKC_EOB(c, p);
 808                             if (c == '\n') {
 809                                 file->line_num++;
 810                                 PEEKC_EOB(c, p);
 811                             } else if (c == '\r') {
 812                                 PEEKC_EOB(c, p);
 813                                 if (c == '\n') {
 814                                     file->line_num++;
 815                                     PEEKC_EOB(c, p);
 816                                 }
 817                             } else {
 818                                 goto after_star;
 819                             }
 820                         }
 821                     }
 822                 } else {
 823                     break;
 824                 }
 825             }
 826         after_star: ;
 827         } else {
 828             /* stray, eob or eof */
 829             file->buf_ptr = p;
 830             c = handle_eob();
 831             p = file->buf_ptr;
 832             if (c == CH_EOF) {
 833                 tcc_error("unexpected end of file in comment");
 834             } else if (c == '\\') {
 835                 p++;
 836             }
 837         }
 838     }
 839  end_of_comment:
 840     p++;
 841     return p;
 842 }
 843
 844 ST_FUNC int set_idnum(int c, int val)
 845 {
 846     int prev = isidnum_table[c - CH_EOF];
 847     isidnum_table[c - CH_EOF] = val;
 848     return prev;
 849 }
 850
 851 #define cinp minp
 852
 853 static inline void skip_spaces(void)
 854 {
 855     while (isidnum_table[ch - CH_EOF] & IS_SPC)
 856         cinp();
 857 }
 858
 859 static inline int check_space(int t, int *spc)
 860 {
 861     if (t < 256 && (isidnum_table[t - CH_EOF] & IS_SPC)) {
 862         if (*spc)
 863             return 1;
 864         *spc = 1;
 865     } else
 866         *spc = 0;
 867     return 0;
 868 }
 869
 870 /* parse a string without interpreting escapes */
 871 static uint8_t *parse_pp_string(uint8_t *p,
 872                                 int sep, CString *str)
 873 {
 874     int c;
 875     p++;
 876     for(;;) {
 877         c = *p;
 878         if (c == sep) {
 879             break;
 880         } else if (c == '\\') {
 881             file->buf_ptr = p;
 882             c = handle_eob();
 883             p = file->buf_ptr;
 884             if (c == CH_EOF) {
 885             unterminated_string:
 886                 /* XXX: indicate line number of start of string */
 887                 tcc_error("missing terminating %c character", sep);
 888             } else if (c == '\\') {
 889                 /* escape : just skip \[\r]\n */
 890                 PEEKC_EOB(c, p);
 891                 if (c == '\n') {
 892                     file->line_num++;
 893                     p++;
 894                 } else if (c == '\r') {
 895                     PEEKC_EOB(c, p);
 896                     if (c != '\n')
 897                         expect("'\n' after '\r'");
 898                     file->line_num++;
 899                     p++;
 900                 } else if (c == CH_EOF) {
 901                     goto unterminated_string;
 902                 } else {
 903                     if (str) {
 904                         cstr_ccat(str, '\\');
 905                         cstr_ccat(str, c);
 906                     }
 907                     p++;
 908                 }
 909             }
 910         } else if (c == '\n') {
 911             file->line_num++;
 912             goto add_char;
 913         } else if (c == '\r') {
 914             PEEKC_EOB(c, p);
 915             if (c != '\n') {
 916                 if (str)
 917                     cstr_ccat(str, '\r');
 918             } else {
 919                 file->line_num++;
 920                 goto add_char;
 921             }
 922         } else {
 923         add_char:
 924             if (str)
 925                 cstr_ccat(str, c);
 926             p++;
 927         }
 928     }
 929     p++;
 930     return p;
 931 }
 932
 933 /* skip block of text until #else, #elif or #endif. skip also pairs of
 934    #if/#endif */
 935 static void preprocess_skip(void)
 936 {
 937     int a, start_of_line, c, in_warn_or_error;
 938     uint8_t *p;
 939
 940     p = file->buf_ptr;
 941     a = 0;
 942 redo_start:
 943     start_of_line = 1;
 944     in_warn_or_error = 0;
 945     for(;;) {
 946     redo_no_start:
 947         c = *p;
 948         switch(c) {
 949         case ' ':
 950         case '\t':
 951         case '\f':
 952         case '\v':
 953         case '\r':
 954             p++;
 955             goto redo_no_start;
 956         case '\n':
 957             file->line_num++;
 958             p++;
 959             goto redo_start;
 960         case '\\':
 961             file->buf_ptr = p;
 962             c = handle_eob();
 963             if (c == CH_EOF) {
 964                 expect("#endif");
 965             } else if (c == '\\') {
 966                 ch = file->buf_ptr[0];
 967                 handle_stray_noerror();
 968             }
 969             p = file->buf_ptr;
 970             goto redo_no_start;
 971         /* skip strings */
 972         case '\"':
 973         case '\'':
 974             if (in_warn_or_error)
 975                 goto _default;
 976             p = parse_pp_string(p, c, NULL);
 977             break;
 978         /* skip comments */
 979         case '/':
 980             if (in_warn_or_error)
 981                 goto _default;
 982             file->buf_ptr = p;
 983             ch = *p;
 984             minp();
 985             p = file->buf_ptr;
 986             if (ch == '*') {
 987                 p = parse_comment(p);
 988             } else if (ch == '/') {
 989                 p = parse_line_comment(p);
 990             }
 991             break;
 992         case '#':
 993             p++;
 994             if (start_of_line) {
 995                 file->buf_ptr = p;
 996                 next_nomacro();
 997                 p = file->buf_ptr;
 998                 if (a == 0 &&
 999                     (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
1000                     goto the_end;
1001                 if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
1002                     a++;
1003                 else if (tok == TOK_ENDIF)
1004                     a--;
1005                 else if( tok == TOK_ERROR || tok == TOK_WARNING)
1006                     in_warn_or_error = 1;
1007                 else if (tok == TOK_LINEFEED)
1008                     goto redo_start;
1009                 else if (parse_flags & PARSE_FLAG_ASM_FILE)
1010                     p = parse_line_comment(p - 1);
1011             } else if (parse_flags & PARSE_FLAG_ASM_FILE)
1012                 p = parse_line_comment(p - 1);
1013             break;
1014 _default:
1015         default:
1016             p++;
1017             break;
1018         }
1019         start_of_line = 0;
1020     }
1021  the_end: ;
1022     file->buf_ptr = p;
1023 }
1024
1025 #if 0
1026 /* return the number of additional 'ints' necessary to store the
1027    token */
1028 static inline int tok_size(const int *p)
1029 {
1030     switch(*p) {
1031         /* 4 bytes */
1032     case TOK_CINT:
1033     case TOK_CUINT:
1034     case TOK_CCHAR:
1035     case TOK_LCHAR:
1036     case TOK_CFLOAT:
1037     case TOK_LINENUM:
1038         return 1 + 1;
1039     case TOK_STR:
1040     case TOK_LSTR:
1041     case TOK_PPNUM:
1042     case TOK_PPSTR:
1043         return 1 + ((sizeof(CString) + ((CString *)(p+1))->size + 3) >> 2);
1044     case TOK_CLONG:
1045     case TOK_CULONG:
1046         return 1 + LONG_SIZE / 4;
1047     case TOK_CDOUBLE:
1048     case TOK_CLLONG:
1049     case TOK_CULLONG:
1050         return 1 + 2;
1051     case TOK_CLDOUBLE:
1052         return 1 + LDOUBLE_SIZE / 4;
1053     default:
1054         return 1 + 0;
1055     }
1056 }
1057 #endif
1058
1059 /* token string handling */
1060 ST_INLN void tok_str_new(TokenString *s)
1061 {
1062     s->str = NULL;
1063     s->len = s->lastlen = 0;
1064     s->allocated_len = 0;
1065     s->last_line_num = -1;
1066 }
1067
1068 ST_FUNC TokenString *tok_str_alloc(void)
1069 {
1070     TokenString *str = tal_realloc(tokstr_alloc, 0, sizeof *str);
1071     tok_str_new(str);
1072     return str;
1073 }
1074
1075 ST_FUNC int *tok_str_dup(TokenString *s)
1076 {
1077     int *str;
1078
1079     str = tal_realloc(tokstr_alloc, 0, s->len * sizeof(int));
1080     memcpy(str, s->str, s->len * sizeof(int));
1081     return str;
1082 }
1083
1084 ST_FUNC void tok_str_free_str(int *str)
1085 {
1086     tal_free(tokstr_alloc, str);
1087 }
1088
1089 ST_FUNC void tok_str_free(TokenString *str)
1090 {
1091     tok_str_free_str(str->str);
1092     tal_free(tokstr_alloc, str);
1093 }
1094
1095 ST_FUNC int *tok_str_realloc(TokenString *s, int new_size)
1096 {
1097     int *str, size;
1098
1099     size = s->allocated_len;
1100     if (size < 16)
1101         size = 16;
1102     while (size < new_size)
1103         size = size * 2;
1104     if (size > s->allocated_len) {
1105         str = tal_realloc(tokstr_alloc, s->str, size * sizeof(int));
1106         s->allocated_len = size;
1107         s->str = str;
1108     }
1109     return s->str;
1110 }
1111
1112 ST_FUNC void tok_str_add(TokenString *s, int t)
1113 {
1114     int len, *str;
1115
1116     len = s->len;
1117     str = s->str;
1118     if (len >= s->allocated_len)
1119         str = tok_str_realloc(s, len + 1);
1120     str[len++] = t;
1121     s->len = len;
1122 }
1123
1124 ST_FUNC void begin_macro(TokenString *str, int alloc)
1125 {
1126     str->alloc = alloc;
1127     str->prev = macro_stack;
1128     str->prev_ptr = macro_ptr;
1129     str->save_line_num = file->line_num;
1130     macro_ptr = str->str;
1131     macro_stack = str;
1132 }
1133
1134 ST_FUNC void end_macro(void)
1135 {
1136     TokenString *str = macro_stack;
1137     macro_stack = str->prev;
1138     macro_ptr = str->prev_ptr;
1139     file->line_num = str->save_line_num;
1140     if (str->alloc != 0) {
1141         if (str->alloc == 2)
1142             str->str = NULL; /* don't free */
1143         tok_str_free(str);
1144     }
1145 }
1146
1147 static void tok_str_add2(TokenString *s, int t, CValue *cv)
1148 {
1149     int len, *str;
1150
1151     len = s->lastlen = s->len;
1152     str = s->str;
1153
1154     /* allocate space for worst case */
1155     if (len + TOK_MAX_SIZE >= s->allocated_len)
1156         str = tok_str_realloc(s, len + TOK_MAX_SIZE + 1);
1157     str[len++] = t;
1158     switch(t) {
1159     case TOK_CINT:
1160     case TOK_CUINT:
1161     case TOK_CCHAR:
1162     case TOK_LCHAR:
1163     case TOK_CFLOAT:
1164     case TOK_LINENUM:
1165 #if LONG_SIZE == 4
1166     case TOK_CLONG:
1167     case TOK_CULONG:
1168 #endif
1169         str[len++] = cv->tab[0];
1170         break;
1171     case TOK_PPNUM:
1172     case TOK_PPSTR:
1173     case TOK_STR:
1174     case TOK_LSTR:
1175         {
1176             /* Insert the string into the int array. */
1177             size_t nb_words =
1178                 1 + (cv->str.size + sizeof(int) - 1) / sizeof(int);
1179             if (len + nb_words >= s->allocated_len)
1180                 str = tok_str_realloc(s, len + nb_words + 1);
1181             str[len] = cv->str.size;
1182             memcpy(&str[len + 1], cv->str.data, cv->str.size);
1183             len += nb_words;
1184         }
1185         break;
1186     case TOK_CDOUBLE:
1187     case TOK_CLLONG:
1188     case TOK_CULLONG:
1189 #if LONG_SIZE == 8
1190     case TOK_CLONG:
1191     case TOK_CULONG:
1192 #endif
1193 #if LDOUBLE_SIZE == 8
1194     case TOK_CLDOUBLE:
1195 #endif
1196         str[len++] = cv->tab[0];
1197         str[len++] = cv->tab[1];
1198         break;
1199 #if LDOUBLE_SIZE == 12
1200     case TOK_CLDOUBLE:
1201         str[len++] = cv->tab[0];
1202         str[len++] = cv->tab[1];
1203         str[len++] = cv->tab[2];
1204 #elif LDOUBLE_SIZE == 16
1205     case TOK_CLDOUBLE:
1206         str[len++] = cv->tab[0];
1207         str[len++] = cv->tab[1];
1208         str[len++] = cv->tab[2];
1209         str[len++] = cv->tab[3];
1210 #elif LDOUBLE_SIZE != 8
1211 #error add long double size support
1212 #endif
1213         break;
1214     default:
1215         break;
1216     }
1217     s->len = len;
1218 }
1219
1220 /* add the current parse token in token string 's' */
1221 ST_FUNC void tok_str_add_tok(TokenString *s)
1222 {
1223     CValue cval;
1224
1225     /* save line number info */
1226     if (file->line_num != s->last_line_num) {
1227         s->last_line_num = file->line_num;
1228         cval.i = s->last_line_num;
1229         tok_str_add2(s, TOK_LINENUM, &cval);
1230     }
1231     tok_str_add2(s, tok, &tokc);
1232 }
1233
1234 /* get a token from an integer array and increment pointer. */
1235 static inline void TOK_GET(int *t, const int **pp, CValue *cv)
1236 {
1237     const int *p = *pp;
1238     int n, *tab;
1239
1240     tab = cv->tab;
1241     switch(*t = *p++) {
1242 #if LONG_SIZE == 4
1243     case TOK_CLONG:
1244 #endif
1245     case TOK_CINT:
1246     case TOK_CCHAR:
1247     case TOK_LCHAR:
1248     case TOK_LINENUM:
1249         cv->i = *p++;
1250         break;
1251 #if LONG_SIZE == 4
1252     case TOK_CULONG:
1253 #endif
1254     case TOK_CUINT:
1255         cv->i = (unsigned)*p++;
1256         break;
1257     case TOK_CFLOAT:
1258         tab[0] = *p++;
1259         break;
1260     case TOK_STR:
1261     case TOK_LSTR:
1262     case TOK_PPNUM:
1263     case TOK_PPSTR:
1264         cv->str.size = *p++;
1265         cv->str.data = p;
1266         p += (cv->str.size + sizeof(int) - 1) / sizeof(int);
1267         break;
1268     case TOK_CDOUBLE:
1269     case TOK_CLLONG:
1270     case TOK_CULLONG:
1271 #if LONG_SIZE == 8
1272     case TOK_CLONG:
1273     case TOK_CULONG:
1274 #endif
1275         n = 2;
1276         goto copy;
1277     case TOK_CLDOUBLE:
1278 #if LDOUBLE_SIZE == 16
1279         n = 4;
1280 #elif LDOUBLE_SIZE == 12
1281         n = 3;
1282 #elif LDOUBLE_SIZE == 8
1283         n = 2;
1284 #else
1285 # error add long double size support
1286 #endif
1287     copy:
1288         do
1289             *tab++ = *p++;
1290         while (--n);
1291         break;
1292     default:
1293         break;
1294     }
1295     *pp = p;
1296 }
1297
1298 static int macro_is_equal(const int *a, const int *b)
1299 {
1300     CValue cv;
1301     int t;
1302
1303     if (!a || !b)
1304         return 1;
1305
1306     while (*a && *b) {
1307         /* first time preallocate macro_equal_buf, next time only reset position to start */
1308         cstr_reset(&macro_equal_buf);
1309         TOK_GET(&t, &a, &cv);
1310         cstr_cat(&macro_equal_buf, get_tok_str(t, &cv), 0);
1311         TOK_GET(&t, &b, &cv);
1312         if (strcmp(macro_equal_buf.data, get_tok_str(t, &cv)))
1313             return 0;
1314     }
1315     return !(*a || *b);
1316 }
1317
1318 /* defines handling */
1319 ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
1320 {
1321     Sym *s, *o;
1322
1323     o = define_find(v);
1324     s = sym_push2(&define_stack, v, macro_type, 0);
1325     s->d = str;
1326     s->next = first_arg;
1327     table_ident[v - TOK_IDENT]->sym_define = s;
1328
1329     if (o && !macro_is_equal(o->d, s->d))
1330         tcc_warning("%s redefined", get_tok_str(v, NULL));
1331 }
1332
1333 /* undefined a define symbol. Its name is just set to zero */
1334 ST_FUNC void define_undef(Sym *s)
1335 {
1336     int v = s->v;
1337     if (v >= TOK_IDENT && v < tok_ident)
1338         table_ident[v - TOK_IDENT]->sym_define = NULL;
1339 }
1340
1341 ST_INLN Sym *define_find(int v)
1342 {
1343     v -= TOK_IDENT;
1344     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1345         return NULL;
1346     return table_ident[v]->sym_define;
1347 }
1348
1349 /* free define stack until top reaches 'b' */
1350 ST_FUNC void free_defines(Sym *b)
1351 {
1352     while (define_stack != b) {
1353         Sym *top = define_stack;
1354         define_stack = top->prev;
1355         tok_str_free_str(top->d);
1356         define_undef(top);
1357         sym_free(top);
1358     }
1359 }
1360
1361 /* label lookup */
1362 ST_FUNC Sym *label_find(int v)
1363 {
1364     v -= TOK_IDENT;
1365     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1366         return NULL;
1367     return table_ident[v]->sym_label;
1368 }
1369
1370 ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
1371 {
1372     Sym *s, **ps;
1373     s = sym_push2(ptop, v, 0, 0);
1374     s->r = flags;
1375     ps = &table_ident[v - TOK_IDENT]->sym_label;
1376     if (ptop == &global_label_stack) {
1377         /* modify the top most local identifier, so that
1378            sym_identifier will point to 's' when popped */
1379         while (*ps != NULL)
1380             ps = &(*ps)->prev_tok;
1381     }
1382     s->prev_tok = *ps;
1383     *ps = s;
1384     return s;
1385 }
1386
1387 /* pop labels until element last is reached. Look if any labels are
1388    undefined. Define symbols if '&&label' was used. */
1389 ST_FUNC void label_pop(Sym **ptop, Sym *slast, int keep)
1390 {
1391     Sym *s, *s1;
1392     for(s = *ptop; s != slast; s = s1) {
1393         s1 = s->prev;
1394         if (s->r == LABEL_DECLARED) {
1395             tcc_warning("label '%s' declared but not used", get_tok_str(s->v, NULL));
1396         } else if (s->r == LABEL_FORWARD) {
1397                 tcc_error("label '%s' used but not defined",
1398                       get_tok_str(s->v, NULL));
1399         } else {
1400             if (s->c) {
1401                 /* define corresponding symbol. A size of
1402                    1 is put. */
1403                 put_extern_sym(s, cur_text_section, s->jnext, 1);
1404             }
1405         }
1406         /* remove label */
1407         table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
1408         if (!keep)
1409             sym_free(s);
1410     }
1411     if (!keep)
1412         *ptop = slast;
1413 }
1414
1415 /* fake the nth "#if defined test_..." for tcc -dt -run */
1416 static void maybe_run_test(TCCState *s)
1417 {
1418     const char *p;
1419     if (s->include_stack_ptr != s->include_stack)
1420         return;
1421     p = get_tok_str(tok, NULL);
1422     if (0 != memcmp(p, "test_", 5))
1423         return;
1424     if (0 != --s->run_test)
1425         return;
1426     fprintf(s->ppfp, "\n[%s]\n" + !(s->dflag & 32), p), fflush(s->ppfp);
1427     define_push(tok, MACRO_OBJ, NULL, NULL);
1428 }
1429
1430 /* eval an expression for #if/#elif */
1431 static int expr_preprocess(void)
1432 {
1433     int c, t;
1434     TokenString *str;
1435
1436     str = tok_str_alloc();
1437     pp_expr = 1;
1438     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1439         next(); /* do macro subst */
1440         if (tok == TOK_DEFINED) {
1441             next_nomacro();
1442             t = tok;
1443             if (t == '(')
1444                 next_nomacro();
1445             if (tok < TOK_IDENT)
1446                 expect("identifier");
1447             if (tcc_state->run_test)
1448                 maybe_run_test(tcc_state);
1449             c = define_find(tok) != 0;
1450             if (t == '(') {
1451                 next_nomacro();
1452                 if (tok != ')')
1453                     expect("')'");
1454             }
1455             tok = TOK_CINT;
1456             tokc.i = c;
1457         } else if (tok >= TOK_IDENT) {
1458             /* if undefined macro */
1459             tok = TOK_CINT;
1460             tokc.i = 0;
1461         }
1462         tok_str_add_tok(str);
1463     }
1464     pp_expr = 0;
1465     tok_str_add(str, -1); /* simulate end of file */
1466     tok_str_add(str, 0);
1467     /* now evaluate C constant expression */
1468     begin_macro(str, 1);
1469     next();
1470     c = expr_const();
1471     end_macro();
1472     return c != 0;
1473 }
1474
1475
1476 /* parse after #define */
1477 ST_FUNC void parse_define(void)
1478 {
1479     Sym *s, *first, **ps;
1480     int v, t, varg, is_vaargs, spc;
1481     int saved_parse_flags = parse_flags;
1482
1483     v = tok;
1484     if (v < TOK_IDENT || v == TOK_DEFINED)
1485         tcc_error("invalid macro name '%s'", get_tok_str(tok, &tokc));
1486     /* XXX: should check if same macro (ANSI) */
1487     first = NULL;
1488     t = MACRO_OBJ;
1489     /* We have to parse the whole define as if not in asm mode, in particular
1490        no line comment with '#' must be ignored.  Also for function
1491        macros the argument list must be parsed without '.' being an ID
1492        character.  */
1493     parse_flags = ((parse_flags & ~PARSE_FLAG_ASM_FILE) | PARSE_FLAG_SPACES);
1494     /* '(' must be just after macro definition for MACRO_FUNC */
1495     next_nomacro_spc();
1496     if (tok == '(') {
1497         int dotid = set_idnum('.', 0);
1498         next_nomacro();
1499         ps = &first;
1500         if (tok != ')') for (;;) {
1501             varg = tok;
1502             next_nomacro();
1503             is_vaargs = 0;
1504             if (varg == TOK_DOTS) {
1505                 varg = TOK___VA_ARGS__;
1506                 is_vaargs = 1;
1507             } else if (tok == TOK_DOTS && gnu_ext) {
1508                 is_vaargs = 1;
1509                 next_nomacro();
1510             }
1511             if (varg < TOK_IDENT)
1512         bad_list:
1513                 tcc_error("bad macro parameter list");
1514             s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
1515             *ps = s;
1516             ps = &s->next;
1517             if (tok == ')')
1518                 break;
1519             if (tok != ',' || is_vaargs)
1520                 goto bad_list;
1521             next_nomacro();
1522         }
1523         next_nomacro_spc();
1524         t = MACRO_FUNC;
1525         set_idnum('.', dotid);
1526     }
1527
1528     tokstr_buf.len = 0;
1529     spc = 2;
1530     parse_flags |= PARSE_FLAG_ACCEPT_STRAYS | PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED;
1531     /* The body of a macro definition should be parsed such that identifiers
1532        are parsed like the file mode determines (i.e. with '.' being an
1533        ID character in asm mode).  But '#' should be retained instead of
1534        regarded as line comment leader, so still don't set ASM_FILE
1535        in parse_flags. */
1536     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1537         /* remove spaces around ## and after '#' */
1538         if (TOK_TWOSHARPS == tok) {
1539             if (2 == spc)
1540                 goto bad_twosharp;
1541             if (1 == spc)
1542                 --tokstr_buf.len;
1543             spc = 3;
1544             tok = TOK_PPJOIN;
1545         } else if ('#' == tok) {
1546             spc = 4;
1547         } else if (check_space(tok, &spc)) {
1548             goto skip;
1549         }
1550         tok_str_add2(&tokstr_buf, tok, &tokc);
1551     skip:
1552         next_nomacro_spc();
1553     }
1554
1555     parse_flags = saved_parse_flags;
1556     if (spc == 1)
1557         --tokstr_buf.len; /* remove trailing space */
1558     tok_str_add(&tokstr_buf, 0);
1559     if (3 == spc)
1560 bad_twosharp:
1561         tcc_error("'##' cannot appear at either end of macro");
1562     define_push(v, t, tok_str_dup(&tokstr_buf), first);
1563 }
1564
1565 static CachedInclude *search_cached_include(TCCState *s1, const char *filename, int add)
1566 {
1567     const unsigned char *s;
1568     unsigned int h;
1569     CachedInclude *e;
1570     int i;
1571
1572     h = TOK_HASH_INIT;
1573     s = (unsigned char *) filename;
1574     while (*s) {
1575 #ifdef _WIN32
1576         h = TOK_HASH_FUNC(h, toup(*s));
1577 #else
1578         h = TOK_HASH_FUNC(h, *s);
1579 #endif
1580         s++;
1581     }
1582     h &= (CACHED_INCLUDES_HASH_SIZE - 1);
1583
1584     i = s1->cached_includes_hash[h];
1585     for(;;) {
1586         if (i == 0)
1587             break;
1588         e = s1->cached_includes[i - 1];
1589         if (0 == PATHCMP(e->filename, filename))
1590             return e;
1591         i = e->hash_next;
1592     }
1593     if (!add)
1594         return NULL;
1595
1596     e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
1597     strcpy(e->filename, filename);
1598     e->ifndef_macro = e->once = 0;
1599     dynarray_add(&s1->cached_includes, &s1->nb_cached_includes, e);
1600     /* add in hash table */
1601     e->hash_next = s1->cached_includes_hash[h];
1602     s1->cached_includes_hash[h] = s1->nb_cached_includes;
1603 #ifdef INC_DEBUG
1604     printf("adding cached '%s'\n", filename);
1605 #endif
1606     return e;
1607 }
1608
1609 static void pragma_parse(TCCState *s1)
1610 {
1611     next_nomacro();
1612     if (tok == TOK_push_macro || tok == TOK_pop_macro) {
1613         int t = tok, v;
1614         Sym *s;
1615
1616         if (next(), tok != '(')
1617             goto pragma_err;
1618         if (next(), tok != TOK_STR)
1619             goto pragma_err;
1620         v = tok_alloc(tokc.str.data, tokc.str.size - 1)->tok;
1621         if (next(), tok != ')')
1622             goto pragma_err;
1623         if (t == TOK_push_macro) {
1624             while (NULL == (s = define_find(v)))
1625                 define_push(v, 0, NULL, NULL);
1626             s->type.ref = s; /* set push boundary */
1627         } else {
1628             for (s = define_stack; s; s = s->prev)
1629                 if (s->v == v && s->type.ref == s) {
1630                     s->type.ref = NULL;
1631                     break;
1632                 }
1633         }
1634         if (s)
1635             table_ident[v - TOK_IDENT]->sym_define = s->d ? s : NULL;
1636         else
1637             tcc_warning("unbalanced #pragma pop_macro");
1638         pp_debug_tok = t, pp_debug_symv = v;
1639
1640     } else if (tok == TOK_once) {
1641         search_cached_include(s1, file->filename, 1)->once = pp_once;
1642
1643     } else if (s1->output_type == TCC_OUTPUT_PREPROCESS) {
1644         /* tcc -E: keep pragmas below unchanged */
1645         unget_tok(' ');
1646         unget_tok(TOK_PRAGMA);
1647         unget_tok('#');
1648         unget_tok(TOK_LINEFEED);
1649
1650     } else if (tok == TOK_pack) {
1651         /* This may be:
1652            #pragma pack(1) // set
1653            #pragma pack() // reset to default
1654            #pragma pack(push,1) // push & set
1655            #pragma pack(pop) // restore previous */
1656         next();
1657         skip('(');
1658         if (tok == TOK_ASM_pop) {
1659             next();
1660             if (s1->pack_stack_ptr <= s1->pack_stack) {
1661             stk_error:
1662                 tcc_error("out of pack stack");
1663             }
1664             s1->pack_stack_ptr--;
1665         } else {
1666             int val = 0;
1667             if (tok != ')') {
1668                 if (tok == TOK_ASM_push) {
1669                     next();
1670                     if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
1671                         goto stk_error;
1672                     s1->pack_stack_ptr++;
1673                     skip(',');
1674                 }
1675                 if (tok != TOK_CINT)
1676                     goto pragma_err;
1677                 val = tokc.i;
1678                 if (val < 1 || val > 16 || (val & (val - 1)) != 0)
1679                     goto pragma_err;
1680                 next();
1681             }
1682             *s1->pack_stack_ptr = val;
1683         }
1684         if (tok != ')')
1685             goto pragma_err;
1686
1687     } else if (tok == TOK_comment) {
1688         char *p; int t;
1689         next();
1690         skip('(');
1691         t = tok;
1692         next();
1693         skip(',');
1694         if (tok != TOK_STR)
1695             goto pragma_err;
1696         p = tcc_strdup((char *)tokc.str.data);
1697         next();
1698         if (tok != ')')
1699             goto pragma_err;
1700         if (t == TOK_lib) {
1701             dynarray_add(&s1->pragma_libs, &s1->nb_pragma_libs, p);
1702         } else {
1703             if (t == TOK_option)
1704                 tcc_set_options(s1, p);
1705             tcc_free(p);
1706         }
1707
1708     } else if (s1->warn_unsupported) {
1709         tcc_warning("#pragma %s is ignored", get_tok_str(tok, &tokc));
1710     }
1711     return;
1712
1713 pragma_err:
1714     tcc_error("malformed #pragma directive");
1715     return;
1716 }
1717
1718 /* is_bof is true if first non space token at beginning of file */
1719 ST_FUNC void preprocess(int is_bof)
1720 {
1721     TCCState *s1 = tcc_state;
1722     int i, c, n, saved_parse_flags;
1723     char buf[1024], *q;
1724     Sym *s;
1725
1726     saved_parse_flags = parse_flags;
1727     parse_flags = PARSE_FLAG_PREPROCESS
1728         | PARSE_FLAG_TOK_NUM
1729         | PARSE_FLAG_TOK_STR
1730         | PARSE_FLAG_LINEFEED
1731         | (parse_flags & PARSE_FLAG_ASM_FILE)
1732         ;
1733
1734     next_nomacro();
1735  redo:
1736     switch(tok) {
1737     case TOK_DEFINE:
1738         pp_debug_tok = tok;
1739         next_nomacro();
1740         pp_debug_symv = tok;
1741         parse_define();
1742         break;
1743     case TOK_UNDEF:
1744         pp_debug_tok = tok;
1745         next_nomacro();
1746         pp_debug_symv = tok;
1747         s = define_find(tok);
1748         /* undefine symbol by putting an invalid name */
1749         if (s)
1750             define_undef(s);
1751         break;
1752     case TOK_INCLUDE:
1753     case TOK_INCLUDE_NEXT:
1754         ch = file->buf_ptr[0];
1755         /* XXX: incorrect if comments : use next_nomacro with a special mode */
1756         skip_spaces();
1757         if (ch == '<') {
1758             c = '>';
1759             goto read_name;
1760         } else if (ch == '\"') {
1761             c = ch;
1762         read_name:
1763             inp();
1764             q = buf;
1765             while (ch != c && ch != '\n' && ch != CH_EOF) {
1766                 if ((q - buf) < sizeof(buf) - 1)
1767                     *q++ = ch;
1768                 if (ch == '\\') {
1769                     if (handle_stray_noerror() == 0)
1770                         --q;
1771                 } else
1772                     inp();
1773             }
1774             *q = '\0';
1775             minp();
1776 #if 0
1777             /* eat all spaces and comments after include */
1778             /* XXX: slightly incorrect */
1779             while (ch1 != '\n' && ch1 != CH_EOF)
1780                 inp();
1781 #endif
1782         } else {
1783             int len;
1784             /* computed #include : concatenate everything up to linefeed,
1785                the result must be one of the two accepted forms.
1786                Don't convert pp-tokens to tokens here.  */
1787             parse_flags = (PARSE_FLAG_PREPROCESS
1788                            | PARSE_FLAG_LINEFEED
1789                            | (parse_flags & PARSE_FLAG_ASM_FILE));
1790             next();
1791             buf[0] = '\0';
1792             while (tok != TOK_LINEFEED) {
1793                 pstrcat(buf, sizeof(buf), get_tok_str(tok, &tokc));
1794                 next();
1795             }
1796             len = strlen(buf);
1797             /* check syntax and remove '<>|""' */
1798             if ((len < 2 || ((buf[0] != '"' || buf[len-1] != '"') &&
1799                              (buf[0] != '<' || buf[len-1] != '>'))))
1800                 tcc_error("'#include' expects \"FILENAME\" or <FILENAME>");
1801             c = buf[len-1];
1802             memmove(buf, buf + 1, len - 2);
1803             buf[len - 2] = '\0';
1804         }
1805
1806         if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
1807             tcc_error("#include recursion too deep");
1808         /* push current file on stack */
1809         *s1->include_stack_ptr++ = file;
1810         i = tok == TOK_INCLUDE_NEXT ? file->include_next_index: 0;
1811         n = 2 + s1->nb_include_paths + s1->nb_sysinclude_paths;
1812         for (; i < n; ++i) {
1813             char buf1[sizeof file->filename];
1814             CachedInclude *e;
1815             const char *path;
1816
1817             if (i == 0) {
1818                 /* check absolute include path */
1819                 if (!IS_ABSPATH(buf))
1820                     continue;
1821                 buf1[0] = 0;
1822
1823             } else if (i == 1) {
1824                 /* search in file's dir if "header.h" */
1825                 if (c != '\"')
1826                     continue;
1827                 /* https://savannah.nongnu.org/bugs/index.php?50847 */
1828                 path = file->true_filename;
1829                 pstrncpy(buf1, path, tcc_basename(path) - path);
1830
1831             } else {
1832                 /* search in all the include paths */
1833                 int j = i - 2, k = j - s1->nb_include_paths;
1834                 path = k < 0 ? s1->include_paths[j] : s1->sysinclude_paths[k];
1835                 pstrcpy(buf1, sizeof(buf1), path);
1836                 pstrcat(buf1, sizeof(buf1), "/");
1837             }
1838
1839             pstrcat(buf1, sizeof(buf1), buf);
1840             e = search_cached_include(s1, buf1, 0);
1841             if (e && (define_find(e->ifndef_macro) || e->once == pp_once)) {
1842                 /* no need to parse the include because the 'ifndef macro'
1843                    is defined (or had #pragma once) */
1844 #ifdef INC_DEBUG
1845                 printf("%s: skipping cached %s\n", file->filename, buf1);
1846 #endif
1847                 goto include_done;
1848             }
1849
1850             if (tcc_open(s1, buf1) < 0)
1851                 continue;
1852
1853             file->include_next_index = i + 1;
1854 #ifdef INC_DEBUG
1855             printf("%s: including %s\n", file->prev->filename, file->filename);
1856 #endif
1857             /* update target deps */
1858             if (s1->gen_deps) {
1859                 dynarray_add(&s1->target_deps, &s1->nb_target_deps,
1860                     tcc_strdup(buf1));
1861             }
1862             /* add include file debug info */
1863             tcc_debug_bincl(tcc_state);
1864             tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
1865             ch = file->buf_ptr[0];
1866             goto the_end;
1867         }
1868         tcc_error("include file '%s' not found", buf);
1869 include_done:
1870         --s1->include_stack_ptr;
1871         break;
1872     case TOK_IFNDEF:
1873         c = 1;
1874         goto do_ifdef;
1875     case TOK_IF:
1876         c = expr_preprocess();
1877         goto do_if;
1878     case TOK_IFDEF:
1879         c = 0;
1880     do_ifdef:
1881         next_nomacro();
1882         if (tok < TOK_IDENT)
1883             tcc_error("invalid argument for '#if%sdef'", c ? "n" : "");
1884         if (is_bof) {
1885             if (c) {
1886 #ifdef INC_DEBUG
1887                 printf("#ifndef %s\n", get_tok_str(tok, NULL));
1888 #endif
1889                 file->ifndef_macro = tok;
1890             }
1891         }
1892         c = (define_find(tok) != 0) ^ c;
1893     do_if:
1894         if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
1895             tcc_error("memory full (ifdef)");
1896         *s1->ifdef_stack_ptr++ = c;
1897         goto test_skip;
1898     case TOK_ELSE:
1899         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1900             tcc_error("#else without matching #if");
1901         if (s1->ifdef_stack_ptr[-1] & 2)
1902             tcc_error("#else after #else");
1903         c = (s1->ifdef_stack_ptr[-1] ^= 3);
1904         goto test_else;
1905     case TOK_ELIF:
1906         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1907             tcc_error("#elif without matching #if");
1908         c = s1->ifdef_stack_ptr[-1];
1909         if (c > 1)
1910             tcc_error("#elif after #else");
1911         /* last #if/#elif expression was true: we skip */
1912         if (c == 1) {
1913             c = 0;
1914         } else {
1915             c = expr_preprocess();
1916             s1->ifdef_stack_ptr[-1] = c;
1917         }
1918     test_else:
1919         if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1)
1920             file->ifndef_macro = 0;
1921     test_skip:
1922         if (!(c & 1)) {
1923             preprocess_skip();
1924             is_bof = 0;
1925             goto redo;
1926         }
1927         break;
1928     case TOK_ENDIF:
1929         if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
1930             tcc_error("#endif without matching #if");
1931         s1->ifdef_stack_ptr--;
1932         /* '#ifndef macro' was at the start of file. Now we check if
1933            an '#endif' is exactly at the end of file */
1934         if (file->ifndef_macro &&
1935             s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
1936             file->ifndef_macro_saved = file->ifndef_macro;
1937             /* need to set to zero to avoid false matches if another
1938                #ifndef at middle of file */
1939             file->ifndef_macro = 0;
1940             while (tok != TOK_LINEFEED)
1941                 next_nomacro();
1942             tok_flags |= TOK_FLAG_ENDIF;
1943             goto the_end;
1944         }
1945         break;
1946     case TOK_PPNUM:
1947         n = strtoul((char*)tokc.str.data, &q, 10);
1948         goto _line_num;
1949     case TOK_LINE:
1950         next();
1951         if (tok != TOK_CINT)
1952     _line_err:
1953             tcc_error("wrong #line format");
1954         n = tokc.i;
1955     _line_num:
1956         next();
1957         if (tok != TOK_LINEFEED) {
1958             if (tok == TOK_STR) {
1959                 if (file->true_filename == file->filename)
1960                     file->true_filename = tcc_strdup(file->filename);
1961                 tcc_debug_putfile(s1, (char *)tokc.str.data);
1962             } else if (parse_flags & PARSE_FLAG_ASM_FILE)
1963                 break;
1964             else
1965                 goto _line_err;
1966             --n;
1967         }
1968         if (file->fd > 0)
1969             total_lines += file->line_num - n;
1970         file->line_num = n;
1971         break;
1972     case TOK_ERROR:
1973     case TOK_WARNING:
1974         c = tok;
1975         ch = file->buf_ptr[0];
1976         skip_spaces();
1977         q = buf;
1978         while (ch != '\n' && ch != CH_EOF) {
1979             if ((q - buf) < sizeof(buf) - 1)
1980                 *q++ = ch;
1981             if (ch == '\\') {
1982                 if (handle_stray_noerror() == 0)
1983                     --q;
1984             } else
1985                 inp();
1986         }
1987         *q = '\0';
1988         if (c == TOK_ERROR)
1989             tcc_error("#error %s", buf);
1990         else
1991             tcc_warning("#warning %s", buf);
1992         break;
1993     case TOK_PRAGMA:
1994         pragma_parse(s1);
1995         break;
1996     case TOK_LINEFEED:
1997         goto the_end;
1998     default:
1999         /* ignore gas line comment in an 'S' file. */
2000         if (saved_parse_flags & PARSE_FLAG_ASM_FILE)
2001             goto ignore;
2002         if (tok == '!' && is_bof)
2003             /* '!' is ignored at beginning to allow C scripts. */
2004             goto ignore;
2005         tcc_warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
2006     ignore:
2007         file->buf_ptr = parse_line_comment(file->buf_ptr - 1);
2008         goto the_end;
2009     }
2010     /* ignore other preprocess commands or #! for C scripts */
2011     while (tok != TOK_LINEFEED)
2012         next_nomacro();
2013  the_end:
2014     parse_flags = saved_parse_flags;
2015 }
2016
2017 /* evaluate escape codes in a string. */
2018 static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
2019 {
2020     int c, n;
2021     const uint8_t *p;
2022
2023     p = buf;
2024     for(;;) {
2025         c = *p;
2026         if (c == '\0')
2027             break;
2028         if (c == '\\') {
2029             p++;
2030             /* escape */
2031             c = *p;
2032             switch(c) {
2033             case '0': case '1': case '2': case '3':
2034             case '4': case '5': case '6': case '7':
2035                 /* at most three octal digits */
2036                 n = c - '0';
2037                 p++;
2038                 c = *p;
2039                 if (isoct(c)) {
2040                     n = n * 8 + c - '0';
2041                     p++;
2042                     c = *p;
2043                     if (isoct(c)) {
2044                         n = n * 8 + c - '0';
2045                         p++;
2046                     }
2047                 }
2048                 c = n;
2049                 goto add_char_nonext;
2050             case 'x':
2051             case 'u':
2052             case 'U':
2053                 p++;
2054                 n = 0;
2055                 for(;;) {
2056                     c = *p;
2057                     if (c >= 'a' && c <= 'f')
2058                         c = c - 'a' + 10;
2059                     else if (c >= 'A' && c <= 'F')
2060                         c = c - 'A' + 10;
2061                     else if (isnum(c))
2062                         c = c - '0';
2063                     else
2064                         break;
2065                     n = n * 16 + c;
2066                     p++;
2067                 }
2068                 c = n;
2069                 goto add_char_nonext;
2070             case 'a':
2071                 c = '\a';
2072                 break;
2073             case 'b':
2074                 c = '\b';
2075                 break;
2076             case 'f':
2077                 c = '\f';
2078                 break;
2079             case 'n':
2080                 c = '\n';
2081                 break;
2082             case 'r':
2083                 c = '\r';
2084                 break;
2085             case 't':
2086                 c = '\t';
2087                 break;
2088             case 'v':
2089                 c = '\v';
2090                 break;
2091             case 'e':
2092                 if (!gnu_ext)
2093                     goto invalid_escape;
2094                 c = 27;
2095                 break;
2096             case '\'':
2097             case '\"':
2098             case '\\':
2099             case '?':
2100                 break;
2101             default:
2102             invalid_escape:
2103                 if (c >= '!' && c <= '~')
2104                     tcc_warning("unknown escape sequence: \'\\%c\'", c);
2105                 else
2106                     tcc_warning("unknown escape sequence: \'\\x%x\'", c);
2107                 break;
2108             }
2109         } else if (is_long && c >= 0x80) {
2110             /* assume we are processing UTF-8 sequence */
2111             /* reference: The Unicode Standard, Version 10.0, ch3.9 */
2112
2113             int cont; /* count of continuation bytes */
2114             int skip; /* how many bytes should skip when error occurred */
2115             int i;
2116
2117             /* decode leading byte */
2118             if (c < 0xC2) {
2119                     skip = 1; goto invalid_utf8_sequence;
2120             } else if (c <= 0xDF) {
2121                     cont = 1; n = c & 0x1f;
2122             } else if (c <= 0xEF) {
2123                     cont = 2; n = c & 0xf;
2124             } else if (c <= 0xF4) {
2125                     cont = 3; n = c & 0x7;
2126             } else {
2127                     skip = 1; goto invalid_utf8_sequence;
2128             }
2129
2130             /* decode continuation bytes */
2131             for (i = 1; i <= cont; i++) {
2132                 int l = 0x80, h = 0xBF;
2133
2134                 /* adjust limit for second byte */
2135                 if (i == 1) {
2136                     switch (c) {
2137                     case 0xE0: l = 0xA0; break;
2138                     case 0xED: h = 0x9F; break;
2139                     case 0xF0: l = 0x90; break;
2140                     case 0xF4: h = 0x8F; break;
2141                     }
2142                 }
2143
2144                 if (p[i] < l || p[i] > h) {
2145                     skip = i; goto invalid_utf8_sequence;
2146                 }
2147
2148                 n = (n << 6) | (p[i] & 0x3f);
2149             }
2150
2151             /* advance pointer */
2152             p += 1 + cont;
2153             c = n;
2154             goto add_char_nonext;
2155
2156             /* error handling */
2157         invalid_utf8_sequence:
2158             tcc_warning("ill-formed UTF-8 subsequence starting with: \'\\x%x\'", c);
2159             c = 0xFFFD;
2160             p += skip;
2161             goto add_char_nonext;
2162
2163         }
2164         p++;
2165     add_char_nonext:
2166         if (!is_long)
2167             cstr_ccat(outstr, c);
2168         else {
2169 #ifdef TCC_TARGET_PE
2170             /* store as UTF-16 */
2171             if (c < 0x10000) {
2172                 cstr_wccat(outstr, c);
2173             } else {
2174                 c -= 0x10000;
2175                 cstr_wccat(outstr, (c >> 10) + 0xD800);
2176                 cstr_wccat(outstr, (c & 0x3FF) + 0xDC00);
2177             }
2178 #else
2179             cstr_wccat(outstr, c);
2180 #endif
2181         }
2182     }
2183     /* add a trailing '\0' */
2184     if (!is_long)
2185         cstr_ccat(outstr, '\0');
2186     else
2187         cstr_wccat(outstr, '\0');
2188 }
2189
2190 static void parse_string(const char *s, int len)
2191 {
2192     uint8_t buf[1000], *p = buf;
2193     int is_long, sep;
2194
2195     if ((is_long = *s == 'L'))
2196         ++s, --len;
2197     sep = *s++;
2198     len -= 2;
2199     if (len >= sizeof buf)
2200         p = tcc_malloc(len + 1);
2201     memcpy(p, s, len);
2202     p[len] = 0;
2203
2204     cstr_reset(&tokcstr);
2205     parse_escape_string(&tokcstr, p, is_long);
2206     if (p != buf)
2207         tcc_free(p);
2208
2209     if (sep == '\'') {
2210         int char_size, i, n, c;
2211         /* XXX: make it portable */
2212         if (!is_long)
2213             tok = TOK_CCHAR, char_size = 1;
2214         else
2215             tok = TOK_LCHAR, char_size = sizeof(nwchar_t);
2216         n = tokcstr.size / char_size - 1;
2217         if (n < 1)
2218             tcc_error("empty character constant");
2219         if (n > 1)
2220             tcc_warning("multi-character character constant");
2221         for (c = i = 0; i < n; ++i) {
2222             if (is_long)
2223                 c = ((nwchar_t *)tokcstr.data)[i];
2224             else
2225                 c = (c << 8) | ((char *)tokcstr.data)[i];
2226         }
2227         tokc.i = c;
2228     } else {
2229         tokc.str.size = tokcstr.size;
2230         tokc.str.data = tokcstr.data;
2231         if (!is_long)
2232             tok = TOK_STR;
2233         else
2234             tok = TOK_LSTR;
2235     }
2236 }
2237
2238 /* we use 64 bit numbers */
2239 #define BN_SIZE 2
2240
2241 /* bn = (bn << shift) | or_val */
2242 static void bn_lshift(unsigned int *bn, int shift, int or_val)
2243 {
2244     int i;
2245     unsigned int v;
2246     for(i=0;i<BN_SIZE;i++) {
2247         v = bn[i];
2248         bn[i] = (v << shift) | or_val;
2249         or_val = v >> (32 - shift);
2250     }
2251 }
2252
2253 static void bn_zero(unsigned int *bn)
2254 {
2255     int i;
2256     for(i=0;i<BN_SIZE;i++) {
2257         bn[i] = 0;
2258     }
2259 }
2260
2261 /* parse number in null terminated string 'p' and return it in the
2262    current token */
2263 static void parse_number(const char *p)
2264 {
2265     int b, t, shift, frac_bits, s, exp_val, ch;
2266     char *q;
2267     unsigned int bn[BN_SIZE];
2268     double d;
2269
2270     /* number */
2271     q = token_buf;
2272     ch = *p++;
2273     t = ch;
2274     ch = *p++;
2275     *q++ = t;
2276     b = 10;
2277     if (t == '.') {
2278         goto float_frac_parse;
2279     } else if (t == '0') {
2280         if (ch == 'x' || ch == 'X') {
2281             q--;
2282             ch = *p++;
2283             b = 16;
2284         } else if (tcc_state->tcc_ext && (ch == 'b' || ch == 'B')) {
2285             q--;
2286             ch = *p++;
2287             b = 2;
2288         }
2289     }
2290     /* parse all digits. cannot check octal numbers at this stage
2291        because of floating point constants */
2292     while (1) {
2293         if (ch >= 'a' && ch <= 'f')
2294             t = ch - 'a' + 10;
2295         else if (ch >= 'A' && ch <= 'F')
2296             t = ch - 'A' + 10;
2297         else if (isnum(ch))
2298             t = ch - '0';
2299         else
2300             break;
2301         if (t >= b)
2302             break;
2303         if (q >= token_buf + STRING_MAX_SIZE) {
2304         num_too_long:
2305             tcc_error("number too long");
2306         }
2307         *q++ = ch;
2308         ch = *p++;
2309     }
2310     if (ch == '.' ||
2311         ((ch == 'e' || ch == 'E') && b == 10) ||
2312         ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
2313         if (b != 10) {
2314             /* NOTE: strtox should support that for hexa numbers, but
2315                non ISOC99 libcs do not support it, so we prefer to do
2316                it by hand */
2317             /* hexadecimal or binary floats */
2318             /* XXX: handle overflows */
2319             *q = '\0';
2320             if (b == 16)
2321                 shift = 4;
2322             else
2323                 shift = 1;
2324             bn_zero(bn);
2325             q = token_buf;
2326             while (1) {
2327                 t = *q++;
2328                 if (t == '\0') {
2329                     break;
2330                 } else if (t >= 'a') {
2331                     t = t - 'a' + 10;
2332                 } else if (t >= 'A') {
2333                     t = t - 'A' + 10;
2334                 } else {
2335                     t = t - '0';
2336                 }
2337                 bn_lshift(bn, shift, t);
2338             }
2339             frac_bits = 0;
2340             if (ch == '.') {
2341                 ch = *p++;
2342                 while (1) {
2343                     t = ch;
2344                     if (t >= 'a' && t <= 'f') {
2345                         t = t - 'a' + 10;
2346                     } else if (t >= 'A' && t <= 'F') {
2347                         t = t - 'A' + 10;
2348                     } else if (t >= '0' && t <= '9') {
2349                         t = t - '0';
2350                     } else {
2351                         break;
2352                     }
2353                     if (t >= b)
2354                         tcc_error("invalid digit");
2355                     bn_lshift(bn, shift, t);
2356                     frac_bits += shift;
2357                     ch = *p++;
2358                 }
2359             }
2360             if (ch != 'p' && ch != 'P')
2361                 expect("exponent");
2362             ch = *p++;
2363             s = 1;
2364             exp_val = 0;
2365             if (ch == '+') {
2366                 ch = *p++;
2367             } else if (ch == '-') {
2368                 s = -1;
2369                 ch = *p++;
2370             }
2371             if (ch < '0' || ch > '9')
2372                 expect("exponent digits");
2373             while (ch >= '0' && ch <= '9') {
2374                 exp_val = exp_val * 10 + ch - '0';
2375                 ch = *p++;
2376             }
2377             exp_val = exp_val * s;
2378
2379             /* now we can generate the number */
2380             /* XXX: should patch directly float number */
2381             d = (double)bn[1] * 4294967296.0 + (double)bn[0];
2382             d = ldexp(d, exp_val - frac_bits);
2383             t = toup(ch);
2384             if (t == 'F') {
2385                 ch = *p++;
2386                 tok = TOK_CFLOAT;
2387                 /* float : should handle overflow */
2388                 tokc.f = (float)d;
2389             } else if (t == 'L') {
2390                 ch = *p++;
2391 #ifdef TCC_TARGET_PE
2392                 tok = TOK_CDOUBLE;
2393                 tokc.d = d;
2394 #else
2395                 tok = TOK_CLDOUBLE;
2396                 /* XXX: not large enough */
2397                 tokc.ld = (long double)d;
2398 #endif
2399             } else {
2400                 tok = TOK_CDOUBLE;
2401                 tokc.d = d;
2402             }
2403         } else {
2404             /* decimal floats */
2405             if (ch == '.') {
2406                 if (q >= token_buf + STRING_MAX_SIZE)
2407                     goto num_too_long;
2408                 *q++ = ch;
2409                 ch = *p++;
2410             float_frac_parse:
2411                 while (ch >= '0' && ch <= '9') {
2412                     if (q >= token_buf + STRING_MAX_SIZE)
2413                         goto num_too_long;
2414                     *q++ = ch;
2415                     ch = *p++;
2416                 }
2417             }
2418             if (ch == 'e' || ch == 'E') {
2419                 if (q >= token_buf + STRING_MAX_SIZE)
2420                     goto num_too_long;
2421                 *q++ = ch;
2422                 ch = *p++;
2423                 if (ch == '-' || ch == '+') {
2424                     if (q >= token_buf + STRING_MAX_SIZE)
2425                         goto num_too_long;
2426                     *q++ = ch;
2427                     ch = *p++;
2428                 }
2429                 if (ch < '0' || ch > '9')
2430                     expect("exponent digits");
2431                 while (ch >= '0' && ch <= '9') {
2432                     if (q >= token_buf + STRING_MAX_SIZE)
2433                         goto num_too_long;
2434                     *q++ = ch;
2435                     ch = *p++;
2436                 }
2437             }
2438             *q = '\0';
2439             t = toup(ch);
2440             errno = 0;
2441             if (t == 'F') {
2442                 ch = *p++;
2443                 tok = TOK_CFLOAT;
2444                 tokc.f = strtof(token_buf, NULL);
2445             } else if (t == 'L') {
2446                 ch = *p++;
2447 #ifdef TCC_TARGET_PE
2448                 tok = TOK_CDOUBLE;
2449                 tokc.d = strtod(token_buf, NULL);
2450 #else
2451                 tok = TOK_CLDOUBLE;
2452                 tokc.ld = strtold(token_buf, NULL);
2453 #endif
2454             } else {
2455                 tok = TOK_CDOUBLE;
2456                 tokc.d = strtod(token_buf, NULL);
2457             }
2458         }
2459     } else {
2460         unsigned long long n, n1;
2461         int lcount, ucount, ov = 0;
2462         const char *p1;
2463
2464         /* integer number */
2465         *q = '\0';
2466         q = token_buf;
2467         if (b == 10 && *q == '0') {
2468             b = 8;
2469             q++;
2470         }
2471         n = 0;
2472         while(1) {
2473             t = *q++;
2474             /* no need for checks except for base 10 / 8 errors */
2475             if (t == '\0')
2476                 break;
2477             else if (t >= 'a')
2478                 t = t - 'a' + 10;
2479             else if (t >= 'A')
2480                 t = t - 'A' + 10;
2481             else
2482                 t = t - '0';
2483             if (t >= b)
2484                 tcc_error("invalid digit");
2485             n1 = n;
2486             n = n * b + t;
2487             /* detect overflow */
2488             if (n1 >= 0x1000000000000000ULL && n / b != n1)
2489                 ov = 1;
2490         }
2491
2492         /* Determine the characteristics (unsigned and/or 64bit) the type of
2493            the constant must have according to the constant suffix(es) */
2494         lcount = ucount = 0;
2495         p1 = p;
2496         for(;;) {
2497             t = toup(ch);
2498             if (t == 'L') {
2499                 if (lcount >= 2)
2500                     tcc_error("three 'l's in integer constant");
2501                 if (lcount && *(p - 1) != ch)
2502                     tcc_error("incorrect integer suffix: %s", p1);
2503                 lcount++;
2504                 ch = *p++;
2505             } else if (t == 'U') {
2506                 if (ucount >= 1)
2507                     tcc_error("two 'u's in integer constant");
2508                 ucount++;
2509                 ch = *p++;
2510             } else {
2511                 break;
2512             }
2513         }
2514
2515         /* Determine if it needs 64 bits and/or unsigned in order to fit */
2516         if (ucount == 0 && b == 10) {
2517             if (lcount <= (LONG_SIZE == 4)) {
2518                 if (n >= 0x80000000U)
2519                     lcount = (LONG_SIZE == 4) + 1;
2520             }
2521             if (n >= 0x8000000000000000ULL)
2522                 ov = 1, ucount = 1;
2523         } else {
2524             if (lcount <= (LONG_SIZE == 4)) {
2525                 if (n >= 0x100000000ULL)
2526                     lcount = (LONG_SIZE == 4) + 1;
2527                 else if (n >= 0x80000000U)
2528                     ucount = 1;
2529             }
2530             if (n >= 0x8000000000000000ULL)
2531                 ucount = 1;
2532         }
2533
2534         if (ov)
2535             tcc_warning("integer constant overflow");
2536
2537         tok = TOK_CINT;
2538         if (lcount) {
2539             tok = TOK_CLONG;
2540             if (lcount == 2)
2541                 tok = TOK_CLLONG;
2542         }
2543         if (ucount)
2544             ++tok; /* TOK_CU... */
2545         tokc.i = n;
2546     }
2547     if (ch)
2548         tcc_error("invalid number\n");
2549 }
2550
2551
2552 #define PARSE2(c1, tok1, c2, tok2)              \
2553     case c1:                                    \
2554         PEEKC(c, p);                            \
2555         if (c == c2) {                          \
2556             p++;                                \
2557             tok = tok2;                         \
2558         } else {                                \
2559             tok = tok1;                         \
2560         }                                       \
2561         break;
2562
2563 /* return next token without macro substitution */
2564 static inline void next_nomacro1(void)
2565 {
2566     int t, c, is_long, len;
2567     TokenSym *ts;
2568     uint8_t *p, *p1;
2569     unsigned int h;
2570
2571     p = file->buf_ptr;
2572  redo_no_start:
2573     c = *p;
2574     switch(c) {
2575     case ' ':
2576     case '\t':
2577         tok = c;
2578         p++;
2579         if (parse_flags & PARSE_FLAG_SPACES)
2580             goto keep_tok_flags;
2581         while (isidnum_table[*p - CH_EOF] & IS_SPC)
2582             ++p;
2583         goto redo_no_start;
2584     case '\f':
2585     case '\v':
2586     case '\r':
2587         p++;
2588         goto redo_no_start;
2589     case '\\':
2590         /* first look if it is in fact an end of buffer */
2591         c = handle_stray1(p);
2592         p = file->buf_ptr;
2593         if (c == '\\')
2594             goto parse_simple;
2595         if (c != CH_EOF)
2596             goto redo_no_start;
2597         {
2598             TCCState *s1 = tcc_state;
2599             if ((parse_flags & PARSE_FLAG_LINEFEED)
2600                 && !(tok_flags & TOK_FLAG_EOF)) {
2601                 tok_flags |= TOK_FLAG_EOF;
2602                 tok = TOK_LINEFEED;
2603                 goto keep_tok_flags;
2604             } else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
2605                 tok = TOK_EOF;
2606             } else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
2607                 tcc_error("missing #endif");
2608             } else if (s1->include_stack_ptr == s1->include_stack) {
2609                 /* no include left : end of file. */
2610                 tok = TOK_EOF;
2611             } else {
2612                 tok_flags &= ~TOK_FLAG_EOF;
2613                 /* pop include file */
2614
2615                 /* test if previous '#endif' was after a #ifdef at
2616                    start of file */
2617                 if (tok_flags & TOK_FLAG_ENDIF) {
2618 #ifdef INC_DEBUG
2619                     printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
2620 #endif
2621                     search_cached_include(s1, file->filename, 1)
2622                         ->ifndef_macro = file->ifndef_macro_saved;
2623                     tok_flags &= ~TOK_FLAG_ENDIF;
2624                 }
2625
2626                 /* add end of include file debug info */
2627                 tcc_debug_eincl(tcc_state);
2628                 /* pop include stack */
2629                 tcc_close();
2630                 s1->include_stack_ptr--;
2631                 p = file->buf_ptr;
2632                 if (p == file->buffer)
2633                     tok_flags = TOK_FLAG_BOF|TOK_FLAG_BOL;
2634                 goto redo_no_start;
2635             }
2636         }
2637         break;
2638
2639     case '\n':
2640         file->line_num++;
2641         tok_flags |= TOK_FLAG_BOL;
2642         p++;
2643 maybe_newline:
2644         if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
2645             goto redo_no_start;
2646         tok = TOK_LINEFEED;
2647         goto keep_tok_flags;
2648
2649     case '#':
2650         /* XXX: simplify */
2651         PEEKC(c, p);
2652         if ((tok_flags & TOK_FLAG_BOL) &&
2653             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2654             file->buf_ptr = p;
2655             preprocess(tok_flags & TOK_FLAG_BOF);
2656             p = file->buf_ptr;
2657             goto maybe_newline;
2658         } else {
2659             if (c == '#') {
2660                 p++;
2661                 tok = TOK_TWOSHARPS;
2662             } else {
2663                 if (parse_flags & PARSE_FLAG_ASM_FILE) {
2664                     p = parse_line_comment(p - 1);
2665                     goto redo_no_start;
2666                 } else {
2667                     tok = '#';
2668                 }
2669             }
2670         }
2671         break;
2672
2673     /* dollar is allowed to start identifiers when not parsing asm */
2674     case '$':
2675         if (!(isidnum_table[c - CH_EOF] & IS_ID)
2676          || (parse_flags & PARSE_FLAG_ASM_FILE))
2677             goto parse_simple;
2678
2679     case 'a': case 'b': case 'c': case 'd':
2680     case 'e': case 'f': case 'g': case 'h':
2681     case 'i': case 'j': case 'k': case 'l':
2682     case 'm': case 'n': case 'o': case 'p':
2683     case 'q': case 'r': case 's': case 't':
2684     case 'u': case 'v': case 'w': case 'x':
2685     case 'y': case 'z':
2686     case 'A': case 'B': case 'C': case 'D':
2687     case 'E': case 'F': case 'G': case 'H':
2688     case 'I': case 'J': case 'K':
2689     case 'M': case 'N': case 'O': case 'P':
2690     case 'Q': case 'R': case 'S': case 'T':
2691     case 'U': case 'V': case 'W': case 'X':
2692     case 'Y': case 'Z':
2693     case '_':
2694     parse_ident_fast:
2695         p1 = p;
2696         h = TOK_HASH_INIT;
2697         h = TOK_HASH_FUNC(h, c);
2698         while (c = *++p, isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2699             h = TOK_HASH_FUNC(h, c);
2700         len = p - p1;
2701         if (c != '\\') {
2702             TokenSym **pts;
2703
2704             /* fast case : no stray found, so we have the full token
2705                and we have already hashed it */
2706             h &= (TOK_HASH_SIZE - 1);
2707             pts = &hash_ident[h];
2708             for(;;) {
2709                 ts = *pts;
2710                 if (!ts)
2711                     break;
2712                 if (ts->len == len && !memcmp(ts->str, p1, len))
2713                     goto token_found;
2714                 pts = &(ts->hash_next);
2715             }
2716             ts = tok_alloc_new(pts, (char *) p1, len);
2717         token_found: ;
2718         } else {
2719             /* slower case */
2720             cstr_reset(&tokcstr);
2721             cstr_cat(&tokcstr, (char *) p1, len);
2722             p--;
2723             PEEKC(c, p);
2724         parse_ident_slow:
2725             while (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2726             {
2727                 cstr_ccat(&tokcstr, c);
2728                 PEEKC(c, p);
2729             }
2730             ts = tok_alloc(tokcstr.data, tokcstr.size);
2731         }
2732         tok = ts->tok;
2733         break;
2734     case 'L':
2735         t = p[1];
2736         if (t != '\\' && t != '\'' && t != '\"') {
2737             /* fast case */
2738             goto parse_ident_fast;
2739         } else {
2740             PEEKC(c, p);
2741             if (c == '\'' || c == '\"') {
2742                 is_long = 1;
2743                 goto str_const;
2744             } else {
2745                 cstr_reset(&tokcstr);
2746                 cstr_ccat(&tokcstr, 'L');
2747                 goto parse_ident_slow;
2748             }
2749         }
2750         break;
2751
2752     case '0': case '1': case '2': case '3':
2753     case '4': case '5': case '6': case '7':
2754     case '8': case '9':
2755         t = c;
2756         PEEKC(c, p);
2757         /* after the first digit, accept digits, alpha, '.' or sign if
2758            prefixed by 'eEpP' */
2759     parse_num:
2760         cstr_reset(&tokcstr);
2761         for(;;) {
2762             cstr_ccat(&tokcstr, t);
2763             if (!((isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2764                   || c == '.'
2765                   || ((c == '+' || c == '-')
2766                       && (((t == 'e' || t == 'E')
2767                             && !(parse_flags & PARSE_FLAG_ASM_FILE
2768                                 /* 0xe+1 is 3 tokens in asm */
2769                                 && ((char*)tokcstr.data)[0] == '0'
2770                                 && toup(((char*)tokcstr.data)[1]) == 'X'))
2771                           || t == 'p' || t == 'P'))))
2772                 break;
2773             t = c;
2774             PEEKC(c, p);
2775         }
2776         /* We add a trailing '\0' to ease parsing */
2777         cstr_ccat(&tokcstr, '\0');
2778         tokc.str.size = tokcstr.size;
2779         tokc.str.data = tokcstr.data;
2780         tok = TOK_PPNUM;
2781         break;
2782
2783     case '.':
2784         /* special dot handling because it can also start a number */
2785         PEEKC(c, p);
2786         if (isnum(c)) {
2787             t = '.';
2788             goto parse_num;
2789         } else if ((isidnum_table['.' - CH_EOF] & IS_ID)
2790                    && (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))) {
2791             *--p = c = '.';
2792             goto parse_ident_fast;
2793         } else if (c == '.') {
2794             PEEKC(c, p);
2795             if (c == '.') {
2796                 p++;
2797                 tok = TOK_DOTS;
2798             } else {
2799                 *--p = '.'; /* may underflow into file->unget[] */
2800                 tok = '.';
2801             }
2802         } else {
2803             tok = '.';
2804         }
2805         break;
2806     case '\'':
2807     case '\"':
2808         is_long = 0;
2809     str_const:
2810         cstr_reset(&tokcstr);
2811         if (is_long)
2812             cstr_ccat(&tokcstr, 'L');
2813         cstr_ccat(&tokcstr, c);
2814         p = parse_pp_string(p, c, &tokcstr);
2815         cstr_ccat(&tokcstr, c);
2816         cstr_ccat(&tokcstr, '\0');
2817         tokc.str.size = tokcstr.size;
2818         tokc.str.data = tokcstr.data;
2819         tok = TOK_PPSTR;
2820         break;
2821
2822     case '<':
2823         PEEKC(c, p);
2824         if (c == '=') {
2825             p++;
2826             tok = TOK_LE;
2827         } else if (c == '<') {
2828             PEEKC(c, p);
2829             if (c == '=') {
2830                 p++;
2831                 tok = TOK_A_SHL;
2832             } else {
2833                 tok = TOK_SHL;
2834             }
2835         } else {
2836             tok = TOK_LT;
2837         }
2838         break;
2839     case '>':
2840         PEEKC(c, p);
2841         if (c == '=') {
2842             p++;
2843             tok = TOK_GE;
2844         } else if (c == '>') {
2845             PEEKC(c, p);
2846             if (c == '=') {
2847                 p++;
2848                 tok = TOK_A_SAR;
2849             } else {
2850                 tok = TOK_SAR;
2851             }
2852         } else {
2853             tok = TOK_GT;
2854         }
2855         break;
2856
2857     case '&':
2858         PEEKC(c, p);
2859         if (c == '&') {
2860             p++;
2861             tok = TOK_LAND;
2862         } else if (c == '=') {
2863             p++;
2864             tok = TOK_A_AND;
2865         } else {
2866             tok = '&';
2867         }
2868         break;
2869
2870     case '|':
2871         PEEKC(c, p);
2872         if (c == '|') {
2873             p++;
2874             tok = TOK_LOR;
2875         } else if (c == '=') {
2876             p++;
2877             tok = TOK_A_OR;
2878         } else {
2879             tok = '|';
2880         }
2881         break;
2882
2883     case '+':
2884         PEEKC(c, p);
2885         if (c == '+') {
2886             p++;
2887             tok = TOK_INC;
2888         } else if (c == '=') {
2889             p++;
2890             tok = TOK_A_ADD;
2891         } else {
2892             tok = '+';
2893         }
2894         break;
2895
2896     case '-':
2897         PEEKC(c, p);
2898         if (c == '-') {
2899             p++;
2900             tok = TOK_DEC;
2901         } else if (c == '=') {
2902             p++;
2903             tok = TOK_A_SUB;
2904         } else if (c == '>') {
2905             p++;
2906             tok = TOK_ARROW;
2907         } else {
2908             tok = '-';
2909         }
2910         break;
2911
2912     PARSE2('!', '!', '=', TOK_NE)
2913     PARSE2('=', '=', '=', TOK_EQ)
2914     PARSE2('*', '*', '=', TOK_A_MUL)
2915     PARSE2('%', '%', '=', TOK_A_MOD)
2916     PARSE2('^', '^', '=', TOK_A_XOR)
2917
2918         /* comments or operator */
2919     case '/':
2920         PEEKC(c, p);
2921         if (c == '*') {
2922             p = parse_comment(p);
2923             /* comments replaced by a blank */
2924             tok = ' ';
2925             goto keep_tok_flags;
2926         } else if (c == '/') {
2927             p = parse_line_comment(p);
2928             tok = ' ';
2929             goto keep_tok_flags;
2930         } else if (c == '=') {
2931             p++;
2932             tok = TOK_A_DIV;
2933         } else {
2934             tok = '/';
2935         }
2936         break;
2937
2938         /* simple tokens */
2939     case '(':
2940     case ')':
2941     case '[':
2942     case ']':
2943     case '{':
2944     case '}':
2945     case ',':
2946     case ';':
2947     case ':':
2948     case '?':
2949     case '~':
2950     case '@': /* only used in assembler */
2951     parse_simple:
2952         tok = c;
2953         p++;
2954         break;
2955     default:
2956         if (c >= 0x80 && c <= 0xFF) /* utf8 identifiers */
2957             goto parse_ident_fast;
2958         if (parse_flags & PARSE_FLAG_ASM_FILE)
2959             goto parse_simple;
2960         tcc_error("unrecognized character \\x%02x", c);
2961         break;
2962     }
2963     tok_flags = 0;
2964 keep_tok_flags:
2965     file->buf_ptr = p;
2966 #if defined(PARSE_DEBUG)
2967     printf("token = %d %s\n", tok, get_tok_str(tok, &tokc));
2968 #endif
2969 }
2970
2971 /* return next token without macro substitution. Can read input from
2972    macro_ptr buffer */
2973 static void next_nomacro_spc(void)
2974 {
2975     if (macro_ptr) {
2976     redo:
2977         tok = *macro_ptr;
2978         if (tok) {
2979             TOK_GET(&tok, &macro_ptr, &tokc);
2980             if (tok == TOK_LINENUM) {
2981                 file->line_num = tokc.i;
2982                 goto redo;
2983             }
2984         }
2985     } else {
2986         next_nomacro1();
2987     }
2988     //printf("token = %s\n", get_tok_str(tok, &tokc));
2989 }
2990
2991 ST_FUNC void next_nomacro(void)
2992 {
2993     do {
2994         next_nomacro_spc();
2995     } while (tok < 256 && (isidnum_table[tok - CH_EOF] & IS_SPC));
2996 }
2997
2998
2999 static void macro_subst(
3000     TokenString *tok_str,
3001     Sym **nested_list,
3002     const int *macro_str
3003     );
3004
3005 /* substitute arguments in replacement lists in macro_str by the values in
3006    args (field d) and return allocated string */
3007 static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
3008 {
3009     int t, t0, t1, spc;
3010     const int *st;
3011     Sym *s;
3012     CValue cval;
3013     TokenString str;
3014     CString cstr;
3015
3016     tok_str_new(&str);
3017     t0 = t1 = 0;
3018     while(1) {
3019         TOK_GET(&t, &macro_str, &cval);
3020         if (!t)
3021             break;
3022         if (t == '#') {
3023             /* stringize */
3024             TOK_GET(&t, &macro_str, &cval);
3025             if (!t)
3026                 goto bad_stringy;
3027             s = sym_find2(args, t);
3028             if (s) {
3029                 cstr_new(&cstr);
3030                 cstr_ccat(&cstr, '\"');
3031                 st = s->d;
3032                 spc = 0;
3033                 while (*st >= 0) {
3034                     TOK_GET(&t, &st, &cval);
3035                     if (t != TOK_PLCHLDR
3036                      && t != TOK_NOSUBST
3037                      && 0 == check_space(t, &spc)) {
3038                         const char *s = get_tok_str(t, &cval);
3039                         while (*s) {
3040                             if (t == TOK_PPSTR && *s != '\'')
3041                                 add_char(&cstr, *s);
3042                             else
3043                                 cstr_ccat(&cstr, *s);
3044                             ++s;
3045                         }
3046                     }
3047                 }
3048                 cstr.size -= spc;
3049                 cstr_ccat(&cstr, '\"');
3050                 cstr_ccat(&cstr, '\0');
3051 #ifdef PP_DEBUG
3052                 printf("\nstringize: <%s>\n", (char *)cstr.data);
3053 #endif
3054                 /* add string */
3055                 cval.str.size = cstr.size;
3056                 cval.str.data = cstr.data;
3057                 tok_str_add2(&str, TOK_PPSTR, &cval);
3058                 cstr_free(&cstr);
3059             } else {
3060         bad_stringy:
3061                 expect("macro parameter after '#'");
3062             }
3063         } else if (t >= TOK_IDENT) {
3064             s = sym_find2(args, t);
3065             if (s) {
3066                 int l0 = str.len;
3067                 st = s->d;
3068                 /* if '##' is present before or after, no arg substitution */
3069                 if (*macro_str == TOK_PPJOIN || t1 == TOK_PPJOIN) {
3070                     /* special case for var arg macros : ## eats the ','
3071                        if empty VA_ARGS variable. */
3072                     if (t1 == TOK_PPJOIN && t0 == ',' && gnu_ext && s->type.t) {
3073                         if (*st <= 0) {
3074                             /* suppress ',' '##' */
3075                             str.len -= 2;
3076                         } else {
3077                             /* suppress '##' and add variable */
3078                             str.len--;
3079                             goto add_var;
3080                         }
3081                     }
3082                 } else {
3083             add_var:
3084                     if (!s->next) {
3085                         /* Expand arguments tokens and store them.  In most
3086                            cases we could also re-expand each argument if
3087                            used multiple times, but not if the argument
3088                            contains the __COUNTER__ macro.  */
3089                         TokenString str2;
3090                         sym_push2(&s->next, s->v, s->type.t, 0);
3091                         tok_str_new(&str2);
3092                         macro_subst(&str2, nested_list, st);
3093                         tok_str_add(&str2, 0);
3094                         s->next->d = str2.str;
3095                     }
3096                     st = s->next->d;
3097                 }
3098                 for(;;) {
3099                     int t2;
3100                     TOK_GET(&t2, &st, &cval);
3101                     if (t2 <= 0)
3102                         break;
3103                     tok_str_add2(&str, t2, &cval);
3104                 }
3105                 if (str.len == l0) /* expanded to empty string */
3106                     tok_str_add(&str, TOK_PLCHLDR);
3107             } else {
3108                 tok_str_add(&str, t);
3109             }
3110         } else {
3111             tok_str_add2(&str, t, &cval);
3112         }
3113         t0 = t1, t1 = t;
3114     }
3115     tok_str_add(&str, 0);
3116     return str.str;
3117 }
3118
3119 static char const ab_month_name[12][4] =
3120 {
3121     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3122     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
3123 };
3124
3125 static int paste_tokens(int t1, CValue *v1, int t2, CValue *v2)
3126 {
3127     CString cstr;
3128     int n, ret = 1;
3129
3130     cstr_new(&cstr);
3131     if (t1 != TOK_PLCHLDR)
3132         cstr_cat(&cstr, get_tok_str(t1, v1), -1);
3133     n = cstr.size;
3134     if (t2 != TOK_PLCHLDR)
3135         cstr_cat(&cstr, get_tok_str(t2, v2), -1);
3136     cstr_ccat(&cstr, '\0');
3137
3138     tcc_open_bf(tcc_state, ":paste:", cstr.size);
3139     memcpy(file->buffer, cstr.data, cstr.size);
3140     tok_flags = 0;
3141     for (;;) {
3142         next_nomacro1();
3143         if (0 == *file->buf_ptr)
3144             break;
3145         if (is_space(tok))
3146             continue;
3147         tcc_warning("pasting \"%.*s\" and \"%s\" does not give a valid"
3148             " preprocessing token", n, cstr.data, (char*)cstr.data + n);
3149         ret = 0;
3150         break;
3151     }
3152     tcc_close();
3153     //printf("paste <%s>\n", (char*)cstr.data);
3154     cstr_free(&cstr);
3155     return ret;
3156 }
3157
3158 /* handle the '##' operator. Return NULL if no '##' seen. Otherwise
3159    return the resulting string (which must be freed). */
3160 static inline int *macro_twosharps(const int *ptr0)
3161 {
3162     int t;
3163     CValue cval;
3164     TokenString macro_str1;
3165     int start_of_nosubsts = -1;
3166     const int *ptr;
3167
3168     /* we search the first '##' */
3169     for (ptr = ptr0;;) {
3170         TOK_GET(&t, &ptr, &cval);
3171         if (t == TOK_PPJOIN)
3172             break;
3173         if (t == 0)
3174             return NULL;
3175     }
3176
3177     tok_str_new(&macro_str1);
3178
3179     //tok_print(" $$$", ptr0);
3180     for (ptr = ptr0;;) {
3181         TOK_GET(&t, &ptr, &cval);
3182         if (t == 0)
3183             break;
3184         if (t == TOK_PPJOIN)
3185             continue;
3186         while (*ptr == TOK_PPJOIN) {
3187             int t1; CValue cv1;
3188             /* given 'a##b', remove nosubsts preceding 'a' */
3189             if (start_of_nosubsts >= 0)
3190                 macro_str1.len = start_of_nosubsts;
3191             /* given 'a##b', remove nosubsts preceding 'b' */
3192             while ((t1 = *++ptr) == TOK_NOSUBST)
3193                 ;
3194             if (t1 && t1 != TOK_PPJOIN) {
3195                 TOK_GET(&t1, &ptr, &cv1);
3196                 if (t != TOK_PLCHLDR || t1 != TOK_PLCHLDR) {
3197                     if (paste_tokens(t, &cval, t1, &cv1)) {
3198                         t = tok, cval = tokc;
3199                     } else {
3200                         tok_str_add2(&macro_str1, t, &cval);
3201                         t = t1, cval = cv1;
3202                     }
3203                 }
3204             }
3205         }
3206         if (t == TOK_NOSUBST) {
3207             if (start_of_nosubsts < 0)
3208                 start_of_nosubsts = macro_str1.len;
3209         } else {
3210             start_of_nosubsts = -1;
3211         }
3212         tok_str_add2(&macro_str1, t, &cval);
3213     }
3214     tok_str_add(&macro_str1, 0);
3215     //tok_print(" ###", macro_str1.str);
3216     return macro_str1.str;
3217 }
3218
3219 /* peek or read [ws_str == NULL] next token from function macro call,
3220    walking up macro levels up to the file if necessary */
3221 static int next_argstream(Sym **nested_list, TokenString *ws_str)
3222 {
3223     int t;
3224     const int *p;
3225     Sym *sa;
3226
3227     for (;;) {
3228         if (macro_ptr) {
3229             p = macro_ptr, t = *p;
3230             if (ws_str) {
3231                 while (is_space(t) || TOK_LINEFEED == t || TOK_PLCHLDR == t)
3232                     tok_str_add(ws_str, t), t = *++p;
3233             }
3234             if (t == 0) {
3235                 end_macro();
3236                 /* also, end of scope for nested defined symbol */
3237                 sa = *nested_list;
3238                 while (sa && sa->v == 0)
3239                     sa = sa->prev;
3240                 if (sa)
3241                     sa->v = 0;
3242                 continue;
3243             }
3244         } else {
3245             ch = handle_eob();
3246             if (ws_str) {
3247                 while (is_space(ch) || ch == '\n' || ch == '/') {
3248                     if (ch == '/') {
3249                         int c;
3250                         uint8_t *p = file->buf_ptr;
3251                         PEEKC(c, p);
3252                         if (c == '*') {
3253                             p = parse_comment(p);
3254                             file->buf_ptr = p - 1;
3255                         } else if (c == '/') {
3256                             p = parse_line_comment(p);
3257                             file->buf_ptr = p - 1;
3258                         } else
3259                             break;
3260                         ch = ' ';
3261                     }
3262                     if (ch == '\n')
3263                         file->line_num++;
3264                     if (!(ch == '\f' || ch == '\v' || ch == '\r'))
3265                         tok_str_add(ws_str, ch);
3266                     cinp();
3267                 }
3268             }
3269             t = ch;
3270         }
3271
3272         if (ws_str)
3273             return t;
3274         next_nomacro_spc();
3275         return tok;
3276     }
3277 }
3278
3279 /* do macro substitution of current token with macro 's' and add
3280    result to (tok_str,tok_len). 'nested_list' is the list of all
3281    macros we got inside to avoid recursing. Return non zero if no
3282    substitution needs to be done */
3283 static int macro_subst_tok(
3284     TokenString *tok_str,
3285     Sym **nested_list,
3286     Sym *s)
3287 {
3288     Sym *args, *sa, *sa1;
3289     int parlevel, t, t1, spc;
3290     TokenString str;
3291     char *cstrval;
3292     CValue cval;
3293     CString cstr;
3294     char buf[32];
3295
3296     /* if symbol is a macro, prepare substitution */
3297     /* special macros */
3298     if (tok == TOK___LINE__ || tok == TOK___COUNTER__) {
3299         t = tok == TOK___LINE__ ? file->line_num : pp_counter++;
3300         snprintf(buf, sizeof(buf), "%d", t);
3301         cstrval = buf;
3302         t1 = TOK_PPNUM;
3303         goto add_cstr1;
3304     } else if (tok == TOK___FILE__) {
3305         cstrval = file->filename;
3306         goto add_cstr;
3307     } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
3308         time_t ti;
3309         struct tm *tm;
3310
3311         time(&ti);
3312         tm = localtime(&ti);
3313         if (tok == TOK___DATE__) {
3314             snprintf(buf, sizeof(buf), "%s %2d %d",
3315                      ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
3316         } else {
3317             snprintf(buf, sizeof(buf), "%02d:%02d:%02d",
3318                      tm->tm_hour, tm->tm_min, tm->tm_sec);
3319         }
3320         cstrval = buf;
3321     add_cstr:
3322         t1 = TOK_STR;
3323     add_cstr1:
3324         cstr_new(&cstr);
3325         cstr_cat(&cstr, cstrval, 0);
3326         cval.str.size = cstr.size;
3327         cval.str.data = cstr.data;
3328         tok_str_add2(tok_str, t1, &cval);
3329         cstr_free(&cstr);
3330     } else if (s->d) {
3331         int saved_parse_flags = parse_flags;
3332         int *joined_str = NULL;
3333         int *mstr = s->d;
3334
3335         if (s->type.t == MACRO_FUNC) {
3336             /* whitespace between macro name and argument list */
3337             TokenString ws_str;
3338             tok_str_new(&ws_str);
3339
3340             spc = 0;
3341             parse_flags |= PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED
3342                 | PARSE_FLAG_ACCEPT_STRAYS;
3343
3344             /* get next token from argument stream */
3345             t = next_argstream(nested_list, &ws_str);
3346             if (t != '(') {
3347                 /* not a macro substitution after all, restore the
3348                  * macro token plus all whitespace we've read.
3349                  * whitespace is intentionally not merged to preserve
3350                  * newlines. */
3351                 parse_flags = saved_parse_flags;
3352                 tok_str_add(tok_str, tok);
3353                 if (parse_flags & PARSE_FLAG_SPACES) {
3354                     int i;
3355                     for (i = 0; i < ws_str.len; i++)
3356                         tok_str_add(tok_str, ws_str.str[i]);
3357                 }
3358                 tok_str_free_str(ws_str.str);
3359                 return 0;
3360             } else {
3361                 tok_str_free_str(ws_str.str);
3362             }
3363             do {
3364                 next_nomacro(); /* eat '(' */
3365             } while (tok == TOK_PLCHLDR);
3366
3367             /* argument macro */
3368             args = NULL;
3369             sa = s->next;
3370             /* NOTE: empty args are allowed, except if no args */
3371             for(;;) {
3372                 do {
3373                     next_argstream(nested_list, NULL);
3374                 } while (is_space(tok) || TOK_LINEFEED == tok);
3375     empty_arg:
3376                 /* handle '()' case */
3377                 if (!args && !sa && tok == ')')
3378                     break;
3379                 if (!sa)
3380                     tcc_error("macro '%s' used with too many args",
3381                           get_tok_str(s->v, 0));
3382                 tok_str_new(&str);
3383                 parlevel = spc = 0;
3384                 /* NOTE: non zero sa->t indicates VA_ARGS */
3385                 while ((parlevel > 0 ||
3386                         (tok != ')' &&
3387                          (tok != ',' || sa->type.t)))) {
3388                     if (tok == TOK_EOF || tok == 0)
3389                         break;
3390                     if (tok == '(')
3391                         parlevel++;
3392                     else if (tok == ')')
3393                         parlevel--;
3394                     if (tok == TOK_LINEFEED)
3395                         tok = ' ';
3396                     if (!check_space(tok, &spc))
3397                         tok_str_add2(&str, tok, &tokc);
3398                     next_argstream(nested_list, NULL);
3399                 }
3400                 if (parlevel)
3401                     expect(")");
3402                 str.len -= spc;
3403                 tok_str_add(&str, -1);
3404                 tok_str_add(&str, 0);
3405                 sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
3406                 sa1->d = str.str;
3407                 sa = sa->next;
3408                 if (tok == ')') {
3409                     /* special case for gcc var args: add an empty
3410                        var arg argument if it is omitted */
3411                     if (sa && sa->type.t && gnu_ext)
3412                         goto empty_arg;
3413                     break;
3414                 }
3415                 if (tok != ',')
3416                     expect(",");
3417             }
3418             if (sa) {
3419                 tcc_error("macro '%s' used with too few args",
3420                       get_tok_str(s->v, 0));
3421             }
3422
3423             parse_flags = saved_parse_flags;
3424
3425             /* now subst each arg */
3426             mstr = macro_arg_subst(nested_list, mstr, args);
3427             /* free memory */
3428             sa = args;
3429             while (sa) {
3430                 sa1 = sa->prev;
3431                 tok_str_free_str(sa->d);
3432                 if (sa->next) {
3433                     tok_str_free_str(sa->next->d);
3434                     sym_free(sa->next);
3435                 }
3436                 sym_free(sa);
3437                 sa = sa1;
3438             }
3439         }
3440
3441         sym_push2(nested_list, s->v, 0, 0);
3442         parse_flags = saved_parse_flags;
3443         joined_str = macro_twosharps(mstr);
3444         macro_subst(tok_str, nested_list, joined_str ? joined_str : mstr);
3445
3446         /* pop nested defined symbol */
3447         sa1 = *nested_list;
3448         *nested_list = sa1->prev;
3449         sym_free(sa1);
3450         if (joined_str)
3451             tok_str_free_str(joined_str);
3452         if (mstr != s->d)
3453             tok_str_free_str(mstr);
3454     }
3455     return 0;
3456 }
3457
3458 /* do macro substitution of macro_str and add result to
3459    (tok_str,tok_len). 'nested_list' is the list of all macros we got
3460    inside to avoid recursing. */
3461 static void macro_subst(
3462     TokenString *tok_str,
3463     Sym **nested_list,
3464     const int *macro_str
3465     )
3466 {
3467     Sym *s;
3468     int t, spc, nosubst;
3469     CValue cval;
3470
3471     spc = nosubst = 0;
3472
3473     while (1) {
3474         TOK_GET(&t, &macro_str, &cval);
3475         if (t <= 0)
3476             break;
3477
3478         if (t >= TOK_IDENT && 0 == nosubst) {
3479             s = define_find(t);
3480             if (s == NULL)
3481                 goto no_subst;
3482
3483             /* if nested substitution, do nothing */
3484             if (sym_find2(*nested_list, t)) {
3485                 /* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
3486                 tok_str_add2(tok_str, TOK_NOSUBST, NULL);
3487                 goto no_subst;
3488             }
3489
3490             {
3491                 TokenString *str = tok_str_alloc();
3492                 str->str = (int*)macro_str;
3493                 begin_macro(str, 2);
3494
3495                 tok = t;
3496                 macro_subst_tok(tok_str, nested_list, s);
3497
3498                 if (macro_stack != str) {
3499                     /* already finished by reading function macro arguments */
3500                     break;
3501                 }
3502
3503                 macro_str = macro_ptr;
3504                 end_macro ();
3505             }
3506             if (tok_str->len)
3507                 spc = is_space(t = tok_str->str[tok_str->lastlen]);
3508         } else {
3509             if (t == '\\' && !(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
3510                 tcc_error("stray '\\' in program");
3511 no_subst:
3512             if (!check_space(t, &spc))
3513                 tok_str_add2(tok_str, t, &cval);
3514
3515             if (nosubst) {
3516                 if (nosubst > 1 && (spc || (++nosubst == 3 && t == '(')))
3517                     continue;
3518                 nosubst = 0;
3519             }
3520             if (t == TOK_NOSUBST)
3521                 nosubst = 1;
3522         }
3523         /* GCC supports 'defined' as result of a macro substitution */
3524         if (t == TOK_DEFINED && pp_expr)
3525             nosubst = 2;
3526     }
3527 }
3528
3529 /* return next token with macro substitution */
3530 ST_FUNC void next(void)
3531 {
3532     /* generate line number info */
3533     if (tcc_state->do_debug)
3534         tcc_debug_line(tcc_state);
3535  redo:
3536     if (parse_flags & PARSE_FLAG_SPACES)
3537         next_nomacro_spc();
3538     else
3539         next_nomacro();
3540
3541     if (macro_ptr) {
3542         if (tok == TOK_NOSUBST || tok == TOK_PLCHLDR) {
3543         /* discard preprocessor markers */
3544             goto redo;
3545         } else if (tok == 0) {
3546             /* end of macro or unget token string */
3547             end_macro();
3548             goto redo;
3549         }
3550     } else if (tok >= TOK_IDENT && (parse_flags & PARSE_FLAG_PREPROCESS)) {
3551         Sym *s;
3552         /* if reading from file, try to substitute macros */
3553         s = define_find(tok);
3554         if (s) {
3555             Sym *nested_list = NULL;
3556             tokstr_buf.len = 0;
3557             macro_subst_tok(&tokstr_buf, &nested_list, s);
3558             tok_str_add(&tokstr_buf, 0);
3559             begin_macro(&tokstr_buf, 0);
3560             goto redo;
3561         }
3562     }
3563     /* convert preprocessor tokens into C tokens */
3564     if (tok == TOK_PPNUM) {
3565         if  (parse_flags & PARSE_FLAG_TOK_NUM)
3566             parse_number((char *)tokc.str.data);
3567     } else if (tok == TOK_PPSTR) {
3568         if (parse_flags & PARSE_FLAG_TOK_STR)
3569             parse_string((char *)tokc.str.data, tokc.str.size - 1);
3570     }
3571 }
3572
3573 /* push back current token and set current token to 'last_tok'. Only
3574    identifier case handled for labels. */
3575 ST_INLN void unget_tok(int last_tok)
3576 {
3577
3578     TokenString *str = tok_str_alloc();
3579     tok_str_add2(str, tok, &tokc);
3580     tok_str_add(str, 0);
3581     begin_macro(str, 1);
3582     tok = last_tok;
3583 }
3584
3585 ST_FUNC void preprocess_start(TCCState *s1, int is_asm)
3586 {
3587     CString cstr;
3588
3589     tccpp_new(s1);
3590
3591     s1->include_stack_ptr = s1->include_stack;
3592     s1->ifdef_stack_ptr = s1->ifdef_stack;
3593     file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
3594     pp_expr = 0;
3595     pp_counter = 0;
3596     pp_debug_tok = pp_debug_symv = 0;
3597     pp_once++;
3598     s1->pack_stack[0] = 0;
3599     s1->pack_stack_ptr = s1->pack_stack;
3600
3601     set_idnum('$', s1->dollars_in_identifiers ? IS_ID : 0);
3602     set_idnum('.', is_asm ? IS_ID : 0);
3603
3604     cstr_new(&cstr);
3605     if (s1->cmdline_defs.size)
3606         cstr_cat(&cstr, s1->cmdline_defs.data, s1->cmdline_defs.size);
3607     cstr_printf(&cstr, "#define __BASE_FILE__ \"%s\"\n", file->filename);
3608     if (is_asm)
3609         cstr_printf(&cstr, "#define __ASSEMBLER__ 1\n");
3610     if (s1->cmdline_incl.size)
3611         cstr_cat(&cstr, s1->cmdline_incl.data, s1->cmdline_incl.size);
3612     //printf("%s\n", (char*)cstr.data);
3613     *s1->include_stack_ptr++ = file;
3614     tcc_open_bf(s1, "<command line>", cstr.size);
3615     memcpy(file->buffer, cstr.data, cstr.size);
3616     cstr_free(&cstr);
3617
3618     parse_flags = is_asm ? PARSE_FLAG_ASM_FILE : 0;
3619     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
3620 }
3621
3622 /* cleanup from error/setjmp */
3623 ST_FUNC void preprocess_end(TCCState *s1)
3624 {
3625     while (macro_stack)
3626         end_macro();
3627     macro_ptr = NULL;
3628     while (file)
3629         tcc_close();
3630     tccpp_delete(s1);
3631 }
3632
3633 ST_FUNC void tccpp_new(TCCState *s)
3634 {
3635     int i, c;
3636     const char *p, *r;
3637
3638     /* init isid table */
3639     for(i = CH_EOF; i<128; i++)
3640         set_idnum(i,
3641             is_space(i) ? IS_SPC
3642             : isid(i) ? IS_ID
3643             : isnum(i) ? IS_NUM
3644             : 0);
3645
3646     for(i = 128; i<256; i++)
3647         set_idnum(i, IS_ID);
3648
3649     /* init allocators */
3650     tal_new(&toksym_alloc, TOKSYM_TAL_LIMIT, TOKSYM_TAL_SIZE);
3651     tal_new(&tokstr_alloc, TOKSTR_TAL_LIMIT, TOKSTR_TAL_SIZE);
3652
3653     memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
3654     memset(s->cached_includes_hash, 0, sizeof s->cached_includes_hash);
3655
3656     cstr_new(&cstr_buf);
3657     cstr_realloc(&cstr_buf, STRING_MAX_SIZE);
3658     tok_str_new(&tokstr_buf);
3659     tok_str_realloc(&tokstr_buf, TOKSTR_MAX_SIZE);
3660
3661     tok_ident = TOK_IDENT;
3662     p = tcc_keywords;
3663     while (*p) {
3664         r = p;
3665         for(;;) {
3666             c = *r++;
3667             if (c == '\0')
3668                 break;
3669         }
3670         tok_alloc(p, r - p - 1);
3671         p = r;
3672     }
3673
3674     /* we add dummy defines for some special macros to speed up tests
3675        and to have working defined() */
3676     define_push(TOK___LINE__, MACRO_OBJ, NULL, NULL);
3677     define_push(TOK___FILE__, MACRO_OBJ, NULL, NULL);
3678     define_push(TOK___DATE__, MACRO_OBJ, NULL, NULL);
3679     define_push(TOK___TIME__, MACRO_OBJ, NULL, NULL);
3680     define_push(TOK___COUNTER__, MACRO_OBJ, NULL, NULL);
3681 }
3682
3683 ST_FUNC void tccpp_delete(TCCState *s)
3684 {
3685     int i, n;
3686
3687     dynarray_reset(&s->cached_includes, &s->nb_cached_includes);
3688
3689     /* free tokens */
3690     n = tok_ident - TOK_IDENT;
3691     if (n > total_idents)
3692         total_idents = n;
3693     for(i = 0; i < n; i++)
3694         tal_free(toksym_alloc, table_ident[i]);
3695     tcc_free(table_ident);
3696     table_ident = NULL;
3697
3698     /* free static buffers */
3699     cstr_free(&tokcstr);
3700     cstr_free(&cstr_buf);
3701     cstr_free(&macro_equal_buf);
3702     tok_str_free_str(tokstr_buf.str);
3703
3704     /* free allocators */
3705     tal_delete(toksym_alloc);
3706     toksym_alloc = NULL;
3707     tal_delete(tokstr_alloc);
3708     tokstr_alloc = NULL;
3709 }
3710
3711 /* ------------------------------------------------------------------------- */
3712 /* tcc -E [-P[1]] [-dD} support */
3713
3714 static void tok_print(const char *msg, const int *str)
3715 {
3716     FILE *fp;
3717     int t, s = 0;
3718     CValue cval;
3719
3720     fp = tcc_state->ppfp;
3721     fprintf(fp, "%s", msg);
3722     while (str) {
3723         TOK_GET(&t, &str, &cval);
3724         if (!t)
3725             break;
3726         fprintf(fp, " %s" + s, get_tok_str(t, &cval)), s = 1;
3727     }
3728     fprintf(fp, "\n");
3729 }
3730
3731 static void pp_line(TCCState *s1, BufferedFile *f, int level)
3732 {
3733     int d = f->line_num - f->line_ref;
3734
3735     if (s1->dflag & 4)
3736         return;
3737
3738     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_NONE) {
3739         ;
3740     } else if (level == 0 && f->line_ref && d < 8) {
3741         while (d > 0)
3742             fputs("\n", s1->ppfp), --d;
3743     } else if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_STD) {
3744         fprintf(s1->ppfp, "#line %d \"%s\"\n", f->line_num, f->filename);
3745     } else {
3746         fprintf(s1->ppfp, "# %d \"%s\"%s\n", f->line_num, f->filename,
3747             level > 0 ? " 1" : level < 0 ? " 2" : "");
3748     }
3749     f->line_ref = f->line_num;
3750 }
3751
3752 static void define_print(TCCState *s1, int v)
3753 {
3754     FILE *fp;
3755     Sym *s;
3756
3757     s = define_find(v);
3758     if (NULL == s || NULL == s->d)
3759         return;
3760
3761     fp = s1->ppfp;
3762     fprintf(fp, "#define %s", get_tok_str(v, NULL));
3763     if (s->type.t == MACRO_FUNC) {
3764         Sym *a = s->next;
3765         fprintf(fp,"(");
3766         if (a)
3767             for (;;) {
3768                 fprintf(fp,"%s", get_tok_str(a->v & ~SYM_FIELD, NULL));
3769                 if (!(a = a->next))
3770                     break;
3771                 fprintf(fp,",");
3772             }
3773         fprintf(fp,")");
3774     }
3775     tok_print("", s->d);
3776 }
3777
3778 static void pp_debug_defines(TCCState *s1)
3779 {
3780     int v, t;
3781     const char *vs;
3782     FILE *fp;
3783
3784     t = pp_debug_tok;
3785     if (t == 0)
3786         return;
3787
3788     file->line_num--;
3789     pp_line(s1, file, 0);
3790     file->line_ref = ++file->line_num;
3791
3792     fp = s1->ppfp;
3793     v = pp_debug_symv;
3794     vs = get_tok_str(v, NULL);
3795     if (t == TOK_DEFINE) {
3796         define_print(s1, v);
3797     } else if (t == TOK_UNDEF) {
3798         fprintf(fp, "#undef %s\n", vs);
3799     } else if (t == TOK_push_macro) {
3800         fprintf(fp, "#pragma push_macro(\"%s\")\n", vs);
3801     } else if (t == TOK_pop_macro) {
3802         fprintf(fp, "#pragma pop_macro(\"%s\")\n", vs);
3803     }
3804     pp_debug_tok = 0;
3805 }
3806
3807 static void pp_debug_builtins(TCCState *s1)
3808 {
3809     int v;
3810     for (v = TOK_IDENT; v < tok_ident; ++v)
3811         define_print(s1, v);
3812 }
3813
3814 /* Add a space between tokens a and b to avoid unwanted textual pasting */
3815 static int pp_need_space(int a, int b)
3816 {
3817     return 'E' == a ? '+' == b || '-' == b
3818         : '+' == a ? TOK_INC == b || '+' == b
3819         : '-' == a ? TOK_DEC == b || '-' == b
3820         : a >= TOK_IDENT ? b >= TOK_IDENT
3821         : a == TOK_PPNUM ? b >= TOK_IDENT
3822         : 0;
3823 }
3824
3825 /* maybe hex like 0x1e */
3826 static int pp_check_he0xE(int t, const char *p)
3827 {
3828     if (t == TOK_PPNUM && toup(strchr(p, 0)[-1]) == 'E')
3829         return 'E';
3830     return t;
3831 }
3832
3833 /* Preprocess the current file */
3834 ST_FUNC int tcc_preprocess(TCCState *s1)
3835 {
3836     BufferedFile **iptr;
3837     int token_seen, spcs, level;
3838     const char *p;
3839     char white[400];
3840
3841     parse_flags = PARSE_FLAG_PREPROCESS
3842                 | (parse_flags & PARSE_FLAG_ASM_FILE)
3843                 | PARSE_FLAG_LINEFEED
3844                 | PARSE_FLAG_SPACES
3845                 | PARSE_FLAG_ACCEPT_STRAYS
3846                 ;
3847     /* Credits to Fabrice Bellard's initial revision to demonstrate its
3848        capability to compile and run itself, provided all numbers are
3849        given as decimals. tcc -E -P10 will do. */
3850     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_P10)
3851         parse_flags |= PARSE_FLAG_TOK_NUM, s1->Pflag = 1;
3852
3853 #ifdef PP_BENCH
3854     /* for PP benchmarks */
3855     do next(); while (tok != TOK_EOF);
3856     return 0;
3857 #endif
3858
3859     if (s1->dflag & 1) {
3860         pp_debug_builtins(s1);
3861         s1->dflag &= ~1;
3862     }
3863
3864     token_seen = TOK_LINEFEED, spcs = 0;
3865     pp_line(s1, file, 0);
3866     for (;;) {
3867         iptr = s1->include_stack_ptr;
3868         next();
3869         if (tok == TOK_EOF)
3870             break;
3871
3872         level = s1->include_stack_ptr - iptr;
3873         if (level) {
3874             if (level > 0)
3875                 pp_line(s1, *iptr, 0);
3876             pp_line(s1, file, level);
3877         }
3878         if (s1->dflag & 7) {
3879             pp_debug_defines(s1);
3880             if (s1->dflag & 4)
3881                 continue;
3882         }
3883
3884         if (is_space(tok)) {
3885             if (spcs < sizeof white - 1)
3886                 white[spcs++] = tok;
3887             continue;
3888         } else if (tok == TOK_LINEFEED) {
3889             spcs = 0;
3890             if (token_seen == TOK_LINEFEED)
3891                 continue;
3892             ++file->line_ref;
3893         } else if (token_seen == TOK_LINEFEED) {
3894             pp_line(s1, file, 0);
3895         } else if (spcs == 0 && pp_need_space(token_seen, tok)) {
3896             white[spcs++] = ' ';
3897         }
3898
3899         white[spcs] = 0, fputs(white, s1->ppfp), spcs = 0;
3900         fputs(p = get_tok_str(tok, &tokc), s1->ppfp);
3901         token_seen = pp_check_he0xE(tok, p);
3902     }
3903     return 0;
3904 }
3905
3906 /* ------------------------------------------------------------------------- */