tccpp.c

   1 /*
   2  *  TCC - Tiny C Compiler
   3  *
   4  *  Copyright (c) 2001-2004 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #define USING_GLOBALS
  22 #include "tcc.h"
  23
  24 /********************************************************/
  25 /* global variables */
  26
  27 ST_DATA int tok_flags;
  28 ST_DATA int parse_flags;
  29
  30 ST_DATA struct BufferedFile *file;
  31 ST_DATA int ch, tok;
  32 ST_DATA CValue tokc;
  33 ST_DATA const int *macro_ptr;
  34 ST_DATA CString tokcstr; /* current parsed string, if any */
  35
  36 /* display benchmark infos */
  37 ST_DATA int total_lines;
  38 ST_DATA int total_bytes;
  39 ST_DATA int tok_ident;
  40 ST_DATA TokenSym **table_ident;
  41
  42 /* ------------------------------------------------------------------------- */
  43
  44 static TokenSym *hash_ident[TOK_HASH_SIZE];
  45 static char token_buf[STRING_MAX_SIZE + 1];
  46 static CString cstr_buf;
  47 static CString macro_equal_buf;
  48 static TokenString tokstr_buf;
  49 static unsigned char isidnum_table[256 - CH_EOF];
  50 static int pp_debug_tok, pp_debug_symv;
  51 static int pp_once;
  52 static int pp_expr;
  53 static int pp_counter;
  54 static void tok_print(const char *msg, const int *str);
  55
  56 static struct TinyAlloc *toksym_alloc;
  57 static struct TinyAlloc *tokstr_alloc;
  58
  59 static TokenString *macro_stack;
  60
  61 static const char tcc_keywords[] =
  62 #define DEF(id, str) str "\0"
  63 #include "tcctok.h"
  64 #undef DEF
  65 ;
  66
  67 /* WARNING: the content of this string encodes token numbers */
  68 static const unsigned char tok_two_chars[] =
  69 /* outdated -- gr
  70     "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
  71     "-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
  72 */{
  73     '<','=', TOK_LE,
  74     '>','=', TOK_GE,
  75     '!','=', TOK_NE,
  76     '&','&', TOK_LAND,
  77     '|','|', TOK_LOR,
  78     '+','+', TOK_INC,
  79     '-','-', TOK_DEC,
  80     '=','=', TOK_EQ,
  81     '<','<', TOK_SHL,
  82     '>','>', TOK_SAR,
  83     '+','=', TOK_A_ADD,
  84     '-','=', TOK_A_SUB,
  85     '*','=', TOK_A_MUL,
  86     '/','=', TOK_A_DIV,
  87     '%','=', TOK_A_MOD,
  88     '&','=', TOK_A_AND,
  89     '^','=', TOK_A_XOR,
  90     '|','=', TOK_A_OR,
  91     '-','>', TOK_ARROW,
  92     '.','.', TOK_TWODOTS,
  93     '#','#', TOK_TWOSHARPS,
  94     0
  95 };
  96
  97 static void next_nomacro_spc(void);
  98
  99 ST_FUNC void skip(int c)
 100 {
 101     if (tok != c)
 102         tcc_error("'%c' expected (got \"%s\")", c, get_tok_str(tok, &tokc));
 103     next();
 104 }
 105
 106 ST_FUNC void expect(const char *msg)
 107 {
 108     tcc_error("%s expected", msg);
 109 }
 110
 111 /* ------------------------------------------------------------------------- */
 112 /* Custom allocator for tiny objects */
 113
 114 #define USE_TAL
 115
 116 #ifndef USE_TAL
 117 #define tal_free(al, p) tcc_free(p)
 118 #define tal_realloc(al, p, size) tcc_realloc(p, size)
 119 #define tal_new(a,b,c)
 120 #define tal_delete(a)
 121 #else
 122 #if !defined(MEM_DEBUG)
 123 #define tal_free(al, p) tal_free_impl(al, p)
 124 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size)
 125 #define TAL_DEBUG_PARAMS
 126 #else
 127 #define TAL_DEBUG 1
 128 //#define TAL_INFO 1 /* collect and dump allocators stats */
 129 #define tal_free(al, p) tal_free_impl(al, p, __FILE__, __LINE__)
 130 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size, __FILE__, __LINE__)
 131 #define TAL_DEBUG_PARAMS , const char *file, int line
 132 #define TAL_DEBUG_FILE_LEN 40
 133 #endif
 134
 135 #define TOKSYM_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenSym in table_ident */
 136 #define TOKSTR_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenString instances */
 137 #define CSTR_TAL_SIZE       (256 * 1024) /* allocator for tiny CString instances */
 138 #define TOKSYM_TAL_LIMIT    256 /* prefer unique limits to distinguish allocators debug msgs */
 139 #define TOKSTR_TAL_LIMIT    128 /* 32 * sizeof(int) */
 140 #define CSTR_TAL_LIMIT      1024
 141
 142 typedef struct TinyAlloc {
 143     unsigned  limit;
 144     unsigned  size;
 145     uint8_t *buffer;
 146     uint8_t *p;
 147     unsigned  nb_allocs;
 148     struct TinyAlloc *next, *top;
 149 #ifdef TAL_INFO
 150     unsigned  nb_peak;
 151     unsigned  nb_total;
 152     unsigned  nb_missed;
 153     uint8_t *peak_p;
 154 #endif
 155 } TinyAlloc;
 156
 157 typedef struct tal_header_t {
 158     unsigned  size;
 159 #ifdef TAL_DEBUG
 160     int     line_num; /* negative line_num used for double free check */
 161     char    file_name[TAL_DEBUG_FILE_LEN + 1];
 162 #endif
 163 } tal_header_t;
 164
 165 /* ------------------------------------------------------------------------- */
 166
 167 static TinyAlloc *tal_new(TinyAlloc **pal, unsigned limit, unsigned size)
 168 {
 169     TinyAlloc *al = tcc_mallocz(sizeof(TinyAlloc));
 170     al->p = al->buffer = tcc_malloc(size);
 171     al->limit = limit;
 172     al->size = size;
 173     if (pal) *pal = al;
 174     return al;
 175 }
 176
 177 static void tal_delete(TinyAlloc *al)
 178 {
 179     TinyAlloc *next;
 180
 181 tail_call:
 182     if (!al)
 183         return;
 184 #ifdef TAL_INFO
 185     fprintf(stderr, "limit=%5d, size=%5g MB, nb_peak=%6d, nb_total=%8d, nb_missed=%6d, usage=%5.1f%%\n",
 186             al->limit, al->size / 1024.0 / 1024.0, al->nb_peak, al->nb_total, al->nb_missed,
 187             (al->peak_p - al->buffer) * 100.0 / al->size);
 188 #endif
 189 #ifdef TAL_DEBUG
 190     if (al->nb_allocs > 0) {
 191         uint8_t *p;
 192         fprintf(stderr, "TAL_DEBUG: memory leak %d chunk(s) (limit= %d)\n",
 193                 al->nb_allocs, al->limit);
 194         p = al->buffer;
 195         while (p < al->p) {
 196             tal_header_t *header = (tal_header_t *)p;
 197             if (header->line_num > 0) {
 198                 fprintf(stderr, "%s:%d: chunk of %d bytes leaked\n",
 199                         header->file_name, header->line_num, header->size);
 200             }
 201             p += header->size + sizeof(tal_header_t);
 202         }
 203 #if MEM_DEBUG-0 == 2
 204         exit(2);
 205 #endif
 206     }
 207 #endif
 208     next = al->next;
 209     tcc_free(al->buffer);
 210     tcc_free(al);
 211     al = next;
 212     goto tail_call;
 213 }
 214
 215 static void tal_free_impl(TinyAlloc *al, void *p TAL_DEBUG_PARAMS)
 216 {
 217     if (!p)
 218         return;
 219 tail_call:
 220     if (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size) {
 221 #ifdef TAL_DEBUG
 222         tal_header_t *header = (((tal_header_t *)p) - 1);
 223         if (header->line_num < 0) {
 224             fprintf(stderr, "%s:%d: TAL_DEBUG: double frees chunk from\n",
 225                     file, line);
 226             fprintf(stderr, "%s:%d: %d bytes\n",
 227                     header->file_name, (int)-header->line_num, (int)header->size);
 228         } else
 229             header->line_num = -header->line_num;
 230 #endif
 231         al->nb_allocs--;
 232         if (!al->nb_allocs)
 233             al->p = al->buffer;
 234     } else if (al->next) {
 235         al = al->next;
 236         goto tail_call;
 237     }
 238     else
 239         tcc_free(p);
 240 }
 241
 242 static void *tal_realloc_impl(TinyAlloc **pal, void *p, unsigned size TAL_DEBUG_PARAMS)
 243 {
 244     tal_header_t *header;
 245     void *ret;
 246     int is_own;
 247     unsigned adj_size = (size + 3) & -4;
 248     TinyAlloc *al = *pal;
 249
 250 tail_call:
 251     is_own = (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size);
 252     if ((!p || is_own) && size <= al->limit) {
 253         if (al->p + adj_size + sizeof(tal_header_t) < al->buffer + al->size) {
 254             header = (tal_header_t *)al->p;
 255             header->size = adj_size;
 256 #ifdef TAL_DEBUG
 257             { int ofs = strlen(file) - TAL_DEBUG_FILE_LEN;
 258             strncpy(header->file_name, file + (ofs > 0 ? ofs : 0), TAL_DEBUG_FILE_LEN);
 259             header->file_name[TAL_DEBUG_FILE_LEN] = 0;
 260             header->line_num = line; }
 261 #endif
 262             ret = al->p + sizeof(tal_header_t);
 263             al->p += adj_size + sizeof(tal_header_t);
 264             if (is_own) {
 265                 header = (((tal_header_t *)p) - 1);
 266                 memcpy(ret, p, header->size);
 267 #ifdef TAL_DEBUG
 268                 header->line_num = -header->line_num;
 269 #endif
 270             } else {
 271                 al->nb_allocs++;
 272             }
 273 #ifdef TAL_INFO
 274             if (al->nb_peak < al->nb_allocs)
 275                 al->nb_peak = al->nb_allocs;
 276             if (al->peak_p < al->p)
 277                 al->peak_p = al->p;
 278             al->nb_total++;
 279 #endif
 280             return ret;
 281         } else if (is_own) {
 282             al->nb_allocs--;
 283             ret = tal_realloc(*pal, 0, size);
 284             header = (((tal_header_t *)p) - 1);
 285             memcpy(ret, p, header->size);
 286 #ifdef TAL_DEBUG
 287             header->line_num = -header->line_num;
 288 #endif
 289             return ret;
 290         }
 291         if (al->next) {
 292             al = al->next;
 293         } else {
 294             TinyAlloc *bottom = al, *next = al->top ? al->top : al;
 295
 296             al = tal_new(pal, next->limit, next->size * 2);
 297             al->next = next;
 298             bottom->top = al;
 299         }
 300         goto tail_call;
 301     }
 302     if (is_own) {
 303         al->nb_allocs--;
 304         ret = tcc_malloc(size);
 305         header = (((tal_header_t *)p) - 1);
 306         memcpy(ret, p, header->size);
 307 #ifdef TAL_DEBUG
 308         header->line_num = -header->line_num;
 309 #endif
 310     } else if (al->next) {
 311         al = al->next;
 312         goto tail_call;
 313     } else
 314         ret = tcc_realloc(p, size);
 315 #ifdef TAL_INFO
 316     al->nb_missed++;
 317 #endif
 318     return ret;
 319 }
 320
 321 #endif /* USE_TAL */
 322
 323 /* ------------------------------------------------------------------------- */
 324 /* CString handling */
 325 static void cstr_realloc(CString *cstr, int new_size)
 326 {
 327     int size;
 328
 329     size = cstr->size_allocated;
 330     if (size < 8)
 331         size = 8; /* no need to allocate a too small first string */
 332     while (size < new_size)
 333         size = size * 2;
 334     cstr->data = tcc_realloc(cstr->data, size);
 335     cstr->size_allocated = size;
 336 }
 337
 338 /* add a byte */
 339 ST_INLN void cstr_ccat(CString *cstr, int ch)
 340 {
 341     int size;
 342     size = cstr->size + 1;
 343     if (size > cstr->size_allocated)
 344         cstr_realloc(cstr, size);
 345     ((unsigned char *)cstr->data)[size - 1] = ch;
 346     cstr->size = size;
 347 }
 348
 349 ST_FUNC void cstr_cat(CString *cstr, const char *str, int len)
 350 {
 351     int size;
 352     if (len <= 0)
 353         len = strlen(str) + 1 + len;
 354     size = cstr->size + len;
 355     if (size > cstr->size_allocated)
 356         cstr_realloc(cstr, size);
 357     memmove(((unsigned char *)cstr->data) + cstr->size, str, len);
 358     cstr->size = size;
 359 }
 360
 361 /* add a wide char */
 362 ST_FUNC void cstr_wccat(CString *cstr, int ch)
 363 {
 364     int size;
 365     size = cstr->size + sizeof(nwchar_t);
 366     if (size > cstr->size_allocated)
 367         cstr_realloc(cstr, size);
 368     *(nwchar_t *)(((unsigned char *)cstr->data) + size - sizeof(nwchar_t)) = ch;
 369     cstr->size = size;
 370 }
 371
 372 ST_FUNC void cstr_new(CString *cstr)
 373 {
 374     memset(cstr, 0, sizeof(CString));
 375 }
 376
 377 /* free string and reset it to NULL */
 378 ST_FUNC void cstr_free(CString *cstr)
 379 {
 380     tcc_free(cstr->data);
 381     cstr_new(cstr);
 382 }
 383
 384 /* reset string to empty */
 385 ST_FUNC void cstr_reset(CString *cstr)
 386 {
 387     cstr->size = 0;
 388 }
 389
 390 ST_FUNC int cstr_printf(CString *cstr, const char *fmt, ...)
 391 {
 392     va_list v;
 393     int len, size;
 394
 395     va_start(v, fmt);
 396     len = vsnprintf(NULL, 0, fmt, v);
 397     va_end(v);
 398     size = cstr->size + len + 1;
 399     if (size > cstr->size_allocated)
 400         cstr_realloc(cstr, size);
 401     va_start(v, fmt);
 402     vsnprintf((char*)cstr->data + cstr->size, size, fmt, v);
 403     va_end(v);
 404     cstr->size += len;
 405     return len;
 406 }
 407
 408 /* XXX: unicode ? */
 409 static void add_char(CString *cstr, int c)
 410 {
 411     if (c == '\'' || c == '\"' || c == '\\') {
 412         /* XXX: could be more precise if char or string */
 413         cstr_ccat(cstr, '\\');
 414     }
 415     if (c >= 32 && c <= 126) {
 416         cstr_ccat(cstr, c);
 417     } else {
 418         cstr_ccat(cstr, '\\');
 419         if (c == '\n') {
 420             cstr_ccat(cstr, 'n');
 421         } else {
 422             cstr_ccat(cstr, '0' + ((c >> 6) & 7));
 423             cstr_ccat(cstr, '0' + ((c >> 3) & 7));
 424             cstr_ccat(cstr, '0' + (c & 7));
 425         }
 426     }
 427 }
 428
 429 /* ------------------------------------------------------------------------- */
 430 /* allocate a new token */
 431 static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
 432 {
 433     TokenSym *ts, **ptable;
 434     int i;
 435
 436     if (tok_ident >= SYM_FIRST_ANOM)
 437         tcc_error("memory full (symbols)");
 438
 439     /* expand token table if needed */
 440     i = tok_ident - TOK_IDENT;
 441     if ((i % TOK_ALLOC_INCR) == 0) {
 442         ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
 443         table_ident = ptable;
 444     }
 445
 446     ts = tal_realloc(toksym_alloc, 0, sizeof(TokenSym) + len);
 447     table_ident[i] = ts;
 448     ts->tok = tok_ident++;
 449     ts->sym_define = NULL;
 450     ts->sym_label = NULL;
 451     ts->sym_struct = NULL;
 452     ts->sym_identifier = NULL;
 453     ts->len = len;
 454     ts->hash_next = NULL;
 455     memcpy(ts->str, str, len);
 456     ts->str[len] = '\0';
 457     *pts = ts;
 458     return ts;
 459 }
 460
 461 #define TOK_HASH_INIT 1
 462 #define TOK_HASH_FUNC(h, c) ((h) + ((h) << 5) + ((h) >> 27) + (c))
 463
 464
 465 /* find a token and add it if not found */
 466 ST_FUNC TokenSym *tok_alloc(const char *str, int len)
 467 {
 468     TokenSym *ts, **pts;
 469     int i;
 470     unsigned int h;
 471
 472     h = TOK_HASH_INIT;
 473     for(i=0;i<len;i++)
 474         h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
 475     h &= (TOK_HASH_SIZE - 1);
 476
 477     pts = &hash_ident[h];
 478     for(;;) {
 479         ts = *pts;
 480         if (!ts)
 481             break;
 482         if (ts->len == len && !memcmp(ts->str, str, len))
 483             return ts;
 484         pts = &(ts->hash_next);
 485     }
 486     return tok_alloc_new(pts, str, len);
 487 }
 488
 489 /* XXX: buffer overflow */
 490 /* XXX: float tokens */
 491 ST_FUNC const char *get_tok_str(int v, CValue *cv)
 492 {
 493     char *p;
 494     int i, len;
 495
 496     cstr_reset(&cstr_buf);
 497     p = cstr_buf.data;
 498
 499     switch(v) {
 500     case TOK_CINT:
 501     case TOK_CUINT:
 502     case TOK_CLONG:
 503     case TOK_CULONG:
 504     case TOK_CLLONG:
 505     case TOK_CULLONG:
 506         /* XXX: not quite exact, but only useful for testing  */
 507 #ifdef _WIN32
 508         sprintf(p, "%u", (unsigned)cv->i);
 509 #else
 510         sprintf(p, "%llu", (unsigned long long)cv->i);
 511 #endif
 512         break;
 513     case TOK_LCHAR:
 514         cstr_ccat(&cstr_buf, 'L');
 515     case TOK_CCHAR:
 516         cstr_ccat(&cstr_buf, '\'');
 517         add_char(&cstr_buf, cv->i);
 518         cstr_ccat(&cstr_buf, '\'');
 519         cstr_ccat(&cstr_buf, '\0');
 520         break;
 521     case TOK_PPNUM:
 522     case TOK_PPSTR:
 523         return (char*)cv->str.data;
 524     case TOK_LSTR:
 525         cstr_ccat(&cstr_buf, 'L');
 526     case TOK_STR:
 527         cstr_ccat(&cstr_buf, '\"');
 528         if (v == TOK_STR) {
 529             len = cv->str.size - 1;
 530             for(i=0;i<len;i++)
 531                 add_char(&cstr_buf, ((unsigned char *)cv->str.data)[i]);
 532         } else {
 533             len = (cv->str.size / sizeof(nwchar_t)) - 1;
 534             for(i=0;i<len;i++)
 535                 add_char(&cstr_buf, ((nwchar_t *)cv->str.data)[i]);
 536         }
 537         cstr_ccat(&cstr_buf, '\"');
 538         cstr_ccat(&cstr_buf, '\0');
 539         break;
 540
 541     case TOK_CFLOAT:
 542         cstr_cat(&cstr_buf, "<float>", 0);
 543         break;
 544     case TOK_CDOUBLE:
 545         cstr_cat(&cstr_buf, "<double>", 0);
 546         break;
 547     case TOK_CLDOUBLE:
 548         cstr_cat(&cstr_buf, "<long double>", 0);
 549         break;
 550     case TOK_LINENUM:
 551         cstr_cat(&cstr_buf, "<linenumber>", 0);
 552         break;
 553
 554     /* above tokens have value, the ones below don't */
 555     case TOK_LT:
 556         v = '<';
 557         goto addv;
 558     case TOK_GT:
 559         v = '>';
 560         goto addv;
 561     case TOK_DOTS:
 562         return strcpy(p, "...");
 563     case TOK_A_SHL:
 564         return strcpy(p, "<<=");
 565     case TOK_A_SAR:
 566         return strcpy(p, ">>=");
 567     case TOK_EOF:
 568         return strcpy(p, "<eof>");
 569     default:
 570         if (v < TOK_IDENT) {
 571             /* search in two bytes table */
 572             const unsigned char *q = tok_two_chars;
 573             while (*q) {
 574                 if (q[2] == v) {
 575                     *p++ = q[0];
 576                     *p++ = q[1];
 577                     *p = '\0';
 578                     return cstr_buf.data;
 579                 }
 580                 q += 3;
 581             }
 582         if (v >= 127) {
 583             sprintf(cstr_buf.data, "<%02x>", v);
 584             return cstr_buf.data;
 585         }
 586         addv:
 587             *p++ = v;
 588             *p = '\0';
 589         } else if (v < tok_ident) {
 590             return table_ident[v - TOK_IDENT]->str;
 591         } else if (v >= SYM_FIRST_ANOM) {
 592             /* special name for anonymous symbol */
 593             sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
 594         } else {
 595             /* should never happen */
 596             return NULL;
 597         }
 598         break;
 599     }
 600     return cstr_buf.data;
 601 }
 602
 603 /* return the current character, handling end of block if necessary
 604    (but not stray) */
 605 ST_FUNC int handle_eob(void)
 606 {
 607     BufferedFile *bf = file;
 608     int len;
 609
 610     /* only tries to read if really end of buffer */
 611     if (bf->buf_ptr >= bf->buf_end) {
 612         if (bf->fd >= 0) {
 613 #if defined(PARSE_DEBUG)
 614             len = 1;
 615 #else
 616             len = IO_BUF_SIZE;
 617 #endif
 618             len = read(bf->fd, bf->buffer, len);
 619             if (len < 0)
 620                 len = 0;
 621         } else {
 622             len = 0;
 623         }
 624         total_bytes += len;
 625         bf->buf_ptr = bf->buffer;
 626         bf->buf_end = bf->buffer + len;
 627         *bf->buf_end = CH_EOB;
 628     }
 629     if (bf->buf_ptr < bf->buf_end) {
 630         return bf->buf_ptr[0];
 631     } else {
 632         bf->buf_ptr = bf->buf_end;
 633         return CH_EOF;
 634     }
 635 }
 636
 637 /* read next char from current input file and handle end of input buffer */
 638 ST_INLN void inp(void)
 639 {
 640     ch = *(++(file->buf_ptr));
 641     /* end of buffer/file handling */
 642     if (ch == CH_EOB)
 643         ch = handle_eob();
 644 }
 645
 646 /* handle '\[\r]\n' */
 647 static int handle_stray_noerror(void)
 648 {
 649     while (ch == '\\') {
 650         inp();
 651         if (ch == '\n') {
 652             file->line_num++;
 653             inp();
 654         } else if (ch == '\r') {
 655             inp();
 656             if (ch != '\n')
 657                 goto fail;
 658             file->line_num++;
 659             inp();
 660         } else {
 661         fail:
 662             return 1;
 663         }
 664     }
 665     return 0;
 666 }
 667
 668 static void handle_stray(void)
 669 {
 670     if (handle_stray_noerror())
 671         tcc_error("stray '\\' in program");
 672 }
 673
 674 /* skip the stray and handle the \\n case. Output an error if
 675    incorrect char after the stray */
 676 static int handle_stray1(uint8_t *p)
 677 {
 678     int c;
 679
 680     file->buf_ptr = p;
 681     if (p >= file->buf_end) {
 682         c = handle_eob();
 683         if (c != '\\')
 684             return c;
 685         p = file->buf_ptr;
 686     }
 687     ch = *p;
 688     if (handle_stray_noerror()) {
 689         if (!(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
 690             tcc_error("stray '\\' in program");
 691         *--file->buf_ptr = '\\';
 692     }
 693     p = file->buf_ptr;
 694     c = *p;
 695     return c;
 696 }
 697
 698 /* handle just the EOB case, but not stray */
 699 #define PEEKC_EOB(c, p)\
 700 {\
 701     p++;\
 702     c = *p;\
 703     if (c == '\\') {\
 704         file->buf_ptr = p;\
 705         c = handle_eob();\
 706         p = file->buf_ptr;\
 707     }\
 708 }
 709
 710 /* handle the complicated stray case */
 711 #define PEEKC(c, p)\
 712 {\
 713     p++;\
 714     c = *p;\
 715     if (c == '\\') {\
 716         c = handle_stray1(p);\
 717         p = file->buf_ptr;\
 718     }\
 719 }
 720
 721 /* input with '\[\r]\n' handling. Note that this function cannot
 722    handle other characters after '\', so you cannot call it inside
 723    strings or comments */
 724 ST_FUNC void minp(void)
 725 {
 726     inp();
 727     if (ch == '\\')
 728         handle_stray();
 729 }
 730
 731 /* single line C++ comments */
 732 static uint8_t *parse_line_comment(uint8_t *p)
 733 {
 734     int c;
 735
 736     p++;
 737     for(;;) {
 738         c = *p;
 739     redo:
 740         if (c == '\n' || c == CH_EOF) {
 741             break;
 742         } else if (c == '\\') {
 743             file->buf_ptr = p;
 744             c = handle_eob();
 745             p = file->buf_ptr;
 746             if (c == '\\') {
 747                 PEEKC_EOB(c, p);
 748                 if (c == '\n') {
 749                     file->line_num++;
 750                     PEEKC_EOB(c, p);
 751                 } else if (c == '\r') {
 752                     PEEKC_EOB(c, p);
 753                     if (c == '\n') {
 754                         file->line_num++;
 755                         PEEKC_EOB(c, p);
 756                     }
 757                 }
 758             } else {
 759                 goto redo;
 760             }
 761         } else {
 762             p++;
 763         }
 764     }
 765     return p;
 766 }
 767
 768 /* C comments */
 769 ST_FUNC uint8_t *parse_comment(uint8_t *p)
 770 {
 771     int c;
 772
 773     p++;
 774     for(;;) {
 775         /* fast skip loop */
 776         for(;;) {
 777             c = *p;
 778             if (c == '\n' || c == '*' || c == '\\')
 779                 break;
 780             p++;
 781             c = *p;
 782             if (c == '\n' || c == '*' || c == '\\')
 783                 break;
 784             p++;
 785         }
 786         /* now we can handle all the cases */
 787         if (c == '\n') {
 788             file->line_num++;
 789             p++;
 790         } else if (c == '*') {
 791             p++;
 792             for(;;) {
 793                 c = *p;
 794                 if (c == '*') {
 795                     p++;
 796                 } else if (c == '/') {
 797                     goto end_of_comment;
 798                 } else if (c == '\\') {
 799                     file->buf_ptr = p;
 800                     c = handle_eob();
 801                     p = file->buf_ptr;
 802                     if (c == CH_EOF)
 803                         tcc_error("unexpected end of file in comment");
 804                     if (c == '\\') {
 805                         /* skip '\[\r]\n', otherwise just skip the stray */
 806                         while (c == '\\') {
 807                             PEEKC_EOB(c, p);
 808                             if (c == '\n') {
 809                                 file->line_num++;
 810                                 PEEKC_EOB(c, p);
 811                             } else if (c == '\r') {
 812                                 PEEKC_EOB(c, p);
 813                                 if (c == '\n') {
 814                                     file->line_num++;
 815                                     PEEKC_EOB(c, p);
 816                                 }
 817                             } else {
 818                                 goto after_star;
 819                             }
 820                         }
 821                     }
 822                 } else {
 823                     break;
 824                 }
 825             }
 826         after_star: ;
 827         } else {
 828             /* stray, eob or eof */
 829             file->buf_ptr = p;
 830             c = handle_eob();
 831             p = file->buf_ptr;
 832             if (c == CH_EOF) {
 833                 tcc_error("unexpected end of file in comment");
 834             } else if (c == '\\') {
 835                 p++;
 836             }
 837         }
 838     }
 839  end_of_comment:
 840     p++;
 841     return p;
 842 }
 843
 844 ST_FUNC int set_idnum(int c, int val)
 845 {
 846     int prev = isidnum_table[c - CH_EOF];
 847     isidnum_table[c - CH_EOF] = val;
 848     return prev;
 849 }
 850
 851 #define cinp minp
 852
 853 static inline void skip_spaces(void)
 854 {
 855     while (isidnum_table[ch - CH_EOF] & IS_SPC)
 856         cinp();
 857 }
 858
 859 static inline int check_space(int t, int *spc)
 860 {
 861     if (t < 256 && (isidnum_table[t - CH_EOF] & IS_SPC)) {
 862         if (*spc)
 863             return 1;
 864         *spc = 1;
 865     } else
 866         *spc = 0;
 867     return 0;
 868 }
 869
 870 /* parse a string without interpreting escapes */
 871 static uint8_t *parse_pp_string(uint8_t *p,
 872                                 int sep, CString *str)
 873 {
 874     int c;
 875     p++;
 876     for(;;) {
 877         c = *p;
 878         if (c == sep) {
 879             break;
 880         } else if (c == '\\') {
 881             file->buf_ptr = p;
 882             c = handle_eob();
 883             p = file->buf_ptr;
 884             if (c == CH_EOF) {
 885             unterminated_string:
 886                 /* XXX: indicate line number of start of string */
 887                 tcc_error("missing terminating %c character", sep);
 888             } else if (c == '\\') {
 889                 /* escape : just skip \[\r]\n */
 890                 PEEKC_EOB(c, p);
 891                 if (c == '\n') {
 892                     file->line_num++;
 893                     p++;
 894                 } else if (c == '\r') {
 895                     PEEKC_EOB(c, p);
 896                     if (c != '\n')
 897                         expect("'\n' after '\r'");
 898                     file->line_num++;
 899                     p++;
 900                 } else if (c == CH_EOF) {
 901                     goto unterminated_string;
 902                 } else {
 903                     if (str) {
 904                         cstr_ccat(str, '\\');
 905                         cstr_ccat(str, c);
 906                     }
 907                     p++;
 908                 }
 909             }
 910         } else if (c == '\n') {
 911             file->line_num++;
 912             goto add_char;
 913         } else if (c == '\r') {
 914             PEEKC_EOB(c, p);
 915             if (c != '\n') {
 916                 if (str)
 917                     cstr_ccat(str, '\r');
 918             } else {
 919                 file->line_num++;
 920                 goto add_char;
 921             }
 922         } else {
 923         add_char:
 924             if (str)
 925                 cstr_ccat(str, c);
 926             p++;
 927         }
 928     }
 929     p++;
 930     return p;
 931 }
 932
 933 /* skip block of text until #else, #elif or #endif. skip also pairs of
 934    #if/#endif */
 935 static void preprocess_skip(void)
 936 {
 937     int a, start_of_line, c, in_warn_or_error;
 938     uint8_t *p;
 939
 940     p = file->buf_ptr;
 941     a = 0;
 942 redo_start:
 943     start_of_line = 1;
 944     in_warn_or_error = 0;
 945     for(;;) {
 946     redo_no_start:
 947         c = *p;
 948         switch(c) {
 949         case ' ':
 950         case '\t':
 951         case '\f':
 952         case '\v':
 953         case '\r':
 954             p++;
 955             goto redo_no_start;
 956         case '\n':
 957             file->line_num++;
 958             p++;
 959             goto redo_start;
 960         case '\\':
 961             file->buf_ptr = p;
 962             c = handle_eob();
 963             if (c == CH_EOF) {
 964                 expect("#endif");
 965             } else if (c == '\\') {
 966                 ch = file->buf_ptr[0];
 967                 handle_stray_noerror();
 968             }
 969             p = file->buf_ptr;
 970             goto redo_no_start;
 971         /* skip strings */
 972         case '\"':
 973         case '\'':
 974             if (in_warn_or_error)
 975                 goto _default;
 976             p = parse_pp_string(p, c, NULL);
 977             break;
 978         /* skip comments */
 979         case '/':
 980             if (in_warn_or_error)
 981                 goto _default;
 982             file->buf_ptr = p;
 983             ch = *p;
 984             minp();
 985             p = file->buf_ptr;
 986             if (ch == '*') {
 987                 p = parse_comment(p);
 988             } else if (ch == '/') {
 989                 p = parse_line_comment(p);
 990             }
 991             break;
 992         case '#':
 993             p++;
 994             if (start_of_line) {
 995                 file->buf_ptr = p;
 996                 next_nomacro();
 997                 p = file->buf_ptr;
 998                 if (a == 0 &&
 999                     (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
1000                     goto the_end;
1001                 if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
1002                     a++;
1003                 else if (tok == TOK_ENDIF)
1004                     a--;
1005                 else if( tok == TOK_ERROR || tok == TOK_WARNING)
1006                     in_warn_or_error = 1;
1007                 else if (tok == TOK_LINEFEED)
1008                     goto redo_start;
1009                 else if (parse_flags & PARSE_FLAG_ASM_FILE)
1010                     p = parse_line_comment(p - 1);
1011             } else if (parse_flags & PARSE_FLAG_ASM_FILE)
1012                 p = parse_line_comment(p - 1);
1013             break;
1014 _default:
1015         default:
1016             p++;
1017             break;
1018         }
1019         start_of_line = 0;
1020     }
1021  the_end: ;
1022     file->buf_ptr = p;
1023 }
1024
1025 #if 0
1026 /* return the number of additional 'ints' necessary to store the
1027    token */
1028 static inline int tok_size(const int *p)
1029 {
1030     switch(*p) {
1031         /* 4 bytes */
1032     case TOK_CINT:
1033     case TOK_CUINT:
1034     case TOK_CCHAR:
1035     case TOK_LCHAR:
1036     case TOK_CFLOAT:
1037     case TOK_LINENUM:
1038         return 1 + 1;
1039     case TOK_STR:
1040     case TOK_LSTR:
1041     case TOK_PPNUM:
1042     case TOK_PPSTR:
1043         return 1 + ((sizeof(CString) + ((CString *)(p+1))->size + 3) >> 2);
1044     case TOK_CLONG:
1045     case TOK_CULONG:
1046         return 1 + LONG_SIZE / 4;
1047     case TOK_CDOUBLE:
1048     case TOK_CLLONG:
1049     case TOK_CULLONG:
1050         return 1 + 2;
1051     case TOK_CLDOUBLE:
1052         return 1 + LDOUBLE_SIZE / 4;
1053     default:
1054         return 1 + 0;
1055     }
1056 }
1057 #endif
1058
1059 /* token string handling */
1060 ST_INLN void tok_str_new(TokenString *s)
1061 {
1062     s->str = NULL;
1063     s->len = s->lastlen = 0;
1064     s->allocated_len = 0;
1065     s->last_line_num = -1;
1066 }
1067
1068 ST_FUNC TokenString *tok_str_alloc(void)
1069 {
1070     TokenString *str = tal_realloc(tokstr_alloc, 0, sizeof *str);
1071     tok_str_new(str);
1072     return str;
1073 }
1074
1075 ST_FUNC int *tok_str_dup(TokenString *s)
1076 {
1077     int *str;
1078
1079     str = tal_realloc(tokstr_alloc, 0, s->len * sizeof(int));
1080     memcpy(str, s->str, s->len * sizeof(int));
1081     return str;
1082 }
1083
1084 ST_FUNC void tok_str_free_str(int *str)
1085 {
1086     tal_free(tokstr_alloc, str);
1087 }
1088
1089 ST_FUNC void tok_str_free(TokenString *str)
1090 {
1091     tok_str_free_str(str->str);
1092     tal_free(tokstr_alloc, str);
1093 }
1094
1095 ST_FUNC int *tok_str_realloc(TokenString *s, int new_size)
1096 {
1097     int *str, size;
1098
1099     size = s->allocated_len;
1100     if (size < 16)
1101         size = 16;
1102     while (size < new_size)
1103         size = size * 2;
1104     if (size > s->allocated_len) {
1105         str = tal_realloc(tokstr_alloc, s->str, size * sizeof(int));
1106         s->allocated_len = size;
1107         s->str = str;
1108     }
1109     return s->str;
1110 }
1111
1112 ST_FUNC void tok_str_add(TokenString *s, int t)
1113 {
1114     int len, *str;
1115
1116     len = s->len;
1117     str = s->str;
1118     if (len >= s->allocated_len)
1119         str = tok_str_realloc(s, len + 1);
1120     str[len++] = t;
1121     s->len = len;
1122 }
1123
1124 ST_FUNC void begin_macro(TokenString *str, int alloc)
1125 {
1126     str->alloc = alloc;
1127     str->prev = macro_stack;
1128     str->prev_ptr = macro_ptr;
1129     str->save_line_num = file->line_num;
1130     macro_ptr = str->str;
1131     macro_stack = str;
1132 }
1133
1134 ST_FUNC void end_macro(void)
1135 {
1136     TokenString *str = macro_stack;
1137     macro_stack = str->prev;
1138     macro_ptr = str->prev_ptr;
1139     file->line_num = str->save_line_num;
1140     if (str->alloc != 0) {
1141         if (str->alloc == 2)
1142             str->str = NULL; /* don't free */
1143         tok_str_free(str);
1144     }
1145 }
1146
1147 static void tok_str_add2(TokenString *s, int t, CValue *cv)
1148 {
1149     int len, *str;
1150
1151     len = s->lastlen = s->len;
1152     str = s->str;
1153
1154     /* allocate space for worst case */
1155     if (len + TOK_MAX_SIZE >= s->allocated_len)
1156         str = tok_str_realloc(s, len + TOK_MAX_SIZE + 1);
1157     str[len++] = t;
1158     switch(t) {
1159     case TOK_CINT:
1160     case TOK_CUINT:
1161     case TOK_CCHAR:
1162     case TOK_LCHAR:
1163     case TOK_CFLOAT:
1164     case TOK_LINENUM:
1165 #if LONG_SIZE == 4
1166     case TOK_CLONG:
1167     case TOK_CULONG:
1168 #endif
1169         str[len++] = cv->tab[0];
1170         break;
1171     case TOK_PPNUM:
1172     case TOK_PPSTR:
1173     case TOK_STR:
1174     case TOK_LSTR:
1175         {
1176             /* Insert the string into the int array. */
1177             size_t nb_words =
1178                 1 + (cv->str.size + sizeof(int) - 1) / sizeof(int);
1179             if (len + nb_words >= s->allocated_len)
1180                 str = tok_str_realloc(s, len + nb_words + 1);
1181             str[len] = cv->str.size;
1182             memcpy(&str[len + 1], cv->str.data, cv->str.size);
1183             len += nb_words;
1184         }
1185         break;
1186     case TOK_CDOUBLE:
1187     case TOK_CLLONG:
1188     case TOK_CULLONG:
1189 #if LONG_SIZE == 8
1190     case TOK_CLONG:
1191     case TOK_CULONG:
1192 #endif
1193 #if LDOUBLE_SIZE == 8
1194     case TOK_CLDOUBLE:
1195 #endif
1196         str[len++] = cv->tab[0];
1197         str[len++] = cv->tab[1];
1198         break;
1199 #if LDOUBLE_SIZE == 12
1200     case TOK_CLDOUBLE:
1201         str[len++] = cv->tab[0];
1202         str[len++] = cv->tab[1];
1203         str[len++] = cv->tab[2];
1204 #elif LDOUBLE_SIZE == 16
1205     case TOK_CLDOUBLE:
1206         str[len++] = cv->tab[0];
1207         str[len++] = cv->tab[1];
1208         str[len++] = cv->tab[2];
1209         str[len++] = cv->tab[3];
1210 #elif LDOUBLE_SIZE != 8
1211 #error add long double size support
1212 #endif
1213         break;
1214     default:
1215         break;
1216     }
1217     s->len = len;
1218 }
1219
1220 /* add the current parse token in token string 's' */
1221 ST_FUNC void tok_str_add_tok(TokenString *s)
1222 {
1223     CValue cval;
1224
1225     /* save line number info */
1226     if (file->line_num != s->last_line_num) {
1227         s->last_line_num = file->line_num;
1228         cval.i = s->last_line_num;
1229         tok_str_add2(s, TOK_LINENUM, &cval);
1230     }
1231     tok_str_add2(s, tok, &tokc);
1232 }
1233
1234 /* get a token from an integer array and increment pointer. */
1235 static inline void TOK_GET(int *t, const int **pp, CValue *cv)
1236 {
1237     const int *p = *pp;
1238     int n, *tab;
1239
1240     tab = cv->tab;
1241     switch(*t = *p++) {
1242 #if LONG_SIZE == 4
1243     case TOK_CLONG:
1244 #endif
1245     case TOK_CINT:
1246     case TOK_CCHAR:
1247     case TOK_LCHAR:
1248     case TOK_LINENUM:
1249         cv->i = *p++;
1250         break;
1251 #if LONG_SIZE == 4
1252     case TOK_CULONG:
1253 #endif
1254     case TOK_CUINT:
1255         cv->i = (unsigned)*p++;
1256         break;
1257     case TOK_CFLOAT:
1258         tab[0] = *p++;
1259         break;
1260     case TOK_STR:
1261     case TOK_LSTR:
1262     case TOK_PPNUM:
1263     case TOK_PPSTR:
1264         cv->str.size = *p++;
1265         cv->str.data = p;
1266         p += (cv->str.size + sizeof(int) - 1) / sizeof(int);
1267         break;
1268     case TOK_CDOUBLE:
1269     case TOK_CLLONG:
1270     case TOK_CULLONG:
1271 #if LONG_SIZE == 8
1272     case TOK_CLONG:
1273     case TOK_CULONG:
1274 #endif
1275         n = 2;
1276         goto copy;
1277     case TOK_CLDOUBLE:
1278 #if LDOUBLE_SIZE == 16
1279         n = 4;
1280 #elif LDOUBLE_SIZE == 12
1281         n = 3;
1282 #elif LDOUBLE_SIZE == 8
1283         n = 2;
1284 #else
1285 # error add long double size support
1286 #endif
1287     copy:
1288         do
1289             *tab++ = *p++;
1290         while (--n);
1291         break;
1292     default:
1293         break;
1294     }
1295     *pp = p;
1296 }
1297
1298 static int macro_is_equal(const int *a, const int *b)
1299 {
1300     CValue cv;
1301     int t;
1302
1303     if (!a || !b)
1304         return 1;
1305
1306     while (*a && *b) {
1307         /* first time preallocate macro_equal_buf, next time only reset position to start */
1308         cstr_reset(&macro_equal_buf);
1309         TOK_GET(&t, &a, &cv);
1310         cstr_cat(&macro_equal_buf, get_tok_str(t, &cv), 0);
1311         TOK_GET(&t, &b, &cv);
1312         if (strcmp(macro_equal_buf.data, get_tok_str(t, &cv)))
1313             return 0;
1314     }
1315     return !(*a || *b);
1316 }
1317
1318 /* defines handling */
1319 ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
1320 {
1321     Sym *s, *o;
1322
1323     o = define_find(v);
1324     s = sym_push2(&define_stack, v, macro_type, 0);
1325     s->d = str;
1326     s->next = first_arg;
1327     table_ident[v - TOK_IDENT]->sym_define = s;
1328
1329     if (o && !macro_is_equal(o->d, s->d))
1330         tcc_warning("%s redefined", get_tok_str(v, NULL));
1331 }
1332
1333 /* undefined a define symbol. Its name is just set to zero */
1334 ST_FUNC void define_undef(Sym *s)
1335 {
1336     int v = s->v;
1337     if (v >= TOK_IDENT && v < tok_ident)
1338         table_ident[v - TOK_IDENT]->sym_define = NULL;
1339 }
1340
1341 ST_INLN Sym *define_find(int v)
1342 {
1343     v -= TOK_IDENT;
1344     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1345         return NULL;
1346     return table_ident[v]->sym_define;
1347 }
1348
1349 /* free define stack until top reaches 'b' */
1350 ST_FUNC void free_defines(Sym *b)
1351 {
1352     while (define_stack != b) {
1353         Sym *top = define_stack;
1354         define_stack = top->prev;
1355         tok_str_free_str(top->d);
1356         define_undef(top);
1357         sym_free(top);
1358     }
1359 }
1360
1361 /* label lookup */
1362 ST_FUNC Sym *label_find(int v)
1363 {
1364     v -= TOK_IDENT;
1365     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1366         return NULL;
1367     return table_ident[v]->sym_label;
1368 }
1369
1370 ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
1371 {
1372     Sym *s, **ps;
1373     s = sym_push2(ptop, v, 0, 0);
1374     s->r = flags;
1375     ps = &table_ident[v - TOK_IDENT]->sym_label;
1376     if (ptop == &global_label_stack) {
1377         /* modify the top most local identifier, so that
1378            sym_identifier will point to 's' when popped */
1379         while (*ps != NULL)
1380             ps = &(*ps)->prev_tok;
1381     }
1382     s->prev_tok = *ps;
1383     *ps = s;
1384     return s;
1385 }
1386
1387 /* pop labels until element last is reached. Look if any labels are
1388    undefined. Define symbols if '&&label' was used. */
1389 ST_FUNC void label_pop(Sym **ptop, Sym *slast, int keep)
1390 {
1391     Sym *s, *s1;
1392     for(s = *ptop; s != slast; s = s1) {
1393         s1 = s->prev;
1394         if (s->r == LABEL_DECLARED) {
1395             tcc_warning("label '%s' declared but not used", get_tok_str(s->v, NULL));
1396         } else if (s->r == LABEL_FORWARD) {
1397                 tcc_error("label '%s' used but not defined",
1398                       get_tok_str(s->v, NULL));
1399         } else {
1400             if (s->c) {
1401                 /* define corresponding symbol. A size of
1402                    1 is put. */
1403                 put_extern_sym(s, cur_text_section, s->jnext, 1);
1404             }
1405         }
1406         /* remove label */
1407         table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
1408         if (!keep)
1409             sym_free(s);
1410     }
1411     if (!keep)
1412         *ptop = slast;
1413 }
1414
1415 /* fake the nth "#if defined test_..." for tcc -dt -run */
1416 static void maybe_run_test(TCCState *s)
1417 {
1418     const char *p;
1419     if (s->include_stack_ptr != s->include_stack)
1420         return;
1421     p = get_tok_str(tok, NULL);
1422     if (0 != memcmp(p, "test_", 5))
1423         return;
1424     if (0 != --s->run_test)
1425         return;
1426     fprintf(s->ppfp, "\n[%s]\n" + !(s->dflag & 32), p), fflush(s->ppfp);
1427     define_push(tok, MACRO_OBJ, NULL, NULL);
1428 }
1429
1430 /* eval an expression for #if/#elif */
1431 static int expr_preprocess(void)
1432 {
1433     int c, t;
1434     TokenString *str;
1435
1436     str = tok_str_alloc();
1437     pp_expr = 1;
1438     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1439         next(); /* do macro subst */
1440         if (tok == TOK_DEFINED) {
1441             next_nomacro();
1442             t = tok;
1443             if (t == '(')
1444                 next_nomacro();
1445             if (tok < TOK_IDENT)
1446                 expect("identifier");
1447             if (tcc_state->run_test)
1448                 maybe_run_test(tcc_state);
1449             c = define_find(tok) != 0;
1450             if (t == '(') {
1451                 next_nomacro();
1452                 if (tok != ')')
1453                     expect("')'");
1454             }
1455             tok = TOK_CINT;
1456             tokc.i = c;
1457         } else if (tok >= TOK_IDENT) {
1458             /* if undefined macro */
1459             tok = TOK_CINT;
1460             tokc.i = 0;
1461         }
1462         tok_str_add_tok(str);
1463     }
1464     pp_expr = 0;
1465     tok_str_add(str, -1); /* simulate end of file */
1466     tok_str_add(str, 0);
1467     /* now evaluate C constant expression */
1468     begin_macro(str, 1);
1469     next();
1470     c = expr_const();
1471     end_macro();
1472     return c != 0;
1473 }
1474
1475
1476 /* parse after #define */
1477 ST_FUNC void parse_define(void)
1478 {
1479     Sym *s, *first, **ps;
1480     int v, t, varg, is_vaargs, spc;
1481     int saved_parse_flags = parse_flags;
1482
1483     v = tok;
1484     if (v < TOK_IDENT || v == TOK_DEFINED)
1485         tcc_error("invalid macro name '%s'", get_tok_str(tok, &tokc));
1486     /* XXX: should check if same macro (ANSI) */
1487     first = NULL;
1488     t = MACRO_OBJ;
1489     /* We have to parse the whole define as if not in asm mode, in particular
1490        no line comment with '#' must be ignored.  Also for function
1491        macros the argument list must be parsed without '.' being an ID
1492        character.  */
1493     parse_flags = ((parse_flags & ~PARSE_FLAG_ASM_FILE) | PARSE_FLAG_SPACES);
1494     /* '(' must be just after macro definition for MACRO_FUNC */
1495     next_nomacro_spc();
1496     if (tok == '(') {
1497         int dotid = set_idnum('.', 0);
1498         next_nomacro();
1499         ps = &first;
1500         if (tok != ')') for (;;) {
1501             varg = tok;
1502             next_nomacro();
1503             is_vaargs = 0;
1504             if (varg == TOK_DOTS) {
1505                 varg = TOK___VA_ARGS__;
1506                 is_vaargs = 1;
1507             } else if (tok == TOK_DOTS && gnu_ext) {
1508                 is_vaargs = 1;
1509                 next_nomacro();
1510             }
1511             if (varg < TOK_IDENT)
1512         bad_list:
1513                 tcc_error("bad macro parameter list");
1514             s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
1515             *ps = s;
1516             ps = &s->next;
1517             if (tok == ')')
1518                 break;
1519             if (tok != ',' || is_vaargs)
1520                 goto bad_list;
1521             next_nomacro();
1522         }
1523         next_nomacro_spc();
1524         t = MACRO_FUNC;
1525         set_idnum('.', dotid);
1526     }
1527
1528     tokstr_buf.len = 0;
1529     spc = 2;
1530     parse_flags |= PARSE_FLAG_ACCEPT_STRAYS | PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED;
1531     /* The body of a macro definition should be parsed such that identifiers
1532        are parsed like the file mode determines (i.e. with '.' being an
1533        ID character in asm mode).  But '#' should be retained instead of
1534        regarded as line comment leader, so still don't set ASM_FILE
1535        in parse_flags. */
1536     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1537         /* remove spaces around ## and after '#' */
1538         if (TOK_TWOSHARPS == tok) {
1539             if (2 == spc)
1540                 goto bad_twosharp;
1541             if (1 == spc)
1542                 --tokstr_buf.len;
1543             spc = 3;
1544             tok = TOK_PPJOIN;
1545         } else if ('#' == tok) {
1546             spc = 4;
1547         } else if (check_space(tok, &spc)) {
1548             goto skip;
1549         }
1550         tok_str_add2(&tokstr_buf, tok, &tokc);
1551     skip:
1552         next_nomacro_spc();
1553     }
1554
1555     parse_flags = saved_parse_flags;
1556     if (spc == 1)
1557         --tokstr_buf.len; /* remove trailing space */
1558     tok_str_add(&tokstr_buf, 0);
1559     if (3 == spc)
1560 bad_twosharp:
1561         tcc_error("'##' cannot appear at either end of macro");
1562     define_push(v, t, tok_str_dup(&tokstr_buf), first);
1563 }
1564
1565 static CachedInclude *search_cached_include(TCCState *s1, const char *filename, int add)
1566 {
1567     const unsigned char *s;
1568     unsigned int h;
1569     CachedInclude *e;
1570     int i;
1571
1572     h = TOK_HASH_INIT;
1573     s = (unsigned char *) filename;
1574     while (*s) {
1575 #ifdef _WIN32
1576         h = TOK_HASH_FUNC(h, toup(*s));
1577 #else
1578         h = TOK_HASH_FUNC(h, *s);
1579 #endif
1580         s++;
1581     }
1582     h &= (CACHED_INCLUDES_HASH_SIZE - 1);
1583
1584     i = s1->cached_includes_hash[h];
1585     for(;;) {
1586         if (i == 0)
1587             break;
1588         e = s1->cached_includes[i - 1];
1589         if (0 == PATHCMP(e->filename, filename))
1590             return e;
1591         i = e->hash_next;
1592     }
1593     if (!add)
1594         return NULL;
1595
1596     e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
1597     strcpy(e->filename, filename);
1598     e->ifndef_macro = e->once = 0;
1599     dynarray_add(&s1->cached_includes, &s1->nb_cached_includes, e);
1600     /* add in hash table */
1601     e->hash_next = s1->cached_includes_hash[h];
1602     s1->cached_includes_hash[h] = s1->nb_cached_includes;
1603 #ifdef INC_DEBUG
1604     printf("adding cached '%s'\n", filename);
1605 #endif
1606     return e;
1607 }
1608
1609 static void pragma_parse(TCCState *s1)
1610 {
1611     next_nomacro();
1612     if (tok == TOK_push_macro || tok == TOK_pop_macro) {
1613         int t = tok, v;
1614         Sym *s;
1615
1616         if (next(), tok != '(')
1617             goto pragma_err;
1618         if (next(), tok != TOK_STR)
1619             goto pragma_err;
1620         v = tok_alloc(tokc.str.data, tokc.str.size - 1)->tok;
1621         if (next(), tok != ')')
1622             goto pragma_err;
1623         if (t == TOK_push_macro) {
1624             while (NULL == (s = define_find(v)))
1625                 define_push(v, 0, NULL, NULL);
1626             s->type.ref = s; /* set push boundary */
1627         } else {
1628             for (s = define_stack; s; s = s->prev)
1629                 if (s->v == v && s->type.ref == s) {
1630                     s->type.ref = NULL;
1631                     break;
1632                 }
1633         }
1634         if (s)
1635             table_ident[v - TOK_IDENT]->sym_define = s->d ? s : NULL;
1636         else
1637             tcc_warning("unbalanced #pragma pop_macro");
1638         pp_debug_tok = t, pp_debug_symv = v;
1639
1640     } else if (tok == TOK_once) {
1641         search_cached_include(s1, file->filename, 1)->once = pp_once;
1642
1643     } else if (s1->output_type == TCC_OUTPUT_PREPROCESS) {
1644         /* tcc -E: keep pragmas below unchanged */
1645         unget_tok(' ');
1646         unget_tok(TOK_PRAGMA);
1647         unget_tok('#');
1648         unget_tok(TOK_LINEFEED);
1649
1650     } else if (tok == TOK_pack) {
1651         /* This may be:
1652            #pragma pack(1) // set
1653            #pragma pack() // reset to default
1654            #pragma pack(push,1) // push & set
1655            #pragma pack(pop) // restore previous */
1656         next();
1657         skip('(');
1658         if (tok == TOK_ASM_pop) {
1659             next();
1660             if (s1->pack_stack_ptr <= s1->pack_stack) {
1661             stk_error:
1662                 tcc_error("out of pack stack");
1663             }
1664             s1->pack_stack_ptr--;
1665         } else {
1666             int val = 0;
1667             if (tok != ')') {
1668                 if (tok == TOK_ASM_push) {
1669                     next();
1670                     if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
1671                         goto stk_error;
1672                     s1->pack_stack_ptr++;
1673                     skip(',');
1674                 }
1675                 if (tok != TOK_CINT)
1676                     goto pragma_err;
1677                 val = tokc.i;
1678                 if (val < 1 || val > 16 || (val & (val - 1)) != 0)
1679                     goto pragma_err;
1680                 next();
1681             }
1682             *s1->pack_stack_ptr = val;
1683         }
1684         if (tok != ')')
1685             goto pragma_err;
1686
1687     } else if (tok == TOK_comment) {
1688         char *p; int t;
1689         next();
1690         skip('(');
1691         t = tok;
1692         next();
1693         skip(',');
1694         if (tok != TOK_STR)
1695             goto pragma_err;
1696         p = tcc_strdup((char *)tokc.str.data);
1697         next();
1698         if (tok != ')')
1699             goto pragma_err;
1700         if (t == TOK_lib) {
1701             dynarray_add(&s1->pragma_libs, &s1->nb_pragma_libs, p);
1702         } else {
1703             if (t == TOK_option)
1704                 tcc_set_options(s1, p);
1705             tcc_free(p);
1706         }
1707
1708     } else if (s1->warn_unsupported) {
1709         tcc_warning("#pragma %s is ignored", get_tok_str(tok, &tokc));
1710     }
1711     return;
1712
1713 pragma_err:
1714     tcc_error("malformed #pragma directive");
1715     return;
1716 }
1717
1718 /* is_bof is true if first non space token at beginning of file */
1719 ST_FUNC void preprocess(int is_bof)
1720 {
1721     TCCState *s1 = tcc_state;
1722     int i, c, n, saved_parse_flags;
1723     char buf[1024], *q;
1724     Sym *s;
1725
1726     saved_parse_flags = parse_flags;
1727     parse_flags = PARSE_FLAG_PREPROCESS
1728         | PARSE_FLAG_TOK_NUM
1729         | PARSE_FLAG_TOK_STR
1730         | PARSE_FLAG_LINEFEED
1731         | (parse_flags & PARSE_FLAG_ASM_FILE)
1732         ;
1733
1734     next_nomacro();
1735  redo:
1736     switch(tok) {
1737     case TOK_DEFINE:
1738         pp_debug_tok = tok;
1739         next_nomacro();
1740         pp_debug_symv = tok;
1741         parse_define();
1742         break;
1743     case TOK_UNDEF:
1744         pp_debug_tok = tok;
1745         next_nomacro();
1746         pp_debug_symv = tok;
1747         s = define_find(tok);
1748         /* undefine symbol by putting an invalid name */
1749         if (s)
1750             define_undef(s);
1751         break;
1752     case TOK_INCLUDE:
1753     case TOK_INCLUDE_NEXT:
1754         ch = file->buf_ptr[0];
1755         /* XXX: incorrect if comments : use next_nomacro with a special mode */
1756         skip_spaces();
1757         if (ch == '<') {
1758             c = '>';
1759             goto read_name;
1760         } else if (ch == '\"') {
1761             c = ch;
1762         read_name:
1763             inp();
1764             q = buf;
1765             while (ch != c && ch != '\n' && ch != CH_EOF) {
1766                 if ((q - buf) < sizeof(buf) - 1)
1767                     *q++ = ch;
1768                 if (ch == '\\') {
1769                     if (handle_stray_noerror() == 0)
1770                         --q;
1771                 } else
1772                     inp();
1773             }
1774             *q = '\0';
1775             minp();
1776 #if 0
1777             /* eat all spaces and comments after include */
1778             /* XXX: slightly incorrect */
1779             while (ch1 != '\n' && ch1 != CH_EOF)
1780                 inp();
1781 #endif
1782         } else {
1783             int len;
1784             /* computed #include : concatenate everything up to linefeed,
1785                the result must be one of the two accepted forms.
1786                Don't convert pp-tokens to tokens here.  */
1787             parse_flags = (PARSE_FLAG_PREPROCESS
1788                            | PARSE_FLAG_LINEFEED
1789                            | (parse_flags & PARSE_FLAG_ASM_FILE));
1790             next();
1791             buf[0] = '\0';
1792             while (tok != TOK_LINEFEED) {
1793                 pstrcat(buf, sizeof(buf), get_tok_str(tok, &tokc));
1794                 next();
1795             }
1796             len = strlen(buf);
1797             /* check syntax and remove '<>|""' */
1798             if ((len < 2 || ((buf[0] != '"' || buf[len-1] != '"') &&
1799                              (buf[0] != '<' || buf[len-1] != '>'))))
1800                 tcc_error("'#include' expects \"FILENAME\" or <FILENAME>");
1801             c = buf[len-1];
1802             memmove(buf, buf + 1, len - 2);
1803             buf[len - 2] = '\0';
1804         }
1805
1806         if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
1807             tcc_error("#include recursion too deep");
1808         /* push current file on stack */
1809         *s1->include_stack_ptr++ = file;
1810         i = tok == TOK_INCLUDE_NEXT ? file->include_next_index: 0;
1811         n = 2 + s1->nb_include_paths + s1->nb_sysinclude_paths;
1812         for (; i < n; ++i) {
1813             char buf1[sizeof file->filename];
1814             CachedInclude *e;
1815             const char *path;
1816
1817             if (i == 0) {
1818                 /* check absolute include path */
1819                 if (!IS_ABSPATH(buf))
1820                     continue;
1821                 buf1[0] = 0;
1822
1823             } else if (i == 1) {
1824                 /* search in file's dir if "header.h" */
1825                 if (c != '\"')
1826                     continue;
1827                 /* https://savannah.nongnu.org/bugs/index.php?50847 */
1828                 path = file->true_filename;
1829                 pstrncpy(buf1, path, tcc_basename(path) - path);
1830
1831             } else {
1832                 /* search in all the include paths */
1833                 int j = i - 2, k = j - s1->nb_include_paths;
1834                 path = k < 0 ? s1->include_paths[j] : s1->sysinclude_paths[k];
1835                 pstrcpy(buf1, sizeof(buf1), path);
1836                 pstrcat(buf1, sizeof(buf1), "/");
1837             }
1838
1839             pstrcat(buf1, sizeof(buf1), buf);
1840             e = search_cached_include(s1, buf1, 0);
1841             if (e && (define_find(e->ifndef_macro) || e->once == pp_once)) {
1842                 /* no need to parse the include because the 'ifndef macro'
1843                    is defined (or had #pragma once) */
1844 #ifdef INC_DEBUG
1845                 printf("%s: skipping cached %s\n", file->filename, buf1);
1846 #endif
1847                 goto include_done;
1848             }
1849
1850             if (tcc_open(s1, buf1) < 0)
1851                 continue;
1852
1853             file->include_next_index = i + 1;
1854 #ifdef INC_DEBUG
1855             printf("%s: including %s\n", file->prev->filename, file->filename);
1856 #endif
1857             /* update target deps */
1858             if (s1->gen_deps) {
1859                 dynarray_add(&s1->target_deps, &s1->nb_target_deps,
1860                     tcc_strdup(buf1));
1861             }
1862             /* add include file debug info */
1863             if (s1->do_debug)
1864                 put_stabs(tcc_state, file->filename, N_BINCL, 0, 0, 0);
1865             tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
1866             ch = file->buf_ptr[0];
1867             goto the_end;
1868         }
1869         tcc_error("include file '%s' not found", buf);
1870 include_done:
1871         --s1->include_stack_ptr;
1872         break;
1873     case TOK_IFNDEF:
1874         c = 1;
1875         goto do_ifdef;
1876     case TOK_IF:
1877         c = expr_preprocess();
1878         goto do_if;
1879     case TOK_IFDEF:
1880         c = 0;
1881     do_ifdef:
1882         next_nomacro();
1883         if (tok < TOK_IDENT)
1884             tcc_error("invalid argument for '#if%sdef'", c ? "n" : "");
1885         if (is_bof) {
1886             if (c) {
1887 #ifdef INC_DEBUG
1888                 printf("#ifndef %s\n", get_tok_str(tok, NULL));
1889 #endif
1890                 file->ifndef_macro = tok;
1891             }
1892         }
1893         c = (define_find(tok) != 0) ^ c;
1894     do_if:
1895         if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
1896             tcc_error("memory full (ifdef)");
1897         *s1->ifdef_stack_ptr++ = c;
1898         goto test_skip;
1899     case TOK_ELSE:
1900         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1901             tcc_error("#else without matching #if");
1902         if (s1->ifdef_stack_ptr[-1] & 2)
1903             tcc_error("#else after #else");
1904         c = (s1->ifdef_stack_ptr[-1] ^= 3);
1905         goto test_else;
1906     case TOK_ELIF:
1907         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1908             tcc_error("#elif without matching #if");
1909         c = s1->ifdef_stack_ptr[-1];
1910         if (c > 1)
1911             tcc_error("#elif after #else");
1912         /* last #if/#elif expression was true: we skip */
1913         if (c == 1) {
1914             c = 0;
1915         } else {
1916             c = expr_preprocess();
1917             s1->ifdef_stack_ptr[-1] = c;
1918         }
1919     test_else:
1920         if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1)
1921             file->ifndef_macro = 0;
1922     test_skip:
1923         if (!(c & 1)) {
1924             preprocess_skip();
1925             is_bof = 0;
1926             goto redo;
1927         }
1928         break;
1929     case TOK_ENDIF:
1930         if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
1931             tcc_error("#endif without matching #if");
1932         s1->ifdef_stack_ptr--;
1933         /* '#ifndef macro' was at the start of file. Now we check if
1934            an '#endif' is exactly at the end of file */
1935         if (file->ifndef_macro &&
1936             s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
1937             file->ifndef_macro_saved = file->ifndef_macro;
1938             /* need to set to zero to avoid false matches if another
1939                #ifndef at middle of file */
1940             file->ifndef_macro = 0;
1941             while (tok != TOK_LINEFEED)
1942                 next_nomacro();
1943             tok_flags |= TOK_FLAG_ENDIF;
1944             goto the_end;
1945         }
1946         break;
1947     case TOK_PPNUM:
1948         n = strtoul((char*)tokc.str.data, &q, 10);
1949         goto _line_num;
1950     case TOK_LINE:
1951         next();
1952         if (tok != TOK_CINT)
1953     _line_err:
1954             tcc_error("wrong #line format");
1955         n = tokc.i;
1956     _line_num:
1957         next();
1958         if (tok != TOK_LINEFEED) {
1959             if (tok == TOK_STR) {
1960                 if (file->true_filename == file->filename)
1961                     file->true_filename = tcc_strdup(file->filename);
1962                 pstrcpy(file->filename, sizeof(file->filename), (char *)tokc.str.data);
1963             } else if (parse_flags & PARSE_FLAG_ASM_FILE)
1964                 break;
1965             else
1966                 goto _line_err;
1967             --n;
1968         }
1969         if (file->fd > 0)
1970             total_lines += file->line_num - n;
1971         file->line_num = n;
1972         if (s1->do_debug)
1973             put_stabs(tcc_state, file->filename, N_BINCL, 0, 0, 0);
1974         break;
1975     case TOK_ERROR:
1976     case TOK_WARNING:
1977         c = tok;
1978         ch = file->buf_ptr[0];
1979         skip_spaces();
1980         q = buf;
1981         while (ch != '\n' && ch != CH_EOF) {
1982             if ((q - buf) < sizeof(buf) - 1)
1983                 *q++ = ch;
1984             if (ch == '\\') {
1985                 if (handle_stray_noerror() == 0)
1986                     --q;
1987             } else
1988                 inp();
1989         }
1990         *q = '\0';
1991         if (c == TOK_ERROR)
1992             tcc_error("#error %s", buf);
1993         else
1994             tcc_warning("#warning %s", buf);
1995         break;
1996     case TOK_PRAGMA:
1997         pragma_parse(s1);
1998         break;
1999     case TOK_LINEFEED:
2000         goto the_end;
2001     default:
2002         /* ignore gas line comment in an 'S' file. */
2003         if (saved_parse_flags & PARSE_FLAG_ASM_FILE)
2004             goto ignore;
2005         if (tok == '!' && is_bof)
2006             /* '!' is ignored at beginning to allow C scripts. */
2007             goto ignore;
2008         tcc_warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
2009     ignore:
2010         file->buf_ptr = parse_line_comment(file->buf_ptr - 1);
2011         goto the_end;
2012     }
2013     /* ignore other preprocess commands or #! for C scripts */
2014     while (tok != TOK_LINEFEED)
2015         next_nomacro();
2016  the_end:
2017     parse_flags = saved_parse_flags;
2018 }
2019
2020 /* evaluate escape codes in a string. */
2021 static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
2022 {
2023     int c, n;
2024     const uint8_t *p;
2025
2026     p = buf;
2027     for(;;) {
2028         c = *p;
2029         if (c == '\0')
2030             break;
2031         if (c == '\\') {
2032             p++;
2033             /* escape */
2034             c = *p;
2035             switch(c) {
2036             case '0': case '1': case '2': case '3':
2037             case '4': case '5': case '6': case '7':
2038                 /* at most three octal digits */
2039                 n = c - '0';
2040                 p++;
2041                 c = *p;
2042                 if (isoct(c)) {
2043                     n = n * 8 + c - '0';
2044                     p++;
2045                     c = *p;
2046                     if (isoct(c)) {
2047                         n = n * 8 + c - '0';
2048                         p++;
2049                     }
2050                 }
2051                 c = n;
2052                 goto add_char_nonext;
2053             case 'x':
2054             case 'u':
2055             case 'U':
2056                 p++;
2057                 n = 0;
2058                 for(;;) {
2059                     c = *p;
2060                     if (c >= 'a' && c <= 'f')
2061                         c = c - 'a' + 10;
2062                     else if (c >= 'A' && c <= 'F')
2063                         c = c - 'A' + 10;
2064                     else if (isnum(c))
2065                         c = c - '0';
2066                     else
2067                         break;
2068                     n = n * 16 + c;
2069                     p++;
2070                 }
2071                 c = n;
2072                 goto add_char_nonext;
2073             case 'a':
2074                 c = '\a';
2075                 break;
2076             case 'b':
2077                 c = '\b';
2078                 break;
2079             case 'f':
2080                 c = '\f';
2081                 break;
2082             case 'n':
2083                 c = '\n';
2084                 break;
2085             case 'r':
2086                 c = '\r';
2087                 break;
2088             case 't':
2089                 c = '\t';
2090                 break;
2091             case 'v':
2092                 c = '\v';
2093                 break;
2094             case 'e':
2095                 if (!gnu_ext)
2096                     goto invalid_escape;
2097                 c = 27;
2098                 break;
2099             case '\'':
2100             case '\"':
2101             case '\\':
2102             case '?':
2103                 break;
2104             default:
2105             invalid_escape:
2106                 if (c >= '!' && c <= '~')
2107                     tcc_warning("unknown escape sequence: \'\\%c\'", c);
2108                 else
2109                     tcc_warning("unknown escape sequence: \'\\x%x\'", c);
2110                 break;
2111             }
2112         } else if (is_long && c >= 0x80) {
2113             /* assume we are processing UTF-8 sequence */
2114             /* reference: The Unicode Standard, Version 10.0, ch3.9 */
2115
2116             int cont; /* count of continuation bytes */
2117             int skip; /* how many bytes should skip when error occurred */
2118             int i;
2119
2120             /* decode leading byte */
2121             if (c < 0xC2) {
2122                     skip = 1; goto invalid_utf8_sequence;
2123             } else if (c <= 0xDF) {
2124                     cont = 1; n = c & 0x1f;
2125             } else if (c <= 0xEF) {
2126                     cont = 2; n = c & 0xf;
2127             } else if (c <= 0xF4) {
2128                     cont = 3; n = c & 0x7;
2129             } else {
2130                     skip = 1; goto invalid_utf8_sequence;
2131             }
2132
2133             /* decode continuation bytes */
2134             for (i = 1; i <= cont; i++) {
2135                 int l = 0x80, h = 0xBF;
2136
2137                 /* adjust limit for second byte */
2138                 if (i == 1) {
2139                     switch (c) {
2140                     case 0xE0: l = 0xA0; break;
2141                     case 0xED: h = 0x9F; break;
2142                     case 0xF0: l = 0x90; break;
2143                     case 0xF4: h = 0x8F; break;
2144                     }
2145                 }
2146
2147                 if (p[i] < l || p[i] > h) {
2148                     skip = i; goto invalid_utf8_sequence;
2149                 }
2150
2151                 n = (n << 6) | (p[i] & 0x3f);
2152             }
2153
2154             /* advance pointer */
2155             p += 1 + cont;
2156             c = n;
2157             goto add_char_nonext;
2158
2159             /* error handling */
2160         invalid_utf8_sequence:
2161             tcc_warning("ill-formed UTF-8 subsequence starting with: \'\\x%x\'", c);
2162             c = 0xFFFD;
2163             p += skip;
2164             goto add_char_nonext;
2165
2166         }
2167         p++;
2168     add_char_nonext:
2169         if (!is_long)
2170             cstr_ccat(outstr, c);
2171         else {
2172 #ifdef TCC_TARGET_PE
2173             /* store as UTF-16 */
2174             if (c < 0x10000) {
2175                 cstr_wccat(outstr, c);
2176             } else {
2177                 c -= 0x10000;
2178                 cstr_wccat(outstr, (c >> 10) + 0xD800);
2179                 cstr_wccat(outstr, (c & 0x3FF) + 0xDC00);
2180             }
2181 #else
2182             cstr_wccat(outstr, c);
2183 #endif
2184         }
2185     }
2186     /* add a trailing '\0' */
2187     if (!is_long)
2188         cstr_ccat(outstr, '\0');
2189     else
2190         cstr_wccat(outstr, '\0');
2191 }
2192
2193 static void parse_string(const char *s, int len)
2194 {
2195     uint8_t buf[1000], *p = buf;
2196     int is_long, sep;
2197
2198     if ((is_long = *s == 'L'))
2199         ++s, --len;
2200     sep = *s++;
2201     len -= 2;
2202     if (len >= sizeof buf)
2203         p = tcc_malloc(len + 1);
2204     memcpy(p, s, len);
2205     p[len] = 0;
2206
2207     cstr_reset(&tokcstr);
2208     parse_escape_string(&tokcstr, p, is_long);
2209     if (p != buf)
2210         tcc_free(p);
2211
2212     if (sep == '\'') {
2213         int char_size, i, n, c;
2214         /* XXX: make it portable */
2215         if (!is_long)
2216             tok = TOK_CCHAR, char_size = 1;
2217         else
2218             tok = TOK_LCHAR, char_size = sizeof(nwchar_t);
2219         n = tokcstr.size / char_size - 1;
2220         if (n < 1)
2221             tcc_error("empty character constant");
2222         if (n > 1)
2223             tcc_warning("multi-character character constant");
2224         for (c = i = 0; i < n; ++i) {
2225             if (is_long)
2226                 c = ((nwchar_t *)tokcstr.data)[i];
2227             else
2228                 c = (c << 8) | ((char *)tokcstr.data)[i];
2229         }
2230         tokc.i = c;
2231     } else {
2232         tokc.str.size = tokcstr.size;
2233         tokc.str.data = tokcstr.data;
2234         if (!is_long)
2235             tok = TOK_STR;
2236         else
2237             tok = TOK_LSTR;
2238     }
2239 }
2240
2241 /* we use 64 bit numbers */
2242 #define BN_SIZE 2
2243
2244 /* bn = (bn << shift) | or_val */
2245 static void bn_lshift(unsigned int *bn, int shift, int or_val)
2246 {
2247     int i;
2248     unsigned int v;
2249     for(i=0;i<BN_SIZE;i++) {
2250         v = bn[i];
2251         bn[i] = (v << shift) | or_val;
2252         or_val = v >> (32 - shift);
2253     }
2254 }
2255
2256 static void bn_zero(unsigned int *bn)
2257 {
2258     int i;
2259     for(i=0;i<BN_SIZE;i++) {
2260         bn[i] = 0;
2261     }
2262 }
2263
2264 /* parse number in null terminated string 'p' and return it in the
2265    current token */
2266 static void parse_number(const char *p)
2267 {
2268     int b, t, shift, frac_bits, s, exp_val, ch;
2269     char *q;
2270     unsigned int bn[BN_SIZE];
2271     double d;
2272
2273     /* number */
2274     q = token_buf;
2275     ch = *p++;
2276     t = ch;
2277     ch = *p++;
2278     *q++ = t;
2279     b = 10;
2280     if (t == '.') {
2281         goto float_frac_parse;
2282     } else if (t == '0') {
2283         if (ch == 'x' || ch == 'X') {
2284             q--;
2285             ch = *p++;
2286             b = 16;
2287         } else if (tcc_state->tcc_ext && (ch == 'b' || ch == 'B')) {
2288             q--;
2289             ch = *p++;
2290             b = 2;
2291         }
2292     }
2293     /* parse all digits. cannot check octal numbers at this stage
2294        because of floating point constants */
2295     while (1) {
2296         if (ch >= 'a' && ch <= 'f')
2297             t = ch - 'a' + 10;
2298         else if (ch >= 'A' && ch <= 'F')
2299             t = ch - 'A' + 10;
2300         else if (isnum(ch))
2301             t = ch - '0';
2302         else
2303             break;
2304         if (t >= b)
2305             break;
2306         if (q >= token_buf + STRING_MAX_SIZE) {
2307         num_too_long:
2308             tcc_error("number too long");
2309         }
2310         *q++ = ch;
2311         ch = *p++;
2312     }
2313     if (ch == '.' ||
2314         ((ch == 'e' || ch == 'E') && b == 10) ||
2315         ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
2316         if (b != 10) {
2317             /* NOTE: strtox should support that for hexa numbers, but
2318                non ISOC99 libcs do not support it, so we prefer to do
2319                it by hand */
2320             /* hexadecimal or binary floats */
2321             /* XXX: handle overflows */
2322             *q = '\0';
2323             if (b == 16)
2324                 shift = 4;
2325             else
2326                 shift = 1;
2327             bn_zero(bn);
2328             q = token_buf;
2329             while (1) {
2330                 t = *q++;
2331                 if (t == '\0') {
2332                     break;
2333                 } else if (t >= 'a') {
2334                     t = t - 'a' + 10;
2335                 } else if (t >= 'A') {
2336                     t = t - 'A' + 10;
2337                 } else {
2338                     t = t - '0';
2339                 }
2340                 bn_lshift(bn, shift, t);
2341             }
2342             frac_bits = 0;
2343             if (ch == '.') {
2344                 ch = *p++;
2345                 while (1) {
2346                     t = ch;
2347                     if (t >= 'a' && t <= 'f') {
2348                         t = t - 'a' + 10;
2349                     } else if (t >= 'A' && t <= 'F') {
2350                         t = t - 'A' + 10;
2351                     } else if (t >= '0' && t <= '9') {
2352                         t = t - '0';
2353                     } else {
2354                         break;
2355                     }
2356                     if (t >= b)
2357                         tcc_error("invalid digit");
2358                     bn_lshift(bn, shift, t);
2359                     frac_bits += shift;
2360                     ch = *p++;
2361                 }
2362             }
2363             if (ch != 'p' && ch != 'P')
2364                 expect("exponent");
2365             ch = *p++;
2366             s = 1;
2367             exp_val = 0;
2368             if (ch == '+') {
2369                 ch = *p++;
2370             } else if (ch == '-') {
2371                 s = -1;
2372                 ch = *p++;
2373             }
2374             if (ch < '0' || ch > '9')
2375                 expect("exponent digits");
2376             while (ch >= '0' && ch <= '9') {
2377                 exp_val = exp_val * 10 + ch - '0';
2378                 ch = *p++;
2379             }
2380             exp_val = exp_val * s;
2381
2382             /* now we can generate the number */
2383             /* XXX: should patch directly float number */
2384             d = (double)bn[1] * 4294967296.0 + (double)bn[0];
2385             d = ldexp(d, exp_val - frac_bits);
2386             t = toup(ch);
2387             if (t == 'F') {
2388                 ch = *p++;
2389                 tok = TOK_CFLOAT;
2390                 /* float : should handle overflow */
2391                 tokc.f = (float)d;
2392             } else if (t == 'L') {
2393                 ch = *p++;
2394 #ifdef TCC_TARGET_PE
2395                 tok = TOK_CDOUBLE;
2396                 tokc.d = d;
2397 #else
2398                 tok = TOK_CLDOUBLE;
2399                 /* XXX: not large enough */
2400                 tokc.ld = (long double)d;
2401 #endif
2402             } else {
2403                 tok = TOK_CDOUBLE;
2404                 tokc.d = d;
2405             }
2406         } else {
2407             /* decimal floats */
2408             if (ch == '.') {
2409                 if (q >= token_buf + STRING_MAX_SIZE)
2410                     goto num_too_long;
2411                 *q++ = ch;
2412                 ch = *p++;
2413             float_frac_parse:
2414                 while (ch >= '0' && ch <= '9') {
2415                     if (q >= token_buf + STRING_MAX_SIZE)
2416                         goto num_too_long;
2417                     *q++ = ch;
2418                     ch = *p++;
2419                 }
2420             }
2421             if (ch == 'e' || ch == 'E') {
2422                 if (q >= token_buf + STRING_MAX_SIZE)
2423                     goto num_too_long;
2424                 *q++ = ch;
2425                 ch = *p++;
2426                 if (ch == '-' || ch == '+') {
2427                     if (q >= token_buf + STRING_MAX_SIZE)
2428                         goto num_too_long;
2429                     *q++ = ch;
2430                     ch = *p++;
2431                 }
2432                 if (ch < '0' || ch > '9')
2433                     expect("exponent digits");
2434                 while (ch >= '0' && ch <= '9') {
2435                     if (q >= token_buf + STRING_MAX_SIZE)
2436                         goto num_too_long;
2437                     *q++ = ch;
2438                     ch = *p++;
2439                 }
2440             }
2441             *q = '\0';
2442             t = toup(ch);
2443             errno = 0;
2444             if (t == 'F') {
2445                 ch = *p++;
2446                 tok = TOK_CFLOAT;
2447                 tokc.f = strtof(token_buf, NULL);
2448             } else if (t == 'L') {
2449                 ch = *p++;
2450 #ifdef TCC_TARGET_PE
2451                 tok = TOK_CDOUBLE;
2452                 tokc.d = strtod(token_buf, NULL);
2453 #else
2454                 tok = TOK_CLDOUBLE;
2455                 tokc.ld = strtold(token_buf, NULL);
2456 #endif
2457             } else {
2458                 tok = TOK_CDOUBLE;
2459                 tokc.d = strtod(token_buf, NULL);
2460             }
2461         }
2462     } else {
2463         unsigned long long n, n1;
2464         int lcount, ucount, ov = 0;
2465         const char *p1;
2466
2467         /* integer number */
2468         *q = '\0';
2469         q = token_buf;
2470         if (b == 10 && *q == '0') {
2471             b = 8;
2472             q++;
2473         }
2474         n = 0;
2475         while(1) {
2476             t = *q++;
2477             /* no need for checks except for base 10 / 8 errors */
2478             if (t == '\0')
2479                 break;
2480             else if (t >= 'a')
2481                 t = t - 'a' + 10;
2482             else if (t >= 'A')
2483                 t = t - 'A' + 10;
2484             else
2485                 t = t - '0';
2486             if (t >= b)
2487                 tcc_error("invalid digit");
2488             n1 = n;
2489             n = n * b + t;
2490             /* detect overflow */
2491             if (n1 >= 0x1000000000000000ULL && n / b != n1)
2492                 ov = 1;
2493         }
2494
2495         /* Determine the characteristics (unsigned and/or 64bit) the type of
2496            the constant must have according to the constant suffix(es) */
2497         lcount = ucount = 0;
2498         p1 = p;
2499         for(;;) {
2500             t = toup(ch);
2501             if (t == 'L') {
2502                 if (lcount >= 2)
2503                     tcc_error("three 'l's in integer constant");
2504                 if (lcount && *(p - 1) != ch)
2505                     tcc_error("incorrect integer suffix: %s", p1);
2506                 lcount++;
2507                 ch = *p++;
2508             } else if (t == 'U') {
2509                 if (ucount >= 1)
2510                     tcc_error("two 'u's in integer constant");
2511                 ucount++;
2512                 ch = *p++;
2513             } else {
2514                 break;
2515             }
2516         }
2517
2518         /* Determine if it needs 64 bits and/or unsigned in order to fit */
2519         if (ucount == 0 && b == 10) {
2520             if (lcount <= (LONG_SIZE == 4)) {
2521                 if (n >= 0x80000000U)
2522                     lcount = (LONG_SIZE == 4) + 1;
2523             }
2524             if (n >= 0x8000000000000000ULL)
2525                 ov = 1, ucount = 1;
2526         } else {
2527             if (lcount <= (LONG_SIZE == 4)) {
2528                 if (n >= 0x100000000ULL)
2529                     lcount = (LONG_SIZE == 4) + 1;
2530                 else if (n >= 0x80000000U)
2531                     ucount = 1;
2532             }
2533             if (n >= 0x8000000000000000ULL)
2534                 ucount = 1;
2535         }
2536
2537         if (ov)
2538             tcc_warning("integer constant overflow");
2539
2540         tok = TOK_CINT;
2541         if (lcount) {
2542             tok = TOK_CLONG;
2543             if (lcount == 2)
2544                 tok = TOK_CLLONG;
2545         }
2546         if (ucount)
2547             ++tok; /* TOK_CU... */
2548         tokc.i = n;
2549     }
2550     if (ch)
2551         tcc_error("invalid number\n");
2552 }
2553
2554
2555 #define PARSE2(c1, tok1, c2, tok2)              \
2556     case c1:                                    \
2557         PEEKC(c, p);                            \
2558         if (c == c2) {                          \
2559             p++;                                \
2560             tok = tok2;                         \
2561         } else {                                \
2562             tok = tok1;                         \
2563         }                                       \
2564         break;
2565
2566 /* return next token without macro substitution */
2567 static inline void next_nomacro1(void)
2568 {
2569     int t, c, is_long, len;
2570     TokenSym *ts;
2571     uint8_t *p, *p1;
2572     unsigned int h;
2573
2574     p = file->buf_ptr;
2575  redo_no_start:
2576     c = *p;
2577     switch(c) {
2578     case ' ':
2579     case '\t':
2580         tok = c;
2581         p++;
2582         if (parse_flags & PARSE_FLAG_SPACES)
2583             goto keep_tok_flags;
2584         while (isidnum_table[*p - CH_EOF] & IS_SPC)
2585             ++p;
2586         goto redo_no_start;
2587     case '\f':
2588     case '\v':
2589     case '\r':
2590         p++;
2591         goto redo_no_start;
2592     case '\\':
2593         /* first look if it is in fact an end of buffer */
2594         c = handle_stray1(p);
2595         p = file->buf_ptr;
2596         if (c == '\\')
2597             goto parse_simple;
2598         if (c != CH_EOF)
2599             goto redo_no_start;
2600         {
2601             TCCState *s1 = tcc_state;
2602             if ((parse_flags & PARSE_FLAG_LINEFEED)
2603                 && !(tok_flags & TOK_FLAG_EOF)) {
2604                 tok_flags |= TOK_FLAG_EOF;
2605                 tok = TOK_LINEFEED;
2606                 goto keep_tok_flags;
2607             } else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
2608                 tok = TOK_EOF;
2609             } else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
2610                 tcc_error("missing #endif");
2611             } else if (s1->include_stack_ptr == s1->include_stack) {
2612                 /* no include left : end of file. */
2613                 tok = TOK_EOF;
2614             } else {
2615                 tok_flags &= ~TOK_FLAG_EOF;
2616                 /* pop include file */
2617
2618                 /* test if previous '#endif' was after a #ifdef at
2619                    start of file */
2620                 if (tok_flags & TOK_FLAG_ENDIF) {
2621 #ifdef INC_DEBUG
2622                     printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
2623 #endif
2624                     search_cached_include(s1, file->filename, 1)
2625                         ->ifndef_macro = file->ifndef_macro_saved;
2626                     tok_flags &= ~TOK_FLAG_ENDIF;
2627                 }
2628
2629                 /* add end of include file debug info */
2630                 if (tcc_state->do_debug) {
2631                     put_stabd(tcc_state, N_EINCL, 0, 0);
2632                 }
2633                 /* pop include stack */
2634                 tcc_close();
2635                 s1->include_stack_ptr--;
2636                 p = file->buf_ptr;
2637                 if (p == file->buffer)
2638                     tok_flags = TOK_FLAG_BOF|TOK_FLAG_BOL;
2639                 goto redo_no_start;
2640             }
2641         }
2642         break;
2643
2644     case '\n':
2645         file->line_num++;
2646         tok_flags |= TOK_FLAG_BOL;
2647         p++;
2648 maybe_newline:
2649         if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
2650             goto redo_no_start;
2651         tok = TOK_LINEFEED;
2652         goto keep_tok_flags;
2653
2654     case '#':
2655         /* XXX: simplify */
2656         PEEKC(c, p);
2657         if ((tok_flags & TOK_FLAG_BOL) &&
2658             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2659             file->buf_ptr = p;
2660             preprocess(tok_flags & TOK_FLAG_BOF);
2661             p = file->buf_ptr;
2662             goto maybe_newline;
2663         } else {
2664             if (c == '#') {
2665                 p++;
2666                 tok = TOK_TWOSHARPS;
2667             } else {
2668                 if (parse_flags & PARSE_FLAG_ASM_FILE) {
2669                     p = parse_line_comment(p - 1);
2670                     goto redo_no_start;
2671                 } else {
2672                     tok = '#';
2673                 }
2674             }
2675         }
2676         break;
2677
2678     /* dollar is allowed to start identifiers when not parsing asm */
2679     case '$':
2680         if (!(isidnum_table[c - CH_EOF] & IS_ID)
2681          || (parse_flags & PARSE_FLAG_ASM_FILE))
2682             goto parse_simple;
2683
2684     case 'a': case 'b': case 'c': case 'd':
2685     case 'e': case 'f': case 'g': case 'h':
2686     case 'i': case 'j': case 'k': case 'l':
2687     case 'm': case 'n': case 'o': case 'p':
2688     case 'q': case 'r': case 's': case 't':
2689     case 'u': case 'v': case 'w': case 'x':
2690     case 'y': case 'z':
2691     case 'A': case 'B': case 'C': case 'D':
2692     case 'E': case 'F': case 'G': case 'H':
2693     case 'I': case 'J': case 'K':
2694     case 'M': case 'N': case 'O': case 'P':
2695     case 'Q': case 'R': case 'S': case 'T':
2696     case 'U': case 'V': case 'W': case 'X':
2697     case 'Y': case 'Z':
2698     case '_':
2699     parse_ident_fast:
2700         p1 = p;
2701         h = TOK_HASH_INIT;
2702         h = TOK_HASH_FUNC(h, c);
2703         while (c = *++p, isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2704             h = TOK_HASH_FUNC(h, c);
2705         len = p - p1;
2706         if (c != '\\') {
2707             TokenSym **pts;
2708
2709             /* fast case : no stray found, so we have the full token
2710                and we have already hashed it */
2711             h &= (TOK_HASH_SIZE - 1);
2712             pts = &hash_ident[h];
2713             for(;;) {
2714                 ts = *pts;
2715                 if (!ts)
2716                     break;
2717                 if (ts->len == len && !memcmp(ts->str, p1, len))
2718                     goto token_found;
2719                 pts = &(ts->hash_next);
2720             }
2721             ts = tok_alloc_new(pts, (char *) p1, len);
2722         token_found: ;
2723         } else {
2724             /* slower case */
2725             cstr_reset(&tokcstr);
2726             cstr_cat(&tokcstr, (char *) p1, len);
2727             p--;
2728             PEEKC(c, p);
2729         parse_ident_slow:
2730             while (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2731             {
2732                 cstr_ccat(&tokcstr, c);
2733                 PEEKC(c, p);
2734             }
2735             ts = tok_alloc(tokcstr.data, tokcstr.size);
2736         }
2737         tok = ts->tok;
2738         break;
2739     case 'L':
2740         t = p[1];
2741         if (t != '\\' && t != '\'' && t != '\"') {
2742             /* fast case */
2743             goto parse_ident_fast;
2744         } else {
2745             PEEKC(c, p);
2746             if (c == '\'' || c == '\"') {
2747                 is_long = 1;
2748                 goto str_const;
2749             } else {
2750                 cstr_reset(&tokcstr);
2751                 cstr_ccat(&tokcstr, 'L');
2752                 goto parse_ident_slow;
2753             }
2754         }
2755         break;
2756
2757     case '0': case '1': case '2': case '3':
2758     case '4': case '5': case '6': case '7':
2759     case '8': case '9':
2760         t = c;
2761         PEEKC(c, p);
2762         /* after the first digit, accept digits, alpha, '.' or sign if
2763            prefixed by 'eEpP' */
2764     parse_num:
2765         cstr_reset(&tokcstr);
2766         for(;;) {
2767             cstr_ccat(&tokcstr, t);
2768             if (!((isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2769                   || c == '.'
2770                   || ((c == '+' || c == '-')
2771                       && (((t == 'e' || t == 'E')
2772                             && !(parse_flags & PARSE_FLAG_ASM_FILE
2773                                 /* 0xe+1 is 3 tokens in asm */
2774                                 && ((char*)tokcstr.data)[0] == '0'
2775                                 && toup(((char*)tokcstr.data)[1]) == 'X'))
2776                           || t == 'p' || t == 'P'))))
2777                 break;
2778             t = c;
2779             PEEKC(c, p);
2780         }
2781         /* We add a trailing '\0' to ease parsing */
2782         cstr_ccat(&tokcstr, '\0');
2783         tokc.str.size = tokcstr.size;
2784         tokc.str.data = tokcstr.data;
2785         tok = TOK_PPNUM;
2786         break;
2787
2788     case '.':
2789         /* special dot handling because it can also start a number */
2790         PEEKC(c, p);
2791         if (isnum(c)) {
2792             t = '.';
2793             goto parse_num;
2794         } else if ((isidnum_table['.' - CH_EOF] & IS_ID)
2795                    && (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))) {
2796             *--p = c = '.';
2797             goto parse_ident_fast;
2798         } else if (c == '.') {
2799             PEEKC(c, p);
2800             if (c == '.') {
2801                 p++;
2802                 tok = TOK_DOTS;
2803             } else {
2804                 *--p = '.'; /* may underflow into file->unget[] */
2805                 tok = '.';
2806             }
2807         } else {
2808             tok = '.';
2809         }
2810         break;
2811     case '\'':
2812     case '\"':
2813         is_long = 0;
2814     str_const:
2815         cstr_reset(&tokcstr);
2816         if (is_long)
2817             cstr_ccat(&tokcstr, 'L');
2818         cstr_ccat(&tokcstr, c);
2819         p = parse_pp_string(p, c, &tokcstr);
2820         cstr_ccat(&tokcstr, c);
2821         cstr_ccat(&tokcstr, '\0');
2822         tokc.str.size = tokcstr.size;
2823         tokc.str.data = tokcstr.data;
2824         tok = TOK_PPSTR;
2825         break;
2826
2827     case '<':
2828         PEEKC(c, p);
2829         if (c == '=') {
2830             p++;
2831             tok = TOK_LE;
2832         } else if (c == '<') {
2833             PEEKC(c, p);
2834             if (c == '=') {
2835                 p++;
2836                 tok = TOK_A_SHL;
2837             } else {
2838                 tok = TOK_SHL;
2839             }
2840         } else {
2841             tok = TOK_LT;
2842         }
2843         break;
2844     case '>':
2845         PEEKC(c, p);
2846         if (c == '=') {
2847             p++;
2848             tok = TOK_GE;
2849         } else if (c == '>') {
2850             PEEKC(c, p);
2851             if (c == '=') {
2852                 p++;
2853                 tok = TOK_A_SAR;
2854             } else {
2855                 tok = TOK_SAR;
2856             }
2857         } else {
2858             tok = TOK_GT;
2859         }
2860         break;
2861
2862     case '&':
2863         PEEKC(c, p);
2864         if (c == '&') {
2865             p++;
2866             tok = TOK_LAND;
2867         } else if (c == '=') {
2868             p++;
2869             tok = TOK_A_AND;
2870         } else {
2871             tok = '&';
2872         }
2873         break;
2874
2875     case '|':
2876         PEEKC(c, p);
2877         if (c == '|') {
2878             p++;
2879             tok = TOK_LOR;
2880         } else if (c == '=') {
2881             p++;
2882             tok = TOK_A_OR;
2883         } else {
2884             tok = '|';
2885         }
2886         break;
2887
2888     case '+':
2889         PEEKC(c, p);
2890         if (c == '+') {
2891             p++;
2892             tok = TOK_INC;
2893         } else if (c == '=') {
2894             p++;
2895             tok = TOK_A_ADD;
2896         } else {
2897             tok = '+';
2898         }
2899         break;
2900
2901     case '-':
2902         PEEKC(c, p);
2903         if (c == '-') {
2904             p++;
2905             tok = TOK_DEC;
2906         } else if (c == '=') {
2907             p++;
2908             tok = TOK_A_SUB;
2909         } else if (c == '>') {
2910             p++;
2911             tok = TOK_ARROW;
2912         } else {
2913             tok = '-';
2914         }
2915         break;
2916
2917     PARSE2('!', '!', '=', TOK_NE)
2918     PARSE2('=', '=', '=', TOK_EQ)
2919     PARSE2('*', '*', '=', TOK_A_MUL)
2920     PARSE2('%', '%', '=', TOK_A_MOD)
2921     PARSE2('^', '^', '=', TOK_A_XOR)
2922
2923         /* comments or operator */
2924     case '/':
2925         PEEKC(c, p);
2926         if (c == '*') {
2927             p = parse_comment(p);
2928             /* comments replaced by a blank */
2929             tok = ' ';
2930             goto keep_tok_flags;
2931         } else if (c == '/') {
2932             p = parse_line_comment(p);
2933             tok = ' ';
2934             goto keep_tok_flags;
2935         } else if (c == '=') {
2936             p++;
2937             tok = TOK_A_DIV;
2938         } else {
2939             tok = '/';
2940         }
2941         break;
2942
2943         /* simple tokens */
2944     case '(':
2945     case ')':
2946     case '[':
2947     case ']':
2948     case '{':
2949     case '}':
2950     case ',':
2951     case ';':
2952     case ':':
2953     case '?':
2954     case '~':
2955     case '@': /* only used in assembler */
2956     parse_simple:
2957         tok = c;
2958         p++;
2959         break;
2960     default:
2961         if (c >= 0x80 && c <= 0xFF) /* utf8 identifiers */
2962             goto parse_ident_fast;
2963         if (parse_flags & PARSE_FLAG_ASM_FILE)
2964             goto parse_simple;
2965         tcc_error("unrecognized character \\x%02x", c);
2966         break;
2967     }
2968     tok_flags = 0;
2969 keep_tok_flags:
2970     file->buf_ptr = p;
2971 #if defined(PARSE_DEBUG)
2972     printf("token = %d %s\n", tok, get_tok_str(tok, &tokc));
2973 #endif
2974 }
2975
2976 /* return next token without macro substitution. Can read input from
2977    macro_ptr buffer */
2978 static void next_nomacro_spc(void)
2979 {
2980     if (macro_ptr) {
2981     redo:
2982         tok = *macro_ptr;
2983         if (tok) {
2984             TOK_GET(&tok, &macro_ptr, &tokc);
2985             if (tok == TOK_LINENUM) {
2986                 file->line_num = tokc.i;
2987                 goto redo;
2988             }
2989         }
2990     } else {
2991         next_nomacro1();
2992     }
2993     //printf("token = %s\n", get_tok_str(tok, &tokc));
2994 }
2995
2996 ST_FUNC void next_nomacro(void)
2997 {
2998     do {
2999         next_nomacro_spc();
3000     } while (tok < 256 && (isidnum_table[tok - CH_EOF] & IS_SPC));
3001 }
3002
3003
3004 static void macro_subst(
3005     TokenString *tok_str,
3006     Sym **nested_list,
3007     const int *macro_str
3008     );
3009
3010 /* substitute arguments in replacement lists in macro_str by the values in
3011    args (field d) and return allocated string */
3012 static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
3013 {
3014     int t, t0, t1, spc;
3015     const int *st;
3016     Sym *s;
3017     CValue cval;
3018     TokenString str;
3019     CString cstr;
3020
3021     tok_str_new(&str);
3022     t0 = t1 = 0;
3023     while(1) {
3024         TOK_GET(&t, &macro_str, &cval);
3025         if (!t)
3026             break;
3027         if (t == '#') {
3028             /* stringize */
3029             TOK_GET(&t, &macro_str, &cval);
3030             if (!t)
3031                 goto bad_stringy;
3032             s = sym_find2(args, t);
3033             if (s) {
3034                 cstr_new(&cstr);
3035                 cstr_ccat(&cstr, '\"');
3036                 st = s->d;
3037                 spc = 0;
3038                 while (*st >= 0) {
3039                     TOK_GET(&t, &st, &cval);
3040                     if (t != TOK_PLCHLDR
3041                      && t != TOK_NOSUBST
3042                      && 0 == check_space(t, &spc)) {
3043                         const char *s = get_tok_str(t, &cval);
3044                         while (*s) {
3045                             if (t == TOK_PPSTR && *s != '\'')
3046                                 add_char(&cstr, *s);
3047                             else
3048                                 cstr_ccat(&cstr, *s);
3049                             ++s;
3050                         }
3051                     }
3052                 }
3053                 cstr.size -= spc;
3054                 cstr_ccat(&cstr, '\"');
3055                 cstr_ccat(&cstr, '\0');
3056 #ifdef PP_DEBUG
3057                 printf("\nstringize: <%s>\n", (char *)cstr.data);
3058 #endif
3059                 /* add string */
3060                 cval.str.size = cstr.size;
3061                 cval.str.data = cstr.data;
3062                 tok_str_add2(&str, TOK_PPSTR, &cval);
3063                 cstr_free(&cstr);
3064             } else {
3065         bad_stringy:
3066                 expect("macro parameter after '#'");
3067             }
3068         } else if (t >= TOK_IDENT) {
3069             s = sym_find2(args, t);
3070             if (s) {
3071                 int l0 = str.len;
3072                 st = s->d;
3073                 /* if '##' is present before or after, no arg substitution */
3074                 if (*macro_str == TOK_PPJOIN || t1 == TOK_PPJOIN) {
3075                     /* special case for var arg macros : ## eats the ','
3076                        if empty VA_ARGS variable. */
3077                     if (t1 == TOK_PPJOIN && t0 == ',' && gnu_ext && s->type.t) {
3078                         if (*st <= 0) {
3079                             /* suppress ',' '##' */
3080                             str.len -= 2;
3081                         } else {
3082                             /* suppress '##' and add variable */
3083                             str.len--;
3084                             goto add_var;
3085                         }
3086                     }
3087                 } else {
3088             add_var:
3089                     if (!s->next) {
3090                         /* Expand arguments tokens and store them.  In most
3091                            cases we could also re-expand each argument if
3092                            used multiple times, but not if the argument
3093                            contains the __COUNTER__ macro.  */
3094                         TokenString str2;
3095                         sym_push2(&s->next, s->v, s->type.t, 0);
3096                         tok_str_new(&str2);
3097                         macro_subst(&str2, nested_list, st);
3098                         tok_str_add(&str2, 0);
3099                         s->next->d = str2.str;
3100                     }
3101                     st = s->next->d;
3102                 }
3103                 for(;;) {
3104                     int t2;
3105                     TOK_GET(&t2, &st, &cval);
3106                     if (t2 <= 0)
3107                         break;
3108                     tok_str_add2(&str, t2, &cval);
3109                 }
3110                 if (str.len == l0) /* expanded to empty string */
3111                     tok_str_add(&str, TOK_PLCHLDR);
3112             } else {
3113                 tok_str_add(&str, t);
3114             }
3115         } else {
3116             tok_str_add2(&str, t, &cval);
3117         }
3118         t0 = t1, t1 = t;
3119     }
3120     tok_str_add(&str, 0);
3121     return str.str;
3122 }
3123
3124 static char const ab_month_name[12][4] =
3125 {
3126     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3127     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
3128 };
3129
3130 static int paste_tokens(int t1, CValue *v1, int t2, CValue *v2)
3131 {
3132     CString cstr;
3133     int n, ret = 1;
3134
3135     cstr_new(&cstr);
3136     if (t1 != TOK_PLCHLDR)
3137         cstr_cat(&cstr, get_tok_str(t1, v1), -1);
3138     n = cstr.size;
3139     if (t2 != TOK_PLCHLDR)
3140         cstr_cat(&cstr, get_tok_str(t2, v2), -1);
3141     cstr_ccat(&cstr, '\0');
3142
3143     tcc_open_bf(tcc_state, ":paste:", cstr.size);
3144     memcpy(file->buffer, cstr.data, cstr.size);
3145     tok_flags = 0;
3146     for (;;) {
3147         next_nomacro1();
3148         if (0 == *file->buf_ptr)
3149             break;
3150         if (is_space(tok))
3151             continue;
3152         tcc_warning("pasting \"%.*s\" and \"%s\" does not give a valid"
3153             " preprocessing token", n, cstr.data, (char*)cstr.data + n);
3154         ret = 0;
3155         break;
3156     }
3157     tcc_close();
3158     //printf("paste <%s>\n", (char*)cstr.data);
3159     cstr_free(&cstr);
3160     return ret;
3161 }
3162
3163 /* handle the '##' operator. Return NULL if no '##' seen. Otherwise
3164    return the resulting string (which must be freed). */
3165 static inline int *macro_twosharps(const int *ptr0)
3166 {
3167     int t;
3168     CValue cval;
3169     TokenString macro_str1;
3170     int start_of_nosubsts = -1;
3171     const int *ptr;
3172
3173     /* we search the first '##' */
3174     for (ptr = ptr0;;) {
3175         TOK_GET(&t, &ptr, &cval);
3176         if (t == TOK_PPJOIN)
3177             break;
3178         if (t == 0)
3179             return NULL;
3180     }
3181
3182     tok_str_new(&macro_str1);
3183
3184     //tok_print(" $$$", ptr0);
3185     for (ptr = ptr0;;) {
3186         TOK_GET(&t, &ptr, &cval);
3187         if (t == 0)
3188             break;
3189         if (t == TOK_PPJOIN)
3190             continue;
3191         while (*ptr == TOK_PPJOIN) {
3192             int t1; CValue cv1;
3193             /* given 'a##b', remove nosubsts preceding 'a' */
3194             if (start_of_nosubsts >= 0)
3195                 macro_str1.len = start_of_nosubsts;
3196             /* given 'a##b', remove nosubsts preceding 'b' */
3197             while ((t1 = *++ptr) == TOK_NOSUBST)
3198                 ;
3199             if (t1 && t1 != TOK_PPJOIN) {
3200                 TOK_GET(&t1, &ptr, &cv1);
3201                 if (t != TOK_PLCHLDR || t1 != TOK_PLCHLDR) {
3202                     if (paste_tokens(t, &cval, t1, &cv1)) {
3203                         t = tok, cval = tokc;
3204                     } else {
3205                         tok_str_add2(&macro_str1, t, &cval);
3206                         t = t1, cval = cv1;
3207                     }
3208                 }
3209             }
3210         }
3211         if (t == TOK_NOSUBST) {
3212             if (start_of_nosubsts < 0)
3213                 start_of_nosubsts = macro_str1.len;
3214         } else {
3215             start_of_nosubsts = -1;
3216         }
3217         tok_str_add2(&macro_str1, t, &cval);
3218     }
3219     tok_str_add(&macro_str1, 0);
3220     //tok_print(" ###", macro_str1.str);
3221     return macro_str1.str;
3222 }
3223
3224 /* peek or read [ws_str == NULL] next token from function macro call,
3225    walking up macro levels up to the file if necessary */
3226 static int next_argstream(Sym **nested_list, TokenString *ws_str)
3227 {
3228     int t;
3229     const int *p;
3230     Sym *sa;
3231
3232     for (;;) {
3233         if (macro_ptr) {
3234             p = macro_ptr, t = *p;
3235             if (ws_str) {
3236                 while (is_space(t) || TOK_LINEFEED == t || TOK_PLCHLDR == t)
3237                     tok_str_add(ws_str, t), t = *++p;
3238             }
3239             if (t == 0) {
3240                 end_macro();
3241                 /* also, end of scope for nested defined symbol */
3242                 sa = *nested_list;
3243                 while (sa && sa->v == 0)
3244                     sa = sa->prev;
3245                 if (sa)
3246                     sa->v = 0;
3247                 continue;
3248             }
3249         } else {
3250             ch = handle_eob();
3251             if (ws_str) {
3252                 while (is_space(ch) || ch == '\n' || ch == '/') {
3253                     if (ch == '/') {
3254                         int c;
3255                         uint8_t *p = file->buf_ptr;
3256                         PEEKC(c, p);
3257                         if (c == '*') {
3258                             p = parse_comment(p);
3259                             file->buf_ptr = p - 1;
3260                         } else if (c == '/') {
3261                             p = parse_line_comment(p);
3262                             file->buf_ptr = p - 1;
3263                         } else
3264                             break;
3265                         ch = ' ';
3266                     }
3267                     if (ch == '\n')
3268                         file->line_num++;
3269                     if (!(ch == '\f' || ch == '\v' || ch == '\r'))
3270                         tok_str_add(ws_str, ch);
3271                     cinp();
3272                 }
3273             }
3274             t = ch;
3275         }
3276
3277         if (ws_str)
3278             return t;
3279         next_nomacro_spc();
3280         return tok;
3281     }
3282 }
3283
3284 /* do macro substitution of current token with macro 's' and add
3285    result to (tok_str,tok_len). 'nested_list' is the list of all
3286    macros we got inside to avoid recursing. Return non zero if no
3287    substitution needs to be done */
3288 static int macro_subst_tok(
3289     TokenString *tok_str,
3290     Sym **nested_list,
3291     Sym *s)
3292 {
3293     Sym *args, *sa, *sa1;
3294     int parlevel, t, t1, spc;
3295     TokenString str;
3296     char *cstrval;
3297     CValue cval;
3298     CString cstr;
3299     char buf[32];
3300
3301     /* if symbol is a macro, prepare substitution */
3302     /* special macros */
3303     if (tok == TOK___LINE__ || tok == TOK___COUNTER__) {
3304         t = tok == TOK___LINE__ ? file->line_num : pp_counter++;
3305         snprintf(buf, sizeof(buf), "%d", t);
3306         cstrval = buf;
3307         t1 = TOK_PPNUM;
3308         goto add_cstr1;
3309     } else if (tok == TOK___FILE__) {
3310         cstrval = file->filename;
3311         goto add_cstr;
3312     } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
3313         time_t ti;
3314         struct tm *tm;
3315
3316         time(&ti);
3317         tm = localtime(&ti);
3318         if (tok == TOK___DATE__) {
3319             snprintf(buf, sizeof(buf), "%s %2d %d",
3320                      ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
3321         } else {
3322             snprintf(buf, sizeof(buf), "%02d:%02d:%02d",
3323                      tm->tm_hour, tm->tm_min, tm->tm_sec);
3324         }
3325         cstrval = buf;
3326     add_cstr:
3327         t1 = TOK_STR;
3328     add_cstr1:
3329         cstr_new(&cstr);
3330         cstr_cat(&cstr, cstrval, 0);
3331         cval.str.size = cstr.size;
3332         cval.str.data = cstr.data;
3333         tok_str_add2(tok_str, t1, &cval);
3334         cstr_free(&cstr);
3335     } else if (s->d) {
3336         int saved_parse_flags = parse_flags;
3337         int *joined_str = NULL;
3338         int *mstr = s->d;
3339
3340         if (s->type.t == MACRO_FUNC) {
3341             /* whitespace between macro name and argument list */
3342             TokenString ws_str;
3343             tok_str_new(&ws_str);
3344
3345             spc = 0;
3346             parse_flags |= PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED
3347                 | PARSE_FLAG_ACCEPT_STRAYS;
3348
3349             /* get next token from argument stream */
3350             t = next_argstream(nested_list, &ws_str);
3351             if (t != '(') {
3352                 /* not a macro substitution after all, restore the
3353                  * macro token plus all whitespace we've read.
3354                  * whitespace is intentionally not merged to preserve
3355                  * newlines. */
3356                 parse_flags = saved_parse_flags;
3357                 tok_str_add(tok_str, tok);
3358                 if (parse_flags & PARSE_FLAG_SPACES) {
3359                     int i;
3360                     for (i = 0; i < ws_str.len; i++)
3361                         tok_str_add(tok_str, ws_str.str[i]);
3362                 }
3363                 tok_str_free_str(ws_str.str);
3364                 return 0;
3365             } else {
3366                 tok_str_free_str(ws_str.str);
3367             }
3368             do {
3369                 next_nomacro(); /* eat '(' */
3370             } while (tok == TOK_PLCHLDR);
3371
3372             /* argument macro */
3373             args = NULL;
3374             sa = s->next;
3375             /* NOTE: empty args are allowed, except if no args */
3376             for(;;) {
3377                 do {
3378                     next_argstream(nested_list, NULL);
3379                 } while (is_space(tok) || TOK_LINEFEED == tok);
3380     empty_arg:
3381                 /* handle '()' case */
3382                 if (!args && !sa && tok == ')')
3383                     break;
3384                 if (!sa)
3385                     tcc_error("macro '%s' used with too many args",
3386                           get_tok_str(s->v, 0));
3387                 tok_str_new(&str);
3388                 parlevel = spc = 0;
3389                 /* NOTE: non zero sa->t indicates VA_ARGS */
3390                 while ((parlevel > 0 ||
3391                         (tok != ')' &&
3392                          (tok != ',' || sa->type.t)))) {
3393                     if (tok == TOK_EOF || tok == 0)
3394                         break;
3395                     if (tok == '(')
3396                         parlevel++;
3397                     else if (tok == ')')
3398                         parlevel--;
3399                     if (tok == TOK_LINEFEED)
3400                         tok = ' ';
3401                     if (!check_space(tok, &spc))
3402                         tok_str_add2(&str, tok, &tokc);
3403                     next_argstream(nested_list, NULL);
3404                 }
3405                 if (parlevel)
3406                     expect(")");
3407                 str.len -= spc;
3408                 tok_str_add(&str, -1);
3409                 tok_str_add(&str, 0);
3410                 sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
3411                 sa1->d = str.str;
3412                 sa = sa->next;
3413                 if (tok == ')') {
3414                     /* special case for gcc var args: add an empty
3415                        var arg argument if it is omitted */
3416                     if (sa && sa->type.t && gnu_ext)
3417                         goto empty_arg;
3418                     break;
3419                 }
3420                 if (tok != ',')
3421                     expect(",");
3422             }
3423             if (sa) {
3424                 tcc_error("macro '%s' used with too few args",
3425                       get_tok_str(s->v, 0));
3426             }
3427
3428             parse_flags = saved_parse_flags;
3429
3430             /* now subst each arg */
3431             mstr = macro_arg_subst(nested_list, mstr, args);
3432             /* free memory */
3433             sa = args;
3434             while (sa) {
3435                 sa1 = sa->prev;
3436                 tok_str_free_str(sa->d);
3437                 if (sa->next) {
3438                     tok_str_free_str(sa->next->d);
3439                     sym_free(sa->next);
3440                 }
3441                 sym_free(sa);
3442                 sa = sa1;
3443             }
3444         }
3445
3446         sym_push2(nested_list, s->v, 0, 0);
3447         parse_flags = saved_parse_flags;
3448         joined_str = macro_twosharps(mstr);
3449         macro_subst(tok_str, nested_list, joined_str ? joined_str : mstr);
3450
3451         /* pop nested defined symbol */
3452         sa1 = *nested_list;
3453         *nested_list = sa1->prev;
3454         sym_free(sa1);
3455         if (joined_str)
3456             tok_str_free_str(joined_str);
3457         if (mstr != s->d)
3458             tok_str_free_str(mstr);
3459     }
3460     return 0;
3461 }
3462
3463 /* do macro substitution of macro_str and add result to
3464    (tok_str,tok_len). 'nested_list' is the list of all macros we got
3465    inside to avoid recursing. */
3466 static void macro_subst(
3467     TokenString *tok_str,
3468     Sym **nested_list,
3469     const int *macro_str
3470     )
3471 {
3472     Sym *s;
3473     int t, spc, nosubst;
3474     CValue cval;
3475
3476     spc = nosubst = 0;
3477
3478     while (1) {
3479         TOK_GET(&t, &macro_str, &cval);
3480         if (t <= 0)
3481             break;
3482
3483         if (t >= TOK_IDENT && 0 == nosubst) {
3484             s = define_find(t);
3485             if (s == NULL)
3486                 goto no_subst;
3487
3488             /* if nested substitution, do nothing */
3489             if (sym_find2(*nested_list, t)) {
3490                 /* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
3491                 tok_str_add2(tok_str, TOK_NOSUBST, NULL);
3492                 goto no_subst;
3493             }
3494
3495             {
3496                 TokenString *str = tok_str_alloc();
3497                 str->str = (int*)macro_str;
3498                 begin_macro(str, 2);
3499
3500                 tok = t;
3501                 macro_subst_tok(tok_str, nested_list, s);
3502
3503                 if (macro_stack != str) {
3504                     /* already finished by reading function macro arguments */
3505                     break;
3506                 }
3507
3508                 macro_str = macro_ptr;
3509                 end_macro ();
3510             }
3511             if (tok_str->len)
3512                 spc = is_space(t = tok_str->str[tok_str->lastlen]);
3513         } else {
3514             if (t == '\\' && !(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
3515                 tcc_error("stray '\\' in program");
3516 no_subst:
3517             if (!check_space(t, &spc))
3518                 tok_str_add2(tok_str, t, &cval);
3519
3520             if (nosubst) {
3521                 if (nosubst > 1 && (spc || (++nosubst == 3 && t == '(')))
3522                     continue;
3523                 nosubst = 0;
3524             }
3525             if (t == TOK_NOSUBST)
3526                 nosubst = 1;
3527         }
3528         /* GCC supports 'defined' as result of a macro substitution */
3529         if (t == TOK_DEFINED && pp_expr)
3530             nosubst = 2;
3531     }
3532 }
3533
3534 /* return next token with macro substitution */
3535 ST_FUNC void next(void)
3536 {
3537     /* generate line number info */
3538     if (tcc_state->do_debug)
3539         tcc_debug_line(tcc_state);
3540  redo:
3541     if (parse_flags & PARSE_FLAG_SPACES)
3542         next_nomacro_spc();
3543     else
3544         next_nomacro();
3545
3546     if (macro_ptr) {
3547         if (tok == TOK_NOSUBST || tok == TOK_PLCHLDR) {
3548         /* discard preprocessor markers */
3549             goto redo;
3550         } else if (tok == 0) {
3551             /* end of macro or unget token string */
3552             end_macro();
3553             goto redo;
3554         }
3555     } else if (tok >= TOK_IDENT && (parse_flags & PARSE_FLAG_PREPROCESS)) {
3556         Sym *s;
3557         /* if reading from file, try to substitute macros */
3558         s = define_find(tok);
3559         if (s) {
3560             Sym *nested_list = NULL;
3561             tokstr_buf.len = 0;
3562             macro_subst_tok(&tokstr_buf, &nested_list, s);
3563             tok_str_add(&tokstr_buf, 0);
3564             begin_macro(&tokstr_buf, 0);
3565             goto redo;
3566         }
3567     }
3568     /* convert preprocessor tokens into C tokens */
3569     if (tok == TOK_PPNUM) {
3570         if  (parse_flags & PARSE_FLAG_TOK_NUM)
3571             parse_number((char *)tokc.str.data);
3572     } else if (tok == TOK_PPSTR) {
3573         if (parse_flags & PARSE_FLAG_TOK_STR)
3574             parse_string((char *)tokc.str.data, tokc.str.size - 1);
3575     }
3576 }
3577
3578 /* push back current token and set current token to 'last_tok'. Only
3579    identifier case handled for labels. */
3580 ST_INLN void unget_tok(int last_tok)
3581 {
3582
3583     TokenString *str = tok_str_alloc();
3584     tok_str_add2(str, tok, &tokc);
3585     tok_str_add(str, 0);
3586     begin_macro(str, 1);
3587     tok = last_tok;
3588 }
3589
3590 ST_FUNC void preprocess_start(TCCState *s1, int is_asm)
3591 {
3592     CString cstr;
3593
3594     tccpp_new(s1);
3595
3596     s1->include_stack_ptr = s1->include_stack;
3597     s1->ifdef_stack_ptr = s1->ifdef_stack;
3598     file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
3599     pp_expr = 0;
3600     pp_counter = 0;
3601     pp_debug_tok = pp_debug_symv = 0;
3602     pp_once++;
3603     pvtop = vtop = vstack - 1;
3604     memset(vtop, 0, sizeof *vtop);
3605     s1->pack_stack[0] = 0;
3606     s1->pack_stack_ptr = s1->pack_stack;
3607
3608     set_idnum('$', s1->dollars_in_identifiers ? IS_ID : 0);
3609     set_idnum('.', is_asm ? IS_ID : 0);
3610
3611     cstr_new(&cstr);
3612     if (is_asm)
3613         cstr_printf(&cstr, "#define __ASSEMBLER__ 1\n");
3614     cstr_printf(&cstr, "#define __BASE_FILE__ \"%s\"\n", file->filename);
3615     if (s1->cmdline_defs.size)
3616         cstr_cat(&cstr, s1->cmdline_defs.data, s1->cmdline_defs.size);
3617     //printf("%s\n", (char*)s1->cmdline_defs.data);
3618     if (s1->cmdline_incl.size)
3619         cstr_cat(&cstr, s1->cmdline_incl.data, s1->cmdline_incl.size);
3620     *s1->include_stack_ptr++ = file;
3621     tcc_open_bf(s1, "<command line>", cstr.size);
3622     memcpy(file->buffer, cstr.data, cstr.size);
3623     cstr_free(&cstr);
3624
3625     parse_flags = is_asm ? PARSE_FLAG_ASM_FILE : 0;
3626     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
3627 }
3628
3629 /* cleanup from error/setjmp */
3630 ST_FUNC void preprocess_end(TCCState *s1)
3631 {
3632     while (macro_stack)
3633         end_macro();
3634     macro_ptr = NULL;
3635
3636     while (file)
3637         tcc_close();
3638
3639     tccpp_delete(s1);
3640 }
3641
3642 ST_FUNC void tccpp_new(TCCState *s)
3643 {
3644     int i, c;
3645     const char *p, *r;
3646
3647     /* init isid table */
3648     for(i = CH_EOF; i<128; i++)
3649         set_idnum(i,
3650             is_space(i) ? IS_SPC
3651             : isid(i) ? IS_ID
3652             : isnum(i) ? IS_NUM
3653             : 0);
3654
3655     for(i = 128; i<256; i++)
3656         set_idnum(i, IS_ID);
3657
3658     /* init allocators */
3659     tal_new(&toksym_alloc, TOKSYM_TAL_LIMIT, TOKSYM_TAL_SIZE);
3660     tal_new(&tokstr_alloc, TOKSTR_TAL_LIMIT, TOKSTR_TAL_SIZE);
3661
3662     memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
3663     memset(s->cached_includes_hash, 0, sizeof s->cached_includes_hash);
3664
3665     cstr_new(&cstr_buf);
3666     cstr_realloc(&cstr_buf, STRING_MAX_SIZE);
3667     tok_str_new(&tokstr_buf);
3668     tok_str_realloc(&tokstr_buf, TOKSTR_MAX_SIZE);
3669
3670     tok_ident = TOK_IDENT;
3671     p = tcc_keywords;
3672     while (*p) {
3673         r = p;
3674         for(;;) {
3675             c = *r++;
3676             if (c == '\0')
3677                 break;
3678         }
3679         tok_alloc(p, r - p - 1);
3680         p = r;
3681     }
3682
3683     /* we add dummy defines for some special macros to speed up tests
3684        and to have working defined() */
3685     define_push(TOK___LINE__, MACRO_OBJ, NULL, NULL);
3686     define_push(TOK___FILE__, MACRO_OBJ, NULL, NULL);
3687     define_push(TOK___DATE__, MACRO_OBJ, NULL, NULL);
3688     define_push(TOK___TIME__, MACRO_OBJ, NULL, NULL);
3689     define_push(TOK___COUNTER__, MACRO_OBJ, NULL, NULL);
3690 }
3691
3692 ST_FUNC void tccpp_delete(TCCState *s)
3693 {
3694     int i, n;
3695
3696     free_defines(NULL);
3697     dynarray_reset(&s->cached_includes, &s->nb_cached_includes);
3698
3699     /* free tokens */
3700     n = tok_ident - TOK_IDENT;
3701     for(i = 0; i < n; i++)
3702         tal_free(toksym_alloc, table_ident[i]);
3703     tcc_free(table_ident);
3704     table_ident = NULL;
3705
3706     /* free static buffers */
3707     cstr_free(&tokcstr);
3708     cstr_free(&cstr_buf);
3709     cstr_free(&macro_equal_buf);
3710     tok_str_free_str(tokstr_buf.str);
3711
3712     /* free allocators */
3713     tal_delete(toksym_alloc);
3714     toksym_alloc = NULL;
3715     tal_delete(tokstr_alloc);
3716     tokstr_alloc = NULL;
3717 }
3718
3719 /* ------------------------------------------------------------------------- */
3720 /* tcc -E [-P[1]] [-dD} support */
3721
3722 static void tok_print(const char *msg, const int *str)
3723 {
3724     FILE *fp;
3725     int t, s = 0;
3726     CValue cval;
3727
3728     fp = tcc_state->ppfp;
3729     fprintf(fp, "%s", msg);
3730     while (str) {
3731         TOK_GET(&t, &str, &cval);
3732         if (!t)
3733             break;
3734         fprintf(fp, " %s" + s, get_tok_str(t, &cval)), s = 1;
3735     }
3736     fprintf(fp, "\n");
3737 }
3738
3739 static void pp_line(TCCState *s1, BufferedFile *f, int level)
3740 {
3741     int d = f->line_num - f->line_ref;
3742
3743     if (s1->dflag & 4)
3744         return;
3745
3746     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_NONE) {
3747         ;
3748     } else if (level == 0 && f->line_ref && d < 8) {
3749         while (d > 0)
3750             fputs("\n", s1->ppfp), --d;
3751     } else if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_STD) {
3752         fprintf(s1->ppfp, "#line %d \"%s\"\n", f->line_num, f->filename);
3753     } else {
3754         fprintf(s1->ppfp, "# %d \"%s\"%s\n", f->line_num, f->filename,
3755             level > 0 ? " 1" : level < 0 ? " 2" : "");
3756     }
3757     f->line_ref = f->line_num;
3758 }
3759
3760 static void define_print(TCCState *s1, int v)
3761 {
3762     FILE *fp;
3763     Sym *s;
3764
3765     s = define_find(v);
3766     if (NULL == s || NULL == s->d)
3767         return;
3768
3769     fp = s1->ppfp;
3770     fprintf(fp, "#define %s", get_tok_str(v, NULL));
3771     if (s->type.t == MACRO_FUNC) {
3772         Sym *a = s->next;
3773         fprintf(fp,"(");
3774         if (a)
3775             for (;;) {
3776                 fprintf(fp,"%s", get_tok_str(a->v & ~SYM_FIELD, NULL));
3777                 if (!(a = a->next))
3778                     break;
3779                 fprintf(fp,",");
3780             }
3781         fprintf(fp,")");
3782     }
3783     tok_print("", s->d);
3784 }
3785
3786 static void pp_debug_defines(TCCState *s1)
3787 {
3788     int v, t;
3789     const char *vs;
3790     FILE *fp;
3791
3792     t = pp_debug_tok;
3793     if (t == 0)
3794         return;
3795
3796     file->line_num--;
3797     pp_line(s1, file, 0);
3798     file->line_ref = ++file->line_num;
3799
3800     fp = s1->ppfp;
3801     v = pp_debug_symv;
3802     vs = get_tok_str(v, NULL);
3803     if (t == TOK_DEFINE) {
3804         define_print(s1, v);
3805     } else if (t == TOK_UNDEF) {
3806         fprintf(fp, "#undef %s\n", vs);
3807     } else if (t == TOK_push_macro) {
3808         fprintf(fp, "#pragma push_macro(\"%s\")\n", vs);
3809     } else if (t == TOK_pop_macro) {
3810         fprintf(fp, "#pragma pop_macro(\"%s\")\n", vs);
3811     }
3812     pp_debug_tok = 0;
3813 }
3814
3815 static void pp_debug_builtins(TCCState *s1)
3816 {
3817     int v;
3818     for (v = TOK_IDENT; v < tok_ident; ++v)
3819         define_print(s1, v);
3820 }
3821
3822 /* Add a space between tokens a and b to avoid unwanted textual pasting */
3823 static int pp_need_space(int a, int b)
3824 {
3825     return 'E' == a ? '+' == b || '-' == b
3826         : '+' == a ? TOK_INC == b || '+' == b
3827         : '-' == a ? TOK_DEC == b || '-' == b
3828         : a >= TOK_IDENT ? b >= TOK_IDENT
3829         : a == TOK_PPNUM ? b >= TOK_IDENT
3830         : 0;
3831 }
3832
3833 /* maybe hex like 0x1e */
3834 static int pp_check_he0xE(int t, const char *p)
3835 {
3836     if (t == TOK_PPNUM && toup(strchr(p, 0)[-1]) == 'E')
3837         return 'E';
3838     return t;
3839 }
3840
3841 /* Preprocess the current file */
3842 ST_FUNC int tcc_preprocess(TCCState *s1)
3843 {
3844     BufferedFile **iptr;
3845     int token_seen, spcs, level;
3846     const char *p;
3847     char white[400];
3848
3849     parse_flags = PARSE_FLAG_PREPROCESS
3850                 | (parse_flags & PARSE_FLAG_ASM_FILE)
3851                 | PARSE_FLAG_LINEFEED
3852                 | PARSE_FLAG_SPACES
3853                 | PARSE_FLAG_ACCEPT_STRAYS
3854                 ;
3855     /* Credits to Fabrice Bellard's initial revision to demonstrate its
3856        capability to compile and run itself, provided all numbers are
3857        given as decimals. tcc -E -P10 will do. */
3858     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_P10)
3859         parse_flags |= PARSE_FLAG_TOK_NUM, s1->Pflag = 1;
3860
3861 #ifdef PP_BENCH
3862     /* for PP benchmarks */
3863     do next(); while (tok != TOK_EOF);
3864     return 0;
3865 #endif
3866
3867     if (s1->dflag & 1) {
3868         pp_debug_builtins(s1);
3869         s1->dflag &= ~1;
3870     }
3871
3872     token_seen = TOK_LINEFEED, spcs = 0;
3873     pp_line(s1, file, 0);
3874     for (;;) {
3875         iptr = s1->include_stack_ptr;
3876         next();
3877         if (tok == TOK_EOF)
3878             break;
3879
3880         level = s1->include_stack_ptr - iptr;
3881         if (level) {
3882             if (level > 0)
3883                 pp_line(s1, *iptr, 0);
3884             pp_line(s1, file, level);
3885         }
3886         if (s1->dflag & 7) {
3887             pp_debug_defines(s1);
3888             if (s1->dflag & 4)
3889                 continue;
3890         }
3891
3892         if (is_space(tok)) {
3893             if (spcs < sizeof white - 1)
3894                 white[spcs++] = tok;
3895             continue;
3896         } else if (tok == TOK_LINEFEED) {
3897             spcs = 0;
3898             if (token_seen == TOK_LINEFEED)
3899                 continue;
3900             ++file->line_ref;
3901         } else if (token_seen == TOK_LINEFEED) {
3902             pp_line(s1, file, 0);
3903         } else if (spcs == 0 && pp_need_space(token_seen, tok)) {
3904             white[spcs++] = ' ';
3905         }
3906
3907         white[spcs] = 0, fputs(white, s1->ppfp), spcs = 0;
3908         fputs(p = get_tok_str(tok, &tokc), s1->ppfp);
3909         token_seen = pp_check_he0xE(tok, p);
3910     }
3911     return 0;
3912 }
3913
3914 /* ------------------------------------------------------------------------- */