tccpp.c

   1 /*
   2  *  TCC - Tiny C Compiler
   3  *
   4  *  Copyright (c) 2001-2004 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #define USING_GLOBALS
  22 #include "tcc.h"
  23
  24 /* #define to 1 to enable (see parse_pp_string()) */
  25 #define ACCEPT_LF_IN_STRINGS 0
  26
  27 /********************************************************/
  28 /* global variables */
  29
  30 ST_DATA int tok_flags;
  31 ST_DATA int parse_flags;
  32
  33 ST_DATA struct BufferedFile *file;
  34 ST_DATA int ch, tok;
  35 ST_DATA CValue tokc;
  36 ST_DATA const int *macro_ptr;
  37 ST_DATA CString tokcstr; /* current parsed string, if any */
  38
  39 /* display benchmark infos */
  40 ST_DATA int tok_ident;
  41 ST_DATA TokenSym **table_ident;
  42
  43 /* ------------------------------------------------------------------------- */
  44
  45 static TokenSym *hash_ident[TOK_HASH_SIZE];
  46 static char token_buf[STRING_MAX_SIZE + 1];
  47 static CString cstr_buf;
  48 static CString macro_equal_buf;
  49 static TokenString tokstr_buf;
  50 static unsigned char isidnum_table[256 - CH_EOF];
  51 static int pp_debug_tok, pp_debug_symv;
  52 static int pp_once;
  53 static int pp_expr;
  54 static int pp_counter;
  55 static void tok_print(const char *msg, const int *str);
  56
  57 static struct TinyAlloc *toksym_alloc;
  58 static struct TinyAlloc *tokstr_alloc;
  59
  60 static TokenString *macro_stack;
  61
  62 static const char tcc_keywords[] =
  63 #define DEF(id, str) str "\0"
  64 #include "tcctok.h"
  65 #undef DEF
  66 ;
  67
  68 /* WARNING: the content of this string encodes token numbers */
  69 static const unsigned char tok_two_chars[] =
  70 /* outdated -- gr
  71     "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
  72     "-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
  73 */{
  74     '<','=', TOK_LE,
  75     '>','=', TOK_GE,
  76     '!','=', TOK_NE,
  77     '&','&', TOK_LAND,
  78     '|','|', TOK_LOR,
  79     '+','+', TOK_INC,
  80     '-','-', TOK_DEC,
  81     '=','=', TOK_EQ,
  82     '<','<', TOK_SHL,
  83     '>','>', TOK_SAR,
  84     '+','=', TOK_A_ADD,
  85     '-','=', TOK_A_SUB,
  86     '*','=', TOK_A_MUL,
  87     '/','=', TOK_A_DIV,
  88     '%','=', TOK_A_MOD,
  89     '&','=', TOK_A_AND,
  90     '^','=', TOK_A_XOR,
  91     '|','=', TOK_A_OR,
  92     '-','>', TOK_ARROW,
  93     '.','.', TOK_TWODOTS,
  94     '#','#', TOK_TWOSHARPS,
  95     '#','#', TOK_PPJOIN,
  96     0
  97 };
  98
  99 static void next_nomacro(void);
 100
 101 ST_FUNC void skip(int c)
 102 {
 103     if (tok != c)
 104         tcc_error("'%c' expected (got \"%s\")", c, get_tok_str(tok, &tokc));
 105     next();
 106 }
 107
 108 ST_FUNC void expect(const char *msg)
 109 {
 110     tcc_error("%s expected", msg);
 111 }
 112
 113 /* ------------------------------------------------------------------------- */
 114 /* Custom allocator for tiny objects */
 115
 116 #define USE_TAL
 117
 118 #ifndef USE_TAL
 119 #define tal_free(al, p) tcc_free(p)
 120 #define tal_realloc(al, p, size) tcc_realloc(p, size)
 121 #define tal_new(a,b,c)
 122 #define tal_delete(a)
 123 #else
 124 #if !defined(MEM_DEBUG)
 125 #define tal_free(al, p) tal_free_impl(al, p)
 126 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size)
 127 #define TAL_DEBUG_PARAMS
 128 #else
 129 #define TAL_DEBUG 1
 130 //#define TAL_INFO 1 /* collect and dump allocators stats */
 131 #define tal_free(al, p) tal_free_impl(al, p, __FILE__, __LINE__)
 132 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size, __FILE__, __LINE__)
 133 #define TAL_DEBUG_PARAMS , const char *file, int line
 134 #define TAL_DEBUG_FILE_LEN 40
 135 #endif
 136
 137 #define TOKSYM_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenSym in table_ident */
 138 #define TOKSTR_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenString instances */
 139 #define CSTR_TAL_SIZE       (256 * 1024) /* allocator for tiny CString instances */
 140 #define TOKSYM_TAL_LIMIT    256 /* prefer unique limits to distinguish allocators debug msgs */
 141 #define TOKSTR_TAL_LIMIT    128 /* 32 * sizeof(int) */
 142 #define CSTR_TAL_LIMIT      1024
 143
 144 typedef struct TinyAlloc {
 145     unsigned  limit;
 146     unsigned  size;
 147     uint8_t *buffer;
 148     uint8_t *p;
 149     unsigned  nb_allocs;
 150     struct TinyAlloc *next, *top;
 151 #ifdef TAL_INFO
 152     unsigned  nb_peak;
 153     unsigned  nb_total;
 154     unsigned  nb_missed;
 155     uint8_t *peak_p;
 156 #endif
 157 } TinyAlloc;
 158
 159 typedef struct tal_header_t {
 160     unsigned  size;
 161 #ifdef TAL_DEBUG
 162     int     line_num; /* negative line_num used for double free check */
 163     char    file_name[TAL_DEBUG_FILE_LEN + 1];
 164 #endif
 165 } tal_header_t;
 166
 167 /* ------------------------------------------------------------------------- */
 168
 169 static TinyAlloc *tal_new(TinyAlloc **pal, unsigned limit, unsigned size)
 170 {
 171     TinyAlloc *al = tcc_mallocz(sizeof(TinyAlloc));
 172     al->p = al->buffer = tcc_malloc(size);
 173     al->limit = limit;
 174     al->size = size;
 175     if (pal) *pal = al;
 176     return al;
 177 }
 178
 179 static void tal_delete(TinyAlloc *al)
 180 {
 181     TinyAlloc *next;
 182
 183 tail_call:
 184     if (!al)
 185         return;
 186 #ifdef TAL_INFO
 187     fprintf(stderr, "limit=%5d, size=%5g MB, nb_peak=%6d, nb_total=%8d, nb_missed=%6d, usage=%5.1f%%\n",
 188             al->limit, al->size / 1024.0 / 1024.0, al->nb_peak, al->nb_total, al->nb_missed,
 189             (al->peak_p - al->buffer) * 100.0 / al->size);
 190 #endif
 191 #ifdef TAL_DEBUG
 192     if (al->nb_allocs > 0) {
 193         uint8_t *p;
 194         fprintf(stderr, "TAL_DEBUG: memory leak %d chunk(s) (limit= %d)\n",
 195                 al->nb_allocs, al->limit);
 196         p = al->buffer;
 197         while (p < al->p) {
 198             tal_header_t *header = (tal_header_t *)p;
 199             if (header->line_num > 0) {
 200                 fprintf(stderr, "%s:%d: chunk of %d bytes leaked\n",
 201                         header->file_name, header->line_num, header->size);
 202             }
 203             p += header->size + sizeof(tal_header_t);
 204         }
 205 #if MEM_DEBUG-0 == 2
 206         exit(2);
 207 #endif
 208     }
 209 #endif
 210     next = al->next;
 211     tcc_free(al->buffer);
 212     tcc_free(al);
 213     al = next;
 214     goto tail_call;
 215 }
 216
 217 static void tal_free_impl(TinyAlloc *al, void *p TAL_DEBUG_PARAMS)
 218 {
 219     if (!p)
 220         return;
 221 tail_call:
 222     if (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size) {
 223 #ifdef TAL_DEBUG
 224         tal_header_t *header = (((tal_header_t *)p) - 1);
 225         if (header->line_num < 0) {
 226             fprintf(stderr, "%s:%d: TAL_DEBUG: double frees chunk from\n",
 227                     file, line);
 228             fprintf(stderr, "%s:%d: %d bytes\n",
 229                     header->file_name, (int)-header->line_num, (int)header->size);
 230         } else
 231             header->line_num = -header->line_num;
 232 #endif
 233         al->nb_allocs--;
 234         if (!al->nb_allocs)
 235             al->p = al->buffer;
 236     } else if (al->next) {
 237         al = al->next;
 238         goto tail_call;
 239     }
 240     else
 241         tcc_free(p);
 242 }
 243
 244 static void *tal_realloc_impl(TinyAlloc **pal, void *p, unsigned size TAL_DEBUG_PARAMS)
 245 {
 246     tal_header_t *header;
 247     void *ret;
 248     int is_own;
 249     unsigned adj_size = (size + 3) & -4;
 250     TinyAlloc *al = *pal;
 251
 252 tail_call:
 253     is_own = (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size);
 254     if ((!p || is_own) && size <= al->limit) {
 255         if (al->p - al->buffer + adj_size + sizeof(tal_header_t) < al->size) {
 256             header = (tal_header_t *)al->p;
 257             header->size = adj_size;
 258 #ifdef TAL_DEBUG
 259             { int ofs = strlen(file) - TAL_DEBUG_FILE_LEN;
 260             strncpy(header->file_name, file + (ofs > 0 ? ofs : 0), TAL_DEBUG_FILE_LEN);
 261             header->file_name[TAL_DEBUG_FILE_LEN] = 0;
 262             header->line_num = line; }
 263 #endif
 264             ret = al->p + sizeof(tal_header_t);
 265             al->p += adj_size + sizeof(tal_header_t);
 266             if (is_own) {
 267                 header = (((tal_header_t *)p) - 1);
 268                 if (p) memcpy(ret, p, header->size);
 269 #ifdef TAL_DEBUG
 270                 header->line_num = -header->line_num;
 271 #endif
 272             } else {
 273                 al->nb_allocs++;
 274             }
 275 #ifdef TAL_INFO
 276             if (al->nb_peak < al->nb_allocs)
 277                 al->nb_peak = al->nb_allocs;
 278             if (al->peak_p < al->p)
 279                 al->peak_p = al->p;
 280             al->nb_total++;
 281 #endif
 282             return ret;
 283         } else if (is_own) {
 284             al->nb_allocs--;
 285             ret = tal_realloc(*pal, 0, size);
 286             header = (((tal_header_t *)p) - 1);
 287             if (p) memcpy(ret, p, header->size);
 288 #ifdef TAL_DEBUG
 289             header->line_num = -header->line_num;
 290 #endif
 291             return ret;
 292         }
 293         if (al->next) {
 294             al = al->next;
 295         } else {
 296             TinyAlloc *bottom = al, *next = al->top ? al->top : al;
 297
 298             al = tal_new(pal, next->limit, next->size * 2);
 299             al->next = next;
 300             bottom->top = al;
 301         }
 302         goto tail_call;
 303     }
 304     if (is_own) {
 305         al->nb_allocs--;
 306         ret = tcc_malloc(size);
 307         header = (((tal_header_t *)p) - 1);
 308         if (p) memcpy(ret, p, header->size);
 309 #ifdef TAL_DEBUG
 310         header->line_num = -header->line_num;
 311 #endif
 312     } else if (al->next) {
 313         al = al->next;
 314         goto tail_call;
 315     } else
 316         ret = tcc_realloc(p, size);
 317 #ifdef TAL_INFO
 318     al->nb_missed++;
 319 #endif
 320     return ret;
 321 }
 322
 323 #endif /* USE_TAL */
 324
 325 /* ------------------------------------------------------------------------- */
 326 /* CString handling */
 327 static void cstr_realloc(CString *cstr, int new_size)
 328 {
 329     int size;
 330
 331     size = cstr->size_allocated;
 332     if (size < 8)
 333         size = 8; /* no need to allocate a too small first string */
 334     while (size < new_size)
 335         size = size * 2;
 336     cstr->data = tcc_realloc(cstr->data, size);
 337     cstr->size_allocated = size;
 338 }
 339
 340 /* add a byte */
 341 ST_INLN void cstr_ccat(CString *cstr, int ch)
 342 {
 343     int size;
 344     size = cstr->size + 1;
 345     if (size > cstr->size_allocated)
 346         cstr_realloc(cstr, size);
 347     ((unsigned char *)cstr->data)[size - 1] = ch;
 348     cstr->size = size;
 349 }
 350
 351 ST_INLN char *unicode_to_utf8 (char *b, uint32_t Uc)
 352 {
 353     if (Uc<0x80) *b++=Uc;
 354     else if (Uc<0x800) *b++=192+Uc/64, *b++=128+Uc%64;
 355     else if (Uc-0xd800u<0x800) goto error;
 356     else if (Uc<0x10000) *b++=224+Uc/4096, *b++=128+Uc/64%64, *b++=128+Uc%64;
 357     else if (Uc<0x110000) *b++=240+Uc/262144, *b++=128+Uc/4096%64, *b++=128+Uc/64%64, *b++=128+Uc%64;
 358     else error: tcc_error("0x%x is not a valid universal character", Uc);
 359     return b;
 360 }
 361
 362 /* add a unicode character expanded into utf8 */
 363 ST_INLN void cstr_u8cat(CString *cstr, int ch)
 364 {
 365     char buf[4], *e;
 366     e = unicode_to_utf8(buf, (uint32_t)ch);
 367     cstr_cat(cstr, buf, e - buf);
 368 }
 369
 370 ST_FUNC void cstr_cat(CString *cstr, const char *str, int len)
 371 {
 372     int size;
 373     if (len <= 0)
 374         len = strlen(str) + 1 + len;
 375     size = cstr->size + len;
 376     if (size > cstr->size_allocated)
 377         cstr_realloc(cstr, size);
 378     memmove(((unsigned char *)cstr->data) + cstr->size, str, len);
 379     cstr->size = size;
 380 }
 381
 382 /* add a wide char */
 383 ST_FUNC void cstr_wccat(CString *cstr, int ch)
 384 {
 385     int size;
 386     size = cstr->size + sizeof(nwchar_t);
 387     if (size > cstr->size_allocated)
 388         cstr_realloc(cstr, size);
 389     *(nwchar_t *)(((unsigned char *)cstr->data) + size - sizeof(nwchar_t)) = ch;
 390     cstr->size = size;
 391 }
 392
 393 ST_FUNC void cstr_new(CString *cstr)
 394 {
 395     memset(cstr, 0, sizeof(CString));
 396 }
 397
 398 /* free string and reset it to NULL */
 399 ST_FUNC void cstr_free(CString *cstr)
 400 {
 401     tcc_free(cstr->data);
 402     cstr_new(cstr);
 403 }
 404
 405 /* reset string to empty */
 406 ST_FUNC void cstr_reset(CString *cstr)
 407 {
 408     cstr->size = 0;
 409 }
 410
 411 ST_FUNC int cstr_vprintf(CString *cstr, const char *fmt, va_list ap)
 412 {
 413     va_list v;
 414     int len, size = 80;
 415     for (;;) {
 416         size += cstr->size;
 417         if (size > cstr->size_allocated)
 418             cstr_realloc(cstr, size);
 419         size = cstr->size_allocated - cstr->size;
 420         va_copy(v, ap);
 421         len = vsnprintf((char*)cstr->data + cstr->size, size, fmt, v);
 422         va_end(v);
 423         if (len > 0 && len < size)
 424             break;
 425         size *= 2;
 426     }
 427     cstr->size += len;
 428     return len;
 429 }
 430
 431 ST_FUNC int cstr_printf(CString *cstr, const char *fmt, ...)
 432 {
 433     va_list ap; int len;
 434     va_start(ap, fmt);
 435     len = cstr_vprintf(cstr, fmt, ap);
 436     va_end(ap);
 437     return len;
 438 }
 439
 440 /* XXX: unicode ? */
 441 static void add_char(CString *cstr, int c)
 442 {
 443     if (c == '\'' || c == '\"' || c == '\\') {
 444         /* XXX: could be more precise if char or string */
 445         cstr_ccat(cstr, '\\');
 446     }
 447     if (c >= 32 && c <= 126) {
 448         cstr_ccat(cstr, c);
 449     } else {
 450         cstr_ccat(cstr, '\\');
 451         if (c == '\n') {
 452             cstr_ccat(cstr, 'n');
 453         } else {
 454             cstr_ccat(cstr, '0' + ((c >> 6) & 7));
 455             cstr_ccat(cstr, '0' + ((c >> 3) & 7));
 456             cstr_ccat(cstr, '0' + (c & 7));
 457         }
 458     }
 459 }
 460
 461 /* ------------------------------------------------------------------------- */
 462 /* allocate a new token */
 463 static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
 464 {
 465     TokenSym *ts, **ptable;
 466     int i;
 467
 468     if (tok_ident >= SYM_FIRST_ANOM)
 469         tcc_error("memory full (symbols)");
 470
 471     /* expand token table if needed */
 472     i = tok_ident - TOK_IDENT;
 473     if ((i % TOK_ALLOC_INCR) == 0) {
 474         ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
 475         table_ident = ptable;
 476     }
 477
 478     ts = tal_realloc(toksym_alloc, 0, sizeof(TokenSym) + len);
 479     table_ident[i] = ts;
 480     ts->tok = tok_ident++;
 481     ts->sym_define = NULL;
 482     ts->sym_label = NULL;
 483     ts->sym_struct = NULL;
 484     ts->sym_identifier = NULL;
 485     ts->len = len;
 486     ts->hash_next = NULL;
 487     memcpy(ts->str, str, len);
 488     ts->str[len] = '\0';
 489     *pts = ts;
 490     return ts;
 491 }
 492
 493 #define TOK_HASH_INIT 1
 494 #define TOK_HASH_FUNC(h, c) ((h) + ((h) << 5) + ((h) >> 27) + (c))
 495
 496
 497 /* find a token and add it if not found */
 498 ST_FUNC TokenSym *tok_alloc(const char *str, int len)
 499 {
 500     TokenSym *ts, **pts;
 501     int i;
 502     unsigned int h;
 503
 504     h = TOK_HASH_INIT;
 505     for(i=0;i<len;i++)
 506         h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
 507     h &= (TOK_HASH_SIZE - 1);
 508
 509     pts = &hash_ident[h];
 510     for(;;) {
 511         ts = *pts;
 512         if (!ts)
 513             break;
 514         if (ts->len == len && !memcmp(ts->str, str, len))
 515             return ts;
 516         pts = &(ts->hash_next);
 517     }
 518     return tok_alloc_new(pts, str, len);
 519 }
 520
 521 ST_FUNC int tok_alloc_const(const char *str)
 522 {
 523     return tok_alloc(str, strlen(str))->tok;
 524 }
 525
 526
 527 /* XXX: buffer overflow */
 528 /* XXX: float tokens */
 529 ST_FUNC const char *get_tok_str(int v, CValue *cv)
 530 {
 531     char *p;
 532     int i, len;
 533
 534     cstr_reset(&cstr_buf);
 535     p = cstr_buf.data;
 536
 537     switch(v) {
 538     case TOK_CINT:
 539     case TOK_CUINT:
 540     case TOK_CLONG:
 541     case TOK_CULONG:
 542     case TOK_CLLONG:
 543     case TOK_CULLONG:
 544         /* XXX: not quite exact, but only useful for testing  */
 545 #ifdef _WIN32
 546         sprintf(p, "%u", (unsigned)cv->i);
 547 #else
 548         sprintf(p, "%llu", (unsigned long long)cv->i);
 549 #endif
 550         break;
 551     case TOK_LCHAR:
 552         cstr_ccat(&cstr_buf, 'L');
 553     case TOK_CCHAR:
 554         cstr_ccat(&cstr_buf, '\'');
 555         add_char(&cstr_buf, cv->i);
 556         cstr_ccat(&cstr_buf, '\'');
 557         cstr_ccat(&cstr_buf, '\0');
 558         break;
 559     case TOK_PPNUM:
 560     case TOK_PPSTR:
 561         return (char*)cv->str.data;
 562     case TOK_LSTR:
 563         cstr_ccat(&cstr_buf, 'L');
 564     case TOK_STR:
 565         cstr_ccat(&cstr_buf, '\"');
 566         if (v == TOK_STR) {
 567             len = cv->str.size - 1;
 568             for(i=0;i<len;i++)
 569                 add_char(&cstr_buf, ((unsigned char *)cv->str.data)[i]);
 570         } else {
 571             len = (cv->str.size / sizeof(nwchar_t)) - 1;
 572             for(i=0;i<len;i++)
 573                 add_char(&cstr_buf, ((nwchar_t *)cv->str.data)[i]);
 574         }
 575         cstr_ccat(&cstr_buf, '\"');
 576         cstr_ccat(&cstr_buf, '\0');
 577         break;
 578
 579     case TOK_CFLOAT:
 580         cstr_cat(&cstr_buf, "<float>", 0);
 581         break;
 582     case TOK_CDOUBLE:
 583         cstr_cat(&cstr_buf, "<double>", 0);
 584         break;
 585     case TOK_CLDOUBLE:
 586         cstr_cat(&cstr_buf, "<long double>", 0);
 587         break;
 588     case TOK_LINENUM:
 589         cstr_cat(&cstr_buf, "<linenumber>", 0);
 590         break;
 591
 592     /* above tokens have value, the ones below don't */
 593     case TOK_LT:
 594         v = '<';
 595         goto addv;
 596     case TOK_GT:
 597         v = '>';
 598         goto addv;
 599     case TOK_DOTS:
 600         return strcpy(p, "...");
 601     case TOK_A_SHL:
 602         return strcpy(p, "<<=");
 603     case TOK_A_SAR:
 604         return strcpy(p, ">>=");
 605     case TOK_EOF:
 606         return strcpy(p, "<eof>");
 607     default:
 608         if (v < TOK_IDENT) {
 609             /* search in two bytes table */
 610             const unsigned char *q = tok_two_chars;
 611             while (*q) {
 612                 if (q[2] == v) {
 613                     *p++ = q[0];
 614                     *p++ = q[1];
 615                     *p = '\0';
 616                     return cstr_buf.data;
 617                 }
 618                 q += 3;
 619             }
 620         if (v >= 127) {
 621             sprintf(cstr_buf.data, "<%02x>", v);
 622             return cstr_buf.data;
 623         }
 624         addv:
 625             *p++ = v;
 626     case 0: /* nameless anonymous symbol */
 627             *p = '\0';
 628         } else if (v < tok_ident) {
 629             return table_ident[v - TOK_IDENT]->str;
 630         } else if (v >= SYM_FIRST_ANOM) {
 631             /* special name for anonymous symbol */
 632             sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
 633         } else {
 634             /* should never happen */
 635             return NULL;
 636         }
 637         break;
 638     }
 639     return cstr_buf.data;
 640 }
 641
 642 /* return the current character, handling end of block if necessary
 643    (but not stray) */
 644 static int handle_eob(void)
 645 {
 646     BufferedFile *bf = file;
 647     int len;
 648
 649     /* only tries to read if really end of buffer */
 650     if (bf->buf_ptr >= bf->buf_end) {
 651         if (bf->fd >= 0) {
 652 #if defined(PARSE_DEBUG)
 653             len = 1;
 654 #else
 655             len = IO_BUF_SIZE;
 656 #endif
 657             len = read(bf->fd, bf->buffer, len);
 658             if (len < 0)
 659                 len = 0;
 660         } else {
 661             len = 0;
 662         }
 663         total_bytes += len;
 664         bf->buf_ptr = bf->buffer;
 665         bf->buf_end = bf->buffer + len;
 666         *bf->buf_end = CH_EOB;
 667     }
 668     if (bf->buf_ptr < bf->buf_end) {
 669         return bf->buf_ptr[0];
 670     } else {
 671         bf->buf_ptr = bf->buf_end;
 672         return CH_EOF;
 673     }
 674 }
 675
 676 /* read next char from current input file and handle end of input buffer */
 677 static inline void inp(void)
 678 {
 679     ch = *(++(file->buf_ptr));
 680     /* end of buffer/file handling */
 681     if (ch == CH_EOB)
 682         ch = handle_eob();
 683 }
 684
 685 /* handle '\[\r]\n' */
 686 static int handle_stray_noerror(void)
 687 {
 688     while (ch == '\\') {
 689         inp();
 690         if (ch == '\n') {
 691             file->line_num++;
 692             inp();
 693         } else if (ch == '\r') {
 694             inp();
 695             if (ch != '\n')
 696                 goto fail;
 697             file->line_num++;
 698             inp();
 699         } else {
 700         fail:
 701             return 1;
 702         }
 703     }
 704     return 0;
 705 }
 706
 707 static void handle_stray(void)
 708 {
 709     if (handle_stray_noerror())
 710         tcc_error("stray '\\' in program");
 711 }
 712
 713 /* skip the stray and handle the \\n case. Output an error if
 714    incorrect char after the stray */
 715 static int handle_stray1(uint8_t *p)
 716 {
 717     int c;
 718
 719     file->buf_ptr = p;
 720     if (p >= file->buf_end) {
 721         c = handle_eob();
 722         if (c != '\\')
 723             return c;
 724         p = file->buf_ptr;
 725     }
 726     ch = *p;
 727     if (handle_stray_noerror()) {
 728         if (!(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
 729             tcc_error("stray '\\' in program");
 730         *--file->buf_ptr = '\\';
 731     }
 732     p = file->buf_ptr;
 733     c = *p;
 734     return c;
 735 }
 736
 737 /* handle just the EOB case, but not stray */
 738 #define PEEKC_EOB(c, p)\
 739 {\
 740     p++;\
 741     c = *p;\
 742     if (c == '\\') {\
 743         file->buf_ptr = p;\
 744         c = handle_eob();\
 745         p = file->buf_ptr;\
 746     }\
 747 }
 748
 749 /* handle the complicated stray case */
 750 #define PEEKC(c, p)\
 751 {\
 752     p++;\
 753     c = *p;\
 754     if (c == '\\') {\
 755         c = handle_stray1(p);\
 756         p = file->buf_ptr;\
 757     }\
 758 }
 759
 760 /* input with '\[\r]\n' handling. Note that this function cannot
 761    handle other characters after '\', so you cannot call it inside
 762    strings or comments */
 763 static void minp(void)
 764 {
 765     inp();
 766     if (ch == '\\')
 767         handle_stray();
 768 }
 769
 770 /* single line C++ comments */
 771 static uint8_t *parse_line_comment(uint8_t *p)
 772 {
 773     int c;
 774
 775     p++;
 776     for(;;) {
 777         c = *p;
 778     redo:
 779         if (c == '\n' || c == CH_EOF) {
 780             break;
 781         } else if (c == '\\') {
 782             file->buf_ptr = p;
 783             c = handle_eob();
 784             p = file->buf_ptr;
 785             if (c == '\\') {
 786                 PEEKC_EOB(c, p);
 787                 if (c == '\n') {
 788                     file->line_num++;
 789                     PEEKC_EOB(c, p);
 790                 } else if (c == '\r') {
 791                     PEEKC_EOB(c, p);
 792                     if (c == '\n') {
 793                         file->line_num++;
 794                         PEEKC_EOB(c, p);
 795                     }
 796                 }
 797             } else {
 798                 goto redo;
 799             }
 800         } else {
 801             p++;
 802         }
 803     }
 804     return p;
 805 }
 806
 807 /* C comments */
 808 static uint8_t *parse_comment(uint8_t *p)
 809 {
 810     int c;
 811
 812     p++;
 813     for(;;) {
 814         /* fast skip loop */
 815         for(;;) {
 816             c = *p;
 817             if (c == '\n' || c == '*' || c == '\\')
 818                 break;
 819             p++;
 820             c = *p;
 821             if (c == '\n' || c == '*' || c == '\\')
 822                 break;
 823             p++;
 824         }
 825         /* now we can handle all the cases */
 826         if (c == '\n') {
 827             file->line_num++;
 828             p++;
 829         } else if (c == '*') {
 830             p++;
 831             for(;;) {
 832                 c = *p;
 833                 if (c == '*') {
 834                     p++;
 835                 } else if (c == '/') {
 836                     goto end_of_comment;
 837                 } else if (c == '\\') {
 838                     file->buf_ptr = p;
 839                     c = handle_eob();
 840                     p = file->buf_ptr;
 841                     if (c == CH_EOF)
 842                         tcc_error("unexpected end of file in comment");
 843                     if (c == '\\') {
 844                         /* skip '\[\r]\n', otherwise just skip the stray */
 845                         while (c == '\\') {
 846                             PEEKC_EOB(c, p);
 847                             if (c == '\n') {
 848                                 file->line_num++;
 849                                 PEEKC_EOB(c, p);
 850                             } else if (c == '\r') {
 851                                 PEEKC_EOB(c, p);
 852                                 if (c == '\n') {
 853                                     file->line_num++;
 854                                     PEEKC_EOB(c, p);
 855                                 }
 856                             } else {
 857                                 goto after_star;
 858                             }
 859                         }
 860                     }
 861                 } else {
 862                     break;
 863                 }
 864             }
 865         after_star: ;
 866         } else {
 867             /* stray, eob or eof */
 868             file->buf_ptr = p;
 869             c = handle_eob();
 870             p = file->buf_ptr;
 871             if (c == CH_EOF) {
 872                 tcc_error("unexpected end of file in comment");
 873             } else if (c == '\\') {
 874                 p++;
 875             }
 876         }
 877     }
 878  end_of_comment:
 879     p++;
 880     return p;
 881 }
 882
 883 ST_FUNC int set_idnum(int c, int val)
 884 {
 885     int prev = isidnum_table[c - CH_EOF];
 886     isidnum_table[c - CH_EOF] = val;
 887     return prev;
 888 }
 889
 890 #define cinp minp
 891
 892 static inline void skip_spaces(void)
 893 {
 894     while (isidnum_table[ch - CH_EOF] & IS_SPC)
 895         cinp();
 896 }
 897
 898 static inline int check_space(int t, int *spc)
 899 {
 900     if (t < 256 && (isidnum_table[t - CH_EOF] & IS_SPC)) {
 901         if (*spc)
 902             return 1;
 903         *spc = 1;
 904     } else
 905         *spc = 0;
 906     return 0;
 907 }
 908
 909 /* parse a string without interpreting escapes */
 910 static uint8_t *parse_pp_string(uint8_t *p,
 911                                 int sep, CString *str)
 912 {
 913     int c;
 914     p++;
 915     for(;;) {
 916         c = *p;
 917         if (c == sep) {
 918             break;
 919         } else if (c == '\\') {
 920             file->buf_ptr = p;
 921             c = handle_eob();
 922             p = file->buf_ptr;
 923             if (c == CH_EOF) {
 924             unterminated_string:
 925                 /* XXX: indicate line number of start of string */
 926                 tcc_error("missing terminating %c character", sep);
 927             } else if (c == '\\') {
 928                 /* escape : just skip \[\r]\n */
 929                 PEEKC_EOB(c, p);
 930                 if (c == '\n') {
 931                     file->line_num++;
 932                     p++;
 933                 } else if (c == '\r') {
 934                     PEEKC_EOB(c, p);
 935                     if (c != '\n')
 936                         expect("'\n' after '\r'");
 937                     file->line_num++;
 938                     p++;
 939                 } else if (c == CH_EOF) {
 940                     goto unterminated_string;
 941                 } else {
 942                     if (str) {
 943                         cstr_ccat(str, '\\');
 944                         cstr_ccat(str, c);
 945                     }
 946                     p++;
 947                 }
 948             }
 949         } else if (c == '\n') {
 950         add_lf:
 951             if (ACCEPT_LF_IN_STRINGS) {
 952                 file->line_num++;
 953                 goto add_char;
 954             } else if (str) { /* not skipping */
 955                 goto unterminated_string;
 956             } else {
 957                 tcc_warning("missing terminating %c character", sep);
 958                 return p;
 959             }
 960         } else if (c == '\r') {
 961             PEEKC_EOB(c, p);
 962             if (c != '\n') {
 963                 if (str)
 964                     cstr_ccat(str, '\r');
 965             } else {
 966                 goto add_lf;
 967             }
 968         } else {
 969         add_char:
 970             if (str)
 971                 cstr_ccat(str, c);
 972             p++;
 973         }
 974     }
 975     p++;
 976     return p;
 977 }
 978
 979 /* skip block of text until #else, #elif or #endif. skip also pairs of
 980    #if/#endif */
 981 static void preprocess_skip(void)
 982 {
 983     int a, start_of_line, c, in_warn_or_error;
 984     uint8_t *p;
 985
 986     p = file->buf_ptr;
 987     a = 0;
 988 redo_start:
 989     start_of_line = 1;
 990     in_warn_or_error = 0;
 991     for(;;) {
 992     redo_no_start:
 993         c = *p;
 994         switch(c) {
 995         case ' ':
 996         case '\t':
 997         case '\f':
 998         case '\v':
 999         case '\r':
1000             p++;
1001             goto redo_no_start;
1002         case '\n':
1003             file->line_num++;
1004             p++;
1005             goto redo_start;
1006         case '\\':
1007             file->buf_ptr = p;
1008             c = handle_eob();
1009             if (c == CH_EOF) {
1010                 expect("#endif");
1011             } else if (c == '\\') {
1012                 ch = file->buf_ptr[0];
1013                 handle_stray_noerror();
1014             }
1015             p = file->buf_ptr;
1016             goto redo_no_start;
1017         /* skip strings */
1018         case '\"':
1019         case '\'':
1020             if (in_warn_or_error)
1021                 goto _default;
1022             tok_flags &= ~TOK_FLAG_BOL;
1023             p = parse_pp_string(p, c, NULL);
1024             break;
1025         /* skip comments */
1026         case '/':
1027             if (in_warn_or_error)
1028                 goto _default;
1029             file->buf_ptr = p;
1030             ch = *p;
1031             minp();
1032             p = file->buf_ptr;
1033             if (ch == '*') {
1034                 p = parse_comment(p);
1035             } else if (ch == '/') {
1036                 p = parse_line_comment(p);
1037             }
1038             break;
1039         case '#':
1040             p++;
1041             if (start_of_line) {
1042                 file->buf_ptr = p;
1043                 next_nomacro();
1044                 p = file->buf_ptr;
1045                 if (a == 0 &&
1046                     (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
1047                     goto the_end;
1048                 if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
1049                     a++;
1050                 else if (tok == TOK_ENDIF)
1051                     a--;
1052                 else if( tok == TOK_ERROR || tok == TOK_WARNING)
1053                     in_warn_or_error = 1;
1054                 else if (tok == TOK_LINEFEED)
1055                     goto redo_start;
1056                 else if (parse_flags & PARSE_FLAG_ASM_FILE)
1057                     p = parse_line_comment(p - 1);
1058             }
1059 #if !defined(TCC_TARGET_ARM)
1060             else if (parse_flags & PARSE_FLAG_ASM_FILE)
1061                 p = parse_line_comment(p - 1);
1062 #else
1063             /* ARM assembly uses '#' for constants */
1064 #endif
1065             break;
1066 _default:
1067         default:
1068             p++;
1069             break;
1070         }
1071         start_of_line = 0;
1072     }
1073  the_end: ;
1074     file->buf_ptr = p;
1075 }
1076
1077 #if 0
1078 /* return the number of additional 'ints' necessary to store the
1079    token */
1080 static inline int tok_size(const int *p)
1081 {
1082     switch(*p) {
1083         /* 4 bytes */
1084     case TOK_CINT:
1085     case TOK_CUINT:
1086     case TOK_CCHAR:
1087     case TOK_LCHAR:
1088     case TOK_CFLOAT:
1089     case TOK_LINENUM:
1090         return 1 + 1;
1091     case TOK_STR:
1092     case TOK_LSTR:
1093     case TOK_PPNUM:
1094     case TOK_PPSTR:
1095         return 1 + ((sizeof(CString) + ((CString *)(p+1))->size + 3) >> 2);
1096     case TOK_CLONG:
1097     case TOK_CULONG:
1098         return 1 + LONG_SIZE / 4;
1099     case TOK_CDOUBLE:
1100     case TOK_CLLONG:
1101     case TOK_CULLONG:
1102         return 1 + 2;
1103     case TOK_CLDOUBLE:
1104         return 1 + LDOUBLE_SIZE / 4;
1105     default:
1106         return 1 + 0;
1107     }
1108 }
1109 #endif
1110
1111 /* token string handling */
1112 ST_INLN void tok_str_new(TokenString *s)
1113 {
1114     s->str = NULL;
1115     s->len = s->lastlen = 0;
1116     s->allocated_len = 0;
1117     s->last_line_num = -1;
1118 }
1119
1120 ST_FUNC TokenString *tok_str_alloc(void)
1121 {
1122     TokenString *str = tal_realloc(tokstr_alloc, 0, sizeof *str);
1123     tok_str_new(str);
1124     return str;
1125 }
1126
1127 ST_FUNC int *tok_str_dup(TokenString *s)
1128 {
1129     int *str;
1130
1131     str = tal_realloc(tokstr_alloc, 0, s->len * sizeof(int));
1132     memcpy(str, s->str, s->len * sizeof(int));
1133     return str;
1134 }
1135
1136 ST_FUNC void tok_str_free_str(int *str)
1137 {
1138     tal_free(tokstr_alloc, str);
1139 }
1140
1141 ST_FUNC void tok_str_free(TokenString *str)
1142 {
1143     tok_str_free_str(str->str);
1144     tal_free(tokstr_alloc, str);
1145 }
1146
1147 ST_FUNC int *tok_str_realloc(TokenString *s, int new_size)
1148 {
1149     int *str, size;
1150
1151     size = s->allocated_len;
1152     if (size < 16)
1153         size = 16;
1154     while (size < new_size)
1155         size = size * 2;
1156     if (size > s->allocated_len) {
1157         str = tal_realloc(tokstr_alloc, s->str, size * sizeof(int));
1158         s->allocated_len = size;
1159         s->str = str;
1160     }
1161     return s->str;
1162 }
1163
1164 ST_FUNC void tok_str_add(TokenString *s, int t)
1165 {
1166     int len, *str;
1167
1168     len = s->len;
1169     str = s->str;
1170     if (len >= s->allocated_len)
1171         str = tok_str_realloc(s, len + 1);
1172     str[len++] = t;
1173     s->len = len;
1174 }
1175
1176 ST_FUNC void begin_macro(TokenString *str, int alloc)
1177 {
1178     str->alloc = alloc;
1179     str->prev = macro_stack;
1180     str->prev_ptr = macro_ptr;
1181     str->save_line_num = file->line_num;
1182     macro_ptr = str->str;
1183     macro_stack = str;
1184 }
1185
1186 ST_FUNC void end_macro(void)
1187 {
1188     TokenString *str = macro_stack;
1189     macro_stack = str->prev;
1190     macro_ptr = str->prev_ptr;
1191     file->line_num = str->save_line_num;
1192     str->len = 0; /* matters if str not alloced, may be tokstr_buf */
1193     if (str->alloc != 0) {
1194         if (str->alloc == 2)
1195             str->str = NULL; /* don't free */
1196         tok_str_free(str);
1197     }
1198 }
1199
1200 static void tok_str_add2(TokenString *s, int t, CValue *cv)
1201 {
1202     int len, *str;
1203
1204     len = s->lastlen = s->len;
1205     str = s->str;
1206
1207     /* allocate space for worst case */
1208     if (len + TOK_MAX_SIZE >= s->allocated_len)
1209         str = tok_str_realloc(s, len + TOK_MAX_SIZE + 1);
1210     str[len++] = t;
1211     switch(t) {
1212     case TOK_CINT:
1213     case TOK_CUINT:
1214     case TOK_CCHAR:
1215     case TOK_LCHAR:
1216     case TOK_CFLOAT:
1217     case TOK_LINENUM:
1218 #if LONG_SIZE == 4
1219     case TOK_CLONG:
1220     case TOK_CULONG:
1221 #endif
1222         str[len++] = cv->tab[0];
1223         break;
1224     case TOK_PPNUM:
1225     case TOK_PPSTR:
1226     case TOK_STR:
1227     case TOK_LSTR:
1228         {
1229             /* Insert the string into the int array. */
1230             size_t nb_words =
1231                 1 + (cv->str.size + sizeof(int) - 1) / sizeof(int);
1232             if (len + nb_words >= s->allocated_len)
1233                 str = tok_str_realloc(s, len + nb_words + 1);
1234             str[len] = cv->str.size;
1235             memcpy(&str[len + 1], cv->str.data, cv->str.size);
1236             len += nb_words;
1237         }
1238         break;
1239     case TOK_CDOUBLE:
1240     case TOK_CLLONG:
1241     case TOK_CULLONG:
1242 #if LONG_SIZE == 8
1243     case TOK_CLONG:
1244     case TOK_CULONG:
1245 #endif
1246 #if LDOUBLE_SIZE == 8
1247     case TOK_CLDOUBLE:
1248 #endif
1249         str[len++] = cv->tab[0];
1250         str[len++] = cv->tab[1];
1251         break;
1252 #if LDOUBLE_SIZE == 12
1253     case TOK_CLDOUBLE:
1254         str[len++] = cv->tab[0];
1255         str[len++] = cv->tab[1];
1256         str[len++] = cv->tab[2];
1257 #elif LDOUBLE_SIZE == 16
1258     case TOK_CLDOUBLE:
1259         str[len++] = cv->tab[0];
1260         str[len++] = cv->tab[1];
1261         str[len++] = cv->tab[2];
1262         str[len++] = cv->tab[3];
1263 #elif LDOUBLE_SIZE != 8
1264 #error add long double size support
1265 #endif
1266         break;
1267     default:
1268         break;
1269     }
1270     s->len = len;
1271 }
1272
1273 /* add the current parse token in token string 's' */
1274 ST_FUNC void tok_str_add_tok(TokenString *s)
1275 {
1276     CValue cval;
1277
1278     /* save line number info */
1279     if (file->line_num != s->last_line_num) {
1280         s->last_line_num = file->line_num;
1281         cval.i = s->last_line_num;
1282         tok_str_add2(s, TOK_LINENUM, &cval);
1283     }
1284     tok_str_add2(s, tok, &tokc);
1285 }
1286
1287 /* get a token from an integer array and increment pointer. */
1288 static inline void tok_get(int *t, const int **pp, CValue *cv)
1289 {
1290     const int *p = *pp;
1291     int n, *tab;
1292
1293     tab = cv->tab;
1294     switch(*t = *p++) {
1295 #if LONG_SIZE == 4
1296     case TOK_CLONG:
1297 #endif
1298     case TOK_CINT:
1299     case TOK_CCHAR:
1300     case TOK_LCHAR:
1301     case TOK_LINENUM:
1302         cv->i = *p++;
1303         break;
1304 #if LONG_SIZE == 4
1305     case TOK_CULONG:
1306 #endif
1307     case TOK_CUINT:
1308         cv->i = (unsigned)*p++;
1309         break;
1310     case TOK_CFLOAT:
1311         tab[0] = *p++;
1312         break;
1313     case TOK_STR:
1314     case TOK_LSTR:
1315     case TOK_PPNUM:
1316     case TOK_PPSTR:
1317         cv->str.size = *p++;
1318         cv->str.data = p;
1319         p += (cv->str.size + sizeof(int) - 1) / sizeof(int);
1320         break;
1321     case TOK_CDOUBLE:
1322     case TOK_CLLONG:
1323     case TOK_CULLONG:
1324 #if LONG_SIZE == 8
1325     case TOK_CLONG:
1326     case TOK_CULONG:
1327 #endif
1328         n = 2;
1329         goto copy;
1330     case TOK_CLDOUBLE:
1331 #if LDOUBLE_SIZE == 16
1332         n = 4;
1333 #elif LDOUBLE_SIZE == 12
1334         n = 3;
1335 #elif LDOUBLE_SIZE == 8
1336         n = 2;
1337 #else
1338 # error add long double size support
1339 #endif
1340     copy:
1341         do
1342             *tab++ = *p++;
1343         while (--n);
1344         break;
1345     default:
1346         break;
1347     }
1348     *pp = p;
1349 }
1350
1351 #if 0
1352 # define TOK_GET(t,p,c) tok_get(t,p,c)
1353 #else
1354 # define TOK_GET(t,p,c) do { \
1355     int _t = **(p); \
1356     if (TOK_HAS_VALUE(_t)) \
1357         tok_get(t, p, c); \
1358     else \
1359         *(t) = _t, ++*(p); \
1360     } while (0)
1361 #endif
1362
1363 static int macro_is_equal(const int *a, const int *b)
1364 {
1365     CValue cv;
1366     int t;
1367
1368     if (!a || !b)
1369         return 1;
1370
1371     while (*a && *b) {
1372         /* first time preallocate macro_equal_buf, next time only reset position to start */
1373         cstr_reset(&macro_equal_buf);
1374         TOK_GET(&t, &a, &cv);
1375         cstr_cat(&macro_equal_buf, get_tok_str(t, &cv), 0);
1376         TOK_GET(&t, &b, &cv);
1377         if (strcmp(macro_equal_buf.data, get_tok_str(t, &cv)))
1378             return 0;
1379     }
1380     return !(*a || *b);
1381 }
1382
1383 /* defines handling */
1384 ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
1385 {
1386     Sym *s, *o;
1387
1388     o = define_find(v);
1389     s = sym_push2(&define_stack, v, macro_type, 0);
1390     s->d = str;
1391     s->next = first_arg;
1392     table_ident[v - TOK_IDENT]->sym_define = s;
1393
1394     if (o && !macro_is_equal(o->d, s->d))
1395         tcc_warning("%s redefined", get_tok_str(v, NULL));
1396 }
1397
1398 /* undefined a define symbol. Its name is just set to zero */
1399 ST_FUNC void define_undef(Sym *s)
1400 {
1401     int v = s->v;
1402     if (v >= TOK_IDENT && v < tok_ident)
1403         table_ident[v - TOK_IDENT]->sym_define = NULL;
1404 }
1405
1406 ST_INLN Sym *define_find(int v)
1407 {
1408     v -= TOK_IDENT;
1409     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1410         return NULL;
1411     return table_ident[v]->sym_define;
1412 }
1413
1414 /* free define stack until top reaches 'b' */
1415 ST_FUNC void free_defines(Sym *b)
1416 {
1417     while (define_stack != b) {
1418         Sym *top = define_stack;
1419         define_stack = top->prev;
1420         tok_str_free_str(top->d);
1421         define_undef(top);
1422         sym_free(top);
1423     }
1424 }
1425
1426 /* label lookup */
1427 ST_FUNC Sym *label_find(int v)
1428 {
1429     v -= TOK_IDENT;
1430     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1431         return NULL;
1432     return table_ident[v]->sym_label;
1433 }
1434
1435 ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
1436 {
1437     Sym *s, **ps;
1438     s = sym_push2(ptop, v, VT_STATIC, 0);
1439     s->r = flags;
1440     ps = &table_ident[v - TOK_IDENT]->sym_label;
1441     if (ptop == &global_label_stack) {
1442         /* modify the top most local identifier, so that
1443            sym_identifier will point to 's' when popped */
1444         while (*ps != NULL)
1445             ps = &(*ps)->prev_tok;
1446     }
1447     s->prev_tok = *ps;
1448     *ps = s;
1449     return s;
1450 }
1451
1452 /* pop labels until element last is reached. Look if any labels are
1453    undefined. Define symbols if '&&label' was used. */
1454 ST_FUNC void label_pop(Sym **ptop, Sym *slast, int keep)
1455 {
1456     Sym *s, *s1;
1457     for(s = *ptop; s != slast; s = s1) {
1458         s1 = s->prev;
1459         if (s->r == LABEL_DECLARED) {
1460             tcc_warning_c(warn_all)("label '%s' declared but not used", get_tok_str(s->v, NULL));
1461         } else if (s->r == LABEL_FORWARD) {
1462                 tcc_error("label '%s' used but not defined",
1463                       get_tok_str(s->v, NULL));
1464         } else {
1465             if (s->c) {
1466                 /* define corresponding symbol. A size of
1467                    1 is put. */
1468                 put_extern_sym(s, cur_text_section, s->jnext, 1);
1469             }
1470         }
1471         /* remove label */
1472         if (s->r != LABEL_GONE)
1473             table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
1474         if (!keep)
1475             sym_free(s);
1476         else
1477             s->r = LABEL_GONE;
1478     }
1479     if (!keep)
1480         *ptop = slast;
1481 }
1482
1483 /* fake the nth "#if defined test_..." for tcc -dt -run */
1484 static void maybe_run_test(TCCState *s)
1485 {
1486     const char *p;
1487     if (s->include_stack_ptr != s->include_stack)
1488         return;
1489     p = get_tok_str(tok, NULL);
1490     if (0 != memcmp(p, "test_", 5))
1491         return;
1492     if (0 != --s->run_test)
1493         return;
1494     fprintf(s->ppfp, &"\n[%s]\n"[!(s->dflag & 32)], p), fflush(s->ppfp);
1495     define_push(tok, MACRO_OBJ, NULL, NULL);
1496 }
1497
1498 static int parse_include(char *buf, size_t size_buf, int last)
1499 {
1500     int c;
1501
1502     skip_spaces();
1503     if (ch == '<') {
1504         c = '>';
1505         goto read_name;
1506     } else if (ch == '\"') {
1507         char *q;
1508
1509         c = ch;
1510     read_name:
1511         inp();
1512         q = buf;
1513         while (ch != c && ch != '\n' && ch != CH_EOF) {
1514             if ((q - buf) < size_buf - 1)
1515                 *q++ = ch;
1516             if (ch == '\\') {
1517                 if (handle_stray_noerror() == 0)
1518                     --q;
1519             } else
1520                 inp();
1521         }
1522         if (ch != c)
1523             goto error;
1524         *q = '\0';
1525         minp();
1526 #if 0
1527         /* eat all spaces and comments after include */
1528         /* XXX: slightly incorrect */
1529         while (ch1 != '\n' && ch1 != CH_EOF)
1530             inp();
1531 #endif
1532     } else {
1533         int len;
1534         /* computed #include : concatenate everything up to linefeed,
1535            the result must be one of the two accepted forms.
1536            Don't convert pp-tokens to tokens here.  */
1537         parse_flags = (PARSE_FLAG_PREPROCESS
1538                        | PARSE_FLAG_LINEFEED
1539                        | (parse_flags & PARSE_FLAG_ASM_FILE));
1540         next();
1541         buf[0] = '\0';
1542         while (tok != TOK_LINEFEED && tok != last) {
1543             pstrcat(buf, size_buf, get_tok_str(tok, &tokc));
1544             next();
1545         }
1546         len = strlen(buf);
1547         if (tok == last)
1548             unget_tok(0);
1549         /* check syntax and remove '<>|""' */
1550         if ((len < 2 || ((buf[0] != '"' || buf[len-1] != '"') &&
1551                          (buf[0] != '<' || buf[len-1] != '>'))))
1552 error:
1553             tcc_error("'#include' expects \"FILENAME\" or <FILENAME>");
1554         c = buf[len-1];
1555         memmove(buf, buf + 1, len - 2);
1556         buf[len - 2] = '\0';
1557     }
1558     return c;
1559 }
1560
1561 static int get_include_file(TCCState *s1, int i, int c,
1562                             char *buf, char *buf1, size_t size_buf1)
1563 {
1564     const char *path;
1565
1566     if (i == 0) {
1567         /* check absolute include path */
1568         if (!IS_ABSPATH(buf))
1569             return -1;
1570         buf1[0] = 0;
1571     } else if (i == 1) {
1572         /* search in file's dir if "header.h" */
1573         if (c != '\"')
1574             return -1;
1575         path = file->true_filename;
1576         pstrncpy(buf1, path, tcc_basename(path) - path);
1577     } else {
1578         /* search in all the include paths */
1579         int j = i - 2, k = j - s1->nb_include_paths;
1580         path = k < 0 ? s1->include_paths[j] : s1->sysinclude_paths[k];
1581         pstrcpy(buf1, size_buf1, path);
1582         pstrcat(buf1, size_buf1, "/");
1583     }
1584     pstrcat(buf1, size_buf1, buf);
1585     return 0;
1586 }
1587
1588 /* eval an expression for #if/#elif */
1589 static int expr_preprocess(TCCState *s1)
1590 {
1591     int c, t;
1592     TokenString *str;
1593
1594     str = tok_str_alloc();
1595     pp_expr = 1;
1596     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1597         next(); /* do macro subst */
1598       redo:
1599         if (tok == TOK_DEFINED) {
1600             next_nomacro();
1601             t = tok;
1602             if (t == '(')
1603                 next_nomacro();
1604             if (tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT)
1605                 c = 1;
1606             else {
1607                 if (tok < TOK_IDENT)
1608                     expect("identifier");
1609                 if (tcc_state->run_test)
1610                     maybe_run_test(tcc_state);
1611                 c = define_find(tok) != 0;
1612             }
1613             if (t == '(') {
1614                 next_nomacro();
1615                 if (tok != ')')
1616                     expect("')'");
1617             }
1618             tok = TOK_CINT;
1619             tokc.i = c;
1620         } else if (tok == TOK___HAS_INCLUDE ||
1621                    tok == TOK___HAS_INCLUDE_NEXT) {
1622             int c, n, i, fd, save_tok = tok;
1623             char buf[1024];
1624
1625             next_nomacro();
1626             if (tok != '(')
1627                 expect("(");
1628             ch = file->buf_ptr[0];
1629             c = parse_include(buf, sizeof(buf), ')');
1630             next_nomacro();
1631             if (tok != ')')
1632                 expect("')'");
1633             tok = TOK_CINT;
1634             tokc.i = 0;
1635             i = save_tok == TOK___HAS_INCLUDE_NEXT ? file->include_next_index + 1 : 0;
1636             n = 2 + s1->nb_include_paths + s1->nb_sysinclude_paths;
1637             for (; i < n; i++) {
1638                 char buf1[sizeof file->filename];
1639
1640                 if (get_include_file(s1, i, c, buf, buf1, sizeof(buf1)))
1641                     continue;
1642                 if ((fd = open(buf1, O_RDONLY | O_BINARY)) >= 0) {
1643                     close(fd);
1644                     tokc.i = 1;
1645                     break;
1646                 }
1647             }
1648         } else if (tok >= TOK_IDENT) {
1649             /* if undefined macro, replace with zero, check for func-like */
1650             t = tok;
1651             tok = TOK_CINT;
1652             tokc.i = 0;
1653             tok_str_add_tok(str);
1654             next();
1655             if (tok == '(')
1656                 tcc_error("function-like macro '%s' is not defined",
1657                           get_tok_str(t, NULL));
1658             goto redo;
1659         }
1660         tok_str_add_tok(str);
1661     }
1662     pp_expr = 0;
1663     tok_str_add(str, -1); /* simulate end of file */
1664     tok_str_add(str, 0);
1665     /* now evaluate C constant expression */
1666     begin_macro(str, 1);
1667     next();
1668     c = expr_const();
1669     end_macro();
1670     return c != 0;
1671 }
1672
1673
1674 /* parse after #define */
1675 ST_FUNC void parse_define(void)
1676 {
1677     Sym *s, *first, **ps;
1678     int v, t, varg, is_vaargs, spc;
1679     int saved_parse_flags = parse_flags;
1680
1681     v = tok;
1682     if (v < TOK_IDENT || v == TOK_DEFINED)
1683         tcc_error("invalid macro name '%s'", get_tok_str(tok, &tokc));
1684     /* XXX: should check if same macro (ANSI) */
1685     first = NULL;
1686     t = MACRO_OBJ;
1687     /* We have to parse the whole define as if not in asm mode, in particular
1688        no line comment with '#' must be ignored.  Also for function
1689        macros the argument list must be parsed without '.' being an ID
1690        character.  */
1691     parse_flags = ((parse_flags & ~PARSE_FLAG_ASM_FILE) | PARSE_FLAG_SPACES);
1692     /* '(' must be just after macro definition for MACRO_FUNC */
1693     next_nomacro();
1694     parse_flags &= ~PARSE_FLAG_SPACES;
1695     if (tok == '(') {
1696         int dotid = set_idnum('.', 0);
1697         next_nomacro();
1698         ps = &first;
1699         if (tok != ')') for (;;) {
1700             varg = tok;
1701             next_nomacro();
1702             is_vaargs = 0;
1703             if (varg == TOK_DOTS) {
1704                 varg = TOK___VA_ARGS__;
1705                 is_vaargs = 1;
1706             } else if (tok == TOK_DOTS && gnu_ext) {
1707                 is_vaargs = 1;
1708                 next_nomacro();
1709             }
1710             if (varg < TOK_IDENT)
1711         bad_list:
1712                 tcc_error("bad macro parameter list");
1713             s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
1714             *ps = s;
1715             ps = &s->next;
1716             if (tok == ')')
1717                 break;
1718             if (tok != ',' || is_vaargs)
1719                 goto bad_list;
1720             next_nomacro();
1721         }
1722         parse_flags |= PARSE_FLAG_SPACES;
1723         next_nomacro();
1724         t = MACRO_FUNC;
1725         set_idnum('.', dotid);
1726     }
1727
1728     tokstr_buf.len = 0;
1729     spc = 2;
1730     parse_flags |= PARSE_FLAG_ACCEPT_STRAYS | PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED;
1731     /* The body of a macro definition should be parsed such that identifiers
1732        are parsed like the file mode determines (i.e. with '.' being an
1733        ID character in asm mode).  But '#' should be retained instead of
1734        regarded as line comment leader, so still don't set ASM_FILE
1735        in parse_flags. */
1736     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1737         /* remove spaces around ## and after '#' */
1738         if (TOK_TWOSHARPS == tok) {
1739             if (2 == spc)
1740                 goto bad_twosharp;
1741             if (1 == spc)
1742                 --tokstr_buf.len;
1743             spc = 3;
1744             tok = TOK_PPJOIN;
1745         } else if ('#' == tok) {
1746             spc = 4;
1747         } else if (check_space(tok, &spc)) {
1748             goto skip;
1749         }
1750         tok_str_add2(&tokstr_buf, tok, &tokc);
1751     skip:
1752         next_nomacro();
1753     }
1754
1755     parse_flags = saved_parse_flags;
1756     if (spc == 1)
1757         --tokstr_buf.len; /* remove trailing space */
1758     tok_str_add(&tokstr_buf, 0);
1759     if (3 == spc)
1760 bad_twosharp:
1761         tcc_error("'##' cannot appear at either end of macro");
1762     define_push(v, t, tok_str_dup(&tokstr_buf), first);
1763 }
1764
1765 static CachedInclude *search_cached_include(TCCState *s1, const char *filename, int add)
1766 {
1767     const unsigned char *s;
1768     unsigned int h;
1769     CachedInclude *e;
1770     int i;
1771
1772     h = TOK_HASH_INIT;
1773     s = (unsigned char *) filename;
1774     while (*s) {
1775 #ifdef _WIN32
1776         h = TOK_HASH_FUNC(h, toup(*s));
1777 #else
1778         h = TOK_HASH_FUNC(h, *s);
1779 #endif
1780         s++;
1781     }
1782     h &= (CACHED_INCLUDES_HASH_SIZE - 1);
1783
1784     i = s1->cached_includes_hash[h];
1785     for(;;) {
1786         if (i == 0)
1787             break;
1788         e = s1->cached_includes[i - 1];
1789         if (0 == PATHCMP(e->filename, filename))
1790             return e;
1791         i = e->hash_next;
1792     }
1793     if (!add)
1794         return NULL;
1795
1796     e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
1797     strcpy(e->filename, filename);
1798     e->ifndef_macro = e->once = 0;
1799     dynarray_add(&s1->cached_includes, &s1->nb_cached_includes, e);
1800     /* add in hash table */
1801     e->hash_next = s1->cached_includes_hash[h];
1802     s1->cached_includes_hash[h] = s1->nb_cached_includes;
1803 #ifdef INC_DEBUG
1804     printf("adding cached '%s'\n", filename);
1805 #endif
1806     return e;
1807 }
1808
1809 static void pragma_parse(TCCState *s1)
1810 {
1811     next_nomacro();
1812     if (tok == TOK_push_macro || tok == TOK_pop_macro) {
1813         int t = tok, v;
1814         Sym *s;
1815
1816         if (next(), tok != '(')
1817             goto pragma_err;
1818         if (next(), tok != TOK_STR)
1819             goto pragma_err;
1820         v = tok_alloc(tokc.str.data, tokc.str.size - 1)->tok;
1821         if (next(), tok != ')')
1822             goto pragma_err;
1823         if (t == TOK_push_macro) {
1824             while (NULL == (s = define_find(v)))
1825                 define_push(v, 0, NULL, NULL);
1826             s->type.ref = s; /* set push boundary */
1827         } else {
1828             for (s = define_stack; s; s = s->prev)
1829                 if (s->v == v && s->type.ref == s) {
1830                     s->type.ref = NULL;
1831                     break;
1832                 }
1833         }
1834         if (s)
1835             table_ident[v - TOK_IDENT]->sym_define = s->d ? s : NULL;
1836         else
1837             tcc_warning("unbalanced #pragma pop_macro");
1838         pp_debug_tok = t, pp_debug_symv = v;
1839
1840     } else if (tok == TOK_once) {
1841         search_cached_include(s1, file->filename, 1)->once = pp_once;
1842
1843     } else if (s1->output_type == TCC_OUTPUT_PREPROCESS) {
1844         /* tcc -E: keep pragmas below unchanged */
1845         unget_tok(' ');
1846         unget_tok(TOK_PRAGMA);
1847         unget_tok('#');
1848         unget_tok(TOK_LINEFEED);
1849
1850     } else if (tok == TOK_pack) {
1851         /* This may be:
1852            #pragma pack(1) // set
1853            #pragma pack() // reset to default
1854            #pragma pack(push) // push current
1855            #pragma pack(push,1) // push & set
1856            #pragma pack(pop) // restore previous */
1857         next();
1858         skip('(');
1859         if (tok == TOK_ASM_pop) {
1860             next();
1861             if (s1->pack_stack_ptr <= s1->pack_stack) {
1862             stk_error:
1863                 tcc_error("out of pack stack");
1864             }
1865             s1->pack_stack_ptr--;
1866         } else {
1867             int val = 0;
1868             if (tok != ')') {
1869                 if (tok == TOK_ASM_push) {
1870                     next();
1871                     if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
1872                         goto stk_error;
1873                     val = *s1->pack_stack_ptr++;
1874                     if (tok != ',')
1875                         goto pack_set;
1876                     next();
1877                 }
1878                 if (tok != TOK_CINT)
1879                     goto pragma_err;
1880                 val = tokc.i;
1881                 if (val < 1 || val > 16 || (val & (val - 1)) != 0)
1882                     goto pragma_err;
1883                 next();
1884             }
1885         pack_set:
1886             *s1->pack_stack_ptr = val;
1887         }
1888         if (tok != ')')
1889             goto pragma_err;
1890
1891     } else if (tok == TOK_comment) {
1892         char *p; int t;
1893         next();
1894         skip('(');
1895         t = tok;
1896         next();
1897         skip(',');
1898         if (tok != TOK_STR)
1899             goto pragma_err;
1900         p = tcc_strdup((char *)tokc.str.data);
1901         next();
1902         if (tok != ')')
1903             goto pragma_err;
1904         if (t == TOK_lib) {
1905             dynarray_add(&s1->pragma_libs, &s1->nb_pragma_libs, p);
1906         } else {
1907             if (t == TOK_option)
1908                 tcc_set_options(s1, p);
1909             tcc_free(p);
1910         }
1911
1912     } else
1913         tcc_warning_c(warn_unsupported)("#pragma %s ignored", get_tok_str(tok, &tokc));
1914     return;
1915
1916 pragma_err:
1917     tcc_error("malformed #pragma directive");
1918     return;
1919 }
1920
1921 /* is_bof is true if first non space token at beginning of file */
1922 ST_FUNC void preprocess(int is_bof)
1923 {
1924     TCCState *s1 = tcc_state;
1925     int i, c, n, saved_parse_flags;
1926     char buf[1024], *q;
1927     Sym *s;
1928
1929     saved_parse_flags = parse_flags;
1930     parse_flags = PARSE_FLAG_PREPROCESS
1931         | PARSE_FLAG_TOK_NUM
1932         | PARSE_FLAG_TOK_STR
1933         | PARSE_FLAG_LINEFEED
1934         | (parse_flags & PARSE_FLAG_ASM_FILE)
1935         ;
1936
1937     next_nomacro();
1938  redo:
1939     switch(tok) {
1940     case TOK_DEFINE:
1941         pp_debug_tok = tok;
1942         next_nomacro();
1943         pp_debug_symv = tok;
1944         parse_define();
1945         break;
1946     case TOK_UNDEF:
1947         pp_debug_tok = tok;
1948         next_nomacro();
1949         pp_debug_symv = tok;
1950         s = define_find(tok);
1951         /* undefine symbol by putting an invalid name */
1952         if (s)
1953             define_undef(s);
1954         break;
1955     case TOK_INCLUDE:
1956     case TOK_INCLUDE_NEXT:
1957         ch = file->buf_ptr[0];
1958         /* XXX: incorrect if comments : use next_nomacro with a special mode */
1959
1960         c = parse_include(buf, sizeof(buf), 0);
1961
1962         if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
1963             tcc_error("#include recursion too deep");
1964         i = tok == TOK_INCLUDE_NEXT ? file->include_next_index + 1 : 0;
1965         n = 2 + s1->nb_include_paths + s1->nb_sysinclude_paths;
1966         for (; i < n; ++i) {
1967             char buf1[sizeof file->filename];
1968             CachedInclude *e;
1969
1970             if (get_include_file(s1, i, c, buf, buf1, sizeof(buf1)))
1971                 continue;
1972             e = search_cached_include(s1, buf1, 0);
1973             if (e && (define_find(e->ifndef_macro) || e->once == pp_once)) {
1974                 /* no need to parse the include because the 'ifndef macro'
1975                    is defined (or had #pragma once) */
1976 #ifdef INC_DEBUG
1977                 printf("%s: skipping cached %s\n", file->filename, buf1);
1978 #endif
1979                 goto include_done;
1980             }
1981
1982             if (tcc_open(s1, buf1) < 0)
1983                 continue;
1984             /* push previous file on stack */
1985             *s1->include_stack_ptr++ = file->prev;
1986             file->include_next_index = i;
1987 #ifdef INC_DEBUG
1988             printf("%s: including %s\n", file->prev->filename, file->filename);
1989 #endif
1990             /* update target deps */
1991             if (s1->gen_deps) {
1992                 BufferedFile *bf = file;
1993                 while (i == 1 && (bf = bf->prev))
1994                     i = bf->include_next_index;
1995                 /* skip system include files */
1996                 if (s1->include_sys_deps || n - i > s1->nb_sysinclude_paths)
1997                     dynarray_add(&s1->target_deps, &s1->nb_target_deps,
1998                         tcc_strdup(buf1));
1999             }
2000             /* add include file debug info */
2001             tcc_debug_bincl(tcc_state);
2002             tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
2003             ch = file->buf_ptr[0];
2004             goto the_end;
2005         }
2006         tcc_error("include file '%s' not found", buf);
2007 include_done:
2008         break;
2009     case TOK_IFNDEF:
2010         c = 1;
2011         goto do_ifdef;
2012     case TOK_IF:
2013         c = expr_preprocess(s1);
2014         goto do_if;
2015     case TOK_IFDEF:
2016         c = 0;
2017     do_ifdef:
2018         next_nomacro();
2019         if (tok < TOK_IDENT)
2020             tcc_error("invalid argument for '#if%sdef'", c ? "n" : "");
2021         if (is_bof) {
2022             if (c) {
2023 #ifdef INC_DEBUG
2024                 printf("#ifndef %s\n", get_tok_str(tok, NULL));
2025 #endif
2026                 file->ifndef_macro = tok;
2027             }
2028         }
2029         if (tok == TOK___HAS_INCLUDE || tok == TOK___HAS_INCLUDE_NEXT)
2030             c = 1 ^ c;
2031         else
2032             c = (define_find(tok) != 0) ^ c;
2033     do_if:
2034         if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
2035             tcc_error("memory full (ifdef)");
2036         *s1->ifdef_stack_ptr++ = c;
2037         goto test_skip;
2038     case TOK_ELSE:
2039         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
2040             tcc_error("#else without matching #if");
2041         if (s1->ifdef_stack_ptr[-1] & 2)
2042             tcc_error("#else after #else");
2043         c = (s1->ifdef_stack_ptr[-1] ^= 3);
2044         goto test_else;
2045     case TOK_ELIF:
2046         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
2047             tcc_error("#elif without matching #if");
2048         c = s1->ifdef_stack_ptr[-1];
2049         if (c > 1)
2050             tcc_error("#elif after #else");
2051         /* last #if/#elif expression was true: we skip */
2052         if (c == 1) {
2053             c = 0;
2054         } else {
2055             c = expr_preprocess(s1);
2056             s1->ifdef_stack_ptr[-1] = c;
2057         }
2058     test_else:
2059         if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1)
2060             file->ifndef_macro = 0;
2061     test_skip:
2062         if (!(c & 1)) {
2063             preprocess_skip();
2064             is_bof = 0;
2065             goto redo;
2066         }
2067         break;
2068     case TOK_ENDIF:
2069         if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
2070             tcc_error("#endif without matching #if");
2071         s1->ifdef_stack_ptr--;
2072         /* '#ifndef macro' was at the start of file. Now we check if
2073            an '#endif' is exactly at the end of file */
2074         if (file->ifndef_macro &&
2075             s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
2076             file->ifndef_macro_saved = file->ifndef_macro;
2077             /* need to set to zero to avoid false matches if another
2078                #ifndef at middle of file */
2079             file->ifndef_macro = 0;
2080             while (tok != TOK_LINEFEED)
2081                 next_nomacro();
2082             tok_flags |= TOK_FLAG_ENDIF;
2083             goto the_end;
2084         }
2085         break;
2086     case TOK_PPNUM:
2087         n = strtoul((char*)tokc.str.data, &q, 10);
2088         goto _line_num;
2089     case TOK_LINE:
2090         next();
2091         if (tok != TOK_CINT)
2092     _line_err:
2093             tcc_error("wrong #line format");
2094         n = tokc.i;
2095     _line_num:
2096         next();
2097         if (tok != TOK_LINEFEED) {
2098             if (tok == TOK_STR) {
2099                 if (file->true_filename == file->filename)
2100                     file->true_filename = tcc_strdup(file->filename);
2101                 q = (char *)tokc.str.data;
2102                 buf[0] = 0;
2103                 if (!IS_ABSPATH(q)) {
2104                     /* prepend directory from real file */
2105                     pstrcpy(buf, sizeof buf, file->true_filename);
2106                     *tcc_basename(buf) = 0;
2107                 }
2108                 pstrcat(buf, sizeof buf, q);
2109                 tcc_debug_putfile(s1, buf);
2110             } else if (parse_flags & PARSE_FLAG_ASM_FILE)
2111                 break;
2112             else
2113                 goto _line_err;
2114             --n;
2115         }
2116         if (file->fd > 0)
2117             total_lines += file->line_num - n;
2118         file->line_num = n;
2119         break;
2120     case TOK_ERROR:
2121     case TOK_WARNING:
2122         c = tok;
2123         ch = file->buf_ptr[0];
2124         skip_spaces();
2125         q = buf;
2126         while (ch != '\n' && ch != CH_EOF) {
2127             if ((q - buf) < sizeof(buf) - 1)
2128                 *q++ = ch;
2129             if (ch == '\\') {
2130                 if (handle_stray_noerror() == 0)
2131                     --q;
2132             } else
2133                 inp();
2134         }
2135         *q = '\0';
2136         if (c == TOK_ERROR)
2137             tcc_error("#error %s", buf);
2138         else
2139             tcc_warning("#warning %s", buf);
2140         break;
2141     case TOK_PRAGMA:
2142         pragma_parse(s1);
2143         break;
2144     case TOK_LINEFEED:
2145         goto the_end;
2146     default:
2147         /* ignore gas line comment in an 'S' file. */
2148         if (saved_parse_flags & PARSE_FLAG_ASM_FILE)
2149             goto ignore;
2150         if (tok == '!' && is_bof)
2151             /* '!' is ignored at beginning to allow C scripts. */
2152             goto ignore;
2153         tcc_warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
2154     ignore:
2155         file->buf_ptr = parse_line_comment(file->buf_ptr - 1);
2156         goto the_end;
2157     }
2158     /* ignore other preprocess commands or #! for C scripts */
2159     while (tok != TOK_LINEFEED)
2160         next_nomacro();
2161  the_end:
2162     parse_flags = saved_parse_flags;
2163 }
2164
2165 /* evaluate escape codes in a string. */
2166 static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
2167 {
2168     int c, n, i;
2169     const uint8_t *p;
2170
2171     p = buf;
2172     for(;;) {
2173         c = *p;
2174         if (c == '\0')
2175             break;
2176         if (c == '\\') {
2177             p++;
2178             /* escape */
2179             c = *p;
2180             switch(c) {
2181             case '0': case '1': case '2': case '3':
2182             case '4': case '5': case '6': case '7':
2183                 /* at most three octal digits */
2184                 n = c - '0';
2185                 p++;
2186                 c = *p;
2187                 if (isoct(c)) {
2188                     n = n * 8 + c - '0';
2189                     p++;
2190                     c = *p;
2191                     if (isoct(c)) {
2192                         n = n * 8 + c - '0';
2193                         p++;
2194                     }
2195                 }
2196                 c = n;
2197                 goto add_char_nonext;
2198             case 'x': i = 0; goto parse_hex_or_ucn;
2199             case 'u': i = 4; goto parse_hex_or_ucn;
2200             case 'U': i = 8; goto parse_hex_or_ucn;
2201     parse_hex_or_ucn:
2202                 p++;
2203                 n = 0;
2204                 do {
2205                     c = *p;
2206                     if (c >= 'a' && c <= 'f')
2207                         c = c - 'a' + 10;
2208                     else if (c >= 'A' && c <= 'F')
2209                         c = c - 'A' + 10;
2210                     else if (isnum(c))
2211                         c = c - '0';
2212                     else if (i > 0)
2213                         expect("more hex digits in universal-character-name");
2214                     else
2215                         goto add_hex_or_ucn;
2216                     n = n * 16 + c;
2217                     p++;
2218                 } while (--i);
2219                 if (is_long) {
2220     add_hex_or_ucn:
2221                     c = n;
2222                     goto add_char_nonext;
2223                 }
2224                 cstr_u8cat(outstr, n);
2225                 continue;
2226             case 'a':
2227                 c = '\a';
2228                 break;
2229             case 'b':
2230                 c = '\b';
2231                 break;
2232             case 'f':
2233                 c = '\f';
2234                 break;
2235             case 'n':
2236                 c = '\n';
2237                 break;
2238             case 'r':
2239                 c = '\r';
2240                 break;
2241             case 't':
2242                 c = '\t';
2243                 break;
2244             case 'v':
2245                 c = '\v';
2246                 break;
2247             case 'e':
2248                 if (!gnu_ext)
2249                     goto invalid_escape;
2250                 c = 27;
2251                 break;
2252             case '\'':
2253             case '\"':
2254             case '\\':
2255             case '?':
2256                 break;
2257             default:
2258             invalid_escape:
2259                 if (c >= '!' && c <= '~')
2260                     tcc_warning("unknown escape sequence: \'\\%c\'", c);
2261                 else
2262                     tcc_warning("unknown escape sequence: \'\\x%x\'", c);
2263                 break;
2264             }
2265         } else if (is_long && c >= 0x80) {
2266             /* assume we are processing UTF-8 sequence */
2267             /* reference: The Unicode Standard, Version 10.0, ch3.9 */
2268
2269             int cont; /* count of continuation bytes */
2270             int skip; /* how many bytes should skip when error occurred */
2271             int i;
2272
2273             /* decode leading byte */
2274             if (c < 0xC2) {
2275                     skip = 1; goto invalid_utf8_sequence;
2276             } else if (c <= 0xDF) {
2277                     cont = 1; n = c & 0x1f;
2278             } else if (c <= 0xEF) {
2279                     cont = 2; n = c & 0xf;
2280             } else if (c <= 0xF4) {
2281                     cont = 3; n = c & 0x7;
2282             } else {
2283                     skip = 1; goto invalid_utf8_sequence;
2284             }
2285
2286             /* decode continuation bytes */
2287             for (i = 1; i <= cont; i++) {
2288                 int l = 0x80, h = 0xBF;
2289
2290                 /* adjust limit for second byte */
2291                 if (i == 1) {
2292                     switch (c) {
2293                     case 0xE0: l = 0xA0; break;
2294                     case 0xED: h = 0x9F; break;
2295                     case 0xF0: l = 0x90; break;
2296                     case 0xF4: h = 0x8F; break;
2297                     }
2298                 }
2299
2300                 if (p[i] < l || p[i] > h) {
2301                     skip = i; goto invalid_utf8_sequence;
2302                 }
2303
2304                 n = (n << 6) | (p[i] & 0x3f);
2305             }
2306
2307             /* advance pointer */
2308             p += 1 + cont;
2309             c = n;
2310             goto add_char_nonext;
2311
2312             /* error handling */
2313         invalid_utf8_sequence:
2314             tcc_warning("ill-formed UTF-8 subsequence starting with: \'\\x%x\'", c);
2315             c = 0xFFFD;
2316             p += skip;
2317             goto add_char_nonext;
2318
2319         }
2320         p++;
2321     add_char_nonext:
2322         if (!is_long)
2323             cstr_ccat(outstr, c);
2324         else {
2325 #ifdef TCC_TARGET_PE
2326             /* store as UTF-16 */
2327             if (c < 0x10000) {
2328                 cstr_wccat(outstr, c);
2329             } else {
2330                 c -= 0x10000;
2331                 cstr_wccat(outstr, (c >> 10) + 0xD800);
2332                 cstr_wccat(outstr, (c & 0x3FF) + 0xDC00);
2333             }
2334 #else
2335             cstr_wccat(outstr, c);
2336 #endif
2337         }
2338     }
2339     /* add a trailing '\0' */
2340     if (!is_long)
2341         cstr_ccat(outstr, '\0');
2342     else
2343         cstr_wccat(outstr, '\0');
2344 }
2345
2346 static void parse_string(const char *s, int len)
2347 {
2348     uint8_t buf[1000], *p = buf;
2349     int is_long, sep;
2350
2351     if ((is_long = *s == 'L'))
2352         ++s, --len;
2353     sep = *s++;
2354     len -= 2;
2355     if (len >= sizeof buf)
2356         p = tcc_malloc(len + 1);
2357     memcpy(p, s, len);
2358     p[len] = 0;
2359
2360     cstr_reset(&tokcstr);
2361     parse_escape_string(&tokcstr, p, is_long);
2362     if (p != buf)
2363         tcc_free(p);
2364
2365     if (sep == '\'') {
2366         int char_size, i, n, c;
2367         /* XXX: make it portable */
2368         if (!is_long)
2369             tok = TOK_CCHAR, char_size = 1;
2370         else
2371             tok = TOK_LCHAR, char_size = sizeof(nwchar_t);
2372         n = tokcstr.size / char_size - 1;
2373         if (n < 1)
2374             tcc_error("empty character constant");
2375         if (n > 1)
2376             tcc_warning_c(warn_all)("multi-character character constant");
2377         for (c = i = 0; i < n; ++i) {
2378             if (is_long)
2379                 c = ((nwchar_t *)tokcstr.data)[i];
2380             else
2381                 c = (c << 8) | ((char *)tokcstr.data)[i];
2382         }
2383         tokc.i = c;
2384     } else {
2385         tokc.str.size = tokcstr.size;
2386         tokc.str.data = tokcstr.data;
2387         if (!is_long)
2388             tok = TOK_STR;
2389         else
2390             tok = TOK_LSTR;
2391     }
2392 }
2393
2394 /* we use 64 bit numbers */
2395 #define BN_SIZE 2
2396
2397 /* bn = (bn << shift) | or_val */
2398 static void bn_lshift(unsigned int *bn, int shift, int or_val)
2399 {
2400     int i;
2401     unsigned int v;
2402     for(i=0;i<BN_SIZE;i++) {
2403         v = bn[i];
2404         bn[i] = (v << shift) | or_val;
2405         or_val = v >> (32 - shift);
2406     }
2407 }
2408
2409 static void bn_zero(unsigned int *bn)
2410 {
2411     int i;
2412     for(i=0;i<BN_SIZE;i++) {
2413         bn[i] = 0;
2414     }
2415 }
2416
2417 /* parse number in null terminated string 'p' and return it in the
2418    current token */
2419 static void parse_number(const char *p)
2420 {
2421     int b, t, shift, frac_bits, s, exp_val, ch;
2422     char *q;
2423     unsigned int bn[BN_SIZE];
2424     double d;
2425
2426     /* number */
2427     q = token_buf;
2428     ch = *p++;
2429     t = ch;
2430     ch = *p++;
2431     *q++ = t;
2432     b = 10;
2433     if (t == '.') {
2434         goto float_frac_parse;
2435     } else if (t == '0') {
2436         if (ch == 'x' || ch == 'X') {
2437             q--;
2438             ch = *p++;
2439             b = 16;
2440         } else if (tcc_state->tcc_ext && (ch == 'b' || ch == 'B')) {
2441             q--;
2442             ch = *p++;
2443             b = 2;
2444         }
2445     }
2446     /* parse all digits. cannot check octal numbers at this stage
2447        because of floating point constants */
2448     while (1) {
2449         if (ch >= 'a' && ch <= 'f')
2450             t = ch - 'a' + 10;
2451         else if (ch >= 'A' && ch <= 'F')
2452             t = ch - 'A' + 10;
2453         else if (isnum(ch))
2454             t = ch - '0';
2455         else
2456             break;
2457         if (t >= b)
2458             break;
2459         if (q >= token_buf + STRING_MAX_SIZE) {
2460         num_too_long:
2461             tcc_error("number too long");
2462         }
2463         *q++ = ch;
2464         ch = *p++;
2465     }
2466     if (ch == '.' ||
2467         ((ch == 'e' || ch == 'E') && b == 10) ||
2468         ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
2469         if (b != 10) {
2470             /* NOTE: strtox should support that for hexa numbers, but
2471                non ISOC99 libcs do not support it, so we prefer to do
2472                it by hand */
2473             /* hexadecimal or binary floats */
2474             /* XXX: handle overflows */
2475             *q = '\0';
2476             if (b == 16)
2477                 shift = 4;
2478             else
2479                 shift = 1;
2480             bn_zero(bn);
2481             q = token_buf;
2482             while (1) {
2483                 t = *q++;
2484                 if (t == '\0') {
2485                     break;
2486                 } else if (t >= 'a') {
2487                     t = t - 'a' + 10;
2488                 } else if (t >= 'A') {
2489                     t = t - 'A' + 10;
2490                 } else {
2491                     t = t - '0';
2492                 }
2493                 bn_lshift(bn, shift, t);
2494             }
2495             frac_bits = 0;
2496             if (ch == '.') {
2497                 ch = *p++;
2498                 while (1) {
2499                     t = ch;
2500                     if (t >= 'a' && t <= 'f') {
2501                         t = t - 'a' + 10;
2502                     } else if (t >= 'A' && t <= 'F') {
2503                         t = t - 'A' + 10;
2504                     } else if (t >= '0' && t <= '9') {
2505                         t = t - '0';
2506                     } else {
2507                         break;
2508                     }
2509                     if (t >= b)
2510                         tcc_error("invalid digit");
2511                     bn_lshift(bn, shift, t);
2512                     frac_bits += shift;
2513                     ch = *p++;
2514                 }
2515             }
2516             if (ch != 'p' && ch != 'P')
2517                 expect("exponent");
2518             ch = *p++;
2519             s = 1;
2520             exp_val = 0;
2521             if (ch == '+') {
2522                 ch = *p++;
2523             } else if (ch == '-') {
2524                 s = -1;
2525                 ch = *p++;
2526             }
2527             if (ch < '0' || ch > '9')
2528                 expect("exponent digits");
2529             while (ch >= '0' && ch <= '9') {
2530                 exp_val = exp_val * 10 + ch - '0';
2531                 ch = *p++;
2532             }
2533             exp_val = exp_val * s;
2534
2535             /* now we can generate the number */
2536             /* XXX: should patch directly float number */
2537             d = (double)bn[1] * 4294967296.0 + (double)bn[0];
2538             d = ldexp(d, exp_val - frac_bits);
2539             t = toup(ch);
2540             if (t == 'F') {
2541                 ch = *p++;
2542                 tok = TOK_CFLOAT;
2543                 /* float : should handle overflow */
2544                 tokc.f = (float)d;
2545             } else if (t == 'L') {
2546                 ch = *p++;
2547 #ifdef TCC_USING_DOUBLE_FOR_LDOUBLE
2548                 tok = TOK_CDOUBLE;
2549                 tokc.d = d;
2550 #else
2551                 tok = TOK_CLDOUBLE;
2552                 /* XXX: not large enough */
2553                 tokc.ld = (long double)d;
2554 #endif
2555             } else {
2556                 tok = TOK_CDOUBLE;
2557                 tokc.d = d;
2558             }
2559         } else {
2560             /* decimal floats */
2561             if (ch == '.') {
2562                 if (q >= token_buf + STRING_MAX_SIZE)
2563                     goto num_too_long;
2564                 *q++ = ch;
2565                 ch = *p++;
2566             float_frac_parse:
2567                 while (ch >= '0' && ch <= '9') {
2568                     if (q >= token_buf + STRING_MAX_SIZE)
2569                         goto num_too_long;
2570                     *q++ = ch;
2571                     ch = *p++;
2572                 }
2573             }
2574             if (ch == 'e' || ch == 'E') {
2575                 if (q >= token_buf + STRING_MAX_SIZE)
2576                     goto num_too_long;
2577                 *q++ = ch;
2578                 ch = *p++;
2579                 if (ch == '-' || ch == '+') {
2580                     if (q >= token_buf + STRING_MAX_SIZE)
2581                         goto num_too_long;
2582                     *q++ = ch;
2583                     ch = *p++;
2584                 }
2585                 if (ch < '0' || ch > '9')
2586                     expect("exponent digits");
2587                 while (ch >= '0' && ch <= '9') {
2588                     if (q >= token_buf + STRING_MAX_SIZE)
2589                         goto num_too_long;
2590                     *q++ = ch;
2591                     ch = *p++;
2592                 }
2593             }
2594             *q = '\0';
2595             t = toup(ch);
2596             errno = 0;
2597             if (t == 'F') {
2598                 ch = *p++;
2599                 tok = TOK_CFLOAT;
2600                 tokc.f = strtof(token_buf, NULL);
2601             } else if (t == 'L') {
2602                 ch = *p++;
2603 #ifdef TCC_USING_DOUBLE_FOR_LDOUBLE
2604                 tok = TOK_CDOUBLE;
2605                 tokc.d = strtod(token_buf, NULL);
2606 #else
2607                 tok = TOK_CLDOUBLE;
2608                 tokc.ld = strtold(token_buf, NULL);
2609 #endif
2610             } else {
2611                 tok = TOK_CDOUBLE;
2612                 tokc.d = strtod(token_buf, NULL);
2613             }
2614         }
2615     } else {
2616         unsigned long long n, n1;
2617         int lcount, ucount, ov = 0;
2618         const char *p1;
2619
2620         /* integer number */
2621         *q = '\0';
2622         q = token_buf;
2623         if (b == 10 && *q == '0') {
2624             b = 8;
2625             q++;
2626         }
2627         n = 0;
2628         while(1) {
2629             t = *q++;
2630             /* no need for checks except for base 10 / 8 errors */
2631             if (t == '\0')
2632                 break;
2633             else if (t >= 'a')
2634                 t = t - 'a' + 10;
2635             else if (t >= 'A')
2636                 t = t - 'A' + 10;
2637             else
2638                 t = t - '0';
2639             if (t >= b)
2640                 tcc_error("invalid digit");
2641             n1 = n;
2642             n = n * b + t;
2643             /* detect overflow */
2644             if (n1 >= 0x1000000000000000ULL && n / b != n1)
2645                 ov = 1;
2646         }
2647
2648         /* Determine the characteristics (unsigned and/or 64bit) the type of
2649            the constant must have according to the constant suffix(es) */
2650         lcount = ucount = 0;
2651         p1 = p;
2652         for(;;) {
2653             t = toup(ch);
2654             if (t == 'L') {
2655                 if (lcount >= 2)
2656                     tcc_error("three 'l's in integer constant");
2657                 if (lcount && *(p - 1) != ch)
2658                     tcc_error("incorrect integer suffix: %s", p1);
2659                 lcount++;
2660                 ch = *p++;
2661             } else if (t == 'U') {
2662                 if (ucount >= 1)
2663                     tcc_error("two 'u's in integer constant");
2664                 ucount++;
2665                 ch = *p++;
2666             } else {
2667                 break;
2668             }
2669         }
2670
2671         /* Determine if it needs 64 bits and/or unsigned in order to fit */
2672         if (ucount == 0 && b == 10) {
2673             if (lcount <= (LONG_SIZE == 4)) {
2674                 if (n >= 0x80000000U)
2675                     lcount = (LONG_SIZE == 4) + 1;
2676             }
2677             if (n >= 0x8000000000000000ULL)
2678                 ov = 1, ucount = 1;
2679         } else {
2680             if (lcount <= (LONG_SIZE == 4)) {
2681                 if (n >= 0x100000000ULL)
2682                     lcount = (LONG_SIZE == 4) + 1;
2683                 else if (n >= 0x80000000U)
2684                     ucount = 1;
2685             }
2686             if (n >= 0x8000000000000000ULL)
2687                 ucount = 1;
2688         }
2689
2690         if (ov)
2691             tcc_warning("integer constant overflow");
2692
2693         tok = TOK_CINT;
2694         if (lcount) {
2695             tok = TOK_CLONG;
2696             if (lcount == 2)
2697                 tok = TOK_CLLONG;
2698         }
2699         if (ucount)
2700             ++tok; /* TOK_CU... */
2701         tokc.i = n;
2702     }
2703     if (ch)
2704         tcc_error("invalid number");
2705 }
2706
2707
2708 #define PARSE2(c1, tok1, c2, tok2)              \
2709     case c1:                                    \
2710         PEEKC(c, p);                            \
2711         if (c == c2) {                          \
2712             p++;                                \
2713             tok = tok2;                         \
2714         } else {                                \
2715             tok = tok1;                         \
2716         }                                       \
2717         break;
2718
2719 /* return next token without macro substitution */
2720 static inline void next_nomacro1(void)
2721 {
2722     int t, c, is_long, len;
2723     TokenSym *ts;
2724     uint8_t *p, *p1;
2725     unsigned int h;
2726
2727     p = file->buf_ptr;
2728  redo_no_start:
2729     c = *p;
2730     switch(c) {
2731     case ' ':
2732     case '\t':
2733         tok = c;
2734         p++;
2735  maybe_space:
2736         if (parse_flags & PARSE_FLAG_SPACES)
2737             goto keep_tok_flags;
2738         while (isidnum_table[*p - CH_EOF] & IS_SPC)
2739             ++p;
2740         goto redo_no_start;
2741     case '\f':
2742     case '\v':
2743     case '\r':
2744         p++;
2745         goto redo_no_start;
2746     case '\\':
2747         /* first look if it is in fact an end of buffer */
2748         c = handle_stray1(p);
2749         p = file->buf_ptr;
2750         if (c == '\\')
2751             goto parse_simple;
2752         if (c != CH_EOF)
2753             goto redo_no_start;
2754         {
2755             TCCState *s1 = tcc_state;
2756             if ((parse_flags & PARSE_FLAG_LINEFEED)
2757                 && !(tok_flags & TOK_FLAG_EOF)) {
2758                 tok_flags |= TOK_FLAG_EOF;
2759                 tok = TOK_LINEFEED;
2760                 goto keep_tok_flags;
2761             } else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
2762                 tok = TOK_EOF;
2763             } else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
2764                 tcc_error("missing #endif");
2765             } else if (s1->include_stack_ptr == s1->include_stack) {
2766                 /* no include left : end of file. */
2767                 tok = TOK_EOF;
2768             } else {
2769                 tok_flags &= ~TOK_FLAG_EOF;
2770                 /* pop include file */
2771
2772                 /* test if previous '#endif' was after a #ifdef at
2773                    start of file */
2774                 if (tok_flags & TOK_FLAG_ENDIF) {
2775 #ifdef INC_DEBUG
2776                     printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
2777 #endif
2778                     search_cached_include(s1, file->filename, 1)
2779                         ->ifndef_macro = file->ifndef_macro_saved;
2780                     tok_flags &= ~TOK_FLAG_ENDIF;
2781                 }
2782
2783                 /* add end of include file debug info */
2784                 tcc_debug_eincl(tcc_state);
2785                 /* pop include stack */
2786                 tcc_close();
2787                 s1->include_stack_ptr--;
2788                 p = file->buf_ptr;
2789                 if (p == file->buffer)
2790                     tok_flags = TOK_FLAG_BOF|TOK_FLAG_BOL;
2791                 goto redo_no_start;
2792             }
2793         }
2794         break;
2795
2796     case '\n':
2797         file->line_num++;
2798         tok_flags |= TOK_FLAG_BOL;
2799         p++;
2800 maybe_newline:
2801         if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
2802             goto redo_no_start;
2803         tok = TOK_LINEFEED;
2804         goto keep_tok_flags;
2805
2806     case '#':
2807         /* XXX: simplify */
2808         PEEKC(c, p);
2809         if ((tok_flags & TOK_FLAG_BOL) &&
2810             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2811             file->buf_ptr = p;
2812             preprocess(tok_flags & TOK_FLAG_BOF);
2813             p = file->buf_ptr;
2814             goto maybe_newline;
2815         } else {
2816             if (c == '#') {
2817                 p++;
2818                 tok = TOK_TWOSHARPS;
2819             } else {
2820 #if !defined(TCC_TARGET_ARM)
2821                 if (parse_flags & PARSE_FLAG_ASM_FILE) {
2822                     p = parse_line_comment(p - 1);
2823                     goto redo_no_start;
2824                 } else
2825 #endif
2826                 {
2827                     tok = '#';
2828                 }
2829             }
2830         }
2831         break;
2832
2833     /* dollar is allowed to start identifiers when not parsing asm */
2834     case '$':
2835         if (!(isidnum_table[c - CH_EOF] & IS_ID)
2836          || (parse_flags & PARSE_FLAG_ASM_FILE))
2837             goto parse_simple;
2838
2839     case 'a': case 'b': case 'c': case 'd':
2840     case 'e': case 'f': case 'g': case 'h':
2841     case 'i': case 'j': case 'k': case 'l':
2842     case 'm': case 'n': case 'o': case 'p':
2843     case 'q': case 'r': case 's': case 't':
2844     case 'u': case 'v': case 'w': case 'x':
2845     case 'y': case 'z':
2846     case 'A': case 'B': case 'C': case 'D':
2847     case 'E': case 'F': case 'G': case 'H':
2848     case 'I': case 'J': case 'K':
2849     case 'M': case 'N': case 'O': case 'P':
2850     case 'Q': case 'R': case 'S': case 'T':
2851     case 'U': case 'V': case 'W': case 'X':
2852     case 'Y': case 'Z':
2853     case '_':
2854     parse_ident_fast:
2855         p1 = p;
2856         h = TOK_HASH_INIT;
2857         h = TOK_HASH_FUNC(h, c);
2858         while (c = *++p, isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2859             h = TOK_HASH_FUNC(h, c);
2860         len = p - p1;
2861         if (c != '\\') {
2862             TokenSym **pts;
2863
2864             /* fast case : no stray found, so we have the full token
2865                and we have already hashed it */
2866             h &= (TOK_HASH_SIZE - 1);
2867             pts = &hash_ident[h];
2868             for(;;) {
2869                 ts = *pts;
2870                 if (!ts)
2871                     break;
2872                 if (ts->len == len && !memcmp(ts->str, p1, len))
2873                     goto token_found;
2874                 pts = &(ts->hash_next);
2875             }
2876             ts = tok_alloc_new(pts, (char *) p1, len);
2877         token_found: ;
2878         } else {
2879             /* slower case */
2880             cstr_reset(&tokcstr);
2881             cstr_cat(&tokcstr, (char *) p1, len);
2882             p--;
2883             PEEKC(c, p);
2884         parse_ident_slow:
2885             while (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2886             {
2887                 cstr_ccat(&tokcstr, c);
2888                 PEEKC(c, p);
2889             }
2890             ts = tok_alloc(tokcstr.data, tokcstr.size);
2891         }
2892         tok = ts->tok;
2893         break;
2894     case 'L':
2895         t = p[1];
2896         if (t != '\\' && t != '\'' && t != '\"') {
2897             /* fast case */
2898             goto parse_ident_fast;
2899         } else {
2900             PEEKC(c, p);
2901             if (c == '\'' || c == '\"') {
2902                 is_long = 1;
2903                 goto str_const;
2904             } else {
2905                 cstr_reset(&tokcstr);
2906                 cstr_ccat(&tokcstr, 'L');
2907                 goto parse_ident_slow;
2908             }
2909         }
2910         break;
2911
2912     case '0': case '1': case '2': case '3':
2913     case '4': case '5': case '6': case '7':
2914     case '8': case '9':
2915         t = c;
2916         PEEKC(c, p);
2917         /* after the first digit, accept digits, alpha, '.' or sign if
2918            prefixed by 'eEpP' */
2919     parse_num:
2920         cstr_reset(&tokcstr);
2921         for(;;) {
2922             cstr_ccat(&tokcstr, t);
2923             if (!((isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2924                   || c == '.'
2925                   || ((c == '+' || c == '-')
2926                       && (((t == 'e' || t == 'E')
2927                             && !(parse_flags & PARSE_FLAG_ASM_FILE
2928                                 /* 0xe+1 is 3 tokens in asm */
2929                                 && ((char*)tokcstr.data)[0] == '0'
2930                                 && toup(((char*)tokcstr.data)[1]) == 'X'))
2931                           || t == 'p' || t == 'P'))))
2932                 break;
2933             t = c;
2934             PEEKC(c, p);
2935         }
2936         /* We add a trailing '\0' to ease parsing */
2937         cstr_ccat(&tokcstr, '\0');
2938         tokc.str.size = tokcstr.size;
2939         tokc.str.data = tokcstr.data;
2940         tok = TOK_PPNUM;
2941         break;
2942
2943     case '.':
2944         /* special dot handling because it can also start a number */
2945         PEEKC(c, p);
2946         if (isnum(c)) {
2947             t = '.';
2948             goto parse_num;
2949         } else if ((isidnum_table['.' - CH_EOF] & IS_ID)
2950                    && (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))) {
2951             *--p = c = '.';
2952             goto parse_ident_fast;
2953         } else if (c == '.') {
2954             PEEKC(c, p);
2955             if (c == '.') {
2956                 p++;
2957                 tok = TOK_DOTS;
2958             } else {
2959                 *--p = '.'; /* may underflow into file->unget[] */
2960                 tok = '.';
2961             }
2962         } else {
2963             tok = '.';
2964         }
2965         break;
2966     case '\'':
2967     case '\"':
2968         is_long = 0;
2969     str_const:
2970         cstr_reset(&tokcstr);
2971         if (is_long)
2972             cstr_ccat(&tokcstr, 'L');
2973         cstr_ccat(&tokcstr, c);
2974         p = parse_pp_string(p, c, &tokcstr);
2975         cstr_ccat(&tokcstr, c);
2976         cstr_ccat(&tokcstr, '\0');
2977         tokc.str.size = tokcstr.size;
2978         tokc.str.data = tokcstr.data;
2979         tok = TOK_PPSTR;
2980         break;
2981
2982     case '<':
2983         PEEKC(c, p);
2984         if (c == '=') {
2985             p++;
2986             tok = TOK_LE;
2987         } else if (c == '<') {
2988             PEEKC(c, p);
2989             if (c == '=') {
2990                 p++;
2991                 tok = TOK_A_SHL;
2992             } else {
2993                 tok = TOK_SHL;
2994             }
2995         } else {
2996             tok = TOK_LT;
2997         }
2998         break;
2999     case '>':
3000         PEEKC(c, p);
3001         if (c == '=') {
3002             p++;
3003             tok = TOK_GE;
3004         } else if (c == '>') {
3005             PEEKC(c, p);
3006             if (c == '=') {
3007                 p++;
3008                 tok = TOK_A_SAR;
3009             } else {
3010                 tok = TOK_SAR;
3011             }
3012         } else {
3013             tok = TOK_GT;
3014         }
3015         break;
3016
3017     case '&':
3018         PEEKC(c, p);
3019         if (c == '&') {
3020             p++;
3021             tok = TOK_LAND;
3022         } else if (c == '=') {
3023             p++;
3024             tok = TOK_A_AND;
3025         } else {
3026             tok = '&';
3027         }
3028         break;
3029
3030     case '|':
3031         PEEKC(c, p);
3032         if (c == '|') {
3033             p++;
3034             tok = TOK_LOR;
3035         } else if (c == '=') {
3036             p++;
3037             tok = TOK_A_OR;
3038         } else {
3039             tok = '|';
3040         }
3041         break;
3042
3043     case '+':
3044         PEEKC(c, p);
3045         if (c == '+') {
3046             p++;
3047             tok = TOK_INC;
3048         } else if (c == '=') {
3049             p++;
3050             tok = TOK_A_ADD;
3051         } else {
3052             tok = '+';
3053         }
3054         break;
3055
3056     case '-':
3057         PEEKC(c, p);
3058         if (c == '-') {
3059             p++;
3060             tok = TOK_DEC;
3061         } else if (c == '=') {
3062             p++;
3063             tok = TOK_A_SUB;
3064         } else if (c == '>') {
3065             p++;
3066             tok = TOK_ARROW;
3067         } else {
3068             tok = '-';
3069         }
3070         break;
3071
3072     PARSE2('!', '!', '=', TOK_NE)
3073     PARSE2('=', '=', '=', TOK_EQ)
3074     PARSE2('*', '*', '=', TOK_A_MUL)
3075     PARSE2('%', '%', '=', TOK_A_MOD)
3076     PARSE2('^', '^', '=', TOK_A_XOR)
3077
3078         /* comments or operator */
3079     case '/':
3080         PEEKC(c, p);
3081         if (c == '*') {
3082             p = parse_comment(p);
3083             /* comments replaced by a blank */
3084             tok = ' ';
3085             goto maybe_space;
3086         } else if (c == '/') {
3087             p = parse_line_comment(p);
3088             tok = ' ';
3089             goto maybe_space;
3090         } else if (c == '=') {
3091             p++;
3092             tok = TOK_A_DIV;
3093         } else {
3094             tok = '/';
3095         }
3096         break;
3097
3098         /* simple tokens */
3099     case '(':
3100     case ')':
3101     case '[':
3102     case ']':
3103     case '{':
3104     case '}':
3105     case ',':
3106     case ';':
3107     case ':':
3108     case '?':
3109     case '~':
3110     case '@': /* only used in assembler */
3111     parse_simple:
3112         tok = c;
3113         p++;
3114         break;
3115     default:
3116         if (c >= 0x80 && c <= 0xFF) /* utf8 identifiers */
3117             goto parse_ident_fast;
3118         if (parse_flags & PARSE_FLAG_ASM_FILE)
3119             goto parse_simple;
3120         tcc_error("unrecognized character \\x%02x", c);
3121         break;
3122     }
3123     tok_flags = 0;
3124 keep_tok_flags:
3125     file->buf_ptr = p;
3126 #if defined(PARSE_DEBUG)
3127     printf("token = %d %s\n", tok, get_tok_str(tok, &tokc));
3128 #endif
3129 }
3130
3131 static void macro_subst(
3132     TokenString *tok_str,
3133     Sym **nested_list,
3134     const int *macro_str
3135     );
3136
3137 /* substitute arguments in replacement lists in macro_str by the values in
3138    args (field d) and return allocated string */
3139 static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
3140 {
3141     int t, t0, t1, spc;
3142     const int *st;
3143     Sym *s;
3144     CValue cval;
3145     TokenString str;
3146     CString cstr;
3147
3148     tok_str_new(&str);
3149     t0 = t1 = 0;
3150     while(1) {
3151         TOK_GET(&t, &macro_str, &cval);
3152         if (!t)
3153             break;
3154         if (t == '#') {
3155             /* stringize */
3156             TOK_GET(&t, &macro_str, &cval);
3157             if (!t)
3158                 goto bad_stringy;
3159             s = sym_find2(args, t);
3160             if (s) {
3161                 cstr_new(&cstr);
3162                 cstr_ccat(&cstr, '\"');
3163                 st = s->d;
3164                 spc = 0;
3165                 while (*st >= 0) {
3166                     TOK_GET(&t, &st, &cval);
3167                     if (t != TOK_PLCHLDR
3168                      && t != TOK_NOSUBST
3169                      && 0 == check_space(t, &spc)) {
3170                         const char *s = get_tok_str(t, &cval);
3171                         while (*s) {
3172                             if (t == TOK_PPSTR && *s != '\'')
3173                                 add_char(&cstr, *s);
3174                             else
3175                                 cstr_ccat(&cstr, *s);
3176                             ++s;
3177                         }
3178                     }
3179                 }
3180                 cstr.size -= spc;
3181                 cstr_ccat(&cstr, '\"');
3182                 cstr_ccat(&cstr, '\0');
3183 #ifdef PP_DEBUG
3184                 printf("\nstringize: <%s>\n", (char *)cstr.data);
3185 #endif
3186                 /* add string */
3187                 cval.str.size = cstr.size;
3188                 cval.str.data = cstr.data;
3189                 tok_str_add2(&str, TOK_PPSTR, &cval);
3190                 cstr_free(&cstr);
3191             } else {
3192         bad_stringy:
3193                 expect("macro parameter after '#'");
3194             }
3195         } else if (t >= TOK_IDENT) {
3196             s = sym_find2(args, t);
3197             if (s) {
3198                 int l0 = str.len;
3199                 st = s->d;
3200                 /* if '##' is present before or after, no arg substitution */
3201                 if (*macro_str == TOK_PPJOIN || t1 == TOK_PPJOIN) {
3202                     /* special case for var arg macros : ## eats the ','
3203                        if empty VA_ARGS variable. */
3204                     if (t1 == TOK_PPJOIN && t0 == ',' && gnu_ext && s->type.t) {
3205                         if (*st <= 0) {
3206                             /* suppress ',' '##' */
3207                             str.len -= 2;
3208                         } else {
3209                             /* suppress '##' and add variable */
3210                             str.len--;
3211                             goto add_var;
3212                         }
3213                     }
3214                 } else {
3215             add_var:
3216                     if (!s->next) {
3217                         /* Expand arguments tokens and store them.  In most
3218                            cases we could also re-expand each argument if
3219                            used multiple times, but not if the argument
3220                            contains the __COUNTER__ macro.  */
3221                         TokenString str2;
3222                         sym_push2(&s->next, s->v, s->type.t, 0);
3223                         tok_str_new(&str2);
3224                         macro_subst(&str2, nested_list, st);
3225                         tok_str_add(&str2, 0);
3226                         s->next->d = str2.str;
3227                     }
3228                     st = s->next->d;
3229                 }
3230                 for(;;) {
3231                     int t2;
3232                     TOK_GET(&t2, &st, &cval);
3233                     if (t2 <= 0)
3234                         break;
3235                     tok_str_add2(&str, t2, &cval);
3236                 }
3237                 if (str.len == l0) /* expanded to empty string */
3238                     tok_str_add(&str, TOK_PLCHLDR);
3239             } else {
3240                 tok_str_add(&str, t);
3241             }
3242         } else {
3243             tok_str_add2(&str, t, &cval);
3244         }
3245         t0 = t1, t1 = t;
3246     }
3247     tok_str_add(&str, 0);
3248     return str.str;
3249 }
3250
3251 static char const ab_month_name[12][4] =
3252 {
3253     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3254     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
3255 };
3256
3257 static int paste_tokens(int t1, CValue *v1, int t2, CValue *v2)
3258 {
3259     CString cstr;
3260     int n, ret = 1;
3261
3262     cstr_new(&cstr);
3263     if (t1 != TOK_PLCHLDR)
3264         cstr_cat(&cstr, get_tok_str(t1, v1), -1);
3265     n = cstr.size;
3266     if (t2 != TOK_PLCHLDR)
3267         cstr_cat(&cstr, get_tok_str(t2, v2), -1);
3268     cstr_ccat(&cstr, '\0');
3269
3270     tcc_open_bf(tcc_state, ":paste:", cstr.size);
3271     memcpy(file->buffer, cstr.data, cstr.size);
3272     tok_flags = 0;
3273     for (;;) {
3274         next_nomacro1();
3275         if (0 == *file->buf_ptr)
3276             break;
3277         if (is_space(tok))
3278             continue;
3279         tcc_warning("pasting \"%.*s\" and \"%s\" does not give a valid"
3280             " preprocessing token", n, (char *)cstr.data, (char*)cstr.data + n);
3281         ret = 0;
3282         break;
3283     }
3284     tcc_close();
3285     //printf("paste <%s>\n", (char*)cstr.data);
3286     cstr_free(&cstr);
3287     return ret;
3288 }
3289
3290 /* handle the '##' operator. Return NULL if no '##' seen. Otherwise
3291    return the resulting string (which must be freed). */
3292 static inline int *macro_twosharps(const int *ptr0)
3293 {
3294     int t;
3295     CValue cval;
3296     TokenString macro_str1;
3297     int start_of_nosubsts = -1;
3298     const int *ptr;
3299
3300     /* we search the first '##' */
3301     for (ptr = ptr0;;) {
3302         TOK_GET(&t, &ptr, &cval);
3303         if (t == TOK_PPJOIN)
3304             break;
3305         if (t == 0)
3306             return NULL;
3307     }
3308
3309     tok_str_new(&macro_str1);
3310
3311     //tok_print(" $$$", ptr0);
3312     for (ptr = ptr0;;) {
3313         TOK_GET(&t, &ptr, &cval);
3314         if (t == 0)
3315             break;
3316         if (t == TOK_PPJOIN)
3317             continue;
3318         while (*ptr == TOK_PPJOIN) {
3319             int t1; CValue cv1;
3320             /* given 'a##b', remove nosubsts preceding 'a' */
3321             if (start_of_nosubsts >= 0)
3322                 macro_str1.len = start_of_nosubsts;
3323             /* given 'a##b', remove nosubsts preceding 'b' */
3324             while ((t1 = *++ptr) == TOK_NOSUBST)
3325                 ;
3326             if (t1 && t1 != TOK_PPJOIN) {
3327                 TOK_GET(&t1, &ptr, &cv1);
3328                 if (t != TOK_PLCHLDR || t1 != TOK_PLCHLDR) {
3329                     if (paste_tokens(t, &cval, t1, &cv1)) {
3330                         t = tok, cval = tokc;
3331                     } else {
3332                         tok_str_add2(&macro_str1, t, &cval);
3333                         t = t1, cval = cv1;
3334                     }
3335                 }
3336             }
3337         }
3338         if (t == TOK_NOSUBST) {
3339             if (start_of_nosubsts < 0)
3340                 start_of_nosubsts = macro_str1.len;
3341         } else {
3342             start_of_nosubsts = -1;
3343         }
3344         tok_str_add2(&macro_str1, t, &cval);
3345     }
3346     tok_str_add(&macro_str1, 0);
3347     //tok_print(" ###", macro_str1.str);
3348     return macro_str1.str;
3349 }
3350
3351 /* peek or read [ws_str == NULL] next token from function macro call,
3352    walking up macro levels up to the file if necessary */
3353 static int next_argstream(Sym **nested_list, TokenString *ws_str)
3354 {
3355     int t;
3356     const int *p;
3357     Sym *sa;
3358
3359     for (;;) {
3360         if (macro_ptr) {
3361             p = macro_ptr, t = *p;
3362             if (ws_str) {
3363                 while (is_space(t) || TOK_LINEFEED == t || TOK_PLCHLDR == t)
3364                     tok_str_add(ws_str, t), t = *++p;
3365             }
3366             if (t == 0) {
3367                 end_macro();
3368                 /* also, end of scope for nested defined symbol */
3369                 sa = *nested_list;
3370                 while (sa && sa->v == 0)
3371                     sa = sa->prev;
3372                 if (sa)
3373                     sa->v = 0;
3374                 continue;
3375             }
3376         } else {
3377             ch = handle_eob();
3378             if (ws_str) {
3379                 while (is_space(ch) || ch == '\n' || ch == '/') {
3380                     if (ch == '/') {
3381                         int c;
3382                         uint8_t *p = file->buf_ptr;
3383                         PEEKC(c, p);
3384                         if (c == '*') {
3385                             p = parse_comment(p);
3386                             file->buf_ptr = p - 1;
3387                         } else if (c == '/') {
3388                             p = parse_line_comment(p);
3389                             file->buf_ptr = p - 1;
3390                         } else
3391                             break;
3392                         ch = ' ';
3393                     }
3394                     if (ch == '\n')
3395                         file->line_num++;
3396                     if (!(ch == '\f' || ch == '\v' || ch == '\r'))
3397                         tok_str_add(ws_str, ch);
3398                     cinp();
3399                 }
3400             }
3401             t = ch;
3402         }
3403
3404         if (ws_str)
3405             return t;
3406         next_nomacro();
3407         return tok;
3408     }
3409 }
3410
3411 /* do macro substitution of current token with macro 's' and add
3412    result to (tok_str,tok_len). 'nested_list' is the list of all
3413    macros we got inside to avoid recursing. Return non zero if no
3414    substitution needs to be done */
3415 static int macro_subst_tok(
3416     TokenString *tok_str,
3417     Sym **nested_list,
3418     Sym *s)
3419 {
3420     Sym *args, *sa, *sa1;
3421     int parlevel, t, t1, spc;
3422     TokenString str;
3423     char *cstrval;
3424     CValue cval;
3425     CString cstr;
3426     char buf[32];
3427
3428     /* if symbol is a macro, prepare substitution */
3429     /* special macros */
3430     if (tok == TOK___LINE__ || tok == TOK___COUNTER__) {
3431         t = tok == TOK___LINE__ ? file->line_num : pp_counter++;
3432         snprintf(buf, sizeof(buf), "%d", t);
3433         cstrval = buf;
3434         t1 = TOK_PPNUM;
3435         goto add_cstr1;
3436     } else if (tok == TOK___FILE__) {
3437         cstrval = file->filename;
3438         goto add_cstr;
3439     } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
3440         time_t ti;
3441         struct tm *tm;
3442
3443         time(&ti);
3444         tm = localtime(&ti);
3445         if (tok == TOK___DATE__) {
3446             snprintf(buf, sizeof(buf), "%s %2d %d",
3447                      ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
3448         } else {
3449             snprintf(buf, sizeof(buf), "%02d:%02d:%02d",
3450                      tm->tm_hour, tm->tm_min, tm->tm_sec);
3451         }
3452         cstrval = buf;
3453     add_cstr:
3454         t1 = TOK_STR;
3455     add_cstr1:
3456         cstr_new(&cstr);
3457         cstr_cat(&cstr, cstrval, 0);
3458         cval.str.size = cstr.size;
3459         cval.str.data = cstr.data;
3460         tok_str_add2(tok_str, t1, &cval);
3461         cstr_free(&cstr);
3462     } else if (s->d) {
3463         int saved_parse_flags = parse_flags;
3464         int *joined_str = NULL;
3465         int *mstr = s->d;
3466
3467         if (s->type.t == MACRO_FUNC) {
3468             /* whitespace between macro name and argument list */
3469             TokenString ws_str;
3470             tok_str_new(&ws_str);
3471
3472             spc = 0;
3473             parse_flags |= PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED
3474                 | PARSE_FLAG_ACCEPT_STRAYS;
3475
3476             /* get next token from argument stream */
3477             t = next_argstream(nested_list, &ws_str);
3478             if (t != '(') {
3479                 /* not a macro substitution after all, restore the
3480                  * macro token plus all whitespace we've read.
3481                  * whitespace is intentionally not merged to preserve
3482                  * newlines. */
3483                 parse_flags = saved_parse_flags;
3484                 tok_str_add(tok_str, tok);
3485                 if (parse_flags & PARSE_FLAG_SPACES) {
3486                     int i;
3487                     for (i = 0; i < ws_str.len; i++)
3488                         tok_str_add(tok_str, ws_str.str[i]);
3489                 }
3490                 tok_str_free_str(ws_str.str);
3491                 return 0;
3492             } else {
3493                 tok_str_free_str(ws_str.str);
3494             }
3495             do {
3496                 next_nomacro(); /* eat '(' */
3497             } while (tok == TOK_PLCHLDR || is_space(tok));
3498
3499             /* argument macro */
3500             args = NULL;
3501             sa = s->next;
3502             /* NOTE: empty args are allowed, except if no args */
3503             for(;;) {
3504                 do {
3505                     next_argstream(nested_list, NULL);
3506                 } while (tok == TOK_PLCHLDR || is_space(tok) ||
3507                          TOK_LINEFEED == tok);
3508     empty_arg:
3509                 /* handle '()' case */
3510                 if (!args && !sa && tok == ')')
3511                     break;
3512                 if (!sa)
3513                     tcc_error("macro '%s' used with too many args",
3514                           get_tok_str(s->v, 0));
3515                 tok_str_new(&str);
3516                 parlevel = spc = 0;
3517                 /* NOTE: non zero sa->t indicates VA_ARGS */
3518                 while ((parlevel > 0 ||
3519                         (tok != ')' &&
3520                          (tok != ',' || sa->type.t)))) {
3521                     if (tok == TOK_EOF || tok == 0)
3522                         break;
3523                     if (tok == '(')
3524                         parlevel++;
3525                     else if (tok == ')')
3526                         parlevel--;
3527                     if (tok == TOK_LINEFEED)
3528                         tok = ' ';
3529                     if (!check_space(tok, &spc))
3530                         tok_str_add2(&str, tok, &tokc);
3531                     next_argstream(nested_list, NULL);
3532                 }
3533                 if (parlevel)
3534                     expect(")");
3535                 str.len -= spc;
3536                 tok_str_add(&str, -1);
3537                 tok_str_add(&str, 0);
3538                 sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
3539                 sa1->d = str.str;
3540                 sa = sa->next;
3541                 if (tok == ')') {
3542                     /* special case for gcc var args: add an empty
3543                        var arg argument if it is omitted */
3544                     if (sa && sa->type.t && gnu_ext)
3545                         goto empty_arg;
3546                     break;
3547                 }
3548                 if (tok != ',')
3549                     expect(",");
3550             }
3551             if (sa) {
3552                 tcc_error("macro '%s' used with too few args",
3553                       get_tok_str(s->v, 0));
3554             }
3555
3556             /* now subst each arg */
3557             mstr = macro_arg_subst(nested_list, mstr, args);
3558             /* free memory */
3559             sa = args;
3560             while (sa) {
3561                 sa1 = sa->prev;
3562                 tok_str_free_str(sa->d);
3563                 if (sa->next) {
3564                     tok_str_free_str(sa->next->d);
3565                     sym_free(sa->next);
3566                 }
3567                 sym_free(sa);
3568                 sa = sa1;
3569             }
3570             parse_flags = saved_parse_flags;
3571         }
3572
3573         sym_push2(nested_list, s->v, 0, 0);
3574         parse_flags = saved_parse_flags;
3575         joined_str = macro_twosharps(mstr);
3576         macro_subst(tok_str, nested_list, joined_str ? joined_str : mstr);
3577
3578         /* pop nested defined symbol */
3579         sa1 = *nested_list;
3580         *nested_list = sa1->prev;
3581         sym_free(sa1);
3582         if (joined_str)
3583             tok_str_free_str(joined_str);
3584         if (mstr != s->d)
3585             tok_str_free_str(mstr);
3586     }
3587     return 0;
3588 }
3589
3590 /* do macro substitution of macro_str and add result to
3591    (tok_str,tok_len). 'nested_list' is the list of all macros we got
3592    inside to avoid recursing. */
3593 static void macro_subst(
3594     TokenString *tok_str,
3595     Sym **nested_list,
3596     const int *macro_str
3597     )
3598 {
3599     Sym *s;
3600     int t, spc, nosubst;
3601     CValue cval;
3602
3603     spc = nosubst = 0;
3604
3605     while (1) {
3606         TOK_GET(&t, &macro_str, &cval);
3607         if (t <= 0)
3608             break;
3609
3610         if (t >= TOK_IDENT && 0 == nosubst) {
3611             s = define_find(t);
3612             if (s == NULL)
3613                 goto no_subst;
3614
3615             /* if nested substitution, do nothing */
3616             if (sym_find2(*nested_list, t)) {
3617                 /* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
3618                 tok_str_add2(tok_str, TOK_NOSUBST, NULL);
3619                 goto no_subst;
3620             }
3621
3622             {
3623                 TokenString *str = tok_str_alloc();
3624                 str->str = (int*)macro_str;
3625                 begin_macro(str, 2);
3626
3627                 tok = t;
3628                 macro_subst_tok(tok_str, nested_list, s);
3629
3630                 if (macro_stack != str) {
3631                     /* already finished by reading function macro arguments */
3632                     break;
3633                 }
3634
3635                 macro_str = macro_ptr;
3636                 end_macro ();
3637             }
3638             if (tok_str->len)
3639                 spc = is_space(t = tok_str->str[tok_str->lastlen]);
3640         } else {
3641 no_subst:
3642             if (!check_space(t, &spc))
3643                 tok_str_add2(tok_str, t, &cval);
3644
3645             if (nosubst) {
3646                 if (nosubst > 1 && (spc || (++nosubst == 3 && t == '(')))
3647                     continue;
3648                 nosubst = 0;
3649             }
3650             if (t == TOK_NOSUBST)
3651                 nosubst = 1;
3652         }
3653         /* GCC supports 'defined' as result of a macro substitution */
3654         if (t == TOK_DEFINED && pp_expr)
3655             nosubst = 2;
3656     }
3657 }
3658
3659 /* return next token without macro substitution. Can read input from
3660    macro_ptr buffer */
3661 static void next_nomacro(void)
3662 {
3663     int t;
3664     if (macro_ptr) {
3665  redo:
3666         t = *macro_ptr;
3667         if (TOK_HAS_VALUE(t)) {
3668             tok_get(&tok, &macro_ptr, &tokc);
3669             if (t == TOK_LINENUM) {
3670                 file->line_num = tokc.i;
3671                 goto redo;
3672             }
3673         } else {
3674             macro_ptr++;
3675             if (t < TOK_IDENT) {
3676                 if (!(parse_flags & PARSE_FLAG_SPACES)
3677                     && (isidnum_table[t - CH_EOF] & IS_SPC))
3678                     goto redo;
3679             }
3680             tok = t;
3681         }
3682     } else {
3683         next_nomacro1();
3684     }
3685 }
3686
3687 /* return next token with macro substitution */
3688 ST_FUNC void next(void)
3689 {
3690     int t;
3691  redo:
3692     next_nomacro();
3693     t = tok;
3694     if (macro_ptr) {
3695         if (!TOK_HAS_VALUE(t)) {
3696             if (t == TOK_NOSUBST || t == TOK_PLCHLDR) {
3697                 /* discard preprocessor markers */
3698                 goto redo;
3699             } else if (t == 0) {
3700                 /* end of macro or unget token string */
3701                 end_macro();
3702                 goto redo;
3703             } else if (t == '\\') {
3704                 if (!(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
3705                     tcc_error("stray '\\' in program");
3706             }
3707             return;
3708         }
3709     } else if (t >= TOK_IDENT && (parse_flags & PARSE_FLAG_PREPROCESS)) {
3710         /* if reading from file, try to substitute macros */
3711         Sym *s = define_find(t);
3712         if (s) {
3713             Sym *nested_list = NULL;
3714             tokstr_buf.len = 0;
3715             macro_subst_tok(&tokstr_buf, &nested_list, s);
3716             tok_str_add(&tokstr_buf, 0);
3717             begin_macro(&tokstr_buf, 0);
3718             goto redo;
3719         }
3720         return;
3721     }
3722     /* convert preprocessor tokens into C tokens */
3723     if (t == TOK_PPNUM) {
3724         if  (parse_flags & PARSE_FLAG_TOK_NUM)
3725             parse_number((char *)tokc.str.data);
3726     } else if (t == TOK_PPSTR) {
3727         if (parse_flags & PARSE_FLAG_TOK_STR)
3728             parse_string((char *)tokc.str.data, tokc.str.size - 1);
3729     }
3730 }
3731
3732 /* push back current token and set current token to 'last_tok'. Only
3733    identifier case handled for labels. */
3734 ST_INLN void unget_tok(int last_tok)
3735 {
3736
3737     TokenString *str = tok_str_alloc();
3738     tok_str_add2(str, tok, &tokc);
3739     tok_str_add(str, 0);
3740     begin_macro(str, 1);
3741     tok = last_tok;
3742 }
3743
3744 /* ------------------------------------------------------------------------- */
3745 /* init preprocessor */
3746
3747 static const char * const target_os_defs =
3748 #ifdef TCC_TARGET_PE
3749     "_WIN32\0"
3750 # if PTR_SIZE == 8
3751     "_WIN64\0"
3752 # endif
3753 #else
3754 # if defined TCC_TARGET_MACHO
3755     "__APPLE__\0"
3756 # elif TARGETOS_FreeBSD
3757     "__FreeBSD__ 12\0"
3758 # elif TARGETOS_FreeBSD_kernel
3759     "__FreeBSD_kernel__\0"
3760 # elif TARGETOS_NetBSD
3761     "__NetBSD__\0"
3762 # elif TARGETOS_OpenBSD
3763     "__OpenBSD__\0"
3764 # else
3765     "__linux__\0"
3766     "__linux\0"
3767 #  if TARGETOS_ANDROID
3768     "__ANDROID__\0"
3769 #  endif
3770 # endif
3771     "__unix__\0"
3772     "__unix\0"
3773 #endif
3774     ;
3775
3776 static void putdef(CString *cs, const char *p)
3777 {
3778     cstr_printf(cs, "#define %s%s\n", p, &" 1"[!!strchr(p, ' ')*2]);
3779 }
3780
3781 static void putdefs(CString *cs, const char *p)
3782 {
3783     while (*p)
3784         putdef(cs, p), p = strchr(p, 0) + 1;
3785 }
3786
3787 static void tcc_predefs(TCCState *s1, CString *cs, int is_asm)
3788 {
3789     int a, b, c;
3790
3791     sscanf(TCC_VERSION, "%d.%d.%d", &a, &b, &c);
3792     cstr_printf(cs, "#define __TINYC__ %d\n", a*10000 + b*100 + c);
3793
3794     putdefs(cs, target_machine_defs);
3795     putdefs(cs, target_os_defs);
3796
3797 #ifdef TCC_TARGET_ARM
3798     if (s1->float_abi == ARM_HARD_FLOAT)
3799       putdef(cs, "__ARM_PCS_VFP");
3800 #endif
3801     if (is_asm)
3802       putdef(cs, "__ASSEMBLER__");
3803     if (s1->output_type == TCC_OUTPUT_PREPROCESS)
3804       putdef(cs, "__TCC_PP__");
3805     if (s1->output_type == TCC_OUTPUT_MEMORY)
3806       putdef(cs, "__TCC_RUN__");
3807     if (s1->char_is_unsigned)
3808       putdef(cs, "__CHAR_UNSIGNED__");
3809     if (s1->optimize > 0)
3810       putdef(cs, "__OPTIMIZE__");
3811     if (s1->option_pthread)
3812       putdef(cs, "_REENTRANT");
3813     if (s1->leading_underscore)
3814       putdef(cs, "__leading_underscore");
3815 #ifdef CONFIG_TCC_BCHECK
3816     if (s1->do_bounds_check)
3817       putdef(cs, "__BOUNDS_CHECKING_ON");
3818 #endif
3819 #ifdef CONFIG_TCC_BACKTRACE
3820     if (s1->do_backtrace)
3821       putdef(cs, "__TCC_BACKTRACE_ENABLED__");
3822 #endif
3823     cstr_printf(cs, "#define __SIZEOF_POINTER__ %d\n", PTR_SIZE);
3824     cstr_printf(cs, "#define __SIZEOF_LONG__ %d\n", LONG_SIZE);
3825     if (!is_asm) {
3826       putdef(cs, "__STDC__");
3827       cstr_printf(cs, "#define __STDC_VERSION__ %dL\n", s1->cversion);
3828       cstr_cat(cs,
3829         /* load more predefs and __builtins */
3830 #if CONFIG_TCC_PREDEFS
3831         #include "tccdefs_.h" /* include as strings */
3832 #else
3833         "#include <tccdefs.h>\n" /* load at runtime */
3834 #endif
3835         , -1);
3836     }
3837     cstr_printf(cs, "#define __BASE_FILE__ \"%s\"\n", file->filename);
3838 }
3839
3840 ST_FUNC void preprocess_start(TCCState *s1, int filetype)
3841 {
3842     int is_asm = !!(filetype & (AFF_TYPE_ASM|AFF_TYPE_ASMPP));
3843     CString cstr;
3844
3845     tccpp_new(s1);
3846
3847     s1->include_stack_ptr = s1->include_stack;
3848     s1->ifdef_stack_ptr = s1->ifdef_stack;
3849     file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
3850     pp_expr = 0;
3851     pp_counter = 0;
3852     pp_debug_tok = pp_debug_symv = 0;
3853     pp_once++;
3854     s1->pack_stack[0] = 0;
3855     s1->pack_stack_ptr = s1->pack_stack;
3856
3857     set_idnum('$', !is_asm && s1->dollars_in_identifiers ? IS_ID : 0);
3858     set_idnum('.', is_asm ? IS_ID : 0);
3859
3860     if (!(filetype & AFF_TYPE_ASM)) {
3861         cstr_new(&cstr);
3862         tcc_predefs(s1, &cstr, is_asm);
3863         if (s1->cmdline_defs.size)
3864           cstr_cat(&cstr, s1->cmdline_defs.data, s1->cmdline_defs.size);
3865         if (s1->cmdline_incl.size)
3866           cstr_cat(&cstr, s1->cmdline_incl.data, s1->cmdline_incl.size);
3867         //printf("%s\n", (char*)cstr.data);
3868         *s1->include_stack_ptr++ = file;
3869         tcc_open_bf(s1, "<command line>", cstr.size);
3870         memcpy(file->buffer, cstr.data, cstr.size);
3871         cstr_free(&cstr);
3872     }
3873
3874     parse_flags = is_asm ? PARSE_FLAG_ASM_FILE : 0;
3875     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
3876 }
3877
3878 /* cleanup from error/setjmp */
3879 ST_FUNC void preprocess_end(TCCState *s1)
3880 {
3881     while (macro_stack)
3882         end_macro();
3883     macro_ptr = NULL;
3884     while (file)
3885         tcc_close();
3886     tccpp_delete(s1);
3887 }
3888
3889 ST_FUNC void tccpp_new(TCCState *s)
3890 {
3891     int i, c;
3892     const char *p, *r;
3893
3894     /* init isid table */
3895     for(i = CH_EOF; i<128; i++)
3896         set_idnum(i,
3897             is_space(i) ? IS_SPC
3898             : isid(i) ? IS_ID
3899             : isnum(i) ? IS_NUM
3900             : 0);
3901
3902     for(i = 128; i<256; i++)
3903         set_idnum(i, IS_ID);
3904
3905     /* init allocators */
3906     tal_new(&toksym_alloc, TOKSYM_TAL_LIMIT, TOKSYM_TAL_SIZE);
3907     tal_new(&tokstr_alloc, TOKSTR_TAL_LIMIT, TOKSTR_TAL_SIZE);
3908
3909     memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
3910     memset(s->cached_includes_hash, 0, sizeof s->cached_includes_hash);
3911
3912     cstr_new(&cstr_buf);
3913     cstr_realloc(&cstr_buf, STRING_MAX_SIZE);
3914     tok_str_new(&tokstr_buf);
3915     tok_str_realloc(&tokstr_buf, TOKSTR_MAX_SIZE);
3916
3917     tok_ident = TOK_IDENT;
3918     p = tcc_keywords;
3919     while (*p) {
3920         r = p;
3921         for(;;) {
3922             c = *r++;
3923             if (c == '\0')
3924                 break;
3925         }
3926         tok_alloc(p, r - p - 1);
3927         p = r;
3928     }
3929
3930     /* we add dummy defines for some special macros to speed up tests
3931        and to have working defined() */
3932     define_push(TOK___LINE__, MACRO_OBJ, NULL, NULL);
3933     define_push(TOK___FILE__, MACRO_OBJ, NULL, NULL);
3934     define_push(TOK___DATE__, MACRO_OBJ, NULL, NULL);
3935     define_push(TOK___TIME__, MACRO_OBJ, NULL, NULL);
3936     define_push(TOK___COUNTER__, MACRO_OBJ, NULL, NULL);
3937 }
3938
3939 ST_FUNC void tccpp_delete(TCCState *s)
3940 {
3941     int i, n;
3942
3943     dynarray_reset(&s->cached_includes, &s->nb_cached_includes);
3944
3945     /* free tokens */
3946     n = tok_ident - TOK_IDENT;
3947     if (n > total_idents)
3948         total_idents = n;
3949     for(i = 0; i < n; i++)
3950         tal_free(toksym_alloc, table_ident[i]);
3951     tcc_free(table_ident);
3952     table_ident = NULL;
3953
3954     /* free static buffers */
3955     cstr_free(&tokcstr);
3956     cstr_free(&cstr_buf);
3957     cstr_free(&macro_equal_buf);
3958     tok_str_free_str(tokstr_buf.str);
3959
3960     /* free allocators */
3961     tal_delete(toksym_alloc);
3962     toksym_alloc = NULL;
3963     tal_delete(tokstr_alloc);
3964     tokstr_alloc = NULL;
3965 }
3966
3967 /* ------------------------------------------------------------------------- */
3968 /* tcc -E [-P[1]] [-dD} support */
3969
3970 static void tok_print(const char *msg, const int *str)
3971 {
3972     FILE *fp;
3973     int t, s = 0;
3974     CValue cval;
3975
3976     fp = tcc_state->ppfp;
3977     fprintf(fp, "%s", msg);
3978     while (str) {
3979         TOK_GET(&t, &str, &cval);
3980         if (!t)
3981             break;
3982         fprintf(fp, &" %s"[s], get_tok_str(t, &cval)), s = 1;
3983     }
3984     fprintf(fp, "\n");
3985 }
3986
3987 static void pp_line(TCCState *s1, BufferedFile *f, int level)
3988 {
3989     int d = f->line_num - f->line_ref;
3990
3991     if (s1->dflag & 4)
3992         return;
3993
3994     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_NONE) {
3995         ;
3996     } else if (level == 0 && f->line_ref && d < 8) {
3997         while (d > 0)
3998             fputs("\n", s1->ppfp), --d;
3999     } else if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_STD) {
4000         fprintf(s1->ppfp, "#line %d \"%s\"\n", f->line_num, f->filename);
4001     } else {
4002         fprintf(s1->ppfp, "# %d \"%s\"%s\n", f->line_num, f->filename,
4003             level > 0 ? " 1" : level < 0 ? " 2" : "");
4004     }
4005     f->line_ref = f->line_num;
4006 }
4007
4008 static void define_print(TCCState *s1, int v)
4009 {
4010     FILE *fp;
4011     Sym *s;
4012
4013     s = define_find(v);
4014     if (NULL == s || NULL == s->d)
4015         return;
4016
4017     fp = s1->ppfp;
4018     fprintf(fp, "#define %s", get_tok_str(v, NULL));
4019     if (s->type.t == MACRO_FUNC) {
4020         Sym *a = s->next;
4021         fprintf(fp,"(");
4022         if (a)
4023             for (;;) {
4024                 fprintf(fp,"%s", get_tok_str(a->v & ~SYM_FIELD, NULL));
4025                 if (!(a = a->next))
4026                     break;
4027                 fprintf(fp,",");
4028             }
4029         fprintf(fp,")");
4030     }
4031     tok_print("", s->d);
4032 }
4033
4034 static void pp_debug_defines(TCCState *s1)
4035 {
4036     int v, t;
4037     const char *vs;
4038     FILE *fp;
4039
4040     t = pp_debug_tok;
4041     if (t == 0)
4042         return;
4043
4044     file->line_num--;
4045     pp_line(s1, file, 0);
4046     file->line_ref = ++file->line_num;
4047
4048     fp = s1->ppfp;
4049     v = pp_debug_symv;
4050     vs = get_tok_str(v, NULL);
4051     if (t == TOK_DEFINE) {
4052         define_print(s1, v);
4053     } else if (t == TOK_UNDEF) {
4054         fprintf(fp, "#undef %s\n", vs);
4055     } else if (t == TOK_push_macro) {
4056         fprintf(fp, "#pragma push_macro(\"%s\")\n", vs);
4057     } else if (t == TOK_pop_macro) {
4058         fprintf(fp, "#pragma pop_macro(\"%s\")\n", vs);
4059     }
4060     pp_debug_tok = 0;
4061 }
4062
4063 static void pp_debug_builtins(TCCState *s1)
4064 {
4065     int v;
4066     for (v = TOK_IDENT; v < tok_ident; ++v)
4067         define_print(s1, v);
4068 }
4069
4070 /* Add a space between tokens a and b to avoid unwanted textual pasting */
4071 static int pp_need_space(int a, int b)
4072 {
4073     return 'E' == a ? '+' == b || '-' == b
4074         : '+' == a ? TOK_INC == b || '+' == b
4075         : '-' == a ? TOK_DEC == b || '-' == b
4076         : a >= TOK_IDENT ? b >= TOK_IDENT
4077         : a == TOK_PPNUM ? b >= TOK_IDENT
4078         : 0;
4079 }
4080
4081 /* maybe hex like 0x1e */
4082 static int pp_check_he0xE(int t, const char *p)
4083 {
4084     if (t == TOK_PPNUM && toup(strchr(p, 0)[-1]) == 'E')
4085         return 'E';
4086     return t;
4087 }
4088
4089 /* Preprocess the current file */
4090 ST_FUNC int tcc_preprocess(TCCState *s1)
4091 {
4092     BufferedFile **iptr;
4093     int token_seen, spcs, level;
4094     const char *p;
4095     char white[400];
4096
4097     parse_flags = PARSE_FLAG_PREPROCESS
4098                 | (parse_flags & PARSE_FLAG_ASM_FILE)
4099                 | PARSE_FLAG_LINEFEED
4100                 | PARSE_FLAG_SPACES
4101                 | PARSE_FLAG_ACCEPT_STRAYS
4102                 ;
4103     /* Credits to Fabrice Bellard's initial revision to demonstrate its
4104        capability to compile and run itself, provided all numbers are
4105        given as decimals. tcc -E -P10 will do. */
4106     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_P10)
4107         parse_flags |= PARSE_FLAG_TOK_NUM, s1->Pflag = 1;
4108
4109     if (s1->do_bench) {
4110         /* for PP benchmarks */
4111         do next(); while (tok != TOK_EOF);
4112         return 0;
4113     }
4114
4115     if (s1->dflag & 1) {
4116         pp_debug_builtins(s1);
4117         s1->dflag &= ~1;
4118     }
4119
4120     token_seen = TOK_LINEFEED, spcs = 0, level = 0;
4121     if (file->prev)
4122         pp_line(s1, file->prev, level++);
4123     pp_line(s1, file, level);
4124     for (;;) {
4125         iptr = s1->include_stack_ptr;
4126         next();
4127         if (tok == TOK_EOF)
4128             break;
4129
4130         level = s1->include_stack_ptr - iptr;
4131         if (level) {
4132             if (level > 0)
4133                 pp_line(s1, *iptr, 0);
4134             pp_line(s1, file, level);
4135         }
4136         if (s1->dflag & 7) {
4137             pp_debug_defines(s1);
4138             if (s1->dflag & 4)
4139                 continue;
4140         }
4141
4142         if (is_space(tok)) {
4143             if (spcs < sizeof white - 1)
4144                 white[spcs++] = tok;
4145             continue;
4146         } else if (tok == TOK_LINEFEED) {
4147             spcs = 0;
4148             if (token_seen == TOK_LINEFEED)
4149                 continue;
4150             ++file->line_ref;
4151         } else if (token_seen == TOK_LINEFEED) {
4152             pp_line(s1, file, 0);
4153         } else if (spcs == 0 && pp_need_space(token_seen, tok)) {
4154             white[spcs++] = ' ';
4155         }
4156
4157         white[spcs] = 0, fputs(white, s1->ppfp), spcs = 0;
4158         fputs(p = get_tok_str(tok, &tokc), s1->ppfp);
4159         token_seen = pp_check_he0xE(tok, p);
4160     }
4161     return 0;
4162 }
4163
4164 /* ------------------------------------------------------------------------- */