tccpp.c

   1 /*
   2  *  TCC - Tiny C Compiler
   3  *
   4  *  Copyright (c) 2001-2004 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #define USING_GLOBALS
  22 #include "tcc.h"
  23
  24 /* #define to 1 to enable (see parse_pp_string()) */
  25 #define ACCEPT_LF_IN_STRINGS 0
  26
  27 /********************************************************/
  28 /* global variables */
  29
  30 ST_DATA int tok_flags;
  31 ST_DATA int parse_flags;
  32
  33 ST_DATA struct BufferedFile *file;
  34 ST_DATA int tok;
  35 ST_DATA CValue tokc;
  36 ST_DATA const int *macro_ptr;
  37 ST_DATA CString tokcstr; /* current parsed string, if any */
  38
  39 /* display benchmark infos */
  40 ST_DATA int tok_ident;
  41 ST_DATA TokenSym **table_ident;
  42
  43 /* ------------------------------------------------------------------------- */
  44
  45 static TokenSym *hash_ident[TOK_HASH_SIZE];
  46 static char token_buf[STRING_MAX_SIZE + 1];
  47 static CString cstr_buf;
  48 static CString macro_equal_buf;
  49 static TokenString tokstr_buf;
  50 static unsigned char isidnum_table[256 - CH_EOF];
  51 static int pp_debug_tok, pp_debug_symv;
  52 static int pp_once;
  53 static int pp_expr;
  54 static int pp_counter;
  55 static void tok_print(const char *msg, const int *str);
  56
  57 static struct TinyAlloc *toksym_alloc;
  58 static struct TinyAlloc *tokstr_alloc;
  59
  60 static TokenString *macro_stack;
  61
  62 static const char tcc_keywords[] =
  63 #define DEF(id, str) str "\0"
  64 #include "tcctok.h"
  65 #undef DEF
  66 ;
  67
  68 /* WARNING: the content of this string encodes token numbers */
  69 static const unsigned char tok_two_chars[] =
  70 /* outdated -- gr
  71     "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253"
  72     "-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";
  73 */{
  74     '<','=', TOK_LE,
  75     '>','=', TOK_GE,
  76     '!','=', TOK_NE,
  77     '&','&', TOK_LAND,
  78     '|','|', TOK_LOR,
  79     '+','+', TOK_INC,
  80     '-','-', TOK_DEC,
  81     '=','=', TOK_EQ,
  82     '<','<', TOK_SHL,
  83     '>','>', TOK_SAR,
  84     '+','=', TOK_A_ADD,
  85     '-','=', TOK_A_SUB,
  86     '*','=', TOK_A_MUL,
  87     '/','=', TOK_A_DIV,
  88     '%','=', TOK_A_MOD,
  89     '&','=', TOK_A_AND,
  90     '^','=', TOK_A_XOR,
  91     '|','=', TOK_A_OR,
  92     '-','>', TOK_ARROW,
  93     '.','.', TOK_TWODOTS,
  94     '#','#', TOK_TWOSHARPS,
  95     '#','#', TOK_PPJOIN,
  96     0
  97 };
  98
  99 static void next_nomacro(void);
 100
 101 ST_FUNC void skip(int c)
 102 {
 103     if (tok != c)
 104         tcc_error("'%c' expected (got \"%s\")", c, get_tok_str(tok, &tokc));
 105     next();
 106 }
 107
 108 ST_FUNC void expect(const char *msg)
 109 {
 110     tcc_error("%s expected", msg);
 111 }
 112
 113 /* ------------------------------------------------------------------------- */
 114 /* Custom allocator for tiny objects */
 115
 116 #define USE_TAL
 117
 118 #ifndef USE_TAL
 119 #define tal_free(al, p) tcc_free(p)
 120 #define tal_realloc(al, p, size) tcc_realloc(p, size)
 121 #define tal_new(a,b,c)
 122 #define tal_delete(a)
 123 #else
 124 #if !defined(MEM_DEBUG)
 125 #define tal_free(al, p) tal_free_impl(al, p)
 126 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size)
 127 #define TAL_DEBUG_PARAMS
 128 #else
 129 #define TAL_DEBUG 1
 130 //#define TAL_INFO 1 /* collect and dump allocators stats */
 131 #define tal_free(al, p) tal_free_impl(al, p, __FILE__, __LINE__)
 132 #define tal_realloc(al, p, size) tal_realloc_impl(&al, p, size, __FILE__, __LINE__)
 133 #define TAL_DEBUG_PARAMS , const char *file, int line
 134 #define TAL_DEBUG_FILE_LEN 40
 135 #endif
 136
 137 #define TOKSYM_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenSym in table_ident */
 138 #define TOKSTR_TAL_SIZE     (768 * 1024) /* allocator for tiny TokenString instances */
 139 #define CSTR_TAL_SIZE       (256 * 1024) /* allocator for tiny CString instances */
 140 #define TOKSYM_TAL_LIMIT    256 /* prefer unique limits to distinguish allocators debug msgs */
 141 #define TOKSTR_TAL_LIMIT    128 /* 32 * sizeof(int) */
 142 #define CSTR_TAL_LIMIT      1024
 143
 144 typedef struct TinyAlloc {
 145     unsigned  limit;
 146     unsigned  size;
 147     uint8_t *buffer;
 148     uint8_t *p;
 149     unsigned  nb_allocs;
 150     struct TinyAlloc *next, *top;
 151 #ifdef TAL_INFO
 152     unsigned  nb_peak;
 153     unsigned  nb_total;
 154     unsigned  nb_missed;
 155     uint8_t *peak_p;
 156 #endif
 157 } TinyAlloc;
 158
 159 typedef struct tal_header_t {
 160     unsigned  size;
 161 #ifdef TAL_DEBUG
 162     int     line_num; /* negative line_num used for double free check */
 163     char    file_name[TAL_DEBUG_FILE_LEN + 1];
 164 #endif
 165 } tal_header_t;
 166
 167 /* ------------------------------------------------------------------------- */
 168
 169 static TinyAlloc *tal_new(TinyAlloc **pal, unsigned limit, unsigned size)
 170 {
 171     TinyAlloc *al = tcc_mallocz(sizeof(TinyAlloc));
 172     al->p = al->buffer = tcc_malloc(size);
 173     al->limit = limit;
 174     al->size = size;
 175     if (pal) *pal = al;
 176     return al;
 177 }
 178
 179 static void tal_delete(TinyAlloc *al)
 180 {
 181     TinyAlloc *next;
 182
 183 tail_call:
 184     if (!al)
 185         return;
 186 #ifdef TAL_INFO
 187     fprintf(stderr, "limit=%5d, size=%5g MB, nb_peak=%6d, nb_total=%8d, nb_missed=%6d, usage=%5.1f%%\n",
 188             al->limit, al->size / 1024.0 / 1024.0, al->nb_peak, al->nb_total, al->nb_missed,
 189             (al->peak_p - al->buffer) * 100.0 / al->size);
 190 #endif
 191 #ifdef TAL_DEBUG
 192     if (al->nb_allocs > 0) {
 193         uint8_t *p;
 194         fprintf(stderr, "TAL_DEBUG: memory leak %d chunk(s) (limit= %d)\n",
 195                 al->nb_allocs, al->limit);
 196         p = al->buffer;
 197         while (p < al->p) {
 198             tal_header_t *header = (tal_header_t *)p;
 199             if (header->line_num > 0) {
 200                 fprintf(stderr, "%s:%d: chunk of %d bytes leaked\n",
 201                         header->file_name, header->line_num, header->size);
 202             }
 203             p += header->size + sizeof(tal_header_t);
 204         }
 205 #if MEM_DEBUG-0 == 2
 206         exit(2);
 207 #endif
 208     }
 209 #endif
 210     next = al->next;
 211     tcc_free(al->buffer);
 212     tcc_free(al);
 213     al = next;
 214     goto tail_call;
 215 }
 216
 217 static void tal_free_impl(TinyAlloc *al, void *p TAL_DEBUG_PARAMS)
 218 {
 219     if (!p)
 220         return;
 221 tail_call:
 222     if (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size) {
 223 #ifdef TAL_DEBUG
 224         tal_header_t *header = (((tal_header_t *)p) - 1);
 225         if (header->line_num < 0) {
 226             fprintf(stderr, "%s:%d: TAL_DEBUG: double frees chunk from\n",
 227                     file, line);
 228             fprintf(stderr, "%s:%d: %d bytes\n",
 229                     header->file_name, (int)-header->line_num, (int)header->size);
 230         } else
 231             header->line_num = -header->line_num;
 232 #endif
 233         al->nb_allocs--;
 234         if (!al->nb_allocs)
 235             al->p = al->buffer;
 236     } else if (al->next) {
 237         al = al->next;
 238         goto tail_call;
 239     }
 240     else
 241         tcc_free(p);
 242 }
 243
 244 static void *tal_realloc_impl(TinyAlloc **pal, void *p, unsigned size TAL_DEBUG_PARAMS)
 245 {
 246     tal_header_t *header;
 247     void *ret;
 248     int is_own;
 249     unsigned adj_size = (size + 3) & -4;
 250     TinyAlloc *al = *pal;
 251
 252 tail_call:
 253     is_own = (al->buffer <= (uint8_t *)p && (uint8_t *)p < al->buffer + al->size);
 254     if ((!p || is_own) && size <= al->limit) {
 255         if (al->p - al->buffer + adj_size + sizeof(tal_header_t) < al->size) {
 256             header = (tal_header_t *)al->p;
 257             header->size = adj_size;
 258 #ifdef TAL_DEBUG
 259             { int ofs = strlen(file) - TAL_DEBUG_FILE_LEN;
 260             strncpy(header->file_name, file + (ofs > 0 ? ofs : 0), TAL_DEBUG_FILE_LEN);
 261             header->file_name[TAL_DEBUG_FILE_LEN] = 0;
 262             header->line_num = line; }
 263 #endif
 264             ret = al->p + sizeof(tal_header_t);
 265             al->p += adj_size + sizeof(tal_header_t);
 266             if (is_own) {
 267                 header = (((tal_header_t *)p) - 1);
 268                 if (p) memcpy(ret, p, header->size);
 269 #ifdef TAL_DEBUG
 270                 header->line_num = -header->line_num;
 271 #endif
 272             } else {
 273                 al->nb_allocs++;
 274             }
 275 #ifdef TAL_INFO
 276             if (al->nb_peak < al->nb_allocs)
 277                 al->nb_peak = al->nb_allocs;
 278             if (al->peak_p < al->p)
 279                 al->peak_p = al->p;
 280             al->nb_total++;
 281 #endif
 282             return ret;
 283         } else if (is_own) {
 284             al->nb_allocs--;
 285             ret = tal_realloc(*pal, 0, size);
 286             header = (((tal_header_t *)p) - 1);
 287             if (p) memcpy(ret, p, header->size);
 288 #ifdef TAL_DEBUG
 289             header->line_num = -header->line_num;
 290 #endif
 291             return ret;
 292         }
 293         if (al->next) {
 294             al = al->next;
 295         } else {
 296             TinyAlloc *bottom = al, *next = al->top ? al->top : al;
 297
 298             al = tal_new(pal, next->limit, next->size * 2);
 299             al->next = next;
 300             bottom->top = al;
 301         }
 302         goto tail_call;
 303     }
 304     if (is_own) {
 305         al->nb_allocs--;
 306         ret = tcc_malloc(size);
 307         header = (((tal_header_t *)p) - 1);
 308         if (p) memcpy(ret, p, header->size);
 309 #ifdef TAL_DEBUG
 310         header->line_num = -header->line_num;
 311 #endif
 312     } else if (al->next) {
 313         al = al->next;
 314         goto tail_call;
 315     } else
 316         ret = tcc_realloc(p, size);
 317 #ifdef TAL_INFO
 318     al->nb_missed++;
 319 #endif
 320     return ret;
 321 }
 322
 323 #endif /* USE_TAL */
 324
 325 /* ------------------------------------------------------------------------- */
 326 /* CString handling */
 327 static void cstr_realloc(CString *cstr, int new_size)
 328 {
 329     int size;
 330
 331     size = cstr->size_allocated;
 332     if (size < 8)
 333         size = 8; /* no need to allocate a too small first string */
 334     while (size < new_size)
 335         size = size * 2;
 336     cstr->data = tcc_realloc(cstr->data, size);
 337     cstr->size_allocated = size;
 338 }
 339
 340 /* add a byte */
 341 ST_INLN void cstr_ccat(CString *cstr, int ch)
 342 {
 343     int size;
 344     size = cstr->size + 1;
 345     if (size > cstr->size_allocated)
 346         cstr_realloc(cstr, size);
 347     ((unsigned char *)cstr->data)[size - 1] = ch;
 348     cstr->size = size;
 349 }
 350
 351 ST_INLN char *unicode_to_utf8 (char *b, uint32_t Uc)
 352 {
 353     if (Uc<0x80) *b++=Uc;
 354     else if (Uc<0x800) *b++=192+Uc/64, *b++=128+Uc%64;
 355     else if (Uc-0xd800u<0x800) goto error;
 356     else if (Uc<0x10000) *b++=224+Uc/4096, *b++=128+Uc/64%64, *b++=128+Uc%64;
 357     else if (Uc<0x110000) *b++=240+Uc/262144, *b++=128+Uc/4096%64, *b++=128+Uc/64%64, *b++=128+Uc%64;
 358     else error: tcc_error("0x%x is not a valid universal character", Uc);
 359     return b;
 360 }
 361
 362 /* add a unicode character expanded into utf8 */
 363 ST_INLN void cstr_u8cat(CString *cstr, int ch)
 364 {
 365     char buf[4], *e;
 366     e = unicode_to_utf8(buf, (uint32_t)ch);
 367     cstr_cat(cstr, buf, e - buf);
 368 }
 369
 370 ST_FUNC void cstr_cat(CString *cstr, const char *str, int len)
 371 {
 372     int size;
 373     if (len <= 0)
 374         len = strlen(str) + 1 + len;
 375     size = cstr->size + len;
 376     if (size > cstr->size_allocated)
 377         cstr_realloc(cstr, size);
 378     memmove(((unsigned char *)cstr->data) + cstr->size, str, len);
 379     cstr->size = size;
 380 }
 381
 382 /* add a wide char */
 383 ST_FUNC void cstr_wccat(CString *cstr, int ch)
 384 {
 385     int size;
 386     size = cstr->size + sizeof(nwchar_t);
 387     if (size > cstr->size_allocated)
 388         cstr_realloc(cstr, size);
 389     *(nwchar_t *)(((unsigned char *)cstr->data) + size - sizeof(nwchar_t)) = ch;
 390     cstr->size = size;
 391 }
 392
 393 ST_FUNC void cstr_new(CString *cstr)
 394 {
 395     memset(cstr, 0, sizeof(CString));
 396 }
 397
 398 /* free string and reset it to NULL */
 399 ST_FUNC void cstr_free(CString *cstr)
 400 {
 401     tcc_free(cstr->data);
 402     cstr_new(cstr);
 403 }
 404
 405 /* reset string to empty */
 406 ST_FUNC void cstr_reset(CString *cstr)
 407 {
 408     cstr->size = 0;
 409 }
 410
 411 ST_FUNC int cstr_vprintf(CString *cstr, const char *fmt, va_list ap)
 412 {
 413     va_list v;
 414     int len, size = 80;
 415     for (;;) {
 416         size += cstr->size;
 417         if (size > cstr->size_allocated)
 418             cstr_realloc(cstr, size);
 419         size = cstr->size_allocated - cstr->size;
 420         va_copy(v, ap);
 421         len = vsnprintf((char*)cstr->data + cstr->size, size, fmt, v);
 422         va_end(v);
 423         if (len > 0 && len < size)
 424             break;
 425         size *= 2;
 426     }
 427     cstr->size += len;
 428     return len;
 429 }
 430
 431 ST_FUNC int cstr_printf(CString *cstr, const char *fmt, ...)
 432 {
 433     va_list ap; int len;
 434     va_start(ap, fmt);
 435     len = cstr_vprintf(cstr, fmt, ap);
 436     va_end(ap);
 437     return len;
 438 }
 439
 440 /* XXX: unicode ? */
 441 static void add_char(CString *cstr, int c)
 442 {
 443     if (c == '\'' || c == '\"' || c == '\\') {
 444         /* XXX: could be more precise if char or string */
 445         cstr_ccat(cstr, '\\');
 446     }
 447     if (c >= 32 && c <= 126) {
 448         cstr_ccat(cstr, c);
 449     } else {
 450         cstr_ccat(cstr, '\\');
 451         if (c == '\n') {
 452             cstr_ccat(cstr, 'n');
 453         } else {
 454             cstr_ccat(cstr, '0' + ((c >> 6) & 7));
 455             cstr_ccat(cstr, '0' + ((c >> 3) & 7));
 456             cstr_ccat(cstr, '0' + (c & 7));
 457         }
 458     }
 459 }
 460
 461 /* ------------------------------------------------------------------------- */
 462 /* allocate a new token */
 463 static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
 464 {
 465     TokenSym *ts, **ptable;
 466     int i;
 467
 468     if (tok_ident >= SYM_FIRST_ANOM)
 469         tcc_error("memory full (symbols)");
 470
 471     /* expand token table if needed */
 472     i = tok_ident - TOK_IDENT;
 473     if ((i % TOK_ALLOC_INCR) == 0) {
 474         ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
 475         table_ident = ptable;
 476     }
 477
 478     ts = tal_realloc(toksym_alloc, 0, sizeof(TokenSym) + len);
 479     table_ident[i] = ts;
 480     ts->tok = tok_ident++;
 481     ts->sym_define = NULL;
 482     ts->sym_label = NULL;
 483     ts->sym_struct = NULL;
 484     ts->sym_identifier = NULL;
 485     ts->len = len;
 486     ts->hash_next = NULL;
 487     memcpy(ts->str, str, len);
 488     ts->str[len] = '\0';
 489     *pts = ts;
 490     return ts;
 491 }
 492
 493 #define TOK_HASH_INIT 1
 494 #define TOK_HASH_FUNC(h, c) ((h) + ((h) << 5) + ((h) >> 27) + (c))
 495
 496
 497 /* find a token and add it if not found */
 498 ST_FUNC TokenSym *tok_alloc(const char *str, int len)
 499 {
 500     TokenSym *ts, **pts;
 501     int i;
 502     unsigned int h;
 503
 504     h = TOK_HASH_INIT;
 505     for(i=0;i<len;i++)
 506         h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
 507     h &= (TOK_HASH_SIZE - 1);
 508
 509     pts = &hash_ident[h];
 510     for(;;) {
 511         ts = *pts;
 512         if (!ts)
 513             break;
 514         if (ts->len == len && !memcmp(ts->str, str, len))
 515             return ts;
 516         pts = &(ts->hash_next);
 517     }
 518     return tok_alloc_new(pts, str, len);
 519 }
 520
 521 ST_FUNC int tok_alloc_const(const char *str)
 522 {
 523     return tok_alloc(str, strlen(str))->tok;
 524 }
 525
 526
 527 /* XXX: buffer overflow */
 528 /* XXX: float tokens */
 529 ST_FUNC const char *get_tok_str(int v, CValue *cv)
 530 {
 531     char *p;
 532     int i, len;
 533
 534     cstr_reset(&cstr_buf);
 535     p = cstr_buf.data;
 536
 537     switch(v) {
 538     case TOK_CINT:
 539     case TOK_CUINT:
 540     case TOK_CLONG:
 541     case TOK_CULONG:
 542     case TOK_CLLONG:
 543     case TOK_CULLONG:
 544         /* XXX: not quite exact, but only useful for testing  */
 545 #ifdef _WIN32
 546         sprintf(p, "%u", (unsigned)cv->i);
 547 #else
 548         sprintf(p, "%llu", (unsigned long long)cv->i);
 549 #endif
 550         break;
 551     case TOK_LCHAR:
 552         cstr_ccat(&cstr_buf, 'L');
 553     case TOK_CCHAR:
 554         cstr_ccat(&cstr_buf, '\'');
 555         add_char(&cstr_buf, cv->i);
 556         cstr_ccat(&cstr_buf, '\'');
 557         cstr_ccat(&cstr_buf, '\0');
 558         break;
 559     case TOK_PPNUM:
 560     case TOK_PPSTR:
 561         return (char*)cv->str.data;
 562     case TOK_LSTR:
 563         cstr_ccat(&cstr_buf, 'L');
 564     case TOK_STR:
 565         cstr_ccat(&cstr_buf, '\"');
 566         if (v == TOK_STR) {
 567             len = cv->str.size - 1;
 568             for(i=0;i<len;i++)
 569                 add_char(&cstr_buf, ((unsigned char *)cv->str.data)[i]);
 570         } else {
 571             len = (cv->str.size / sizeof(nwchar_t)) - 1;
 572             for(i=0;i<len;i++)
 573                 add_char(&cstr_buf, ((nwchar_t *)cv->str.data)[i]);
 574         }
 575         cstr_ccat(&cstr_buf, '\"');
 576         cstr_ccat(&cstr_buf, '\0');
 577         break;
 578
 579     case TOK_CFLOAT:
 580         cstr_cat(&cstr_buf, "<float>", 0);
 581         break;
 582     case TOK_CDOUBLE:
 583         cstr_cat(&cstr_buf, "<double>", 0);
 584         break;
 585     case TOK_CLDOUBLE:
 586         cstr_cat(&cstr_buf, "<long double>", 0);
 587         break;
 588     case TOK_LINENUM:
 589         cstr_cat(&cstr_buf, "<linenumber>", 0);
 590         break;
 591
 592     /* above tokens have value, the ones below don't */
 593     case TOK_LT:
 594         v = '<';
 595         goto addv;
 596     case TOK_GT:
 597         v = '>';
 598         goto addv;
 599     case TOK_DOTS:
 600         return strcpy(p, "...");
 601     case TOK_A_SHL:
 602         return strcpy(p, "<<=");
 603     case TOK_A_SAR:
 604         return strcpy(p, ">>=");
 605     case TOK_EOF:
 606         return strcpy(p, "<eof>");
 607     default:
 608         if (v < TOK_IDENT) {
 609             /* search in two bytes table */
 610             const unsigned char *q = tok_two_chars;
 611             while (*q) {
 612                 if (q[2] == v) {
 613                     *p++ = q[0];
 614                     *p++ = q[1];
 615                     *p = '\0';
 616                     return cstr_buf.data;
 617                 }
 618                 q += 3;
 619             }
 620         if (v >= 127) {
 621             sprintf(cstr_buf.data, "<%02x>", v);
 622             return cstr_buf.data;
 623         }
 624         addv:
 625             *p++ = v;
 626     case 0: /* nameless anonymous symbol */
 627             *p = '\0';
 628         } else if (v < tok_ident) {
 629             return table_ident[v - TOK_IDENT]->str;
 630         } else if (v >= SYM_FIRST_ANOM) {
 631             /* special name for anonymous symbol */
 632             sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
 633         } else {
 634             /* should never happen */
 635             return NULL;
 636         }
 637         break;
 638     }
 639     return cstr_buf.data;
 640 }
 641
 642 static inline int check_space(int t, int *spc)
 643 {
 644     if (t < 256 && (isidnum_table[t - CH_EOF] & IS_SPC)) {
 645         if (*spc)
 646             return 1;
 647         *spc = 1;
 648     } else
 649         *spc = 0;
 650     return 0;
 651 }
 652
 653 /* return the current character, handling end of block if necessary
 654    (but not stray) */
 655 static int handle_eob(void)
 656 {
 657     BufferedFile *bf = file;
 658     int len;
 659
 660     /* only tries to read if really end of buffer */
 661     if (bf->buf_ptr >= bf->buf_end) {
 662         if (bf->fd >= 0) {
 663 #if defined(PARSE_DEBUG)
 664             len = 1;
 665 #else
 666             len = IO_BUF_SIZE;
 667 #endif
 668             len = read(bf->fd, bf->buffer, len);
 669             if (len < 0)
 670                 len = 0;
 671         } else {
 672             len = 0;
 673         }
 674         total_bytes += len;
 675         bf->buf_ptr = bf->buffer;
 676         bf->buf_end = bf->buffer + len;
 677         *bf->buf_end = CH_EOB;
 678     }
 679     if (bf->buf_ptr < bf->buf_end) {
 680         return bf->buf_ptr[0];
 681     } else {
 682         bf->buf_ptr = bf->buf_end;
 683         return CH_EOF;
 684     }
 685 }
 686
 687 /* read next char from current input file and handle end of input buffer */
 688 static int next_c(void)
 689 {
 690     int ch = *++file->buf_ptr;
 691     /* end of buffer/file handling */
 692     if (ch == CH_EOB && file->buf_ptr >= file->buf_end)
 693         ch = handle_eob();
 694     return ch;
 695 }
 696
 697 /* input with '\[\r]\n' handling. */
 698 static int handle_stray_noerror(int err)
 699 {
 700     int ch;
 701     while ((ch = next_c()) == '\\') {
 702         ch = next_c();
 703         if (ch == '\n') {
 704     newl:
 705             file->line_num++;
 706         } else {
 707             if (ch == '\r') {
 708                 ch = next_c();
 709                 if (ch == '\n')
 710                     goto newl;
 711                 *--file->buf_ptr = '\r';
 712             }
 713             if (err)
 714                 tcc_error("stray '\\' in program");
 715             /* may take advantage of 'BufferedFile.unget[4}' */
 716             return *--file->buf_ptr = '\\';
 717         }
 718     }
 719     return ch;
 720 }
 721
 722 #define ninp() handle_stray_noerror(0)
 723
 724 /* handle '\\' in strings, comments and skipped regions */
 725 static int handle_bs(uint8_t **p)
 726 {
 727     int c;
 728     file->buf_ptr = *p - 1;
 729     c = ninp();
 730     *p = file->buf_ptr;
 731     return c;
 732 }
 733
 734 /* skip the stray and handle the \\n case. Output an error if
 735    incorrect char after the stray */
 736 static int handle_stray(uint8_t **p)
 737 {
 738     int c;
 739     file->buf_ptr = *p - 1;
 740     c = handle_stray_noerror(!(parse_flags & PARSE_FLAG_ACCEPT_STRAYS));
 741     *p = file->buf_ptr;
 742     return c;
 743 }
 744
 745 /* handle the complicated stray case */
 746 #define PEEKC(c, p)\
 747 {\
 748     c = *++p;\
 749     if (c == '\\')\
 750         c = handle_stray(&p); \
 751 }
 752
 753 static int skip_spaces(void)
 754 {
 755     int ch;
 756     --file->buf_ptr;
 757     do {
 758         ch = ninp();
 759     } while (isidnum_table[ch - CH_EOF] & IS_SPC);
 760     return ch;
 761 }
 762
 763 /* single line C++ comments */
 764 static uint8_t *parse_line_comment(uint8_t *p)
 765 {
 766     int c;
 767     for(;;) {
 768         for (;;) {
 769             c = *++p;
 770     redo:
 771             if (c == '\n' || c == '\\')
 772                 break;
 773             c = *++p;
 774             if (c == '\n' || c == '\\')
 775                 break;
 776         }
 777         if (c == '\n')
 778             break;
 779         c = handle_bs(&p);
 780         if (c == CH_EOF)
 781             break;
 782         if (c != '\\')
 783             goto redo;
 784     }
 785     return p;
 786 }
 787
 788 /* C comments */
 789 static uint8_t *parse_comment(uint8_t *p)
 790 {
 791     int c;
 792     for(;;) {
 793         /* fast skip loop */
 794         for(;;) {
 795             c = *++p;
 796         redo:
 797             if (c == '\n' || c == '*' || c == '\\')
 798                 break;
 799             c = *++p;
 800             if (c == '\n' || c == '*' || c == '\\')
 801                 break;
 802         }
 803         /* now we can handle all the cases */
 804         if (c == '\n') {
 805             file->line_num++;
 806         } else if (c == '*') {
 807             do {
 808                 c = *++p;
 809             } while (c == '*');
 810             if (c == '\\')
 811                 c = handle_bs(&p);
 812             if (c == '/')
 813                 break;
 814             goto check_eof;
 815         } else {
 816             c = handle_bs(&p);
 817         check_eof:
 818             if (c == CH_EOF)
 819                 tcc_error("unexpected end of file in comment");
 820             if (c != '\\')
 821                 goto redo;
 822         }
 823     }
 824     return p + 1;
 825 }
 826
 827 /* parse a string without interpreting escapes */
 828 static uint8_t *parse_pp_string(uint8_t *p, int sep, CString *str)
 829 {
 830     int c;
 831     for(;;) {
 832         c = *++p;
 833     redo:
 834         if (c == sep) {
 835             break;
 836         } else if (c == '\\') {
 837             c = handle_bs(&p);
 838             if (c == CH_EOF) {
 839         unterminated_string:
 840                 /* XXX: indicate line number of start of string */
 841                 tcc_error("missing terminating %c character", sep);
 842             } else if (c == '\\') {
 843                 if (str)
 844                     cstr_ccat(str, c);
 845                 c = *++p;
 846                 /* add char after '\\' unconditionally */
 847                 if (c == '\\') {
 848                     c = handle_bs(&p);
 849                     if (c == CH_EOF)
 850                         goto unterminated_string;
 851                 }
 852                 goto add_char;
 853             } else {
 854                 goto redo;
 855             }
 856         } else if (c == '\n') {
 857         add_lf:
 858             if (ACCEPT_LF_IN_STRINGS) {
 859                 file->line_num++;
 860                 goto add_char;
 861             } else if (str) { /* not skipping */
 862                 goto unterminated_string;
 863             } else {
 864                 //tcc_warning("missing terminating %c character", sep);
 865                 return p;
 866             }
 867         } else if (c == '\r') {
 868             c = *++p;
 869             if (c == '\\')
 870                 c = handle_bs(&p);
 871             if (c == '\n')
 872                 goto add_lf;
 873             if (c == CH_EOF)
 874                 goto unterminated_string;
 875             if (str)
 876                 cstr_ccat(str, '\r');
 877             goto redo;
 878         } else {
 879         add_char:
 880             if (str)
 881                 cstr_ccat(str, c);
 882         }
 883     }
 884     p++;
 885     return p;
 886 }
 887
 888 /* skip block of text until #else, #elif or #endif. skip also pairs of
 889    #if/#endif */
 890 static void preprocess_skip(void)
 891 {
 892     int a, start_of_line, c, in_warn_or_error;
 893     uint8_t *p;
 894
 895     p = file->buf_ptr;
 896     a = 0;
 897 redo_start:
 898     start_of_line = 1;
 899     in_warn_or_error = 0;
 900     for(;;) {
 901     redo_no_start:
 902         c = *p;
 903         switch(c) {
 904         case ' ':
 905         case '\t':
 906         case '\f':
 907         case '\v':
 908         case '\r':
 909             p++;
 910             goto redo_no_start;
 911         case '\n':
 912             file->line_num++;
 913             p++;
 914             goto redo_start;
 915         case '\\':
 916             c = handle_bs(&p);
 917             if (c == CH_EOF)
 918                 expect("#endif");
 919             if (c == '\\')
 920                 ++p;
 921             goto redo_no_start;
 922         /* skip strings */
 923         case '\"':
 924         case '\'':
 925             if (in_warn_or_error)
 926                 goto _default;
 927             tok_flags &= ~TOK_FLAG_BOL;
 928             p = parse_pp_string(p, c, NULL);
 929             break;
 930         /* skip comments */
 931         case '/':
 932             if (in_warn_or_error)
 933                 goto _default;
 934             ++p;
 935             c = handle_bs(&p);
 936             if (c == '*') {
 937                 p = parse_comment(p);
 938             } else if (c == '/') {
 939                 p = parse_line_comment(p);
 940             }
 941             break;
 942         case '#':
 943             p++;
 944             if (start_of_line) {
 945                 file->buf_ptr = p;
 946                 next_nomacro();
 947                 p = file->buf_ptr;
 948                 if (a == 0 &&
 949                     (tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
 950                     goto the_end;
 951                 if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
 952                     a++;
 953                 else if (tok == TOK_ENDIF)
 954                     a--;
 955                 else if( tok == TOK_ERROR || tok == TOK_WARNING)
 956                     in_warn_or_error = 1;
 957                 else if (tok == TOK_LINEFEED)
 958                     goto redo_start;
 959                 else if (parse_flags & PARSE_FLAG_ASM_FILE)
 960                     p = parse_line_comment(p - 1);
 961             }
 962 #if !defined(TCC_TARGET_ARM)
 963             else if (parse_flags & PARSE_FLAG_ASM_FILE)
 964                 p = parse_line_comment(p - 1);
 965 #else
 966             /* ARM assembly uses '#' for constants */
 967 #endif
 968             break;
 969 _default:
 970         default:
 971             p++;
 972             break;
 973         }
 974         start_of_line = 0;
 975     }
 976  the_end: ;
 977     file->buf_ptr = p;
 978 }
 979
 980 #if 0
 981 /* return the number of additional 'ints' necessary to store the
 982    token */
 983 static inline int tok_size(const int *p)
 984 {
 985     switch(*p) {
 986         /* 4 bytes */
 987     case TOK_CINT:
 988     case TOK_CUINT:
 989     case TOK_CCHAR:
 990     case TOK_LCHAR:
 991     case TOK_CFLOAT:
 992     case TOK_LINENUM:
 993         return 1 + 1;
 994     case TOK_STR:
 995     case TOK_LSTR:
 996     case TOK_PPNUM:
 997     case TOK_PPSTR:
 998         return 1 + ((sizeof(CString) + ((CString *)(p+1))->size + 3) >> 2);
 999     case TOK_CLONG:
1000     case TOK_CULONG:
1001         return 1 + LONG_SIZE / 4;
1002     case TOK_CDOUBLE:
1003     case TOK_CLLONG:
1004     case TOK_CULLONG:
1005         return 1 + 2;
1006     case TOK_CLDOUBLE:
1007         return 1 + LDOUBLE_SIZE / 4;
1008     default:
1009         return 1 + 0;
1010     }
1011 }
1012 #endif
1013
1014 /* token string handling */
1015 ST_INLN void tok_str_new(TokenString *s)
1016 {
1017     s->str = NULL;
1018     s->len = s->lastlen = 0;
1019     s->allocated_len = 0;
1020     s->last_line_num = -1;
1021 }
1022
1023 ST_FUNC TokenString *tok_str_alloc(void)
1024 {
1025     TokenString *str = tal_realloc(tokstr_alloc, 0, sizeof *str);
1026     tok_str_new(str);
1027     return str;
1028 }
1029
1030 ST_FUNC int *tok_str_dup(TokenString *s)
1031 {
1032     int *str;
1033
1034     str = tal_realloc(tokstr_alloc, 0, s->len * sizeof(int));
1035     memcpy(str, s->str, s->len * sizeof(int));
1036     return str;
1037 }
1038
1039 ST_FUNC void tok_str_free_str(int *str)
1040 {
1041     tal_free(tokstr_alloc, str);
1042 }
1043
1044 ST_FUNC void tok_str_free(TokenString *str)
1045 {
1046     tok_str_free_str(str->str);
1047     tal_free(tokstr_alloc, str);
1048 }
1049
1050 ST_FUNC int *tok_str_realloc(TokenString *s, int new_size)
1051 {
1052     int *str, size;
1053
1054     size = s->allocated_len;
1055     if (size < 16)
1056         size = 16;
1057     while (size < new_size)
1058         size = size * 2;
1059     if (size > s->allocated_len) {
1060         str = tal_realloc(tokstr_alloc, s->str, size * sizeof(int));
1061         s->allocated_len = size;
1062         s->str = str;
1063     }
1064     return s->str;
1065 }
1066
1067 ST_FUNC void tok_str_add(TokenString *s, int t)
1068 {
1069     int len, *str;
1070
1071     len = s->len;
1072     str = s->str;
1073     if (len >= s->allocated_len)
1074         str = tok_str_realloc(s, len + 1);
1075     str[len++] = t;
1076     s->len = len;
1077 }
1078
1079 ST_FUNC void begin_macro(TokenString *str, int alloc)
1080 {
1081     str->alloc = alloc;
1082     str->prev = macro_stack;
1083     str->prev_ptr = macro_ptr;
1084     str->save_line_num = file->line_num;
1085     macro_ptr = str->str;
1086     macro_stack = str;
1087 }
1088
1089 ST_FUNC void end_macro(void)
1090 {
1091     TokenString *str = macro_stack;
1092     macro_stack = str->prev;
1093     macro_ptr = str->prev_ptr;
1094     file->line_num = str->save_line_num;
1095     str->len = 0; /* matters if str not alloced, may be tokstr_buf */
1096     if (str->alloc != 0) {
1097         if (str->alloc == 2)
1098             str->str = NULL; /* don't free */
1099         tok_str_free(str);
1100     }
1101 }
1102
1103 static void tok_str_add2(TokenString *s, int t, CValue *cv)
1104 {
1105     int len, *str;
1106
1107     len = s->lastlen = s->len;
1108     str = s->str;
1109
1110     /* allocate space for worst case */
1111     if (len + TOK_MAX_SIZE >= s->allocated_len)
1112         str = tok_str_realloc(s, len + TOK_MAX_SIZE + 1);
1113     str[len++] = t;
1114     switch(t) {
1115     case TOK_CINT:
1116     case TOK_CUINT:
1117     case TOK_CCHAR:
1118     case TOK_LCHAR:
1119     case TOK_CFLOAT:
1120     case TOK_LINENUM:
1121 #if LONG_SIZE == 4
1122     case TOK_CLONG:
1123     case TOK_CULONG:
1124 #endif
1125         str[len++] = cv->tab[0];
1126         break;
1127     case TOK_PPNUM:
1128     case TOK_PPSTR:
1129     case TOK_STR:
1130     case TOK_LSTR:
1131         {
1132             /* Insert the string into the int array. */
1133             size_t nb_words =
1134                 1 + (cv->str.size + sizeof(int) - 1) / sizeof(int);
1135             if (len + nb_words >= s->allocated_len)
1136                 str = tok_str_realloc(s, len + nb_words + 1);
1137             str[len] = cv->str.size;
1138             memcpy(&str[len + 1], cv->str.data, cv->str.size);
1139             len += nb_words;
1140         }
1141         break;
1142     case TOK_CDOUBLE:
1143     case TOK_CLLONG:
1144     case TOK_CULLONG:
1145 #if LONG_SIZE == 8
1146     case TOK_CLONG:
1147     case TOK_CULONG:
1148 #endif
1149 #if LDOUBLE_SIZE == 8
1150     case TOK_CLDOUBLE:
1151 #endif
1152         str[len++] = cv->tab[0];
1153         str[len++] = cv->tab[1];
1154         break;
1155 #if LDOUBLE_SIZE == 12
1156     case TOK_CLDOUBLE:
1157         str[len++] = cv->tab[0];
1158         str[len++] = cv->tab[1];
1159         str[len++] = cv->tab[2];
1160 #elif LDOUBLE_SIZE == 16
1161     case TOK_CLDOUBLE:
1162         str[len++] = cv->tab[0];
1163         str[len++] = cv->tab[1];
1164         str[len++] = cv->tab[2];
1165         str[len++] = cv->tab[3];
1166 #elif LDOUBLE_SIZE != 8
1167 #error add long double size support
1168 #endif
1169         break;
1170     default:
1171         break;
1172     }
1173     s->len = len;
1174 }
1175
1176 /* add the current parse token in token string 's' */
1177 ST_FUNC void tok_str_add_tok(TokenString *s)
1178 {
1179     CValue cval;
1180
1181     /* save line number info */
1182     if (file->line_num != s->last_line_num) {
1183         s->last_line_num = file->line_num;
1184         cval.i = s->last_line_num;
1185         tok_str_add2(s, TOK_LINENUM, &cval);
1186     }
1187     tok_str_add2(s, tok, &tokc);
1188 }
1189
1190 /* get a token from an integer array and increment pointer. */
1191 static inline void tok_get(int *t, const int **pp, CValue *cv)
1192 {
1193     const int *p = *pp;
1194     int n, *tab;
1195
1196     tab = cv->tab;
1197     switch(*t = *p++) {
1198 #if LONG_SIZE == 4
1199     case TOK_CLONG:
1200 #endif
1201     case TOK_CINT:
1202     case TOK_CCHAR:
1203     case TOK_LCHAR:
1204     case TOK_LINENUM:
1205         cv->i = *p++;
1206         break;
1207 #if LONG_SIZE == 4
1208     case TOK_CULONG:
1209 #endif
1210     case TOK_CUINT:
1211         cv->i = (unsigned)*p++;
1212         break;
1213     case TOK_CFLOAT:
1214         tab[0] = *p++;
1215         break;
1216     case TOK_STR:
1217     case TOK_LSTR:
1218     case TOK_PPNUM:
1219     case TOK_PPSTR:
1220         cv->str.size = *p++;
1221         cv->str.data = p;
1222         p += (cv->str.size + sizeof(int) - 1) / sizeof(int);
1223         break;
1224     case TOK_CDOUBLE:
1225     case TOK_CLLONG:
1226     case TOK_CULLONG:
1227 #if LONG_SIZE == 8
1228     case TOK_CLONG:
1229     case TOK_CULONG:
1230 #endif
1231         n = 2;
1232         goto copy;
1233     case TOK_CLDOUBLE:
1234 #if LDOUBLE_SIZE == 16
1235         n = 4;
1236 #elif LDOUBLE_SIZE == 12
1237         n = 3;
1238 #elif LDOUBLE_SIZE == 8
1239         n = 2;
1240 #else
1241 # error add long double size support
1242 #endif
1243     copy:
1244         do
1245             *tab++ = *p++;
1246         while (--n);
1247         break;
1248     default:
1249         break;
1250     }
1251     *pp = p;
1252 }
1253
1254 #if 0
1255 # define TOK_GET(t,p,c) tok_get(t,p,c)
1256 #else
1257 # define TOK_GET(t,p,c) do { \
1258     int _t = **(p); \
1259     if (TOK_HAS_VALUE(_t)) \
1260         tok_get(t, p, c); \
1261     else \
1262         *(t) = _t, ++*(p); \
1263     } while (0)
1264 #endif
1265
1266 static int macro_is_equal(const int *a, const int *b)
1267 {
1268     CValue cv;
1269     int t;
1270
1271     if (!a || !b)
1272         return 1;
1273
1274     while (*a && *b) {
1275         /* first time preallocate macro_equal_buf, next time only reset position to start */
1276         cstr_reset(&macro_equal_buf);
1277         TOK_GET(&t, &a, &cv);
1278         cstr_cat(&macro_equal_buf, get_tok_str(t, &cv), 0);
1279         TOK_GET(&t, &b, &cv);
1280         if (strcmp(macro_equal_buf.data, get_tok_str(t, &cv)))
1281             return 0;
1282     }
1283     return !(*a || *b);
1284 }
1285
1286 /* defines handling */
1287 ST_INLN void define_push(int v, int macro_type, int *str, Sym *first_arg)
1288 {
1289     Sym *s, *o;
1290
1291     o = define_find(v);
1292     s = sym_push2(&define_stack, v, macro_type, 0);
1293     s->d = str;
1294     s->next = first_arg;
1295     table_ident[v - TOK_IDENT]->sym_define = s;
1296
1297     if (o && !macro_is_equal(o->d, s->d))
1298         tcc_warning("%s redefined", get_tok_str(v, NULL));
1299 }
1300
1301 /* undefined a define symbol. Its name is just set to zero */
1302 ST_FUNC void define_undef(Sym *s)
1303 {
1304     int v = s->v;
1305     if (v >= TOK_IDENT && v < tok_ident)
1306         table_ident[v - TOK_IDENT]->sym_define = NULL;
1307 }
1308
1309 ST_INLN Sym *define_find(int v)
1310 {
1311     v -= TOK_IDENT;
1312     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1313         return NULL;
1314     return table_ident[v]->sym_define;
1315 }
1316
1317 /* free define stack until top reaches 'b' */
1318 ST_FUNC void free_defines(Sym *b)
1319 {
1320     while (define_stack != b) {
1321         Sym *top = define_stack;
1322         define_stack = top->prev;
1323         tok_str_free_str(top->d);
1324         define_undef(top);
1325         sym_free(top);
1326     }
1327 }
1328
1329 /* label lookup */
1330 ST_FUNC Sym *label_find(int v)
1331 {
1332     v -= TOK_IDENT;
1333     if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
1334         return NULL;
1335     return table_ident[v]->sym_label;
1336 }
1337
1338 ST_FUNC Sym *label_push(Sym **ptop, int v, int flags)
1339 {
1340     Sym *s, **ps;
1341     s = sym_push2(ptop, v, VT_STATIC, 0);
1342     s->r = flags;
1343     ps = &table_ident[v - TOK_IDENT]->sym_label;
1344     if (ptop == &global_label_stack) {
1345         /* modify the top most local identifier, so that
1346            sym_identifier will point to 's' when popped */
1347         while (*ps != NULL)
1348             ps = &(*ps)->prev_tok;
1349     }
1350     s->prev_tok = *ps;
1351     *ps = s;
1352     return s;
1353 }
1354
1355 /* pop labels until element last is reached. Look if any labels are
1356    undefined. Define symbols if '&&label' was used. */
1357 ST_FUNC void label_pop(Sym **ptop, Sym *slast, int keep)
1358 {
1359     Sym *s, *s1;
1360     for(s = *ptop; s != slast; s = s1) {
1361         s1 = s->prev;
1362         if (s->r == LABEL_DECLARED) {
1363             tcc_warning_c(warn_all)("label '%s' declared but not used", get_tok_str(s->v, NULL));
1364         } else if (s->r == LABEL_FORWARD) {
1365                 tcc_error("label '%s' used but not defined",
1366                       get_tok_str(s->v, NULL));
1367         } else {
1368             if (s->c) {
1369                 /* define corresponding symbol. A size of
1370                    1 is put. */
1371                 put_extern_sym(s, cur_text_section, s->jnext, 1);
1372             }
1373         }
1374         /* remove label */
1375         if (s->r != LABEL_GONE)
1376             table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
1377         if (!keep)
1378             sym_free(s);
1379         else
1380             s->r = LABEL_GONE;
1381     }
1382     if (!keep)
1383         *ptop = slast;
1384 }
1385
1386 /* fake the nth "#if defined test_..." for tcc -dt -run */
1387 static void maybe_run_test(TCCState *s)
1388 {
1389     const char *p;
1390     if (s->include_stack_ptr != s->include_stack)
1391         return;
1392     p = get_tok_str(tok, NULL);
1393     if (0 != memcmp(p, "test_", 5))
1394         return;
1395     if (0 != --s->run_test)
1396         return;
1397     fprintf(s->ppfp, &"\n[%s]\n"[!(s->dflag & 32)], p), fflush(s->ppfp);
1398     define_push(tok, MACRO_OBJ, NULL, NULL);
1399 }
1400
1401 static CachedInclude *
1402 search_cached_include(TCCState *s1, const char *filename, int add);
1403
1404 static int parse_include(TCCState *s1, int do_next, int test)
1405 {
1406     int c, i;
1407     CString cs;
1408     char name[1024], buf[1024], *p;
1409     CachedInclude *e;
1410
1411     cstr_new(&cs);
1412     c = skip_spaces();
1413     if (c == '<' || c == '\"') {
1414         file->buf_ptr = parse_pp_string(file->buf_ptr, c == '<' ? '>' : c, &cs);
1415         next_nomacro();
1416     } else {
1417         /* computed #include : concatenate tokens until result is one of
1418            the two accepted forms.  Don't convert pp-tokens to tokens here. */
1419         parse_flags = PARSE_FLAG_PREPROCESS
1420                     | PARSE_FLAG_LINEFEED
1421                     | (parse_flags & PARSE_FLAG_ASM_FILE);
1422         for (;;) {
1423             next();
1424             p = cs.data, i = cs.size - 1;
1425             if (i > 0
1426                 && ((p[0] == '"' && p[i] == '"')
1427                  || (p[0] == '<' && p[i] == '>')))
1428                 break;
1429             if (tok == TOK_LINEFEED)
1430                 tcc_error("'#include' expects \"FILENAME\" or <FILENAME>");
1431             cstr_cat(&cs, get_tok_str(tok, &tokc), -1);
1432         }
1433         c = p[0];
1434         /* remove '<>|""' */
1435         memmove(p, p + 1, cs.size -= 2);
1436     }
1437     cstr_ccat(&cs, '\0');
1438     pstrcpy(name, sizeof name, cs.data);
1439     cstr_free(&cs);
1440
1441     i = do_next ? file->include_next_index : -1;
1442     for (;;) {
1443         ++i;
1444         if (i == 0) {
1445             /* check absolute include path */
1446             if (!IS_ABSPATH(name))
1447                 continue;
1448             buf[0] = '\0';
1449         } else if (i == 1) {
1450             /* search in file's dir if "header.h" */
1451             if (c != '\"')
1452                 continue;
1453             p = file->true_filename;
1454             pstrncpy(buf, p, tcc_basename(p) - p);
1455         } else {
1456             int j = i - 2, k = j - s1->nb_include_paths;
1457             if (k < 0)
1458                 p = s1->include_paths[j];
1459             else if (k < s1->nb_sysinclude_paths)
1460                 p = s1->sysinclude_paths[k];
1461             else if (test)
1462                 return 0;
1463             else
1464                 tcc_error("include file '%s' not found", name);
1465             pstrcpy(buf, sizeof buf, p);
1466             pstrcat(buf, sizeof buf, "/");
1467         }
1468         pstrcat(buf, sizeof buf, name);
1469         e = search_cached_include(s1, buf, 0);
1470         if (e && (define_find(e->ifndef_macro) || e->once == pp_once)) {
1471             /* no need to parse the include because the 'ifndef macro'
1472                is defined (or had #pragma once) */
1473 #ifdef INC_DEBUG
1474             printf("%s: skipping cached %s\n", file->filename, buf);
1475 #endif
1476             return 1;
1477         }
1478         if (tcc_open(s1, buf) >= 0)
1479             break;
1480     }
1481
1482     if (test) {
1483         tcc_close();
1484     } else {
1485         if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
1486             tcc_error("#include recursion too deep");
1487         /* push previous file on stack */
1488         *s1->include_stack_ptr++ = file->prev;
1489         file->include_next_index = i;
1490 #ifdef INC_DEBUG
1491         printf("%s: including %s\n", file->prev->filename, file->filename);
1492 #endif
1493         /* update target deps */
1494         if (s1->gen_deps) {
1495             BufferedFile *bf = file;
1496             while (i == 1 && (bf = bf->prev))
1497                 i = bf->include_next_index;
1498             /* skip system include files */
1499             if (s1->include_sys_deps || i - 2 < s1->nb_include_paths)
1500                 dynarray_add(&s1->target_deps, &s1->nb_target_deps,
1501                     tcc_strdup(buf));
1502         }
1503         /* add include file debug info */
1504         tcc_debug_bincl(s1);
1505         tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
1506     }
1507     return 1;
1508 }
1509
1510 /* eval an expression for #if/#elif */
1511 static int expr_preprocess(TCCState *s1)
1512 {
1513     int c, t;
1514     TokenString *str;
1515
1516     str = tok_str_alloc();
1517     pp_expr = 1;
1518     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1519         next(); /* do macro subst */
1520       redo:
1521         if (tok == TOK_DEFINED) {
1522             next_nomacro();
1523             t = tok;
1524             if (t == '(')
1525                 next_nomacro();
1526             if (tok < TOK_IDENT)
1527                 expect("identifier");
1528             if (s1->run_test)
1529                 maybe_run_test(s1);
1530             c = 0;
1531             if (define_find(tok)
1532                 || tok == TOK___HAS_INCLUDE
1533                 || tok == TOK___HAS_INCLUDE_NEXT)
1534                 c = 1;
1535             if (t == '(') {
1536                 next_nomacro();
1537                 if (tok != ')')
1538                     expect("')'");
1539             }
1540             tok = TOK_CINT;
1541             tokc.i = c;
1542         } else if (tok == TOK___HAS_INCLUDE ||
1543                    tok == TOK___HAS_INCLUDE_NEXT) {
1544             t = tok;
1545             next_nomacro();
1546             if (tok != '(')
1547                 expect("(");
1548             c = parse_include(s1, t - TOK___HAS_INCLUDE, 1);
1549             if (tok != ')')
1550                 expect("')'");
1551             tok = TOK_CINT;
1552             tokc.i = c;
1553         } else if (tok >= TOK_IDENT) {
1554             /* if undefined macro, replace with zero, check for func-like */
1555             t = tok;
1556             tok = TOK_CINT;
1557             tokc.i = 0;
1558             tok_str_add_tok(str);
1559             next();
1560             if (tok == '(')
1561                 tcc_error("function-like macro '%s' is not defined",
1562                           get_tok_str(t, NULL));
1563             goto redo;
1564         }
1565         tok_str_add_tok(str);
1566     }
1567     pp_expr = 0;
1568     tok_str_add(str, -1); /* simulate end of file */
1569     tok_str_add(str, 0);
1570     /* now evaluate C constant expression */
1571     begin_macro(str, 1);
1572     next();
1573     c = expr_const();
1574     end_macro();
1575     return c != 0;
1576 }
1577
1578
1579 /* parse after #define */
1580 ST_FUNC void parse_define(void)
1581 {
1582     Sym *s, *first, **ps;
1583     int v, t, varg, is_vaargs, spc;
1584     int saved_parse_flags = parse_flags;
1585
1586     v = tok;
1587     if (v < TOK_IDENT || v == TOK_DEFINED)
1588         tcc_error("invalid macro name '%s'", get_tok_str(tok, &tokc));
1589     /* XXX: should check if same macro (ANSI) */
1590     first = NULL;
1591     t = MACRO_OBJ;
1592     /* We have to parse the whole define as if not in asm mode, in particular
1593        no line comment with '#' must be ignored.  Also for function
1594        macros the argument list must be parsed without '.' being an ID
1595        character.  */
1596     parse_flags = ((parse_flags & ~PARSE_FLAG_ASM_FILE) | PARSE_FLAG_SPACES);
1597     /* '(' must be just after macro definition for MACRO_FUNC */
1598     next_nomacro();
1599     parse_flags &= ~PARSE_FLAG_SPACES;
1600     if (tok == '(') {
1601         int dotid = set_idnum('.', 0);
1602         next_nomacro();
1603         ps = &first;
1604         if (tok != ')') for (;;) {
1605             varg = tok;
1606             next_nomacro();
1607             is_vaargs = 0;
1608             if (varg == TOK_DOTS) {
1609                 varg = TOK___VA_ARGS__;
1610                 is_vaargs = 1;
1611             } else if (tok == TOK_DOTS && gnu_ext) {
1612                 is_vaargs = 1;
1613                 next_nomacro();
1614             }
1615             if (varg < TOK_IDENT)
1616         bad_list:
1617                 tcc_error("bad macro parameter list");
1618             s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
1619             *ps = s;
1620             ps = &s->next;
1621             if (tok == ')')
1622                 break;
1623             if (tok != ',' || is_vaargs)
1624                 goto bad_list;
1625             next_nomacro();
1626         }
1627         parse_flags |= PARSE_FLAG_SPACES;
1628         next_nomacro();
1629         t = MACRO_FUNC;
1630         set_idnum('.', dotid);
1631     }
1632
1633     tokstr_buf.len = 0;
1634     spc = 2;
1635     parse_flags |= PARSE_FLAG_ACCEPT_STRAYS | PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED;
1636     /* The body of a macro definition should be parsed such that identifiers
1637        are parsed like the file mode determines (i.e. with '.' being an
1638        ID character in asm mode).  But '#' should be retained instead of
1639        regarded as line comment leader, so still don't set ASM_FILE
1640        in parse_flags. */
1641     while (tok != TOK_LINEFEED && tok != TOK_EOF) {
1642         /* remove spaces around ## and after '#' */
1643         if (TOK_TWOSHARPS == tok) {
1644             if (2 == spc)
1645                 goto bad_twosharp;
1646             if (1 == spc)
1647                 --tokstr_buf.len;
1648             spc = 3;
1649             tok = TOK_PPJOIN;
1650         } else if ('#' == tok) {
1651             spc = 4;
1652         } else if (check_space(tok, &spc)) {
1653             goto skip;
1654         }
1655         tok_str_add2(&tokstr_buf, tok, &tokc);
1656     skip:
1657         next_nomacro();
1658     }
1659
1660     parse_flags = saved_parse_flags;
1661     if (spc == 1)
1662         --tokstr_buf.len; /* remove trailing space */
1663     tok_str_add(&tokstr_buf, 0);
1664     if (3 == spc)
1665 bad_twosharp:
1666         tcc_error("'##' cannot appear at either end of macro");
1667     define_push(v, t, tok_str_dup(&tokstr_buf), first);
1668 }
1669
1670 static CachedInclude *search_cached_include(TCCState *s1, const char *filename, int add)
1671 {
1672     const unsigned char *s;
1673     unsigned int h;
1674     CachedInclude *e;
1675     int i;
1676
1677     h = TOK_HASH_INIT;
1678     s = (unsigned char *) filename;
1679     while (*s) {
1680 #ifdef _WIN32
1681         h = TOK_HASH_FUNC(h, toup(*s));
1682 #else
1683         h = TOK_HASH_FUNC(h, *s);
1684 #endif
1685         s++;
1686     }
1687     h &= (CACHED_INCLUDES_HASH_SIZE - 1);
1688
1689     i = s1->cached_includes_hash[h];
1690     for(;;) {
1691         if (i == 0)
1692             break;
1693         e = s1->cached_includes[i - 1];
1694         if (0 == PATHCMP(e->filename, filename))
1695             return e;
1696         i = e->hash_next;
1697     }
1698     if (!add)
1699         return NULL;
1700
1701     e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
1702     strcpy(e->filename, filename);
1703     e->ifndef_macro = e->once = 0;
1704     dynarray_add(&s1->cached_includes, &s1->nb_cached_includes, e);
1705     /* add in hash table */
1706     e->hash_next = s1->cached_includes_hash[h];
1707     s1->cached_includes_hash[h] = s1->nb_cached_includes;
1708 #ifdef INC_DEBUG
1709     printf("adding cached '%s'\n", filename);
1710 #endif
1711     return e;
1712 }
1713
1714 static void pragma_parse(TCCState *s1)
1715 {
1716     next_nomacro();
1717     if (tok == TOK_push_macro || tok == TOK_pop_macro) {
1718         int t = tok, v;
1719         Sym *s;
1720
1721         if (next(), tok != '(')
1722             goto pragma_err;
1723         if (next(), tok != TOK_STR)
1724             goto pragma_err;
1725         v = tok_alloc(tokc.str.data, tokc.str.size - 1)->tok;
1726         if (next(), tok != ')')
1727             goto pragma_err;
1728         if (t == TOK_push_macro) {
1729             while (NULL == (s = define_find(v)))
1730                 define_push(v, 0, NULL, NULL);
1731             s->type.ref = s; /* set push boundary */
1732         } else {
1733             for (s = define_stack; s; s = s->prev)
1734                 if (s->v == v && s->type.ref == s) {
1735                     s->type.ref = NULL;
1736                     break;
1737                 }
1738         }
1739         if (s)
1740             table_ident[v - TOK_IDENT]->sym_define = s->d ? s : NULL;
1741         else
1742             tcc_warning("unbalanced #pragma pop_macro");
1743         pp_debug_tok = t, pp_debug_symv = v;
1744
1745     } else if (tok == TOK_once) {
1746         search_cached_include(s1, file->filename, 1)->once = pp_once;
1747
1748     } else if (s1->output_type == TCC_OUTPUT_PREPROCESS) {
1749         /* tcc -E: keep pragmas below unchanged */
1750         unget_tok(' ');
1751         unget_tok(TOK_PRAGMA);
1752         unget_tok('#');
1753         unget_tok(TOK_LINEFEED);
1754
1755     } else if (tok == TOK_pack) {
1756         /* This may be:
1757            #pragma pack(1) // set
1758            #pragma pack() // reset to default
1759            #pragma pack(push) // push current
1760            #pragma pack(push,1) // push & set
1761            #pragma pack(pop) // restore previous */
1762         next();
1763         skip('(');
1764         if (tok == TOK_ASM_pop) {
1765             next();
1766             if (s1->pack_stack_ptr <= s1->pack_stack) {
1767             stk_error:
1768                 tcc_error("out of pack stack");
1769             }
1770             s1->pack_stack_ptr--;
1771         } else {
1772             int val = 0;
1773             if (tok != ')') {
1774                 if (tok == TOK_ASM_push) {
1775                     next();
1776                     if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
1777                         goto stk_error;
1778                     val = *s1->pack_stack_ptr++;
1779                     if (tok != ',')
1780                         goto pack_set;
1781                     next();
1782                 }
1783                 if (tok != TOK_CINT)
1784                     goto pragma_err;
1785                 val = tokc.i;
1786                 if (val < 1 || val > 16 || (val & (val - 1)) != 0)
1787                     goto pragma_err;
1788                 next();
1789             }
1790         pack_set:
1791             *s1->pack_stack_ptr = val;
1792         }
1793         if (tok != ')')
1794             goto pragma_err;
1795
1796     } else if (tok == TOK_comment) {
1797         char *p; int t;
1798         next();
1799         skip('(');
1800         t = tok;
1801         next();
1802         skip(',');
1803         if (tok != TOK_STR)
1804             goto pragma_err;
1805         p = tcc_strdup((char *)tokc.str.data);
1806         next();
1807         if (tok != ')')
1808             goto pragma_err;
1809         if (t == TOK_lib) {
1810             dynarray_add(&s1->pragma_libs, &s1->nb_pragma_libs, p);
1811         } else {
1812             if (t == TOK_option)
1813                 tcc_set_options(s1, p);
1814             tcc_free(p);
1815         }
1816
1817     } else
1818         tcc_warning_c(warn_unsupported)("#pragma %s ignored", get_tok_str(tok, &tokc));
1819     return;
1820
1821 pragma_err:
1822     tcc_error("malformed #pragma directive");
1823     return;
1824 }
1825
1826 /* is_bof is true if first non space token at beginning of file */
1827 ST_FUNC void preprocess(int is_bof)
1828 {
1829     TCCState *s1 = tcc_state;
1830     int c, n, saved_parse_flags;
1831     char buf[1024], *q;
1832     Sym *s;
1833
1834     saved_parse_flags = parse_flags;
1835     parse_flags = PARSE_FLAG_PREPROCESS
1836         | PARSE_FLAG_TOK_NUM
1837         | PARSE_FLAG_TOK_STR
1838         | PARSE_FLAG_LINEFEED
1839         | (parse_flags & PARSE_FLAG_ASM_FILE)
1840         ;
1841
1842     next_nomacro();
1843  redo:
1844     switch(tok) {
1845     case TOK_DEFINE:
1846         pp_debug_tok = tok;
1847         next_nomacro();
1848         pp_debug_symv = tok;
1849         parse_define();
1850         break;
1851     case TOK_UNDEF:
1852         pp_debug_tok = tok;
1853         next_nomacro();
1854         pp_debug_symv = tok;
1855         s = define_find(tok);
1856         /* undefine symbol by putting an invalid name */
1857         if (s)
1858             define_undef(s);
1859         break;
1860     case TOK_INCLUDE:
1861     case TOK_INCLUDE_NEXT:
1862         parse_include(s1, tok - TOK_INCLUDE, 0);
1863         break;
1864     case TOK_IFNDEF:
1865         c = 1;
1866         goto do_ifdef;
1867     case TOK_IF:
1868         c = expr_preprocess(s1);
1869         goto do_if;
1870     case TOK_IFDEF:
1871         c = 0;
1872     do_ifdef:
1873         next_nomacro();
1874         if (tok < TOK_IDENT)
1875             tcc_error("invalid argument for '#if%sdef'", c ? "n" : "");
1876         if (is_bof) {
1877             if (c) {
1878 #ifdef INC_DEBUG
1879                 printf("#ifndef %s\n", get_tok_str(tok, NULL));
1880 #endif
1881                 file->ifndef_macro = tok;
1882             }
1883         }
1884         if (define_find(tok)
1885             || tok == TOK___HAS_INCLUDE
1886             || tok == TOK___HAS_INCLUDE_NEXT)
1887             c ^= 1;
1888     do_if:
1889         if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
1890             tcc_error("memory full (ifdef)");
1891         *s1->ifdef_stack_ptr++ = c;
1892         goto test_skip;
1893     case TOK_ELSE:
1894         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1895             tcc_error("#else without matching #if");
1896         if (s1->ifdef_stack_ptr[-1] & 2)
1897             tcc_error("#else after #else");
1898         c = (s1->ifdef_stack_ptr[-1] ^= 3);
1899         goto test_else;
1900     case TOK_ELIF:
1901         if (s1->ifdef_stack_ptr == s1->ifdef_stack)
1902             tcc_error("#elif without matching #if");
1903         c = s1->ifdef_stack_ptr[-1];
1904         if (c > 1)
1905             tcc_error("#elif after #else");
1906         /* last #if/#elif expression was true: we skip */
1907         if (c == 1) {
1908             c = 0;
1909         } else {
1910             c = expr_preprocess(s1);
1911             s1->ifdef_stack_ptr[-1] = c;
1912         }
1913     test_else:
1914         if (s1->ifdef_stack_ptr == file->ifdef_stack_ptr + 1)
1915             file->ifndef_macro = 0;
1916     test_skip:
1917         if (!(c & 1)) {
1918             preprocess_skip();
1919             is_bof = 0;
1920             goto redo;
1921         }
1922         break;
1923     case TOK_ENDIF:
1924         if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
1925             tcc_error("#endif without matching #if");
1926         s1->ifdef_stack_ptr--;
1927         /* '#ifndef macro' was at the start of file. Now we check if
1928            an '#endif' is exactly at the end of file */
1929         if (file->ifndef_macro &&
1930             s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
1931             file->ifndef_macro_saved = file->ifndef_macro;
1932             /* need to set to zero to avoid false matches if another
1933                #ifndef at middle of file */
1934             file->ifndef_macro = 0;
1935             while (tok != TOK_LINEFEED)
1936                 next_nomacro();
1937             tok_flags |= TOK_FLAG_ENDIF;
1938             goto the_end;
1939         }
1940         break;
1941     case TOK_PPNUM:
1942         n = strtoul((char*)tokc.str.data, &q, 10);
1943         goto _line_num;
1944     case TOK_LINE:
1945         next();
1946         if (tok != TOK_CINT)
1947     _line_err:
1948             tcc_error("wrong #line format");
1949         n = tokc.i;
1950     _line_num:
1951         next();
1952         if (tok != TOK_LINEFEED) {
1953             if (tok == TOK_STR) {
1954                 if (file->true_filename == file->filename)
1955                     file->true_filename = tcc_strdup(file->filename);
1956                 q = (char *)tokc.str.data;
1957                 buf[0] = 0;
1958                 if (!IS_ABSPATH(q)) {
1959                     /* prepend directory from real file */
1960                     pstrcpy(buf, sizeof buf, file->true_filename);
1961                     *tcc_basename(buf) = 0;
1962                 }
1963                 pstrcat(buf, sizeof buf, q);
1964                 tcc_debug_putfile(s1, buf);
1965             } else if (parse_flags & PARSE_FLAG_ASM_FILE)
1966                 break;
1967             else
1968                 goto _line_err;
1969             --n;
1970         }
1971         if (file->fd > 0)
1972             total_lines += file->line_num - n;
1973         file->line_num = n;
1974         break;
1975     case TOK_ERROR:
1976     case TOK_WARNING:
1977         q = buf;
1978         c = skip_spaces();
1979         while (c != '\n' && c != CH_EOF) {
1980             if ((q - buf) < sizeof(buf) - 1)
1981                 *q++ = c;
1982             c = ninp();
1983         }
1984         *q = '\0';
1985         if (tok == TOK_ERROR)
1986             tcc_error("#error %s", buf);
1987         else
1988             tcc_warning("#warning %s", buf);
1989         break;
1990     case TOK_PRAGMA:
1991         pragma_parse(s1);
1992         break;
1993     case TOK_LINEFEED:
1994         goto the_end;
1995     default:
1996         /* ignore gas line comment in an 'S' file. */
1997         if (saved_parse_flags & PARSE_FLAG_ASM_FILE)
1998             goto ignore;
1999         if (tok == '!' && is_bof)
2000             /* '!' is ignored at beginning to allow C scripts. */
2001             goto ignore;
2002         tcc_warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
2003     ignore:
2004         file->buf_ptr = parse_line_comment(file->buf_ptr - 1);
2005         goto the_end;
2006     }
2007     /* ignore other preprocess commands or #! for C scripts */
2008     while (tok != TOK_LINEFEED)
2009         next_nomacro();
2010  the_end:
2011     parse_flags = saved_parse_flags;
2012 }
2013
2014 /* evaluate escape codes in a string. */
2015 static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
2016 {
2017     int c, n, i;
2018     const uint8_t *p;
2019
2020     p = buf;
2021     for(;;) {
2022         c = *p;
2023         if (c == '\0')
2024             break;
2025         if (c == '\\') {
2026             p++;
2027             /* escape */
2028             c = *p;
2029             switch(c) {
2030             case '0': case '1': case '2': case '3':
2031             case '4': case '5': case '6': case '7':
2032                 /* at most three octal digits */
2033                 n = c - '0';
2034                 p++;
2035                 c = *p;
2036                 if (isoct(c)) {
2037                     n = n * 8 + c - '0';
2038                     p++;
2039                     c = *p;
2040                     if (isoct(c)) {
2041                         n = n * 8 + c - '0';
2042                         p++;
2043                     }
2044                 }
2045                 c = n;
2046                 goto add_char_nonext;
2047             case 'x': i = 0; goto parse_hex_or_ucn;
2048             case 'u': i = 4; goto parse_hex_or_ucn;
2049             case 'U': i = 8; goto parse_hex_or_ucn;
2050     parse_hex_or_ucn:
2051                 p++;
2052                 n = 0;
2053                 do {
2054                     c = *p;
2055                     if (c >= 'a' && c <= 'f')
2056                         c = c - 'a' + 10;
2057                     else if (c >= 'A' && c <= 'F')
2058                         c = c - 'A' + 10;
2059                     else if (isnum(c))
2060                         c = c - '0';
2061                     else if (i > 0)
2062                         expect("more hex digits in universal-character-name");
2063                     else
2064                         goto add_hex_or_ucn;
2065                     n = n * 16 + c;
2066                     p++;
2067                 } while (--i);
2068                 if (is_long) {
2069     add_hex_or_ucn:
2070                     c = n;
2071                     goto add_char_nonext;
2072                 }
2073                 cstr_u8cat(outstr, n);
2074                 continue;
2075             case 'a':
2076                 c = '\a';
2077                 break;
2078             case 'b':
2079                 c = '\b';
2080                 break;
2081             case 'f':
2082                 c = '\f';
2083                 break;
2084             case 'n':
2085                 c = '\n';
2086                 break;
2087             case 'r':
2088                 c = '\r';
2089                 break;
2090             case 't':
2091                 c = '\t';
2092                 break;
2093             case 'v':
2094                 c = '\v';
2095                 break;
2096             case 'e':
2097                 if (!gnu_ext)
2098                     goto invalid_escape;
2099                 c = 27;
2100                 break;
2101             case '\'':
2102             case '\"':
2103             case '\\':
2104             case '?':
2105                 break;
2106             default:
2107             invalid_escape:
2108                 if (c >= '!' && c <= '~')
2109                     tcc_warning("unknown escape sequence: \'\\%c\'", c);
2110                 else
2111                     tcc_warning("unknown escape sequence: \'\\x%x\'", c);
2112                 break;
2113             }
2114         } else if (is_long && c >= 0x80) {
2115             /* assume we are processing UTF-8 sequence */
2116             /* reference: The Unicode Standard, Version 10.0, ch3.9 */
2117
2118             int cont; /* count of continuation bytes */
2119             int skip; /* how many bytes should skip when error occurred */
2120             int i;
2121
2122             /* decode leading byte */
2123             if (c < 0xC2) {
2124                     skip = 1; goto invalid_utf8_sequence;
2125             } else if (c <= 0xDF) {
2126                     cont = 1; n = c & 0x1f;
2127             } else if (c <= 0xEF) {
2128                     cont = 2; n = c & 0xf;
2129             } else if (c <= 0xF4) {
2130                     cont = 3; n = c & 0x7;
2131             } else {
2132                     skip = 1; goto invalid_utf8_sequence;
2133             }
2134
2135             /* decode continuation bytes */
2136             for (i = 1; i <= cont; i++) {
2137                 int l = 0x80, h = 0xBF;
2138
2139                 /* adjust limit for second byte */
2140                 if (i == 1) {
2141                     switch (c) {
2142                     case 0xE0: l = 0xA0; break;
2143                     case 0xED: h = 0x9F; break;
2144                     case 0xF0: l = 0x90; break;
2145                     case 0xF4: h = 0x8F; break;
2146                     }
2147                 }
2148
2149                 if (p[i] < l || p[i] > h) {
2150                     skip = i; goto invalid_utf8_sequence;
2151                 }
2152
2153                 n = (n << 6) | (p[i] & 0x3f);
2154             }
2155
2156             /* advance pointer */
2157             p += 1 + cont;
2158             c = n;
2159             goto add_char_nonext;
2160
2161             /* error handling */
2162         invalid_utf8_sequence:
2163             tcc_warning("ill-formed UTF-8 subsequence starting with: \'\\x%x\'", c);
2164             c = 0xFFFD;
2165             p += skip;
2166             goto add_char_nonext;
2167
2168         }
2169         p++;
2170     add_char_nonext:
2171         if (!is_long)
2172             cstr_ccat(outstr, c);
2173         else {
2174 #ifdef TCC_TARGET_PE
2175             /* store as UTF-16 */
2176             if (c < 0x10000) {
2177                 cstr_wccat(outstr, c);
2178             } else {
2179                 c -= 0x10000;
2180                 cstr_wccat(outstr, (c >> 10) + 0xD800);
2181                 cstr_wccat(outstr, (c & 0x3FF) + 0xDC00);
2182             }
2183 #else
2184             cstr_wccat(outstr, c);
2185 #endif
2186         }
2187     }
2188     /* add a trailing '\0' */
2189     if (!is_long)
2190         cstr_ccat(outstr, '\0');
2191     else
2192         cstr_wccat(outstr, '\0');
2193 }
2194
2195 static void parse_string(const char *s, int len)
2196 {
2197     uint8_t buf[1000], *p = buf;
2198     int is_long, sep;
2199
2200     if ((is_long = *s == 'L'))
2201         ++s, --len;
2202     sep = *s++;
2203     len -= 2;
2204     if (len >= sizeof buf)
2205         p = tcc_malloc(len + 1);
2206     memcpy(p, s, len);
2207     p[len] = 0;
2208
2209     cstr_reset(&tokcstr);
2210     parse_escape_string(&tokcstr, p, is_long);
2211     if (p != buf)
2212         tcc_free(p);
2213
2214     if (sep == '\'') {
2215         int char_size, i, n, c;
2216         /* XXX: make it portable */
2217         if (!is_long)
2218             tok = TOK_CCHAR, char_size = 1;
2219         else
2220             tok = TOK_LCHAR, char_size = sizeof(nwchar_t);
2221         n = tokcstr.size / char_size - 1;
2222         if (n < 1)
2223             tcc_error("empty character constant");
2224         if (n > 1)
2225             tcc_warning_c(warn_all)("multi-character character constant");
2226         for (c = i = 0; i < n; ++i) {
2227             if (is_long)
2228                 c = ((nwchar_t *)tokcstr.data)[i];
2229             else
2230                 c = (c << 8) | ((char *)tokcstr.data)[i];
2231         }
2232         tokc.i = c;
2233     } else {
2234         tokc.str.size = tokcstr.size;
2235         tokc.str.data = tokcstr.data;
2236         if (!is_long)
2237             tok = TOK_STR;
2238         else
2239             tok = TOK_LSTR;
2240     }
2241 }
2242
2243 /* we use 64 bit numbers */
2244 #define BN_SIZE 2
2245
2246 /* bn = (bn << shift) | or_val */
2247 static void bn_lshift(unsigned int *bn, int shift, int or_val)
2248 {
2249     int i;
2250     unsigned int v;
2251     for(i=0;i<BN_SIZE;i++) {
2252         v = bn[i];
2253         bn[i] = (v << shift) | or_val;
2254         or_val = v >> (32 - shift);
2255     }
2256 }
2257
2258 static void bn_zero(unsigned int *bn)
2259 {
2260     int i;
2261     for(i=0;i<BN_SIZE;i++) {
2262         bn[i] = 0;
2263     }
2264 }
2265
2266 /* parse number in null terminated string 'p' and return it in the
2267    current token */
2268 static void parse_number(const char *p)
2269 {
2270     int b, t, shift, frac_bits, s, exp_val, ch;
2271     char *q;
2272     unsigned int bn[BN_SIZE];
2273     double d;
2274
2275     /* number */
2276     q = token_buf;
2277     ch = *p++;
2278     t = ch;
2279     ch = *p++;
2280     *q++ = t;
2281     b = 10;
2282     if (t == '.') {
2283         goto float_frac_parse;
2284     } else if (t == '0') {
2285         if (ch == 'x' || ch == 'X') {
2286             q--;
2287             ch = *p++;
2288             b = 16;
2289         } else if (tcc_state->tcc_ext && (ch == 'b' || ch == 'B')) {
2290             q--;
2291             ch = *p++;
2292             b = 2;
2293         }
2294     }
2295     /* parse all digits. cannot check octal numbers at this stage
2296        because of floating point constants */
2297     while (1) {
2298         if (ch >= 'a' && ch <= 'f')
2299             t = ch - 'a' + 10;
2300         else if (ch >= 'A' && ch <= 'F')
2301             t = ch - 'A' + 10;
2302         else if (isnum(ch))
2303             t = ch - '0';
2304         else
2305             break;
2306         if (t >= b)
2307             break;
2308         if (q >= token_buf + STRING_MAX_SIZE) {
2309         num_too_long:
2310             tcc_error("number too long");
2311         }
2312         *q++ = ch;
2313         ch = *p++;
2314     }
2315     if (ch == '.' ||
2316         ((ch == 'e' || ch == 'E') && b == 10) ||
2317         ((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
2318         if (b != 10) {
2319             /* NOTE: strtox should support that for hexa numbers, but
2320                non ISOC99 libcs do not support it, so we prefer to do
2321                it by hand */
2322             /* hexadecimal or binary floats */
2323             /* XXX: handle overflows */
2324             *q = '\0';
2325             if (b == 16)
2326                 shift = 4;
2327             else
2328                 shift = 1;
2329             bn_zero(bn);
2330             q = token_buf;
2331             while (1) {
2332                 t = *q++;
2333                 if (t == '\0') {
2334                     break;
2335                 } else if (t >= 'a') {
2336                     t = t - 'a' + 10;
2337                 } else if (t >= 'A') {
2338                     t = t - 'A' + 10;
2339                 } else {
2340                     t = t - '0';
2341                 }
2342                 bn_lshift(bn, shift, t);
2343             }
2344             frac_bits = 0;
2345             if (ch == '.') {
2346                 ch = *p++;
2347                 while (1) {
2348                     t = ch;
2349                     if (t >= 'a' && t <= 'f') {
2350                         t = t - 'a' + 10;
2351                     } else if (t >= 'A' && t <= 'F') {
2352                         t = t - 'A' + 10;
2353                     } else if (t >= '0' && t <= '9') {
2354                         t = t - '0';
2355                     } else {
2356                         break;
2357                     }
2358                     if (t >= b)
2359                         tcc_error("invalid digit");
2360                     bn_lshift(bn, shift, t);
2361                     frac_bits += shift;
2362                     ch = *p++;
2363                 }
2364             }
2365             if (ch != 'p' && ch != 'P')
2366                 expect("exponent");
2367             ch = *p++;
2368             s = 1;
2369             exp_val = 0;
2370             if (ch == '+') {
2371                 ch = *p++;
2372             } else if (ch == '-') {
2373                 s = -1;
2374                 ch = *p++;
2375             }
2376             if (ch < '0' || ch > '9')
2377                 expect("exponent digits");
2378             while (ch >= '0' && ch <= '9') {
2379                 exp_val = exp_val * 10 + ch - '0';
2380                 ch = *p++;
2381             }
2382             exp_val = exp_val * s;
2383
2384             /* now we can generate the number */
2385             /* XXX: should patch directly float number */
2386             d = (double)bn[1] * 4294967296.0 + (double)bn[0];
2387             d = ldexp(d, exp_val - frac_bits);
2388             t = toup(ch);
2389             if (t == 'F') {
2390                 ch = *p++;
2391                 tok = TOK_CFLOAT;
2392                 /* float : should handle overflow */
2393                 tokc.f = (float)d;
2394             } else if (t == 'L') {
2395                 ch = *p++;
2396 #ifdef TCC_USING_DOUBLE_FOR_LDOUBLE
2397                 tok = TOK_CDOUBLE;
2398                 tokc.d = d;
2399 #else
2400                 tok = TOK_CLDOUBLE;
2401                 /* XXX: not large enough */
2402                 tokc.ld = (long double)d;
2403 #endif
2404             } else {
2405                 tok = TOK_CDOUBLE;
2406                 tokc.d = d;
2407             }
2408         } else {
2409             /* decimal floats */
2410             if (ch == '.') {
2411                 if (q >= token_buf + STRING_MAX_SIZE)
2412                     goto num_too_long;
2413                 *q++ = ch;
2414                 ch = *p++;
2415             float_frac_parse:
2416                 while (ch >= '0' && ch <= '9') {
2417                     if (q >= token_buf + STRING_MAX_SIZE)
2418                         goto num_too_long;
2419                     *q++ = ch;
2420                     ch = *p++;
2421                 }
2422             }
2423             if (ch == 'e' || ch == 'E') {
2424                 if (q >= token_buf + STRING_MAX_SIZE)
2425                     goto num_too_long;
2426                 *q++ = ch;
2427                 ch = *p++;
2428                 if (ch == '-' || ch == '+') {
2429                     if (q >= token_buf + STRING_MAX_SIZE)
2430                         goto num_too_long;
2431                     *q++ = ch;
2432                     ch = *p++;
2433                 }
2434                 if (ch < '0' || ch > '9')
2435                     expect("exponent digits");
2436                 while (ch >= '0' && ch <= '9') {
2437                     if (q >= token_buf + STRING_MAX_SIZE)
2438                         goto num_too_long;
2439                     *q++ = ch;
2440                     ch = *p++;
2441                 }
2442             }
2443             *q = '\0';
2444             t = toup(ch);
2445             errno = 0;
2446             if (t == 'F') {
2447                 ch = *p++;
2448                 tok = TOK_CFLOAT;
2449                 tokc.f = strtof(token_buf, NULL);
2450             } else if (t == 'L') {
2451                 ch = *p++;
2452 #ifdef TCC_USING_DOUBLE_FOR_LDOUBLE
2453                 tok = TOK_CDOUBLE;
2454                 tokc.d = strtod(token_buf, NULL);
2455 #else
2456                 tok = TOK_CLDOUBLE;
2457                 tokc.ld = strtold(token_buf, NULL);
2458 #endif
2459             } else {
2460                 tok = TOK_CDOUBLE;
2461                 tokc.d = strtod(token_buf, NULL);
2462             }
2463         }
2464     } else {
2465         unsigned long long n, n1;
2466         int lcount, ucount, ov = 0;
2467         const char *p1;
2468
2469         /* integer number */
2470         *q = '\0';
2471         q = token_buf;
2472         if (b == 10 && *q == '0') {
2473             b = 8;
2474             q++;
2475         }
2476         n = 0;
2477         while(1) {
2478             t = *q++;
2479             /* no need for checks except for base 10 / 8 errors */
2480             if (t == '\0')
2481                 break;
2482             else if (t >= 'a')
2483                 t = t - 'a' + 10;
2484             else if (t >= 'A')
2485                 t = t - 'A' + 10;
2486             else
2487                 t = t - '0';
2488             if (t >= b)
2489                 tcc_error("invalid digit");
2490             n1 = n;
2491             n = n * b + t;
2492             /* detect overflow */
2493             if (n1 >= 0x1000000000000000ULL && n / b != n1)
2494                 ov = 1;
2495         }
2496
2497         /* Determine the characteristics (unsigned and/or 64bit) the type of
2498            the constant must have according to the constant suffix(es) */
2499         lcount = ucount = 0;
2500         p1 = p;
2501         for(;;) {
2502             t = toup(ch);
2503             if (t == 'L') {
2504                 if (lcount >= 2)
2505                     tcc_error("three 'l's in integer constant");
2506                 if (lcount && *(p - 1) != ch)
2507                     tcc_error("incorrect integer suffix: %s", p1);
2508                 lcount++;
2509                 ch = *p++;
2510             } else if (t == 'U') {
2511                 if (ucount >= 1)
2512                     tcc_error("two 'u's in integer constant");
2513                 ucount++;
2514                 ch = *p++;
2515             } else {
2516                 break;
2517             }
2518         }
2519
2520         /* Determine if it needs 64 bits and/or unsigned in order to fit */
2521         if (ucount == 0 && b == 10) {
2522             if (lcount <= (LONG_SIZE == 4)) {
2523                 if (n >= 0x80000000U)
2524                     lcount = (LONG_SIZE == 4) + 1;
2525             }
2526             if (n >= 0x8000000000000000ULL)
2527                 ov = 1, ucount = 1;
2528         } else {
2529             if (lcount <= (LONG_SIZE == 4)) {
2530                 if (n >= 0x100000000ULL)
2531                     lcount = (LONG_SIZE == 4) + 1;
2532                 else if (n >= 0x80000000U)
2533                     ucount = 1;
2534             }
2535             if (n >= 0x8000000000000000ULL)
2536                 ucount = 1;
2537         }
2538
2539         if (ov)
2540             tcc_warning("integer constant overflow");
2541
2542         tok = TOK_CINT;
2543         if (lcount) {
2544             tok = TOK_CLONG;
2545             if (lcount == 2)
2546                 tok = TOK_CLLONG;
2547         }
2548         if (ucount)
2549             ++tok; /* TOK_CU... */
2550         tokc.i = n;
2551     }
2552     if (ch)
2553         tcc_error("invalid number");
2554 }
2555
2556
2557 #define PARSE2(c1, tok1, c2, tok2)              \
2558     case c1:                                    \
2559         PEEKC(c, p);                            \
2560         if (c == c2) {                          \
2561             p++;                                \
2562             tok = tok2;                         \
2563         } else {                                \
2564             tok = tok1;                         \
2565         }                                       \
2566         break;
2567
2568 /* return next token without macro substitution */
2569 static inline void next_nomacro1(void)
2570 {
2571     int t, c, is_long, len;
2572     TokenSym *ts;
2573     uint8_t *p, *p1;
2574     unsigned int h;
2575
2576     p = file->buf_ptr;
2577  redo_no_start:
2578     c = *p;
2579     switch(c) {
2580     case ' ':
2581     case '\t':
2582         tok = c;
2583         p++;
2584  maybe_space:
2585         if (parse_flags & PARSE_FLAG_SPACES)
2586             goto keep_tok_flags;
2587         while (isidnum_table[*p - CH_EOF] & IS_SPC)
2588             ++p;
2589         goto redo_no_start;
2590     case '\f':
2591     case '\v':
2592     case '\r':
2593         p++;
2594         goto redo_no_start;
2595     case '\\':
2596         /* first look if it is in fact an end of buffer */
2597         c = handle_stray(&p);
2598         if (c == '\\')
2599             goto parse_simple;
2600         if (c == CH_EOF) {
2601             TCCState *s1 = tcc_state;
2602             if ((parse_flags & PARSE_FLAG_LINEFEED)
2603                 && !(tok_flags & TOK_FLAG_EOF)) {
2604                 tok_flags |= TOK_FLAG_EOF;
2605                 tok = TOK_LINEFEED;
2606                 goto keep_tok_flags;
2607             } else if (!(parse_flags & PARSE_FLAG_PREPROCESS)) {
2608                 tok = TOK_EOF;
2609             } else if (s1->ifdef_stack_ptr != file->ifdef_stack_ptr) {
2610                 tcc_error("missing #endif");
2611             } else if (s1->include_stack_ptr == s1->include_stack) {
2612                 /* no include left : end of file. */
2613                 tok = TOK_EOF;
2614             } else {
2615                 tok_flags &= ~TOK_FLAG_EOF;
2616                 /* pop include file */
2617
2618                 /* test if previous '#endif' was after a #ifdef at
2619                    start of file */
2620                 if (tok_flags & TOK_FLAG_ENDIF) {
2621 #ifdef INC_DEBUG
2622                     printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
2623 #endif
2624                     search_cached_include(s1, file->filename, 1)
2625                         ->ifndef_macro = file->ifndef_macro_saved;
2626                     tok_flags &= ~TOK_FLAG_ENDIF;
2627                 }
2628
2629                 /* add end of include file debug info */
2630                 tcc_debug_eincl(tcc_state);
2631                 /* pop include stack */
2632                 tcc_close();
2633                 s1->include_stack_ptr--;
2634                 p = file->buf_ptr;
2635                 if (p == file->buffer)
2636                     tok_flags = TOK_FLAG_BOF|TOK_FLAG_BOL;
2637                 goto redo_no_start;
2638             }
2639         } else {
2640             goto redo_no_start;
2641         }
2642         break;
2643
2644     case '\n':
2645         file->line_num++;
2646         tok_flags |= TOK_FLAG_BOL;
2647         p++;
2648 maybe_newline:
2649         if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
2650             goto redo_no_start;
2651         tok = TOK_LINEFEED;
2652         goto keep_tok_flags;
2653
2654     case '#':
2655         /* XXX: simplify */
2656         PEEKC(c, p);
2657         if ((tok_flags & TOK_FLAG_BOL) &&
2658             (parse_flags & PARSE_FLAG_PREPROCESS)) {
2659             file->buf_ptr = p;
2660             preprocess(tok_flags & TOK_FLAG_BOF);
2661             p = file->buf_ptr;
2662             goto maybe_newline;
2663         } else {
2664             if (c == '#') {
2665                 p++;
2666                 tok = TOK_TWOSHARPS;
2667             } else {
2668 #if !defined(TCC_TARGET_ARM)
2669                 if (parse_flags & PARSE_FLAG_ASM_FILE) {
2670                     p = parse_line_comment(p - 1);
2671                     goto redo_no_start;
2672                 } else
2673 #endif
2674                 {
2675                     tok = '#';
2676                 }
2677             }
2678         }
2679         break;
2680
2681     /* dollar is allowed to start identifiers when not parsing asm */
2682     case '$':
2683         if (!(isidnum_table[c - CH_EOF] & IS_ID)
2684          || (parse_flags & PARSE_FLAG_ASM_FILE))
2685             goto parse_simple;
2686
2687     case 'a': case 'b': case 'c': case 'd':
2688     case 'e': case 'f': case 'g': case 'h':
2689     case 'i': case 'j': case 'k': case 'l':
2690     case 'm': case 'n': case 'o': case 'p':
2691     case 'q': case 'r': case 's': case 't':
2692     case 'u': case 'v': case 'w': case 'x':
2693     case 'y': case 'z':
2694     case 'A': case 'B': case 'C': case 'D':
2695     case 'E': case 'F': case 'G': case 'H':
2696     case 'I': case 'J': case 'K':
2697     case 'M': case 'N': case 'O': case 'P':
2698     case 'Q': case 'R': case 'S': case 'T':
2699     case 'U': case 'V': case 'W': case 'X':
2700     case 'Y': case 'Z':
2701     case '_':
2702     parse_ident_fast:
2703         p1 = p;
2704         h = TOK_HASH_INIT;
2705         h = TOK_HASH_FUNC(h, c);
2706         while (c = *++p, isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2707             h = TOK_HASH_FUNC(h, c);
2708         len = p - p1;
2709         if (c != '\\') {
2710             TokenSym **pts;
2711
2712             /* fast case : no stray found, so we have the full token
2713                and we have already hashed it */
2714             h &= (TOK_HASH_SIZE - 1);
2715             pts = &hash_ident[h];
2716             for(;;) {
2717                 ts = *pts;
2718                 if (!ts)
2719                     break;
2720                 if (ts->len == len && !memcmp(ts->str, p1, len))
2721                     goto token_found;
2722                 pts = &(ts->hash_next);
2723             }
2724             ts = tok_alloc_new(pts, (char *) p1, len);
2725         token_found: ;
2726         } else {
2727             /* slower case */
2728             cstr_reset(&tokcstr);
2729             cstr_cat(&tokcstr, (char *) p1, len);
2730             p--;
2731             PEEKC(c, p);
2732         parse_ident_slow:
2733             while (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2734             {
2735                 cstr_ccat(&tokcstr, c);
2736                 PEEKC(c, p);
2737             }
2738             ts = tok_alloc(tokcstr.data, tokcstr.size);
2739         }
2740         tok = ts->tok;
2741         break;
2742     case 'L':
2743         t = p[1];
2744         if (t != '\\' && t != '\'' && t != '\"') {
2745             /* fast case */
2746             goto parse_ident_fast;
2747         } else {
2748             PEEKC(c, p);
2749             if (c == '\'' || c == '\"') {
2750                 is_long = 1;
2751                 goto str_const;
2752             } else {
2753                 cstr_reset(&tokcstr);
2754                 cstr_ccat(&tokcstr, 'L');
2755                 goto parse_ident_slow;
2756             }
2757         }
2758         break;
2759
2760     case '0': case '1': case '2': case '3':
2761     case '4': case '5': case '6': case '7':
2762     case '8': case '9':
2763         t = c;
2764         PEEKC(c, p);
2765         /* after the first digit, accept digits, alpha, '.' or sign if
2766            prefixed by 'eEpP' */
2767     parse_num:
2768         cstr_reset(&tokcstr);
2769         for(;;) {
2770             cstr_ccat(&tokcstr, t);
2771             if (!((isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))
2772                   || c == '.'
2773                   || ((c == '+' || c == '-')
2774                       && (((t == 'e' || t == 'E')
2775                             && !(parse_flags & PARSE_FLAG_ASM_FILE
2776                                 /* 0xe+1 is 3 tokens in asm */
2777                                 && ((char*)tokcstr.data)[0] == '0'
2778                                 && toup(((char*)tokcstr.data)[1]) == 'X'))
2779                           || t == 'p' || t == 'P'))))
2780                 break;
2781             t = c;
2782             PEEKC(c, p);
2783         }
2784         /* We add a trailing '\0' to ease parsing */
2785         cstr_ccat(&tokcstr, '\0');
2786         tokc.str.size = tokcstr.size;
2787         tokc.str.data = tokcstr.data;
2788         tok = TOK_PPNUM;
2789         break;
2790
2791     case '.':
2792         /* special dot handling because it can also start a number */
2793         PEEKC(c, p);
2794         if (isnum(c)) {
2795             t = '.';
2796             goto parse_num;
2797         } else if ((isidnum_table['.' - CH_EOF] & IS_ID)
2798                    && (isidnum_table[c - CH_EOF] & (IS_ID|IS_NUM))) {
2799             *--p = c = '.';
2800             goto parse_ident_fast;
2801         } else if (c == '.') {
2802             PEEKC(c, p);
2803             if (c == '.') {
2804                 p++;
2805                 tok = TOK_DOTS;
2806             } else {
2807                 *--p = '.'; /* may underflow into file->unget[] */
2808                 tok = '.';
2809             }
2810         } else {
2811             tok = '.';
2812         }
2813         break;
2814     case '\'':
2815     case '\"':
2816         is_long = 0;
2817     str_const:
2818         cstr_reset(&tokcstr);
2819         if (is_long)
2820             cstr_ccat(&tokcstr, 'L');
2821         cstr_ccat(&tokcstr, c);
2822         p = parse_pp_string(p, c, &tokcstr);
2823         cstr_ccat(&tokcstr, c);
2824         cstr_ccat(&tokcstr, '\0');
2825         tokc.str.size = tokcstr.size;
2826         tokc.str.data = tokcstr.data;
2827         tok = TOK_PPSTR;
2828         break;
2829
2830     case '<':
2831         PEEKC(c, p);
2832         if (c == '=') {
2833             p++;
2834             tok = TOK_LE;
2835         } else if (c == '<') {
2836             PEEKC(c, p);
2837             if (c == '=') {
2838                 p++;
2839                 tok = TOK_A_SHL;
2840             } else {
2841                 tok = TOK_SHL;
2842             }
2843         } else {
2844             tok = TOK_LT;
2845         }
2846         break;
2847     case '>':
2848         PEEKC(c, p);
2849         if (c == '=') {
2850             p++;
2851             tok = TOK_GE;
2852         } else if (c == '>') {
2853             PEEKC(c, p);
2854             if (c == '=') {
2855                 p++;
2856                 tok = TOK_A_SAR;
2857             } else {
2858                 tok = TOK_SAR;
2859             }
2860         } else {
2861             tok = TOK_GT;
2862         }
2863         break;
2864
2865     case '&':
2866         PEEKC(c, p);
2867         if (c == '&') {
2868             p++;
2869             tok = TOK_LAND;
2870         } else if (c == '=') {
2871             p++;
2872             tok = TOK_A_AND;
2873         } else {
2874             tok = '&';
2875         }
2876         break;
2877
2878     case '|':
2879         PEEKC(c, p);
2880         if (c == '|') {
2881             p++;
2882             tok = TOK_LOR;
2883         } else if (c == '=') {
2884             p++;
2885             tok = TOK_A_OR;
2886         } else {
2887             tok = '|';
2888         }
2889         break;
2890
2891     case '+':
2892         PEEKC(c, p);
2893         if (c == '+') {
2894             p++;
2895             tok = TOK_INC;
2896         } else if (c == '=') {
2897             p++;
2898             tok = TOK_A_ADD;
2899         } else {
2900             tok = '+';
2901         }
2902         break;
2903
2904     case '-':
2905         PEEKC(c, p);
2906         if (c == '-') {
2907             p++;
2908             tok = TOK_DEC;
2909         } else if (c == '=') {
2910             p++;
2911             tok = TOK_A_SUB;
2912         } else if (c == '>') {
2913             p++;
2914             tok = TOK_ARROW;
2915         } else {
2916             tok = '-';
2917         }
2918         break;
2919
2920     PARSE2('!', '!', '=', TOK_NE)
2921     PARSE2('=', '=', '=', TOK_EQ)
2922     PARSE2('*', '*', '=', TOK_A_MUL)
2923     PARSE2('%', '%', '=', TOK_A_MOD)
2924     PARSE2('^', '^', '=', TOK_A_XOR)
2925
2926         /* comments or operator */
2927     case '/':
2928         PEEKC(c, p);
2929         if (c == '*') {
2930             p = parse_comment(p);
2931             /* comments replaced by a blank */
2932             tok = ' ';
2933             goto maybe_space;
2934         } else if (c == '/') {
2935             p = parse_line_comment(p);
2936             tok = ' ';
2937             goto maybe_space;
2938         } else if (c == '=') {
2939             p++;
2940             tok = TOK_A_DIV;
2941         } else {
2942             tok = '/';
2943         }
2944         break;
2945
2946         /* simple tokens */
2947     case '(':
2948     case ')':
2949     case '[':
2950     case ']':
2951     case '{':
2952     case '}':
2953     case ',':
2954     case ';':
2955     case ':':
2956     case '?':
2957     case '~':
2958     case '@': /* only used in assembler */
2959     parse_simple:
2960         tok = c;
2961         p++;
2962         break;
2963     default:
2964         if (c >= 0x80 && c <= 0xFF) /* utf8 identifiers */
2965             goto parse_ident_fast;
2966         if (parse_flags & PARSE_FLAG_ASM_FILE)
2967             goto parse_simple;
2968         tcc_error("unrecognized character \\x%02x", c);
2969         break;
2970     }
2971     tok_flags = 0;
2972 keep_tok_flags:
2973     file->buf_ptr = p;
2974 #if defined(PARSE_DEBUG)
2975     printf("token = %d %s\n", tok, get_tok_str(tok, &tokc));
2976 #endif
2977 }
2978
2979 static void macro_subst(
2980     TokenString *tok_str,
2981     Sym **nested_list,
2982     const int *macro_str
2983     );
2984
2985 /* substitute arguments in replacement lists in macro_str by the values in
2986    args (field d) and return allocated string */
2987 static int *macro_arg_subst(Sym **nested_list, const int *macro_str, Sym *args)
2988 {
2989     int t, t0, t1, spc;
2990     const int *st;
2991     Sym *s;
2992     CValue cval;
2993     TokenString str;
2994     CString cstr;
2995
2996     tok_str_new(&str);
2997     t0 = t1 = 0;
2998     while(1) {
2999         TOK_GET(&t, &macro_str, &cval);
3000         if (!t)
3001             break;
3002         if (t == '#') {
3003             /* stringize */
3004             TOK_GET(&t, &macro_str, &cval);
3005             if (!t)
3006                 goto bad_stringy;
3007             s = sym_find2(args, t);
3008             if (s) {
3009                 cstr_new(&cstr);
3010                 cstr_ccat(&cstr, '\"');
3011                 st = s->d;
3012                 spc = 0;
3013                 while (*st >= 0) {
3014                     TOK_GET(&t, &st, &cval);
3015                     if (t != TOK_PLCHLDR
3016                      && t != TOK_NOSUBST
3017                      && 0 == check_space(t, &spc)) {
3018                         const char *s = get_tok_str(t, &cval);
3019                         while (*s) {
3020                             if (t == TOK_PPSTR && *s != '\'')
3021                                 add_char(&cstr, *s);
3022                             else
3023                                 cstr_ccat(&cstr, *s);
3024                             ++s;
3025                         }
3026                     }
3027                 }
3028                 cstr.size -= spc;
3029                 cstr_ccat(&cstr, '\"');
3030                 cstr_ccat(&cstr, '\0');
3031 #ifdef PP_DEBUG
3032                 printf("\nstringize: <%s>\n", (char *)cstr.data);
3033 #endif
3034                 /* add string */
3035                 cval.str.size = cstr.size;
3036                 cval.str.data = cstr.data;
3037                 tok_str_add2(&str, TOK_PPSTR, &cval);
3038                 cstr_free(&cstr);
3039             } else {
3040         bad_stringy:
3041                 expect("macro parameter after '#'");
3042             }
3043         } else if (t >= TOK_IDENT) {
3044             s = sym_find2(args, t);
3045             if (s) {
3046                 int l0 = str.len;
3047                 st = s->d;
3048                 /* if '##' is present before or after, no arg substitution */
3049                 if (*macro_str == TOK_PPJOIN || t1 == TOK_PPJOIN) {
3050                     /* special case for var arg macros : ## eats the ','
3051                        if empty VA_ARGS variable. */
3052                     if (t1 == TOK_PPJOIN && t0 == ',' && gnu_ext && s->type.t) {
3053                         if (*st <= 0) {
3054                             /* suppress ',' '##' */
3055                             str.len -= 2;
3056                         } else {
3057                             /* suppress '##' and add variable */
3058                             str.len--;
3059                             goto add_var;
3060                         }
3061                     }
3062                 } else {
3063             add_var:
3064                     if (!s->next) {
3065                         /* Expand arguments tokens and store them.  In most
3066                            cases we could also re-expand each argument if
3067                            used multiple times, but not if the argument
3068                            contains the __COUNTER__ macro.  */
3069                         TokenString str2;
3070                         sym_push2(&s->next, s->v, s->type.t, 0);
3071                         tok_str_new(&str2);
3072                         macro_subst(&str2, nested_list, st);
3073                         tok_str_add(&str2, 0);
3074                         s->next->d = str2.str;
3075                     }
3076                     st = s->next->d;
3077                 }
3078                 for(;;) {
3079                     int t2;
3080                     TOK_GET(&t2, &st, &cval);
3081                     if (t2 <= 0)
3082                         break;
3083                     tok_str_add2(&str, t2, &cval);
3084                 }
3085                 if (str.len == l0) /* expanded to empty string */
3086                     tok_str_add(&str, TOK_PLCHLDR);
3087             } else {
3088                 tok_str_add(&str, t);
3089             }
3090         } else {
3091             tok_str_add2(&str, t, &cval);
3092         }
3093         t0 = t1, t1 = t;
3094     }
3095     tok_str_add(&str, 0);
3096     return str.str;
3097 }
3098
3099 static char const ab_month_name[12][4] =
3100 {
3101     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
3102     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
3103 };
3104
3105 static int paste_tokens(int t1, CValue *v1, int t2, CValue *v2)
3106 {
3107     CString cstr;
3108     int n, ret = 1;
3109
3110     cstr_new(&cstr);
3111     if (t1 != TOK_PLCHLDR)
3112         cstr_cat(&cstr, get_tok_str(t1, v1), -1);
3113     n = cstr.size;
3114     if (t2 != TOK_PLCHLDR)
3115         cstr_cat(&cstr, get_tok_str(t2, v2), -1);
3116     cstr_ccat(&cstr, '\0');
3117
3118     tcc_open_bf(tcc_state, ":paste:", cstr.size);
3119     memcpy(file->buffer, cstr.data, cstr.size);
3120     tok_flags = 0;
3121     for (;;) {
3122         next_nomacro1();
3123         if (0 == *file->buf_ptr)
3124             break;
3125         if (is_space(tok))
3126             continue;
3127         tcc_warning("pasting \"%.*s\" and \"%s\" does not give a valid"
3128             " preprocessing token", n, (char *)cstr.data, (char*)cstr.data + n);
3129         ret = 0;
3130         break;
3131     }
3132     tcc_close();
3133     //printf("paste <%s>\n", (char*)cstr.data);
3134     cstr_free(&cstr);
3135     return ret;
3136 }
3137
3138 /* handle the '##' operator. Return NULL if no '##' seen. Otherwise
3139    return the resulting string (which must be freed). */
3140 static inline int *macro_twosharps(const int *ptr0)
3141 {
3142     int t;
3143     CValue cval;
3144     TokenString macro_str1;
3145     int start_of_nosubsts = -1;
3146     const int *ptr;
3147
3148     /* we search the first '##' */
3149     for (ptr = ptr0;;) {
3150         TOK_GET(&t, &ptr, &cval);
3151         if (t == TOK_PPJOIN)
3152             break;
3153         if (t == 0)
3154             return NULL;
3155     }
3156
3157     tok_str_new(&macro_str1);
3158
3159     //tok_print(" $$$", ptr0);
3160     for (ptr = ptr0;;) {
3161         TOK_GET(&t, &ptr, &cval);
3162         if (t == 0)
3163             break;
3164         if (t == TOK_PPJOIN)
3165             continue;
3166         while (*ptr == TOK_PPJOIN) {
3167             int t1; CValue cv1;
3168             /* given 'a##b', remove nosubsts preceding 'a' */
3169             if (start_of_nosubsts >= 0)
3170                 macro_str1.len = start_of_nosubsts;
3171             /* given 'a##b', remove nosubsts preceding 'b' */
3172             while ((t1 = *++ptr) == TOK_NOSUBST)
3173                 ;
3174             if (t1 && t1 != TOK_PPJOIN) {
3175                 TOK_GET(&t1, &ptr, &cv1);
3176                 if (t != TOK_PLCHLDR || t1 != TOK_PLCHLDR) {
3177                     if (paste_tokens(t, &cval, t1, &cv1)) {
3178                         t = tok, cval = tokc;
3179                     } else {
3180                         tok_str_add2(&macro_str1, t, &cval);
3181                         t = t1, cval = cv1;
3182                     }
3183                 }
3184             }
3185         }
3186         if (t == TOK_NOSUBST) {
3187             if (start_of_nosubsts < 0)
3188                 start_of_nosubsts = macro_str1.len;
3189         } else {
3190             start_of_nosubsts = -1;
3191         }
3192         tok_str_add2(&macro_str1, t, &cval);
3193     }
3194     tok_str_add(&macro_str1, 0);
3195     //tok_print(" ###", macro_str1.str);
3196     return macro_str1.str;
3197 }
3198
3199 /* peek or read [ws_str == NULL] next token from function macro call,
3200    walking up macro levels up to the file if necessary */
3201 static int next_argstream(Sym **nested_list, TokenString *ws_str)
3202 {
3203     int t;
3204     const int *p;
3205     Sym *sa;
3206
3207     for (;;) {
3208         if (macro_ptr) {
3209             p = macro_ptr, t = *p;
3210             if (ws_str) {
3211                 while (is_space(t) || TOK_LINEFEED == t || TOK_PLCHLDR == t)
3212                     tok_str_add(ws_str, t), t = *++p;
3213             }
3214             if (t == 0) {
3215                 end_macro();
3216                 /* also, end of scope for nested defined symbol */
3217                 sa = *nested_list;
3218                 while (sa && sa->v == 0)
3219                     sa = sa->prev;
3220                 if (sa)
3221                     sa->v = 0;
3222                 continue;
3223             }
3224         } else {
3225             uint8_t *p = file->buf_ptr;
3226             int ch = handle_bs(&p);
3227             if (ws_str) {
3228                 while (is_space(ch) || ch == '\n' || ch == '/') {
3229                     if (ch == '/') {
3230                         int c;
3231                         PEEKC(c, p);
3232                         if (c == '*') {
3233                             p = parse_comment(p) - 1;
3234                         } else if (c == '/') {
3235                             p = parse_line_comment(p) - 1;
3236                         } else {
3237                             break;
3238                         }
3239                         ch = ' ';
3240                     }
3241                     if (ch == '\n')
3242                         file->line_num++;
3243                     if (!(ch == '\f' || ch == '\v' || ch == '\r'))
3244                         tok_str_add(ws_str, ch);
3245                     PEEKC(ch, p);
3246                 }
3247             }
3248             file->buf_ptr = p;
3249             t = ch;
3250         }
3251
3252         if (ws_str)
3253             return t;
3254         next_nomacro();
3255         return tok;
3256     }
3257 }
3258
3259 /* do macro substitution of current token with macro 's' and add
3260    result to (tok_str,tok_len). 'nested_list' is the list of all
3261    macros we got inside to avoid recursing. Return non zero if no
3262    substitution needs to be done */
3263 static int macro_subst_tok(
3264     TokenString *tok_str,
3265     Sym **nested_list,
3266     Sym *s)
3267 {
3268     Sym *args, *sa, *sa1;
3269     int parlevel, t, t1, spc;
3270     TokenString str;
3271     char *cstrval;
3272     CValue cval;
3273     CString cstr;
3274     char buf[32];
3275
3276     /* if symbol is a macro, prepare substitution */
3277     /* special macros */
3278     if (tok == TOK___LINE__ || tok == TOK___COUNTER__) {
3279         t = tok == TOK___LINE__ ? file->line_num : pp_counter++;
3280         snprintf(buf, sizeof(buf), "%d", t);
3281         cstrval = buf;
3282         t1 = TOK_PPNUM;
3283         goto add_cstr1;
3284     } else if (tok == TOK___FILE__) {
3285         cstrval = file->filename;
3286         goto add_cstr;
3287     } else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
3288         time_t ti;
3289         struct tm *tm;
3290
3291         time(&ti);
3292         tm = localtime(&ti);
3293         if (tok == TOK___DATE__) {
3294             snprintf(buf, sizeof(buf), "%s %2d %d",
3295                      ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
3296         } else {
3297             snprintf(buf, sizeof(buf), "%02d:%02d:%02d",
3298                      tm->tm_hour, tm->tm_min, tm->tm_sec);
3299         }
3300         cstrval = buf;
3301     add_cstr:
3302         t1 = TOK_STR;
3303     add_cstr1:
3304         cstr_new(&cstr);
3305         cstr_cat(&cstr, cstrval, 0);
3306         cval.str.size = cstr.size;
3307         cval.str.data = cstr.data;
3308         tok_str_add2(tok_str, t1, &cval);
3309         cstr_free(&cstr);
3310     } else if (s->d) {
3311         int saved_parse_flags = parse_flags;
3312         int *joined_str = NULL;
3313         int *mstr = s->d;
3314
3315         if (s->type.t == MACRO_FUNC) {
3316             /* whitespace between macro name and argument list */
3317             TokenString ws_str;
3318             tok_str_new(&ws_str);
3319
3320             spc = 0;
3321             parse_flags |= PARSE_FLAG_SPACES | PARSE_FLAG_LINEFEED
3322                 | PARSE_FLAG_ACCEPT_STRAYS;
3323
3324             /* get next token from argument stream */
3325             t = next_argstream(nested_list, &ws_str);
3326             if (t != '(') {
3327                 /* not a macro substitution after all, restore the
3328                  * macro token plus all whitespace we've read.
3329                  * whitespace is intentionally not merged to preserve
3330                  * newlines. */
3331                 parse_flags = saved_parse_flags;
3332                 tok_str_add(tok_str, tok);
3333                 if (parse_flags & PARSE_FLAG_SPACES) {
3334                     int i;
3335                     for (i = 0; i < ws_str.len; i++)
3336                         tok_str_add(tok_str, ws_str.str[i]);
3337                 }
3338                 tok_str_free_str(ws_str.str);
3339                 return 0;
3340             } else {
3341                 tok_str_free_str(ws_str.str);
3342             }
3343             do {
3344                 next_nomacro(); /* eat '(' */
3345             } while (tok == TOK_PLCHLDR || is_space(tok));
3346
3347             /* argument macro */
3348             args = NULL;
3349             sa = s->next;
3350             /* NOTE: empty args are allowed, except if no args */
3351             for(;;) {
3352                 do {
3353                     next_argstream(nested_list, NULL);
3354                 } while (tok == TOK_PLCHLDR || is_space(tok) ||
3355                          TOK_LINEFEED == tok);
3356     empty_arg:
3357                 /* handle '()' case */
3358                 if (!args && !sa && tok == ')')
3359                     break;
3360                 if (!sa)
3361                     tcc_error("macro '%s' used with too many args",
3362                           get_tok_str(s->v, 0));
3363                 tok_str_new(&str);
3364                 parlevel = spc = 0;
3365                 /* NOTE: non zero sa->t indicates VA_ARGS */
3366                 while ((parlevel > 0 ||
3367                         (tok != ')' &&
3368                          (tok != ',' || sa->type.t)))) {
3369                     if (tok == TOK_EOF || tok == 0)
3370                         break;
3371                     if (tok == '(')
3372                         parlevel++;
3373                     else if (tok == ')')
3374                         parlevel--;
3375                     if (tok == TOK_LINEFEED)
3376                         tok = ' ';
3377                     if (!check_space(tok, &spc))
3378                         tok_str_add2(&str, tok, &tokc);
3379                     next_argstream(nested_list, NULL);
3380                 }
3381                 if (parlevel)
3382                     expect(")");
3383                 str.len -= spc;
3384                 tok_str_add(&str, -1);
3385                 tok_str_add(&str, 0);
3386                 sa1 = sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, 0);
3387                 sa1->d = str.str;
3388                 sa = sa->next;
3389                 if (tok == ')') {
3390                     /* special case for gcc var args: add an empty
3391                        var arg argument if it is omitted */
3392                     if (sa && sa->type.t && gnu_ext)
3393                         goto empty_arg;
3394                     break;
3395                 }
3396                 if (tok != ',')
3397                     expect(",");
3398             }
3399             if (sa) {
3400                 tcc_error("macro '%s' used with too few args",
3401                       get_tok_str(s->v, 0));
3402             }
3403
3404             /* now subst each arg */
3405             mstr = macro_arg_subst(nested_list, mstr, args);
3406             /* free memory */
3407             sa = args;
3408             while (sa) {
3409                 sa1 = sa->prev;
3410                 tok_str_free_str(sa->d);
3411                 if (sa->next) {
3412                     tok_str_free_str(sa->next->d);
3413                     sym_free(sa->next);
3414                 }
3415                 sym_free(sa);
3416                 sa = sa1;
3417             }
3418             parse_flags = saved_parse_flags;
3419         }
3420
3421         sym_push2(nested_list, s->v, 0, 0);
3422         parse_flags = saved_parse_flags;
3423         joined_str = macro_twosharps(mstr);
3424         macro_subst(tok_str, nested_list, joined_str ? joined_str : mstr);
3425
3426         /* pop nested defined symbol */
3427         sa1 = *nested_list;
3428         *nested_list = sa1->prev;
3429         sym_free(sa1);
3430         if (joined_str)
3431             tok_str_free_str(joined_str);
3432         if (mstr != s->d)
3433             tok_str_free_str(mstr);
3434     }
3435     return 0;
3436 }
3437
3438 /* do macro substitution of macro_str and add result to
3439    (tok_str,tok_len). 'nested_list' is the list of all macros we got
3440    inside to avoid recursing. */
3441 static void macro_subst(
3442     TokenString *tok_str,
3443     Sym **nested_list,
3444     const int *macro_str
3445     )
3446 {
3447     Sym *s;
3448     int t, spc, nosubst;
3449     CValue cval;
3450
3451     spc = nosubst = 0;
3452
3453     while (1) {
3454         TOK_GET(&t, &macro_str, &cval);
3455         if (t <= 0)
3456             break;
3457
3458         if (t >= TOK_IDENT && 0 == nosubst) {
3459             s = define_find(t);
3460             if (s == NULL)
3461                 goto no_subst;
3462
3463             /* if nested substitution, do nothing */
3464             if (sym_find2(*nested_list, t)) {
3465                 /* and mark it as TOK_NOSUBST, so it doesn't get subst'd again */
3466                 tok_str_add2(tok_str, TOK_NOSUBST, NULL);
3467                 goto no_subst;
3468             }
3469
3470             {
3471                 TokenString *str = tok_str_alloc();
3472                 str->str = (int*)macro_str;
3473                 begin_macro(str, 2);
3474
3475                 tok = t;
3476                 macro_subst_tok(tok_str, nested_list, s);
3477
3478                 if (macro_stack != str) {
3479                     /* already finished by reading function macro arguments */
3480                     break;
3481                 }
3482
3483                 macro_str = macro_ptr;
3484                 end_macro ();
3485             }
3486             if (tok_str->len)
3487                 spc = is_space(t = tok_str->str[tok_str->lastlen]);
3488         } else {
3489 no_subst:
3490             if (!check_space(t, &spc))
3491                 tok_str_add2(tok_str, t, &cval);
3492
3493             if (nosubst) {
3494                 if (nosubst > 1 && (spc || (++nosubst == 3 && t == '(')))
3495                     continue;
3496                 nosubst = 0;
3497             }
3498             if (t == TOK_NOSUBST)
3499                 nosubst = 1;
3500         }
3501         /* GCC supports 'defined' as result of a macro substitution */
3502         if (t == TOK_DEFINED && pp_expr)
3503             nosubst = 2;
3504     }
3505 }
3506
3507 /* return next token without macro substitution. Can read input from
3508    macro_ptr buffer */
3509 static void next_nomacro(void)
3510 {
3511     int t;
3512     if (macro_ptr) {
3513  redo:
3514         t = *macro_ptr;
3515         if (TOK_HAS_VALUE(t)) {
3516             tok_get(&tok, &macro_ptr, &tokc);
3517             if (t == TOK_LINENUM) {
3518                 file->line_num = tokc.i;
3519                 goto redo;
3520             }
3521         } else {
3522             macro_ptr++;
3523             if (t < TOK_IDENT) {
3524                 if (!(parse_flags & PARSE_FLAG_SPACES)
3525                     && (isidnum_table[t - CH_EOF] & IS_SPC))
3526                     goto redo;
3527             }
3528             tok = t;
3529         }
3530     } else {
3531         next_nomacro1();
3532     }
3533 }
3534
3535 /* return next token with macro substitution */
3536 ST_FUNC void next(void)
3537 {
3538     int t;
3539  redo:
3540     next_nomacro();
3541     t = tok;
3542     if (macro_ptr) {
3543         if (!TOK_HAS_VALUE(t)) {
3544             if (t == TOK_NOSUBST || t == TOK_PLCHLDR) {
3545                 /* discard preprocessor markers */
3546                 goto redo;
3547             } else if (t == 0) {
3548                 /* end of macro or unget token string */
3549                 end_macro();
3550                 goto redo;
3551             } else if (t == '\\') {
3552                 if (!(parse_flags & PARSE_FLAG_ACCEPT_STRAYS))
3553                     tcc_error("stray '\\' in program");
3554             }
3555             return;
3556         }
3557     } else if (t >= TOK_IDENT && (parse_flags & PARSE_FLAG_PREPROCESS)) {
3558         /* if reading from file, try to substitute macros */
3559         Sym *s = define_find(t);
3560         if (s) {
3561             Sym *nested_list = NULL;
3562             tokstr_buf.len = 0;
3563             macro_subst_tok(&tokstr_buf, &nested_list, s);
3564             tok_str_add(&tokstr_buf, 0);
3565             begin_macro(&tokstr_buf, 0);
3566             goto redo;
3567         }
3568         return;
3569     }
3570     /* convert preprocessor tokens into C tokens */
3571     if (t == TOK_PPNUM) {
3572         if  (parse_flags & PARSE_FLAG_TOK_NUM)
3573             parse_number((char *)tokc.str.data);
3574     } else if (t == TOK_PPSTR) {
3575         if (parse_flags & PARSE_FLAG_TOK_STR)
3576             parse_string((char *)tokc.str.data, tokc.str.size - 1);
3577     }
3578 }
3579
3580 /* push back current token and set current token to 'last_tok'. Only
3581    identifier case handled for labels. */
3582 ST_INLN void unget_tok(int last_tok)
3583 {
3584
3585     TokenString *str = tok_str_alloc();
3586     tok_str_add2(str, tok, &tokc);
3587     tok_str_add(str, 0);
3588     begin_macro(str, 1);
3589     tok = last_tok;
3590 }
3591
3592 /* ------------------------------------------------------------------------- */
3593 /* init preprocessor */
3594
3595 static const char * const target_os_defs =
3596 #ifdef TCC_TARGET_PE
3597     "_WIN32\0"
3598 # if PTR_SIZE == 8
3599     "_WIN64\0"
3600 # endif
3601 #else
3602 # if defined TCC_TARGET_MACHO
3603     "__APPLE__\0"
3604 # elif TARGETOS_FreeBSD
3605     "__FreeBSD__ 12\0"
3606 # elif TARGETOS_FreeBSD_kernel
3607     "__FreeBSD_kernel__\0"
3608 # elif TARGETOS_NetBSD
3609     "__NetBSD__\0"
3610 # elif TARGETOS_OpenBSD
3611     "__OpenBSD__\0"
3612 # else
3613     "__linux__\0"
3614     "__linux\0"
3615 #  if TARGETOS_ANDROID
3616     "__ANDROID__\0"
3617 #  endif
3618 # endif
3619     "__unix__\0"
3620     "__unix\0"
3621 #endif
3622     ;
3623
3624 static void putdef(CString *cs, const char *p)
3625 {
3626     cstr_printf(cs, "#define %s%s\n", p, &" 1"[!!strchr(p, ' ')*2]);
3627 }
3628
3629 static void putdefs(CString *cs, const char *p)
3630 {
3631     while (*p)
3632         putdef(cs, p), p = strchr(p, 0) + 1;
3633 }
3634
3635 static void tcc_predefs(TCCState *s1, CString *cs, int is_asm)
3636 {
3637     int a, b, c;
3638
3639     sscanf(TCC_VERSION, "%d.%d.%d", &a, &b, &c);
3640     cstr_printf(cs, "#define __TINYC__ %d\n", a*10000 + b*100 + c);
3641
3642     putdefs(cs, target_machine_defs);
3643     putdefs(cs, target_os_defs);
3644
3645 #ifdef TCC_TARGET_ARM
3646     if (s1->float_abi == ARM_HARD_FLOAT)
3647       putdef(cs, "__ARM_PCS_VFP");
3648 #endif
3649     if (is_asm)
3650       putdef(cs, "__ASSEMBLER__");
3651     if (s1->output_type == TCC_OUTPUT_PREPROCESS)
3652       putdef(cs, "__TCC_PP__");
3653     if (s1->output_type == TCC_OUTPUT_MEMORY)
3654       putdef(cs, "__TCC_RUN__");
3655     if (s1->char_is_unsigned)
3656       putdef(cs, "__CHAR_UNSIGNED__");
3657     if (s1->optimize > 0)
3658       putdef(cs, "__OPTIMIZE__");
3659     if (s1->option_pthread)
3660       putdef(cs, "_REENTRANT");
3661     if (s1->leading_underscore)
3662       putdef(cs, "__leading_underscore");
3663 #ifdef CONFIG_TCC_BCHECK
3664     if (s1->do_bounds_check)
3665       putdef(cs, "__BOUNDS_CHECKING_ON");
3666 #endif
3667 #ifdef CONFIG_TCC_BACKTRACE
3668     if (s1->do_backtrace)
3669       putdef(cs, "__TCC_BACKTRACE_ENABLED__");
3670 #endif
3671     cstr_printf(cs, "#define __SIZEOF_POINTER__ %d\n", PTR_SIZE);
3672     cstr_printf(cs, "#define __SIZEOF_LONG__ %d\n", LONG_SIZE);
3673     if (!is_asm) {
3674       putdef(cs, "__STDC__");
3675       cstr_printf(cs, "#define __STDC_VERSION__ %dL\n", s1->cversion);
3676       cstr_cat(cs,
3677         /* load more predefs and __builtins */
3678 #if CONFIG_TCC_PREDEFS
3679         #include "tccdefs_.h" /* include as strings */
3680 #else
3681         "#include <tccdefs.h>\n" /* load at runtime */
3682 #endif
3683         , -1);
3684     }
3685     cstr_printf(cs, "#define __BASE_FILE__ \"%s\"\n", file->filename);
3686 }
3687
3688 ST_FUNC void preprocess_start(TCCState *s1, int filetype)
3689 {
3690     int is_asm = !!(filetype & (AFF_TYPE_ASM|AFF_TYPE_ASMPP));
3691     CString cstr;
3692
3693     tccpp_new(s1);
3694
3695     s1->include_stack_ptr = s1->include_stack;
3696     s1->ifdef_stack_ptr = s1->ifdef_stack;
3697     file->ifdef_stack_ptr = s1->ifdef_stack_ptr;
3698     pp_expr = 0;
3699     pp_counter = 0;
3700     pp_debug_tok = pp_debug_symv = 0;
3701     pp_once++;
3702     s1->pack_stack[0] = 0;
3703     s1->pack_stack_ptr = s1->pack_stack;
3704
3705     set_idnum('$', !is_asm && s1->dollars_in_identifiers ? IS_ID : 0);
3706     set_idnum('.', is_asm ? IS_ID : 0);
3707
3708     if (!(filetype & AFF_TYPE_ASM)) {
3709         cstr_new(&cstr);
3710         tcc_predefs(s1, &cstr, is_asm);
3711         if (s1->cmdline_defs.size)
3712           cstr_cat(&cstr, s1->cmdline_defs.data, s1->cmdline_defs.size);
3713         if (s1->cmdline_incl.size)
3714           cstr_cat(&cstr, s1->cmdline_incl.data, s1->cmdline_incl.size);
3715         //printf("%s\n", (char*)cstr.data);
3716         *s1->include_stack_ptr++ = file;
3717         tcc_open_bf(s1, "<command line>", cstr.size);
3718         memcpy(file->buffer, cstr.data, cstr.size);
3719         cstr_free(&cstr);
3720     }
3721
3722     parse_flags = is_asm ? PARSE_FLAG_ASM_FILE : 0;
3723     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
3724 }
3725
3726 /* cleanup from error/setjmp */
3727 ST_FUNC void preprocess_end(TCCState *s1)
3728 {
3729     while (macro_stack)
3730         end_macro();
3731     macro_ptr = NULL;
3732     while (file)
3733         tcc_close();
3734     tccpp_delete(s1);
3735 }
3736
3737 ST_FUNC int set_idnum(int c, int val)
3738 {
3739     int prev = isidnum_table[c - CH_EOF];
3740     isidnum_table[c - CH_EOF] = val;
3741     return prev;
3742 }
3743
3744 ST_FUNC void tccpp_new(TCCState *s)
3745 {
3746     int i, c;
3747     const char *p, *r;
3748
3749     /* init isid table */
3750     for(i = CH_EOF; i<128; i++)
3751         set_idnum(i,
3752             is_space(i) ? IS_SPC
3753             : isid(i) ? IS_ID
3754             : isnum(i) ? IS_NUM
3755             : 0);
3756
3757     for(i = 128; i<256; i++)
3758         set_idnum(i, IS_ID);
3759
3760     /* init allocators */
3761     tal_new(&toksym_alloc, TOKSYM_TAL_LIMIT, TOKSYM_TAL_SIZE);
3762     tal_new(&tokstr_alloc, TOKSTR_TAL_LIMIT, TOKSTR_TAL_SIZE);
3763
3764     memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));
3765     memset(s->cached_includes_hash, 0, sizeof s->cached_includes_hash);
3766
3767     cstr_new(&cstr_buf);
3768     cstr_realloc(&cstr_buf, STRING_MAX_SIZE);
3769     tok_str_new(&tokstr_buf);
3770     tok_str_realloc(&tokstr_buf, TOKSTR_MAX_SIZE);
3771
3772     tok_ident = TOK_IDENT;
3773     p = tcc_keywords;
3774     while (*p) {
3775         r = p;
3776         for(;;) {
3777             c = *r++;
3778             if (c == '\0')
3779                 break;
3780         }
3781         tok_alloc(p, r - p - 1);
3782         p = r;
3783     }
3784
3785     /* we add dummy defines for some special macros to speed up tests
3786        and to have working defined() */
3787     define_push(TOK___LINE__, MACRO_OBJ, NULL, NULL);
3788     define_push(TOK___FILE__, MACRO_OBJ, NULL, NULL);
3789     define_push(TOK___DATE__, MACRO_OBJ, NULL, NULL);
3790     define_push(TOK___TIME__, MACRO_OBJ, NULL, NULL);
3791     define_push(TOK___COUNTER__, MACRO_OBJ, NULL, NULL);
3792 }
3793
3794 ST_FUNC void tccpp_delete(TCCState *s)
3795 {
3796     int i, n;
3797
3798     dynarray_reset(&s->cached_includes, &s->nb_cached_includes);
3799
3800     /* free tokens */
3801     n = tok_ident - TOK_IDENT;
3802     if (n > total_idents)
3803         total_idents = n;
3804     for(i = 0; i < n; i++)
3805         tal_free(toksym_alloc, table_ident[i]);
3806     tcc_free(table_ident);
3807     table_ident = NULL;
3808
3809     /* free static buffers */
3810     cstr_free(&tokcstr);
3811     cstr_free(&cstr_buf);
3812     cstr_free(&macro_equal_buf);
3813     tok_str_free_str(tokstr_buf.str);
3814
3815     /* free allocators */
3816     tal_delete(toksym_alloc);
3817     toksym_alloc = NULL;
3818     tal_delete(tokstr_alloc);
3819     tokstr_alloc = NULL;
3820 }
3821
3822 /* ------------------------------------------------------------------------- */
3823 /* tcc -E [-P[1]] [-dD} support */
3824
3825 static void tok_print(const char *msg, const int *str)
3826 {
3827     FILE *fp;
3828     int t, s = 0;
3829     CValue cval;
3830
3831     fp = tcc_state->ppfp;
3832     fprintf(fp, "%s", msg);
3833     while (str) {
3834         TOK_GET(&t, &str, &cval);
3835         if (!t)
3836             break;
3837         fprintf(fp, &" %s"[s], get_tok_str(t, &cval)), s = 1;
3838     }
3839     fprintf(fp, "\n");
3840 }
3841
3842 static void pp_line(TCCState *s1, BufferedFile *f, int level)
3843 {
3844     int d = f->line_num - f->line_ref;
3845
3846     if (s1->dflag & 4)
3847         return;
3848
3849     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_NONE) {
3850         ;
3851     } else if (level == 0 && f->line_ref && d < 8) {
3852         while (d > 0)
3853             fputs("\n", s1->ppfp), --d;
3854     } else if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_STD) {
3855         fprintf(s1->ppfp, "#line %d \"%s\"\n", f->line_num, f->filename);
3856     } else {
3857         fprintf(s1->ppfp, "# %d \"%s\"%s\n", f->line_num, f->filename,
3858             level > 0 ? " 1" : level < 0 ? " 2" : "");
3859     }
3860     f->line_ref = f->line_num;
3861 }
3862
3863 static void define_print(TCCState *s1, int v)
3864 {
3865     FILE *fp;
3866     Sym *s;
3867
3868     s = define_find(v);
3869     if (NULL == s || NULL == s->d)
3870         return;
3871
3872     fp = s1->ppfp;
3873     fprintf(fp, "#define %s", get_tok_str(v, NULL));
3874     if (s->type.t == MACRO_FUNC) {
3875         Sym *a = s->next;
3876         fprintf(fp,"(");
3877         if (a)
3878             for (;;) {
3879                 fprintf(fp,"%s", get_tok_str(a->v & ~SYM_FIELD, NULL));
3880                 if (!(a = a->next))
3881                     break;
3882                 fprintf(fp,",");
3883             }
3884         fprintf(fp,")");
3885     }
3886     tok_print("", s->d);
3887 }
3888
3889 static void pp_debug_defines(TCCState *s1)
3890 {
3891     int v, t;
3892     const char *vs;
3893     FILE *fp;
3894
3895     t = pp_debug_tok;
3896     if (t == 0)
3897         return;
3898
3899     file->line_num--;
3900     pp_line(s1, file, 0);
3901     file->line_ref = ++file->line_num;
3902
3903     fp = s1->ppfp;
3904     v = pp_debug_symv;
3905     vs = get_tok_str(v, NULL);
3906     if (t == TOK_DEFINE) {
3907         define_print(s1, v);
3908     } else if (t == TOK_UNDEF) {
3909         fprintf(fp, "#undef %s\n", vs);
3910     } else if (t == TOK_push_macro) {
3911         fprintf(fp, "#pragma push_macro(\"%s\")\n", vs);
3912     } else if (t == TOK_pop_macro) {
3913         fprintf(fp, "#pragma pop_macro(\"%s\")\n", vs);
3914     }
3915     pp_debug_tok = 0;
3916 }
3917
3918 static void pp_debug_builtins(TCCState *s1)
3919 {
3920     int v;
3921     for (v = TOK_IDENT; v < tok_ident; ++v)
3922         define_print(s1, v);
3923 }
3924
3925 /* Add a space between tokens a and b to avoid unwanted textual pasting */
3926 static int pp_need_space(int a, int b)
3927 {
3928     return 'E' == a ? '+' == b || '-' == b
3929         : '+' == a ? TOK_INC == b || '+' == b
3930         : '-' == a ? TOK_DEC == b || '-' == b
3931         : a >= TOK_IDENT ? b >= TOK_IDENT
3932         : a == TOK_PPNUM ? b >= TOK_IDENT
3933         : 0;
3934 }
3935
3936 /* maybe hex like 0x1e */
3937 static int pp_check_he0xE(int t, const char *p)
3938 {
3939     if (t == TOK_PPNUM && toup(strchr(p, 0)[-1]) == 'E')
3940         return 'E';
3941     return t;
3942 }
3943
3944 /* Preprocess the current file */
3945 ST_FUNC int tcc_preprocess(TCCState *s1)
3946 {
3947     BufferedFile **iptr;
3948     int token_seen, spcs, level;
3949     const char *p;
3950     char white[400];
3951
3952     parse_flags = PARSE_FLAG_PREPROCESS
3953                 | (parse_flags & PARSE_FLAG_ASM_FILE)
3954                 | PARSE_FLAG_LINEFEED
3955                 | PARSE_FLAG_SPACES
3956                 | PARSE_FLAG_ACCEPT_STRAYS
3957                 ;
3958     /* Credits to Fabrice Bellard's initial revision to demonstrate its
3959        capability to compile and run itself, provided all numbers are
3960        given as decimals. tcc -E -P10 will do. */
3961     if (s1->Pflag == LINE_MACRO_OUTPUT_FORMAT_P10)
3962         parse_flags |= PARSE_FLAG_TOK_NUM, s1->Pflag = 1;
3963
3964     if (s1->do_bench) {
3965         /* for PP benchmarks */
3966         do next(); while (tok != TOK_EOF);
3967         return 0;
3968     }
3969
3970     if (s1->dflag & 1) {
3971         pp_debug_builtins(s1);
3972         s1->dflag &= ~1;
3973     }
3974
3975     token_seen = TOK_LINEFEED, spcs = 0, level = 0;
3976     if (file->prev)
3977         pp_line(s1, file->prev, level++);
3978     pp_line(s1, file, level);
3979     for (;;) {
3980         iptr = s1->include_stack_ptr;
3981         next();
3982         if (tok == TOK_EOF)
3983             break;
3984
3985         level = s1->include_stack_ptr - iptr;
3986         if (level) {
3987             if (level > 0)
3988                 pp_line(s1, *iptr, 0);
3989             pp_line(s1, file, level);
3990         }
3991         if (s1->dflag & 7) {
3992             pp_debug_defines(s1);
3993             if (s1->dflag & 4)
3994                 continue;
3995         }
3996
3997         if (is_space(tok)) {
3998             if (spcs < sizeof white - 1)
3999                 white[spcs++] = tok;
4000             continue;
4001         } else if (tok == TOK_LINEFEED) {
4002             spcs = 0;
4003             if (token_seen == TOK_LINEFEED)
4004                 continue;
4005             ++file->line_ref;
4006         } else if (token_seen == TOK_LINEFEED) {
4007             pp_line(s1, file, 0);
4008         } else if (spcs == 0 && pp_need_space(token_seen, tok)) {
4009             white[spcs++] = ' ';
4010         }
4011
4012         white[spcs] = 0, fputs(white, s1->ppfp), spcs = 0;
4013         fputs(p = get_tok_str(tok, &tokc), s1->ppfp);
4014         token_seen = pp_check_he0xE(tok, p);
4015     }
4016     return 0;
4017 }
4018
4019 /* ------------------------------------------------------------------------- */