preproc.c

   1 #define __STDC_WANT_LIB_EXT2__ 1
   2 #include <stdio.h>
   3 #include <string.h>
   4 #include <ctype.h>
   5 #include <assert.h>
   6 #include "preproc.h"
   7 #include "tokenizer.h"
   8 #include "tglist.h"
   9 #include "hbmap.h"
  10
  11 #define MACRO_FLAG_OBJECTLIKE (1U<<31)
  12 #define MACRO_FLAG_VARIADIC (1U<<30)
  13 #define MACRO_ARGCOUNT_MASK (~(0|MACRO_FLAG_OBJECTLIKE|MACRO_FLAG_VARIADIC))
  14
  15 #define OBJECTLIKE(M) (M->num_args & MACRO_FLAG_OBJECTLIKE)
  16 #define FUNCTIONLIKE(M) (!(OBJECTLIKE(M)))
  17 #define MACRO_ARGCOUNT(M) (M->num_args & MACRO_ARGCOUNT_MASK)
  18 #define MACRO_VARIADIC(M) (M->num_args & MACRO_FLAG_VARIADIC)
  19
  20 #define MAX_RECURSION 32
  21
  22 static unsigned string_hash(const char* s) {
  23         uint_fast32_t h = 0;
  24         while (*s) {
  25                 h = 16*h + *s++;
  26                 h ^= h>>24 & 0xf0;
  27         }
  28         return h & 0xfffffff;
  29 }
  30
  31 struct macro {
  32         unsigned num_args;
  33         FILE* str_contents;
  34         char *str_contents_buf;
  35         tglist(char*) argnames;
  36 };
  37
  38 struct cpp {
  39         tglist(char*) includedirs;
  40         hbmap(char*, struct macro, 128) *macros;
  41         const char *last_file;
  42         int last_line;
  43         struct tokenizer *tchain[MAX_RECURSION];
  44 };
  45
  46 static int token_needs_string(struct token *tok) {
  47         switch(tok->type) {
  48                 case TT_IDENTIFIER:
  49                 case TT_WIDECHAR_LIT:
  50                 case TT_WIDESTRING_LIT:
  51                 case TT_SQSTRING_LIT:
  52                 case TT_DQSTRING_LIT:
  53                 case TT_ELLIPSIS:
  54                 case TT_HEX_INT_LIT:
  55                 case TT_OCT_INT_LIT:
  56                 case TT_DEC_INT_LIT:
  57                 case TT_FLOAT_LIT:
  58                 case TT_UNKNOWN:
  59                         return 1;
  60                 default:
  61                         return 0;
  62         }
  63 }
  64
  65 static void tokenizer_from_file(struct tokenizer *t, FILE* f) {
  66         tokenizer_init(t, f, TF_PARSE_STRINGS);
  67         tokenizer_set_filename(t, "<macro>");
  68         tokenizer_rewind(t);
  69 }
  70
  71 static int strptrcmp(const void *a, const void *b) {
  72         const char * const *x = a;
  73         const char * const *y = b;
  74         return strcmp(*x, *y);
  75 }
  76
  77 static struct macro* get_macro(struct cpp *cpp, const char *name) {
  78         return hbmap_get(cpp->macros, name);
  79 }
  80
  81 static void add_macro(struct cpp *cpp, const char *name, struct macro*m) {
  82         hbmap_insert(cpp->macros, name, *m);
  83 }
  84
  85 static int undef_macro(struct cpp *cpp, const char *name) {
  86         hbmap_iter k = hbmap_find(cpp->macros, name);
  87         if(k == (hbmap_iter) -1) return 0;
  88         struct macro *m = &hbmap_getval(cpp->macros, k);
  89         free(hbmap_getkey(cpp->macros, k));
  90         if(m->str_contents) fclose(m->str_contents);
  91         free(m->str_contents_buf);
  92         tglist_free_values(&m->argnames);
  93         tglist_free_items(&m->argnames);
  94         hbmap_delete(cpp->macros, k);
  95         return 1;
  96 }
  97
  98 static void free_macros(struct cpp *cpp) {
  99         hbmap_iter i;
 100         hbmap_foreach(cpp->macros, i) {
 101                 while(hbmap_iter_index_valid(cpp->macros, i))
 102                         undef_macro(cpp, hbmap_getkey(cpp->macros, i));
 103         }
 104         hbmap_fini(cpp->macros, 1);
 105         free(cpp->macros);
 106 }
 107
 108 static void error_or_warning(const char *err, const char* type, struct tokenizer *t, struct token *curr) {
 109         unsigned column = curr ? curr->column : t->column;
 110         unsigned line  = curr ? curr->line : t->line;
 111         fprintf(stderr, "<%s> %u:%u %s: '%s'\n", t->filename, line, column, type, err);
 112         fprintf(stderr, "%s\n", t->buf);
 113         for(int i = 0; i < strlen(t->buf); i++)
 114                 fprintf(stderr, "^");
 115         fprintf(stderr, "\n");
 116 }
 117 static void error(const char *err, struct tokenizer *t, struct token *curr) {
 118         error_or_warning(err, "error", t, curr);
 119 }
 120 static void warning(const char *err, struct tokenizer *t, struct token *curr) {
 121         error_or_warning(err, "warning", t, curr);
 122 }
 123
 124 static void emit(FILE *out, const char *s) {
 125         fprintf(out, "%s", s);
 126 }
 127
 128 static int x_tokenizer_next_of(struct tokenizer *t, struct token *tok, int fail_unk) {
 129         int ret = tokenizer_next(t, tok);
 130         if(tok->type == TT_OVERFLOW) {
 131                 error("max token length of 4095 exceeded!", t, tok);
 132                 return 0;
 133         } else if (fail_unk && ret == 0) {
 134                 error("tokenizer encountered unknown token", t, tok);
 135                 return 0;
 136         } else if (tok->type == TT_SEP && tok->value == '\t') {
 137                 tok->value = ' ';
 138         }
 139         return 1;
 140 }
 141
 142 #define tokenizer_next(T, TOK) x_tokenizer_next_of(T, TOK, 0)
 143 #define x_tokenizer_next(T, TOK) x_tokenizer_next_of(T, TOK, 1)
 144
 145 static int is_whitespace_token(struct token *token)
 146 {
 147         return token->type == TT_SEP &&
 148                 (token->value == ' ' || token->value == '\t');
 149 }
 150
 151 /* return index of matching item in values array, or -1 on error */
 152 static int expect(struct tokenizer *t, enum tokentype tt, const char* values[], struct token *token)
 153 {
 154         int ret;
 155         do {
 156                 ret = tokenizer_next(t, token);
 157                 if(ret == 0 || token->type == TT_EOF) goto err;
 158         } while(is_whitespace_token(token));
 159
 160         if(token->type != tt) {
 161 err:
 162                 error("unexpected token", t, token);
 163                 return -1;
 164         }
 165         int i = 0;
 166         while(values[i]) {
 167                 if(!strcmp(values[i], t->buf))
 168                         return i;
 169                 ++i;
 170         }
 171         return -1;
 172 }
 173
 174 static int is_char(struct token *tok, int ch) {
 175         return tok->type == TT_SEP && tok->value == ch;
 176 }
 177
 178 static void flush_whitespace(FILE *out, int *ws_count) {
 179         while(*ws_count > 0) {
 180                 emit(out, " ");
 181                 --(*ws_count);
 182         }
 183 }
 184
 185 /* skips until the next non-whitespace token (if the current one is one too)*/
 186 static int eat_whitespace(struct tokenizer *t, struct token *token, int *count) {
 187         *count = 0;
 188         int ret = 1;
 189         while (is_whitespace_token(token)) {
 190                 ++(*count);
 191                 ret = x_tokenizer_next(t, token);
 192                 if(!ret) break;
 193         }
 194         return ret;
 195 }
 196 /* fetches the next token until it is non-whitespace */
 197 static int skip_next_and_ws(struct tokenizer *t, struct token *tok) {
 198         int ret = tokenizer_next(t, tok);
 199         if(!ret) return ret;
 200         int ws_count;
 201         ret = eat_whitespace(t, tok, &ws_count);
 202         return ret;
 203 }
 204
 205 static void emit_token(FILE* out, struct token *tok, const char* strbuf) {
 206         if(tok->type == TT_SEP) {
 207                 fprintf(out, "%c", tok->value);
 208         } else if(strbuf && token_needs_string(tok)) {
 209                 fprintf(out, "%s", strbuf);
 210         } else {
 211                 fprintf(stderr, "oops, dunno how to handle tt %d (%s)\n", (int) tok->type, strbuf);
 212         }
 213 }
 214
 215 int parse_file(struct cpp* cpp, FILE *f, const char*, FILE *out);
 216 static int include_file(struct cpp* cpp, struct tokenizer *t, FILE* out) {
 217         static const char* inc_chars[] = { "\"", "<", 0};
 218         static const char* inc_chars_end[] = { "\"", ">", 0};
 219         struct token tok;
 220         tokenizer_set_flags(t, 0); // disable string tokenization
 221
 222         int inc1sep = expect(t, TT_SEP, inc_chars, &tok);
 223         if(inc1sep == -1) {
 224                 error("expected one of [\"<]", t, &tok);
 225                 return 0;
 226         }
 227         int ret = tokenizer_read_until(t, inc_chars_end[inc1sep], 1);
 228         if(!ret) {
 229                 error("error parsing filename", t, &tok);
 230                 return 0;
 231         }
 232         // TODO: different path lookup depending on whether " or <
 233         size_t i;
 234         FILE *f = 0;
 235         tglist_foreach(&cpp->includedirs, i) {
 236                 char buf[512];
 237                 snprintf(buf, sizeof buf, "%s/%s", tglist_get(&cpp->includedirs, i), t->buf);
 238                 f = fopen(buf, "r");
 239                 if(f) break;
 240         }
 241         if(!f) {
 242                 fprintf(stderr, "%s: ", t->buf);
 243                 perror("fopen");
 244                 return 0;
 245         }
 246         const char *fn = strdup(t->buf);
 247         assert(tokenizer_next(t, &tok) && is_char(&tok, inc_chars_end[inc1sep][0]));
 248
 249         tokenizer_set_flags(t, TF_PARSE_STRINGS);
 250         return parse_file(cpp, f, fn, out);
 251 }
 252
 253 static int emit_error_or_warning(struct tokenizer *t, int is_error) {
 254         int ws_count;
 255         int ret = tokenizer_skip_chars(t, " \t", &ws_count);
 256         if(!ret) return ret;
 257         struct token tmp = {.column = t->column, .line = t->line};
 258         ret = tokenizer_read_until(t, "\n", 1);
 259         if(is_error) {
 260                 error(t->buf, t, &tmp);
 261                 return 0;
 262         }
 263         warning(t->buf, t, &tmp);
 264         return 1;
 265 }
 266
 267 static FILE *freopen_r(FILE *f, char **buf, size_t *size) {
 268         fflush(f);
 269         fclose(f);
 270         return fmemopen(*buf, *size, "r");
 271 }
 272
 273 static int consume_nl_and_ws(struct tokenizer *t, struct token *tok, int expected) {
 274         if(!x_tokenizer_next(t, tok)) {
 275 err:
 276                 error("unexpected", t, tok);
 277                 return 0;
 278         }
 279         if(expected) {
 280                 if(tok->type != TT_SEP || tok->value != expected) goto err;
 281                 switch(expected) {
 282                         case '\\' : expected = '\n'; break;
 283                         case '\n' : expected = 0; break;
 284                 }
 285         } else {
 286                 if(is_whitespace_token(tok)) ;
 287                 else if(is_char(tok, '\\')) expected = '\n';
 288                 else return 1;
 289         }
 290         return consume_nl_and_ws(t, tok, expected);
 291 }
 292
 293 static int expand_macro(struct cpp *cpp, struct tokenizer *t, FILE* out, const char* name, unsigned rec_level, char *visited[]);
 294
 295 static int parse_macro(struct cpp *cpp, struct tokenizer *t) {
 296         int ws_count;
 297         int ret = tokenizer_skip_chars(t, " \t", &ws_count);
 298         if(!ret) return ret;
 299         struct token curr; //tmp = {.column = t->column, .line = t->line};
 300         ret = tokenizer_next(t, &curr) && curr.type != TT_EOF;
 301         if(!ret) {
 302                 error("parsing macro name", t, &curr);
 303                 return ret;
 304         }
 305         if(curr.type != TT_IDENTIFIER) {
 306                 error("expected identifier", t, &curr);
 307                 return 0;
 308         }
 309         const char* macroname = strdup(t->buf);
 310 #ifdef DEBUG
 311         fprintf(stderr, "parsing macro %s\n", macroname);
 312 #endif
 313         int redefined = 0;
 314         if(get_macro(cpp, macroname)) {
 315                 if(!strcmp(macroname, "defined")) {
 316                         error("\"defined\" cannot be used as a macro name", t, &curr);
 317                         return 0;
 318                 }
 319                 redefined = 1;
 320         }
 321
 322         struct macro new = { 0 };
 323         unsigned macro_flags = MACRO_FLAG_OBJECTLIKE;
 324         tglist_init(&new.argnames);
 325
 326         ret = x_tokenizer_next(t, &curr) && curr.type != TT_EOF;
 327         if(!ret) return ret;
 328
 329         if (is_char(&curr, '(')) {
 330                 macro_flags = 0;
 331                 unsigned expected = 0;
 332                 while(1) {
 333                         /* process next function argument identifier */
 334                         ret = consume_nl_and_ws(t, &curr, expected);
 335                         if(!ret) {
 336                                 error("unexpected", t, &curr);
 337                                 return ret;
 338                         }
 339                         expected = 0;
 340                         if(curr.type == TT_SEP) {
 341                                 switch(curr.value) {
 342                                 case '\\':
 343                                         expected = '\n';
 344                                         continue;
 345                                 case ',':
 346                                         continue;
 347                                 case ')':
 348                                         ret = tokenizer_skip_chars(t, " \t", &ws_count);
 349                                         if(!ret) return ret;
 350                                         goto break_loop1;
 351                                 default:
 352                                         error("unexpected character", t, &curr);
 353                                         return 0;
 354                                 }
 355                         } else if(!(curr.type == TT_IDENTIFIER || curr.type == TT_ELLIPSIS)) {
 356                                 error("expected identifier for macro arg", t, &curr);
 357                                 return 0;
 358                         }
 359                         {
 360                                 if(curr.type == TT_ELLIPSIS) {
 361                                         if(macro_flags & MACRO_FLAG_VARIADIC) {
 362                                                 error("\"...\" isn't the last parameter", t, &curr);
 363                                                 return 0;
 364                                         }
 365                                         macro_flags |= MACRO_FLAG_VARIADIC;
 366                                 }
 367                                 char *tmps = strdup(t->buf);
 368                                 tglist_add(&new.argnames, tmps);
 369                         }
 370                         ++new.num_args;
 371                 }
 372                 break_loop1:;
 373         } else if(is_whitespace_token(&curr)) {
 374                 ret = tokenizer_skip_chars(t, " \t", &ws_count);
 375                 if(!ret) return ret;
 376         } else if(is_char(&curr, '\n')) {
 377                 /* content-less macro */
 378                 goto done;
 379         }
 380
 381         struct FILE_container {
 382                 FILE *f;
 383                 char *buf;
 384                 size_t len;
 385         } contents;
 386         contents.f = open_memstream(&contents.buf, &contents.len);
 387
 388         int backslash_seen = 0;
 389         while(1) {
 390                 /* ignore unknown tokens in macro body */
 391                 ret = tokenizer_next(t, &curr);
 392                 if(!ret) return 0;
 393                 if(curr.type == TT_EOF) break;
 394                 if (curr.type == TT_SEP) {
 395                         if(curr.value == '\\')
 396                                 backslash_seen = 1;
 397                         else {
 398                                 if(curr.value == '\n' && !backslash_seen) break;
 399                                 emit_token(contents.f, &curr, t->buf);
 400                                 backslash_seen = 0;
 401                         }
 402                 } else {
 403                         emit_token(contents.f, &curr, t->buf);
 404                 }
 405         }
 406         new.str_contents = freopen_r(contents.f, &contents.buf, &contents.len);
 407         new.str_contents_buf = contents.buf;
 408 done:
 409         if(redefined) {
 410                 struct macro *old = get_macro(cpp, macroname);
 411                 char *s_old = old->str_contents_buf ? old->str_contents_buf : "";
 412                 char *s_new = new.str_contents_buf ? new.str_contents_buf : "";
 413                 if(strcmp(s_old, s_new)) {
 414                         char buf[128];
 415                         sprintf(buf, "redefinition of macro %s", macroname);
 416                         warning(buf, t, 0);
 417                 }
 418         }
 419         new.num_args |= macro_flags;
 420         add_macro(cpp, macroname, &new);
 421         return 1;
 422 }
 423
 424 static size_t macro_arglist_pos(struct macro *m, const char* iden) {
 425         size_t i;
 426         for(i = 0; i < tglist_getsize(&m->argnames); i++) {
 427                 char *item = tglist_get(&m->argnames, i);
 428                 if(!strcmp(item, iden)) return i;
 429         }
 430         return (size_t) -1;
 431 }
 432
 433
 434 struct macro_info {
 435         const char *name;
 436         unsigned nest;
 437         unsigned first;
 438         unsigned last;
 439 };
 440
 441 static int was_visited(const char *name, char*visited[], unsigned rec_level) {
 442         int x;
 443         for(x = rec_level; x >= 0; --x) {
 444                 if(!strcmp(visited[x], name)) return 1;
 445         }
 446         return 0;
 447 }
 448
 449 unsigned get_macro_info(struct cpp* cpp,
 450         struct tokenizer *t,
 451         struct macro_info *mi_list, size_t *mi_cnt,
 452         unsigned nest, unsigned tpos, const char *name,
 453         char* visited[], unsigned rec_level
 454         ) {
 455         int brace_lvl = 0;
 456         while(1) {
 457                 struct token tok;
 458                 int ret = tokenizer_next(t, &tok);
 459                 if(!ret || tok.type == TT_EOF) break;
 460 #ifdef DEBUG
 461                 fprintf(stderr, "(%s) nest %d, brace %u t: %s\n", name, nest, brace_lvl, t->buf);
 462 #endif
 463                 struct macro* m = 0;
 464                 if(tok.type == TT_IDENTIFIER && (m = get_macro(cpp, t->buf)) && !was_visited(t->buf, visited, rec_level)) {
 465                         const char* newname = strdup(t->buf);
 466                         if(FUNCTIONLIKE(m)) {
 467                                 if(tokenizer_peek(t) == '(') {
 468                                         unsigned tpos_save = tpos;
 469                                         tpos = get_macro_info(cpp, t, mi_list, mi_cnt, nest+1, tpos+1, newname, visited, rec_level);
 470                                         mi_list[*mi_cnt] = (struct macro_info) {
 471                                                 .name = newname,
 472                                                 .nest=nest+1,
 473                                                 .first = tpos_save,
 474                                                 .last = tpos + 1};
 475                                         ++(*mi_cnt);
 476                                 } else {
 477                                         /* suppress expansion */
 478                                 }
 479                         } else {
 480                                 mi_list[*mi_cnt] = (struct macro_info) {
 481                                         .name = newname,
 482                                         .nest=nest+1,
 483                                         .first = tpos,
 484                                         .last = tpos + 1};
 485                                 ++(*mi_cnt);
 486                         }
 487                 } else if(is_char(&tok, '(')) {
 488                         ++brace_lvl;
 489                 } else if(is_char(&tok, ')')) {
 490                         --brace_lvl;
 491                         if(brace_lvl == 0 && nest != 0) break;
 492                 }
 493                 ++tpos;
 494         }
 495         return tpos;
 496 }
 497
 498 struct FILE_container {
 499         FILE *f;
 500         char *buf;
 501         size_t len;
 502         struct tokenizer t;
 503 };
 504
 505 static void free_file_container(struct FILE_container *fc) {
 506         fclose(fc->f);
 507         free(fc->buf);
 508 }
 509
 510 static int mem_tokenizers_join(
 511         struct FILE_container* org, struct FILE_container *inj,
 512         struct FILE_container* result,
 513         int first, off_t lastpos) {
 514         result->f = open_memstream(&result->buf, &result->len);
 515         size_t i;
 516         struct token tok;
 517         int ret;
 518         tokenizer_rewind(&org->t);
 519         for(i=0; i<first; ++i) {
 520                 ret = tokenizer_next(&org->t, &tok);
 521                 assert(ret && tok.type != TT_EOF);
 522                 emit_token(result->f, &tok, org->t.buf);
 523         }
 524         int cnt = 0, last = first;
 525         while(1) {
 526                 ret = tokenizer_next(&inj->t, &tok);
 527                 if(!ret || tok.type == TT_EOF) break;
 528                 emit_token(result->f, &tok, inj->t.buf);
 529                 ++cnt;
 530         }
 531         while(tokenizer_ftello(&org->t) < lastpos) {
 532                 ret = tokenizer_next(&org->t, &tok);
 533                 last++;
 534         }
 535
 536         int diff = cnt - ((int) last - (int) first);
 537
 538         while(1) {
 539                 ret = tokenizer_next(&org->t, &tok);
 540                 if(!ret || tok.type == TT_EOF) break;
 541                 emit_token(result->f, &tok, org->t.buf);
 542         }
 543
 544         result->f = freopen_r(result->f, &result->buf, &result->len);
 545         tokenizer_from_file(&result->t, result->f);
 546         return diff;
 547 }
 548
 549 static int tchain_parens_follows(struct cpp *cpp, int rec_level) {
 550         int i, c = 0;
 551         for(i=rec_level;i>=0;--i) {
 552                 c = tokenizer_peek(cpp->tchain[i]);
 553                 if(c == EOF) continue;
 554                 if(c == '(') return i;
 555                 else break;
 556         }
 557         return -1;
 558 }
 559
 560 static int stringify(struct cpp *ccp, struct tokenizer *t, FILE* output) {
 561         int ret = 1;
 562         struct token tok;
 563         emit(output, "\"");
 564         while(1) {
 565                 ret = tokenizer_next(t, &tok);
 566                 if(!ret) return ret;
 567                 if(tok.type == TT_EOF) break;
 568                 if(is_char(&tok, '\n')) continue;
 569                 if(is_char(&tok, '\\') && tokenizer_peek(t) == '\n') continue;
 570                 if(tok.type == TT_DQSTRING_LIT) {
 571                         char *s = t->buf;
 572                         char buf[2] = {0};
 573                         while(*s) {
 574                                 if(*s == '\"') {
 575                                         emit(output, "\\\"");
 576                                 } else if (*s == '\\') {
 577                                         emit(output, "\\\\");
 578                                 } else {
 579                                         buf[0] = *s;
 580                                         emit(output, buf);
 581                                 }
 582                                 ++s;
 583                         }
 584                 } else
 585                         emit_token(output, &tok, t->buf);
 586         }
 587         emit(output, "\"");
 588         return ret;
 589 }
 590
 591 /* rec_level -1 serves as a magic value to signal we're using
 592    expand_macro from the if-evaluator code, which means activating
 593    the "define" macro */
 594 static int expand_macro(struct cpp* cpp, struct tokenizer *t, FILE* out, const char* name, unsigned rec_level, char* visited[]) {
 595         int is_define = !strcmp(name, "defined");
 596
 597         struct macro *m;
 598         if(is_define && rec_level != -1)
 599                 m = NULL;
 600         else m = get_macro(cpp, name);
 601         if(!m) {
 602                 emit(out, name);
 603                 return 1;
 604         }
 605         if(rec_level == -1) rec_level = 0;
 606         if(rec_level >= MAX_RECURSION) {
 607                 error("max recursion level reached", t, 0);
 608                 return 0;
 609         }
 610 #ifdef DEBUG
 611         fprintf(stderr, "lvl %u: expanding macro %s (%s)\n", rec_level, name, m->str_contents_buf);
 612 #endif
 613
 614         if(rec_level == 0 && strcmp(t->filename, "<macro>")) {
 615                 cpp->last_file = t->filename;
 616                 cpp->last_line = t->line;
 617         }
 618         if(!strcmp(name, "__FILE__")) {
 619                 emit(out, "\"");
 620                 emit(out, cpp->last_file);
 621                 emit(out, "\"");
 622                 return 1;
 623         } else if(!strcmp(name, "__LINE__")) {
 624                 char buf[64];
 625                 sprintf(buf, "%d", cpp->last_line);
 626                 emit(out, buf);
 627                 return 1;
 628         }
 629
 630         if(visited[rec_level]) free(visited[rec_level]);
 631         visited[rec_level] = strdup(name);
 632         cpp->tchain[rec_level] = t;
 633
 634         size_t i;
 635         struct token tok;
 636         unsigned num_args = MACRO_ARGCOUNT(m);
 637         struct FILE_container *argvalues = calloc(MACRO_VARIADIC(m) ? num_args + 1 : num_args, sizeof(struct FILE_container));
 638
 639         for(i=0; i < num_args; i++)
 640                 argvalues[i].f = open_memstream(&argvalues[i].buf, &argvalues[i].len);
 641
 642         /* replace named arguments in the contents of the macro call */
 643         if(FUNCTIONLIKE(m)) {
 644                 int ret;
 645                 if((ret = tokenizer_peek(t)) != '(') {
 646                         /* function-like macro shall not be expanded if not followed by '(' */
 647                         if(ret == EOF && rec_level > 0 && (ret = tchain_parens_follows(cpp, rec_level-1)) != -1) {
 648                                 // warning("Replacement text involved subsequent text", t, 0);
 649                                 t = cpp->tchain[ret];
 650                         } else {
 651                                 emit(out, name);
 652                                 goto cleanup;
 653                         }
 654                 }
 655                 ret = x_tokenizer_next(t, &tok);
 656                 assert(ret && is_char(&tok, '('));
 657
 658                 unsigned curr_arg = 0, need_arg = 1, parens = 0;
 659                 int ws_count;
 660                 if(!tokenizer_skip_chars(t, " \t", &ws_count)) return 0;
 661
 662                 int varargs = 0;
 663                 if(num_args == 1 && MACRO_VARIADIC(m)) varargs = 1;
 664                 while(1) {
 665                         int ret = tokenizer_next(t, &tok);
 666                         if(!ret) return 0;
 667                         if( tok.type == TT_EOF) {
 668                                 fprintf(stderr, "warning EOF\n");
 669                                 break;
 670                         }
 671                         if(!parens && is_char(&tok, ',') && !varargs) {
 672                                 if(need_arg && !ws_count) {
 673                                         /* empty argument is OK */
 674                                 }
 675                                 need_arg = 1;
 676                                 if(!varargs) curr_arg++;
 677                                 if(curr_arg + 1 == num_args && MACRO_VARIADIC(m)) {
 678                                         varargs = 1;
 679                                 } else if(curr_arg >= num_args) {
 680                                         error("too many arguments for function macro", t, &tok);
 681                                         return 0;
 682                                 }
 683                                 ret = tokenizer_skip_chars(t, " \t", &ws_count);
 684                                 if(!ret) return ret;
 685                                 continue;
 686                         } else if(is_char(&tok, '(')) {
 687                                 ++parens;
 688                         } else if(is_char(&tok, ')')) {
 689                                 if(!parens) {
 690                                         if(curr_arg + num_args && curr_arg < num_args-1) {
 691                                                 error("too few args for function macro", t, &tok);
 692                                                 return 0;
 693                                         }
 694                                         break;
 695                                 }
 696                                 --parens;
 697                         } else if(is_char(&tok, '\\')) {
 698                                 if(tokenizer_peek(t) == '\n') continue;
 699                         }
 700                         need_arg = 0;
 701                         emit_token(argvalues[curr_arg].f, &tok, t->buf);
 702                 }
 703         }
 704
 705         for(i=0; i < num_args; i++) {
 706                 argvalues[i].f = freopen_r(argvalues[i].f, &argvalues[i].buf, &argvalues[i].len);
 707                 tokenizer_from_file(&argvalues[i].t, argvalues[i].f);
 708 #ifdef DEBUG
 709                 fprintf(stderr, "macro argument %i: %s\n", (int) i, argvalues[i].buf);
 710 #endif
 711         }
 712
 713         if(is_define) {
 714                 if(get_macro(cpp, argvalues[0].buf))
 715                         emit(out, "1");
 716                 else
 717                         emit(out, "0");
 718         }
 719
 720         if(!m->str_contents) goto cleanup;
 721
 722         struct FILE_container cwae = {0}; /* contents_with_args_expanded */
 723         cwae.f = open_memstream(&cwae.buf, &cwae.len);
 724         FILE* output = cwae.f;
 725
 726         struct tokenizer t2;
 727         tokenizer_from_file(&t2, m->str_contents);
 728         int hash_count = 0;
 729         int ws_count = 0;
 730         while(1) {
 731                 int ret;
 732                 ret = tokenizer_next(&t2, &tok);
 733                 if(!ret) return 0;
 734                 if(tok.type == TT_EOF) break;
 735                 if(tok.type == TT_IDENTIFIER) {
 736                         flush_whitespace(output, &ws_count);
 737                         char *id = t2.buf;
 738                         if(MACRO_VARIADIC(m) && !strcmp(t2.buf, "__VA_ARGS__")) {
 739                                 id = "...";
 740                         }
 741                         size_t arg_nr = macro_arglist_pos(m, id);
 742                         if(arg_nr != (size_t) -1) {
 743                                 tokenizer_rewind(&argvalues[arg_nr].t);
 744                                 if(hash_count == 1) ret = stringify(cpp, &argvalues[arg_nr].t, output);
 745                                 else while(1) {
 746                                         ret = tokenizer_next(&argvalues[arg_nr].t, &tok);
 747                                         if(!ret) return ret;
 748                                         if(tok.type == TT_EOF) break;
 749                                         emit_token(output, &tok, argvalues[arg_nr].t.buf);
 750                                 }
 751                                 hash_count = 0;
 752                         } else {
 753                                 if(hash_count == 1) {
 754                 hash_err:
 755                                         error("'#' is not followed by macro parameter", &t2, &tok);
 756                                         return 0;
 757                                 }
 758                                 emit_token(output, &tok, t2.buf);
 759                         }
 760                 } else if(is_char(&tok, '#')) {
 761                         if(hash_count) {
 762                                 goto hash_err;
 763                         }
 764                         while(1) {
 765                                 ++hash_count;
 766                                 /* in a real cpp we'd need to look for '\\' first */
 767                                 while(tokenizer_peek(&t2) == '\n') {
 768                                         x_tokenizer_next(&t2, &tok);
 769                                 }
 770                                 if(tokenizer_peek(&t2) == '#') x_tokenizer_next(&t2, &tok);
 771                                 else break;
 772                         }
 773                         if(hash_count == 1) flush_whitespace(output, &ws_count);
 774                         else if(hash_count > 2) {
 775                                 error("only two '#' characters allowed for macro expansion", &t2, &tok);
 776                                 return 0;
 777                         }
 778                         if(hash_count == 2)
 779                                 ret = tokenizer_skip_chars(&t2, " \t\n", &ws_count);
 780                         else
 781                                 ret = tokenizer_skip_chars(&t2, " \t", &ws_count);
 782
 783                         if(!ret) return ret;
 784                         ws_count = 0;
 785
 786                 } else if(is_whitespace_token(&tok)) {
 787                         ws_count++;
 788                 } else {
 789                         if(hash_count == 1) goto hash_err;
 790                         flush_whitespace(output, &ws_count);
 791                         emit_token(output, &tok, t2.buf);
 792                 }
 793         }
 794         flush_whitespace(output, &ws_count);
 795
 796         /* we need to expand macros after the macro arguments have been inserted */
 797         if(1) {
 798                 cwae.f = freopen_r(cwae.f, &cwae.buf, &cwae.len);
 799 #ifdef DEBUG
 800                 fprintf(stderr, "contents with args expanded: %s\n", cwae.buf);
 801 #endif
 802                 tokenizer_from_file(&cwae.t, cwae.f);
 803                 size_t mac_cnt = 0;
 804                 while(1) {
 805                         int ret = tokenizer_next(&cwae.t, &tok);
 806                         if(!ret) return ret;
 807                         if(tok.type == TT_EOF) break;
 808                         if(tok.type == TT_IDENTIFIER && get_macro(cpp, cwae.t.buf))
 809                                 ++mac_cnt;
 810                 }
 811
 812                 tokenizer_rewind(&cwae.t);
 813                 struct macro_info *mcs = calloc(mac_cnt, sizeof(struct macro_info));
 814                 {
 815                         size_t mac_iter = 0;
 816                         get_macro_info(cpp, &cwae.t, mcs, &mac_iter, 0, 0, "null", visited, rec_level);
 817                         /* some of the macros might not expand at this stage (without braces)*/
 818                         while(mac_cnt && mcs[mac_cnt-1].name == 0)
 819                                 --mac_cnt;
 820                 }
 821                 size_t i; int depth = 0;
 822                 for(i = 0; i < mac_cnt; ++i) {
 823                         if(mcs[i].nest > depth) depth = mcs[i].nest;
 824                 }
 825                 while(depth > -1) {
 826                         for(i = 0; i < mac_cnt; ++i) if(mcs[i].nest == depth) {
 827                                 struct macro_info *mi = &mcs[i];
 828                                 tokenizer_rewind(&cwae.t);
 829                                 size_t j;
 830                                 struct token utok;
 831                                 for(j = 0; j < mi->first+1; ++j)
 832                                         tokenizer_next(&cwae.t, &utok);
 833                                 struct FILE_container t2 = {0}, tmp = {0};
 834                                 t2.f = open_memstream(&t2.buf, &t2.len);
 835                                 if(!expand_macro(cpp, &cwae.t, t2.f, mi->name, rec_level+1, visited))
 836                                         return 0;
 837                                 t2.f = freopen_r(t2.f, &t2.buf, &t2.len);
 838                                 tokenizer_from_file(&t2.t, t2.f);
 839                                 /* manipulating the stream in case more stuff has been consumed */
 840                                 off_t cwae_pos = tokenizer_ftello(&cwae.t);
 841                                 tokenizer_rewind(&cwae.t);
 842 #ifdef DEBUG
 843                                 fprintf(stderr, "merging %s with %s\n", cwae.buf, t2.buf);
 844 #endif
 845                                 int diff = mem_tokenizers_join(&cwae, &t2, &tmp, mi->first, cwae_pos);
 846                                 free_file_container(&cwae);
 847                                 free_file_container(&t2);
 848                                 cwae = tmp;
 849 #ifdef DEBUG
 850                                 fprintf(stderr, "result: %s\n", cwae.buf);
 851 #endif
 852                                 if(diff == 0) continue;
 853                                 for(j = 0; j < mac_cnt; ++j) {
 854                                         if(j == i) continue;
 855                                         struct macro_info *mi2 = &mcs[j];
 856                                         /* modified element mi can be either inside, after or before
 857                                            another macro. the after case doesn't affect us. */
 858                                         if(mi->first >= mi2->first && mi->last <= mi2->last) {
 859                                                 /* inside m2 */
 860                                                 mi2->last += diff;
 861                                         } else if (mi->first < mi2->first) {
 862                                                 /* before m2 */
 863                                                 mi2->first += diff;
 864                                                 mi2->last += diff;
 865                                         }
 866                                 }
 867                         }
 868                         --depth;
 869                 }
 870                 tokenizer_rewind(&cwae.t);
 871                 while(1) {
 872                         struct macro *ma;
 873                         tokenizer_next(&cwae.t, &tok);
 874                         if(tok.type == TT_EOF) break;
 875                         if(tok.type == TT_IDENTIFIER && tokenizer_peek(&cwae.t) == EOF &&
 876                            (ma = get_macro(cpp, cwae.t.buf)) && FUNCTIONLIKE(ma) && tchain_parens_follows(cpp, rec_level) != -1
 877                         ) {
 878                                 int ret = expand_macro(cpp, &cwae.t, out, cwae.t.buf, rec_level+1, visited);
 879                                 if(!ret) return ret;
 880                         } else
 881                                 emit_token(out, &tok, cwae.t.buf);
 882                 }
 883                 free(mcs);
 884         }
 885
 886         free_file_container(&cwae);
 887
 888 cleanup:
 889         for(i=0; i < num_args; i++) {
 890                 fclose(argvalues[i].f);
 891                 free(argvalues[i].buf);
 892         }
 893         free(argvalues);
 894         return 1;
 895 }
 896
 897 #define TT_LAND TT_CUSTOM+0
 898 #define TT_LOR TT_CUSTOM+1
 899 #define TT_LTE TT_CUSTOM+2
 900 #define TT_GTE TT_CUSTOM+3
 901 #define TT_SHL TT_CUSTOM+4
 902 #define TT_SHR TT_CUSTOM+5
 903 #define TT_EQ TT_CUSTOM+6
 904 #define TT_NEQ TT_CUSTOM+7
 905 #define TT_LT TT_CUSTOM+8
 906 #define TT_GT TT_CUSTOM+9
 907 #define TT_BAND TT_CUSTOM+10
 908 #define TT_BOR TT_CUSTOM+11
 909 #define TT_XOR TT_CUSTOM+12
 910 #define TT_NEG TT_CUSTOM+13
 911 #define TT_PLUS TT_CUSTOM+14
 912 #define TT_MINUS TT_CUSTOM+15
 913 #define TT_MUL TT_CUSTOM+16
 914 #define TT_DIV TT_CUSTOM+17
 915 #define TT_MOD TT_CUSTOM+18
 916 #define TT_LPAREN TT_CUSTOM+19
 917 #define TT_RPAREN TT_CUSTOM+20
 918 #define TT_LNOT TT_CUSTOM+21
 919
 920 #define TTINT(X) X-TT_CUSTOM
 921 #define TTENT(X, Y) [TTINT(X)] = Y
 922
 923 static int bp(int tokentype) {
 924         static const int bplist[] = {
 925                 TTENT(TT_LOR, 1 << 4),
 926                 TTENT(TT_LAND, 1 << 5),
 927                 TTENT(TT_BOR, 1 << 6),
 928                 TTENT(TT_XOR, 1 << 7),
 929                 TTENT(TT_BAND, 1 << 8),
 930                 TTENT(TT_EQ, 1 << 9),
 931                 TTENT(TT_NEQ, 1 << 9),
 932                 TTENT(TT_LTE, 1 << 10),
 933                 TTENT(TT_GTE, 1 << 10),
 934                 TTENT(TT_LT, 1 << 10),
 935                 TTENT(TT_GT, 1 << 10),
 936                 TTENT(TT_SHL, 1 << 11),
 937                 TTENT(TT_SHR, 1 << 11),
 938                 TTENT(TT_PLUS, 1 << 12),
 939                 TTENT(TT_MINUS, 1 << 12),
 940                 TTENT(TT_MUL, 1 << 13),
 941                 TTENT(TT_DIV, 1 << 13),
 942                 TTENT(TT_MOD, 1 << 13),
 943                 TTENT(TT_NEG, 1 << 14),
 944                 TTENT(TT_LNOT, 1 << 14),
 945                 TTENT(TT_LPAREN, 1 << 15),
 946 //              TTENT(TT_RPAREN, 1 << 15),
 947 //              TTENT(TT_LPAREN, 0),
 948                 TTENT(TT_RPAREN, 0),
 949         };
 950         if(TTINT(tokentype) < sizeof(bplist)/sizeof(bplist[0])) return bplist[TTINT(tokentype)];
 951         return 0;
 952 }
 953
 954 static int expr(struct tokenizer *t, int rbp, int *err);
 955
 956 static int charlit_to_int(const char *lit) {
 957         if(lit[1] == '\\') switch(lit[2]) {
 958                 case '0': return 0;
 959                 case 'n': return 10;
 960                 case 't': return 9;
 961                 case 'r': return 13;
 962                 case 'x': return strtol(lit+3, NULL, 16);
 963                 default: return lit[2];
 964         }
 965         return lit[1];
 966 }
 967
 968 static int nud(struct tokenizer *t, struct token *tok, int *err) {
 969         switch((unsigned) tok->type) {
 970                 case TT_IDENTIFIER: return 0;
 971                 case TT_WIDECHAR_LIT:
 972                 case TT_SQSTRING_LIT:  return charlit_to_int(t->buf);
 973                 case TT_HEX_INT_LIT:
 974                 case TT_OCT_INT_LIT:
 975                 case TT_DEC_INT_LIT:
 976                         return strtol(t->buf, NULL, 0);
 977                 case TT_NEG:   return ~ expr(t, bp(tok->type), err);
 978                 case TT_PLUS:  return expr(t, bp(tok->type), err);
 979                 case TT_MINUS: return - expr(t, bp(tok->type), err);
 980                 case TT_LNOT:  return !expr(t, bp(tok->type), err);
 981                 case TT_LPAREN: {
 982                         int inner = expr(t, 0, err);
 983                         if(0!=expect(t, TT_RPAREN, (const char*[]){")", 0}, tok)) {
 984                                 error("missing ')'", t, tok);
 985                                 return 0;
 986                         }
 987                         return inner;
 988                 }
 989                 case TT_FLOAT_LIT:
 990                         error("floating constant in preprocessor expression", t, tok);
 991                         *err = 1;
 992                         return 0;
 993                 case TT_RPAREN:
 994                 default:
 995                         error("unexpected token", t, tok);
 996                         *err = 1;
 997                         return 0;
 998         }
 999 }
1000
1001 static int led(struct tokenizer *t, int left, struct token *tok, int *err) {
1002         int right;
1003         switch((unsigned) tok->type) {
1004                 case TT_LAND:
1005                 case TT_LOR:
1006                         right = expr(t, bp(tok->type), err);
1007                         if(tok->type == TT_LAND) return left && right;
1008                         return left || right;
1009                 case TT_LTE:  return left <= expr(t, bp(tok->type), err);
1010                 case TT_GTE:  return left >= expr(t, bp(tok->type), err);
1011                 case TT_SHL:  return left << expr(t, bp(tok->type), err);
1012                 case TT_SHR:  return left >> expr(t, bp(tok->type), err);
1013                 case TT_EQ:   return left == expr(t, bp(tok->type), err);
1014                 case TT_NEQ:  return left != expr(t, bp(tok->type), err);
1015                 case TT_LT:   return left <  expr(t, bp(tok->type), err);
1016                 case TT_GT:   return left >  expr(t, bp(tok->type), err);
1017                 case TT_BAND: return left &  expr(t, bp(tok->type), err);
1018                 case TT_BOR:  return left |  expr(t, bp(tok->type), err);
1019                 case TT_XOR:  return left ^  expr(t, bp(tok->type), err);
1020                 case TT_PLUS: return left +  expr(t, bp(tok->type), err);
1021                 case TT_MINUS:return left -  expr(t, bp(tok->type), err);
1022                 case TT_MUL:  return left *  expr(t, bp(tok->type), err);
1023                 case TT_DIV:
1024                 case TT_MOD:
1025                         right = expr(t, bp(tok->type), err);
1026                         if(right == 0)  {
1027                                 error("eval: div by zero", t, tok);
1028                                 *err = 1;
1029                         }
1030                         else if(tok->type == TT_DIV) return left / right;
1031                         else if(tok->type == TT_MOD) return left % right;
1032                         return 0;
1033                 default:
1034                         error("eval: unexpect token", t, tok);
1035                         *err = 1;
1036                         return 0;
1037         }
1038 }
1039
1040
1041 static int tokenizer_peek_next_non_ws(struct tokenizer *t, struct token *tok)
1042 {
1043         int ret;
1044         while(1) {
1045                 ret = tokenizer_peek_token(t, tok);
1046                 if(is_whitespace_token(tok))
1047                         x_tokenizer_next(t, tok);
1048                 else break;
1049         }
1050         return ret;
1051 }
1052
1053 static int expr(struct tokenizer *t, int rbp, int*err) {
1054         struct token tok;
1055         int ret = skip_next_and_ws(t, &tok);
1056         if(tok.type == TT_EOF) return 0;
1057         int left = nud(t, &tok, err);
1058         while(1) {
1059                 ret = tokenizer_peek_next_non_ws(t, &tok);
1060                 if(bp(tok.type) <= rbp) break;
1061                 ret = tokenizer_next(t, &tok);
1062                 if(tok.type == TT_EOF) break;
1063                 left = led(t, left, &tok, err);
1064         }
1065         (void) ret;
1066         return left;
1067 }
1068
1069 static int do_eval(struct tokenizer *t, int *result) {
1070         tokenizer_register_custom_token(t, TT_LAND, "&&");
1071         tokenizer_register_custom_token(t, TT_LOR, "||");
1072         tokenizer_register_custom_token(t, TT_LTE, "<=");
1073         tokenizer_register_custom_token(t, TT_GTE, ">=");
1074         tokenizer_register_custom_token(t, TT_SHL, "<<");
1075         tokenizer_register_custom_token(t, TT_SHR, ">>");
1076         tokenizer_register_custom_token(t, TT_EQ, "==");
1077         tokenizer_register_custom_token(t, TT_NEQ, "!=");
1078
1079         tokenizer_register_custom_token(t, TT_LT, "<");
1080         tokenizer_register_custom_token(t, TT_GT, ">");
1081
1082         tokenizer_register_custom_token(t, TT_BAND, "&");
1083         tokenizer_register_custom_token(t, TT_BOR, "|");
1084         tokenizer_register_custom_token(t, TT_XOR, "^");
1085         tokenizer_register_custom_token(t, TT_NEG, "~");
1086
1087         tokenizer_register_custom_token(t, TT_PLUS, "+");
1088         tokenizer_register_custom_token(t, TT_MINUS, "-");
1089         tokenizer_register_custom_token(t, TT_MUL, "*");
1090         tokenizer_register_custom_token(t, TT_DIV, "/");
1091         tokenizer_register_custom_token(t, TT_MOD, "%");
1092
1093         tokenizer_register_custom_token(t, TT_LPAREN, "(");
1094         tokenizer_register_custom_token(t, TT_RPAREN, ")");
1095         tokenizer_register_custom_token(t, TT_LNOT, "!");
1096
1097         int err = 0;
1098         *result = expr(t, 0, &err);
1099 #ifdef DEBUG
1100         fprintf(stderr, "eval result: %d\n", *result);
1101 #endif
1102         return !err;
1103 }
1104
1105 static int evaluate_condition(struct cpp *cpp, struct tokenizer *t, int *result, char *visited[]) {
1106         int ret, backslash_seen = 0;
1107         struct token curr;
1108         char *bufp;
1109         size_t size;
1110         int tflags = tokenizer_get_flags(t);
1111         tokenizer_set_flags(t, tflags | TF_PARSE_WIDE_STRINGS);
1112         ret = tokenizer_next(t, &curr);
1113         if(!ret) return ret;
1114         if(!is_whitespace_token(&curr)) {
1115                 error("expected whitespace after if/elif", t, &curr);
1116                 return 0;
1117         }
1118         FILE *f = open_memstream(&bufp, &size);
1119         while(1) {
1120                 ret = tokenizer_next(t, &curr);
1121                 if(!ret) return ret;
1122                 if(curr.type == TT_IDENTIFIER) {
1123                         if(!expand_macro(cpp, t, f, t->buf, -1, visited)) return 0;
1124                 } else if(curr.type == TT_SEP) {
1125                         if(curr.value == '\\')
1126                                 backslash_seen = 1;
1127                         else {
1128                                 if(curr.value == '\n') {
1129                                         if(!backslash_seen) break;
1130                                 } else {
1131                                         emit_token(f, &curr, t->buf);
1132                                 }
1133                                 backslash_seen = 0;
1134                         }
1135                 } else {
1136                         emit_token(f, &curr, t->buf);
1137                 }
1138         }
1139         f = freopen_r(f, &bufp, &size);
1140         if(!f || size == 0) {
1141                 error("#(el)if with no expression", t, &curr);
1142                 return 0;
1143         }
1144 #ifdef DEBUG
1145         fprintf(stderr, "evaluating condition %s\n", bufp);
1146 #endif
1147         struct tokenizer t2;
1148         tokenizer_from_file(&t2, f);
1149         ret = do_eval(&t2, result);
1150         fclose(f);
1151         free(bufp);
1152         tokenizer_set_flags(t, tflags);
1153         return ret;
1154 }
1155
1156 static void free_visited(char *visited[]) {
1157         size_t i;
1158         for(i=0; i< MAX_RECURSION; i++)
1159                 if(visited[i]) free(visited[i]);
1160
1161 }
1162
1163 int parse_file(struct cpp *cpp, FILE *f, const char *fn, FILE *out) {
1164         struct tokenizer t;
1165         struct token curr;
1166         tokenizer_init(&t, f, TF_PARSE_STRINGS);
1167         tokenizer_set_filename(&t, fn);
1168         tokenizer_register_marker(&t, MT_MULTILINE_COMMENT_START, "/*"); /**/
1169         tokenizer_register_marker(&t, MT_MULTILINE_COMMENT_END, "*/");
1170         tokenizer_register_marker(&t, MT_SINGLELINE_COMMENT_START, "//");
1171         int ret, newline=1, ws_count = 0;
1172
1173         int if_level = 0, if_level_active = 0, if_level_satisfied = 0;
1174
1175 #define all_levels_active() (if_level_active == if_level)
1176 #define prev_level_active() (if_level_active == if_level-1)
1177 #define set_level(X, V) do { \
1178                 if(if_level_active > X) if_level_active = X; \
1179                 if(if_level_satisfied > X) if_level_satisfied = X; \
1180                 if(V != -1) { \
1181                         if(V) if_level_active = X; \
1182                         else if(if_level_active == X) if_level_active = X-1; \
1183                         if(V && if_level_active == X) if_level_satisfied = X; \
1184                 } \
1185                 if_level = X; \
1186         } while(0)
1187 #define skip_conditional_block (if_level > if_level_active)
1188
1189         static const char* directives[] = {"include", "error", "warning", "define", "undef", "if", "elif", "else", "ifdef", "ifndef", "endif", "line", "pragma", 0};
1190         while((ret = tokenizer_next(&t, &curr)) && curr.type != TT_EOF) {
1191                 newline = curr.column == 0;
1192                 if(newline) {
1193                         ret = eat_whitespace(&t, &curr, &ws_count);
1194                         if(!ret) return ret;
1195                 }
1196                 if(curr.type == TT_EOF) break;
1197                 if(skip_conditional_block && !(newline && is_char(&curr, '#'))) continue;
1198                 if(is_char(&curr, '#')) {
1199                         if(!newline) {
1200                                 error("stray #", &t, &curr);
1201                                 return 0;
1202                         }
1203                         int index = expect(&t, TT_IDENTIFIER, directives, &curr);
1204                         if(index == -1) {
1205                                 if(skip_conditional_block) continue;
1206                                 error("invalid preprocessing directive", &t, &curr);
1207                                 return 0;
1208                         }
1209                         if(skip_conditional_block) switch(index) {
1210                                 case 0: case 1: case 2: case 3: case 4:
1211                                         continue;
1212                                 default: break;
1213                         }
1214                         switch(index) {
1215                         case 0:
1216                                 ret = include_file(cpp, &t, out);
1217                                 if(!ret) return ret;
1218                                 break;
1219                         case 1:
1220                                 ret = emit_error_or_warning(&t, 1);
1221                                 if(!ret) return ret;
1222                                 break;
1223                         case 2:
1224                                 ret = emit_error_or_warning(&t, 0);
1225                                 if(!ret) return ret;
1226                                 break;
1227                         case 3:
1228                                 ret = parse_macro(cpp, &t);
1229                                 if(!ret) return ret;
1230                                 break;
1231                         case 4:
1232                                 if(!skip_next_and_ws(&t, &curr)) return 0;
1233                                 if(curr.type != TT_IDENTIFIER) {
1234                                         error("expected identifier", &t, &curr);
1235                                         return 0;
1236                                 }
1237                                 undef_macro(cpp, t.buf);
1238                                 break;
1239                         case 5: // if
1240                                 if(all_levels_active()) {
1241                                         char* visited[MAX_RECURSION] = {0};
1242                                         if(!evaluate_condition(cpp, &t, &ret, visited)) return 0;
1243                                         free_visited(visited);
1244                                         set_level(if_level + 1, ret);
1245                                 } else {
1246                                         set_level(if_level + 1, 0);
1247                                 }
1248                                 break;
1249                         case 6: // elif
1250                                 if(prev_level_active() && if_level_satisfied < if_level) {
1251                                         char* visited[MAX_RECURSION] = {0};
1252                                         if(!evaluate_condition(cpp, &t, &ret, visited)) return 0;
1253                                         free_visited(visited);
1254                                         if(ret) {
1255                                                 if_level_active = if_level;
1256                                                 if_level_satisfied = if_level;
1257                                         }
1258                                 } else if(if_level_active == if_level) {
1259                                         --if_level_active;
1260                                 }
1261                                 break;
1262                         case 7: // else
1263                                 if(prev_level_active() && if_level_satisfied < if_level) {
1264                                         if(1) {
1265                                                 if_level_active = if_level;
1266                                                 if_level_satisfied = if_level;
1267                                         }
1268                                 } else if(if_level_active == if_level) {
1269                                         --if_level_active;
1270                                 }
1271                                 break;
1272                         case 8: // ifdef
1273                         case 9: // ifndef
1274                                 if(!skip_next_and_ws(&t, &curr) || curr.type == TT_EOF) return 0;
1275                                 ret = !!get_macro(cpp, t.buf);
1276                                 if(index == 9) ret = !ret;
1277
1278                                 if(all_levels_active()) {
1279                                         set_level(if_level + 1, ret);
1280                                 } else {
1281                                         set_level(if_level + 1, 0);
1282                                 }
1283                                 break;
1284                         case 10: // endif
1285                                 set_level(if_level-1, -1);
1286                                 break;
1287                         case 11: // line
1288                                 ret = tokenizer_read_until(&t, "\n", 1);
1289                                 if(!ret) {
1290                                         error("unknown", &t, &curr);
1291                                         return 0;
1292                                 }
1293                                 break;
1294                         case 12: // pragma
1295                                 emit(out, "#pragma");
1296                                 while((ret = x_tokenizer_next(&t, &curr)) && curr.type != TT_EOF) {
1297                                         emit_token(out, &curr, t.buf);
1298                                         if(is_char(&curr, '\n')) break;
1299                                 }
1300                                 if(!ret) return ret;
1301                                 break;
1302                         default:
1303                                 break;
1304                         }
1305                         continue;
1306                 } else {
1307                         while(ws_count) {
1308                                 emit(out, " ");
1309                                 --ws_count;
1310                         }
1311                 }
1312 #if DEBUG
1313                 fprintf(stderr, "(stdin:%u,%u) ", curr.line, curr.column);
1314                 if(curr.type == TT_SEP)
1315                         fprintf(stderr, "separator: %c\n", curr.value == '\n'? ' ' : curr.value);
1316                 else
1317                         fprintf(stderr, "%s: %s\n", tokentype_to_str(curr.type), t.buf);
1318 #endif
1319                 if(curr.type == TT_IDENTIFIER) {
1320                         char* visited[MAX_RECURSION] = {0};
1321                         if(!expand_macro(cpp, &t, out, t.buf, 0, visited))
1322                                 return 0;
1323                         free_visited(visited);
1324                 } else {
1325                         emit_token(out, &curr, t.buf);
1326                 }
1327         }
1328         if(if_level) {
1329                 error("unterminated #if", &t, &curr);
1330                 return 0;
1331         }
1332         return 1;
1333 }
1334
1335 struct cpp * cpp_new(void) {
1336         struct cpp* ret = calloc(1, sizeof(struct cpp));
1337         if(!ret) return ret;
1338         tglist_init(&ret->includedirs);
1339         cpp_add_includedir(ret, ".");
1340         ret->macros = hbmap_new(strptrcmp, string_hash, 128);
1341         struct macro m = {.num_args = 1};
1342         add_macro(ret, strdup("defined"), &m);
1343         m.num_args = MACRO_FLAG_OBJECTLIKE;
1344         add_macro(ret, strdup("__FILE__"), &m);
1345         add_macro(ret, strdup("__LINE__"), &m);
1346         return ret;
1347 }
1348
1349 void cpp_free(struct cpp*cpp) {
1350         free_macros(cpp);
1351         tglist_free_values(&cpp->includedirs);
1352         tglist_free_items(&cpp->includedirs);
1353 }
1354
1355 void cpp_add_includedir(struct cpp *cpp, const char* includedir) {
1356         tglist_add(&cpp->includedirs, strdup(includedir));
1357 }
1358
1359 int cpp_add_define(struct cpp *cpp, const char *mdecl) {
1360         struct FILE_container tmp = {0};
1361         tmp.f = open_memstream(&tmp.buf, &tmp.len);
1362         fprintf(tmp.f, "%s\n", mdecl);
1363         tmp.f = freopen_r(tmp.f, &tmp.buf, &tmp.len);
1364         tokenizer_from_file(&tmp.t, tmp.f);
1365         int ret = parse_macro(cpp, &tmp.t);
1366         free_file_container(&tmp);
1367         return ret;
1368 }
1369
1370 int cpp_run(struct cpp *cpp, FILE* in, FILE* out, const char* inname) {
1371 #ifdef DEBUG
1372         setvbuf(out, NULL, _IONBF, 0);
1373 #endif
1374         return parse_file(cpp, in, inname, out);
1375 }