libcpp/traditional.c

   1 /* CPP Library - traditional lexical analysis and macro expansion.
   2    Copyright (C) 2002-2014 Free Software Foundation, Inc.
   3    Contributed by Neil Booth, May 2002
   4
   5 This program is free software; you can redistribute it and/or modify it
   6 under the terms of the GNU General Public License as published by the
   7 Free Software Foundation; either version 3, or (at your option) any
   8 later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program; see the file COPYING3.  If not see
  17 <http://www.gnu.org/licenses/>.  */
  18
  19 #include "config.h"
  20 #include "system.h"
  21 #include "cpplib.h"
  22 #include "internal.h"
  23
  24 /* The replacement text of a function-like macro is stored as a
  25    contiguous sequence of aligned blocks, each representing the text
  26    between subsequent parameters.
  27
  28    Each block comprises the text between its surrounding parameters,
  29    the length of that text, and the one-based index of the following
  30    parameter.  The final block in the replacement text is easily
  31    recognizable as it has an argument index of zero.  */
  32
  33 struct block
  34 {
  35   unsigned int text_len;
  36   unsigned short arg_index;
  37   uchar text[1];
  38 };
  39
  40 #define BLOCK_HEADER_LEN offsetof (struct block, text)
  41 #define BLOCK_LEN(TEXT_LEN) CPP_ALIGN (BLOCK_HEADER_LEN + (TEXT_LEN))
  42
  43 /* Structure holding information about a function-like macro
  44    invocation.  */
  45 struct fun_macro
  46 {
  47   /* Memory buffer holding the trad_arg array.  */
  48   _cpp_buff *buff;
  49
  50   /* An array of size the number of macro parameters + 1, containing
  51      the offsets of the start of each macro argument in the output
  52      buffer.  The argument continues until the character before the
  53      start of the next one.  */
  54   size_t *args;
  55
  56   /* The hashnode of the macro.  */
  57   cpp_hashnode *node;
  58
  59   /* The offset of the macro name in the output buffer.  */
  60   size_t offset;
  61
  62   /* The line the macro name appeared on.  */
  63   source_location line;
  64
  65   /* Zero-based index of argument being currently lexed.  */
  66   unsigned int argc;
  67 };
  68
  69 /* Lexing state.  It is mostly used to prevent macro expansion.  */
  70 enum ls {ls_none = 0,           /* Normal state.  */
  71          ls_fun_open,           /* When looking for '('.  */
  72          ls_fun_close,          /* When looking for ')'.  */
  73          ls_defined,            /* After defined.  */
  74          ls_defined_close,      /* Looking for ')' of defined().  */
  75          ls_hash,               /* After # in preprocessor conditional.  */
  76          ls_predicate,          /* After the predicate, maybe paren?  */
  77          ls_answer,             /* In answer to predicate.  */
  78          ls_has_include,        /* After __has_include__.  */
  79          ls_has_include_close,  /* Looking for ')' of __has_include__.  */
  80          ls_has_attribute,      /* After __has_attribute__.  */
  81          ls_has_attribute_close}; /* Looking for ')' of __has_attribute__.  */
  82
  83 /* Lexing TODO: Maybe handle space in escaped newlines.  Stop lex.c
  84    from recognizing comments and directives during its lexing pass.  */
  85
  86 static const uchar *skip_whitespace (cpp_reader *, const uchar *, int);
  87 static cpp_hashnode *lex_identifier (cpp_reader *, const uchar *);
  88 static const uchar *copy_comment (cpp_reader *, const uchar *, int);
  89 static void check_output_buffer (cpp_reader *, size_t);
  90 static void push_replacement_text (cpp_reader *, cpp_hashnode *);
  91 static bool scan_parameters (cpp_reader *, cpp_macro *);
  92 static bool recursive_macro (cpp_reader *, cpp_hashnode *);
  93 static void save_replacement_text (cpp_reader *, cpp_macro *, unsigned int);
  94 static void maybe_start_funlike (cpp_reader *, cpp_hashnode *, const uchar *,
  95                                  struct fun_macro *);
  96 static void save_argument (struct fun_macro *, size_t);
  97 static void replace_args_and_push (cpp_reader *, struct fun_macro *);
  98 static size_t canonicalize_text (uchar *, const uchar *, size_t, uchar *);
  99
 100 /* Ensures we have N bytes' space in the output buffer, and
 101    reallocates it if not.  */
 102 static void
 103 check_output_buffer (cpp_reader *pfile, size_t n)
 104 {
 105   /* We might need two bytes to terminate an unterminated comment, and
 106      one more to terminate the line with a NUL.  */
 107   n += 2 + 1;
 108
 109   if (n > (size_t) (pfile->out.limit - pfile->out.cur))
 110     {
 111       size_t size = pfile->out.cur - pfile->out.base;
 112       size_t new_size = (size + n) * 3 / 2;
 113
 114       pfile->out.base = XRESIZEVEC (unsigned char, pfile->out.base, new_size);
 115       pfile->out.limit = pfile->out.base + new_size;
 116       pfile->out.cur = pfile->out.base + size;
 117     }
 118 }
 119
 120 /* Skip a C-style block comment in a macro as a result of -CC.
 121    Buffer->cur points to the initial asterisk of the comment.  */
 122 static void
 123 skip_macro_block_comment (cpp_reader *pfile)
 124 {
 125   const uchar *cur = pfile->buffer->cur;
 126
 127   cur++;
 128   if (*cur == '/')
 129     cur++;
 130
 131   /* People like decorating comments with '*', so check for '/'
 132      instead for efficiency.  */
 133   while(! (*cur++ == '/' && cur[-2] == '*') )
 134     ;
 135
 136   pfile->buffer->cur = cur;
 137 }
 138
 139 /* CUR points to the asterisk introducing a comment in the current
 140    context.  IN_DEFINE is true if we are in the replacement text of a
 141    macro.
 142
 143    The asterisk and following comment is copied to the buffer pointed
 144    to by pfile->out.cur, which must be of sufficient size.
 145    Unterminated comments are diagnosed, and correctly terminated in
 146    the output.  pfile->out.cur is updated depending upon IN_DEFINE,
 147    -C, -CC and pfile->state.in_directive.
 148
 149    Returns a pointer to the first character after the comment in the
 150    input buffer.  */
 151 static const uchar *
 152 copy_comment (cpp_reader *pfile, const uchar *cur, int in_define)
 153 {
 154   bool unterminated, copy = false;
 155   source_location src_loc = pfile->line_table->highest_line;
 156   cpp_buffer *buffer = pfile->buffer;
 157
 158   buffer->cur = cur;
 159   if (pfile->context->prev)
 160     unterminated = false, skip_macro_block_comment (pfile);
 161   else
 162     unterminated = _cpp_skip_block_comment (pfile);
 163
 164   if (unterminated)
 165     cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0,
 166                          "unterminated comment");
 167
 168   /* Comments in directives become spaces so that tokens are properly
 169      separated when the ISO preprocessor re-lexes the line.  The
 170      exception is #define.  */
 171   if (pfile->state.in_directive)
 172     {
 173       if (in_define)
 174         {
 175           if (CPP_OPTION (pfile, discard_comments_in_macro_exp))
 176             pfile->out.cur--;
 177           else
 178             copy = true;
 179         }
 180       else
 181         pfile->out.cur[-1] = ' ';
 182     }
 183   else if (CPP_OPTION (pfile, discard_comments))
 184     pfile->out.cur--;
 185   else
 186     copy = true;
 187
 188   if (copy)
 189     {
 190       size_t len = (size_t) (buffer->cur - cur);
 191       memcpy (pfile->out.cur, cur, len);
 192       pfile->out.cur += len;
 193       if (unterminated)
 194         {
 195           *pfile->out.cur++ = '*';
 196           *pfile->out.cur++ = '/';
 197         }
 198     }
 199
 200   return buffer->cur;
 201 }
 202
 203 /* CUR points to any character in the input buffer.  Skips over all
 204    contiguous horizontal white space and NULs, including comments if
 205    SKIP_COMMENTS, until reaching the first non-horizontal-whitespace
 206    character or the end of the current context.  Escaped newlines are
 207    removed.
 208
 209    The whitespace is copied verbatim to the output buffer, except that
 210    comments are handled as described in copy_comment().
 211    pfile->out.cur is updated.
 212
 213    Returns a pointer to the first character after the whitespace in
 214    the input buffer.  */
 215 static const uchar *
 216 skip_whitespace (cpp_reader *pfile, const uchar *cur, int skip_comments)
 217 {
 218   uchar *out = pfile->out.cur;
 219
 220   for (;;)
 221     {
 222       unsigned int c = *cur++;
 223       *out++ = c;
 224
 225       if (is_nvspace (c))
 226         continue;
 227
 228       if (c == '/' && *cur == '*' && skip_comments)
 229         {
 230           pfile->out.cur = out;
 231           cur = copy_comment (pfile, cur, false /* in_define */);
 232           out = pfile->out.cur;
 233           continue;
 234         }
 235
 236       out--;
 237       break;
 238     }
 239
 240   pfile->out.cur = out;
 241   return cur - 1;
 242 }
 243
 244 /* Lexes and outputs an identifier starting at CUR, which is assumed
 245    to point to a valid first character of an identifier.  Returns
 246    the hashnode, and updates out.cur.  */
 247 static cpp_hashnode *
 248 lex_identifier (cpp_reader *pfile, const uchar *cur)
 249 {
 250   size_t len;
 251   uchar *out = pfile->out.cur;
 252   cpp_hashnode *result;
 253
 254   do
 255     *out++ = *cur++;
 256   while (is_numchar (*cur));
 257
 258   CUR (pfile->context) = cur;
 259   len = out - pfile->out.cur;
 260   result = CPP_HASHNODE (ht_lookup (pfile->hash_table, pfile->out.cur,
 261                                     len, HT_ALLOC));
 262   pfile->out.cur = out;
 263   return result;
 264 }
 265
 266 /* Overlays the true file buffer temporarily with text of length LEN
 267    starting at START.  The true buffer is restored upon calling
 268    restore_buff().  */
 269 void
 270 _cpp_overlay_buffer (cpp_reader *pfile, const uchar *start, size_t len)
 271 {
 272   cpp_buffer *buffer = pfile->buffer;
 273
 274   pfile->overlaid_buffer = buffer;
 275   pfile->saved_cur = buffer->cur;
 276   pfile->saved_rlimit = buffer->rlimit;
 277   pfile->saved_line_base = buffer->next_line;
 278   buffer->need_line = false;
 279
 280   buffer->cur = start;
 281   buffer->line_base = start;
 282   buffer->rlimit = start + len;
 283 }
 284
 285 /* Restores a buffer overlaid by _cpp_overlay_buffer().  */
 286 void
 287 _cpp_remove_overlay (cpp_reader *pfile)
 288 {
 289   cpp_buffer *buffer = pfile->overlaid_buffer;
 290
 291   buffer->cur = pfile->saved_cur;
 292   buffer->rlimit = pfile->saved_rlimit;
 293   buffer->line_base = pfile->saved_line_base;
 294   buffer->need_line = true;
 295
 296   pfile->overlaid_buffer = NULL;
 297 }
 298
 299 /* Reads a logical line into the output buffer.  Returns TRUE if there
 300    is more text left in the buffer.  */
 301 bool
 302 _cpp_read_logical_line_trad (cpp_reader *pfile)
 303 {
 304   do
 305     {
 306       if (pfile->buffer->need_line && !_cpp_get_fresh_line (pfile))
 307         return false;
 308     }
 309   while (!_cpp_scan_out_logical_line (pfile, NULL) || pfile->state.skipping);
 310
 311   return pfile->buffer != NULL;
 312 }
 313
 314 /* Set up state for finding the opening '(' of a function-like
 315    macro.  */
 316 static void
 317 maybe_start_funlike (cpp_reader *pfile, cpp_hashnode *node, const uchar *start, struct fun_macro *macro)
 318 {
 319   unsigned int n = node->value.macro->paramc + 1;
 320
 321   if (macro->buff)
 322     _cpp_release_buff (pfile, macro->buff);
 323   macro->buff = _cpp_get_buff (pfile, n * sizeof (size_t));
 324   macro->args = (size_t *) BUFF_FRONT (macro->buff);
 325   macro->node = node;
 326   macro->offset = start - pfile->out.base;
 327   macro->argc = 0;
 328 }
 329
 330 /* Save the OFFSET of the start of the next argument to MACRO.  */
 331 static void
 332 save_argument (struct fun_macro *macro, size_t offset)
 333 {
 334   macro->argc++;
 335   if (macro->argc <= macro->node->value.macro->paramc)
 336     macro->args[macro->argc] = offset;
 337 }
 338
 339 /* Copies the next logical line in the current buffer (starting at
 340    buffer->cur) to the output buffer.  The output is guaranteed to
 341    terminate with a NUL character.  buffer->cur is updated.
 342
 343    If MACRO is non-NULL, then we are scanning the replacement list of
 344    MACRO, and we call save_replacement_text() every time we meet an
 345    argument.  */
 346 bool
 347 _cpp_scan_out_logical_line (cpp_reader *pfile, cpp_macro *macro)
 348 {
 349   bool result = true;
 350   cpp_context *context;
 351   const uchar *cur;
 352   uchar *out;
 353   struct fun_macro fmacro;
 354   unsigned int c, paren_depth = 0, quote;
 355   enum ls lex_state = ls_none;
 356   bool header_ok;
 357   const uchar *start_of_input_line;
 358
 359   fmacro.buff = NULL;
 360   fmacro.args = NULL;
 361   fmacro.node = NULL;
 362   fmacro.offset = 0;
 363   fmacro.line = 0;
 364   fmacro.argc = 0;
 365
 366   quote = 0;
 367   header_ok = pfile->state.angled_headers;
 368   CUR (pfile->context) = pfile->buffer->cur;
 369   RLIMIT (pfile->context) = pfile->buffer->rlimit;
 370   pfile->out.cur = pfile->out.base;
 371   pfile->out.first_line = pfile->line_table->highest_line;
 372   /* start_of_input_line is needed to make sure that directives really,
 373      really start at the first character of the line.  */
 374   start_of_input_line = pfile->buffer->cur;
 375  new_context:
 376   context = pfile->context;
 377   cur = CUR (context);
 378   check_output_buffer (pfile, RLIMIT (context) - cur);
 379   out = pfile->out.cur;
 380
 381   for (;;)
 382     {
 383       if (!context->prev
 384           && cur >= pfile->buffer->notes[pfile->buffer->cur_note].pos)
 385         {
 386           pfile->buffer->cur = cur;
 387           _cpp_process_line_notes (pfile, false);
 388         }
 389       c = *cur++;
 390       *out++ = c;
 391
 392       /* Whitespace should "continue" out of the switch,
 393          non-whitespace should "break" out of it.  */
 394       switch (c)
 395         {
 396         case ' ':
 397         case '\t':
 398         case '\f':
 399         case '\v':
 400         case '\0':
 401           continue;
 402
 403         case '\n':
 404           /* If this is a macro's expansion, pop it.  */
 405           if (context->prev)
 406             {
 407               pfile->out.cur = out - 1;
 408               _cpp_pop_context (pfile);
 409               goto new_context;
 410             }
 411
 412           /* Omit the newline from the output buffer.  */
 413           pfile->out.cur = out - 1;
 414           pfile->buffer->cur = cur;
 415           pfile->buffer->need_line = true;
 416           CPP_INCREMENT_LINE (pfile, 0);
 417
 418           if ((lex_state == ls_fun_open || lex_state == ls_fun_close)
 419               && !pfile->state.in_directive
 420               && _cpp_get_fresh_line (pfile))
 421             {
 422               /* Newlines in arguments become a space, but we don't
 423                  clear any in-progress quote.  */
 424               if (lex_state == ls_fun_close)
 425                 out[-1] = ' ';
 426               cur = pfile->buffer->cur;
 427               continue;
 428             }
 429           goto done;
 430
 431         case '<':
 432           if (header_ok)
 433             quote = '>';
 434           break;
 435         case '>':
 436           if (c == quote)
 437             quote = 0;
 438           break;
 439
 440         case '"':
 441         case '\'':
 442           if (c == quote)
 443             quote = 0;
 444           else if (!quote)
 445             quote = c;
 446           break;
 447
 448         case '\\':
 449           /* Skip escaped quotes here, it's easier than above.  */
 450           if (*cur == '\\' || *cur == '"' || *cur == '\'')
 451             *out++ = *cur++;
 452           break;
 453
 454         case '/':
 455           /* Traditional CPP does not recognize comments within
 456              literals.  */
 457           if (!quote && *cur == '*')
 458             {
 459               pfile->out.cur = out;
 460               cur = copy_comment (pfile, cur, macro != 0);
 461               out = pfile->out.cur;
 462               continue;
 463             }
 464           break;
 465
 466         case '_':
 467         case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
 468         case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
 469         case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
 470         case 's': case 't': case 'u': case 'v': case 'w': case 'x':
 471         case 'y': case 'z':
 472         case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
 473         case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
 474         case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
 475         case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
 476         case 'Y': case 'Z':
 477           if (!pfile->state.skipping && (quote == 0 || macro))
 478             {
 479               cpp_hashnode *node;
 480               uchar *out_start = out - 1;
 481
 482               pfile->out.cur = out_start;
 483               node = lex_identifier (pfile, cur - 1);
 484               out = pfile->out.cur;
 485               cur = CUR (context);
 486
 487               if (node->type == NT_MACRO
 488                   /* Should we expand for ls_answer?  */
 489                   && (lex_state == ls_none || lex_state == ls_fun_open)
 490                   && !pfile->state.prevent_expansion)
 491                 {
 492                   /* Macros invalidate MI optimization.  */
 493                   pfile->mi_valid = false;
 494                   if (! (node->flags & NODE_BUILTIN)
 495                       && node->value.macro->fun_like)
 496                     {
 497                       maybe_start_funlike (pfile, node, out_start, &fmacro);
 498                       lex_state = ls_fun_open;
 499                       fmacro.line = pfile->line_table->highest_line;
 500                       continue;
 501                     }
 502                   else if (!recursive_macro (pfile, node))
 503                     {
 504                       /* Remove the object-like macro's name from the
 505                          output, and push its replacement text.  */
 506                       pfile->out.cur = out_start;
 507                       push_replacement_text (pfile, node);
 508                       lex_state = ls_none;
 509                       goto new_context;
 510                     }
 511                 }
 512               else if (macro && (node->flags & NODE_MACRO_ARG) != 0)
 513                 {
 514                   /* Found a parameter in the replacement text of a
 515                      #define.  Remove its name from the output.  */
 516                   pfile->out.cur = out_start;
 517                   save_replacement_text (pfile, macro, node->value.arg_index);
 518                   out = pfile->out.base;
 519                 }
 520               else if (lex_state == ls_hash)
 521                 {
 522                   lex_state = ls_predicate;
 523                   continue;
 524                 }
 525               else if (pfile->state.in_expression
 526                        && node == pfile->spec_nodes.n_defined)
 527                 {
 528                   lex_state = ls_defined;
 529                   continue;
 530                 }
 531               else if (pfile->state.in_expression
 532                        && (node == pfile->spec_nodes.n__has_include__
 533                         || node == pfile->spec_nodes.n__has_include_next__))
 534                 {
 535                   lex_state = ls_has_include;
 536                   continue;
 537                 }
 538               else if (pfile->state.in_expression
 539                        && node == pfile->spec_nodes.n__has_attribute__)
 540                 {
 541                   lex_state = ls_has_attribute;
 542                   continue;
 543                 }
 544             }
 545           break;
 546
 547         case '(':
 548           if (quote == 0)
 549             {
 550               paren_depth++;
 551               if (lex_state == ls_fun_open)
 552                 {
 553                   if (recursive_macro (pfile, fmacro.node))
 554                     lex_state = ls_none;
 555                   else
 556                     {
 557                       lex_state = ls_fun_close;
 558                       paren_depth = 1;
 559                       out = pfile->out.base + fmacro.offset;
 560                       fmacro.args[0] = fmacro.offset;
 561                     }
 562                 }
 563               else if (lex_state == ls_predicate)
 564                 lex_state = ls_answer;
 565               else if (lex_state == ls_defined)
 566                 lex_state = ls_defined_close;
 567               else if (lex_state == ls_has_include)
 568                 lex_state = ls_has_include_close;
 569               else if (lex_state == ls_has_attribute)
 570                 lex_state = ls_has_attribute_close;
 571             }
 572           break;
 573
 574         case ',':
 575           if (quote == 0 && lex_state == ls_fun_close && paren_depth == 1)
 576             save_argument (&fmacro, out - pfile->out.base);
 577           break;
 578
 579         case ')':
 580           if (quote == 0)
 581             {
 582               paren_depth--;
 583               if (lex_state == ls_fun_close && paren_depth == 0)
 584                 {
 585                   cpp_macro *m = fmacro.node->value.macro;
 586
 587                   m->used = 1;
 588                   lex_state = ls_none;
 589                   save_argument (&fmacro, out - pfile->out.base);
 590
 591                   /* A single zero-length argument is no argument.  */
 592                   if (fmacro.argc == 1
 593                       && m->paramc == 0
 594                       && out == pfile->out.base + fmacro.offset + 1)
 595                     fmacro.argc = 0;
 596
 597                   if (_cpp_arguments_ok (pfile, m, fmacro.node, fmacro.argc))
 598                     {
 599                       /* Remove the macro's invocation from the
 600                          output, and push its replacement text.  */
 601                       pfile->out.cur = (pfile->out.base
 602                                              + fmacro.offset);
 603                       CUR (context) = cur;
 604                       replace_args_and_push (pfile, &fmacro);
 605                       goto new_context;
 606                     }
 607                 }
 608               else if (lex_state == ls_answer || lex_state == ls_defined_close
 609                         || lex_state == ls_has_include_close
 610                         || lex_state == ls_has_attribute_close)
 611                 lex_state = ls_none;
 612             }
 613           break;
 614
 615         case '#':
 616           if (cur - 1 == start_of_input_line
 617               /* A '#' from a macro doesn't start a directive.  */
 618               && !pfile->context->prev
 619               && !pfile->state.in_directive)
 620             {
 621               /* A directive.  With the way _cpp_handle_directive
 622                  currently works, we only want to call it if either we
 623                  know the directive is OK, or we want it to fail and
 624                  be removed from the output.  If we want it to be
 625                  passed through (the assembler case) then we must not
 626                  call _cpp_handle_directive.  */
 627               pfile->out.cur = out;
 628               cur = skip_whitespace (pfile, cur, true /* skip_comments */);
 629               out = pfile->out.cur;
 630
 631               if (*cur == '\n')
 632                 {
 633                   /* Null directive.  Ignore it and don't invalidate
 634                      the MI optimization.  */
 635                   pfile->buffer->need_line = true;
 636                   CPP_INCREMENT_LINE (pfile, 0);
 637                   result = false;
 638                   goto done;
 639                 }
 640               else
 641                 {
 642                   bool do_it = false;
 643
 644                   if (is_numstart (*cur)
 645                       && CPP_OPTION (pfile, lang) != CLK_ASM)
 646                     do_it = true;
 647                   else if (is_idstart (*cur))
 648                     /* Check whether we know this directive, but don't
 649                        advance.  */
 650                     do_it = lex_identifier (pfile, cur)->is_directive;
 651
 652                   if (do_it || CPP_OPTION (pfile, lang) != CLK_ASM)
 653                     {
 654                       /* This is a kludge.  We want to have the ISO
 655                          preprocessor lex the next token.  */
 656                       pfile->buffer->cur = cur;
 657                       _cpp_handle_directive (pfile, false /* indented */);
 658                       result = false;
 659                       goto done;
 660                     }
 661                 }
 662             }
 663
 664           if (pfile->state.in_expression)
 665             {
 666               lex_state = ls_hash;
 667               continue;
 668             }
 669           break;
 670
 671         default:
 672           break;
 673         }
 674
 675       /* Non-whitespace disables MI optimization and stops treating
 676          '<' as a quote in #include.  */
 677       header_ok = false;
 678       if (!pfile->state.in_directive)
 679         pfile->mi_valid = false;
 680
 681       if (lex_state == ls_none)
 682         continue;
 683
 684       /* Some of these transitions of state are syntax errors.  The
 685          ISO preprocessor will issue errors later.  */
 686       if (lex_state == ls_fun_open)
 687         /* Missing '('.  */
 688         lex_state = ls_none;
 689       else if (lex_state == ls_hash
 690                || lex_state == ls_predicate
 691                || lex_state == ls_defined
 692                || lex_state == ls_has_include
 693                || lex_state == ls_has_attribute)
 694         lex_state = ls_none;
 695
 696       /* ls_answer and ls_defined_close keep going until ')'.  */
 697     }
 698
 699  done:
 700   if (fmacro.buff)
 701     _cpp_release_buff (pfile, fmacro.buff);
 702
 703   if (lex_state == ls_fun_close)
 704     cpp_error_with_line (pfile, CPP_DL_ERROR, fmacro.line, 0,
 705                          "unterminated argument list invoking macro \"%s\"",
 706                          NODE_NAME (fmacro.node));
 707   return result;
 708 }
 709
 710 /* Push a context holding the replacement text of the macro NODE on
 711    the context stack.  NODE is either object-like, or a function-like
 712    macro with no arguments.  */
 713 static void
 714 push_replacement_text (cpp_reader *pfile, cpp_hashnode *node)
 715 {
 716   size_t len;
 717   const uchar *text;
 718   uchar *buf;
 719
 720   if (node->flags & NODE_BUILTIN)
 721     {
 722       text = _cpp_builtin_macro_text (pfile, node);
 723       len = ustrlen (text);
 724       buf = _cpp_unaligned_alloc (pfile, len + 1);
 725       memcpy (buf, text, len);
 726       buf[len]='\n';
 727       text = buf;
 728     }
 729   else
 730     {
 731       cpp_macro *macro = node->value.macro;
 732       macro->used = 1;
 733       text = macro->exp.text;
 734       macro->traditional = 1;
 735       len = macro->count;
 736     }
 737
 738   _cpp_push_text_context (pfile, node, text, len);
 739 }
 740
 741 /* Returns TRUE if traditional macro recursion is detected.  */
 742 static bool
 743 recursive_macro (cpp_reader *pfile, cpp_hashnode *node)
 744 {
 745   bool recursing = !!(node->flags & NODE_DISABLED);
 746
 747   /* Object-like macros that are already expanding are necessarily
 748      recursive.
 749
 750      However, it is possible to have traditional function-like macros
 751      that are not infinitely recursive but recurse to any given depth.
 752      Further, it is easy to construct examples that get ever longer
 753      until the point they stop recursing.  So there is no easy way to
 754      detect true recursion; instead we assume any expansion more than
 755      20 deep since the first invocation of this macro must be
 756      recursing.  */
 757   if (recursing && node->value.macro->fun_like)
 758     {
 759       size_t depth = 0;
 760       cpp_context *context = pfile->context;
 761
 762       do
 763         {
 764           depth++;
 765           if (context->c.macro == node && depth > 20)
 766             break;
 767           context = context->prev;
 768         }
 769       while (context);
 770       recursing = context != NULL;
 771     }
 772
 773   if (recursing)
 774     cpp_error (pfile, CPP_DL_ERROR,
 775                "detected recursion whilst expanding macro \"%s\"",
 776                NODE_NAME (node));
 777
 778   return recursing;
 779 }
 780
 781 /* Return the length of the replacement text of a function-like or
 782    object-like non-builtin macro.  */
 783 size_t
 784 _cpp_replacement_text_len (const cpp_macro *macro)
 785 {
 786   size_t len;
 787
 788   if (macro->fun_like && (macro->paramc != 0))
 789     {
 790       const uchar *exp;
 791
 792       len = 0;
 793       for (exp = macro->exp.text;;)
 794         {
 795           struct block *b = (struct block *) exp;
 796
 797           len += b->text_len;
 798           if (b->arg_index == 0)
 799             break;
 800           len += NODE_LEN (macro->params[b->arg_index - 1]);
 801           exp += BLOCK_LEN (b->text_len);
 802         }
 803     }
 804   else
 805     len = macro->count;
 806
 807   return len;
 808 }
 809
 810 /* Copy the replacement text of MACRO to DEST, which must be of
 811    sufficient size.  It is not NUL-terminated.  The next character is
 812    returned.  */
 813 uchar *
 814 _cpp_copy_replacement_text (const cpp_macro *macro, uchar *dest)
 815 {
 816   if (macro->fun_like && (macro->paramc != 0))
 817     {
 818       const uchar *exp;
 819
 820       for (exp = macro->exp.text;;)
 821         {
 822           struct block *b = (struct block *) exp;
 823           cpp_hashnode *param;
 824
 825           memcpy (dest, b->text, b->text_len);
 826           dest += b->text_len;
 827           if (b->arg_index == 0)
 828             break;
 829           param = macro->params[b->arg_index - 1];
 830           memcpy (dest, NODE_NAME (param), NODE_LEN (param));
 831           dest += NODE_LEN (param);
 832           exp += BLOCK_LEN (b->text_len);
 833         }
 834     }
 835   else
 836     {
 837       memcpy (dest, macro->exp.text, macro->count);
 838       dest += macro->count;
 839     }
 840
 841   return dest;
 842 }
 843
 844 /* Push a context holding the replacement text of the macro NODE on
 845    the context stack.  NODE is either object-like, or a function-like
 846    macro with no arguments.  */
 847 static void
 848 replace_args_and_push (cpp_reader *pfile, struct fun_macro *fmacro)
 849 {
 850   cpp_macro *macro = fmacro->node->value.macro;
 851
 852   if (macro->paramc == 0)
 853     push_replacement_text (pfile, fmacro->node);
 854   else
 855     {
 856       const uchar *exp;
 857       uchar *p;
 858       _cpp_buff *buff;
 859       size_t len = 0;
 860       int cxtquote = 0;
 861
 862       /* Get an estimate of the length of the argument-replaced text.
 863          This is a worst case estimate, assuming that every replacement
 864          text character needs quoting.  */
 865       for (exp = macro->exp.text;;)
 866         {
 867           struct block *b = (struct block *) exp;
 868
 869           len += b->text_len;
 870           if (b->arg_index == 0)
 871             break;
 872           len += 2 * (fmacro->args[b->arg_index]
 873                       - fmacro->args[b->arg_index - 1] - 1);
 874           exp += BLOCK_LEN (b->text_len);
 875         }
 876
 877       /* Allocate room for the expansion plus \n.  */
 878       buff = _cpp_get_buff (pfile, len + 1);
 879
 880       /* Copy the expansion and replace arguments.  */
 881       /* Accumulate actual length, including quoting as necessary */
 882       p = BUFF_FRONT (buff);
 883       len = 0;
 884       for (exp = macro->exp.text;;)
 885         {
 886           struct block *b = (struct block *) exp;
 887           size_t arglen;
 888           int argquote;
 889           uchar *base;
 890           uchar *in;
 891
 892           len += b->text_len;
 893           /* Copy the non-argument text literally, keeping
 894              track of whether matching quotes have been seen. */
 895           for (arglen = b->text_len, in = b->text; arglen > 0; arglen--)
 896             {
 897               if (*in == '"')
 898                 cxtquote = ! cxtquote;
 899               *p++ = *in++;
 900             }
 901           /* Done if no more arguments */
 902           if (b->arg_index == 0)
 903             break;
 904           arglen = (fmacro->args[b->arg_index]
 905                     - fmacro->args[b->arg_index - 1] - 1);
 906           base = pfile->out.base + fmacro->args[b->arg_index - 1];
 907           in = base;
 908 #if 0
 909           /* Skip leading whitespace in the text for the argument to
 910              be substituted. To be compatible with gcc 2.95, we would
 911              also need to trim trailing whitespace. Gcc 2.95 trims
 912              leading and trailing whitespace, which may be a bug.  The
 913              current gcc testsuite explicitly checks that this leading
 914              and trailing whitespace in actual arguments is
 915              preserved. */
 916           while (arglen > 0 && is_space (*in))
 917             {
 918               in++;
 919               arglen--;
 920             }
 921 #endif
 922           for (argquote = 0; arglen > 0; arglen--)
 923             {
 924               if (cxtquote && *in == '"')
 925                 {
 926                   if (in > base && *(in-1) != '\\')
 927                     argquote = ! argquote;
 928                   /* Always add backslash before double quote if argument
 929                      is expanded in a quoted context */
 930                   *p++ = '\\';
 931                   len++;
 932                 }
 933               else if (cxtquote && argquote && *in == '\\')
 934                 {
 935                   /* Always add backslash before a backslash in an argument
 936                      that is expanded in a quoted context and also in the
 937                      range of a quoted context in the argument itself. */
 938                   *p++ = '\\';
 939                   len++;
 940                 }
 941               *p++ = *in++;
 942               len++;
 943             }
 944           exp += BLOCK_LEN (b->text_len);
 945         }
 946
 947       /* \n-terminate.  */
 948       *p = '\n';
 949       _cpp_push_text_context (pfile, fmacro->node, BUFF_FRONT (buff), len);
 950
 951       /* So we free buffer allocation when macro is left.  */
 952       pfile->context->buff = buff;
 953     }
 954 }
 955
 956 /* Read and record the parameters, if any, of a function-like macro
 957    definition.  Destroys pfile->out.cur.
 958
 959    Returns true on success, false on failure (syntax error or a
 960    duplicate parameter).  On success, CUR (pfile->context) is just
 961    past the closing parenthesis.  */
 962 static bool
 963 scan_parameters (cpp_reader *pfile, cpp_macro *macro)
 964 {
 965   const uchar *cur = CUR (pfile->context) + 1;
 966   bool ok;
 967
 968   for (;;)
 969     {
 970       cur = skip_whitespace (pfile, cur, true /* skip_comments */);
 971
 972       if (is_idstart (*cur))
 973         {
 974           struct cpp_hashnode *id = lex_identifier (pfile, cur);
 975           ok = false;
 976           if (_cpp_save_parameter (pfile, macro, id, id))
 977             break;
 978           cur = skip_whitespace (pfile, CUR (pfile->context),
 979                                  true /* skip_comments */);
 980           if (*cur == ',')
 981             {
 982               cur++;
 983               continue;
 984             }
 985           ok = (*cur == ')');
 986           break;
 987         }
 988
 989       ok = (*cur == ')' && macro->paramc == 0);
 990       break;
 991     }
 992
 993   if (!ok)
 994     cpp_error (pfile, CPP_DL_ERROR, "syntax error in macro parameter list");
 995
 996   CUR (pfile->context) = cur + (*cur == ')');
 997
 998   return ok;
 999 }
1000
1001 /* Save the text from pfile->out.base to pfile->out.cur as
1002    the replacement text for the current macro, followed by argument
1003    ARG_INDEX, with zero indicating the end of the replacement
1004    text.  */
1005 static void
1006 save_replacement_text (cpp_reader *pfile, cpp_macro *macro,
1007                        unsigned int arg_index)
1008 {
1009   size_t len = pfile->out.cur - pfile->out.base;
1010   uchar *exp;
1011
1012   if (macro->paramc == 0)
1013     {
1014       /* Object-like and function-like macros without parameters
1015          simply store their \n-terminated replacement text.  */
1016       exp = _cpp_unaligned_alloc (pfile, len + 1);
1017       memcpy (exp, pfile->out.base, len);
1018       exp[len] = '\n';
1019       macro->exp.text = exp;
1020       macro->traditional = 1;
1021       macro->count = len;
1022     }
1023   else
1024     {
1025       /* Store the text's length (unsigned int), the argument index
1026          (unsigned short, base 1) and then the text.  */
1027       size_t blen = BLOCK_LEN (len);
1028       struct block *block;
1029
1030       if (macro->count + blen > BUFF_ROOM (pfile->a_buff))
1031         _cpp_extend_buff (pfile, &pfile->a_buff, macro->count + blen);
1032
1033       exp = BUFF_FRONT (pfile->a_buff);
1034       block = (struct block *) (exp + macro->count);
1035       macro->exp.text = exp;
1036       macro->traditional = 1;
1037
1038       /* Write out the block information.  */
1039       block->text_len = len;
1040       block->arg_index = arg_index;
1041       memcpy (block->text, pfile->out.base, len);
1042
1043       /* Lex the rest into the start of the output buffer.  */
1044       pfile->out.cur = pfile->out.base;
1045
1046       macro->count += blen;
1047
1048       /* If we've finished, commit the memory.  */
1049       if (arg_index == 0)
1050         BUFF_FRONT (pfile->a_buff) += macro->count;
1051     }
1052 }
1053
1054 /* Analyze and save the replacement text of a macro.  Returns true on
1055    success.  */
1056 bool
1057 _cpp_create_trad_definition (cpp_reader *pfile, cpp_macro *macro)
1058 {
1059   const uchar *cur;
1060   uchar *limit;
1061   cpp_context *context = pfile->context;
1062
1063   /* The context has not been set up for command line defines, and CUR
1064      has not been updated for the macro name for in-file defines.  */
1065   pfile->out.cur = pfile->out.base;
1066   CUR (context) = pfile->buffer->cur;
1067   RLIMIT (context) = pfile->buffer->rlimit;
1068   check_output_buffer (pfile, RLIMIT (context) - CUR (context));
1069
1070   /* Is this a function-like macro?  */
1071   if (* CUR (context) == '(')
1072     {
1073       bool ok = scan_parameters (pfile, macro);
1074
1075       /* Remember the params so we can clear NODE_MACRO_ARG flags.  */
1076       macro->params = (cpp_hashnode **) BUFF_FRONT (pfile->a_buff);
1077
1078       /* Setting macro to NULL indicates an error occurred, and
1079          prevents unnecessary work in _cpp_scan_out_logical_line.  */
1080       if (!ok)
1081         macro = NULL;
1082       else
1083         {
1084           BUFF_FRONT (pfile->a_buff) = (uchar *) &macro->params[macro->paramc];
1085           macro->fun_like = 1;
1086         }
1087     }
1088
1089   /* Skip leading whitespace in the replacement text.  */
1090   pfile->buffer->cur
1091     = skip_whitespace (pfile, CUR (context),
1092                        CPP_OPTION (pfile, discard_comments_in_macro_exp));
1093
1094   pfile->state.prevent_expansion++;
1095   _cpp_scan_out_logical_line (pfile, macro);
1096   pfile->state.prevent_expansion--;
1097
1098   if (!macro)
1099     return false;
1100
1101   /* Skip trailing white space.  */
1102   cur = pfile->out.base;
1103   limit = pfile->out.cur;
1104   while (limit > cur && is_space (limit[-1]))
1105     limit--;
1106   pfile->out.cur = limit;
1107   save_replacement_text (pfile, macro, 0);
1108
1109   return true;
1110 }
1111
1112 /* Copy SRC of length LEN to DEST, but convert all contiguous
1113    whitespace to a single space, provided it is not in quotes.  The
1114    quote currently in effect is pointed to by PQUOTE, and is updated
1115    by the function.  Returns the number of bytes copied.  */
1116 static size_t
1117 canonicalize_text (uchar *dest, const uchar *src, size_t len, uchar *pquote)
1118 {
1119   uchar *orig_dest = dest;
1120   uchar quote = *pquote;
1121
1122   while (len)
1123     {
1124       if (is_space (*src) && !quote)
1125         {
1126           do
1127             src++, len--;
1128           while (len && is_space (*src));
1129           *dest++ = ' ';
1130         }
1131       else
1132         {
1133           if (*src == '\'' || *src == '"')
1134             {
1135               if (!quote)
1136                 quote = *src;
1137               else if (quote == *src)
1138                 quote = 0;
1139             }
1140           *dest++ = *src++, len--;
1141         }
1142     }
1143
1144   *pquote = quote;
1145   return dest - orig_dest;
1146 }
1147
1148 /* Returns true if MACRO1 and MACRO2 have expansions different other
1149    than in the form of their whitespace.  */
1150 bool
1151 _cpp_expansions_different_trad (const cpp_macro *macro1,
1152                                 const cpp_macro *macro2)
1153 {
1154   uchar *p1 = XNEWVEC (uchar, macro1->count + macro2->count);
1155   uchar *p2 = p1 + macro1->count;
1156   uchar quote1 = 0, quote2 = 0;
1157   bool mismatch;
1158   size_t len1, len2;
1159
1160   if (macro1->paramc > 0)
1161     {
1162       const uchar *exp1 = macro1->exp.text, *exp2 = macro2->exp.text;
1163
1164       mismatch = true;
1165       for (;;)
1166         {
1167           struct block *b1 = (struct block *) exp1;
1168           struct block *b2 = (struct block *) exp2;
1169
1170           if (b1->arg_index != b2->arg_index)
1171             break;
1172
1173           len1 = canonicalize_text (p1, b1->text, b1->text_len, &quote1);
1174           len2 = canonicalize_text (p2, b2->text, b2->text_len, &quote2);
1175           if (len1 != len2 || memcmp (p1, p2, len1))
1176             break;
1177           if (b1->arg_index == 0)
1178             {
1179               mismatch = false;
1180               break;
1181             }
1182           exp1 += BLOCK_LEN (b1->text_len);
1183           exp2 += BLOCK_LEN (b2->text_len);
1184         }
1185     }
1186   else
1187     {
1188       len1 = canonicalize_text (p1, macro1->exp.text, macro1->count, &quote1);
1189       len2 = canonicalize_text (p2, macro2->exp.text, macro2->count, &quote2);
1190       mismatch = (len1 != len2 || memcmp (p1, p2, len1));
1191     }
1192
1193   free (p1);
1194   return mismatch;
1195 }