libcpp/traditional.c

   1 /* CPP Library - traditional lexical analysis and macro expansion.
   2    Copyright (C) 2002-2020 Free Software Foundation, Inc.
   3    Contributed by Neil Booth, May 2002
   4
   5 This program is free software; you can redistribute it and/or modify it
   6 under the terms of the GNU General Public License as published by the
   7 Free Software Foundation; either version 3, or (at your option) any
   8 later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program; see the file COPYING3.  If not see
  17 <http://www.gnu.org/licenses/>.  */
  18
  19 #include "config.h"
  20 #include "system.h"
  21 #include "cpplib.h"
  22 #include "internal.h"
  23
  24 /* The replacement text of a function-like macro is stored as a
  25    contiguous sequence of aligned blocks, each representing the text
  26    between subsequent parameters.
  27
  28    Each block comprises the text between its surrounding parameters,
  29    the length of that text, and the one-based index of the following
  30    parameter.  The final block in the replacement text is easily
  31    recognizable as it has an argument index of zero.  */
  32
  33 struct block
  34 {
  35   unsigned int text_len;
  36   unsigned short arg_index;
  37   uchar text[1];
  38 };
  39
  40 #define BLOCK_HEADER_LEN offsetof (struct block, text)
  41 #define BLOCK_LEN(TEXT_LEN) CPP_ALIGN (BLOCK_HEADER_LEN + (TEXT_LEN))
  42
  43 /* Structure holding information about a function-like macro
  44    invocation.  */
  45 struct fun_macro
  46 {
  47   /* Memory buffer holding the trad_arg array.  */
  48   _cpp_buff *buff;
  49
  50   /* An array of size the number of macro parameters + 1, containing
  51      the offsets of the start of each macro argument in the output
  52      buffer.  The argument continues until the character before the
  53      start of the next one.  */
  54   size_t *args;
  55
  56   /* The hashnode of the macro.  */
  57   cpp_hashnode *node;
  58
  59   /* The offset of the macro name in the output buffer.  */
  60   size_t offset;
  61
  62   /* The line the macro name appeared on.  */
  63   location_t line;
  64
  65   /* Number of parameters.  */
  66   unsigned int paramc;
  67
  68   /* Zero-based index of argument being currently lexed.  */
  69   unsigned int argc;
  70 };
  71
  72 /* Lexing state.  It is mostly used to prevent macro expansion.  */
  73 enum ls {ls_none = 0,           /* Normal state.  */
  74          ls_fun_open,           /* When looking for '('.  */
  75          ls_fun_close,          /* When looking for ')'.  */
  76          ls_defined,            /* After defined.  */
  77          ls_defined_close,      /* Looking for ')' of defined().  */
  78          ls_hash,               /* After # in preprocessor conditional.  */
  79          ls_predicate,          /* After the predicate, maybe paren?  */
  80          ls_answer              /* In answer to predicate.  */
  81 };
  82
  83 /* Lexing TODO: Maybe handle space in escaped newlines.  Stop lex.c
  84    from recognizing comments and directives during its lexing pass.  */
  85
  86 static const uchar *skip_whitespace (cpp_reader *, const uchar *, int);
  87 static cpp_hashnode *lex_identifier (cpp_reader *, const uchar *);
  88 static const uchar *copy_comment (cpp_reader *, const uchar *, int);
  89 static void check_output_buffer (cpp_reader *, size_t);
  90 static void push_replacement_text (cpp_reader *, cpp_hashnode *);
  91 static bool scan_parameters (cpp_reader *, unsigned *);
  92 static bool recursive_macro (cpp_reader *, cpp_hashnode *);
  93 static void save_replacement_text (cpp_reader *, cpp_macro *, unsigned int);
  94 static void maybe_start_funlike (cpp_reader *, cpp_hashnode *, const uchar *,
  95                                  struct fun_macro *);
  96 static void save_argument (struct fun_macro *, size_t);
  97 static void replace_args_and_push (cpp_reader *, struct fun_macro *);
  98 static size_t canonicalize_text (uchar *, const uchar *, size_t, uchar *);
  99
 100 /* Ensures we have N bytes' space in the output buffer, and
 101    reallocates it if not.  */
 102 static void
 103 check_output_buffer (cpp_reader *pfile, size_t n)
 104 {
 105   /* We might need two bytes to terminate an unterminated comment, and
 106      one more to terminate the line with a NUL.  */
 107   n += 2 + 1;
 108
 109   if (n > (size_t) (pfile->out.limit - pfile->out.cur))
 110     {
 111       size_t size = pfile->out.cur - pfile->out.base;
 112       size_t new_size = (size + n) * 3 / 2;
 113
 114       pfile->out.base = XRESIZEVEC (unsigned char, pfile->out.base, new_size);
 115       pfile->out.limit = pfile->out.base + new_size;
 116       pfile->out.cur = pfile->out.base + size;
 117     }
 118 }
 119
 120 /* Skip a C-style block comment in a macro as a result of -CC.
 121    PFILE->buffer->cur points to the initial asterisk of the comment,
 122    change it to point to after the '*' and '/' characters that terminate it.
 123    Return true if the macro has not been termined, in that case set
 124    PFILE->buffer->cur to the end of the buffer.  */
 125 static bool
 126 skip_macro_block_comment (cpp_reader *pfile)
 127 {
 128   const uchar *cur = pfile->buffer->cur;
 129
 130   cur++;
 131   if (*cur == '/')
 132     cur++;
 133
 134   /* People like decorating comments with '*', so check for '/'
 135      instead for efficiency.  */
 136   while (! (*cur++ == '/' && cur[-2] == '*'))
 137     if (cur[-1] == '\n')
 138       {
 139         pfile->buffer->cur = cur - 1;
 140         return true;
 141       }
 142
 143   pfile->buffer->cur = cur;
 144   return false;
 145 }
 146
 147 /* CUR points to the asterisk introducing a comment in the current
 148    context.  IN_DEFINE is true if we are in the replacement text of a
 149    macro.
 150
 151    The asterisk and following comment is copied to the buffer pointed
 152    to by pfile->out.cur, which must be of sufficient size.
 153    Unterminated comments are diagnosed, and correctly terminated in
 154    the output.  pfile->out.cur is updated depending upon IN_DEFINE,
 155    -C, -CC and pfile->state.in_directive.
 156
 157    Returns a pointer to the first character after the comment in the
 158    input buffer.  */
 159 static const uchar *
 160 copy_comment (cpp_reader *pfile, const uchar *cur, int in_define)
 161 {
 162   bool unterminated, copy = false;
 163   location_t src_loc = pfile->line_table->highest_line;
 164   cpp_buffer *buffer = pfile->buffer;
 165
 166   buffer->cur = cur;
 167   if (pfile->context->prev)
 168     unterminated = skip_macro_block_comment (pfile);
 169   else
 170     unterminated = _cpp_skip_block_comment (pfile);
 171
 172   if (unterminated)
 173     cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0,
 174                          "unterminated comment");
 175
 176   /* Comments in directives become spaces so that tokens are properly
 177      separated when the ISO preprocessor re-lexes the line.  The
 178      exception is #define.  */
 179   if (pfile->state.in_directive)
 180     {
 181       if (in_define)
 182         {
 183           if (CPP_OPTION (pfile, discard_comments_in_macro_exp))
 184             pfile->out.cur--;
 185           else
 186             copy = true;
 187         }
 188       else
 189         pfile->out.cur[-1] = ' ';
 190     }
 191   else if (CPP_OPTION (pfile, discard_comments))
 192     pfile->out.cur--;
 193   else
 194     copy = true;
 195
 196   if (copy)
 197     {
 198       size_t len = (size_t) (buffer->cur - cur);
 199       memcpy (pfile->out.cur, cur, len);
 200       pfile->out.cur += len;
 201       if (unterminated)
 202         {
 203           *pfile->out.cur++ = '*';
 204           *pfile->out.cur++ = '/';
 205         }
 206     }
 207
 208   return buffer->cur;
 209 }
 210
 211 /* CUR points to any character in the input buffer.  Skips over all
 212    contiguous horizontal white space and NULs, including comments if
 213    SKIP_COMMENTS, until reaching the first non-horizontal-whitespace
 214    character or the end of the current context.  Escaped newlines are
 215    removed.
 216
 217    The whitespace is copied verbatim to the output buffer, except that
 218    comments are handled as described in copy_comment().
 219    pfile->out.cur is updated.
 220
 221    Returns a pointer to the first character after the whitespace in
 222    the input buffer.  */
 223 static const uchar *
 224 skip_whitespace (cpp_reader *pfile, const uchar *cur, int skip_comments)
 225 {
 226   uchar *out = pfile->out.cur;
 227
 228   for (;;)
 229     {
 230       unsigned int c = *cur++;
 231       *out++ = c;
 232
 233       if (is_nvspace (c))
 234         continue;
 235
 236       if (c == '/' && *cur == '*' && skip_comments)
 237         {
 238           pfile->out.cur = out;
 239           cur = copy_comment (pfile, cur, false /* in_define */);
 240           out = pfile->out.cur;
 241           continue;
 242         }
 243
 244       out--;
 245       break;
 246     }
 247
 248   pfile->out.cur = out;
 249   return cur - 1;
 250 }
 251
 252 /* Lexes and outputs an identifier starting at CUR, which is assumed
 253    to point to a valid first character of an identifier.  Returns
 254    the hashnode, and updates out.cur.  */
 255 static cpp_hashnode *
 256 lex_identifier (cpp_reader *pfile, const uchar *cur)
 257 {
 258   size_t len;
 259   uchar *out = pfile->out.cur;
 260   cpp_hashnode *result;
 261
 262   do
 263     *out++ = *cur++;
 264   while (is_numchar (*cur));
 265
 266   CUR (pfile->context) = cur;
 267   len = out - pfile->out.cur;
 268   result = CPP_HASHNODE (ht_lookup (pfile->hash_table, pfile->out.cur,
 269                                     len, HT_ALLOC));
 270   pfile->out.cur = out;
 271   return result;
 272 }
 273
 274 /* Overlays the true file buffer temporarily with text of length LEN
 275    starting at START.  The true buffer is restored upon calling
 276    restore_buff().  */
 277 void
 278 _cpp_overlay_buffer (cpp_reader *pfile, const uchar *start, size_t len)
 279 {
 280   cpp_buffer *buffer = pfile->buffer;
 281
 282   pfile->overlaid_buffer = buffer;
 283   pfile->saved_cur = buffer->cur;
 284   pfile->saved_rlimit = buffer->rlimit;
 285   pfile->saved_line_base = buffer->next_line;
 286   buffer->need_line = false;
 287
 288   buffer->cur = start;
 289   buffer->line_base = start;
 290   buffer->rlimit = start + len;
 291 }
 292
 293 /* Restores a buffer overlaid by _cpp_overlay_buffer().  */
 294 void
 295 _cpp_remove_overlay (cpp_reader *pfile)
 296 {
 297   cpp_buffer *buffer = pfile->overlaid_buffer;
 298
 299   buffer->cur = pfile->saved_cur;
 300   buffer->rlimit = pfile->saved_rlimit;
 301   buffer->line_base = pfile->saved_line_base;
 302   buffer->need_line = true;
 303
 304   pfile->overlaid_buffer = NULL;
 305 }
 306
 307 /* Reads a logical line into the output buffer.  Returns TRUE if there
 308    is more text left in the buffer.  */
 309 bool
 310 _cpp_read_logical_line_trad (cpp_reader *pfile)
 311 {
 312   do
 313     {
 314       if (pfile->buffer->need_line && !_cpp_get_fresh_line (pfile))
 315         {
 316           /* Now pop the buffer that _cpp_get_fresh_line did not.  */
 317           _cpp_pop_buffer (pfile);
 318           return false;
 319         }
 320     }
 321   while (!_cpp_scan_out_logical_line (pfile, NULL, false)
 322          || pfile->state.skipping);
 323
 324   return pfile->buffer != NULL;
 325 }
 326
 327 /* Return true if NODE is a fun_like macro.  */
 328 static inline bool
 329 fun_like_macro (cpp_hashnode *node)
 330 {
 331   if (cpp_builtin_macro_p (node))
 332     return (node->value.builtin == BT_HAS_ATTRIBUTE
 333             || node->value.builtin == BT_HAS_BUILTIN
 334             || node->value.builtin == BT_HAS_INCLUDE
 335             || node->value.builtin == BT_HAS_INCLUDE_NEXT);
 336   return node->value.macro->fun_like;
 337 }
 338
 339 /* Set up state for finding the opening '(' of a function-like
 340    macro.  */
 341 static void
 342 maybe_start_funlike (cpp_reader *pfile, cpp_hashnode *node, const uchar *start,
 343                      struct fun_macro *macro)
 344 {
 345   unsigned int n;
 346   if (cpp_builtin_macro_p (node))
 347     n = 1;
 348   else
 349     n = node->value.macro->paramc;
 350
 351   if (macro->buff)
 352     _cpp_release_buff (pfile, macro->buff);
 353   macro->buff = _cpp_get_buff (pfile, (n + 1) * sizeof (size_t));
 354   macro->args = (size_t *) BUFF_FRONT (macro->buff);
 355   macro->node = node;
 356   macro->offset = start - pfile->out.base;
 357   macro->paramc = n;
 358   macro->argc = 0;
 359 }
 360
 361 /* Save the OFFSET of the start of the next argument to MACRO.  */
 362 static void
 363 save_argument (struct fun_macro *macro, size_t offset)
 364 {
 365   macro->argc++;
 366   if (macro->argc <= macro->paramc)
 367     macro->args[macro->argc] = offset;
 368 }
 369
 370 /* Copies the next logical line in the current buffer (starting at
 371    buffer->cur) to the output buffer.  The output is guaranteed to
 372    terminate with a NUL character.  buffer->cur is updated.
 373
 374    If MACRO is non-NULL, then we are scanning the replacement list of
 375    MACRO, and we call save_replacement_text() every time we meet an
 376    argument.
 377
 378    If BUILTIN_MACRO_ARG is true, this is called to macro expand
 379    arguments of builtin function-like macros.  */
 380 bool
 381 _cpp_scan_out_logical_line (cpp_reader *pfile, cpp_macro *macro,
 382                             bool builtin_macro_arg)
 383 {
 384   bool result = true;
 385   cpp_context *context;
 386   const uchar *cur;
 387   uchar *out;
 388   struct fun_macro fmacro;
 389   unsigned int c, paren_depth = 0, quote;
 390   enum ls lex_state = ls_none;
 391   bool header_ok;
 392   const uchar *start_of_input_line;
 393
 394   fmacro.buff = NULL;
 395   fmacro.args = NULL;
 396   fmacro.node = NULL;
 397   fmacro.offset = 0;
 398   fmacro.line = 0;
 399   fmacro.paramc = 0;
 400   fmacro.argc = 0;
 401
 402   quote = 0;
 403   header_ok = pfile->state.angled_headers;
 404   CUR (pfile->context) = pfile->buffer->cur;
 405   RLIMIT (pfile->context) = pfile->buffer->rlimit;
 406   if (!builtin_macro_arg)
 407     {
 408       pfile->out.cur = pfile->out.base;
 409       pfile->out.first_line = pfile->line_table->highest_line;
 410     }
 411   /* start_of_input_line is needed to make sure that directives really,
 412      really start at the first character of the line.  */
 413   start_of_input_line = pfile->buffer->cur;
 414  new_context:
 415   context = pfile->context;
 416   cur = CUR (context);
 417   check_output_buffer (pfile, RLIMIT (context) - cur);
 418   out = pfile->out.cur;
 419
 420   for (;;)
 421     {
 422       if (!context->prev
 423           && !builtin_macro_arg
 424           && cur >= pfile->buffer->notes[pfile->buffer->cur_note].pos)
 425         {
 426           pfile->buffer->cur = cur;
 427           _cpp_process_line_notes (pfile, false);
 428         }
 429       c = *cur++;
 430       *out++ = c;
 431
 432       /* Whitespace should "continue" out of the switch,
 433          non-whitespace should "break" out of it.  */
 434       switch (c)
 435         {
 436         case ' ':
 437         case '\t':
 438         case '\f':
 439         case '\v':
 440         case '\0':
 441           continue;
 442
 443         case '\n':
 444           /* If this is a macro's expansion, pop it.  */
 445           if (context->prev)
 446             {
 447               pfile->out.cur = out - 1;
 448               _cpp_pop_context (pfile);
 449               goto new_context;
 450             }
 451
 452           /* Omit the newline from the output buffer.  */
 453           pfile->out.cur = out - 1;
 454           pfile->buffer->cur = cur;
 455           if (builtin_macro_arg)
 456             goto done;
 457           pfile->buffer->need_line = true;
 458           CPP_INCREMENT_LINE (pfile, 0);
 459
 460           if ((lex_state == ls_fun_open || lex_state == ls_fun_close)
 461               && !pfile->state.in_directive
 462               && _cpp_get_fresh_line (pfile))
 463             {
 464               /* Newlines in arguments become a space, but we don't
 465                  clear any in-progress quote.  */
 466               if (lex_state == ls_fun_close)
 467                 out[-1] = ' ';
 468               cur = pfile->buffer->cur;
 469               continue;
 470             }
 471           goto done;
 472
 473         case '<':
 474           if (header_ok)
 475             quote = '>';
 476           break;
 477         case '>':
 478           if (c == quote)
 479             quote = 0;
 480           break;
 481
 482         case '"':
 483         case '\'':
 484           if (c == quote)
 485             quote = 0;
 486           else if (!quote)
 487             quote = c;
 488           break;
 489
 490         case '\\':
 491           /* Skip escaped quotes here, it's easier than above.  */
 492           if (*cur == '\\' || *cur == '"' || *cur == '\'')
 493             *out++ = *cur++;
 494           break;
 495
 496         case '/':
 497           /* Traditional CPP does not recognize comments within
 498              literals.  */
 499           if (!quote && *cur == '*')
 500             {
 501               pfile->out.cur = out;
 502               cur = copy_comment (pfile, cur, macro != 0);
 503               out = pfile->out.cur;
 504               continue;
 505             }
 506           break;
 507
 508         case '_':
 509         case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
 510         case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
 511         case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
 512         case 's': case 't': case 'u': case 'v': case 'w': case 'x':
 513         case 'y': case 'z':
 514         case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
 515         case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
 516         case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
 517         case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
 518         case 'Y': case 'Z':
 519           if (!pfile->state.skipping && (quote == 0 || macro))
 520             {
 521               cpp_hashnode *node;
 522               uchar *out_start = out - 1;
 523
 524               pfile->out.cur = out_start;
 525               node = lex_identifier (pfile, cur - 1);
 526               out = pfile->out.cur;
 527               cur = CUR (context);
 528
 529               if (cpp_macro_p (node)
 530                   /* Should we expand for ls_answer?  */
 531                   && (lex_state == ls_none || lex_state == ls_fun_open)
 532                   && !pfile->state.prevent_expansion)
 533                 {
 534                   /* Macros invalidate MI optimization.  */
 535                   pfile->mi_valid = false;
 536                   if (fun_like_macro (node))
 537                     {
 538                       maybe_start_funlike (pfile, node, out_start, &fmacro);
 539                       lex_state = ls_fun_open;
 540                       fmacro.line = pfile->line_table->highest_line;
 541                       continue;
 542                     }
 543                   else if (!recursive_macro (pfile, node))
 544                     {
 545                       /* Remove the object-like macro's name from the
 546                          output, and push its replacement text.  */
 547                       pfile->out.cur = out_start;
 548                       push_replacement_text (pfile, node);
 549                       lex_state = ls_none;
 550                       goto new_context;
 551                     }
 552                 }
 553               else if (macro && node->type == NT_MACRO_ARG)
 554                 {
 555                   /* Found a parameter in the replacement text of a
 556                      #define.  Remove its name from the output.  */
 557                   pfile->out.cur = out_start;
 558                   save_replacement_text (pfile, macro, node->value.arg_index);
 559                   out = pfile->out.base;
 560                 }
 561               else if (lex_state == ls_hash)
 562                 {
 563                   lex_state = ls_predicate;
 564                   continue;
 565                 }
 566               else if (pfile->state.in_expression
 567                        && node == pfile->spec_nodes.n_defined)
 568                 {
 569                   lex_state = ls_defined;
 570                   continue;
 571                 }
 572             }
 573           break;
 574
 575         case '(':
 576           if (quote == 0)
 577             {
 578               paren_depth++;
 579               if (lex_state == ls_fun_open)
 580                 {
 581                   if (recursive_macro (pfile, fmacro.node))
 582                     lex_state = ls_none;
 583                   else
 584                     {
 585                       lex_state = ls_fun_close;
 586                       paren_depth = 1;
 587                       out = pfile->out.base + fmacro.offset;
 588                       fmacro.args[0] = fmacro.offset;
 589                     }
 590                 }
 591               else if (lex_state == ls_predicate)
 592                 lex_state = ls_answer;
 593               else if (lex_state == ls_defined)
 594                 lex_state = ls_defined_close;
 595             }
 596           break;
 597
 598         case ',':
 599           if (quote == 0 && lex_state == ls_fun_close && paren_depth == 1)
 600             save_argument (&fmacro, out - pfile->out.base);
 601           break;
 602
 603         case ')':
 604           if (quote == 0)
 605             {
 606               paren_depth--;
 607               if (lex_state == ls_fun_close && paren_depth == 0)
 608                 {
 609                   if (cpp_builtin_macro_p (fmacro.node))
 610                     {
 611                       /* Handle builtin function-like macros like
 612                          __has_attribute.  The already parsed arguments
 613                          are put into a buffer, which is then preprocessed
 614                          and the result is fed to _cpp_push_text_context
 615                          with disabled expansion, where the ISO preprocessor
 616                          parses it.  While in traditional preprocessing
 617                          macro arguments aren't immediately expanded, they in
 618                          the end are because the macro with replaced arguments
 619                          is preprocessed again.  For the builtin function-like
 620                          macros we need the argument immediately though,
 621                          if we don't preprocess them, they would behave
 622                          very differently from ISO preprocessor handling
 623                          of those builtin macros.  So, this handling is
 624                          more similar to traditional preprocessing of
 625                          #if directives, where we also keep preprocessing
 626                          until everything is expanded, and then feed the
 627                          result with disabled expansion to ISO preprocessor
 628                          for handling the directives.  */
 629                       lex_state = ls_none;
 630                       save_argument (&fmacro, out - pfile->out.base);
 631                       cpp_macro m;
 632                       memset (&m, '\0', sizeof (m));
 633                       m.paramc = fmacro.paramc;
 634                       if (_cpp_arguments_ok (pfile, &m, fmacro.node,
 635                                              fmacro.argc))
 636                         {
 637                           size_t len = fmacro.args[1] - fmacro.args[0];
 638                           uchar *buf;
 639
 640                           /* Remove the macro's invocation from the
 641                              output, and push its replacement text.  */
 642                           pfile->out.cur = pfile->out.base + fmacro.offset;
 643                           CUR (context) = cur;
 644                           buf = _cpp_unaligned_alloc (pfile, len + 2);
 645                           buf[0] = '(';
 646                           memcpy (buf + 1, pfile->out.base + fmacro.args[0],
 647                                   len);
 648                           buf[len + 1] = '\n';
 649
 650                           const unsigned char *ctx_rlimit = RLIMIT (context);
 651                           const unsigned char *saved_cur = pfile->buffer->cur;
 652                           const unsigned char *saved_rlimit
 653                             = pfile->buffer->rlimit;
 654                           const unsigned char *saved_line_base
 655                             = pfile->buffer->line_base;
 656                           bool saved_need_line = pfile->buffer->need_line;
 657                           cpp_buffer *saved_overlaid_buffer
 658                             = pfile->overlaid_buffer;
 659                           pfile->buffer->cur = buf;
 660                           pfile->buffer->line_base = buf;
 661                           pfile->buffer->rlimit = buf + len + 1;
 662                           pfile->buffer->need_line = false;
 663                           pfile->overlaid_buffer = pfile->buffer;
 664                           bool saved_in_directive = pfile->state.in_directive;
 665                           pfile->state.in_directive = true;
 666                           cpp_context *saved_prev_context = context->prev;
 667                           context->prev = NULL;
 668
 669                           _cpp_scan_out_logical_line (pfile, NULL, true);
 670
 671                           pfile->state.in_directive = saved_in_directive;
 672                           check_output_buffer (pfile, 1);
 673                           *pfile->out.cur = '\n';
 674                           pfile->buffer->cur = pfile->out.base + fmacro.offset;
 675                           pfile->buffer->line_base = pfile->buffer->cur;
 676                           pfile->buffer->rlimit = pfile->out.cur;
 677                           CUR (context) = pfile->buffer->cur;
 678                           RLIMIT (context) = pfile->buffer->rlimit;
 679
 680                           pfile->state.prevent_expansion++;
 681                           const uchar *text
 682                             = _cpp_builtin_macro_text (pfile, fmacro.node);
 683                           pfile->state.prevent_expansion--;
 684
 685                           context->prev = saved_prev_context;
 686                           pfile->buffer->cur = saved_cur;
 687                           pfile->buffer->rlimit = saved_rlimit;
 688                           pfile->buffer->line_base = saved_line_base;
 689                           pfile->buffer->need_line = saved_need_line;
 690                           pfile->overlaid_buffer = saved_overlaid_buffer;
 691                           pfile->out.cur = pfile->out.base + fmacro.offset;
 692                           CUR (context) = cur;
 693                           RLIMIT (context) = ctx_rlimit;
 694                           len = ustrlen (text);
 695                           buf = _cpp_unaligned_alloc (pfile, len + 1);
 696                           memcpy (buf, text, len);
 697                           buf[len] = '\n';
 698                           text = buf;
 699                           _cpp_push_text_context (pfile, fmacro.node,
 700                                                   text, len);
 701                           goto new_context;
 702                         }
 703                       break;
 704                     }
 705
 706                   cpp_macro *m = fmacro.node->value.macro;
 707
 708                   m->used = 1;
 709                   lex_state = ls_none;
 710                   save_argument (&fmacro, out - pfile->out.base);
 711
 712                   /* A single zero-length argument is no argument.  */
 713                   if (fmacro.argc == 1
 714                       && m->paramc == 0
 715                       && out == pfile->out.base + fmacro.offset + 1)
 716                     fmacro.argc = 0;
 717
 718                   if (_cpp_arguments_ok (pfile, m, fmacro.node, fmacro.argc))
 719                     {
 720                       /* Remove the macro's invocation from the
 721                          output, and push its replacement text.  */
 722                       pfile->out.cur = pfile->out.base + fmacro.offset;
 723                       CUR (context) = cur;
 724                       replace_args_and_push (pfile, &fmacro);
 725                       goto new_context;
 726                     }
 727                 }
 728               else if (lex_state == ls_answer || lex_state == ls_defined_close)
 729                 lex_state = ls_none;
 730             }
 731           break;
 732
 733         case '#':
 734           if (cur - 1 == start_of_input_line
 735               /* A '#' from a macro doesn't start a directive.  */
 736               && !pfile->context->prev
 737               && !pfile->state.in_directive)
 738             {
 739               /* A directive.  With the way _cpp_handle_directive
 740                  currently works, we only want to call it if either we
 741                  know the directive is OK, or we want it to fail and
 742                  be removed from the output.  If we want it to be
 743                  passed through (the assembler case) then we must not
 744                  call _cpp_handle_directive.  */
 745               pfile->out.cur = out;
 746               cur = skip_whitespace (pfile, cur, true /* skip_comments */);
 747               out = pfile->out.cur;
 748
 749               if (*cur == '\n')
 750                 {
 751                   /* Null directive.  Ignore it and don't invalidate
 752                      the MI optimization.  */
 753                   pfile->buffer->need_line = true;
 754                   CPP_INCREMENT_LINE (pfile, 0);
 755                   result = false;
 756                   goto done;
 757                 }
 758               else
 759                 {
 760                   bool do_it = false;
 761
 762                   if (is_numstart (*cur)
 763                       && CPP_OPTION (pfile, lang) != CLK_ASM)
 764                     do_it = true;
 765                   else if (is_idstart (*cur))
 766                     /* Check whether we know this directive, but don't
 767                        advance.  */
 768                     do_it = lex_identifier (pfile, cur)->is_directive;
 769
 770                   if (do_it || CPP_OPTION (pfile, lang) != CLK_ASM)
 771                     {
 772                       /* This is a kludge.  We want to have the ISO
 773                          preprocessor lex the next token.  */
 774                       pfile->buffer->cur = cur;
 775                       _cpp_handle_directive (pfile, false /* indented */);
 776                       result = false;
 777                       goto done;
 778                     }
 779                 }
 780             }
 781
 782           if (pfile->state.in_expression)
 783             {
 784               lex_state = ls_hash;
 785               continue;
 786             }
 787           break;
 788
 789         default:
 790           break;
 791         }
 792
 793       /* Non-whitespace disables MI optimization and stops treating
 794          '<' as a quote in #include.  */
 795       header_ok = false;
 796       if (!pfile->state.in_directive)
 797         pfile->mi_valid = false;
 798
 799       if (lex_state == ls_none)
 800         continue;
 801
 802       /* Some of these transitions of state are syntax errors.  The
 803          ISO preprocessor will issue errors later.  */
 804       if (lex_state == ls_fun_open)
 805         /* Missing '('.  */
 806         lex_state = ls_none;
 807       else if (lex_state == ls_hash
 808                || lex_state == ls_predicate
 809                || lex_state == ls_defined)
 810         lex_state = ls_none;
 811
 812       /* ls_answer and ls_defined_close keep going until ')'.  */
 813     }
 814
 815  done:
 816   if (fmacro.buff)
 817     _cpp_release_buff (pfile, fmacro.buff);
 818
 819   if (lex_state == ls_fun_close)
 820     cpp_error_with_line (pfile, CPP_DL_ERROR, fmacro.line, 0,
 821                          "unterminated argument list invoking macro \"%s\"",
 822                          NODE_NAME (fmacro.node));
 823   return result;
 824 }
 825
 826 /* Push a context holding the replacement text of the macro NODE on
 827    the context stack.  NODE is either object-like, or a function-like
 828    macro with no arguments.  */
 829 static void
 830 push_replacement_text (cpp_reader *pfile, cpp_hashnode *node)
 831 {
 832   size_t len;
 833   const uchar *text;
 834   uchar *buf;
 835
 836   if (cpp_builtin_macro_p (node))
 837     {
 838       text = _cpp_builtin_macro_text (pfile, node);
 839       len = ustrlen (text);
 840       buf = _cpp_unaligned_alloc (pfile, len + 1);
 841       memcpy (buf, text, len);
 842       buf[len] = '\n';
 843       text = buf;
 844     }
 845   else
 846     {
 847       cpp_macro *macro = node->value.macro;
 848       macro->used = 1;
 849       text = macro->exp.text;
 850       len = macro->count;
 851     }
 852
 853   _cpp_push_text_context (pfile, node, text, len);
 854 }
 855
 856 /* Returns TRUE if traditional macro recursion is detected.  */
 857 static bool
 858 recursive_macro (cpp_reader *pfile, cpp_hashnode *node)
 859 {
 860   bool recursing = !!(node->flags & NODE_DISABLED);
 861
 862   /* Object-like macros that are already expanding are necessarily
 863      recursive.
 864
 865      However, it is possible to have traditional function-like macros
 866      that are not infinitely recursive but recurse to any given depth.
 867      Further, it is easy to construct examples that get ever longer
 868      until the point they stop recursing.  So there is no easy way to
 869      detect true recursion; instead we assume any expansion more than
 870      20 deep since the first invocation of this macro must be
 871      recursing.  */
 872   if (recursing && fun_like_macro (node))
 873     {
 874       size_t depth = 0;
 875       cpp_context *context = pfile->context;
 876
 877       do
 878         {
 879           depth++;
 880           if (context->c.macro == node && depth > 20)
 881             break;
 882           context = context->prev;
 883         }
 884       while (context);
 885       recursing = context != NULL;
 886     }
 887
 888   if (recursing)
 889     cpp_error (pfile, CPP_DL_ERROR,
 890                "detected recursion whilst expanding macro \"%s\"",
 891                NODE_NAME (node));
 892
 893   return recursing;
 894 }
 895
 896 /* Return the length of the replacement text of a function-like or
 897    object-like non-builtin macro.  */
 898 size_t
 899 _cpp_replacement_text_len (const cpp_macro *macro)
 900 {
 901   size_t len;
 902
 903   if (macro->fun_like && (macro->paramc != 0))
 904     {
 905       const uchar *exp;
 906
 907       len = 0;
 908       for (exp = macro->exp.text;;)
 909         {
 910           struct block *b = (struct block *) exp;
 911
 912           len += b->text_len;
 913           if (b->arg_index == 0)
 914             break;
 915           len += NODE_LEN (macro->parm.params[b->arg_index - 1]);
 916           exp += BLOCK_LEN (b->text_len);
 917         }
 918     }
 919   else
 920     len = macro->count;
 921
 922   return len;
 923 }
 924
 925 /* Copy the replacement text of MACRO to DEST, which must be of
 926    sufficient size.  It is not NUL-terminated.  The next character is
 927    returned.  */
 928 uchar *
 929 _cpp_copy_replacement_text (const cpp_macro *macro, uchar *dest)
 930 {
 931   if (macro->fun_like && (macro->paramc != 0))
 932     {
 933       const uchar *exp;
 934
 935       for (exp = macro->exp.text;;)
 936         {
 937           struct block *b = (struct block *) exp;
 938           cpp_hashnode *param;
 939
 940           memcpy (dest, b->text, b->text_len);
 941           dest += b->text_len;
 942           if (b->arg_index == 0)
 943             break;
 944           param = macro->parm.params[b->arg_index - 1];
 945           memcpy (dest, NODE_NAME (param), NODE_LEN (param));
 946           dest += NODE_LEN (param);
 947           exp += BLOCK_LEN (b->text_len);
 948         }
 949     }
 950   else
 951     {
 952       memcpy (dest, macro->exp.text, macro->count);
 953       dest += macro->count;
 954     }
 955
 956   return dest;
 957 }
 958
 959 /* Push a context holding the replacement text of the macro NODE on
 960    the context stack.  NODE is either object-like, or a function-like
 961    macro with no arguments.  */
 962 static void
 963 replace_args_and_push (cpp_reader *pfile, struct fun_macro *fmacro)
 964 {
 965   cpp_macro *macro = fmacro->node->value.macro;
 966
 967   if (macro->paramc == 0)
 968     push_replacement_text (pfile, fmacro->node);
 969   else
 970     {
 971       const uchar *exp;
 972       uchar *p;
 973       _cpp_buff *buff;
 974       size_t len = 0;
 975       int cxtquote = 0;
 976
 977       /* Get an estimate of the length of the argument-replaced text.
 978          This is a worst case estimate, assuming that every replacement
 979          text character needs quoting.  */
 980       for (exp = macro->exp.text;;)
 981         {
 982           struct block *b = (struct block *) exp;
 983
 984           len += b->text_len;
 985           if (b->arg_index == 0)
 986             break;
 987           len += 2 * (fmacro->args[b->arg_index]
 988                       - fmacro->args[b->arg_index - 1] - 1);
 989           exp += BLOCK_LEN (b->text_len);
 990         }
 991
 992       /* Allocate room for the expansion plus \n.  */
 993       buff = _cpp_get_buff (pfile, len + 1);
 994
 995       /* Copy the expansion and replace arguments.  */
 996       /* Accumulate actual length, including quoting as necessary */
 997       p = BUFF_FRONT (buff);
 998       len = 0;
 999       for (exp = macro->exp.text;;)
1000         {
1001           struct block *b = (struct block *) exp;
1002           size_t arglen;
1003           int argquote;
1004           uchar *base;
1005           uchar *in;
1006
1007           len += b->text_len;
1008           /* Copy the non-argument text literally, keeping
1009              track of whether matching quotes have been seen. */
1010           for (arglen = b->text_len, in = b->text; arglen > 0; arglen--)
1011             {
1012               if (*in == '"')
1013                 cxtquote = ! cxtquote;
1014               *p++ = *in++;
1015             }
1016           /* Done if no more arguments */
1017           if (b->arg_index == 0)
1018             break;
1019           arglen = (fmacro->args[b->arg_index]
1020                     - fmacro->args[b->arg_index - 1] - 1);
1021           base = pfile->out.base + fmacro->args[b->arg_index - 1];
1022           in = base;
1023 #if 0
1024           /* Skip leading whitespace in the text for the argument to
1025              be substituted. To be compatible with gcc 2.95, we would
1026              also need to trim trailing whitespace. Gcc 2.95 trims
1027              leading and trailing whitespace, which may be a bug.  The
1028              current gcc testsuite explicitly checks that this leading
1029              and trailing whitespace in actual arguments is
1030              preserved. */
1031           while (arglen > 0 && is_space (*in))
1032             {
1033               in++;
1034               arglen--;
1035             }
1036 #endif
1037           for (argquote = 0; arglen > 0; arglen--)
1038             {
1039               if (cxtquote && *in == '"')
1040                 {
1041                   if (in > base && *(in-1) != '\\')
1042                     argquote = ! argquote;
1043                   /* Always add backslash before double quote if argument
1044                      is expanded in a quoted context */
1045                   *p++ = '\\';
1046                   len++;
1047                 }
1048               else if (cxtquote && argquote && *in == '\\')
1049                 {
1050                   /* Always add backslash before a backslash in an argument
1051                      that is expanded in a quoted context and also in the
1052                      range of a quoted context in the argument itself. */
1053                   *p++ = '\\';
1054                   len++;
1055                 }
1056               *p++ = *in++;
1057               len++;
1058             }
1059           exp += BLOCK_LEN (b->text_len);
1060         }
1061
1062       /* \n-terminate.  */
1063       *p = '\n';
1064       _cpp_push_text_context (pfile, fmacro->node, BUFF_FRONT (buff), len);
1065
1066       /* So we free buffer allocation when macro is left.  */
1067       pfile->context->buff = buff;
1068     }
1069 }
1070
1071 /* Read and record the parameters, if any, of a function-like macro
1072    definition.  Destroys pfile->out.cur.
1073
1074    Returns true on success, false on failure (syntax error or a
1075    duplicate parameter).  On success, CUR (pfile->context) is just
1076    past the closing parenthesis.  */
1077 static bool
1078 scan_parameters (cpp_reader *pfile, unsigned *n_ptr)
1079 {
1080   const uchar *cur = CUR (pfile->context) + 1;
1081   bool ok;
1082
1083   unsigned nparms = 0;
1084   for (;;)
1085     {
1086       cur = skip_whitespace (pfile, cur, true /* skip_comments */);
1087
1088       if (is_idstart (*cur))
1089         {
1090           struct cpp_hashnode *id = lex_identifier (pfile, cur);
1091           ok = false;
1092           if (!_cpp_save_parameter (pfile, nparms, id, id))
1093             break;
1094           nparms++;
1095           cur = skip_whitespace (pfile, CUR (pfile->context),
1096                                  true /* skip_comments */);
1097           if (*cur == ',')
1098             {
1099               cur++;
1100               continue;
1101             }
1102           ok = (*cur == ')');
1103           break;
1104         }
1105
1106       ok = (*cur == ')' && !nparms);
1107       break;
1108     }
1109
1110   *n_ptr = nparms;
1111
1112   if (!ok)
1113     cpp_error (pfile, CPP_DL_ERROR, "syntax error in macro parameter list");
1114
1115   CUR (pfile->context) = cur + (*cur == ')');
1116
1117   return ok;
1118 }
1119
1120 /* Save the text from pfile->out.base to pfile->out.cur as
1121    the replacement text for the current macro, followed by argument
1122    ARG_INDEX, with zero indicating the end of the replacement
1123    text.  */
1124 static void
1125 save_replacement_text (cpp_reader *pfile, cpp_macro *macro,
1126                        unsigned int arg_index)
1127 {
1128   size_t len = pfile->out.cur - pfile->out.base;
1129   uchar *exp;
1130
1131   if (macro->paramc == 0)
1132     {
1133       /* Object-like and function-like macros without parameters
1134          simply store their \n-terminated replacement text.  */
1135       exp = _cpp_unaligned_alloc (pfile, len + 1);
1136       memcpy (exp, pfile->out.base, len);
1137       exp[len] = '\n';
1138       macro->exp.text = exp;
1139       macro->count = len;
1140     }
1141   else
1142     {
1143       /* Store the text's length (unsigned int), the argument index
1144          (unsigned short, base 1) and then the text.  */
1145       size_t blen = BLOCK_LEN (len);
1146       struct block *block;
1147
1148       if (macro->count + blen > BUFF_ROOM (pfile->a_buff))
1149         _cpp_extend_buff (pfile, &pfile->a_buff, macro->count + blen);
1150
1151       exp = BUFF_FRONT (pfile->a_buff);
1152       block = (struct block *) (exp + macro->count);
1153       macro->exp.text = exp;
1154
1155       /* Write out the block information.  */
1156       block->text_len = len;
1157       block->arg_index = arg_index;
1158       memcpy (block->text, pfile->out.base, len);
1159
1160       /* Lex the rest into the start of the output buffer.  */
1161       pfile->out.cur = pfile->out.base;
1162
1163       macro->count += blen;
1164
1165       /* If we've finished, commit the memory.  */
1166       if (arg_index == 0)
1167         BUFF_FRONT (pfile->a_buff) += macro->count;
1168     }
1169 }
1170
1171 /* Analyze and save the replacement text of a macro.  Returns true on
1172    success.  */
1173 cpp_macro *
1174 _cpp_create_trad_definition (cpp_reader *pfile)
1175 {
1176   const uchar *cur;
1177   uchar *limit;
1178   cpp_context *context = pfile->context;
1179   unsigned nparms = 0;
1180   int fun_like = 0;
1181   cpp_hashnode **params = NULL;
1182
1183   /* The context has not been set up for command line defines, and CUR
1184      has not been updated for the macro name for in-file defines.  */
1185   pfile->out.cur = pfile->out.base;
1186   CUR (context) = pfile->buffer->cur;
1187   RLIMIT (context) = pfile->buffer->rlimit;
1188   check_output_buffer (pfile, RLIMIT (context) - CUR (context));
1189
1190   /* Is this a function-like macro?  */
1191   if (* CUR (context) == '(')
1192     {
1193       fun_like = +1;
1194       if (scan_parameters (pfile, &nparms))
1195         params = (cpp_hashnode **)_cpp_commit_buff
1196           (pfile, sizeof (cpp_hashnode *) * nparms);
1197       else
1198         fun_like = -1;
1199     }
1200
1201   cpp_macro *macro = NULL;
1202
1203   if (fun_like >= 0)
1204     {
1205       macro = _cpp_new_macro (pfile, cmk_traditional,
1206                               _cpp_aligned_alloc (pfile, sizeof (cpp_macro)));
1207       macro->parm.params = params;
1208       macro->paramc = nparms;
1209       macro->fun_like = fun_like != 0;
1210     }
1211
1212   /* Skip leading whitespace in the replacement text.  */
1213   pfile->buffer->cur
1214     = skip_whitespace (pfile, CUR (context),
1215                        CPP_OPTION (pfile, discard_comments_in_macro_exp));
1216
1217   pfile->state.prevent_expansion++;
1218   _cpp_scan_out_logical_line (pfile, macro, false);
1219   pfile->state.prevent_expansion--;
1220
1221   _cpp_unsave_parameters (pfile, nparms);
1222
1223   if (macro)
1224     {
1225       /* Skip trailing white space.  */
1226       cur = pfile->out.base;
1227       limit = pfile->out.cur;
1228       while (limit > cur && is_space (limit[-1]))
1229         limit--;
1230       pfile->out.cur = limit;
1231       save_replacement_text (pfile, macro, 0);
1232     }
1233
1234   return macro;
1235 }
1236
1237 /* Copy SRC of length LEN to DEST, but convert all contiguous
1238    whitespace to a single space, provided it is not in quotes.  The
1239    quote currently in effect is pointed to by PQUOTE, and is updated
1240    by the function.  Returns the number of bytes copied.  */
1241 static size_t
1242 canonicalize_text (uchar *dest, const uchar *src, size_t len, uchar *pquote)
1243 {
1244   uchar *orig_dest = dest;
1245   uchar quote = *pquote;
1246
1247   while (len)
1248     {
1249       if (is_space (*src) && !quote)
1250         {
1251           do
1252             src++, len--;
1253           while (len && is_space (*src));
1254           *dest++ = ' ';
1255         }
1256       else
1257         {
1258           if (*src == '\'' || *src == '"')
1259             {
1260               if (!quote)
1261                 quote = *src;
1262               else if (quote == *src)
1263                 quote = 0;
1264             }
1265           *dest++ = *src++, len--;
1266         }
1267     }
1268
1269   *pquote = quote;
1270   return dest - orig_dest;
1271 }
1272
1273 /* Returns true if MACRO1 and MACRO2 have expansions different other
1274    than in the form of their whitespace.  */
1275 bool
1276 _cpp_expansions_different_trad (const cpp_macro *macro1,
1277                                 const cpp_macro *macro2)
1278 {
1279   uchar *p1 = XNEWVEC (uchar, macro1->count + macro2->count);
1280   uchar *p2 = p1 + macro1->count;
1281   uchar quote1 = 0, quote2 = 0;
1282   bool mismatch;
1283   size_t len1, len2;
1284
1285   if (macro1->paramc > 0)
1286     {
1287       const uchar *exp1 = macro1->exp.text, *exp2 = macro2->exp.text;
1288
1289       mismatch = true;
1290       for (;;)
1291         {
1292           struct block *b1 = (struct block *) exp1;
1293           struct block *b2 = (struct block *) exp2;
1294
1295           if (b1->arg_index != b2->arg_index)
1296             break;
1297
1298           len1 = canonicalize_text (p1, b1->text, b1->text_len, &quote1);
1299           len2 = canonicalize_text (p2, b2->text, b2->text_len, &quote2);
1300           if (len1 != len2 || memcmp (p1, p2, len1))
1301             break;
1302           if (b1->arg_index == 0)
1303             {
1304               mismatch = false;
1305               break;
1306             }
1307           exp1 += BLOCK_LEN (b1->text_len);
1308           exp2 += BLOCK_LEN (b2->text_len);
1309         }
1310     }
1311   else
1312     {
1313       len1 = canonicalize_text (p1, macro1->exp.text, macro1->count, &quote1);
1314       len2 = canonicalize_text (p2, macro2->exp.text, macro2->count, &quote2);
1315       mismatch = (len1 != len2 || memcmp (p1, p2, len1));
1316     }
1317
1318   free (p1);
1319   return mismatch;
1320 }