libcpp/traditional.c

   1 /* CPP Library - traditional lexical analysis and macro expansion.
   2    Copyright (C) 2002-2020 Free Software Foundation, Inc.
   3    Contributed by Neil Booth, May 2002
   4
   5 This program is free software; you can redistribute it and/or modify it
   6 under the terms of the GNU General Public License as published by the
   7 Free Software Foundation; either version 3, or (at your option) any
   8 later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program; see the file COPYING3.  If not see
  17 <http://www.gnu.org/licenses/>.  */
  18
  19 #include "config.h"
  20 #include "system.h"
  21 #include "cpplib.h"
  22 #include "internal.h"
  23
  24 /* The replacement text of a function-like macro is stored as a
  25    contiguous sequence of aligned blocks, each representing the text
  26    between subsequent parameters.
  27
  28    Each block comprises the text between its surrounding parameters,
  29    the length of that text, and the one-based index of the following
  30    parameter.  The final block in the replacement text is easily
  31    recognizable as it has an argument index of zero.  */
  32
  33 struct block
  34 {
  35   unsigned int text_len;
  36   unsigned short arg_index;
  37   uchar text[1];
  38 };
  39
  40 #define BLOCK_HEADER_LEN offsetof (struct block, text)
  41 #define BLOCK_LEN(TEXT_LEN) CPP_ALIGN (BLOCK_HEADER_LEN + (TEXT_LEN))
  42
  43 /* Structure holding information about a function-like macro
  44    invocation.  */
  45 struct fun_macro
  46 {
  47   /* Memory buffer holding the trad_arg array.  */
  48   _cpp_buff *buff;
  49
  50   /* An array of size the number of macro parameters + 1, containing
  51      the offsets of the start of each macro argument in the output
  52      buffer.  The argument continues until the character before the
  53      start of the next one.  */
  54   size_t *args;
  55
  56   /* The hashnode of the macro.  */
  57   cpp_hashnode *node;
  58
  59   /* The offset of the macro name in the output buffer.  */
  60   size_t offset;
  61
  62   /* The line the macro name appeared on.  */
  63   location_t line;
  64
  65   /* Number of parameters.  */
  66   unsigned int paramc;
  67
  68   /* Zero-based index of argument being currently lexed.  */
  69   unsigned int argc;
  70 };
  71
  72 /* Lexing state.  It is mostly used to prevent macro expansion.  */
  73 enum ls {ls_none = 0,           /* Normal state.  */
  74          ls_fun_open,           /* When looking for '('.  */
  75          ls_fun_close,          /* When looking for ')'.  */
  76          ls_defined,            /* After defined.  */
  77          ls_defined_close,      /* Looking for ')' of defined().  */
  78          ls_hash,               /* After # in preprocessor conditional.  */
  79          ls_predicate,          /* After the predicate, maybe paren?  */
  80          ls_answer              /* In answer to predicate.  */
  81 };
  82
  83 /* Lexing TODO: Maybe handle space in escaped newlines.  Stop lex.c
  84    from recognizing comments and directives during its lexing pass.  */
  85
  86 static const uchar *skip_whitespace (cpp_reader *, const uchar *, int);
  87 static cpp_hashnode *lex_identifier (cpp_reader *, const uchar *);
  88 static const uchar *copy_comment (cpp_reader *, const uchar *, int);
  89 static void check_output_buffer (cpp_reader *, size_t);
  90 static void push_replacement_text (cpp_reader *, cpp_hashnode *);
  91 static bool scan_parameters (cpp_reader *, unsigned *);
  92 static bool recursive_macro (cpp_reader *, cpp_hashnode *);
  93 static void save_replacement_text (cpp_reader *, cpp_macro *, unsigned int);
  94 static void maybe_start_funlike (cpp_reader *, cpp_hashnode *, const uchar *,
  95                                  struct fun_macro *);
  96 static void save_argument (struct fun_macro *, size_t);
  97 static void replace_args_and_push (cpp_reader *, struct fun_macro *);
  98 static size_t canonicalize_text (uchar *, const uchar *, size_t, uchar *);
  99
 100 /* Ensures we have N bytes' space in the output buffer, and
 101    reallocates it if not.  */
 102 static void
 103 check_output_buffer (cpp_reader *pfile, size_t n)
 104 {
 105   /* We might need two bytes to terminate an unterminated comment, and
 106      one more to terminate the line with a NUL.  */
 107   n += 2 + 1;
 108
 109   if (n > (size_t) (pfile->out.limit - pfile->out.cur))
 110     {
 111       size_t size = pfile->out.cur - pfile->out.base;
 112       size_t new_size = (size + n) * 3 / 2;
 113
 114       pfile->out.base = XRESIZEVEC (unsigned char, pfile->out.base, new_size);
 115       pfile->out.limit = pfile->out.base + new_size;
 116       pfile->out.cur = pfile->out.base + size;
 117     }
 118 }
 119
 120 /* Skip a C-style block comment in a macro as a result of -CC.
 121    PFILE->buffer->cur points to the initial asterisk of the comment,
 122    change it to point to after the '*' and '/' characters that terminate it.
 123    Return true if the macro has not been termined, in that case set
 124    PFILE->buffer->cur to the end of the buffer.  */
 125 static bool
 126 skip_macro_block_comment (cpp_reader *pfile)
 127 {
 128   const uchar *cur = pfile->buffer->cur;
 129
 130   cur++;
 131   if (*cur == '/')
 132     cur++;
 133
 134   /* People like decorating comments with '*', so check for '/'
 135      instead for efficiency.  */
 136   while (! (*cur++ == '/' && cur[-2] == '*'))
 137     if (cur[-1] == '\n')
 138       {
 139         pfile->buffer->cur = cur - 1;
 140         return true;
 141       }
 142
 143   pfile->buffer->cur = cur;
 144   return false;
 145 }
 146
 147 /* CUR points to the asterisk introducing a comment in the current
 148    context.  IN_DEFINE is true if we are in the replacement text of a
 149    macro.
 150
 151    The asterisk and following comment is copied to the buffer pointed
 152    to by pfile->out.cur, which must be of sufficient size.
 153    Unterminated comments are diagnosed, and correctly terminated in
 154    the output.  pfile->out.cur is updated depending upon IN_DEFINE,
 155    -C, -CC and pfile->state.in_directive.
 156
 157    Returns a pointer to the first character after the comment in the
 158    input buffer.  */
 159 static const uchar *
 160 copy_comment (cpp_reader *pfile, const uchar *cur, int in_define)
 161 {
 162   bool unterminated, copy = false;
 163   location_t src_loc = pfile->line_table->highest_line;
 164   cpp_buffer *buffer = pfile->buffer;
 165
 166   buffer->cur = cur;
 167   if (pfile->context->prev)
 168     unterminated = skip_macro_block_comment (pfile);
 169   else
 170     unterminated = _cpp_skip_block_comment (pfile);
 171
 172   if (unterminated)
 173     cpp_error_with_line (pfile, CPP_DL_ERROR, src_loc, 0,
 174                          "unterminated comment");
 175
 176   /* Comments in directives become spaces so that tokens are properly
 177      separated when the ISO preprocessor re-lexes the line.  The
 178      exception is #define.  */
 179   if (pfile->state.in_directive)
 180     {
 181       if (in_define)
 182         {
 183           if (CPP_OPTION (pfile, discard_comments_in_macro_exp))
 184             pfile->out.cur--;
 185           else
 186             copy = true;
 187         }
 188       else
 189         pfile->out.cur[-1] = ' ';
 190     }
 191   else if (CPP_OPTION (pfile, discard_comments))
 192     pfile->out.cur--;
 193   else
 194     copy = true;
 195
 196   if (copy)
 197     {
 198       size_t len = (size_t) (buffer->cur - cur);
 199       memcpy (pfile->out.cur, cur, len);
 200       pfile->out.cur += len;
 201       if (unterminated)
 202         {
 203           *pfile->out.cur++ = '*';
 204           *pfile->out.cur++ = '/';
 205         }
 206     }
 207
 208   return buffer->cur;
 209 }
 210
 211 /* CUR points to any character in the input buffer.  Skips over all
 212    contiguous horizontal white space and NULs, including comments if
 213    SKIP_COMMENTS, until reaching the first non-horizontal-whitespace
 214    character or the end of the current context.  Escaped newlines are
 215    removed.
 216
 217    The whitespace is copied verbatim to the output buffer, except that
 218    comments are handled as described in copy_comment().
 219    pfile->out.cur is updated.
 220
 221    Returns a pointer to the first character after the whitespace in
 222    the input buffer.  */
 223 static const uchar *
 224 skip_whitespace (cpp_reader *pfile, const uchar *cur, int skip_comments)
 225 {
 226   uchar *out = pfile->out.cur;
 227
 228   for (;;)
 229     {
 230       unsigned int c = *cur++;
 231       *out++ = c;
 232
 233       if (is_nvspace (c))
 234         continue;
 235
 236       if (c == '/' && *cur == '*' && skip_comments)
 237         {
 238           pfile->out.cur = out;
 239           cur = copy_comment (pfile, cur, false /* in_define */);
 240           out = pfile->out.cur;
 241           continue;
 242         }
 243
 244       out--;
 245       break;
 246     }
 247
 248   pfile->out.cur = out;
 249   return cur - 1;
 250 }
 251
 252 /* Lexes and outputs an identifier starting at CUR, which is assumed
 253    to point to a valid first character of an identifier.  Returns
 254    the hashnode, and updates out.cur.  */
 255 static cpp_hashnode *
 256 lex_identifier (cpp_reader *pfile, const uchar *cur)
 257 {
 258   size_t len;
 259   uchar *out = pfile->out.cur;
 260   cpp_hashnode *result;
 261
 262   do
 263     *out++ = *cur++;
 264   while (is_numchar (*cur));
 265
 266   CUR (pfile->context) = cur;
 267   len = out - pfile->out.cur;
 268   result = CPP_HASHNODE (ht_lookup (pfile->hash_table, pfile->out.cur,
 269                                     len, HT_ALLOC));
 270   pfile->out.cur = out;
 271   return result;
 272 }
 273
 274 /* Overlays the true file buffer temporarily with text of length LEN
 275    starting at START.  The true buffer is restored upon calling
 276    restore_buff().  */
 277 void
 278 _cpp_overlay_buffer (cpp_reader *pfile, const uchar *start, size_t len)
 279 {
 280   cpp_buffer *buffer = pfile->buffer;
 281
 282   pfile->overlaid_buffer = buffer;
 283   pfile->saved_cur = buffer->cur;
 284   pfile->saved_rlimit = buffer->rlimit;
 285   pfile->saved_line_base = buffer->next_line;
 286   buffer->need_line = false;
 287
 288   buffer->cur = start;
 289   buffer->line_base = start;
 290   buffer->rlimit = start + len;
 291 }
 292
 293 /* Restores a buffer overlaid by _cpp_overlay_buffer().  */
 294 void
 295 _cpp_remove_overlay (cpp_reader *pfile)
 296 {
 297   cpp_buffer *buffer = pfile->overlaid_buffer;
 298
 299   buffer->cur = pfile->saved_cur;
 300   buffer->rlimit = pfile->saved_rlimit;
 301   buffer->line_base = pfile->saved_line_base;
 302   buffer->need_line = true;
 303
 304   pfile->overlaid_buffer = NULL;
 305 }
 306
 307 /* Reads a logical line into the output buffer.  Returns TRUE if there
 308    is more text left in the buffer.  */
 309 bool
 310 _cpp_read_logical_line_trad (cpp_reader *pfile)
 311 {
 312   do
 313     {
 314       if (pfile->buffer->need_line && !_cpp_get_fresh_line (pfile))
 315         return false;
 316     }
 317   while (!_cpp_scan_out_logical_line (pfile, NULL, false)
 318          || pfile->state.skipping);
 319
 320   return pfile->buffer != NULL;
 321 }
 322
 323 /* Return true if NODE is a fun_like macro.  */
 324 static inline bool
 325 fun_like_macro (cpp_hashnode *node)
 326 {
 327   if (cpp_builtin_macro_p (node))
 328     return (node->value.builtin == BT_HAS_ATTRIBUTE
 329             || node->value.builtin == BT_HAS_BUILTIN);
 330   return node->value.macro->fun_like;
 331 }
 332
 333 /* Set up state for finding the opening '(' of a function-like
 334    macro.  */
 335 static void
 336 maybe_start_funlike (cpp_reader *pfile, cpp_hashnode *node, const uchar *start,
 337                      struct fun_macro *macro)
 338 {
 339   unsigned int n;
 340   if (cpp_builtin_macro_p (node))
 341     n = 1;
 342   else
 343     n = node->value.macro->paramc;
 344
 345   if (macro->buff)
 346     _cpp_release_buff (pfile, macro->buff);
 347   macro->buff = _cpp_get_buff (pfile, (n + 1) * sizeof (size_t));
 348   macro->args = (size_t *) BUFF_FRONT (macro->buff);
 349   macro->node = node;
 350   macro->offset = start - pfile->out.base;
 351   macro->paramc = n;
 352   macro->argc = 0;
 353 }
 354
 355 /* Save the OFFSET of the start of the next argument to MACRO.  */
 356 static void
 357 save_argument (struct fun_macro *macro, size_t offset)
 358 {
 359   macro->argc++;
 360   if (macro->argc <= macro->paramc)
 361     macro->args[macro->argc] = offset;
 362 }
 363
 364 /* Copies the next logical line in the current buffer (starting at
 365    buffer->cur) to the output buffer.  The output is guaranteed to
 366    terminate with a NUL character.  buffer->cur is updated.
 367
 368    If MACRO is non-NULL, then we are scanning the replacement list of
 369    MACRO, and we call save_replacement_text() every time we meet an
 370    argument.
 371
 372    If BUILTIN_MACRO_ARG is true, this is called to macro expand
 373    arguments of builtin function-like macros.  */
 374 bool
 375 _cpp_scan_out_logical_line (cpp_reader *pfile, cpp_macro *macro,
 376                             bool builtin_macro_arg)
 377 {
 378   bool result = true;
 379   cpp_context *context;
 380   const uchar *cur;
 381   uchar *out;
 382   struct fun_macro fmacro;
 383   unsigned int c, paren_depth = 0, quote;
 384   enum ls lex_state = ls_none;
 385   bool header_ok;
 386   const uchar *start_of_input_line;
 387
 388   fmacro.buff = NULL;
 389   fmacro.args = NULL;
 390   fmacro.node = NULL;
 391   fmacro.offset = 0;
 392   fmacro.line = 0;
 393   fmacro.paramc = 0;
 394   fmacro.argc = 0;
 395
 396   quote = 0;
 397   header_ok = pfile->state.angled_headers;
 398   CUR (pfile->context) = pfile->buffer->cur;
 399   RLIMIT (pfile->context) = pfile->buffer->rlimit;
 400   if (!builtin_macro_arg)
 401     {
 402       pfile->out.cur = pfile->out.base;
 403       pfile->out.first_line = pfile->line_table->highest_line;
 404     }
 405   /* start_of_input_line is needed to make sure that directives really,
 406      really start at the first character of the line.  */
 407   start_of_input_line = pfile->buffer->cur;
 408  new_context:
 409   context = pfile->context;
 410   cur = CUR (context);
 411   check_output_buffer (pfile, RLIMIT (context) - cur);
 412   out = pfile->out.cur;
 413
 414   for (;;)
 415     {
 416       if (!context->prev
 417           && !builtin_macro_arg
 418           && cur >= pfile->buffer->notes[pfile->buffer->cur_note].pos)
 419         {
 420           pfile->buffer->cur = cur;
 421           _cpp_process_line_notes (pfile, false);
 422         }
 423       c = *cur++;
 424       *out++ = c;
 425
 426       /* Whitespace should "continue" out of the switch,
 427          non-whitespace should "break" out of it.  */
 428       switch (c)
 429         {
 430         case ' ':
 431         case '\t':
 432         case '\f':
 433         case '\v':
 434         case '\0':
 435           continue;
 436
 437         case '\n':
 438           /* If this is a macro's expansion, pop it.  */
 439           if (context->prev)
 440             {
 441               pfile->out.cur = out - 1;
 442               _cpp_pop_context (pfile);
 443               goto new_context;
 444             }
 445
 446           /* Omit the newline from the output buffer.  */
 447           pfile->out.cur = out - 1;
 448           pfile->buffer->cur = cur;
 449           if (builtin_macro_arg)
 450             goto done;
 451           pfile->buffer->need_line = true;
 452           CPP_INCREMENT_LINE (pfile, 0);
 453
 454           if ((lex_state == ls_fun_open || lex_state == ls_fun_close)
 455               && !pfile->state.in_directive
 456               && _cpp_get_fresh_line (pfile))
 457             {
 458               /* Newlines in arguments become a space, but we don't
 459                  clear any in-progress quote.  */
 460               if (lex_state == ls_fun_close)
 461                 out[-1] = ' ';
 462               cur = pfile->buffer->cur;
 463               continue;
 464             }
 465           goto done;
 466
 467         case '<':
 468           if (header_ok)
 469             quote = '>';
 470           break;
 471         case '>':
 472           if (c == quote)
 473             quote = 0;
 474           break;
 475
 476         case '"':
 477         case '\'':
 478           if (c == quote)
 479             quote = 0;
 480           else if (!quote)
 481             quote = c;
 482           break;
 483
 484         case '\\':
 485           /* Skip escaped quotes here, it's easier than above.  */
 486           if (*cur == '\\' || *cur == '"' || *cur == '\'')
 487             *out++ = *cur++;
 488           break;
 489
 490         case '/':
 491           /* Traditional CPP does not recognize comments within
 492              literals.  */
 493           if (!quote && *cur == '*')
 494             {
 495               pfile->out.cur = out;
 496               cur = copy_comment (pfile, cur, macro != 0);
 497               out = pfile->out.cur;
 498               continue;
 499             }
 500           break;
 501
 502         case '_':
 503         case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
 504         case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
 505         case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
 506         case 's': case 't': case 'u': case 'v': case 'w': case 'x':
 507         case 'y': case 'z':
 508         case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
 509         case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
 510         case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
 511         case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
 512         case 'Y': case 'Z':
 513           if (!pfile->state.skipping && (quote == 0 || macro))
 514             {
 515               cpp_hashnode *node;
 516               uchar *out_start = out - 1;
 517
 518               pfile->out.cur = out_start;
 519               node = lex_identifier (pfile, cur - 1);
 520               out = pfile->out.cur;
 521               cur = CUR (context);
 522
 523               if (cpp_macro_p (node)
 524                   /* Should we expand for ls_answer?  */
 525                   && (lex_state == ls_none || lex_state == ls_fun_open)
 526                   && !pfile->state.prevent_expansion)
 527                 {
 528                   /* Macros invalidate MI optimization.  */
 529                   pfile->mi_valid = false;
 530                   if (fun_like_macro (node))
 531                     {
 532                       maybe_start_funlike (pfile, node, out_start, &fmacro);
 533                       lex_state = ls_fun_open;
 534                       fmacro.line = pfile->line_table->highest_line;
 535                       continue;
 536                     }
 537                   else if (!recursive_macro (pfile, node))
 538                     {
 539                       /* Remove the object-like macro's name from the
 540                          output, and push its replacement text.  */
 541                       pfile->out.cur = out_start;
 542                       push_replacement_text (pfile, node);
 543                       lex_state = ls_none;
 544                       goto new_context;
 545                     }
 546                 }
 547               else if (macro && node->type == NT_MACRO_ARG)
 548                 {
 549                   /* Found a parameter in the replacement text of a
 550                      #define.  Remove its name from the output.  */
 551                   pfile->out.cur = out_start;
 552                   save_replacement_text (pfile, macro, node->value.arg_index);
 553                   out = pfile->out.base;
 554                 }
 555               else if (lex_state == ls_hash)
 556                 {
 557                   lex_state = ls_predicate;
 558                   continue;
 559                 }
 560               else if (pfile->state.in_expression
 561                        && node == pfile->spec_nodes.n_defined)
 562                 {
 563                   lex_state = ls_defined;
 564                   continue;
 565                 }
 566             }
 567           break;
 568
 569         case '(':
 570           if (quote == 0)
 571             {
 572               paren_depth++;
 573               if (lex_state == ls_fun_open)
 574                 {
 575                   if (recursive_macro (pfile, fmacro.node))
 576                     lex_state = ls_none;
 577                   else
 578                     {
 579                       lex_state = ls_fun_close;
 580                       paren_depth = 1;
 581                       out = pfile->out.base + fmacro.offset;
 582                       fmacro.args[0] = fmacro.offset;
 583                     }
 584                 }
 585               else if (lex_state == ls_predicate)
 586                 lex_state = ls_answer;
 587               else if (lex_state == ls_defined)
 588                 lex_state = ls_defined_close;
 589             }
 590           break;
 591
 592         case ',':
 593           if (quote == 0 && lex_state == ls_fun_close && paren_depth == 1)
 594             save_argument (&fmacro, out - pfile->out.base);
 595           break;
 596
 597         case ')':
 598           if (quote == 0)
 599             {
 600               paren_depth--;
 601               if (lex_state == ls_fun_close && paren_depth == 0)
 602                 {
 603                   if (cpp_builtin_macro_p (fmacro.node))
 604                     {
 605                       /* Handle builtin function-like macros like
 606                          __has_attribute.  The already parsed arguments
 607                          are put into a buffer, which is then preprocessed
 608                          and the result is fed to _cpp_push_text_context
 609                          with disabled expansion, where the ISO preprocessor
 610                          parses it.  While in traditional preprocessing
 611                          macro arguments aren't immediately expanded, they in
 612                          the end are because the macro with replaced arguments
 613                          is preprocessed again.  For the builtin function-like
 614                          macros we need the argument immediately though,
 615                          if we don't preprocess them, they would behave
 616                          very differently from ISO preprocessor handling
 617                          of those builtin macros.  So, this handling is
 618                          more similar to traditional preprocessing of
 619                          #if directives, where we also keep preprocessing
 620                          until everything is expanded, and then feed the
 621                          result with disabled expansion to ISO preprocessor
 622                          for handling the directives.  */
 623                       lex_state = ls_none;
 624                       save_argument (&fmacro, out - pfile->out.base);
 625                       cpp_macro m;
 626                       memset (&m, '\0', sizeof (m));
 627                       m.paramc = fmacro.paramc;
 628                       if (_cpp_arguments_ok (pfile, &m, fmacro.node,
 629                                              fmacro.argc))
 630                         {
 631                           size_t len = fmacro.args[1] - fmacro.args[0];
 632                           uchar *buf;
 633
 634                           /* Remove the macro's invocation from the
 635                              output, and push its replacement text.  */
 636                           pfile->out.cur = pfile->out.base + fmacro.offset;
 637                           CUR (context) = cur;
 638                           buf = _cpp_unaligned_alloc (pfile, len + 2);
 639                           buf[0] = '(';
 640                           memcpy (buf + 1, pfile->out.base + fmacro.args[0],
 641                                   len);
 642                           buf[len + 1] = '\n';
 643
 644                           const unsigned char *ctx_rlimit = RLIMIT (context);
 645                           const unsigned char *saved_cur = pfile->buffer->cur;
 646                           const unsigned char *saved_rlimit
 647                             = pfile->buffer->rlimit;
 648                           const unsigned char *saved_line_base
 649                             = pfile->buffer->line_base;
 650                           bool saved_need_line = pfile->buffer->need_line;
 651                           cpp_buffer *saved_overlaid_buffer
 652                             = pfile->overlaid_buffer;
 653                           pfile->buffer->cur = buf;
 654                           pfile->buffer->line_base = buf;
 655                           pfile->buffer->rlimit = buf + len + 1;
 656                           pfile->buffer->need_line = false;
 657                           pfile->overlaid_buffer = pfile->buffer;
 658                           bool saved_in_directive = pfile->state.in_directive;
 659                           pfile->state.in_directive = true;
 660                           cpp_context *saved_prev_context = context->prev;
 661                           context->prev = NULL;
 662
 663                           _cpp_scan_out_logical_line (pfile, NULL, true);
 664
 665                           pfile->state.in_directive = saved_in_directive;
 666                           check_output_buffer (pfile, 1);
 667                           *pfile->out.cur = '\n';
 668                           pfile->buffer->cur = pfile->out.base + fmacro.offset;
 669                           pfile->buffer->line_base = pfile->buffer->cur;
 670                           pfile->buffer->rlimit = pfile->out.cur;
 671                           CUR (context) = pfile->buffer->cur;
 672                           RLIMIT (context) = pfile->buffer->rlimit;
 673
 674                           pfile->state.prevent_expansion++;
 675                           const uchar *text
 676                             = _cpp_builtin_macro_text (pfile, fmacro.node);
 677                           pfile->state.prevent_expansion--;
 678
 679                           context->prev = saved_prev_context;
 680                           pfile->buffer->cur = saved_cur;
 681                           pfile->buffer->rlimit = saved_rlimit;
 682                           pfile->buffer->line_base = saved_line_base;
 683                           pfile->buffer->need_line = saved_need_line;
 684                           pfile->overlaid_buffer = saved_overlaid_buffer;
 685                           pfile->out.cur = pfile->out.base + fmacro.offset;
 686                           CUR (context) = cur;
 687                           RLIMIT (context) = ctx_rlimit;
 688                           len = ustrlen (text);
 689                           buf = _cpp_unaligned_alloc (pfile, len + 1);
 690                           memcpy (buf, text, len);
 691                           buf[len] = '\n';
 692                           text = buf;
 693                           _cpp_push_text_context (pfile, fmacro.node,
 694                                                   text, len);
 695                           goto new_context;
 696                         }
 697                       break;
 698                     }
 699
 700                   cpp_macro *m = fmacro.node->value.macro;
 701
 702                   m->used = 1;
 703                   lex_state = ls_none;
 704                   save_argument (&fmacro, out - pfile->out.base);
 705
 706                   /* A single zero-length argument is no argument.  */
 707                   if (fmacro.argc == 1
 708                       && m->paramc == 0
 709                       && out == pfile->out.base + fmacro.offset + 1)
 710                     fmacro.argc = 0;
 711
 712                   if (_cpp_arguments_ok (pfile, m, fmacro.node, fmacro.argc))
 713                     {
 714                       /* Remove the macro's invocation from the
 715                          output, and push its replacement text.  */
 716                       pfile->out.cur = pfile->out.base + fmacro.offset;
 717                       CUR (context) = cur;
 718                       replace_args_and_push (pfile, &fmacro);
 719                       goto new_context;
 720                     }
 721                 }
 722               else if (lex_state == ls_answer || lex_state == ls_defined_close)
 723                 lex_state = ls_none;
 724             }
 725           break;
 726
 727         case '#':
 728           if (cur - 1 == start_of_input_line
 729               /* A '#' from a macro doesn't start a directive.  */
 730               && !pfile->context->prev
 731               && !pfile->state.in_directive)
 732             {
 733               /* A directive.  With the way _cpp_handle_directive
 734                  currently works, we only want to call it if either we
 735                  know the directive is OK, or we want it to fail and
 736                  be removed from the output.  If we want it to be
 737                  passed through (the assembler case) then we must not
 738                  call _cpp_handle_directive.  */
 739               pfile->out.cur = out;
 740               cur = skip_whitespace (pfile, cur, true /* skip_comments */);
 741               out = pfile->out.cur;
 742
 743               if (*cur == '\n')
 744                 {
 745                   /* Null directive.  Ignore it and don't invalidate
 746                      the MI optimization.  */
 747                   pfile->buffer->need_line = true;
 748                   CPP_INCREMENT_LINE (pfile, 0);
 749                   result = false;
 750                   goto done;
 751                 }
 752               else
 753                 {
 754                   bool do_it = false;
 755
 756                   if (is_numstart (*cur)
 757                       && CPP_OPTION (pfile, lang) != CLK_ASM)
 758                     do_it = true;
 759                   else if (is_idstart (*cur))
 760                     /* Check whether we know this directive, but don't
 761                        advance.  */
 762                     do_it = lex_identifier (pfile, cur)->is_directive;
 763
 764                   if (do_it || CPP_OPTION (pfile, lang) != CLK_ASM)
 765                     {
 766                       /* This is a kludge.  We want to have the ISO
 767                          preprocessor lex the next token.  */
 768                       pfile->buffer->cur = cur;
 769                       _cpp_handle_directive (pfile, false /* indented */);
 770                       result = false;
 771                       goto done;
 772                     }
 773                 }
 774             }
 775
 776           if (pfile->state.in_expression)
 777             {
 778               lex_state = ls_hash;
 779               continue;
 780             }
 781           break;
 782
 783         default:
 784           break;
 785         }
 786
 787       /* Non-whitespace disables MI optimization and stops treating
 788          '<' as a quote in #include.  */
 789       header_ok = false;
 790       if (!pfile->state.in_directive)
 791         pfile->mi_valid = false;
 792
 793       if (lex_state == ls_none)
 794         continue;
 795
 796       /* Some of these transitions of state are syntax errors.  The
 797          ISO preprocessor will issue errors later.  */
 798       if (lex_state == ls_fun_open)
 799         /* Missing '('.  */
 800         lex_state = ls_none;
 801       else if (lex_state == ls_hash
 802                || lex_state == ls_predicate
 803                || lex_state == ls_defined)
 804         lex_state = ls_none;
 805
 806       /* ls_answer and ls_defined_close keep going until ')'.  */
 807     }
 808
 809  done:
 810   if (fmacro.buff)
 811     _cpp_release_buff (pfile, fmacro.buff);
 812
 813   if (lex_state == ls_fun_close)
 814     cpp_error_with_line (pfile, CPP_DL_ERROR, fmacro.line, 0,
 815                          "unterminated argument list invoking macro \"%s\"",
 816                          NODE_NAME (fmacro.node));
 817   return result;
 818 }
 819
 820 /* Push a context holding the replacement text of the macro NODE on
 821    the context stack.  NODE is either object-like, or a function-like
 822    macro with no arguments.  */
 823 static void
 824 push_replacement_text (cpp_reader *pfile, cpp_hashnode *node)
 825 {
 826   size_t len;
 827   const uchar *text;
 828   uchar *buf;
 829
 830   if (cpp_builtin_macro_p (node))
 831     {
 832       text = _cpp_builtin_macro_text (pfile, node);
 833       len = ustrlen (text);
 834       buf = _cpp_unaligned_alloc (pfile, len + 1);
 835       memcpy (buf, text, len);
 836       buf[len] = '\n';
 837       text = buf;
 838     }
 839   else
 840     {
 841       cpp_macro *macro = node->value.macro;
 842       macro->used = 1;
 843       text = macro->exp.text;
 844       len = macro->count;
 845     }
 846
 847   _cpp_push_text_context (pfile, node, text, len);
 848 }
 849
 850 /* Returns TRUE if traditional macro recursion is detected.  */
 851 static bool
 852 recursive_macro (cpp_reader *pfile, cpp_hashnode *node)
 853 {
 854   bool recursing = !!(node->flags & NODE_DISABLED);
 855
 856   /* Object-like macros that are already expanding are necessarily
 857      recursive.
 858
 859      However, it is possible to have traditional function-like macros
 860      that are not infinitely recursive but recurse to any given depth.
 861      Further, it is easy to construct examples that get ever longer
 862      until the point they stop recursing.  So there is no easy way to
 863      detect true recursion; instead we assume any expansion more than
 864      20 deep since the first invocation of this macro must be
 865      recursing.  */
 866   if (recursing && fun_like_macro (node))
 867     {
 868       size_t depth = 0;
 869       cpp_context *context = pfile->context;
 870
 871       do
 872         {
 873           depth++;
 874           if (context->c.macro == node && depth > 20)
 875             break;
 876           context = context->prev;
 877         }
 878       while (context);
 879       recursing = context != NULL;
 880     }
 881
 882   if (recursing)
 883     cpp_error (pfile, CPP_DL_ERROR,
 884                "detected recursion whilst expanding macro \"%s\"",
 885                NODE_NAME (node));
 886
 887   return recursing;
 888 }
 889
 890 /* Return the length of the replacement text of a function-like or
 891    object-like non-builtin macro.  */
 892 size_t
 893 _cpp_replacement_text_len (const cpp_macro *macro)
 894 {
 895   size_t len;
 896
 897   if (macro->fun_like && (macro->paramc != 0))
 898     {
 899       const uchar *exp;
 900
 901       len = 0;
 902       for (exp = macro->exp.text;;)
 903         {
 904           struct block *b = (struct block *) exp;
 905
 906           len += b->text_len;
 907           if (b->arg_index == 0)
 908             break;
 909           len += NODE_LEN (macro->parm.params[b->arg_index - 1]);
 910           exp += BLOCK_LEN (b->text_len);
 911         }
 912     }
 913   else
 914     len = macro->count;
 915
 916   return len;
 917 }
 918
 919 /* Copy the replacement text of MACRO to DEST, which must be of
 920    sufficient size.  It is not NUL-terminated.  The next character is
 921    returned.  */
 922 uchar *
 923 _cpp_copy_replacement_text (const cpp_macro *macro, uchar *dest)
 924 {
 925   if (macro->fun_like && (macro->paramc != 0))
 926     {
 927       const uchar *exp;
 928
 929       for (exp = macro->exp.text;;)
 930         {
 931           struct block *b = (struct block *) exp;
 932           cpp_hashnode *param;
 933
 934           memcpy (dest, b->text, b->text_len);
 935           dest += b->text_len;
 936           if (b->arg_index == 0)
 937             break;
 938           param = macro->parm.params[b->arg_index - 1];
 939           memcpy (dest, NODE_NAME (param), NODE_LEN (param));
 940           dest += NODE_LEN (param);
 941           exp += BLOCK_LEN (b->text_len);
 942         }
 943     }
 944   else
 945     {
 946       memcpy (dest, macro->exp.text, macro->count);
 947       dest += macro->count;
 948     }
 949
 950   return dest;
 951 }
 952
 953 /* Push a context holding the replacement text of the macro NODE on
 954    the context stack.  NODE is either object-like, or a function-like
 955    macro with no arguments.  */
 956 static void
 957 replace_args_and_push (cpp_reader *pfile, struct fun_macro *fmacro)
 958 {
 959   cpp_macro *macro = fmacro->node->value.macro;
 960
 961   if (macro->paramc == 0)
 962     push_replacement_text (pfile, fmacro->node);
 963   else
 964     {
 965       const uchar *exp;
 966       uchar *p;
 967       _cpp_buff *buff;
 968       size_t len = 0;
 969       int cxtquote = 0;
 970
 971       /* Get an estimate of the length of the argument-replaced text.
 972          This is a worst case estimate, assuming that every replacement
 973          text character needs quoting.  */
 974       for (exp = macro->exp.text;;)
 975         {
 976           struct block *b = (struct block *) exp;
 977
 978           len += b->text_len;
 979           if (b->arg_index == 0)
 980             break;
 981           len += 2 * (fmacro->args[b->arg_index]
 982                       - fmacro->args[b->arg_index - 1] - 1);
 983           exp += BLOCK_LEN (b->text_len);
 984         }
 985
 986       /* Allocate room for the expansion plus \n.  */
 987       buff = _cpp_get_buff (pfile, len + 1);
 988
 989       /* Copy the expansion and replace arguments.  */
 990       /* Accumulate actual length, including quoting as necessary */
 991       p = BUFF_FRONT (buff);
 992       len = 0;
 993       for (exp = macro->exp.text;;)
 994         {
 995           struct block *b = (struct block *) exp;
 996           size_t arglen;
 997           int argquote;
 998           uchar *base;
 999           uchar *in;
1000
1001           len += b->text_len;
1002           /* Copy the non-argument text literally, keeping
1003              track of whether matching quotes have been seen. */
1004           for (arglen = b->text_len, in = b->text; arglen > 0; arglen--)
1005             {
1006               if (*in == '"')
1007                 cxtquote = ! cxtquote;
1008               *p++ = *in++;
1009             }
1010           /* Done if no more arguments */
1011           if (b->arg_index == 0)
1012             break;
1013           arglen = (fmacro->args[b->arg_index]
1014                     - fmacro->args[b->arg_index - 1] - 1);
1015           base = pfile->out.base + fmacro->args[b->arg_index - 1];
1016           in = base;
1017 #if 0
1018           /* Skip leading whitespace in the text for the argument to
1019              be substituted. To be compatible with gcc 2.95, we would
1020              also need to trim trailing whitespace. Gcc 2.95 trims
1021              leading and trailing whitespace, which may be a bug.  The
1022              current gcc testsuite explicitly checks that this leading
1023              and trailing whitespace in actual arguments is
1024              preserved. */
1025           while (arglen > 0 && is_space (*in))
1026             {
1027               in++;
1028               arglen--;
1029             }
1030 #endif
1031           for (argquote = 0; arglen > 0; arglen--)
1032             {
1033               if (cxtquote && *in == '"')
1034                 {
1035                   if (in > base && *(in-1) != '\\')
1036                     argquote = ! argquote;
1037                   /* Always add backslash before double quote if argument
1038                      is expanded in a quoted context */
1039                   *p++ = '\\';
1040                   len++;
1041                 }
1042               else if (cxtquote && argquote && *in == '\\')
1043                 {
1044                   /* Always add backslash before a backslash in an argument
1045                      that is expanded in a quoted context and also in the
1046                      range of a quoted context in the argument itself. */
1047                   *p++ = '\\';
1048                   len++;
1049                 }
1050               *p++ = *in++;
1051               len++;
1052             }
1053           exp += BLOCK_LEN (b->text_len);
1054         }
1055
1056       /* \n-terminate.  */
1057       *p = '\n';
1058       _cpp_push_text_context (pfile, fmacro->node, BUFF_FRONT (buff), len);
1059
1060       /* So we free buffer allocation when macro is left.  */
1061       pfile->context->buff = buff;
1062     }
1063 }
1064
1065 /* Read and record the parameters, if any, of a function-like macro
1066    definition.  Destroys pfile->out.cur.
1067
1068    Returns true on success, false on failure (syntax error or a
1069    duplicate parameter).  On success, CUR (pfile->context) is just
1070    past the closing parenthesis.  */
1071 static bool
1072 scan_parameters (cpp_reader *pfile, unsigned *n_ptr)
1073 {
1074   const uchar *cur = CUR (pfile->context) + 1;
1075   bool ok;
1076
1077   unsigned nparms = 0;
1078   for (;;)
1079     {
1080       cur = skip_whitespace (pfile, cur, true /* skip_comments */);
1081
1082       if (is_idstart (*cur))
1083         {
1084           struct cpp_hashnode *id = lex_identifier (pfile, cur);
1085           ok = false;
1086           if (!_cpp_save_parameter (pfile, nparms, id, id))
1087             break;
1088           nparms++;
1089           cur = skip_whitespace (pfile, CUR (pfile->context),
1090                                  true /* skip_comments */);
1091           if (*cur == ',')
1092             {
1093               cur++;
1094               continue;
1095             }
1096           ok = (*cur == ')');
1097           break;
1098         }
1099
1100       ok = (*cur == ')' && !nparms);
1101       break;
1102     }
1103
1104   *n_ptr = nparms;
1105
1106   if (!ok)
1107     cpp_error (pfile, CPP_DL_ERROR, "syntax error in macro parameter list");
1108
1109   CUR (pfile->context) = cur + (*cur == ')');
1110
1111   return ok;
1112 }
1113
1114 /* Save the text from pfile->out.base to pfile->out.cur as
1115    the replacement text for the current macro, followed by argument
1116    ARG_INDEX, with zero indicating the end of the replacement
1117    text.  */
1118 static void
1119 save_replacement_text (cpp_reader *pfile, cpp_macro *macro,
1120                        unsigned int arg_index)
1121 {
1122   size_t len = pfile->out.cur - pfile->out.base;
1123   uchar *exp;
1124
1125   if (macro->paramc == 0)
1126     {
1127       /* Object-like and function-like macros without parameters
1128          simply store their \n-terminated replacement text.  */
1129       exp = _cpp_unaligned_alloc (pfile, len + 1);
1130       memcpy (exp, pfile->out.base, len);
1131       exp[len] = '\n';
1132       macro->exp.text = exp;
1133       macro->count = len;
1134     }
1135   else
1136     {
1137       /* Store the text's length (unsigned int), the argument index
1138          (unsigned short, base 1) and then the text.  */
1139       size_t blen = BLOCK_LEN (len);
1140       struct block *block;
1141
1142       if (macro->count + blen > BUFF_ROOM (pfile->a_buff))
1143         _cpp_extend_buff (pfile, &pfile->a_buff, macro->count + blen);
1144
1145       exp = BUFF_FRONT (pfile->a_buff);
1146       block = (struct block *) (exp + macro->count);
1147       macro->exp.text = exp;
1148
1149       /* Write out the block information.  */
1150       block->text_len = len;
1151       block->arg_index = arg_index;
1152       memcpy (block->text, pfile->out.base, len);
1153
1154       /* Lex the rest into the start of the output buffer.  */
1155       pfile->out.cur = pfile->out.base;
1156
1157       macro->count += blen;
1158
1159       /* If we've finished, commit the memory.  */
1160       if (arg_index == 0)
1161         BUFF_FRONT (pfile->a_buff) += macro->count;
1162     }
1163 }
1164
1165 /* Analyze and save the replacement text of a macro.  Returns true on
1166    success.  */
1167 cpp_macro *
1168 _cpp_create_trad_definition (cpp_reader *pfile)
1169 {
1170   const uchar *cur;
1171   uchar *limit;
1172   cpp_context *context = pfile->context;
1173   unsigned nparms = 0;
1174   int fun_like = 0;
1175   cpp_hashnode **params = NULL;
1176
1177   /* The context has not been set up for command line defines, and CUR
1178      has not been updated for the macro name for in-file defines.  */
1179   pfile->out.cur = pfile->out.base;
1180   CUR (context) = pfile->buffer->cur;
1181   RLIMIT (context) = pfile->buffer->rlimit;
1182   check_output_buffer (pfile, RLIMIT (context) - CUR (context));
1183
1184   /* Is this a function-like macro?  */
1185   if (* CUR (context) == '(')
1186     {
1187       fun_like = +1;
1188       if (scan_parameters (pfile, &nparms))
1189         params = (cpp_hashnode **)_cpp_commit_buff
1190           (pfile, sizeof (cpp_hashnode *) * nparms);
1191       else
1192         fun_like = -1;
1193     }
1194
1195   cpp_macro *macro = NULL;
1196
1197   if (fun_like >= 0)
1198     {
1199       macro = _cpp_new_macro (pfile, cmk_traditional,
1200                               _cpp_aligned_alloc (pfile, sizeof (cpp_macro)));
1201       macro->parm.params = params;
1202       macro->paramc = nparms;
1203       macro->fun_like = fun_like != 0;
1204     }
1205
1206   /* Skip leading whitespace in the replacement text.  */
1207   pfile->buffer->cur
1208     = skip_whitespace (pfile, CUR (context),
1209                        CPP_OPTION (pfile, discard_comments_in_macro_exp));
1210
1211   pfile->state.prevent_expansion++;
1212   _cpp_scan_out_logical_line (pfile, macro, false);
1213   pfile->state.prevent_expansion--;
1214
1215   _cpp_unsave_parameters (pfile, nparms);
1216
1217   if (macro)
1218     {
1219       /* Skip trailing white space.  */
1220       cur = pfile->out.base;
1221       limit = pfile->out.cur;
1222       while (limit > cur && is_space (limit[-1]))
1223         limit--;
1224       pfile->out.cur = limit;
1225       save_replacement_text (pfile, macro, 0);
1226     }
1227
1228   return macro;
1229 }
1230
1231 /* Copy SRC of length LEN to DEST, but convert all contiguous
1232    whitespace to a single space, provided it is not in quotes.  The
1233    quote currently in effect is pointed to by PQUOTE, and is updated
1234    by the function.  Returns the number of bytes copied.  */
1235 static size_t
1236 canonicalize_text (uchar *dest, const uchar *src, size_t len, uchar *pquote)
1237 {
1238   uchar *orig_dest = dest;
1239   uchar quote = *pquote;
1240
1241   while (len)
1242     {
1243       if (is_space (*src) && !quote)
1244         {
1245           do
1246             src++, len--;
1247           while (len && is_space (*src));
1248           *dest++ = ' ';
1249         }
1250       else
1251         {
1252           if (*src == '\'' || *src == '"')
1253             {
1254               if (!quote)
1255                 quote = *src;
1256               else if (quote == *src)
1257                 quote = 0;
1258             }
1259           *dest++ = *src++, len--;
1260         }
1261     }
1262
1263   *pquote = quote;
1264   return dest - orig_dest;
1265 }
1266
1267 /* Returns true if MACRO1 and MACRO2 have expansions different other
1268    than in the form of their whitespace.  */
1269 bool
1270 _cpp_expansions_different_trad (const cpp_macro *macro1,
1271                                 const cpp_macro *macro2)
1272 {
1273   uchar *p1 = XNEWVEC (uchar, macro1->count + macro2->count);
1274   uchar *p2 = p1 + macro1->count;
1275   uchar quote1 = 0, quote2 = 0;
1276   bool mismatch;
1277   size_t len1, len2;
1278
1279   if (macro1->paramc > 0)
1280     {
1281       const uchar *exp1 = macro1->exp.text, *exp2 = macro2->exp.text;
1282
1283       mismatch = true;
1284       for (;;)
1285         {
1286           struct block *b1 = (struct block *) exp1;
1287           struct block *b2 = (struct block *) exp2;
1288
1289           if (b1->arg_index != b2->arg_index)
1290             break;
1291
1292           len1 = canonicalize_text (p1, b1->text, b1->text_len, &quote1);
1293           len2 = canonicalize_text (p2, b2->text, b2->text_len, &quote2);
1294           if (len1 != len2 || memcmp (p1, p2, len1))
1295             break;
1296           if (b1->arg_index == 0)
1297             {
1298               mismatch = false;
1299               break;
1300             }
1301           exp1 += BLOCK_LEN (b1->text_len);
1302           exp2 += BLOCK_LEN (b2->text_len);
1303         }
1304     }
1305   else
1306     {
1307       len1 = canonicalize_text (p1, macro1->exp.text, macro1->count, &quote1);
1308       len2 = canonicalize_text (p2, macro2->exp.text, macro2->count, &quote2);
1309       mismatch = (len1 != len2 || memcmp (p1, p2, len1));
1310     }
1311
1312   free (p1);
1313   return mismatch;
1314 }