gcc/cpptrad.c

   1 /* CPP Library - traditional lexical analysis and macro expansion.
   2    Copyright (C) 2002 Free Software Foundation, Inc.
   3    Contributed by Neil Booth, May 2002
   4
   5 This program is free software; you can redistribute it and/or modify it
   6 under the terms of the GNU General Public License as published by the
   7 Free Software Foundation; either version 2, or (at your option) any
   8 later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program; if not, write to the Free Software
  17 Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
  18
  19 #include "config.h"
  20 #include "system.h"
  21 #include "cpplib.h"
  22 #include "cpphash.h"
  23
  24 /* The replacement text of a function-like macro is stored as a
  25    contiguous sequence of aligned blocks, each representing the text
  26    between subsequent parameters.
  27
  28    Each block comprises the text between its surrounding parameters,
  29    the length of that text, and the one-based index of the following
  30    parameter.  The final block in the replacement text is easily
  31    recognizable as it has an argument index of zero.  */
  32
  33 struct block
  34 {
  35   unsigned int text_len;
  36   unsigned short arg_index;
  37   uchar text[1];
  38 };
  39
  40 #define BLOCK_HEADER_LEN offsetof (struct block, text)
  41 #define BLOCK_LEN(TEXT_LEN) CPP_ALIGN (BLOCK_HEADER_LEN + (TEXT_LEN))
  42
  43 /* Structure holding information about a function-like macro
  44    invocation.  */
  45 struct fun_macro
  46 {
  47   /* Memory buffer holding the trad_arg array.  */
  48   _cpp_buff *buff;
  49
  50   /* An array of size the number of macro parameters + 1, containing
  51      the offsets of the start of each macro argument in the output
  52      buffer.  The argument continues until the character before the
  53      start of the next one.  */
  54   size_t *args;
  55
  56   /* The hashnode of the macro.  */
  57   cpp_hashnode *node;
  58
  59   /* The offset of the macro name in the output buffer.  */
  60   size_t offset;
  61
  62   /* The line the macro name appeared on.  */
  63   unsigned int line;
  64
  65   /* Zero-based index of argument being currently lexed.  */
  66   unsigned int argc;
  67 };
  68
  69 /* Lexing state.  It is mostly used to prevent macro expansion.  */
  70 enum ls {ls_none = 0,           /* Normal state.  */
  71          ls_fun_open,           /* When looking for '('.  */
  72          ls_fun_close,          /* When looking for ')'.  */
  73          ls_defined,            /* After defined.  */
  74          ls_defined_close,      /* Looking for ')' of defined().  */
  75          ls_hash,               /* After # in preprocessor conditional.  */
  76          ls_predicate,          /* After the predicate, maybe paren?  */
  77          ls_answer};            /* In answer to predicate.  */
  78
  79 /* Lexing TODO: Maybe handle space in escaped newlines.  Stop cpplex.c
  80    from recognizing comments and directives during its lexing pass.  */
  81
  82 static const uchar *handle_newline PARAMS ((cpp_reader *, const uchar *));
  83 static const uchar *skip_escaped_newlines PARAMS ((cpp_reader *,
  84                                                    const uchar *));
  85 static const uchar *skip_whitespace PARAMS ((cpp_reader *, const uchar *,
  86                                              int));
  87 static cpp_hashnode *lex_identifier PARAMS ((cpp_reader *, const uchar *));
  88 static const uchar *copy_comment PARAMS ((cpp_reader *, const uchar *, int));
  89 static void scan_out_logical_line PARAMS ((cpp_reader *pfile, cpp_macro *));
  90 static void check_output_buffer PARAMS ((cpp_reader *, size_t));
  91 static void push_replacement_text PARAMS ((cpp_reader *, cpp_hashnode *));
  92 static bool scan_parameters PARAMS ((cpp_reader *, cpp_macro *));
  93 static bool recursive_macro PARAMS ((cpp_reader *, cpp_hashnode *));
  94 static void save_replacement_text PARAMS ((cpp_reader *, cpp_macro *,
  95                                            unsigned int));
  96 static void maybe_start_funlike PARAMS ((cpp_reader *, cpp_hashnode *,
  97                                          const uchar *, struct fun_macro *));
  98 static void save_argument PARAMS ((struct fun_macro *, size_t));
  99 static void replace_args_and_push PARAMS ((cpp_reader *, struct fun_macro *));
 100 static size_t canonicalize_text PARAMS ((uchar *, const uchar *, size_t,
 101                                          uchar *));
 102
 103 /* Ensures we have N bytes' space in the output buffer, and
 104    reallocates it if not.  */
 105 static void
 106 check_output_buffer (pfile, n)
 107      cpp_reader *pfile;
 108      size_t n;
 109 {
 110   /* We might need two bytes to terminate an unterminated comment, and
 111      one more to terminate the line with a NUL.  */
 112   n += 2 + 1;
 113
 114   if (n > (size_t) (pfile->out.limit - pfile->out.cur))
 115     {
 116       size_t size = pfile->out.cur - pfile->out.base;
 117       size_t new_size = (size + n) * 3 / 2;
 118
 119       pfile->out.base
 120         = (uchar *) xrealloc (pfile->out.base, new_size);
 121       pfile->out.limit = pfile->out.base + new_size;
 122       pfile->out.cur = pfile->out.base + size;
 123     }
 124 }
 125
 126 /* To be called whenever a newline character is encountered in the
 127    input file, at CUR.  Handles DOS, Mac and Unix ends of line, and
 128    increments pfile->line.
 129
 130    Returns a pointer the character after the newline sequence.  */
 131 static const uchar *
 132 handle_newline (pfile, cur)
 133      cpp_reader *pfile;
 134      const uchar *cur;
 135 {
 136   pfile->line++;
 137   if (cur[0] + cur[1] == '\r' + '\n')
 138     cur++;
 139   return cur + 1;
 140 }
 141
 142 /* CUR points to any character in the current context, not necessarily
 143    a backslash.  Advances CUR until all escaped newlines are skipped,
 144    and returns the new position without updating the context.
 145
 146    Warns if a file buffer ends in an escaped newline.  */
 147 static const uchar *
 148 skip_escaped_newlines (pfile, cur)
 149      cpp_reader *pfile;
 150      const uchar *cur;
 151 {
 152   const uchar *orig_cur = cur;
 153
 154   while (*cur == '\\' && is_vspace (cur[1]))
 155     cur = handle_newline (pfile, cur + 1);
 156
 157   if (cur != orig_cur && cur == RLIMIT (pfile->context) && pfile->buffer->inc)
 158     cpp_error (pfile, DL_PEDWARN, "backslash-newline at end of file");
 159
 160   return cur;
 161 }
 162
 163 /* CUR points to the asterisk introducing a comment in the current
 164    context.  IN_DEFINE is true if we are in the replacement text of a
 165    macro.
 166
 167    The asterisk and following comment is copied to the buffer pointed
 168    to by pfile->out.cur, which must be of sufficient size.
 169    Unterminated comments are diagnosed, and correctly terminated in
 170    the output.  pfile->out.cur is updated depending upon IN_DEFINE,
 171    -C, -CC and pfile->state.in_directive.
 172
 173    Returns a pointer to the first character after the comment in the
 174    input buffer.  */
 175 static const uchar *
 176 copy_comment (pfile, cur, in_define)
 177      cpp_reader *pfile;
 178      const uchar *cur;
 179      int in_define;
 180 {
 181   unsigned int from_line = pfile->line;
 182   const uchar *limit = RLIMIT (pfile->context);
 183   uchar *out = pfile->out.cur;
 184
 185   do
 186     {
 187       unsigned int c = *cur++;
 188       *out++ = c;
 189
 190       if (c == '/')
 191         {
 192           /* An immediate slash does not terminate the comment.  */
 193           if (out[-2] == '*' && out - 2 > pfile->out.cur)
 194             goto done;
 195
 196           if (*cur == '*' && cur[1] != '/'
 197               && CPP_OPTION (pfile, warn_comments))
 198             cpp_error_with_line (pfile, DL_WARNING, pfile->line, 0,
 199                                  "\"/*\" within comment");
 200         }
 201       else if (is_vspace (c))
 202         {
 203           cur = handle_newline (pfile, cur - 1);
 204           /* Canonicalize newline sequences and skip escaped ones.  */
 205           if (out[-2] == '\\')
 206             out -= 2;
 207           else
 208             out[-1] = '\n';
 209         }
 210     }
 211   while (cur < limit);
 212
 213   cpp_error_with_line (pfile, DL_ERROR, from_line, 0, "unterminated comment");
 214   *out++ = '*';
 215   *out++ = '/';
 216
 217  done:
 218   /* Comments in directives become spaces so that tokens are properly
 219      separated when the ISO preprocessor re-lexes the line.  The
 220      exception is #define.  */
 221   if (pfile->state.in_directive)
 222     {
 223       if (in_define)
 224         {
 225           if (CPP_OPTION (pfile, discard_comments_in_macro_exp))
 226             pfile->out.cur--;
 227           else
 228             pfile->out.cur = out;
 229         }
 230       else
 231         pfile->out.cur[-1] = ' ';
 232     }
 233   else if (CPP_OPTION (pfile, discard_comments))
 234     pfile->out.cur--;
 235   else
 236     pfile->out.cur = out;
 237
 238   return cur;
 239 }
 240
 241 /* CUR points to any character in the input buffer.  Skips over all
 242    contiguous horizontal white space and NULs, including comments if
 243    SKIP_COMMENTS, until reaching the first non-horizontal-whitespace
 244    character or the end of the current context.  Escaped newlines are
 245    removed.
 246
 247    The whitespace is copied verbatim to the output buffer, except that
 248    comments are handled as described in copy_comment().
 249    pfile->out.cur is updated.
 250
 251    Returns a pointer to the first character after the whitespace in
 252    the input buffer.  */
 253 static const uchar *
 254 skip_whitespace (pfile, cur, skip_comments)
 255      cpp_reader *pfile;
 256      const uchar *cur;
 257      int skip_comments;
 258 {
 259   uchar *out = pfile->out.cur;
 260
 261   for (;;)
 262     {
 263       unsigned int c = *cur++;
 264       *out++ = c;
 265
 266       if (is_nvspace (c) && c)
 267         continue;
 268
 269       if (!c && cur - 1 != RLIMIT (pfile->context))
 270         continue;
 271
 272       if (c == '/' && skip_comments)
 273         {
 274           const uchar *tmp = skip_escaped_newlines (pfile, cur);
 275           if (*tmp == '*')
 276             {
 277               pfile->out.cur = out;
 278               cur = copy_comment (pfile, tmp, false /* in_define */);
 279               out = pfile->out.cur;
 280               continue;
 281             }
 282         }
 283
 284       out--;
 285       if (c == '\\' && is_vspace (*cur))
 286         {
 287           cur = skip_escaped_newlines (pfile, cur - 1);
 288           continue;
 289         }
 290
 291       break;
 292     }
 293
 294   pfile->out.cur = out;
 295   return cur - 1;
 296 }
 297
 298 /* Lexes and outputs an identifier starting at CUR, which is assumed
 299    to point to a valid first character of an identifier.  Returns
 300    the hashnode, and updates out.cur.  */
 301 static cpp_hashnode *
 302 lex_identifier (pfile, cur)
 303      cpp_reader *pfile;
 304      const uchar *cur;
 305 {
 306   size_t len;
 307   uchar *out = pfile->out.cur;
 308   cpp_hashnode *result;
 309
 310   do
 311     {
 312       do
 313         *out++ = *cur++;
 314       while (is_numchar (*cur));
 315       cur = skip_escaped_newlines (pfile, cur);
 316     }
 317   while (is_numchar (*cur));
 318
 319   CUR (pfile->context) = cur;
 320   len = out - pfile->out.cur;
 321   result = (cpp_hashnode *) ht_lookup (pfile->hash_table, pfile->out.cur,
 322                                        len, HT_ALLOC);
 323   pfile->out.cur = out;
 324   return result;
 325 }
 326
 327 /* Overlays the true file buffer temporarily with text of length LEN
 328    starting at START.  The true buffer is restored upon calling
 329    restore_buff().  */
 330 void
 331 _cpp_overlay_buffer (pfile, start, len)
 332      cpp_reader *pfile;
 333      const uchar *start;
 334      size_t len;
 335 {
 336   cpp_buffer *buffer = pfile->buffer;
 337
 338   pfile->overlaid_buffer = buffer;
 339   buffer->saved_cur = buffer->cur;
 340   buffer->saved_rlimit = buffer->rlimit;
 341
 342   buffer->cur = start;
 343   buffer->rlimit = start + len;
 344
 345   pfile->saved_line = pfile->line;
 346 }
 347
 348 /* Restores a buffer overlaid by _cpp_overlay_buffer().  */
 349 void
 350 _cpp_remove_overlay (pfile)
 351      cpp_reader *pfile;
 352 {
 353   cpp_buffer *buffer = pfile->overlaid_buffer;
 354
 355   buffer->cur = buffer->saved_cur;
 356   buffer->rlimit = buffer->saved_rlimit;
 357
 358   pfile->line = pfile->saved_line;
 359 }
 360
 361 /* Reads a logical line into the output buffer.  Returns TRUE if there
 362    is more text left in the buffer.  */
 363 bool
 364 _cpp_read_logical_line_trad (pfile)
 365      cpp_reader *pfile;
 366 {
 367   do
 368     {
 369       if (pfile->buffer->cur == pfile->buffer->rlimit)
 370         {
 371           bool stop = true;
 372
 373           /* Don't pop the last buffer.  */
 374           if (pfile->buffer->prev)
 375             {
 376               stop = pfile->buffer->return_at_eof;
 377               _cpp_pop_buffer (pfile);
 378             }
 379
 380           if (stop)
 381             return false;
 382         }
 383
 384       scan_out_logical_line (pfile, NULL);
 385     }
 386   while (pfile->state.skipping);
 387
 388   return true;
 389 }
 390
 391 /* Set up state for finding the opening '(' of a function-like
 392    macro.  */
 393 static void
 394 maybe_start_funlike (pfile, node, start, macro)
 395      cpp_reader *pfile;
 396      cpp_hashnode *node;
 397      const uchar *start;
 398      struct fun_macro *macro;
 399 {
 400   unsigned int n = node->value.macro->paramc + 1;
 401
 402   if (macro->buff)
 403     _cpp_release_buff (pfile, macro->buff);
 404   macro->buff = _cpp_get_buff (pfile, n * sizeof (size_t));
 405   macro->args = (size_t *) BUFF_FRONT (macro->buff);
 406   macro->node = node;
 407   macro->offset = start - pfile->out.base;
 408   macro->argc = 0;
 409 }
 410
 411 /* Save the OFFSET of the start of the next argument to MACRO.  */
 412 static void
 413 save_argument (macro, offset)
 414      struct fun_macro *macro;
 415      size_t offset;
 416 {
 417   macro->argc++;
 418   if (macro->argc <= macro->node->value.macro->paramc)
 419     macro->args[macro->argc] = offset;
 420 }
 421
 422 /* Copies the next logical line in the current buffer (starting at
 423    buffer->cur) to the output buffer.  The output is guaranteed to
 424    terminate with a NUL character.  buffer->cur is updated.
 425
 426    If MACRO is non-NULL, then we are scanning the replacement list of
 427    MACRO, and we call save_replacement_text() every time we meet an
 428    argument.  */
 429 static void
 430 scan_out_logical_line (pfile, macro)
 431      cpp_reader *pfile;
 432      cpp_macro *macro;
 433 {
 434   cpp_context *context;
 435   const uchar *cur;
 436   uchar *out;
 437   struct fun_macro fmacro;
 438   unsigned int c, paren_depth = 0, quote;
 439   enum ls lex_state = ls_none;
 440
 441   fmacro.buff = NULL;
 442
 443  start_logical_line:
 444   quote = 0;
 445   CUR (pfile->context) = pfile->buffer->cur;
 446   RLIMIT (pfile->context) = pfile->buffer->rlimit;
 447   pfile->out.cur = pfile->out.base;
 448   pfile->out.first_line = pfile->line;
 449  new_context:
 450   context = pfile->context;
 451   cur = CUR (context);
 452   check_output_buffer (pfile, RLIMIT (context) - cur);
 453   out = pfile->out.cur;
 454
 455   for (;;)
 456     {
 457       c = *cur++;
 458       *out++ = c;
 459
 460       /* Whitespace should "continue" out of the switch,
 461          non-whitespace should "break" out of it.  */
 462       switch (c)
 463         {
 464         case ' ':
 465         case '\t':
 466         case '\f':
 467         case '\v':
 468           continue;
 469
 470         case '\0':
 471           if (cur - 1 != RLIMIT (context))
 472             continue;
 473
 474           /* If this is a macro's expansion, pop it.  */
 475           if (context->prev)
 476             {
 477               pfile->out.cur = out - 1;
 478               _cpp_pop_context (pfile);
 479               goto new_context;
 480             }
 481
 482           /* Premature end of file.  Fake a new line.  */
 483           cur--;
 484           if (!pfile->buffer->from_stage3)
 485             cpp_error (pfile, DL_PEDWARN, "no newline at end of file");
 486           pfile->line++;
 487           goto done;
 488
 489         case '\r': case '\n':
 490           cur = handle_newline (pfile, cur - 1);
 491           if ((lex_state == ls_fun_open || lex_state == ls_fun_close)
 492               && !pfile->state.in_directive)
 493             {
 494               /* Newlines in arguments become a space, but we don't
 495                  clear any in-progress quote.  */
 496               if (lex_state == ls_fun_close)
 497                 out[-1] = ' ';
 498               continue;
 499             }
 500           goto done;
 501
 502         case '<':
 503           if (pfile->state.angled_headers && !quote)
 504             quote = '>';
 505           break;
 506         case '>':
 507           if (c == quote)
 508             {
 509               pfile->state.angled_headers = false;
 510               quote = 0;
 511             }
 512           break;
 513
 514         case '"':
 515         case '\'':
 516           if (c == quote)
 517             quote = 0;
 518           else if (!quote)
 519             quote = c;
 520           break;
 521
 522         case '\\':
 523           if (is_vspace (*cur))
 524             {
 525               out--;
 526               cur = skip_escaped_newlines (pfile, cur - 1);
 527               continue;
 528             }
 529           else
 530             {
 531               /* Skip escaped quotes here, it's easier than above, but
 532                  take care to first skip escaped newlines.  */
 533               cur = skip_escaped_newlines (pfile, cur);
 534               if (*cur == '\\' || *cur == '"' || *cur == '\'')
 535                 *out++ = *cur++;
 536             }
 537           break;
 538
 539         case '/':
 540           /* Traditional CPP does not recognize comments within
 541              literals.  */
 542           if (!quote)
 543             {
 544               cur = skip_escaped_newlines (pfile, cur);
 545               if (*cur == '*')
 546                 {
 547                   pfile->out.cur = out;
 548                   cur = copy_comment (pfile, cur, macro != 0);
 549                   out = pfile->out.cur;
 550                   continue;
 551                 }
 552             }
 553           break;
 554
 555         case '_':
 556         case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
 557         case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
 558         case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
 559         case 's': case 't': case 'u': case 'v': case 'w': case 'x':
 560         case 'y': case 'z':
 561         case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
 562         case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
 563         case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
 564         case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
 565         case 'Y': case 'Z':
 566           if (!pfile->state.skipping && (quote == 0 || macro))
 567             {
 568               cpp_hashnode *node;
 569               uchar *out_start = out - 1;
 570
 571               pfile->out.cur = out_start;
 572               node = lex_identifier (pfile, cur - 1);
 573               out = pfile->out.cur;
 574               cur = CUR (context);
 575
 576               if (node->type == NT_MACRO
 577                   /* Should we expand for ls_answer?  */
 578                   && (lex_state == ls_none || lex_state == ls_fun_open)
 579                   && !pfile->state.prevent_expansion)
 580                 {
 581                   /* Macros invalidate MI optimization.  */
 582                   pfile->mi_valid = false;
 583                   if (! (node->flags & NODE_BUILTIN)
 584                       && node->value.macro->fun_like)
 585                     {
 586                       maybe_start_funlike (pfile, node, out_start, &fmacro);
 587                       lex_state = ls_fun_open;
 588                       fmacro.line = pfile->line;
 589                       continue;
 590                     }
 591                   else if (!recursive_macro (pfile, node))
 592                     {
 593                       /* Remove the object-like macro's name from the
 594                          output, and push its replacement text.  */
 595                       pfile->out.cur = out_start;
 596                       push_replacement_text (pfile, node);
 597                       lex_state = ls_none;
 598                       goto new_context;
 599                     }
 600                 }
 601               else if (macro && node->arg_index)
 602                 {
 603                   /* Found a parameter in the replacement text of a
 604                      #define.  Remove its name from the output.  */
 605                   pfile->out.cur = out_start;
 606                   save_replacement_text (pfile, macro, node->arg_index);
 607                   out = pfile->out.base;
 608                 }
 609               else if (lex_state == ls_hash)
 610                 {
 611                   lex_state = ls_predicate;
 612                   continue;
 613                 }
 614               else if (pfile->state.in_expression
 615                        && node == pfile->spec_nodes.n_defined)
 616                 {
 617                   lex_state = ls_defined;
 618                   continue;
 619                 }
 620             }
 621           break;
 622
 623         case '(':
 624           if (quote == 0)
 625             {
 626               paren_depth++;
 627               if (lex_state == ls_fun_open)
 628                 {
 629                   if (recursive_macro (pfile, fmacro.node))
 630                     lex_state = ls_none;
 631                   else
 632                     {
 633                       lex_state = ls_fun_close;
 634                       paren_depth = 1;
 635                       out = pfile->out.base + fmacro.offset;
 636                       fmacro.args[0] = fmacro.offset;
 637                     }
 638                 }
 639               else if (lex_state == ls_predicate)
 640                 lex_state = ls_answer;
 641               else if (lex_state == ls_defined)
 642                 lex_state = ls_defined_close;
 643             }
 644           break;
 645
 646         case ',':
 647           if (quote == 0 && lex_state == ls_fun_close && paren_depth == 1)
 648             save_argument (&fmacro, out - pfile->out.base);
 649           break;
 650
 651         case ')':
 652           if (quote == 0)
 653             {
 654               paren_depth--;
 655               if (lex_state == ls_fun_close && paren_depth == 0)
 656                 {
 657                   cpp_macro *m = fmacro.node->value.macro;
 658
 659                   lex_state = ls_none;
 660                   save_argument (&fmacro, out - pfile->out.base);
 661
 662                   /* A single zero-length argument is no argument.  */
 663                   if (fmacro.argc == 1
 664                       && m->paramc == 0
 665                       && out == pfile->out.base + fmacro.offset + 1)
 666                     fmacro.argc = 0;
 667
 668                   if (_cpp_arguments_ok (pfile, m, fmacro.node, fmacro.argc))
 669                     {
 670                       /* Remove the macro's invocation from the
 671                          output, and push its replacement text.  */
 672                       pfile->out.cur = (pfile->out.base
 673                                              + fmacro.offset);
 674                       CUR (context) = cur;
 675                       replace_args_and_push (pfile, &fmacro);
 676                       goto new_context;
 677                     }
 678                 }
 679               else if (lex_state == ls_answer || lex_state == ls_defined_close)
 680                 lex_state = ls_none;
 681             }
 682           break;
 683
 684         case '#':
 685           if (out - 1 == pfile->out.base && !pfile->state.in_directive)
 686             {
 687               /* A directive.  With the way _cpp_handle_directive
 688                  currently works, we only want to call it if either we
 689                  know the directive is OK, or we want it to fail and
 690                  be removed from the output.  If we want it to be
 691                  passed through (the assembler case) then we must not
 692                  call _cpp_handle_directive.  */
 693               pfile->out.cur = out;
 694               cur = skip_whitespace (pfile, cur, true /* skip_comments */);
 695               out = pfile->out.cur;
 696
 697               if (is_vspace (*cur))
 698                 {
 699                   /* Null directive.  Ignore it and don't invalidate
 700                      the MI optimization.  */
 701                   out = pfile->out.base;
 702                   continue;
 703                 }
 704               else
 705                 {
 706                   bool do_it = false;
 707
 708                   if (is_numstart (*cur))
 709                     do_it = true;
 710                   else if (is_idstart (*cur))
 711                     /* Check whether we know this directive, but don't
 712                        advance.  */
 713                     do_it = lex_identifier (pfile, cur)->directive_index != 0;
 714
 715                   if (do_it || CPP_OPTION (pfile, lang) != CLK_ASM)
 716                     {
 717                       /* This is a kludge.  We want to have the ISO
 718                          preprocessor lex the next token.  */
 719                       pfile->buffer->cur = cur;
 720                       _cpp_handle_directive (pfile, false /* indented */);
 721                       /* #include changes pfile->buffer so we need to
 722                          update the limits of the current context.  */
 723                       goto start_logical_line;
 724                     }
 725                 }
 726             }
 727
 728           if (pfile->state.in_expression)
 729             {
 730               lex_state = ls_hash;
 731               continue;
 732             }
 733           break;
 734
 735         default:
 736           break;
 737         }
 738
 739       /* Non-whitespace disables MI optimization.  */
 740       if (!pfile->state.in_directive)
 741         pfile->mi_valid = false;
 742
 743       if (lex_state == ls_none)
 744         continue;
 745
 746       /* Some of these transitions of state are syntax errors.  The
 747          ISO preprocessor will issue errors later.  */
 748       if (lex_state == ls_fun_open)
 749         /* Missing '('.  */
 750         lex_state = ls_none;
 751       else if (lex_state == ls_hash
 752                || lex_state == ls_predicate
 753                || lex_state == ls_defined)
 754         lex_state = ls_none;
 755
 756       /* ls_answer and ls_defined_close keep going until ')'.  */
 757     }
 758
 759  done:
 760   out[-1] = '\0';
 761   pfile->buffer->cur = cur;
 762   pfile->out.cur = out - 1;
 763   if (fmacro.buff)
 764     _cpp_release_buff (pfile, fmacro.buff);
 765
 766   if (lex_state == ls_fun_close)
 767     cpp_error_with_line (pfile, DL_ERROR, fmacro.line, 0,
 768                          "unterminated argument list invoking macro \"%s\"",
 769                          NODE_NAME (fmacro.node));
 770 }
 771
 772 /* Push a context holding the replacement text of the macro NODE on
 773    the context stack.  NODE is either object-like, or a function-like
 774    macro with no arguments.  */
 775 static void
 776 push_replacement_text (pfile, node)
 777      cpp_reader *pfile;
 778      cpp_hashnode *node;
 779 {
 780   size_t len;
 781   const uchar *text;
 782
 783   if (node->flags & NODE_BUILTIN)
 784     {
 785       text = _cpp_builtin_macro_text (pfile, node);
 786       len = ustrlen (text);
 787     }
 788   else
 789     {
 790       cpp_macro *macro = node->value.macro;
 791       text = macro->exp.text;
 792       len = macro->count;
 793     }
 794
 795   _cpp_push_text_context (pfile, node, text, len);
 796 }
 797
 798 /* Returns TRUE if traditional macro recursion is detected.  */
 799 static bool
 800 recursive_macro (pfile, node)
 801      cpp_reader *pfile;
 802      cpp_hashnode *node;
 803 {
 804   bool recursing = node->flags & NODE_DISABLED;
 805
 806   /* Object-like macros that are already expanding are necessarily
 807      recursive.
 808
 809      However, it is possible to have traditional function-like macros
 810      that are not infinitely recursive but recurse to any given depth.
 811      Further, it is easy to construct examples that get ever longer
 812      until the point they stop recursing.  So there is no easy way to
 813      detect true recursion; instead we assume any expansion more than
 814      20 deep since the first invocation of this macro must be
 815      recursing.  */
 816   if (recursing && node->value.macro->fun_like)
 817     {
 818       size_t depth = 0;
 819       cpp_context *context = pfile->context;
 820
 821       do
 822         {
 823           depth++;
 824           if (context->macro == node && depth > 20)
 825             break;
 826           context = context->prev;
 827         }
 828       while (context);
 829       recursing = context != NULL;
 830     }
 831
 832   if (recursing)
 833     cpp_error (pfile, DL_ERROR,
 834                "detected recursion whilst expanding macro \"%s\"",
 835                NODE_NAME (node));
 836
 837   return recursing;
 838 }
 839
 840 /* Return the length of the replacement text of a function-like or
 841    object-like non-builtin macro.  */
 842 size_t
 843 _cpp_replacement_text_len (macro)
 844      const cpp_macro *macro;
 845 {
 846   size_t len;
 847
 848   if (macro->fun_like)
 849     {
 850       const uchar *exp;
 851
 852       len = 0;
 853       for (exp = macro->exp.text;;)
 854         {
 855           struct block *b = (struct block *) exp;
 856
 857           len += b->text_len;
 858           if (b->arg_index == 0)
 859             break;
 860           len += NODE_LEN (macro->params[b->arg_index - 1]);
 861           exp += BLOCK_LEN (b->text_len);
 862         }
 863     }
 864   else
 865     len = macro->count;
 866
 867   return len;
 868 }
 869
 870 /* Copy the replacement text of MACRO to DEST, which must be of
 871    sufficient size.  It is not NUL-terminated.  The next character is
 872    returned.  */
 873 uchar *
 874 _cpp_copy_replacement_text (macro, dest)
 875      const cpp_macro *macro;
 876      uchar *dest;
 877 {
 878   if (macro->fun_like)
 879     {
 880       const uchar *exp;
 881
 882       for (exp = macro->exp.text;;)
 883         {
 884           struct block *b = (struct block *) exp;
 885           cpp_hashnode *param;
 886
 887           memcpy (dest, b->text, b->text_len);
 888           dest += b->text_len;
 889           if (b->arg_index == 0)
 890             break;
 891           param = macro->params[b->arg_index - 1];
 892           memcpy (dest, NODE_NAME (param), NODE_LEN (param));
 893           dest += NODE_LEN (param);
 894           exp += BLOCK_LEN (b->text_len);
 895         }
 896     }
 897   else
 898     {
 899       memcpy (dest, macro->exp.text, macro->count);
 900       dest += macro->count;
 901     }
 902
 903   return dest;
 904 }
 905
 906 /* Push a context holding the replacement text of the macro NODE on
 907    the context stack.  NODE is either object-like, or a function-like
 908    macro with no arguments.  */
 909 static void
 910 replace_args_and_push (pfile, fmacro)
 911      cpp_reader *pfile;
 912      struct fun_macro *fmacro;
 913 {
 914   cpp_macro *macro = fmacro->node->value.macro;
 915
 916   if (macro->paramc == 0)
 917     push_replacement_text (pfile, fmacro->node);
 918   else
 919     {
 920       const uchar *exp;
 921       uchar *p;
 922       _cpp_buff *buff;
 923       size_t len = 0;
 924
 925       /* Calculate the length of the argument-replaced text.  */
 926       for (exp = macro->exp.text;;)
 927         {
 928           struct block *b = (struct block *) exp;
 929
 930           len += b->text_len;
 931           if (b->arg_index == 0)
 932             break;
 933           len += (fmacro->args[b->arg_index]
 934                   - fmacro->args[b->arg_index - 1] - 1);
 935           exp += BLOCK_LEN (b->text_len);
 936         }
 937
 938       /* Allocate room for the expansion plus NUL.  */
 939       buff = _cpp_get_buff (pfile, len + 1);
 940
 941       /* Copy the expansion and replace arguments.  */
 942       p = BUFF_FRONT (buff);
 943       for (exp = macro->exp.text;;)
 944         {
 945           struct block *b = (struct block *) exp;
 946           size_t arglen;
 947
 948           memcpy (p, b->text, b->text_len);
 949           p += b->text_len;
 950           if (b->arg_index == 0)
 951             break;
 952           arglen = (fmacro->args[b->arg_index]
 953                     - fmacro->args[b->arg_index - 1] - 1);
 954           memcpy (p, pfile->out.base + fmacro->args[b->arg_index - 1],
 955                   arglen);
 956           p += arglen;
 957           exp += BLOCK_LEN (b->text_len);
 958         }
 959
 960       /* NUL-terminate.  */
 961       *p = '\0';
 962       _cpp_push_text_context (pfile, fmacro->node, BUFF_FRONT (buff), len);
 963
 964       /* So we free buffer allocation when macro is left.  */
 965       pfile->context->buff = buff;
 966     }
 967 }
 968
 969 /* Read and record the parameters, if any, of a function-like macro
 970    definition.  Destroys pfile->out.cur.
 971
 972    Returns true on success, false on failure (syntax error or a
 973    duplicate parameter).  On success, CUR (pfile->context) is just
 974    past the closing parenthesis.  */
 975 static bool
 976 scan_parameters (pfile, macro)
 977      cpp_reader *pfile;
 978      cpp_macro *macro;
 979 {
 980   const uchar *cur = CUR (pfile->context) + 1;
 981   bool ok;
 982
 983   for (;;)
 984     {
 985       cur = skip_whitespace (pfile, cur, true /* skip_comments */);
 986
 987       if (is_idstart (*cur))
 988         {
 989           ok = false;
 990           if (_cpp_save_parameter (pfile, macro, lex_identifier (pfile, cur)))
 991             break;
 992           cur = skip_whitespace (pfile, CUR (pfile->context),
 993                                  true /* skip_comments */);
 994           if (*cur == ',')
 995             {
 996               cur++;
 997               continue;
 998             }
 999           ok = (*cur == ')');
1000           break;
1001         }
1002
1003       ok = (*cur == ')' && macro->paramc == 0);
1004       break;
1005     }
1006
1007   CUR (pfile->context) = cur + (*cur == ')');
1008
1009   return ok;
1010 }
1011
1012 /* Save the text from pfile->out.base to pfile->out.cur as
1013    the replacement text for the current macro, followed by argument
1014    ARG_INDEX, with zero indicating the end of the replacement
1015    text.  */
1016 static void
1017 save_replacement_text (pfile, macro, arg_index)
1018      cpp_reader *pfile;
1019      cpp_macro *macro;
1020      unsigned int arg_index;
1021 {
1022   size_t len = pfile->out.cur - pfile->out.base;
1023   uchar *exp;
1024
1025   if (macro->paramc == 0)
1026     {
1027       /* Object-like and function-like macros without parameters
1028          simply store their NUL-terminated replacement text.  */
1029       exp = _cpp_unaligned_alloc (pfile, len + 1);
1030       memcpy (exp, pfile->out.base, len);
1031       exp[len] = '\0';
1032       macro->exp.text = exp;
1033       macro->count = len;
1034     }
1035   else
1036     {
1037       /* Store the text's length (unsigned int), the argument index
1038          (unsigned short, base 1) and then the text.  */
1039       size_t blen = BLOCK_LEN (len);
1040       struct block *block;
1041
1042       if (macro->count + blen > BUFF_ROOM (pfile->a_buff))
1043         _cpp_extend_buff (pfile, &pfile->a_buff, macro->count + blen);
1044
1045       exp = BUFF_FRONT (pfile->a_buff);
1046       block = (struct block *) (exp + macro->count);
1047       macro->exp.text = exp;
1048
1049       /* Write out the block information.  */
1050       block->text_len = len;
1051       block->arg_index = arg_index;
1052       memcpy (block->text, pfile->out.base, len);
1053
1054       /* Lex the rest into the start of the output buffer.  */
1055       pfile->out.cur = pfile->out.base;
1056
1057       macro->count += blen;
1058
1059       /* If we've finished, commit the memory.  */
1060       if (arg_index == 0)
1061         BUFF_FRONT (pfile->a_buff) += macro->count;
1062     }
1063 }
1064
1065 /* Analyze and save the replacement text of a macro.  Returns true on
1066    success.  */
1067 bool
1068 _cpp_create_trad_definition (pfile, macro)
1069      cpp_reader *pfile;
1070      cpp_macro *macro;
1071 {
1072   const uchar *cur;
1073   uchar *limit;
1074   cpp_context *context = pfile->context;
1075
1076   /* The context has not been set up for command line defines, and CUR
1077      has not been updated for the macro name for in-file defines.  */
1078   pfile->out.cur = pfile->out.base;
1079   CUR (context) = pfile->buffer->cur;
1080   RLIMIT (context) = pfile->buffer->rlimit;
1081   check_output_buffer (pfile, RLIMIT (context) - CUR (context));
1082
1083   /* Is this a function-like macro?  */
1084   if (* CUR (context) == '(')
1085     {
1086       /* Setting macro to NULL indicates an error occurred, and
1087          prevents unnecessary work in scan_out_logical_line.  */
1088       if (!scan_parameters (pfile, macro))
1089         macro = NULL;
1090       else
1091         {
1092           /* Success.  Commit the parameter array.  */
1093           macro->params = (cpp_hashnode **) BUFF_FRONT (pfile->a_buff);
1094           BUFF_FRONT (pfile->a_buff) = (uchar *) &macro->params[macro->paramc];
1095           macro->fun_like = 1;
1096         }
1097     }
1098
1099   /* Skip leading whitespace in the replacement text.  */
1100   pfile->buffer->cur
1101     = skip_whitespace (pfile, CUR (context),
1102                        CPP_OPTION (pfile, discard_comments_in_macro_exp));
1103
1104   pfile->state.prevent_expansion++;
1105   scan_out_logical_line (pfile, macro);
1106   pfile->state.prevent_expansion--;
1107
1108   if (!macro)
1109     return false;
1110
1111   /* Skip trailing white space.  */
1112   cur = pfile->out.base;
1113   limit = pfile->out.cur;
1114   while (limit > cur && is_space (limit[-1]))
1115     limit--;
1116   pfile->out.cur = limit;
1117   save_replacement_text (pfile, macro, 0);
1118
1119   return true;
1120 }
1121
1122 /* Copy SRC of length LEN to DEST, but convert all contiguous
1123    whitespace to a single space, provided it is not in quotes.  The
1124    quote currently in effect is pointed to by PQUOTE, and is updated
1125    by the function.  Returns the number of bytes copied.  */
1126 static size_t
1127 canonicalize_text (dest, src, len, pquote)
1128      uchar *dest;
1129      const uchar *src;
1130      size_t len;
1131      uchar *pquote;
1132 {
1133   uchar *orig_dest = dest;
1134   uchar quote = *pquote;
1135
1136   while (len)
1137     {
1138       if (is_space (*src) && !quote)
1139         {
1140           do
1141             src++, len--;
1142           while (len && is_space (*src));
1143           *dest++ = ' ';
1144         }
1145       else
1146         {
1147           if (*src == '\'' || *src == '"')
1148             {
1149               if (!quote)
1150                 quote = *src;
1151               else if (quote == *src)
1152                 quote = 0;
1153             }
1154           *dest++ = *src++, len--;
1155         }
1156     }
1157
1158   *pquote = quote;
1159   return dest - orig_dest;
1160 }
1161
1162 /* Returns true if MACRO1 and MACRO2 have expansions different other
1163    than in the form of their whitespace.  */
1164 bool
1165 _cpp_expansions_different_trad (macro1, macro2)
1166      const cpp_macro *macro1, *macro2;
1167 {
1168   uchar *p1 = xmalloc (macro1->count + macro2->count);
1169   uchar *p2 = p1 + macro1->count;
1170   uchar quote1 = 0, quote2 = 0;
1171   bool mismatch;
1172   size_t len1, len2;
1173
1174   if (macro1->paramc > 0)
1175     {
1176       const uchar *exp1 = macro1->exp.text, *exp2 = macro2->exp.text;
1177
1178       mismatch = true;
1179       for (;;)
1180         {
1181           struct block *b1 = (struct block *) exp1;
1182           struct block *b2 = (struct block *) exp2;
1183
1184           if (b1->arg_index != b2->arg_index)
1185             break;
1186
1187           len1 = canonicalize_text (p1, b1->text, b1->text_len, &quote1);
1188           len2 = canonicalize_text (p2, b2->text, b2->text_len, &quote2);
1189           if (len1 != len2 || memcmp (p1, p2, len1))
1190             break;
1191           if (b1->arg_index == 0)
1192             {
1193               mismatch = false;
1194               break;
1195             }
1196           exp1 += BLOCK_LEN (b1->text_len);
1197           exp2 += BLOCK_LEN (b2->text_len);
1198         }
1199     }
1200   else
1201     {
1202       len1 = canonicalize_text (p1, macro1->exp.text, macro1->count, &quote1);
1203       len2 = canonicalize_text (p2, macro2->exp.text, macro2->count, &quote2);
1204       mismatch = (len1 != len2 || memcmp (p1, p2, len1));
1205     }
1206
1207   free (p1);
1208   return mismatch;
1209 }