libcpp/macro.c

   1 /* Part of CPP library.  (Macro and #define handling.)
   2    Copyright (C) 1986-2018 Free Software Foundation, Inc.
   3    Written by Per Bothner, 1994.
   4    Based on CCCP program by Paul Rubin, June 1986
   5    Adapted to ANSI C, Richard Stallman, Jan 1987
   6
   7 This program is free software; you can redistribute it and/or modify it
   8 under the terms of the GNU General Public License as published by the
   9 Free Software Foundation; either version 3, or (at your option) any
  10 later version.
  11
  12 This program is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 GNU General Public License for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; see the file COPYING3.  If not see
  19 <http://www.gnu.org/licenses/>.
  20
  21  In other words, you are welcome to use, share and improve this program.
  22  You are forbidden to forbid anyone else to use, share and improve
  23  what you give them.   Help stamp out software-hoarding!  */
  24
  25 #include "config.h"
  26 #include "system.h"
  27 #include "cpplib.h"
  28 #include "internal.h"
  29
  30 typedef struct macro_arg macro_arg;
  31 /* This structure represents the tokens of a macro argument.  These
  32    tokens can be macro themselves, in which case they can be either
  33    expanded or unexpanded.  When they are expanded, this data
  34    structure keeps both the expanded and unexpanded forms.  */
  35 struct macro_arg
  36 {
  37   const cpp_token **first;      /* First token in unexpanded argument.  */
  38   const cpp_token **expanded;   /* Macro-expanded argument.  */
  39   const cpp_token *stringified; /* Stringified argument.  */
  40   unsigned int count;           /* # of tokens in argument.  */
  41   unsigned int expanded_count;  /* # of tokens in expanded argument.  */
  42   source_location *virt_locs;   /* Where virtual locations for
  43                                    unexpanded tokens are stored.  */
  44   source_location *expanded_virt_locs; /* Where virtual locations for
  45                                           expanded tokens are
  46                                           stored.  */
  47 };
  48
  49 /* The kind of macro tokens which the instance of
  50    macro_arg_token_iter is supposed to iterate over.  */
  51 enum macro_arg_token_kind {
  52   MACRO_ARG_TOKEN_NORMAL,
  53   /* This is a macro argument token that got transformed into a string
  54      literal, e.g. #foo.  */
  55   MACRO_ARG_TOKEN_STRINGIFIED,
  56   /* This is a token resulting from the expansion of a macro
  57      argument that was itself a macro.  */
  58   MACRO_ARG_TOKEN_EXPANDED
  59 };
  60
  61 /* An iterator over tokens coming from a function-like macro
  62    argument.  */
  63 typedef struct macro_arg_token_iter macro_arg_token_iter;
  64 struct macro_arg_token_iter
  65 {
  66   /* Whether or not -ftrack-macro-expansion is used.  */
  67   bool track_macro_exp_p;
  68   /* The kind of token over which we are supposed to iterate.  */
  69   enum macro_arg_token_kind kind;
  70   /* A pointer to the current token pointed to by the iterator.  */
  71   const cpp_token **token_ptr;
  72   /* A pointer to the "full" location of the current token.  If
  73      -ftrack-macro-expansion is used this location tracks loci across
  74      macro expansion.  */
  75   const source_location *location_ptr;
  76 #if CHECKING_P
  77   /* The number of times the iterator went forward. This useful only
  78      when checking is enabled.  */
  79   size_t num_forwards;
  80 #endif
  81 };
  82
  83 /* Saved data about an identifier being used as a macro argument
  84    name.  */
  85 struct macro_arg_saved_data {
  86   /* The canonical (UTF-8) spelling of this identifier.  */
  87   cpp_hashnode *canonical_node;
  88   /* The previous value of this identifier.  */
  89   union _cpp_hashnode_value value;
  90 };
  91
  92 static const char *vaopt_paste_error =
  93   N_("'##' cannot appear at either end of __VA_OPT__");
  94
  95 /* A class for tracking __VA_OPT__ state while iterating over a
  96    sequence of tokens.  This is used during both macro definition and
  97    expansion.  */
  98 class vaopt_state {
  99
 100  public:
 101
 102   /* Initialize the state tracker.  ANY_ARGS is true if variable
 103      arguments were provided to the macro invocation.  */
 104   vaopt_state (cpp_reader *pfile, bool is_variadic, bool any_args)
 105     : m_pfile (pfile),
 106     m_allowed (any_args),
 107     m_variadic (is_variadic),
 108     m_last_was_paste (false),
 109     m_state (0),
 110     m_paste_location (0),
 111     m_location (0)
 112   {
 113   }
 114
 115   enum update_type
 116   {
 117     ERROR,
 118     DROP,
 119     INCLUDE,
 120     BEGIN,
 121     END
 122   };
 123
 124   /* Given a token, update the state of this tracker and return a
 125      boolean indicating whether the token should be be included in the
 126      expansion.  */
 127   update_type update (const cpp_token *token)
 128   {
 129     /* If the macro isn't variadic, just don't bother.  */
 130     if (!m_variadic)
 131       return INCLUDE;
 132
 133     if (token->type == CPP_NAME
 134         && token->val.node.node == m_pfile->spec_nodes.n__VA_OPT__)
 135       {
 136         if (m_state > 0)
 137           {
 138             cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
 139                           "__VA_OPT__ may not appear in a __VA_OPT__");
 140             return ERROR;
 141           }
 142         ++m_state;
 143         m_location = token->src_loc;
 144         return BEGIN;
 145       }
 146     else if (m_state == 1)
 147       {
 148         if (token->type != CPP_OPEN_PAREN)
 149           {
 150             cpp_error_at (m_pfile, CPP_DL_ERROR, m_location,
 151                           "__VA_OPT__ must be followed by an "
 152                           "open parenthesis");
 153             return ERROR;
 154           }
 155         ++m_state;
 156         return DROP;
 157       }
 158     else if (m_state >= 2)
 159       {
 160         if (m_state == 2 && token->type == CPP_PASTE)
 161           {
 162             cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
 163                           vaopt_paste_error);
 164             return ERROR;
 165           }
 166         /* Advance states before further considering this token, in
 167            case we see a close paren immediately after the open
 168            paren.  */
 169         if (m_state == 2)
 170           ++m_state;
 171
 172         bool was_paste = m_last_was_paste;
 173         m_last_was_paste = false;
 174         if (token->type == CPP_PASTE)
 175           {
 176             m_last_was_paste = true;
 177             m_paste_location = token->src_loc;
 178           }
 179         else if (token->type == CPP_OPEN_PAREN)
 180           ++m_state;
 181         else if (token->type == CPP_CLOSE_PAREN)
 182           {
 183             --m_state;
 184             if (m_state == 2)
 185               {
 186                 /* Saw the final paren.  */
 187                 m_state = 0;
 188
 189                 if (was_paste)
 190                   {
 191                     cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
 192                                   vaopt_paste_error);
 193                     return ERROR;
 194                   }
 195
 196                 return END;
 197               }
 198           }
 199         return m_allowed ? INCLUDE : DROP;
 200       }
 201
 202     /* Nothing to do with __VA_OPT__.  */
 203     return INCLUDE;
 204   }
 205
 206   /* Ensure that any __VA_OPT__ was completed.  If ok, return true.
 207      Otherwise, issue an error and return false.  */
 208   bool completed ()
 209   {
 210     if (m_variadic && m_state != 0)
 211       cpp_error_at (m_pfile, CPP_DL_ERROR, m_location,
 212                     "unterminated __VA_OPT__");
 213     return m_state == 0;
 214   }
 215
 216  private:
 217
 218   /* The cpp_reader.  */
 219   cpp_reader *m_pfile;
 220
 221   /* True if there were varargs.  */
 222   bool m_allowed;
 223   /* True if the macro is variadic.  */
 224   bool m_variadic;
 225   /* If true, the previous token was ##.  This is used to detect when
 226      a paste occurs at the end of the sequence.  */
 227   bool m_last_was_paste;
 228
 229   /* The state variable:
 230      0 means not parsing
 231      1 means __VA_OPT__ seen, looking for "("
 232      2 means "(" seen (so the next token can't be "##")
 233      >= 3 means looking for ")", the number encodes the paren depth.  */
 234   int m_state;
 235
 236   /* The location of the paste token.  */
 237   source_location m_paste_location;
 238
 239   /* Location of the __VA_OPT__ token.  */
 240   source_location m_location;
 241 };
 242
 243 /* Macro expansion.  */
 244
 245 static int enter_macro_context (cpp_reader *, cpp_hashnode *,
 246                                 const cpp_token *, source_location);
 247 static int builtin_macro (cpp_reader *, cpp_hashnode *,
 248                           source_location, source_location);
 249 static void push_ptoken_context (cpp_reader *, cpp_hashnode *, _cpp_buff *,
 250                                  const cpp_token **, unsigned int);
 251 static void push_extended_tokens_context (cpp_reader *, cpp_hashnode *,
 252                                           _cpp_buff *, source_location *,
 253                                           const cpp_token **, unsigned int);
 254 static _cpp_buff *collect_args (cpp_reader *, const cpp_hashnode *,
 255                                 _cpp_buff **, unsigned *);
 256 static cpp_context *next_context (cpp_reader *);
 257 static const cpp_token *padding_token (cpp_reader *, const cpp_token *);
 258 static void expand_arg (cpp_reader *, macro_arg *);
 259 static const cpp_token *new_string_token (cpp_reader *, uchar *, unsigned int);
 260 static const cpp_token *stringify_arg (cpp_reader *, macro_arg *);
 261 static void paste_all_tokens (cpp_reader *, const cpp_token *);
 262 static bool paste_tokens (cpp_reader *, source_location,
 263                           const cpp_token **, const cpp_token *);
 264 static void alloc_expanded_arg_mem (cpp_reader *, macro_arg *, size_t);
 265 static void ensure_expanded_arg_room (cpp_reader *, macro_arg *, size_t, size_t *);
 266 static void delete_macro_args (_cpp_buff*, unsigned num_args);
 267 static void set_arg_token (macro_arg *, const cpp_token *,
 268                            source_location, size_t,
 269                            enum macro_arg_token_kind,
 270                            bool);
 271 static const source_location *get_arg_token_location (const macro_arg *,
 272                                                       enum macro_arg_token_kind);
 273 static const cpp_token **arg_token_ptr_at (const macro_arg *,
 274                                            size_t,
 275                                            enum macro_arg_token_kind,
 276                                            source_location **virt_location);
 277
 278 static void macro_arg_token_iter_init (macro_arg_token_iter *, bool,
 279                                        enum macro_arg_token_kind,
 280                                        const macro_arg *,
 281                                        const cpp_token **);
 282 static const cpp_token *macro_arg_token_iter_get_token
 283 (const macro_arg_token_iter *it);
 284 static source_location macro_arg_token_iter_get_location
 285 (const macro_arg_token_iter *);
 286 static void macro_arg_token_iter_forward (macro_arg_token_iter *);
 287 static _cpp_buff *tokens_buff_new (cpp_reader *, size_t,
 288                                    source_location **);
 289 static size_t tokens_buff_count (_cpp_buff *);
 290 static const cpp_token **tokens_buff_last_token_ptr (_cpp_buff *);
 291 static inline const cpp_token **tokens_buff_put_token_to (const cpp_token **,
 292                                                           source_location *,
 293                                                           const cpp_token *,
 294                                                           source_location,
 295                                                           source_location,
 296                                                           const line_map_macro *,
 297                                                           unsigned int);
 298
 299 static const cpp_token **tokens_buff_add_token (_cpp_buff *,
 300                                                 source_location *,
 301                                                 const cpp_token *,
 302                                                 source_location,
 303                                                 source_location,
 304                                                 const line_map_macro *,
 305                                                 unsigned int);
 306 static inline void tokens_buff_remove_last_token (_cpp_buff *);
 307 static void replace_args (cpp_reader *, cpp_hashnode *, cpp_macro *,
 308                           macro_arg *, source_location);
 309 static _cpp_buff *funlike_invocation_p (cpp_reader *, cpp_hashnode *,
 310                                         _cpp_buff **, unsigned *);
 311 static bool create_iso_definition (cpp_reader *, cpp_macro *);
 312
 313 /* #define directive parsing and handling.  */
 314
 315 static cpp_token *alloc_expansion_token (cpp_reader *, cpp_macro *);
 316 static cpp_token *lex_expansion_token (cpp_reader *, cpp_macro *);
 317 static bool warn_of_redefinition (cpp_reader *, cpp_hashnode *,
 318                                   const cpp_macro *);
 319 static bool parse_params (cpp_reader *, cpp_macro *);
 320 static void check_trad_stringification (cpp_reader *, const cpp_macro *,
 321                                         const cpp_string *);
 322 static bool reached_end_of_context (cpp_context *);
 323 static void consume_next_token_from_context (cpp_reader *pfile,
 324                                              const cpp_token **,
 325                                              source_location *);
 326 static const cpp_token* cpp_get_token_1 (cpp_reader *, source_location *);
 327
 328 static cpp_hashnode* macro_of_context (cpp_context *context);
 329
 330 static bool in_macro_expansion_p (cpp_reader *pfile);
 331
 332 /* Statistical counter tracking the number of macros that got
 333    expanded.  */
 334 unsigned num_expanded_macros_counter = 0;
 335 /* Statistical counter tracking the total number tokens resulting
 336    from macro expansion.  */
 337 unsigned num_macro_tokens_counter = 0;
 338
 339 /* Emits a warning if NODE is a macro defined in the main file that
 340    has not been used.  */
 341 int
 342 _cpp_warn_if_unused_macro (cpp_reader *pfile, cpp_hashnode *node,
 343                            void *v ATTRIBUTE_UNUSED)
 344 {
 345   if (node->type == NT_MACRO && !(node->flags & NODE_BUILTIN))
 346     {
 347       cpp_macro *macro = node->value.macro;
 348
 349       if (!macro->used
 350           && MAIN_FILE_P (linemap_check_ordinary
 351                             (linemap_lookup (pfile->line_table,
 352                                              macro->line))))
 353         cpp_warning_with_line (pfile, CPP_W_UNUSED_MACROS, macro->line, 0,
 354                                "macro \"%s\" is not used", NODE_NAME (node));
 355     }
 356
 357   return 1;
 358 }
 359
 360 /* Allocates and returns a CPP_STRING token, containing TEXT of length
 361    LEN, after null-terminating it.  TEXT must be in permanent storage.  */
 362 static const cpp_token *
 363 new_string_token (cpp_reader *pfile, unsigned char *text, unsigned int len)
 364 {
 365   cpp_token *token = _cpp_temp_token (pfile);
 366
 367   text[len] = '\0';
 368   token->type = CPP_STRING;
 369   token->val.str.len = len;
 370   token->val.str.text = text;
 371   token->flags = 0;
 372   return token;
 373 }
 374
 375 static const char * const monthnames[] =
 376 {
 377   "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 378   "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 379 };
 380
 381 /* Helper function for builtin_macro.  Returns the text generated by
 382    a builtin macro. */
 383 const uchar *
 384 _cpp_builtin_macro_text (cpp_reader *pfile, cpp_hashnode *node,
 385                          source_location loc)
 386 {
 387   const uchar *result = NULL;
 388   linenum_type number = 1;
 389
 390   switch (node->value.builtin)
 391     {
 392     default:
 393       cpp_error (pfile, CPP_DL_ICE, "invalid built-in macro \"%s\"",
 394                  NODE_NAME (node));
 395       break;
 396
 397     case BT_TIMESTAMP:
 398       {
 399         if (CPP_OPTION (pfile, warn_date_time))
 400           cpp_warning (pfile, CPP_W_DATE_TIME, "macro \"%s\" might prevent "
 401                        "reproducible builds", NODE_NAME (node));
 402
 403         cpp_buffer *pbuffer = cpp_get_buffer (pfile);
 404         if (pbuffer->timestamp == NULL)
 405           {
 406             /* Initialize timestamp value of the assotiated file. */
 407             struct _cpp_file *file = cpp_get_file (pbuffer);
 408             if (file)
 409               {
 410                 /* Generate __TIMESTAMP__ string, that represents
 411                    the date and time of the last modification
 412                    of the current source file. The string constant
 413                    looks like "Sun Sep 16 01:03:52 1973".  */
 414                 struct tm *tb = NULL;
 415                 struct stat *st = _cpp_get_file_stat (file);
 416                 if (st)
 417                   tb = localtime (&st->st_mtime);
 418                 if (tb)
 419                   {
 420                     char *str = asctime (tb);
 421                     size_t len = strlen (str);
 422                     unsigned char *buf = _cpp_unaligned_alloc (pfile, len + 2);
 423                     buf[0] = '"';
 424                     strcpy ((char *) buf + 1, str);
 425                     buf[len] = '"';
 426                     pbuffer->timestamp = buf;
 427                   }
 428                 else
 429                   {
 430                     cpp_errno (pfile, CPP_DL_WARNING,
 431                         "could not determine file timestamp");
 432                     pbuffer->timestamp = UC"\"??? ??? ?? ??:??:?? ????\"";
 433                   }
 434               }
 435           }
 436         result = pbuffer->timestamp;
 437       }
 438       break;
 439     case BT_FILE:
 440     case BT_BASE_FILE:
 441       {
 442         unsigned int len;
 443         const char *name;
 444         uchar *buf;
 445
 446         if (node->value.builtin == BT_FILE)
 447           name = linemap_get_expansion_filename (pfile->line_table,
 448                                                  pfile->line_table->highest_line);
 449         else
 450           {
 451             name = _cpp_get_file_name (pfile->main_file);
 452             if (!name)
 453               abort ();
 454           }
 455         if (pfile->cb.remap_filename)
 456           name = pfile->cb.remap_filename (name);
 457         len = strlen (name);
 458         buf = _cpp_unaligned_alloc (pfile, len * 2 + 3);
 459         result = buf;
 460         *buf = '"';
 461         buf = cpp_quote_string (buf + 1, (const unsigned char *) name, len);
 462         *buf++ = '"';
 463         *buf = '\0';
 464       }
 465       break;
 466
 467     case BT_INCLUDE_LEVEL:
 468       /* The line map depth counts the primary source as level 1, but
 469          historically __INCLUDE_DEPTH__ has called the primary source
 470          level 0.  */
 471       number = pfile->line_table->depth - 1;
 472       break;
 473
 474     case BT_SPECLINE:
 475       /* If __LINE__ is embedded in a macro, it must expand to the
 476          line of the macro's invocation, not its definition.
 477          Otherwise things like assert() will not work properly.
 478          See WG14 N1911, WG21 N4220 sec 6.5, and PR 61861.  */
 479       if (CPP_OPTION (pfile, traditional))
 480         loc = pfile->line_table->highest_line;
 481       else
 482         loc = linemap_resolve_location (pfile->line_table, loc,
 483                                         LRK_MACRO_EXPANSION_POINT, NULL);
 484       number = linemap_get_expansion_line (pfile->line_table, loc);
 485       break;
 486
 487       /* __STDC__ has the value 1 under normal circumstances.
 488          However, if (a) we are in a system header, (b) the option
 489          stdc_0_in_system_headers is true (set by target config), and
 490          (c) we are not in strictly conforming mode, then it has the
 491          value 0.  (b) and (c) are already checked in cpp_init_builtins.  */
 492     case BT_STDC:
 493       if (cpp_in_system_header (pfile))
 494         number = 0;
 495       else
 496         number = 1;
 497       break;
 498
 499     case BT_DATE:
 500     case BT_TIME:
 501       if (CPP_OPTION (pfile, warn_date_time))
 502         cpp_warning (pfile, CPP_W_DATE_TIME, "macro \"%s\" might prevent "
 503                      "reproducible builds", NODE_NAME (node));
 504       if (pfile->date == NULL)
 505         {
 506           /* Allocate __DATE__ and __TIME__ strings from permanent
 507              storage.  We only do this once, and don't generate them
 508              at init time, because time() and localtime() are very
 509              slow on some systems.  */
 510           time_t tt;
 511           struct tm *tb = NULL;
 512
 513           /* Set a reproducible timestamp for __DATE__ and __TIME__ macro
 514              if SOURCE_DATE_EPOCH is defined.  */
 515           if (pfile->source_date_epoch == (time_t) -2
 516               && pfile->cb.get_source_date_epoch != NULL)
 517             pfile->source_date_epoch = pfile->cb.get_source_date_epoch (pfile);
 518
 519           if (pfile->source_date_epoch >= (time_t) 0)
 520             tb = gmtime (&pfile->source_date_epoch);
 521           else
 522             {
 523               /* (time_t) -1 is a legitimate value for "number of seconds
 524                  since the Epoch", so we have to do a little dance to
 525                  distinguish that from a genuine error.  */
 526               errno = 0;
 527               tt = time (NULL);
 528               if (tt != (time_t)-1 || errno == 0)
 529                 tb = localtime (&tt);
 530             }
 531
 532           if (tb)
 533             {
 534               pfile->date = _cpp_unaligned_alloc (pfile,
 535                                                   sizeof ("\"Oct 11 1347\""));
 536               sprintf ((char *) pfile->date, "\"%s %2d %4d\"",
 537                        monthnames[tb->tm_mon], tb->tm_mday,
 538                        tb->tm_year + 1900);
 539
 540               pfile->time = _cpp_unaligned_alloc (pfile,
 541                                                   sizeof ("\"12:34:56\""));
 542               sprintf ((char *) pfile->time, "\"%02d:%02d:%02d\"",
 543                        tb->tm_hour, tb->tm_min, tb->tm_sec);
 544             }
 545           else
 546             {
 547               cpp_errno (pfile, CPP_DL_WARNING,
 548                          "could not determine date and time");
 549
 550               pfile->date = UC"\"??? ?? ????\"";
 551               pfile->time = UC"\"??:??:??\"";
 552             }
 553         }
 554
 555       if (node->value.builtin == BT_DATE)
 556         result = pfile->date;
 557       else
 558         result = pfile->time;
 559       break;
 560
 561     case BT_COUNTER:
 562       if (CPP_OPTION (pfile, directives_only) && pfile->state.in_directive)
 563         cpp_error (pfile, CPP_DL_ERROR,
 564             "__COUNTER__ expanded inside directive with -fdirectives-only");
 565       number = pfile->counter++;
 566       break;
 567
 568     case BT_HAS_ATTRIBUTE:
 569       number = pfile->cb.has_attribute (pfile);
 570       break;
 571     }
 572
 573   if (result == NULL)
 574     {
 575       /* 21 bytes holds all NUL-terminated unsigned 64-bit numbers.  */
 576       result = _cpp_unaligned_alloc (pfile, 21);
 577       sprintf ((char *) result, "%u", number);
 578     }
 579
 580   return result;
 581 }
 582
 583 /* Convert builtin macros like __FILE__ to a token and push it on the
 584    context stack.  Also handles _Pragma, for which a new token may not
 585    be created.  Returns 1 if it generates a new token context, 0 to
 586    return the token to the caller.  LOC is the location of the expansion
 587    point of the macro.  */
 588 static int
 589 builtin_macro (cpp_reader *pfile, cpp_hashnode *node,
 590                source_location loc, source_location expand_loc)
 591 {
 592   const uchar *buf;
 593   size_t len;
 594   char *nbuf;
 595
 596   if (node->value.builtin == BT_PRAGMA)
 597     {
 598       /* Don't interpret _Pragma within directives.  The standard is
 599          not clear on this, but to me this makes most sense.  */
 600       if (pfile->state.in_directive)
 601         return 0;
 602
 603       return _cpp_do__Pragma (pfile, loc);
 604     }
 605
 606   buf = _cpp_builtin_macro_text (pfile, node, expand_loc);
 607   len = ustrlen (buf);
 608   nbuf = (char *) alloca (len + 1);
 609   memcpy (nbuf, buf, len);
 610   nbuf[len]='\n';
 611
 612   cpp_push_buffer (pfile, (uchar *) nbuf, len, /* from_stage3 */ true);
 613   _cpp_clean_line (pfile);
 614
 615   /* Set pfile->cur_token as required by _cpp_lex_direct.  */
 616   pfile->cur_token = _cpp_temp_token (pfile);
 617   cpp_token *token = _cpp_lex_direct (pfile);
 618   /* We should point to the expansion point of the builtin macro.  */
 619   token->src_loc = loc;
 620   if (pfile->context->tokens_kind == TOKENS_KIND_EXTENDED)
 621     {
 622       /* We are tracking tokens resulting from macro expansion.
 623          Create a macro line map and generate a virtual location for
 624          the token resulting from the expansion of the built-in
 625          macro.  */
 626       source_location *virt_locs = NULL;
 627       _cpp_buff *token_buf = tokens_buff_new (pfile, 1, &virt_locs);
 628       const line_map_macro * map =
 629         linemap_enter_macro (pfile->line_table, node, loc, 1);
 630       tokens_buff_add_token (token_buf, virt_locs, token,
 631                              pfile->line_table->builtin_location,
 632                              pfile->line_table->builtin_location,
 633                             map, /*macro_token_index=*/0);
 634       push_extended_tokens_context (pfile, node, token_buf, virt_locs,
 635                                     (const cpp_token **)token_buf->base,
 636                                     1);
 637     }
 638   else
 639     _cpp_push_token_context (pfile, NULL, token, 1);
 640   if (pfile->buffer->cur != pfile->buffer->rlimit)
 641     cpp_error (pfile, CPP_DL_ICE, "invalid built-in macro \"%s\"",
 642                NODE_NAME (node));
 643   _cpp_pop_buffer (pfile);
 644
 645   return 1;
 646 }
 647
 648 /* Copies SRC, of length LEN, to DEST, adding backslashes before all
 649    backslashes and double quotes. DEST must be of sufficient size.
 650    Returns a pointer to the end of the string.  */
 651 uchar *
 652 cpp_quote_string (uchar *dest, const uchar *src, unsigned int len)
 653 {
 654   while (len--)
 655     {
 656       uchar c = *src++;
 657
 658       switch (c)
 659         {
 660         case '\n':
 661           /* Naked LF can appear in raw string literals  */
 662           c = 'n';
 663           /* FALLTHROUGH */
 664
 665         case '\\':
 666         case '"':
 667           *dest++ = '\\';
 668           /* FALLTHROUGH */
 669
 670         default:
 671           *dest++ = c;
 672         }
 673     }
 674
 675   return dest;
 676 }
 677
 678 /* Convert a token sequence ARG to a single string token according to
 679    the rules of the ISO C #-operator.  */
 680 static const cpp_token *
 681 stringify_arg (cpp_reader *pfile, macro_arg *arg)
 682 {
 683   unsigned char *dest;
 684   unsigned int i, escape_it, backslash_count = 0;
 685   const cpp_token *source = NULL;
 686   size_t len;
 687
 688   if (BUFF_ROOM (pfile->u_buff) < 3)
 689     _cpp_extend_buff (pfile, &pfile->u_buff, 3);
 690   dest = BUFF_FRONT (pfile->u_buff);
 691   *dest++ = '"';
 692
 693   /* Loop, reading in the argument's tokens.  */
 694   for (i = 0; i < arg->count; i++)
 695     {
 696       const cpp_token *token = arg->first[i];
 697
 698       if (token->type == CPP_PADDING)
 699         {
 700           if (source == NULL
 701               || (!(source->flags & PREV_WHITE)
 702                   && token->val.source == NULL))
 703             source = token->val.source;
 704           continue;
 705         }
 706
 707       escape_it = (token->type == CPP_STRING || token->type == CPP_CHAR
 708                    || token->type == CPP_WSTRING || token->type == CPP_WCHAR
 709                    || token->type == CPP_STRING32 || token->type == CPP_CHAR32
 710                    || token->type == CPP_STRING16 || token->type == CPP_CHAR16
 711                    || token->type == CPP_UTF8STRING || token->type == CPP_UTF8CHAR
 712                    || cpp_userdef_string_p (token->type)
 713                    || cpp_userdef_char_p (token->type));
 714
 715       /* Room for each char being written in octal, initial space and
 716          final quote and NUL.  */
 717       len = cpp_token_len (token);
 718       if (escape_it)
 719         len *= 4;
 720       len += 3;
 721
 722       if ((size_t) (BUFF_LIMIT (pfile->u_buff) - dest) < len)
 723         {
 724           size_t len_so_far = dest - BUFF_FRONT (pfile->u_buff);
 725           _cpp_extend_buff (pfile, &pfile->u_buff, len);
 726           dest = BUFF_FRONT (pfile->u_buff) + len_so_far;
 727         }
 728
 729       /* Leading white space?  */
 730       if (dest - 1 != BUFF_FRONT (pfile->u_buff))
 731         {
 732           if (source == NULL)
 733             source = token;
 734           if (source->flags & PREV_WHITE)
 735             *dest++ = ' ';
 736         }
 737       source = NULL;
 738
 739       if (escape_it)
 740         {
 741           _cpp_buff *buff = _cpp_get_buff (pfile, len);
 742           unsigned char *buf = BUFF_FRONT (buff);
 743           len = cpp_spell_token (pfile, token, buf, true) - buf;
 744           dest = cpp_quote_string (dest, buf, len);
 745           _cpp_release_buff (pfile, buff);
 746         }
 747       else
 748         dest = cpp_spell_token (pfile, token, dest, true);
 749
 750       if (token->type == CPP_OTHER && token->val.str.text[0] == '\\')
 751         backslash_count++;
 752       else
 753         backslash_count = 0;
 754     }
 755
 756   /* Ignore the final \ of invalid string literals.  */
 757   if (backslash_count & 1)
 758     {
 759       cpp_error (pfile, CPP_DL_WARNING,
 760                  "invalid string literal, ignoring final '\\'");
 761       dest--;
 762     }
 763
 764   /* Commit the memory, including NUL, and return the token.  */
 765   *dest++ = '"';
 766   len = dest - BUFF_FRONT (pfile->u_buff);
 767   BUFF_FRONT (pfile->u_buff) = dest + 1;
 768   return new_string_token (pfile, dest - len, len);
 769 }
 770
 771 /* Try to paste two tokens.  On success, return nonzero.  In any
 772    case, PLHS is updated to point to the pasted token, which is
 773    guaranteed to not have the PASTE_LEFT flag set.  LOCATION is
 774    the virtual location used for error reporting.  */
 775 static bool
 776 paste_tokens (cpp_reader *pfile, source_location location,
 777               const cpp_token **plhs, const cpp_token *rhs)
 778 {
 779   unsigned char *buf, *end, *lhsend;
 780   cpp_token *lhs;
 781   unsigned int len;
 782
 783   len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 1;
 784   buf = (unsigned char *) alloca (len);
 785   end = lhsend = cpp_spell_token (pfile, *plhs, buf, true);
 786
 787   /* Avoid comment headers, since they are still processed in stage 3.
 788      It is simpler to insert a space here, rather than modifying the
 789      lexer to ignore comments in some circumstances.  Simply returning
 790      false doesn't work, since we want to clear the PASTE_LEFT flag.  */
 791   if ((*plhs)->type == CPP_DIV && rhs->type != CPP_EQ)
 792     *end++ = ' ';
 793   /* In one obscure case we might see padding here.  */
 794   if (rhs->type != CPP_PADDING)
 795     end = cpp_spell_token (pfile, rhs, end, true);
 796   *end = '\n';
 797
 798   cpp_push_buffer (pfile, buf, end - buf, /* from_stage3 */ true);
 799   _cpp_clean_line (pfile);
 800
 801   /* Set pfile->cur_token as required by _cpp_lex_direct.  */
 802   pfile->cur_token = _cpp_temp_token (pfile);
 803   lhs = _cpp_lex_direct (pfile);
 804   if (pfile->buffer->cur != pfile->buffer->rlimit)
 805     {
 806       source_location saved_loc = lhs->src_loc;
 807
 808       _cpp_pop_buffer (pfile);
 809       _cpp_backup_tokens (pfile, 1);
 810       *lhsend = '\0';
 811
 812       /* We have to remove the PASTE_LEFT flag from the old lhs, but
 813          we want to keep the new location.  */
 814       *lhs = **plhs;
 815       *plhs = lhs;
 816       lhs->src_loc = saved_loc;
 817       lhs->flags &= ~PASTE_LEFT;
 818
 819       /* Mandatory error for all apart from assembler.  */
 820       if (CPP_OPTION (pfile, lang) != CLK_ASM)
 821         cpp_error_with_line (pfile, CPP_DL_ERROR, location, 0,
 822          "pasting \"%s\" and \"%s\" does not give a valid preprocessing token",
 823                    buf, cpp_token_as_text (pfile, rhs));
 824       return false;
 825     }
 826
 827   *plhs = lhs;
 828   _cpp_pop_buffer (pfile);
 829   return true;
 830 }
 831
 832 /* Handles an arbitrarily long sequence of ## operators, with initial
 833    operand LHS.  This implementation is left-associative,
 834    non-recursive, and finishes a paste before handling succeeding
 835    ones.  If a paste fails, we back up to the RHS of the failing ##
 836    operator before pushing the context containing the result of prior
 837    successful pastes, with the effect that the RHS appears in the
 838    output stream after the pasted LHS normally.  */
 839 static void
 840 paste_all_tokens (cpp_reader *pfile, const cpp_token *lhs)
 841 {
 842   const cpp_token *rhs = NULL;
 843   cpp_context *context = pfile->context;
 844   source_location virt_loc = 0;
 845
 846   /* We are expanding a macro and we must have been called on a token
 847      that appears at the left hand side of a ## operator.  */
 848   if (macro_of_context (pfile->context) == NULL
 849       || (!(lhs->flags & PASTE_LEFT)))
 850     abort ();
 851
 852   if (context->tokens_kind == TOKENS_KIND_EXTENDED)
 853     /* The caller must have called consume_next_token_from_context
 854        right before calling us.  That has incremented the pointer to
 855        the current virtual location.  So it now points to the location
 856        of the token that comes right after *LHS.  We want the
 857        resulting pasted token to have the location of the current
 858        *LHS, though.  */
 859     virt_loc = context->c.mc->cur_virt_loc[-1];
 860   else
 861     /* We are not tracking macro expansion.  So the best virtual
 862        location we can get here is the expansion point of the macro we
 863        are currently expanding.  */
 864     virt_loc = pfile->invocation_location;
 865
 866   do
 867     {
 868       /* Take the token directly from the current context.  We can do
 869          this, because we are in the replacement list of either an
 870          object-like macro, or a function-like macro with arguments
 871          inserted.  In either case, the constraints to #define
 872          guarantee we have at least one more token.  */
 873       if (context->tokens_kind == TOKENS_KIND_DIRECT)
 874         rhs = FIRST (context).token++;
 875       else if (context->tokens_kind == TOKENS_KIND_INDIRECT)
 876         rhs = *FIRST (context).ptoken++;
 877       else if (context->tokens_kind == TOKENS_KIND_EXTENDED)
 878         {
 879           /* So we are in presence of an extended token context, which
 880              means that each token in this context has a virtual
 881              location attached to it.  So let's not forget to update
 882              the pointer to the current virtual location of the
 883              current token when we update the pointer to the current
 884              token */
 885
 886           rhs = *FIRST (context).ptoken++;
 887           /* context->c.mc must be non-null, as if we were not in a
 888              macro context, context->tokens_kind could not be equal to
 889              TOKENS_KIND_EXTENDED.  */
 890           context->c.mc->cur_virt_loc++;
 891         }
 892
 893       if (rhs->type == CPP_PADDING)
 894         {
 895           if (rhs->flags & PASTE_LEFT)
 896             abort ();
 897         }
 898       if (!paste_tokens (pfile, virt_loc, &lhs, rhs))
 899         break;
 900     }
 901   while (rhs->flags & PASTE_LEFT);
 902
 903   /* Put the resulting token in its own context.  */
 904   if (context->tokens_kind == TOKENS_KIND_EXTENDED)
 905     {
 906       source_location *virt_locs = NULL;
 907       _cpp_buff *token_buf = tokens_buff_new (pfile, 1, &virt_locs);
 908       tokens_buff_add_token (token_buf, virt_locs, lhs,
 909                              virt_loc, 0, NULL, 0);
 910       push_extended_tokens_context (pfile, context->c.mc->macro_node,
 911                                     token_buf, virt_locs,
 912                                     (const cpp_token **)token_buf->base, 1);
 913     }
 914   else
 915     _cpp_push_token_context (pfile, NULL, lhs, 1);
 916 }
 917
 918 /* Returns TRUE if the number of arguments ARGC supplied in an
 919    invocation of the MACRO referenced by NODE is valid.  An empty
 920    invocation to a macro with no parameters should pass ARGC as zero.
 921
 922    Note that MACRO cannot necessarily be deduced from NODE, in case
 923    NODE was redefined whilst collecting arguments.  */
 924 bool
 925 _cpp_arguments_ok (cpp_reader *pfile, cpp_macro *macro, const cpp_hashnode *node, unsigned int argc)
 926 {
 927   if (argc == macro->paramc)
 928     return true;
 929
 930   if (argc < macro->paramc)
 931     {
 932       /* In C++2a (here the va_opt flag is used), and also as a GNU
 933          extension, variadic arguments are allowed to not appear in
 934          the invocation at all.
 935          e.g. #define debug(format, args...) something
 936          debug("string");
 937
 938          This is exactly the same as if an empty variadic list had been
 939          supplied - debug("string", ).  */
 940
 941       if (argc + 1 == macro->paramc && macro->variadic)
 942         {
 943           if (CPP_PEDANTIC (pfile) && ! macro->syshdr
 944               && ! CPP_OPTION (pfile, va_opt))
 945             {
 946               if (CPP_OPTION (pfile, cplusplus))
 947                 cpp_error (pfile, CPP_DL_PEDWARN,
 948                            "ISO C++11 requires at least one argument "
 949                            "for the \"...\" in a variadic macro");
 950               else
 951                 cpp_error (pfile, CPP_DL_PEDWARN,
 952                            "ISO C99 requires at least one argument "
 953                            "for the \"...\" in a variadic macro");
 954             }
 955           return true;
 956         }
 957
 958       cpp_error (pfile, CPP_DL_ERROR,
 959                  "macro \"%s\" requires %u arguments, but only %u given",
 960                  NODE_NAME (node), macro->paramc, argc);
 961     }
 962   else
 963     cpp_error (pfile, CPP_DL_ERROR,
 964                "macro \"%s\" passed %u arguments, but takes just %u",
 965                NODE_NAME (node), argc, macro->paramc);
 966
 967   return false;
 968 }
 969
 970 /* Reads and returns the arguments to a function-like macro
 971    invocation.  Assumes the opening parenthesis has been processed.
 972    If there is an error, emits an appropriate diagnostic and returns
 973    NULL.  Each argument is terminated by a CPP_EOF token, for the
 974    future benefit of expand_arg().  If there are any deferred
 975    #pragma directives among macro arguments, store pointers to the
 976    CPP_PRAGMA ... CPP_PRAGMA_EOL tokens into *PRAGMA_BUFF buffer.
 977
 978    What is returned is the buffer that contains the memory allocated
 979    to hold the macro arguments.  NODE is the name of the macro this
 980    function is dealing with.  If NUM_ARGS is non-NULL, *NUM_ARGS is
 981    set to the actual number of macro arguments allocated in the
 982    returned buffer.  */
 983 static _cpp_buff *
 984 collect_args (cpp_reader *pfile, const cpp_hashnode *node,
 985               _cpp_buff **pragma_buff, unsigned *num_args)
 986 {
 987   _cpp_buff *buff, *base_buff;
 988   cpp_macro *macro;
 989   macro_arg *args, *arg;
 990   const cpp_token *token;
 991   unsigned int argc;
 992   source_location virt_loc;
 993   bool track_macro_expansion_p = CPP_OPTION (pfile, track_macro_expansion);
 994   unsigned num_args_alloced = 0;
 995
 996   macro = node->value.macro;
 997   if (macro->paramc)
 998     argc = macro->paramc;
 999   else
1000     argc = 1;
1001
1002 #define DEFAULT_NUM_TOKENS_PER_MACRO_ARG 50
1003 #define ARG_TOKENS_EXTENT 1000
1004
1005   buff = _cpp_get_buff (pfile, argc * (DEFAULT_NUM_TOKENS_PER_MACRO_ARG
1006                                        * sizeof (cpp_token *)
1007                                        + sizeof (macro_arg)));
1008   base_buff = buff;
1009   args = (macro_arg *) buff->base;
1010   memset (args, 0, argc * sizeof (macro_arg));
1011   buff->cur = (unsigned char *) &args[argc];
1012   arg = args, argc = 0;
1013
1014   /* Collect the tokens making up each argument.  We don't yet know
1015      how many arguments have been supplied, whether too many or too
1016      few.  Hence the slightly bizarre usage of "argc" and "arg".  */
1017   do
1018     {
1019       unsigned int paren_depth = 0;
1020       unsigned int ntokens = 0;
1021       unsigned virt_locs_capacity = DEFAULT_NUM_TOKENS_PER_MACRO_ARG;
1022       num_args_alloced++;
1023
1024       argc++;
1025       arg->first = (const cpp_token **) buff->cur;
1026       if (track_macro_expansion_p)
1027         {
1028           virt_locs_capacity = DEFAULT_NUM_TOKENS_PER_MACRO_ARG;
1029           arg->virt_locs = XNEWVEC (source_location,
1030                                     virt_locs_capacity);
1031         }
1032
1033       for (;;)
1034         {
1035           /* Require space for 2 new tokens (including a CPP_EOF).  */
1036           if ((unsigned char *) &arg->first[ntokens + 2] > buff->limit)
1037             {
1038               buff = _cpp_append_extend_buff (pfile, buff,
1039                                               ARG_TOKENS_EXTENT
1040                                               * sizeof (cpp_token *));
1041               arg->first = (const cpp_token **) buff->cur;
1042             }
1043           if (track_macro_expansion_p
1044               && (ntokens + 2 > virt_locs_capacity))
1045             {
1046               virt_locs_capacity += ARG_TOKENS_EXTENT;
1047               arg->virt_locs = XRESIZEVEC (source_location,
1048                                            arg->virt_locs,
1049                                            virt_locs_capacity);
1050             }
1051
1052           token = cpp_get_token_1 (pfile, &virt_loc);
1053
1054           if (token->type == CPP_PADDING)
1055             {
1056               /* Drop leading padding.  */
1057               if (ntokens == 0)
1058                 continue;
1059             }
1060           else if (token->type == CPP_OPEN_PAREN)
1061             paren_depth++;
1062           else if (token->type == CPP_CLOSE_PAREN)
1063             {
1064               if (paren_depth-- == 0)
1065                 break;
1066             }
1067           else if (token->type == CPP_COMMA)
1068             {
1069               /* A comma does not terminate an argument within
1070                  parentheses or as part of a variable argument.  */
1071               if (paren_depth == 0
1072                   && ! (macro->variadic && argc == macro->paramc))
1073                 break;
1074             }
1075           else if (token->type == CPP_EOF
1076                    || (token->type == CPP_HASH && token->flags & BOL))
1077             break;
1078           else if (token->type == CPP_PRAGMA)
1079             {
1080               cpp_token *newtok = _cpp_temp_token (pfile);
1081
1082               /* CPP_PRAGMA token lives in directive_result, which will
1083                  be overwritten on the next directive.  */
1084               *newtok = *token;
1085               token = newtok;
1086               do
1087                 {
1088                   if (*pragma_buff == NULL
1089                       || BUFF_ROOM (*pragma_buff) < sizeof (cpp_token *))
1090                     {
1091                       _cpp_buff *next;
1092                       if (*pragma_buff == NULL)
1093                         *pragma_buff
1094                           = _cpp_get_buff (pfile, 32 * sizeof (cpp_token *));
1095                       else
1096                         {
1097                           next = *pragma_buff;
1098                           *pragma_buff
1099                             = _cpp_get_buff (pfile,
1100                                              (BUFF_FRONT (*pragma_buff)
1101                                               - (*pragma_buff)->base) * 2);
1102                           (*pragma_buff)->next = next;
1103                         }
1104                     }
1105                   *(const cpp_token **) BUFF_FRONT (*pragma_buff) = token;
1106                   BUFF_FRONT (*pragma_buff) += sizeof (cpp_token *);
1107                   if (token->type == CPP_PRAGMA_EOL)
1108                     break;
1109                   token = cpp_get_token_1 (pfile, &virt_loc);
1110                 }
1111               while (token->type != CPP_EOF);
1112
1113               /* In deferred pragmas parsing_args and prevent_expansion
1114                  had been changed, reset it.  */
1115               pfile->state.parsing_args = 2;
1116               pfile->state.prevent_expansion = 1;
1117
1118               if (token->type == CPP_EOF)
1119                 break;
1120               else
1121                 continue;
1122             }
1123           set_arg_token (arg, token, virt_loc,
1124                          ntokens, MACRO_ARG_TOKEN_NORMAL,
1125                          CPP_OPTION (pfile, track_macro_expansion));
1126           ntokens++;
1127         }
1128
1129       /* Drop trailing padding.  */
1130       while (ntokens > 0 && arg->first[ntokens - 1]->type == CPP_PADDING)
1131         ntokens--;
1132
1133       arg->count = ntokens;
1134       set_arg_token (arg, &pfile->eof, pfile->eof.src_loc,
1135                      ntokens, MACRO_ARG_TOKEN_NORMAL,
1136                      CPP_OPTION (pfile, track_macro_expansion));
1137
1138       /* Terminate the argument.  Excess arguments loop back and
1139          overwrite the final legitimate argument, before failing.  */
1140       if (argc <= macro->paramc)
1141         {
1142           buff->cur = (unsigned char *) &arg->first[ntokens + 1];
1143           if (argc != macro->paramc)
1144             arg++;
1145         }
1146     }
1147   while (token->type != CPP_CLOSE_PAREN && token->type != CPP_EOF);
1148
1149   if (token->type == CPP_EOF)
1150     {
1151       /* We still need the CPP_EOF to end directives, and to end
1152          pre-expansion of a macro argument.  Step back is not
1153          unconditional, since we don't want to return a CPP_EOF to our
1154          callers at the end of an -include-d file.  */
1155       if (pfile->context->prev || pfile->state.in_directive)
1156         _cpp_backup_tokens (pfile, 1);
1157       cpp_error (pfile, CPP_DL_ERROR,
1158                  "unterminated argument list invoking macro \"%s\"",
1159                  NODE_NAME (node));
1160     }
1161   else
1162     {
1163       /* A single empty argument is counted as no argument.  */
1164       if (argc == 1 && macro->paramc == 0 && args[0].count == 0)
1165         argc = 0;
1166       if (_cpp_arguments_ok (pfile, macro, node, argc))
1167         {
1168           /* GCC has special semantics for , ## b where b is a varargs
1169              parameter: we remove the comma if b was omitted entirely.
1170              If b was merely an empty argument, the comma is retained.
1171              If the macro takes just one (varargs) parameter, then we
1172              retain the comma only if we are standards conforming.
1173
1174              If FIRST is NULL replace_args () swallows the comma.  */
1175           if (macro->variadic && (argc < macro->paramc
1176                                   || (argc == 1 && args[0].count == 0
1177                                       && !CPP_OPTION (pfile, std))))
1178             args[macro->paramc - 1].first = NULL;
1179           if (num_args)
1180             *num_args = num_args_alloced;
1181           return base_buff;
1182         }
1183     }
1184
1185   /* An error occurred.  */
1186   _cpp_release_buff (pfile, base_buff);
1187   return NULL;
1188 }
1189
1190 /* Search for an opening parenthesis to the macro of NODE, in such a
1191    way that, if none is found, we don't lose the information in any
1192    intervening padding tokens.  If we find the parenthesis, collect
1193    the arguments and return the buffer containing them.  PRAGMA_BUFF
1194    argument is the same as in collect_args.  If NUM_ARGS is non-NULL,
1195    *NUM_ARGS is set to the number of arguments contained in the
1196    returned buffer.  */
1197 static _cpp_buff *
1198 funlike_invocation_p (cpp_reader *pfile, cpp_hashnode *node,
1199                       _cpp_buff **pragma_buff, unsigned *num_args)
1200 {
1201   const cpp_token *token, *padding = NULL;
1202
1203   for (;;)
1204     {
1205       token = cpp_get_token (pfile);
1206       if (token->type != CPP_PADDING)
1207         break;
1208       if (padding == NULL
1209           || (!(padding->flags & PREV_WHITE) && token->val.source == NULL))
1210         padding = token;
1211     }
1212
1213   if (token->type == CPP_OPEN_PAREN)
1214     {
1215       pfile->state.parsing_args = 2;
1216       return collect_args (pfile, node, pragma_buff, num_args);
1217     }
1218
1219   /* CPP_EOF can be the end of macro arguments, or the end of the
1220      file.  We mustn't back up over the latter.  Ugh.  */
1221   if (token->type != CPP_EOF || token == &pfile->eof)
1222     {
1223       /* Back up.  We may have skipped padding, in which case backing
1224          up more than one token when expanding macros is in general
1225          too difficult.  We re-insert it in its own context.  */
1226       _cpp_backup_tokens (pfile, 1);
1227       if (padding)
1228         _cpp_push_token_context (pfile, NULL, padding, 1);
1229     }
1230
1231   return NULL;
1232 }
1233
1234 /* Return the real number of tokens in the expansion of MACRO.  */
1235 static inline unsigned int
1236 macro_real_token_count (const cpp_macro *macro)
1237 {
1238   unsigned int i;
1239   if (__builtin_expect (!macro->extra_tokens, true))
1240     return macro->count;
1241   for (i = 0; i < macro->count; i++)
1242     if (macro->exp.tokens[i].type == CPP_PASTE)
1243       return i;
1244   abort ();
1245 }
1246
1247 /* Push the context of a macro with hash entry NODE onto the context
1248    stack.  If we can successfully expand the macro, we push a context
1249    containing its yet-to-be-rescanned replacement list and return one.
1250    If there were additionally any unexpanded deferred #pragma
1251    directives among macro arguments, push another context containing
1252    the pragma tokens before the yet-to-be-rescanned replacement list
1253    and return two.  Otherwise, we don't push a context and return
1254    zero. LOCATION is the location of the expansion point of the
1255    macro.  */
1256 static int
1257 enter_macro_context (cpp_reader *pfile, cpp_hashnode *node,
1258                      const cpp_token *result, source_location location)
1259 {
1260   /* The presence of a macro invalidates a file's controlling macro.  */
1261   pfile->mi_valid = false;
1262
1263   pfile->state.angled_headers = false;
1264
1265   /* From here to when we push the context for the macro later down
1266      this function, we need to flag the fact that we are about to
1267      expand a macro.  This is useful when -ftrack-macro-expansion is
1268      turned off.  In that case, we need to record the location of the
1269      expansion point of the top-most macro we are about to to expand,
1270      into pfile->invocation_location.  But we must not record any such
1271      location once the process of expanding the macro starts; that is,
1272      we must not do that recording between now and later down this
1273      function where set this flag to FALSE.  */
1274   pfile->about_to_expand_macro_p = true;
1275
1276   if ((node->flags & NODE_BUILTIN) && !(node->flags & NODE_USED))
1277     {
1278       node->flags |= NODE_USED;
1279       if ((!pfile->cb.user_builtin_macro
1280            || !pfile->cb.user_builtin_macro (pfile, node))
1281           && pfile->cb.used_define)
1282         pfile->cb.used_define (pfile, pfile->directive_line, node);
1283     }
1284
1285   /* Handle standard macros.  */
1286   if (! (node->flags & NODE_BUILTIN))
1287     {
1288       cpp_macro *macro = node->value.macro;
1289       _cpp_buff *pragma_buff = NULL;
1290
1291       if (macro->fun_like)
1292         {
1293           _cpp_buff *buff;
1294           unsigned num_args = 0;
1295
1296           pfile->state.prevent_expansion++;
1297           pfile->keep_tokens++;
1298           pfile->state.parsing_args = 1;
1299           buff = funlike_invocation_p (pfile, node, &pragma_buff,
1300                                        &num_args);
1301           pfile->state.parsing_args = 0;
1302           pfile->keep_tokens--;
1303           pfile->state.prevent_expansion--;
1304
1305           if (buff == NULL)
1306             {
1307               if (CPP_WTRADITIONAL (pfile) && ! node->value.macro->syshdr)
1308                 cpp_warning (pfile, CPP_W_TRADITIONAL,
1309  "function-like macro \"%s\" must be used with arguments in traditional C",
1310                              NODE_NAME (node));
1311
1312               if (pragma_buff)
1313                 _cpp_release_buff (pfile, pragma_buff);
1314
1315               pfile->about_to_expand_macro_p = false;
1316               return 0;
1317             }
1318
1319           if (macro->paramc > 0)
1320             replace_args (pfile, node, macro,
1321                           (macro_arg *) buff->base,
1322                           location);
1323           /* Free the memory used by the arguments of this
1324              function-like macro.  This memory has been allocated by
1325              funlike_invocation_p and by replace_args.  */
1326           delete_macro_args (buff, num_args);
1327         }
1328
1329       /* Disable the macro within its expansion.  */
1330       node->flags |= NODE_DISABLED;
1331
1332       if (!(node->flags & NODE_USED))
1333         {
1334           node->flags |= NODE_USED;
1335           if (pfile->cb.used_define)
1336             pfile->cb.used_define (pfile, pfile->directive_line, node);
1337         }
1338
1339       if (pfile->cb.used)
1340         pfile->cb.used (pfile, location, node);
1341
1342       macro->used = 1;
1343
1344       if (macro->paramc == 0)
1345         {
1346           unsigned tokens_count = macro_real_token_count (macro);
1347           if (CPP_OPTION (pfile, track_macro_expansion))
1348             {
1349               unsigned int i;
1350               const cpp_token *src = macro->exp.tokens;
1351               const line_map_macro *map;
1352               source_location *virt_locs = NULL;
1353               _cpp_buff *macro_tokens
1354                 = tokens_buff_new (pfile, tokens_count, &virt_locs);
1355
1356               /* Create a macro map to record the locations of the
1357                  tokens that are involved in the expansion. LOCATION
1358                  is the location of the macro expansion point.  */
1359               map = linemap_enter_macro (pfile->line_table,
1360                                          node, location, tokens_count);
1361               for (i = 0; i < tokens_count; ++i)
1362                 {
1363                   tokens_buff_add_token (macro_tokens, virt_locs,
1364                                          src, src->src_loc,
1365                                          src->src_loc, map, i);
1366                   ++src;
1367                 }
1368               push_extended_tokens_context (pfile, node,
1369                                             macro_tokens,
1370                                             virt_locs,
1371                                             (const cpp_token **)
1372                                             macro_tokens->base,
1373                                             tokens_count);
1374             }
1375           else
1376             _cpp_push_token_context (pfile, node, macro->exp.tokens,
1377                                      tokens_count);
1378           num_macro_tokens_counter += tokens_count;
1379         }
1380
1381       if (pragma_buff)
1382         {
1383           if (!pfile->state.in_directive)
1384             _cpp_push_token_context (pfile, NULL,
1385                                      padding_token (pfile, result), 1);
1386           do
1387             {
1388               unsigned tokens_count;
1389               _cpp_buff *tail = pragma_buff->next;
1390               pragma_buff->next = NULL;
1391               tokens_count = ((const cpp_token **) BUFF_FRONT (pragma_buff)
1392                               - (const cpp_token **) pragma_buff->base);
1393               push_ptoken_context (pfile, NULL, pragma_buff,
1394                                    (const cpp_token **) pragma_buff->base,
1395                                    tokens_count);
1396               pragma_buff = tail;
1397               if (!CPP_OPTION (pfile, track_macro_expansion))
1398                 num_macro_tokens_counter += tokens_count;
1399
1400             }
1401           while (pragma_buff != NULL);
1402           pfile->about_to_expand_macro_p = false;
1403           return 2;
1404         }
1405
1406       pfile->about_to_expand_macro_p = false;
1407       return 1;
1408     }
1409
1410   pfile->about_to_expand_macro_p = false;
1411   /* Handle built-in macros and the _Pragma operator.  */
1412   {
1413     source_location loc, expand_loc;
1414
1415     if (/* The top-level macro invocation that triggered the expansion
1416            we are looking at is with a standard macro ...*/
1417         !(pfile->top_most_macro_node->flags & NODE_BUILTIN)
1418         /* ... and it's a function-like macro invocation.  */
1419         && pfile->top_most_macro_node->value.macro->fun_like)
1420       {
1421         /* Then the location of the end of the macro invocation is the
1422            location of the closing parenthesis.  */
1423         loc = pfile->cur_token[-1].src_loc;
1424         expand_loc = loc;
1425       }
1426     else
1427       {
1428         /* Otherwise, the location of the end of the macro invocation is
1429            the location of the expansion point of that top-level macro
1430            invocation.  */
1431         loc = location;
1432         expand_loc = pfile->invocation_location;
1433       }
1434
1435     return builtin_macro (pfile, node, loc, expand_loc);
1436   }
1437 }
1438
1439 /* De-allocate the memory used by BUFF which is an array of instances
1440    of macro_arg.  NUM_ARGS is the number of instances of macro_arg
1441    present in BUFF.  */
1442 static void
1443 delete_macro_args (_cpp_buff *buff, unsigned num_args)
1444 {
1445   macro_arg *macro_args;
1446   unsigned i;
1447
1448   if (buff == NULL)
1449     return;
1450
1451   macro_args = (macro_arg *) buff->base;
1452
1453   /* Walk instances of macro_arg to free their expanded tokens as well
1454      as their macro_arg::virt_locs members.  */
1455   for (i = 0; i < num_args; ++i)
1456     {
1457       if (macro_args[i].expanded)
1458         {
1459           free (macro_args[i].expanded);
1460           macro_args[i].expanded = NULL;
1461         }
1462       if (macro_args[i].virt_locs)
1463         {
1464           free (macro_args[i].virt_locs);
1465           macro_args[i].virt_locs = NULL;
1466         }
1467       if (macro_args[i].expanded_virt_locs)
1468         {
1469           free (macro_args[i].expanded_virt_locs);
1470           macro_args[i].expanded_virt_locs = NULL;
1471         }
1472     }
1473   _cpp_free_buff (buff);
1474 }
1475
1476 /* Set the INDEXth token of the macro argument ARG. TOKEN is the token
1477    to set, LOCATION is its virtual location.  "Virtual" location means
1478    the location that encodes loci across macro expansion. Otherwise
1479    it has to be TOKEN->SRC_LOC.  KIND is the kind of tokens the
1480    argument ARG is supposed to contain.  Note that ARG must be
1481    tailored so that it has enough room to contain INDEX + 1 numbers of
1482    tokens, at least.  */
1483 static void
1484 set_arg_token (macro_arg *arg, const cpp_token *token,
1485                source_location location, size_t index,
1486                enum macro_arg_token_kind kind,
1487                bool track_macro_exp_p)
1488 {
1489   const cpp_token **token_ptr;
1490   source_location *loc = NULL;
1491
1492   token_ptr =
1493     arg_token_ptr_at (arg, index, kind,
1494                       track_macro_exp_p ? &loc : NULL);
1495   *token_ptr = token;
1496
1497   if (loc != NULL)
1498     {
1499       /* We can't set the location of a stringified argument
1500          token and we can't set any location if we aren't tracking
1501          macro expansion locations.   */
1502       gcc_checking_assert (kind != MACRO_ARG_TOKEN_STRINGIFIED
1503                            && track_macro_exp_p);
1504       *loc = location;
1505     }
1506 }
1507
1508 /* Get the pointer to the location of the argument token of the
1509    function-like macro argument ARG.  This function must be called
1510    only when we -ftrack-macro-expansion is on.  */
1511 static const source_location *
1512 get_arg_token_location (const macro_arg *arg,
1513                         enum macro_arg_token_kind kind)
1514 {
1515   const source_location *loc = NULL;
1516   const cpp_token **token_ptr =
1517     arg_token_ptr_at (arg, 0, kind, (source_location **) &loc);
1518
1519   if (token_ptr == NULL)
1520     return NULL;
1521
1522   return loc;
1523 }
1524
1525 /* Return the pointer to the INDEXth token of the macro argument ARG.
1526    KIND specifies the kind of token the macro argument ARG contains.
1527    If VIRT_LOCATION is non NULL, *VIRT_LOCATION is set to the address
1528    of the virtual location of the returned token if the
1529    -ftrack-macro-expansion flag is on; otherwise, it's set to the
1530    spelling location of the returned token.  */
1531 static const cpp_token **
1532 arg_token_ptr_at (const macro_arg *arg, size_t index,
1533                   enum macro_arg_token_kind kind,
1534                   source_location **virt_location)
1535 {
1536   const cpp_token **tokens_ptr = NULL;
1537
1538   switch (kind)
1539     {
1540     case MACRO_ARG_TOKEN_NORMAL:
1541       tokens_ptr = arg->first;
1542       break;
1543     case MACRO_ARG_TOKEN_STRINGIFIED:
1544       tokens_ptr = (const cpp_token **) &arg->stringified;
1545       break;
1546     case MACRO_ARG_TOKEN_EXPANDED:
1547         tokens_ptr = arg->expanded;
1548       break;
1549     }
1550
1551   if (tokens_ptr == NULL)
1552     /* This can happen for e.g, an empty token argument to a
1553        funtion-like macro.  */
1554     return tokens_ptr;
1555
1556   if (virt_location)
1557     {
1558       if (kind == MACRO_ARG_TOKEN_NORMAL)
1559         *virt_location = &arg->virt_locs[index];
1560       else if (kind == MACRO_ARG_TOKEN_EXPANDED)
1561         *virt_location = &arg->expanded_virt_locs[index];
1562       else if (kind == MACRO_ARG_TOKEN_STRINGIFIED)
1563         *virt_location =
1564           (source_location *) &tokens_ptr[index]->src_loc;
1565     }
1566   return &tokens_ptr[index];
1567 }
1568
1569 /* Initialize an iterator so that it iterates over the tokens of a
1570    function-like macro argument.  KIND is the kind of tokens we want
1571    ITER to iterate over. TOKEN_PTR points the first token ITER will
1572    iterate over.  */
1573 static void
1574 macro_arg_token_iter_init (macro_arg_token_iter *iter,
1575                            bool track_macro_exp_p,
1576                            enum macro_arg_token_kind kind,
1577                            const macro_arg *arg,
1578                            const cpp_token **token_ptr)
1579 {
1580   iter->track_macro_exp_p = track_macro_exp_p;
1581   iter->kind = kind;
1582   iter->token_ptr = token_ptr;
1583   /* Unconditionally initialize this so that the compiler doesn't warn
1584      about iter->location_ptr being possibly uninitialized later after
1585      this code has been inlined somewhere.  */
1586   iter->location_ptr = NULL;
1587   if (track_macro_exp_p)
1588     iter->location_ptr = get_arg_token_location (arg, kind);
1589 #if CHECKING_P
1590   iter->num_forwards = 0;
1591   if (track_macro_exp_p
1592       && token_ptr != NULL
1593       && iter->location_ptr == NULL)
1594     abort ();
1595 #endif
1596 }
1597
1598 /* Move the iterator one token forward. Note that if IT was
1599    initialized on an argument that has a stringified token, moving it
1600    forward doesn't make sense as a stringified token is essentially one
1601    string.  */
1602 static void
1603 macro_arg_token_iter_forward (macro_arg_token_iter *it)
1604 {
1605   switch (it->kind)
1606     {
1607     case MACRO_ARG_TOKEN_NORMAL:
1608     case MACRO_ARG_TOKEN_EXPANDED:
1609       it->token_ptr++;
1610       if (it->track_macro_exp_p)
1611         it->location_ptr++;
1612       break;
1613     case MACRO_ARG_TOKEN_STRINGIFIED:
1614 #if CHECKING_P
1615       if (it->num_forwards > 0)
1616         abort ();
1617 #endif
1618       break;
1619     }
1620
1621 #if CHECKING_P
1622   it->num_forwards++;
1623 #endif
1624 }
1625
1626 /* Return the token pointed to by the iterator.  */
1627 static const cpp_token *
1628 macro_arg_token_iter_get_token (const macro_arg_token_iter *it)
1629 {
1630 #if CHECKING_P
1631   if (it->kind == MACRO_ARG_TOKEN_STRINGIFIED
1632       && it->num_forwards > 0)
1633     abort ();
1634 #endif
1635   if (it->token_ptr == NULL)
1636     return NULL;
1637   return *it->token_ptr;
1638 }
1639
1640 /* Return the location of the token pointed to by the iterator.*/
1641 static source_location
1642 macro_arg_token_iter_get_location (const macro_arg_token_iter *it)
1643 {
1644 #if CHECKING_P
1645   if (it->kind == MACRO_ARG_TOKEN_STRINGIFIED
1646       && it->num_forwards > 0)
1647     abort ();
1648 #endif
1649   if (it->track_macro_exp_p)
1650     return *it->location_ptr;
1651   else
1652     return (*it->token_ptr)->src_loc;
1653 }
1654
1655 /* Return the index of a token [resulting from macro expansion] inside
1656    the total list of tokens resulting from a given macro
1657    expansion. The index can be different depending on whether if we
1658    want each tokens resulting from function-like macro arguments
1659    expansion to have a different location or not.
1660
1661    E.g, consider this function-like macro:
1662
1663         #define M(x) x - 3
1664
1665    Then consider us "calling" it (and thus expanding it) like:
1666
1667        M(1+4)
1668
1669    It will be expanded into:
1670
1671        1+4-3
1672
1673    Let's consider the case of the token '4'.
1674
1675    Its index can be 2 (it's the third token of the set of tokens
1676    resulting from the expansion) or it can be 0 if we consider that
1677    all tokens resulting from the expansion of the argument "1+2" have
1678    the same index, which is 0. In this later case, the index of token
1679    '-' would then be 1 and the index of token '3' would be 2.
1680
1681    The later case is useful to use less memory e.g, for the case of
1682    the user using the option -ftrack-macro-expansion=1.
1683
1684    ABSOLUTE_TOKEN_INDEX is the index of the macro argument token we
1685    are interested in.  CUR_REPLACEMENT_TOKEN is the token of the macro
1686    parameter (inside the macro replacement list) that corresponds to
1687    the macro argument for which ABSOLUTE_TOKEN_INDEX is a token index
1688    of.
1689
1690    If we refer to the example above, for the '4' argument token,
1691    ABSOLUTE_TOKEN_INDEX would be set to 2, and CUR_REPLACEMENT_TOKEN
1692    would be set to the token 'x', in the replacement list "x - 3" of
1693    macro M.
1694
1695    This is a subroutine of replace_args.  */
1696 inline static unsigned
1697 expanded_token_index (cpp_reader *pfile, cpp_macro *macro,
1698                       const cpp_token *cur_replacement_token,
1699                       unsigned absolute_token_index)
1700 {
1701   if (CPP_OPTION (pfile, track_macro_expansion) > 1)
1702     return absolute_token_index;
1703   return cur_replacement_token - macro->exp.tokens;
1704 }
1705
1706 /* Copy whether PASTE_LEFT is set from SRC to *PASTE_FLAG.  */
1707
1708 static void
1709 copy_paste_flag (cpp_reader *pfile, const cpp_token **paste_flag,
1710                  const cpp_token *src)
1711 {
1712   cpp_token *token = _cpp_temp_token (pfile);
1713   token->type = (*paste_flag)->type;
1714   token->val = (*paste_flag)->val;
1715   if (src->flags & PASTE_LEFT)
1716     token->flags = (*paste_flag)->flags | PASTE_LEFT;
1717   else
1718     token->flags = (*paste_flag)->flags & ~PASTE_LEFT;
1719   *paste_flag = token;
1720 }
1721
1722 /* True IFF the last token emitted into BUFF (if any) is PTR.  */
1723
1724 static bool
1725 last_token_is (_cpp_buff *buff, const cpp_token **ptr)
1726 {
1727   return (ptr && tokens_buff_last_token_ptr (buff) == ptr);
1728 }
1729
1730 /* Replace the parameters in a function-like macro of NODE with the
1731    actual ARGS, and place the result in a newly pushed token context.
1732    Expand each argument before replacing, unless it is operated upon
1733    by the # or ## operators. EXPANSION_POINT_LOC is the location of
1734    the expansion point of the macro. E.g, the location of the
1735    function-like macro invocation.  */
1736 static void
1737 replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
1738               macro_arg *args, source_location expansion_point_loc)
1739 {
1740   unsigned int i, total;
1741   const cpp_token *src, *limit;
1742   const cpp_token **first = NULL;
1743   macro_arg *arg;
1744   _cpp_buff *buff = NULL;
1745   source_location *virt_locs = NULL;
1746   unsigned int exp_count;
1747   const line_map_macro *map = NULL;
1748   int track_macro_exp;
1749
1750   /* First, fully macro-expand arguments, calculating the number of
1751      tokens in the final expansion as we go.  The ordering of the if
1752      statements below is subtle; we must handle stringification before
1753      pasting.  */
1754
1755   /* EXP_COUNT is the number of tokens in the macro replacement
1756      list.  TOTAL is the number of tokens /after/ macro parameters
1757      have been replaced by their arguments.   */
1758   exp_count = macro_real_token_count (macro);
1759   total = exp_count;
1760   limit = macro->exp.tokens + exp_count;
1761
1762   for (src = macro->exp.tokens; src < limit; src++)
1763     if (src->type == CPP_MACRO_ARG)
1764       {
1765         /* Leading and trailing padding tokens.  */
1766         total += 2;
1767         /* Account for leading and padding tokens in exp_count too.
1768            This is going to be important later down this function,
1769            when we want to handle the case of (track_macro_exp <
1770            2).  */
1771         exp_count += 2;
1772
1773         /* We have an argument.  If it is not being stringified or
1774            pasted it is macro-replaced before insertion.  */
1775         arg = &args[src->val.macro_arg.arg_no - 1];
1776
1777         if (src->flags & STRINGIFY_ARG)
1778           {
1779             if (!arg->stringified)
1780               arg->stringified = stringify_arg (pfile, arg);
1781           }
1782         else if ((src->flags & PASTE_LEFT)
1783                  || (src > macro->exp.tokens && (src[-1].flags & PASTE_LEFT)))
1784           total += arg->count - 1;
1785         else
1786           {
1787             if (!arg->expanded)
1788               expand_arg (pfile, arg);
1789             total += arg->expanded_count - 1;
1790           }
1791       }
1792
1793   /* When the compiler is called with the -ftrack-macro-expansion
1794      flag, we need to keep track of the location of each token that
1795      results from macro expansion.
1796
1797      A token resulting from macro expansion is not a new token. It is
1798      simply the same token as the token coming from the macro
1799      definition.  The new things that are allocated are the buffer
1800      that holds the tokens resulting from macro expansion and a new
1801      location that records many things like the locus of the expansion
1802      point as well as the original locus inside the definition of the
1803      macro.  This location is called a virtual location.
1804
1805      So the buffer BUFF holds a set of cpp_token*, and the buffer
1806      VIRT_LOCS holds the virtual locations of the tokens held by BUFF.
1807
1808      Both of these two buffers are going to be hung off of the macro
1809      context, when the latter is pushed.  The memory allocated to
1810      store the tokens and their locations is going to be freed once
1811      the context of macro expansion is popped.
1812
1813      As far as tokens are concerned, the memory overhead of
1814      -ftrack-macro-expansion is proportional to the number of
1815      macros that get expanded multiplied by sizeof (source_location).
1816      The good news is that extra memory gets freed when the macro
1817      context is freed, i.e shortly after the macro got expanded.  */
1818
1819   /* Is the -ftrack-macro-expansion flag in effect?  */
1820   track_macro_exp = CPP_OPTION (pfile, track_macro_expansion);
1821
1822   /* Now allocate memory space for tokens and locations resulting from
1823      the macro expansion, copy the tokens and replace the arguments.
1824      This memory must be freed when the context of the macro MACRO is
1825      popped.  */
1826   buff = tokens_buff_new (pfile, total, track_macro_exp ? &virt_locs : NULL);
1827
1828   first = (const cpp_token **) buff->base;
1829
1830   /* Create a macro map to record the locations of the tokens that are
1831      involved in the expansion.  Note that the expansion point is set
1832      to the location of the closing parenthesis.  Otherwise, the
1833      subsequent map created for the first token that comes after the
1834      macro map might have a wrong line number.  That would lead to
1835      tokens with wrong line numbers after the macro expansion.  This
1836      adds up to the memory overhead of the -ftrack-macro-expansion
1837      flag; for every macro that is expanded, a "macro map" is
1838      created.  */
1839   if (track_macro_exp)
1840     {
1841       int num_macro_tokens = total;
1842       if (track_macro_exp < 2)
1843         /* Then the number of macro tokens won't take in account the
1844            fact that function-like macro arguments can expand to
1845            multiple tokens. This is to save memory at the expense of
1846            accuracy.
1847
1848            Suppose we have #define SQUARE(A) A * A
1849
1850            And then we do SQUARE(2+3)
1851
1852            Then the tokens 2, +, 3, will have the same location,
1853            saying they come from the expansion of the argument A.  */
1854         num_macro_tokens = exp_count;
1855       map = linemap_enter_macro (pfile->line_table, node,
1856                                  expansion_point_loc,
1857                                  num_macro_tokens);
1858     }
1859   i = 0;
1860   vaopt_state vaopt_tracker (pfile, macro->variadic,
1861                              args[macro->paramc - 1].count > 0);
1862   const cpp_token **vaopt_start = NULL;
1863   for (src = macro->exp.tokens; src < limit; src++)
1864     {
1865       unsigned int arg_tokens_count;
1866       macro_arg_token_iter from;
1867       const cpp_token **paste_flag = NULL;
1868       const cpp_token **tmp_token_ptr;
1869
1870       /* __VA_OPT__ handling.  */
1871       vaopt_state::update_type vostate = vaopt_tracker.update (src);
1872       if (vostate != vaopt_state::INCLUDE)
1873         {
1874           if (vostate == vaopt_state::BEGIN)
1875             {
1876               /* Padding on the left of __VA_OPT__ (unless RHS of ##).  */
1877               if (src != macro->exp.tokens && !(src[-1].flags & PASTE_LEFT))
1878                 {
1879                   const cpp_token *t = padding_token (pfile, src);
1880                   unsigned index = expanded_token_index (pfile, macro, src, i);
1881                   /* Allocate a virtual location for the padding token and
1882                      append the token and its location to BUFF and
1883                      VIRT_LOCS.   */
1884                   tokens_buff_add_token (buff, virt_locs, t,
1885                                          t->src_loc, t->src_loc,
1886                                          map, index);
1887                 }
1888               vaopt_start = tokens_buff_last_token_ptr (buff);
1889             }
1890           else if (vostate == vaopt_state::END)
1891             {
1892               const cpp_token **start = vaopt_start;
1893               vaopt_start = NULL;
1894
1895               /* Remove any tail padding from inside the __VA_OPT__.  */
1896               paste_flag = tokens_buff_last_token_ptr (buff);
1897               while (paste_flag && paste_flag != start
1898                      && (*paste_flag)->type == CPP_PADDING)
1899                 {
1900                   tokens_buff_remove_last_token (buff);
1901                   paste_flag = tokens_buff_last_token_ptr (buff);
1902                 }
1903
1904               if (src->flags & PASTE_LEFT)
1905                 {
1906                   /* With a non-empty __VA_OPT__ on the LHS of ##, the last
1907                      token should be flagged PASTE_LEFT.  */
1908                   if (paste_flag && (*paste_flag)->type != CPP_PADDING)
1909                     copy_paste_flag (pfile, paste_flag, src);
1910                 }
1911               else
1912                 {
1913                   /* Otherwise, avoid paste on RHS, __VA_OPT__(c)d or
1914                      __VA_OPT__(c)__VA_OPT__(d).  */
1915                   const cpp_token *t = &pfile->avoid_paste;
1916                   tokens_buff_add_token (buff, virt_locs,
1917                                          t, t->src_loc, t->src_loc,
1918                                          NULL, 0);
1919                 }
1920             }
1921           continue;
1922         }
1923
1924       if (src->type != CPP_MACRO_ARG)
1925         {
1926           /* Allocate a virtual location for token SRC, and add that
1927              token and its virtual location into the buffers BUFF and
1928              VIRT_LOCS.  */
1929           unsigned index = expanded_token_index (pfile, macro, src, i);
1930           tokens_buff_add_token (buff, virt_locs, src,
1931                                  src->src_loc, src->src_loc,
1932                                  map, index);
1933           i += 1;
1934           continue;
1935         }
1936
1937       paste_flag = 0;
1938       arg = &args[src->val.macro_arg.arg_no - 1];
1939       /* SRC is a macro parameter that we need to replace with its
1940          corresponding argument.  So at some point we'll need to
1941          iterate over the tokens of the macro argument and copy them
1942          into the "place" now holding the correspondig macro
1943          parameter.  We are going to use the iterator type
1944          macro_argo_token_iter to handle that iterating.  The 'if'
1945          below is to initialize the iterator depending on the type of
1946          tokens the macro argument has.  It also does some adjustment
1947          related to padding tokens and some pasting corner cases.  */
1948       if (src->flags & STRINGIFY_ARG)
1949         {
1950           arg_tokens_count = 1;
1951           macro_arg_token_iter_init (&from,
1952                                      CPP_OPTION (pfile,
1953                                                  track_macro_expansion),
1954                                      MACRO_ARG_TOKEN_STRINGIFIED,
1955                                      arg, &arg->stringified);
1956         }
1957       else if (src->flags & PASTE_LEFT)
1958         {
1959           arg_tokens_count = arg->count;
1960           macro_arg_token_iter_init (&from,
1961                                      CPP_OPTION (pfile,
1962                                                  track_macro_expansion),
1963                                      MACRO_ARG_TOKEN_NORMAL,
1964                                      arg, arg->first);
1965         }
1966       else if (src != macro->exp.tokens && (src[-1].flags & PASTE_LEFT))
1967         {
1968           int num_toks;
1969           arg_tokens_count = arg->count;
1970           macro_arg_token_iter_init (&from,
1971                                      CPP_OPTION (pfile,
1972                                                  track_macro_expansion),
1973                                      MACRO_ARG_TOKEN_NORMAL,
1974                                      arg, arg->first);
1975
1976           num_toks = tokens_buff_count (buff);
1977
1978           if (num_toks != 0)
1979             {
1980               /* So the current parameter token is pasted to the previous
1981                  token in the replacement list.  Let's look at what
1982                  we have as previous and current arguments.  */
1983
1984               /* This is the previous argument's token ...  */
1985               tmp_token_ptr = tokens_buff_last_token_ptr (buff);
1986
1987               if ((*tmp_token_ptr)->type == CPP_COMMA
1988                   && macro->variadic
1989                   && src->val.macro_arg.arg_no == macro->paramc)
1990                 {
1991                   /* ... which is a comma; and the current parameter
1992                      is the last parameter of a variadic function-like
1993                      macro.  If the argument to the current last
1994                      parameter is NULL, then swallow the comma,
1995                      otherwise drop the paste flag.  */
1996                   if (macro_arg_token_iter_get_token (&from) == NULL)
1997                     tokens_buff_remove_last_token (buff);
1998                   else
1999                     paste_flag = tmp_token_ptr;
2000                 }
2001               /* Remove the paste flag if the RHS is a placemarker, unless the
2002                  previous emitted token is at the beginning of __VA_OPT__;
2003                  placemarkers within __VA_OPT__ are ignored in that case.  */
2004               else if (arg_tokens_count == 0
2005                        && tmp_token_ptr != vaopt_start)
2006                 paste_flag = tmp_token_ptr;
2007             }
2008         }
2009       else
2010         {
2011           arg_tokens_count = arg->expanded_count;
2012           macro_arg_token_iter_init (&from,
2013                                      CPP_OPTION (pfile,
2014                                                  track_macro_expansion),
2015                                      MACRO_ARG_TOKEN_EXPANDED,
2016                                      arg, arg->expanded);
2017
2018           if (last_token_is (buff, vaopt_start))
2019             {
2020               /* We're expanding an arg at the beginning of __VA_OPT__.
2021                  Skip padding. */
2022               while (arg_tokens_count)
2023                 {
2024                   const cpp_token *t = macro_arg_token_iter_get_token (&from);
2025                   if (t->type != CPP_PADDING)
2026                     break;
2027                   macro_arg_token_iter_forward (&from);
2028                   --arg_tokens_count;
2029                 }
2030             }
2031         }
2032
2033       /* Padding on the left of an argument (unless RHS of ##).  */
2034       if ((!pfile->state.in_directive || pfile->state.directive_wants_padding)
2035           && src != macro->exp.tokens && !(src[-1].flags & PASTE_LEFT)
2036           && !last_token_is (buff, vaopt_start))
2037         {
2038           const cpp_token *t = padding_token (pfile, src);
2039           unsigned index = expanded_token_index (pfile, macro, src, i);
2040           /* Allocate a virtual location for the padding token and
2041              append the token and its location to BUFF and
2042              VIRT_LOCS.   */
2043           tokens_buff_add_token (buff, virt_locs, t,
2044                                  t->src_loc, t->src_loc,
2045                                  map, index);
2046         }
2047
2048       if (arg_tokens_count)
2049         {
2050           /* So now we've got the number of tokens that make up the
2051              argument that is going to replace the current parameter
2052              in the macro's replacement list.  */
2053           unsigned int j;
2054           for (j = 0; j < arg_tokens_count; ++j)
2055             {
2056               /* So if track_macro_exp is < 2, the user wants to
2057                  save extra memory while tracking macro expansion
2058                  locations.  So in that case here is what we do:
2059
2060                  Suppose we have #define SQUARE(A) A * A
2061
2062                  And then we do SQUARE(2+3)
2063
2064                  Then the tokens 2, +, 3, will have the same location,
2065                  saying they come from the expansion of the argument
2066                  A.
2067
2068               So that means we are going to ignore the COUNT tokens
2069               resulting from the expansion of the current macro
2070               argument. In other words all the ARG_TOKENS_COUNT tokens
2071               resulting from the expansion of the macro argument will
2072               have the index I.  Normally, each of those tokens should
2073               have index I+J.  */
2074               unsigned token_index = i;
2075               unsigned index;
2076               if (track_macro_exp > 1)
2077                 token_index += j;
2078
2079               index = expanded_token_index (pfile, macro, src, token_index);
2080               tokens_buff_add_token (buff, virt_locs,
2081                                      macro_arg_token_iter_get_token (&from),
2082                                      macro_arg_token_iter_get_location (&from),
2083                                      src->src_loc, map, index);
2084               macro_arg_token_iter_forward (&from);
2085             }
2086
2087           /* With a non-empty argument on the LHS of ##, the last
2088              token should be flagged PASTE_LEFT.  */
2089           if (src->flags & PASTE_LEFT)
2090             paste_flag
2091               = (const cpp_token **) tokens_buff_last_token_ptr (buff);
2092         }
2093       else if (CPP_PEDANTIC (pfile) && ! CPP_OPTION (pfile, c99)
2094                && ! macro->syshdr && ! cpp_in_system_header (pfile))
2095         {
2096           if (CPP_OPTION (pfile, cplusplus))
2097             cpp_pedwarning (pfile, CPP_W_PEDANTIC,
2098                             "invoking macro %s argument %d: "
2099                             "empty macro arguments are undefined"
2100                             " in ISO C++98",
2101                             NODE_NAME (node), src->val.macro_arg.arg_no);
2102           else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat))
2103             cpp_pedwarning (pfile,
2104                             CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
2105                             ? CPP_W_C90_C99_COMPAT : CPP_W_PEDANTIC,
2106                             "invoking macro %s argument %d: "
2107                             "empty macro arguments are undefined"
2108                             " in ISO C90",
2109                             NODE_NAME (node), src->val.macro_arg.arg_no);
2110         }
2111       else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
2112                && ! CPP_OPTION (pfile, cplusplus)
2113                && ! macro->syshdr && ! cpp_in_system_header (pfile))
2114         cpp_warning (pfile, CPP_W_C90_C99_COMPAT,
2115                      "invoking macro %s argument %d: "
2116                      "empty macro arguments are undefined"
2117                      " in ISO C90",
2118                      NODE_NAME (node), src->val.macro_arg.arg_no);
2119
2120       /* Avoid paste on RHS (even case count == 0).  */
2121       if (!pfile->state.in_directive && !(src->flags & PASTE_LEFT)
2122           && !last_token_is (buff, vaopt_start))
2123         {
2124           const cpp_token *t = &pfile->avoid_paste;
2125           tokens_buff_add_token (buff, virt_locs,
2126                                  t, t->src_loc, t->src_loc,
2127                                  NULL, 0);
2128         }
2129
2130       /* Add a new paste flag, or remove an unwanted one.  */
2131       if (paste_flag)
2132         copy_paste_flag (pfile, paste_flag, src);
2133
2134       i += arg_tokens_count;
2135     }
2136
2137   if (track_macro_exp)
2138     push_extended_tokens_context (pfile, node, buff, virt_locs, first,
2139                                   tokens_buff_count (buff));
2140   else
2141     push_ptoken_context (pfile, node, buff, first,
2142                          tokens_buff_count (buff));
2143
2144   num_macro_tokens_counter += tokens_buff_count (buff);
2145 }
2146
2147 /* Return a special padding token, with padding inherited from SOURCE.  */
2148 static const cpp_token *
2149 padding_token (cpp_reader *pfile, const cpp_token *source)
2150 {
2151   cpp_token *result = _cpp_temp_token (pfile);
2152
2153   result->type = CPP_PADDING;
2154
2155   /* Data in GCed data structures cannot be made const so far, so we
2156      need a cast here.  */
2157   result->val.source = (cpp_token *) source;
2158   result->flags = 0;
2159   return result;
2160 }
2161
2162 /* Get a new uninitialized context.  Create a new one if we cannot
2163    re-use an old one.  */
2164 static cpp_context *
2165 next_context (cpp_reader *pfile)
2166 {
2167   cpp_context *result = pfile->context->next;
2168
2169   if (result == 0)
2170     {
2171       result = XNEW (cpp_context);
2172       memset (result, 0, sizeof (cpp_context));
2173       result->prev = pfile->context;
2174       result->next = 0;
2175       pfile->context->next = result;
2176     }
2177
2178   pfile->context = result;
2179   return result;
2180 }
2181
2182 /* Push a list of pointers to tokens.  */
2183 static void
2184 push_ptoken_context (cpp_reader *pfile, cpp_hashnode *macro, _cpp_buff *buff,
2185                      const cpp_token **first, unsigned int count)
2186 {
2187   cpp_context *context = next_context (pfile);
2188
2189   context->tokens_kind = TOKENS_KIND_INDIRECT;
2190   context->c.macro = macro;
2191   context->buff = buff;
2192   FIRST (context).ptoken = first;
2193   LAST (context).ptoken = first + count;
2194 }
2195
2196 /* Push a list of tokens.
2197
2198    A NULL macro means that we should continue the current macro
2199    expansion, in essence.  That means that if we are currently in a
2200    macro expansion context, we'll make the new pfile->context refer to
2201    the current macro.  */
2202 void
2203 _cpp_push_token_context (cpp_reader *pfile, cpp_hashnode *macro,
2204                          const cpp_token *first, unsigned int count)
2205 {
2206   cpp_context *context;
2207
2208    if (macro == NULL)
2209      macro = macro_of_context (pfile->context);
2210
2211    context = next_context (pfile);
2212    context->tokens_kind = TOKENS_KIND_DIRECT;
2213    context->c.macro = macro;
2214    context->buff = NULL;
2215    FIRST (context).token = first;
2216    LAST (context).token = first + count;
2217 }
2218
2219 /* Build a context containing a list of tokens as well as their
2220    virtual locations and push it.  TOKENS_BUFF is the buffer that
2221    contains the tokens pointed to by FIRST.  If TOKENS_BUFF is
2222    non-NULL, it means that the context owns it, meaning that
2223    _cpp_pop_context will free it as well as VIRT_LOCS_BUFF that
2224    contains the virtual locations.
2225
2226    A NULL macro means that we should continue the current macro
2227    expansion, in essence.  That means that if we are currently in a
2228    macro expansion context, we'll make the new pfile->context refer to
2229    the current macro.  */
2230 static void
2231 push_extended_tokens_context (cpp_reader *pfile,
2232                               cpp_hashnode *macro,
2233                               _cpp_buff *token_buff,
2234                               source_location *virt_locs,
2235                               const cpp_token **first,
2236                               unsigned int count)
2237 {
2238   cpp_context *context;
2239   macro_context *m;
2240
2241   if (macro == NULL)
2242     macro = macro_of_context (pfile->context);
2243
2244   context = next_context (pfile);
2245   context->tokens_kind = TOKENS_KIND_EXTENDED;
2246   context->buff = token_buff;
2247
2248   m = XNEW (macro_context);
2249   m->macro_node = macro;
2250   m->virt_locs = virt_locs;
2251   m->cur_virt_loc = virt_locs;
2252   context->c.mc = m;
2253   FIRST (context).ptoken = first;
2254   LAST (context).ptoken = first + count;
2255 }
2256
2257 /* Push a traditional macro's replacement text.  */
2258 void
2259 _cpp_push_text_context (cpp_reader *pfile, cpp_hashnode *macro,
2260                         const uchar *start, size_t len)
2261 {
2262   cpp_context *context = next_context (pfile);
2263
2264   context->tokens_kind = TOKENS_KIND_DIRECT;
2265   context->c.macro = macro;
2266   context->buff = NULL;
2267   CUR (context) = start;
2268   RLIMIT (context) = start + len;
2269   macro->flags |= NODE_DISABLED;
2270 }
2271
2272 /* Creates a buffer that holds tokens a.k.a "token buffer", usually
2273    for the purpose of storing them on a cpp_context. If VIRT_LOCS is
2274    non-null (which means that -ftrack-macro-expansion is on),
2275    *VIRT_LOCS is set to a newly allocated buffer that is supposed to
2276    hold the virtual locations of the tokens resulting from macro
2277    expansion.  */
2278 static _cpp_buff*
2279 tokens_buff_new (cpp_reader *pfile, size_t len,
2280                  source_location **virt_locs)
2281 {
2282   size_t tokens_size = len * sizeof (cpp_token *);
2283   size_t locs_size = len * sizeof (source_location);
2284
2285   if (virt_locs != NULL)
2286     *virt_locs = XNEWVEC (source_location, locs_size);
2287   return _cpp_get_buff (pfile, tokens_size);
2288 }
2289
2290 /* Returns the number of tokens contained in a token buffer.  The
2291    buffer holds a set of cpp_token*.  */
2292 static size_t
2293 tokens_buff_count (_cpp_buff *buff)
2294 {
2295   return (BUFF_FRONT (buff) - buff->base) / sizeof (cpp_token *);
2296 }
2297
2298 /* Return a pointer to the last token contained in the token buffer
2299    BUFF.  */
2300 static const cpp_token **
2301 tokens_buff_last_token_ptr (_cpp_buff *buff)
2302 {
2303   if (BUFF_FRONT (buff) == buff->base)
2304     return NULL;
2305   return &((const cpp_token **) BUFF_FRONT (buff))[-1];
2306 }
2307
2308 /* Remove the last token contained in the token buffer TOKENS_BUFF.
2309    If VIRT_LOCS_BUFF is non-NULL,  it should point at the buffer
2310    containing the virtual locations of the tokens in TOKENS_BUFF; in
2311    which case the function updates that buffer as well.   */
2312 static inline void
2313 tokens_buff_remove_last_token (_cpp_buff *tokens_buff)
2314
2315 {
2316   if (BUFF_FRONT (tokens_buff) > tokens_buff->base)
2317     BUFF_FRONT (tokens_buff) =
2318       (unsigned char *) &((cpp_token **) BUFF_FRONT (tokens_buff))[-1];
2319 }
2320
2321 /* Insert a token into the token buffer at the position pointed to by
2322    DEST.  Note that the buffer is not enlarged so the previous token
2323    that was at *DEST is overwritten.  VIRT_LOC_DEST, if non-null,
2324    means -ftrack-macro-expansion is effect; it then points to where to
2325    insert the virtual location of TOKEN.  TOKEN is the token to
2326    insert.  VIRT_LOC is the virtual location of the token, i.e, the
2327    location possibly encoding its locus across macro expansion.  If
2328    TOKEN is an argument of a function-like macro (inside a macro
2329    replacement list), PARM_DEF_LOC is the spelling location of the
2330    macro parameter that TOKEN is replacing, in the replacement list of
2331    the macro.  If TOKEN is not an argument of a function-like macro or
2332    if it doesn't come from a macro expansion, then VIRT_LOC can just
2333    be set to the same value as PARM_DEF_LOC.  If MAP is non null, it
2334    means TOKEN comes from a macro expansion and MAP is the macro map
2335    associated to the macro.  MACRO_TOKEN_INDEX points to the index of
2336    the token in the macro map; it is not considered if MAP is NULL.
2337
2338    Upon successful completion this function returns the a pointer to
2339    the position of the token coming right after the insertion
2340    point.  */
2341 static inline const cpp_token **
2342 tokens_buff_put_token_to (const cpp_token **dest,
2343                           source_location *virt_loc_dest,
2344                           const cpp_token *token,
2345                           source_location virt_loc,
2346                           source_location parm_def_loc,
2347                           const line_map_macro *map,
2348                           unsigned int macro_token_index)
2349 {
2350   source_location macro_loc = virt_loc;
2351   const cpp_token **result;
2352
2353   if (virt_loc_dest)
2354     {
2355       /* -ftrack-macro-expansion is on.  */
2356       if (map)
2357         macro_loc = linemap_add_macro_token (map, macro_token_index,
2358                                              virt_loc, parm_def_loc);
2359       *virt_loc_dest = macro_loc;
2360     }
2361   *dest = token;
2362   result = &dest[1];
2363
2364   return result;
2365 }
2366
2367 /* Adds a token at the end of the tokens contained in BUFFER.  Note
2368    that this function doesn't enlarge BUFFER when the number of tokens
2369    reaches BUFFER's size; it aborts in that situation.
2370
2371    TOKEN is the token to append. VIRT_LOC is the virtual location of
2372    the token, i.e, the location possibly encoding its locus across
2373    macro expansion. If TOKEN is an argument of a function-like macro
2374    (inside a macro replacement list), PARM_DEF_LOC is the location of
2375    the macro parameter that TOKEN is replacing.  If TOKEN doesn't come
2376    from a macro expansion, then VIRT_LOC can just be set to the same
2377    value as PARM_DEF_LOC.  If MAP is non null, it means TOKEN comes
2378    from a macro expansion and MAP is the macro map associated to the
2379    macro.  MACRO_TOKEN_INDEX points to the index of the token in the
2380    macro map; It is not considered if MAP is NULL.  If VIRT_LOCS is
2381    non-null, it means -ftrack-macro-expansion is on; in which case
2382    this function adds the virtual location DEF_LOC to the VIRT_LOCS
2383    array, at the same index as the one of TOKEN in BUFFER.  Upon
2384    successful completion this function returns the a pointer to the
2385    position of the token coming right after the insertion point.  */
2386 static const cpp_token **
2387 tokens_buff_add_token (_cpp_buff *buffer,
2388                        source_location *virt_locs,
2389                        const cpp_token *token,
2390                        source_location virt_loc,
2391                        source_location parm_def_loc,
2392                        const line_map_macro *map,
2393                        unsigned int macro_token_index)
2394 {
2395   const cpp_token **result;
2396   source_location *virt_loc_dest = NULL;
2397   unsigned token_index =
2398     (BUFF_FRONT (buffer) - buffer->base) / sizeof (cpp_token *);
2399
2400   /* Abort if we pass the end the buffer.  */
2401   if (BUFF_FRONT (buffer) > BUFF_LIMIT (buffer))
2402     abort ();
2403
2404   if (virt_locs != NULL)
2405     virt_loc_dest = &virt_locs[token_index];
2406
2407   result =
2408     tokens_buff_put_token_to ((const cpp_token **) BUFF_FRONT (buffer),
2409                               virt_loc_dest, token, virt_loc, parm_def_loc,
2410                               map, macro_token_index);
2411
2412   BUFF_FRONT (buffer) = (unsigned char *) result;
2413   return result;
2414 }
2415
2416 /* Allocate space for the function-like macro argument ARG to store
2417    the tokens resulting from the macro-expansion of the tokens that
2418    make up ARG itself. That space is allocated in ARG->expanded and
2419    needs to be freed using free.  */
2420 static void
2421 alloc_expanded_arg_mem (cpp_reader *pfile, macro_arg *arg, size_t capacity)
2422 {
2423   gcc_checking_assert (arg->expanded == NULL
2424                        && arg->expanded_virt_locs == NULL);
2425
2426   arg->expanded = XNEWVEC (const cpp_token *, capacity);
2427   if (CPP_OPTION (pfile, track_macro_expansion))
2428     arg->expanded_virt_locs = XNEWVEC (source_location, capacity);
2429
2430 }
2431
2432 /* If necessary, enlarge ARG->expanded to so that it can contain SIZE
2433    tokens.  */
2434 static void
2435 ensure_expanded_arg_room (cpp_reader *pfile, macro_arg *arg,
2436                           size_t size, size_t *expanded_capacity)
2437 {
2438   if (size <= *expanded_capacity)
2439     return;
2440
2441   size *= 2;
2442
2443   arg->expanded =
2444     XRESIZEVEC (const cpp_token *, arg->expanded, size);
2445   *expanded_capacity = size;
2446
2447   if (CPP_OPTION (pfile, track_macro_expansion))
2448     {
2449       if (arg->expanded_virt_locs == NULL)
2450         arg->expanded_virt_locs = XNEWVEC (source_location, size);
2451       else
2452         arg->expanded_virt_locs = XRESIZEVEC (source_location,
2453                                               arg->expanded_virt_locs,
2454                                               size);
2455     }
2456 }
2457
2458 /* Expand an argument ARG before replacing parameters in a
2459    function-like macro.  This works by pushing a context with the
2460    argument's tokens, and then expanding that into a temporary buffer
2461    as if it were a normal part of the token stream.  collect_args()
2462    has terminated the argument's tokens with a CPP_EOF so that we know
2463    when we have fully expanded the argument.  */
2464 static void
2465 expand_arg (cpp_reader *pfile, macro_arg *arg)
2466 {
2467   size_t capacity;
2468   bool saved_warn_trad;
2469   bool track_macro_exp_p = CPP_OPTION (pfile, track_macro_expansion);
2470
2471   if (arg->count == 0
2472       || arg->expanded != NULL)
2473     return;
2474
2475   /* Don't warn about funlike macros when pre-expanding.  */
2476   saved_warn_trad = CPP_WTRADITIONAL (pfile);
2477   CPP_WTRADITIONAL (pfile) = 0;
2478
2479   /* Loop, reading in the tokens of the argument.  */
2480   capacity = 256;
2481   alloc_expanded_arg_mem (pfile, arg, capacity);
2482
2483   if (track_macro_exp_p)
2484     push_extended_tokens_context (pfile, NULL, NULL,
2485                                   arg->virt_locs,
2486                                   arg->first,
2487                                   arg->count + 1);
2488   else
2489     push_ptoken_context (pfile, NULL, NULL,
2490                          arg->first, arg->count + 1);
2491
2492   for (;;)
2493     {
2494       const cpp_token *token;
2495       source_location location;
2496
2497       ensure_expanded_arg_room (pfile, arg, arg->expanded_count + 1,
2498                                 &capacity);
2499
2500       token = cpp_get_token_1 (pfile, &location);
2501
2502       if (token->type == CPP_EOF)
2503         break;
2504
2505       set_arg_token (arg, token, location,
2506                      arg->expanded_count, MACRO_ARG_TOKEN_EXPANDED,
2507                      CPP_OPTION (pfile, track_macro_expansion));
2508       arg->expanded_count++;
2509     }
2510
2511   _cpp_pop_context (pfile);
2512
2513   CPP_WTRADITIONAL (pfile) = saved_warn_trad;
2514 }
2515
2516 /* Returns the macro associated to the current context if we are in
2517    the context a macro expansion, NULL otherwise.  */
2518 static cpp_hashnode*
2519 macro_of_context (cpp_context *context)
2520 {
2521   if (context == NULL)
2522     return NULL;
2523
2524   return (context->tokens_kind == TOKENS_KIND_EXTENDED)
2525     ? context->c.mc->macro_node
2526     : context->c.macro;
2527 }
2528
2529 /* Return TRUE iff we are expanding a macro or are about to start
2530    expanding one.  If we are effectively expanding a macro, the
2531    function macro_of_context returns a pointer to the macro being
2532    expanded.  */
2533 static bool
2534 in_macro_expansion_p (cpp_reader *pfile)
2535 {
2536   if (pfile == NULL)
2537     return false;
2538
2539   return (pfile->about_to_expand_macro_p
2540           || macro_of_context (pfile->context));
2541 }
2542
2543 /* Pop the current context off the stack, re-enabling the macro if the
2544    context represented a macro's replacement list.  Initially the
2545    context structure was not freed so that we can re-use it later, but
2546    now we do free it to reduce peak memory consumption.  */
2547 void
2548 _cpp_pop_context (cpp_reader *pfile)
2549 {
2550   cpp_context *context = pfile->context;
2551
2552   /* We should not be popping the base context.  */
2553   if (context == &pfile->base_context)
2554     abort ();
2555
2556   if (context->c.macro)
2557     {
2558       cpp_hashnode *macro;
2559       if (context->tokens_kind == TOKENS_KIND_EXTENDED)
2560         {
2561           macro_context *mc = context->c.mc;
2562           macro = mc->macro_node;
2563           /* If context->buff is set, it means the life time of tokens
2564              is bound to the life time of this context; so we must
2565              free the tokens; that means we must free the virtual
2566              locations of these tokens too.  */
2567           if (context->buff && mc->virt_locs)
2568             {
2569               free (mc->virt_locs);
2570               mc->virt_locs = NULL;
2571             }
2572           free (mc);
2573           context->c.mc = NULL;
2574         }
2575       else
2576         macro = context->c.macro;
2577
2578       /* Beware that MACRO can be NULL in cases like when we are
2579          called from expand_arg.  In those cases, a dummy context with
2580          tokens is pushed just for the purpose of walking them using
2581          cpp_get_token_1.  In that case, no 'macro' field is set into
2582          the dummy context.  */
2583       if (macro != NULL
2584           /* Several contiguous macro expansion contexts can be
2585              associated to the same macro; that means it's the same
2586              macro expansion that spans across all these (sub)
2587              contexts.  So we should re-enable an expansion-disabled
2588              macro only when we are sure we are really out of that
2589              macro expansion.  */
2590           && macro_of_context (context->prev) != macro)
2591         macro->flags &= ~NODE_DISABLED;
2592
2593       if (macro == pfile->top_most_macro_node && context->prev == NULL)
2594         /* We are popping the context of the top-most macro node.  */
2595         pfile->top_most_macro_node = NULL;
2596     }
2597
2598   if (context->buff)
2599     {
2600       /* Decrease memory peak consumption by freeing the memory used
2601          by the context.  */
2602       _cpp_free_buff (context->buff);
2603     }
2604
2605   pfile->context = context->prev;
2606   /* decrease peak memory consumption by feeing the context.  */
2607   pfile->context->next = NULL;
2608   free (context);
2609 }
2610
2611 /* Return TRUE if we reached the end of the set of tokens stored in
2612    CONTEXT, FALSE otherwise.  */
2613 static inline bool
2614 reached_end_of_context (cpp_context *context)
2615 {
2616   if (context->tokens_kind == TOKENS_KIND_DIRECT)
2617       return FIRST (context).token == LAST (context).token;
2618   else if (context->tokens_kind == TOKENS_KIND_INDIRECT
2619            || context->tokens_kind == TOKENS_KIND_EXTENDED)
2620     return FIRST (context).ptoken == LAST (context).ptoken;
2621   else
2622     abort ();
2623 }
2624
2625 /* Consume the next token contained in the current context of PFILE,
2626    and return it in *TOKEN. It's "full location" is returned in
2627    *LOCATION. If -ftrack-macro-location is in effeect, fFull location"
2628    means the location encoding the locus of the token across macro
2629    expansion; otherwise it's just is the "normal" location of the
2630    token which (*TOKEN)->src_loc.  */
2631 static inline void
2632 consume_next_token_from_context (cpp_reader *pfile,
2633                                  const cpp_token ** token,
2634                                  source_location *location)
2635 {
2636   cpp_context *c = pfile->context;
2637
2638   if ((c)->tokens_kind == TOKENS_KIND_DIRECT)
2639     {
2640       *token = FIRST (c).token;
2641       *location = (*token)->src_loc;
2642       FIRST (c).token++;
2643     }
2644   else if ((c)->tokens_kind == TOKENS_KIND_INDIRECT)
2645     {
2646       *token = *FIRST (c).ptoken;
2647       *location = (*token)->src_loc;
2648       FIRST (c).ptoken++;
2649     }
2650   else if ((c)->tokens_kind == TOKENS_KIND_EXTENDED)
2651     {
2652       macro_context *m = c->c.mc;
2653       *token = *FIRST (c).ptoken;
2654       if (m->virt_locs)
2655         {
2656           *location = *m->cur_virt_loc;
2657           m->cur_virt_loc++;
2658         }
2659       else
2660         *location = (*token)->src_loc;
2661       FIRST (c).ptoken++;
2662     }
2663   else
2664     abort ();
2665 }
2666
2667 /* In the traditional mode of the preprocessor, if we are currently in
2668    a directive, the location of a token must be the location of the
2669    start of the directive line.  This function returns the proper
2670    location if we are in the traditional mode, and just returns
2671    LOCATION otherwise.  */
2672
2673 static inline source_location
2674 maybe_adjust_loc_for_trad_cpp (cpp_reader *pfile, source_location location)
2675 {
2676   if (CPP_OPTION (pfile, traditional))
2677     {
2678       if (pfile->state.in_directive)
2679         return pfile->directive_line;
2680     }
2681   return location;
2682 }
2683
2684 /* Routine to get a token as well as its location.
2685
2686    Macro expansions and directives are transparently handled,
2687    including entering included files.  Thus tokens are post-macro
2688    expansion, and after any intervening directives.  External callers
2689    see CPP_EOF only at EOF.  Internal callers also see it when meeting
2690    a directive inside a macro call, when at the end of a directive and
2691    state.in_directive is still 1, and at the end of argument
2692    pre-expansion.
2693
2694    LOC is an out parameter; *LOC is set to the location "as expected
2695    by the user".  Please read the comment of
2696    cpp_get_token_with_location to learn more about the meaning of this
2697    location.  */
2698 static const cpp_token*
2699 cpp_get_token_1 (cpp_reader *pfile, source_location *location)
2700 {
2701   const cpp_token *result;
2702   /* This token is a virtual token that either encodes a location
2703      related to macro expansion or a spelling location.  */
2704   source_location virt_loc = 0;
2705   /* pfile->about_to_expand_macro_p can be overriden by indirect calls
2706      to functions that push macro contexts.  So let's save it so that
2707      we can restore it when we are about to leave this routine.  */
2708   bool saved_about_to_expand_macro = pfile->about_to_expand_macro_p;
2709
2710   for (;;)
2711     {
2712       cpp_hashnode *node;
2713       cpp_context *context = pfile->context;
2714
2715       /* Context->prev == 0 <=> base context.  */
2716       if (!context->prev)
2717         {
2718           result = _cpp_lex_token (pfile);
2719           virt_loc = result->src_loc;
2720         }
2721       else if (!reached_end_of_context (context))
2722         {
2723           consume_next_token_from_context (pfile, &result,
2724                                            &virt_loc);
2725           if (result->flags & PASTE_LEFT)
2726             {
2727               paste_all_tokens (pfile, result);
2728               if (pfile->state.in_directive)
2729                 continue;
2730               result = padding_token (pfile, result);
2731               goto out;
2732             }
2733         }
2734       else
2735         {
2736           if (pfile->context->c.macro)
2737             ++num_expanded_macros_counter;
2738           _cpp_pop_context (pfile);
2739           if (pfile->state.in_directive)
2740             continue;
2741           result = &pfile->avoid_paste;
2742           goto out;
2743         }
2744
2745       if (pfile->state.in_directive && result->type == CPP_COMMENT)
2746         continue;
2747
2748       if (result->type != CPP_NAME)
2749         break;
2750
2751       node = result->val.node.node;
2752
2753       if (node->type != NT_MACRO || (result->flags & NO_EXPAND))
2754         break;
2755
2756       if (!(node->flags & NODE_DISABLED))
2757         {
2758           int ret = 0;
2759           /* If not in a macro context, and we're going to start an
2760              expansion, record the location and the top level macro
2761              about to be expanded.  */
2762           if (!in_macro_expansion_p (pfile))
2763             {
2764               pfile->invocation_location = result->src_loc;
2765               pfile->top_most_macro_node = node;
2766             }
2767           if (pfile->state.prevent_expansion)
2768             break;
2769
2770           /* Conditional macros require that a predicate be evaluated
2771              first.  */
2772           if ((node->flags & NODE_CONDITIONAL) != 0)
2773             {
2774               if (pfile->cb.macro_to_expand)
2775                 {
2776                   bool whitespace_after;
2777                   const cpp_token *peek_tok = cpp_peek_token (pfile, 0);
2778
2779                   whitespace_after = (peek_tok->type == CPP_PADDING
2780                                       || (peek_tok->flags & PREV_WHITE));
2781                   node = pfile->cb.macro_to_expand (pfile, result);
2782                   if (node)
2783                     ret = enter_macro_context (pfile, node, result,
2784                                                virt_loc);
2785                   else if (whitespace_after)
2786                     {
2787                       /* If macro_to_expand hook returned NULL and it
2788                          ate some tokens, see if we don't need to add
2789                          a padding token in between this and the
2790                          next token.  */
2791                       peek_tok = cpp_peek_token (pfile, 0);
2792                       if (peek_tok->type != CPP_PADDING
2793                           && (peek_tok->flags & PREV_WHITE) == 0)
2794                         _cpp_push_token_context (pfile, NULL,
2795                                                  padding_token (pfile,
2796                                                                 peek_tok), 1);
2797                     }
2798                 }
2799             }
2800           else
2801             ret = enter_macro_context (pfile, node, result,
2802                                        virt_loc);
2803           if (ret)
2804             {
2805               if (pfile->state.in_directive || ret == 2)
2806                 continue;
2807               result = padding_token (pfile, result);
2808               goto out;
2809             }
2810         }
2811       else
2812         {
2813           /* Flag this token as always unexpandable.  FIXME: move this
2814              to collect_args()?.  */
2815           cpp_token *t = _cpp_temp_token (pfile);
2816           t->type = result->type;
2817           t->flags = result->flags | NO_EXPAND;
2818           t->val = result->val;
2819           result = t;
2820         }
2821
2822       break;
2823     }
2824
2825  out:
2826   if (location != NULL)
2827     {
2828       if (virt_loc == 0)
2829         virt_loc = result->src_loc;
2830       *location = virt_loc;
2831
2832       if (!CPP_OPTION (pfile, track_macro_expansion)
2833           && macro_of_context (pfile->context) != NULL)
2834         /* We are in a macro expansion context, are not tracking
2835            virtual location, but were asked to report the location
2836            of the expansion point of the macro being expanded.  */
2837         *location = pfile->invocation_location;
2838
2839       *location = maybe_adjust_loc_for_trad_cpp (pfile, *location);
2840     }
2841
2842   pfile->about_to_expand_macro_p = saved_about_to_expand_macro;
2843   return result;
2844 }
2845
2846 /* External routine to get a token.  Also used nearly everywhere
2847    internally, except for places where we know we can safely call
2848    _cpp_lex_token directly, such as lexing a directive name.
2849
2850    Macro expansions and directives are transparently handled,
2851    including entering included files.  Thus tokens are post-macro
2852    expansion, and after any intervening directives.  External callers
2853    see CPP_EOF only at EOF.  Internal callers also see it when meeting
2854    a directive inside a macro call, when at the end of a directive and
2855    state.in_directive is still 1, and at the end of argument
2856    pre-expansion.  */
2857 const cpp_token *
2858 cpp_get_token (cpp_reader *pfile)
2859 {
2860   return cpp_get_token_1 (pfile, NULL);
2861 }
2862
2863 /* Like cpp_get_token, but also returns a virtual token location
2864    separate from the spelling location carried by the returned token.
2865
2866    LOC is an out parameter; *LOC is set to the location "as expected
2867    by the user".  This matters when a token results from macro
2868    expansion; in that case the token's spelling location indicates the
2869    locus of the token in the definition of the macro but *LOC
2870    virtually encodes all the other meaningful locuses associated to
2871    the token.
2872
2873    What? virtual location? Yes, virtual location.
2874
2875    If the token results from macro expansion and if macro expansion
2876    location tracking is enabled its virtual location encodes (at the
2877    same time):
2878
2879    - the spelling location of the token
2880
2881    - the locus of the macro expansion point
2882
2883    - the locus of the point where the token got instantiated as part
2884      of the macro expansion process.
2885
2886    You have to use the linemap API to get the locus you are interested
2887    in from a given virtual location.
2888
2889    Note however that virtual locations are not necessarily ordered for
2890    relations '<' and '>'.  One must use the function
2891    linemap_location_before_p instead of using the relational operator
2892    '<'.
2893
2894    If macro expansion tracking is off and if the token results from
2895    macro expansion the virtual location is the expansion point of the
2896    macro that got expanded.
2897
2898    When the token doesn't result from macro expansion, the virtual
2899    location is just the same thing as its spelling location.  */
2900
2901 const cpp_token *
2902 cpp_get_token_with_location (cpp_reader *pfile, source_location *loc)
2903 {
2904   return cpp_get_token_1 (pfile, loc);
2905 }
2906
2907 /* Returns true if we're expanding an object-like macro that was
2908    defined in a system header.  Just checks the macro at the top of
2909    the stack.  Used for diagnostic suppression.  */
2910 int
2911 cpp_sys_macro_p (cpp_reader *pfile)
2912 {
2913   cpp_hashnode *node = NULL;
2914
2915   if (pfile->context->tokens_kind == TOKENS_KIND_EXTENDED)
2916     node = pfile->context->c.mc->macro_node;
2917   else
2918     node = pfile->context->c.macro;
2919
2920   return node && node->value.macro && node->value.macro->syshdr;
2921 }
2922
2923 /* Read each token in, until end of the current file.  Directives are
2924    transparently processed.  */
2925 void
2926 cpp_scan_nooutput (cpp_reader *pfile)
2927 {
2928   /* Request a CPP_EOF token at the end of this file, rather than
2929      transparently continuing with the including file.  */
2930   pfile->buffer->return_at_eof = true;
2931
2932   pfile->state.discarding_output++;
2933   pfile->state.prevent_expansion++;
2934
2935   if (CPP_OPTION (pfile, traditional))
2936     while (_cpp_read_logical_line_trad (pfile))
2937       ;
2938   else
2939     while (cpp_get_token (pfile)->type != CPP_EOF)
2940       ;
2941
2942   pfile->state.discarding_output--;
2943   pfile->state.prevent_expansion--;
2944 }
2945
2946 /* Step back one or more tokens obtained from the lexer.  */
2947 void
2948 _cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)
2949 {
2950   pfile->lookaheads += count;
2951   while (count--)
2952     {
2953       pfile->cur_token--;
2954       if (pfile->cur_token == pfile->cur_run->base
2955           /* Possible with -fpreprocessed and no leading #line.  */
2956           && pfile->cur_run->prev != NULL)
2957         {
2958           pfile->cur_run = pfile->cur_run->prev;
2959           pfile->cur_token = pfile->cur_run->limit;
2960         }
2961     }
2962 }
2963
2964 /* Step back one (or more) tokens.  Can only step back more than 1 if
2965    they are from the lexer, and not from macro expansion.  */
2966 void
2967 _cpp_backup_tokens (cpp_reader *pfile, unsigned int count)
2968 {
2969   if (pfile->context->prev == NULL)
2970     _cpp_backup_tokens_direct (pfile, count);
2971   else
2972     {
2973       if (count != 1)
2974         abort ();
2975       if (pfile->context->tokens_kind == TOKENS_KIND_DIRECT)
2976         FIRST (pfile->context).token--;
2977       else if (pfile->context->tokens_kind == TOKENS_KIND_INDIRECT)
2978         FIRST (pfile->context).ptoken--;
2979       else if (pfile->context->tokens_kind == TOKENS_KIND_EXTENDED)
2980         {
2981           FIRST (pfile->context).ptoken--;
2982           if (pfile->context->c.macro)
2983             {
2984               macro_context *m = pfile->context->c.mc;
2985               m->cur_virt_loc--;
2986               gcc_checking_assert (m->cur_virt_loc >= m->virt_locs);
2987             }
2988           else
2989             abort ();
2990         }
2991       else
2992         abort ();
2993     }
2994 }
2995
2996 /* #define directive parsing and handling.  */
2997
2998 /* Returns nonzero if a macro redefinition warning is required.  */
2999 static bool
3000 warn_of_redefinition (cpp_reader *pfile, cpp_hashnode *node,
3001                       const cpp_macro *macro2)
3002 {
3003   const cpp_macro *macro1;
3004   unsigned int i;
3005
3006   /* Some redefinitions need to be warned about regardless.  */
3007   if (node->flags & NODE_WARN)
3008     return true;
3009
3010   /* Suppress warnings for builtins that lack the NODE_WARN flag,
3011      unless Wbuiltin-macro-redefined.  */
3012   if (node->flags & NODE_BUILTIN
3013       && (!pfile->cb.user_builtin_macro
3014           || !pfile->cb.user_builtin_macro (pfile, node)))
3015     return CPP_OPTION (pfile, warn_builtin_macro_redefined);
3016
3017   /* Redefinitions of conditional (context-sensitive) macros, on
3018      the other hand, must be allowed silently.  */
3019   if (node->flags & NODE_CONDITIONAL)
3020     return false;
3021
3022   /* Redefinition of a macro is allowed if and only if the old and new
3023      definitions are the same.  (6.10.3 paragraph 2).  */
3024   macro1 = node->value.macro;
3025
3026   /* Don't check count here as it can be different in valid
3027      traditional redefinitions with just whitespace differences.  */
3028   if (macro1->paramc != macro2->paramc
3029       || macro1->fun_like != macro2->fun_like
3030       || macro1->variadic != macro2->variadic)
3031     return true;
3032
3033   /* Check parameter spellings.  */
3034   for (i = 0; i < macro1->paramc; i++)
3035     if (macro1->params[i] != macro2->params[i])
3036       return true;
3037
3038   /* Check the replacement text or tokens.  */
3039   if (CPP_OPTION (pfile, traditional))
3040     return _cpp_expansions_different_trad (macro1, macro2);
3041
3042   if (macro1->count != macro2->count)
3043     return true;
3044
3045   for (i = 0; i < macro1->count; i++)
3046     if (!_cpp_equiv_tokens (&macro1->exp.tokens[i], &macro2->exp.tokens[i]))
3047       return true;
3048
3049   return false;
3050 }
3051
3052 /* Free the definition of hashnode H.  */
3053 void
3054 _cpp_free_definition (cpp_hashnode *h)
3055 {
3056   /* Macros and assertions no longer have anything to free.  */
3057   h->type = NT_VOID;
3058   /* Clear builtin flag in case of redefinition.  */
3059   h->flags &= ~(NODE_BUILTIN | NODE_DISABLED | NODE_USED);
3060 }
3061
3062 /* Save parameter NODE (spelling SPELLING) to the parameter list of
3063    macro MACRO.  Returns zero on success, nonzero if the parameter is
3064    a duplicate.  */
3065 bool
3066 _cpp_save_parameter (cpp_reader *pfile, cpp_macro *macro, cpp_hashnode *node,
3067                      cpp_hashnode *spelling)
3068 {
3069   unsigned int len;
3070   /* Constraint 6.10.3.6 - duplicate parameter names.  */
3071   if (node->flags & NODE_MACRO_ARG)
3072     {
3073       cpp_error (pfile, CPP_DL_ERROR, "duplicate macro parameter \"%s\"",
3074                  NODE_NAME (node));
3075       return true;
3076     }
3077
3078   if (BUFF_ROOM (pfile->a_buff)
3079       < (macro->paramc + 1) * sizeof (cpp_hashnode *))
3080     _cpp_extend_buff (pfile, &pfile->a_buff, sizeof (cpp_hashnode *));
3081
3082   ((cpp_hashnode **) BUFF_FRONT (pfile->a_buff))[macro->paramc++] = spelling;
3083   node->flags |= NODE_MACRO_ARG;
3084   len = macro->paramc * sizeof (struct macro_arg_saved_data);
3085   if (len > pfile->macro_buffer_len)
3086     {
3087       pfile->macro_buffer = XRESIZEVEC (unsigned char, pfile->macro_buffer,
3088                                         len);
3089       pfile->macro_buffer_len = len;
3090     }
3091   struct macro_arg_saved_data save;
3092   save.value = node->value;
3093   save.canonical_node = node;
3094   ((struct macro_arg_saved_data *) pfile->macro_buffer)[macro->paramc - 1]
3095     = save;
3096
3097   node->value.arg_index  = macro->paramc;
3098   return false;
3099 }
3100
3101 /* Check the syntax of the parameters in a MACRO definition.  Returns
3102    false if an error occurs.  */
3103 static bool
3104 parse_params (cpp_reader *pfile, cpp_macro *macro)
3105 {
3106   unsigned int prev_ident = 0;
3107
3108   for (;;)
3109     {
3110       const cpp_token *token = _cpp_lex_token (pfile);
3111
3112       switch (token->type)
3113         {
3114         default:
3115           /* Allow/ignore comments in parameter lists if we are
3116              preserving comments in macro expansions.  */
3117           if (token->type == CPP_COMMENT
3118               && ! CPP_OPTION (pfile, discard_comments_in_macro_exp))
3119             continue;
3120
3121           cpp_error (pfile, CPP_DL_ERROR,
3122                      "\"%s\" may not appear in macro parameter list",
3123                      cpp_token_as_text (pfile, token));
3124           return false;
3125
3126         case CPP_NAME:
3127           if (prev_ident)
3128             {
3129               cpp_error (pfile, CPP_DL_ERROR,
3130                          "macro parameters must be comma-separated");
3131               return false;
3132             }
3133           prev_ident = 1;
3134
3135           if (_cpp_save_parameter (pfile, macro, token->val.node.node,
3136                                    token->val.node.spelling))
3137             return false;
3138           continue;
3139
3140         case CPP_CLOSE_PAREN:
3141           if (prev_ident || macro->paramc == 0)
3142             return true;
3143
3144           /* Fall through to pick up the error.  */
3145           /* FALLTHRU */
3146         case CPP_COMMA:
3147           if (!prev_ident)
3148             {
3149               cpp_error (pfile, CPP_DL_ERROR, "parameter name missing");
3150               return false;
3151             }
3152           prev_ident = 0;
3153           continue;
3154
3155         case CPP_ELLIPSIS:
3156           macro->variadic = 1;
3157           if (!prev_ident)
3158             {
3159               _cpp_save_parameter (pfile, macro,
3160                                    pfile->spec_nodes.n__VA_ARGS__,
3161                                    pfile->spec_nodes.n__VA_ARGS__);
3162               pfile->state.va_args_ok = 1;
3163               if (! CPP_OPTION (pfile, c99)
3164                   && CPP_OPTION (pfile, cpp_pedantic)
3165                   && CPP_OPTION (pfile, warn_variadic_macros))
3166                 {
3167                   if (CPP_OPTION (pfile, cplusplus))
3168                     cpp_pedwarning
3169                         (pfile, CPP_W_VARIADIC_MACROS,
3170                         "anonymous variadic macros were introduced in C++11");
3171                   else
3172                     cpp_pedwarning
3173                         (pfile, CPP_W_VARIADIC_MACROS,
3174                         "anonymous variadic macros were introduced in C99");
3175                 }
3176               else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
3177                        && ! CPP_OPTION (pfile, cplusplus))
3178                 cpp_error (pfile, CPP_DL_WARNING,
3179                            "anonymous variadic macros were introduced in C99");
3180             }
3181           else if (CPP_OPTION (pfile, cpp_pedantic)
3182                    && CPP_OPTION (pfile, warn_variadic_macros))
3183             {
3184               if (CPP_OPTION (pfile, cplusplus))
3185                 cpp_pedwarning (pfile, CPP_W_VARIADIC_MACROS,
3186                             "ISO C++ does not permit named variadic macros");
3187               else
3188                 cpp_pedwarning (pfile, CPP_W_VARIADIC_MACROS,
3189                             "ISO C does not permit named variadic macros");
3190             }
3191
3192           /* We're at the end, and just expect a closing parenthesis.  */
3193           token = _cpp_lex_token (pfile);
3194           if (token->type == CPP_CLOSE_PAREN)
3195             return true;
3196           /* Fall through.  */
3197
3198         case CPP_EOF:
3199           cpp_error (pfile, CPP_DL_ERROR, "missing ')' in macro parameter list");
3200           return false;
3201         }
3202     }
3203 }
3204
3205 /* Allocate room for a token from a macro's replacement list.  */
3206 static cpp_token *
3207 alloc_expansion_token (cpp_reader *pfile, cpp_macro *macro)
3208 {
3209   if (BUFF_ROOM (pfile->a_buff) < (macro->count + 1) * sizeof (cpp_token))
3210     _cpp_extend_buff (pfile, &pfile->a_buff, sizeof (cpp_token));
3211
3212   return &((cpp_token *) BUFF_FRONT (pfile->a_buff))[macro->count++];
3213 }
3214
3215 /* Lex a token from the expansion of MACRO, but mark parameters as we
3216    find them and warn of traditional stringification.  */
3217 static cpp_token *
3218 lex_expansion_token (cpp_reader *pfile, cpp_macro *macro)
3219 {
3220   cpp_token *token, *saved_cur_token;
3221
3222   saved_cur_token = pfile->cur_token;
3223   pfile->cur_token = alloc_expansion_token (pfile, macro);
3224   token = _cpp_lex_direct (pfile);
3225   pfile->cur_token = saved_cur_token;
3226
3227   /* Is this a parameter?  */
3228   if (token->type == CPP_NAME
3229       && (token->val.node.node->flags & NODE_MACRO_ARG) != 0)
3230     {
3231       cpp_hashnode *spelling = token->val.node.spelling;
3232       token->type = CPP_MACRO_ARG;
3233       token->val.macro_arg.arg_no = token->val.node.node->value.arg_index;
3234       token->val.macro_arg.spelling = spelling;
3235     }
3236   else if (CPP_WTRADITIONAL (pfile) && macro->paramc > 0
3237            && (token->type == CPP_STRING || token->type == CPP_CHAR))
3238     check_trad_stringification (pfile, macro, &token->val.str);
3239
3240   return token;
3241 }
3242
3243 static bool
3244 create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
3245 {
3246   cpp_token *token;
3247   const cpp_token *ctoken;
3248   bool following_paste_op = false;
3249   const char *paste_op_error_msg =
3250     N_("'##' cannot appear at either end of a macro expansion");
3251   unsigned int num_extra_tokens = 0;
3252
3253   /* Get the first token of the expansion (or the '(' of a
3254      function-like macro).  */
3255   ctoken = _cpp_lex_token (pfile);
3256
3257   if (ctoken->type == CPP_OPEN_PAREN && !(ctoken->flags & PREV_WHITE))
3258     {
3259       bool ok = parse_params (pfile, macro);
3260       macro->params = (cpp_hashnode **) BUFF_FRONT (pfile->a_buff);
3261       if (!ok)
3262         return false;
3263
3264       /* Success.  Commit or allocate the parameter array.  */
3265       if (pfile->hash_table->alloc_subobject)
3266         {
3267           cpp_hashnode **params =
3268             (cpp_hashnode **) pfile->hash_table->alloc_subobject
3269             (sizeof (cpp_hashnode *) * macro->paramc);
3270           memcpy (params, macro->params,
3271                   sizeof (cpp_hashnode *) * macro->paramc);
3272           macro->params = params;
3273         }
3274       else
3275         BUFF_FRONT (pfile->a_buff) = (uchar *) &macro->params[macro->paramc];
3276       macro->fun_like = 1;
3277     }
3278   else if (ctoken->type != CPP_EOF && !(ctoken->flags & PREV_WHITE))
3279     {
3280       /* While ISO C99 requires whitespace before replacement text
3281          in a macro definition, ISO C90 with TC1 allows characters
3282          from the basic source character set there.  */
3283       if (CPP_OPTION (pfile, c99))
3284         {
3285           if (CPP_OPTION (pfile, cplusplus))
3286             cpp_error (pfile, CPP_DL_PEDWARN,
3287                        "ISO C++11 requires whitespace after the macro name");
3288           else
3289             cpp_error (pfile, CPP_DL_PEDWARN,
3290                        "ISO C99 requires whitespace after the macro name");
3291         }
3292       else
3293         {
3294           int warntype = CPP_DL_WARNING;
3295           switch (ctoken->type)
3296             {
3297             case CPP_ATSIGN:
3298             case CPP_AT_NAME:
3299             case CPP_OBJC_STRING:
3300               /* '@' is not in basic character set.  */
3301               warntype = CPP_DL_PEDWARN;
3302               break;
3303             case CPP_OTHER:
3304               /* Basic character set sans letters, digits and _.  */
3305               if (strchr ("!\"#%&'()*+,-./:;<=>?[\\]^{|}~",
3306                           ctoken->val.str.text[0]) == NULL)
3307                 warntype = CPP_DL_PEDWARN;
3308               break;
3309             default:
3310               /* All other tokens start with a character from basic
3311                  character set.  */
3312               break;
3313             }
3314           cpp_error (pfile, warntype,
3315                      "missing whitespace after the macro name");
3316         }
3317     }
3318
3319   if (macro->fun_like)
3320     token = lex_expansion_token (pfile, macro);
3321   else
3322     {
3323       token = alloc_expansion_token (pfile, macro);
3324       *token = *ctoken;
3325     }
3326
3327   /* The argument doesn't matter here.  */
3328   vaopt_state vaopt_tracker (pfile, macro->variadic, true);
3329
3330   for (;;)
3331     {
3332       /* Check the stringifying # constraint 6.10.3.2.1 of
3333          function-like macros when lexing the subsequent token.  */
3334       if (macro->count > 1 && token[-1].type == CPP_HASH && macro->fun_like)
3335         {
3336           if (token->type == CPP_MACRO_ARG)
3337             {
3338               if (token->flags & PREV_WHITE)
3339                 token->flags |= SP_PREV_WHITE;
3340               if (token[-1].flags & DIGRAPH)
3341                 token->flags |= SP_DIGRAPH;
3342               token->flags &= ~PREV_WHITE;
3343               token->flags |= STRINGIFY_ARG;
3344               token->flags |= token[-1].flags & PREV_WHITE;
3345               token[-1] = token[0];
3346               macro->count--;
3347             }
3348           /* Let assembler get away with murder.  */
3349           else if (CPP_OPTION (pfile, lang) != CLK_ASM)
3350             {
3351               cpp_error (pfile, CPP_DL_ERROR,
3352                          "'#' is not followed by a macro parameter");
3353               return false;
3354             }
3355         }
3356
3357       if (token->type == CPP_EOF)
3358         {
3359           /* Paste operator constraint 6.10.3.3.1:
3360              Token-paste ##, can appear in both object-like and
3361              function-like macros, but not at the end.  */
3362           if (following_paste_op)
3363             {
3364               cpp_error (pfile, CPP_DL_ERROR, paste_op_error_msg);
3365               return false;
3366             }
3367           break;
3368         }
3369
3370       /* Paste operator constraint 6.10.3.3.1.  */
3371       if (token->type == CPP_PASTE)
3372         {
3373           /* Token-paste ##, can appear in both object-like and
3374              function-like macros, but not at the beginning.  */
3375           if (macro->count == 1)
3376             {
3377               cpp_error (pfile, CPP_DL_ERROR, paste_op_error_msg);
3378               return false;
3379             }
3380
3381           if (token[-1].flags & PASTE_LEFT)
3382             {
3383               macro->extra_tokens = 1;
3384               num_extra_tokens++;
3385               token->val.token_no = macro->count - 1;
3386             }
3387           else
3388             {
3389               --macro->count;
3390               token[-1].flags |= PASTE_LEFT;
3391               if (token->flags & DIGRAPH)
3392                 token[-1].flags |= SP_DIGRAPH;
3393               if (token->flags & PREV_WHITE)
3394                 token[-1].flags |= SP_PREV_WHITE;
3395             }
3396         }
3397
3398       if (vaopt_tracker.update (token) == vaopt_state::ERROR)
3399         return false;
3400
3401       following_paste_op = (token->type == CPP_PASTE);
3402       token = lex_expansion_token (pfile, macro);
3403     }
3404
3405   if (!vaopt_tracker.completed ())
3406     return false;
3407
3408   macro->exp.tokens = (cpp_token *) BUFF_FRONT (pfile->a_buff);
3409   macro->traditional = 0;
3410
3411   /* Don't count the CPP_EOF.  */
3412   macro->count--;
3413
3414   /* Clear whitespace on first token for warn_of_redefinition().  */
3415   if (macro->count)
3416     macro->exp.tokens[0].flags &= ~PREV_WHITE;
3417
3418   /* Commit or allocate the memory.  */
3419   if (pfile->hash_table->alloc_subobject)
3420     {
3421       cpp_token *tokns =
3422         (cpp_token *) pfile->hash_table->alloc_subobject (sizeof (cpp_token)
3423                                                           * macro->count);
3424       if (num_extra_tokens)
3425         {
3426           /* Place second and subsequent ## or %:%: tokens in
3427              sequences of consecutive such tokens at the end of the
3428              list to preserve information about where they appear, how
3429              they are spelt and whether they are preceded by
3430              whitespace without otherwise interfering with macro
3431              expansion.  */
3432           cpp_token *normal_dest = tokns;
3433           cpp_token *extra_dest = tokns + macro->count - num_extra_tokens;
3434           unsigned int i;
3435           for (i = 0; i < macro->count; i++)
3436             {
3437               if (macro->exp.tokens[i].type == CPP_PASTE)
3438                 *extra_dest++ = macro->exp.tokens[i];
3439               else
3440                 *normal_dest++ = macro->exp.tokens[i];
3441             }
3442         }
3443       else
3444         memcpy (tokns, macro->exp.tokens, sizeof (cpp_token) * macro->count);
3445       macro->exp.tokens = tokns;
3446     }
3447   else
3448     BUFF_FRONT (pfile->a_buff) = (uchar *) &macro->exp.tokens[macro->count];
3449
3450   return true;
3451 }
3452
3453 /* Parse a macro and save its expansion.  Returns nonzero on success.  */
3454 bool
3455 _cpp_create_definition (cpp_reader *pfile, cpp_hashnode *node)
3456 {
3457   cpp_macro *macro;
3458   unsigned int i;
3459   bool ok;
3460
3461   if (pfile->hash_table->alloc_subobject)
3462     macro = (cpp_macro *) pfile->hash_table->alloc_subobject
3463       (sizeof (cpp_macro));
3464   else
3465     macro = (cpp_macro *) _cpp_aligned_alloc (pfile, sizeof (cpp_macro));
3466   macro->line = pfile->directive_line;
3467   macro->params = 0;
3468   macro->paramc = 0;
3469   macro->variadic = 0;
3470   macro->used = !CPP_OPTION (pfile, warn_unused_macros);
3471   macro->count = 0;
3472   macro->fun_like = 0;
3473   macro->extra_tokens = 0;
3474   /* To suppress some diagnostics.  */
3475   macro->syshdr = pfile->buffer && pfile->buffer->sysp != 0;
3476
3477   if (CPP_OPTION (pfile, traditional))
3478     ok = _cpp_create_trad_definition (pfile, macro);
3479   else
3480     {
3481       ok = create_iso_definition (pfile, macro);
3482
3483       /* We set the type for SEEN_EOL() in directives.c.
3484
3485          Longer term we should lex the whole line before coming here,
3486          and just copy the expansion.  */
3487
3488       /* Stop the lexer accepting __VA_ARGS__.  */
3489       pfile->state.va_args_ok = 0;
3490     }
3491
3492   /* Clear the fast argument lookup indices.  */
3493   for (i = macro->paramc; i-- > 0; )
3494     {
3495       struct macro_arg_saved_data *save =
3496         &((struct macro_arg_saved_data *) pfile->macro_buffer)[i];
3497       struct cpp_hashnode *node = save->canonical_node;
3498       node->flags &= ~ NODE_MACRO_ARG;
3499       node->value = save->value;
3500     }
3501
3502   if (!ok)
3503     return ok;
3504
3505   if (node->type == NT_MACRO)
3506     {
3507       if (CPP_OPTION (pfile, warn_unused_macros))
3508         _cpp_warn_if_unused_macro (pfile, node, NULL);
3509
3510       if (warn_of_redefinition (pfile, node, macro))
3511         {
3512           const int reason = ((node->flags & NODE_BUILTIN)
3513                               && !(node->flags & NODE_WARN))
3514                              ? CPP_W_BUILTIN_MACRO_REDEFINED : CPP_W_NONE;
3515
3516           bool warned =
3517             cpp_pedwarning_with_line (pfile, reason,
3518                                       pfile->directive_line, 0,
3519                                       "\"%s\" redefined", NODE_NAME (node));
3520
3521           if (warned && node->type == NT_MACRO && !(node->flags & NODE_BUILTIN))
3522             cpp_error_with_line (pfile, CPP_DL_NOTE,
3523                                  node->value.macro->line, 0,
3524                          "this is the location of the previous definition");
3525         }
3526     }
3527
3528   if (node->type != NT_VOID)
3529     _cpp_free_definition (node);
3530
3531   /* Enter definition in hash table.  */
3532   node->type = NT_MACRO;
3533   node->value.macro = macro;
3534   if (! ustrncmp (NODE_NAME (node), DSC ("__STDC_"))
3535       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_FORMAT_MACROS")
3536       /* __STDC_LIMIT_MACROS and __STDC_CONSTANT_MACROS are mentioned
3537          in the C standard, as something that one must use in C++.
3538          However DR#593 and C++11 indicate that they play no role in C++.
3539          We special-case them anyway.  */
3540       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_LIMIT_MACROS")
3541       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_CONSTANT_MACROS"))
3542     node->flags |= NODE_WARN;
3543
3544   /* If user defines one of the conditional macros, remove the
3545      conditional flag */
3546   node->flags &= ~NODE_CONDITIONAL;
3547
3548   return ok;
3549 }
3550
3551 /* Warn if a token in STRING matches one of a function-like MACRO's
3552    parameters.  */
3553 static void
3554 check_trad_stringification (cpp_reader *pfile, const cpp_macro *macro,
3555                             const cpp_string *string)
3556 {
3557   unsigned int i, len;
3558   const uchar *p, *q, *limit;
3559
3560   /* Loop over the string.  */
3561   limit = string->text + string->len - 1;
3562   for (p = string->text + 1; p < limit; p = q)
3563     {
3564       /* Find the start of an identifier.  */
3565       while (p < limit && !is_idstart (*p))
3566         p++;
3567
3568       /* Find the end of the identifier.  */
3569       q = p;
3570       while (q < limit && is_idchar (*q))
3571         q++;
3572
3573       len = q - p;
3574
3575       /* Loop over the function macro arguments to see if the
3576          identifier inside the string matches one of them.  */
3577       for (i = 0; i < macro->paramc; i++)
3578         {
3579           const cpp_hashnode *node = macro->params[i];
3580
3581           if (NODE_LEN (node) == len
3582               && !memcmp (p, NODE_NAME (node), len))
3583             {
3584               cpp_warning (pfile, CPP_W_TRADITIONAL,
3585            "macro argument \"%s\" would be stringified in traditional C",
3586                          NODE_NAME (node));
3587               break;
3588             }
3589         }
3590     }
3591 }
3592
3593 /* Returns true of NODE is a function-like macro.  */
3594 bool
3595 cpp_fun_like_macro_p (cpp_hashnode *node)
3596 {
3597   return (node->type == NT_MACRO
3598           && (node->flags & (NODE_BUILTIN | NODE_MACRO_ARG)) == 0
3599           && node->value.macro->fun_like);
3600 }
3601
3602 /* Returns the name, arguments and expansion of a macro, in a format
3603    suitable to be read back in again, and therefore also for DWARF 2
3604    debugging info.  e.g. "PASTE(X, Y) X ## Y", or "MACNAME EXPANSION".
3605    Caller is expected to generate the "#define" bit if needed.  The
3606    returned text is temporary, and automatically freed later.  */
3607 const unsigned char *
3608 cpp_macro_definition (cpp_reader *pfile, cpp_hashnode *node)
3609 {
3610   unsigned int i, len;
3611   const cpp_macro *macro;
3612   unsigned char *buffer;
3613
3614   if (node->type != NT_MACRO || (node->flags & NODE_BUILTIN))
3615     {
3616       if (node->type != NT_MACRO
3617           || !pfile->cb.user_builtin_macro
3618           || !pfile->cb.user_builtin_macro (pfile, node))
3619         {
3620           cpp_error (pfile, CPP_DL_ICE,
3621                      "invalid hash type %d in cpp_macro_definition",
3622                      node->type);
3623           return 0;
3624         }
3625     }
3626
3627   macro = node->value.macro;
3628   /* Calculate length.  */
3629   len = NODE_LEN (node) * 10 + 2;               /* ' ' and NUL.  */
3630   if (macro->fun_like)
3631     {
3632       len += 4;         /* "()" plus possible final ".." of named
3633                            varargs (we have + 1 below).  */
3634       for (i = 0; i < macro->paramc; i++)
3635         len += NODE_LEN (macro->params[i]) + 1; /* "," */
3636     }
3637
3638   /* This should match below where we fill in the buffer.  */
3639   if (CPP_OPTION (pfile, traditional))
3640     len += _cpp_replacement_text_len (macro);
3641   else
3642     {
3643       unsigned int count = macro_real_token_count (macro);
3644       for (i = 0; i < count; i++)
3645         {
3646           cpp_token *token = &macro->exp.tokens[i];
3647
3648           if (token->type == CPP_MACRO_ARG)
3649             len += NODE_LEN (token->val.macro_arg.spelling);
3650           else
3651             len += cpp_token_len (token);
3652
3653           if (token->flags & STRINGIFY_ARG)
3654             len++;                      /* "#" */
3655           if (token->flags & PASTE_LEFT)
3656             len += 3;           /* " ##" */
3657           if (token->flags & PREV_WHITE)
3658             len++;              /* " " */
3659         }
3660     }
3661
3662   if (len > pfile->macro_buffer_len)
3663     {
3664       pfile->macro_buffer = XRESIZEVEC (unsigned char,
3665                                         pfile->macro_buffer, len);
3666       pfile->macro_buffer_len = len;
3667     }
3668
3669   /* Fill in the buffer.  Start with the macro name.  */
3670   buffer = pfile->macro_buffer;
3671   buffer = _cpp_spell_ident_ucns (buffer, node);
3672
3673   /* Parameter names.  */
3674   if (macro->fun_like)
3675     {
3676       *buffer++ = '(';
3677       for (i = 0; i < macro->paramc; i++)
3678         {
3679           cpp_hashnode *param = macro->params[i];
3680
3681           if (param != pfile->spec_nodes.n__VA_ARGS__)
3682             {
3683               memcpy (buffer, NODE_NAME (param), NODE_LEN (param));
3684               buffer += NODE_LEN (param);
3685             }
3686
3687           if (i + 1 < macro->paramc)
3688             /* Don't emit a space after the comma here; we're trying
3689                to emit a Dwarf-friendly definition, and the Dwarf spec
3690                forbids spaces in the argument list.  */
3691             *buffer++ = ',';
3692           else if (macro->variadic)
3693             *buffer++ = '.', *buffer++ = '.', *buffer++ = '.';
3694         }
3695       *buffer++ = ')';
3696     }
3697
3698   /* The Dwarf spec requires a space after the macro name, even if the
3699      definition is the empty string.  */
3700   *buffer++ = ' ';
3701
3702   if (CPP_OPTION (pfile, traditional))
3703     buffer = _cpp_copy_replacement_text (macro, buffer);
3704   else if (macro->count)
3705   /* Expansion tokens.  */
3706     {
3707       unsigned int count = macro_real_token_count (macro);
3708       for (i = 0; i < count; i++)
3709         {
3710           cpp_token *token = &macro->exp.tokens[i];
3711
3712           if (token->flags & PREV_WHITE)
3713             *buffer++ = ' ';
3714           if (token->flags & STRINGIFY_ARG)
3715             *buffer++ = '#';
3716
3717           if (token->type == CPP_MACRO_ARG)
3718             {
3719               memcpy (buffer,
3720                       NODE_NAME (token->val.macro_arg.spelling),
3721                       NODE_LEN (token->val.macro_arg.spelling));
3722               buffer += NODE_LEN (token->val.macro_arg.spelling);
3723             }
3724           else
3725             buffer = cpp_spell_token (pfile, token, buffer, true);
3726
3727           if (token->flags & PASTE_LEFT)
3728             {
3729               *buffer++ = ' ';
3730               *buffer++ = '#';
3731               *buffer++ = '#';
3732               /* Next has PREV_WHITE; see _cpp_create_definition.  */
3733             }
3734         }
3735     }
3736
3737   *buffer = '\0';
3738   return pfile->macro_buffer;
3739 }
3740
3741 /* Get the line at which the macro was defined.  */
3742
3743 source_location
3744 cpp_macro_definition_location (cpp_hashnode *node)
3745 {
3746   return node->value.macro->line;
3747 }