libcpp/macro.c

   1 /* Part of CPP library.  (Macro and #define handling.)
   2    Copyright (C) 1986-2018 Free Software Foundation, Inc.
   3    Written by Per Bothner, 1994.
   4    Based on CCCP program by Paul Rubin, June 1986
   5    Adapted to ANSI C, Richard Stallman, Jan 1987
   6
   7 This program is free software; you can redistribute it and/or modify it
   8 under the terms of the GNU General Public License as published by the
   9 Free Software Foundation; either version 3, or (at your option) any
  10 later version.
  11
  12 This program is distributed in the hope that it will be useful,
  13 but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 GNU General Public License for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; see the file COPYING3.  If not see
  19 <http://www.gnu.org/licenses/>.
  20
  21  In other words, you are welcome to use, share and improve this program.
  22  You are forbidden to forbid anyone else to use, share and improve
  23  what you give them.   Help stamp out software-hoarding!  */
  24
  25 #include "config.h"
  26 #include "system.h"
  27 #include "cpplib.h"
  28 #include "internal.h"
  29
  30 typedef struct macro_arg macro_arg;
  31 /* This structure represents the tokens of a macro argument.  These
  32    tokens can be macro themselves, in which case they can be either
  33    expanded or unexpanded.  When they are expanded, this data
  34    structure keeps both the expanded and unexpanded forms.  */
  35 struct macro_arg
  36 {
  37   const cpp_token **first;      /* First token in unexpanded argument.  */
  38   const cpp_token **expanded;   /* Macro-expanded argument.  */
  39   const cpp_token *stringified; /* Stringified argument.  */
  40   unsigned int count;           /* # of tokens in argument.  */
  41   unsigned int expanded_count;  /* # of tokens in expanded argument.  */
  42   source_location *virt_locs;   /* Where virtual locations for
  43                                    unexpanded tokens are stored.  */
  44   source_location *expanded_virt_locs; /* Where virtual locations for
  45                                           expanded tokens are
  46                                           stored.  */
  47 };
  48
  49 /* The kind of macro tokens which the instance of
  50    macro_arg_token_iter is supposed to iterate over.  */
  51 enum macro_arg_token_kind {
  52   MACRO_ARG_TOKEN_NORMAL,
  53   /* This is a macro argument token that got transformed into a string
  54      litteral, e.g. #foo.  */
  55   MACRO_ARG_TOKEN_STRINGIFIED,
  56   /* This is a token resulting from the expansion of a macro
  57      argument that was itself a macro.  */
  58   MACRO_ARG_TOKEN_EXPANDED
  59 };
  60
  61 /* An iterator over tokens coming from a function-like macro
  62    argument.  */
  63 typedef struct macro_arg_token_iter macro_arg_token_iter;
  64 struct macro_arg_token_iter
  65 {
  66   /* Whether or not -ftrack-macro-expansion is used.  */
  67   bool track_macro_exp_p;
  68   /* The kind of token over which we are supposed to iterate.  */
  69   enum macro_arg_token_kind kind;
  70   /* A pointer to the current token pointed to by the iterator.  */
  71   const cpp_token **token_ptr;
  72   /* A pointer to the "full" location of the current token.  If
  73      -ftrack-macro-expansion is used this location tracks loci across
  74      macro expansion.  */
  75   const source_location *location_ptr;
  76 #if CHECKING_P
  77   /* The number of times the iterator went forward. This useful only
  78      when checking is enabled.  */
  79   size_t num_forwards;
  80 #endif
  81 };
  82
  83 /* Saved data about an identifier being used as a macro argument
  84    name.  */
  85 struct macro_arg_saved_data {
  86   /* The canonical (UTF-8) spelling of this identifier.  */
  87   cpp_hashnode *canonical_node;
  88   /* The previous value of this identifier.  */
  89   union _cpp_hashnode_value value;
  90 };
  91
  92 static const char *vaopt_paste_error =
  93   N_("'##' cannot appear at either end of __VA_OPT__");
  94
  95 /* A class for tracking __VA_OPT__ state while iterating over a
  96    sequence of tokens.  This is used during both macro definition and
  97    expansion.  */
  98 class vaopt_state {
  99
 100  public:
 101
 102   /* Initialize the state tracker.  ANY_ARGS is true if variable
 103      arguments were provided to the macro invocation.  */
 104   vaopt_state (cpp_reader *pfile, bool is_variadic, bool any_args)
 105     : m_pfile (pfile),
 106     m_allowed (any_args),
 107     m_variadic (is_variadic),
 108     m_state (0),
 109     m_last_was_paste (false),
 110     m_paste_location (0),
 111     m_location (0)
 112   {
 113   }
 114
 115   enum update_type
 116   {
 117     ERROR,
 118     DROP,
 119     INCLUDE
 120   };
 121
 122   /* Given a token, update the state of this tracker and return a
 123      boolean indicating whether the token should be be included in the
 124      expansion.  */
 125   update_type update (const cpp_token *token)
 126   {
 127     /* If the macro isn't variadic, just don't bother.  */
 128     if (!m_variadic)
 129       return INCLUDE;
 130
 131     if (token->type == CPP_NAME
 132         && token->val.node.node == m_pfile->spec_nodes.n__VA_OPT__)
 133       {
 134         if (m_state > 0)
 135           {
 136             cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
 137                           "__VA_OPT__ may not appear in a __VA_OPT__");
 138             return ERROR;
 139           }
 140         ++m_state;
 141         m_location = token->src_loc;
 142         return DROP;
 143       }
 144     else if (m_state == 1)
 145       {
 146         if (token->type != CPP_OPEN_PAREN)
 147           {
 148             cpp_error_at (m_pfile, CPP_DL_ERROR, m_location,
 149                           "__VA_OPT__ must be followed by an "
 150                           "open parenthesis");
 151             return ERROR;
 152           }
 153         ++m_state;
 154         return DROP;
 155       }
 156     else if (m_state >= 2)
 157       {
 158         if (m_state == 2 && token->type == CPP_PASTE)
 159           {
 160             cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
 161                           vaopt_paste_error);
 162             return ERROR;
 163           }
 164         /* Advance states before further considering this token, in
 165            case we see a close paren immediately after the open
 166            paren.  */
 167         if (m_state == 2)
 168           ++m_state;
 169
 170         bool was_paste = m_last_was_paste;
 171         m_last_was_paste = false;
 172         if (token->type == CPP_PASTE)
 173           {
 174             m_last_was_paste = true;
 175             m_paste_location = token->src_loc;
 176           }
 177         else if (token->type == CPP_OPEN_PAREN)
 178           ++m_state;
 179         else if (token->type == CPP_CLOSE_PAREN)
 180           {
 181             --m_state;
 182             if (m_state == 2)
 183               {
 184                 /* Saw the final paren.  */
 185                 m_state = 0;
 186
 187                 if (was_paste)
 188                   {
 189                     cpp_error_at (m_pfile, CPP_DL_ERROR, token->src_loc,
 190                                   vaopt_paste_error);
 191                     return ERROR;
 192                   }
 193
 194                 return DROP;
 195               }
 196           }
 197         return m_allowed ? INCLUDE : DROP;
 198       }
 199
 200     /* Nothing to do with __VA_OPT__.  */
 201     return INCLUDE;
 202   }
 203
 204   /* Ensure that any __VA_OPT__ was completed.  If ok, return true.
 205      Otherwise, issue an error and return false.  */
 206   bool completed ()
 207   {
 208     if (m_variadic && m_state != 0)
 209       cpp_error_at (m_pfile, CPP_DL_ERROR, m_location,
 210                     "unterminated __VA_OPT__");
 211     return m_state == 0;
 212   }
 213
 214  private:
 215
 216   /* The cpp_reader.  */
 217   cpp_reader *m_pfile;
 218
 219   /* True if there were varargs.  */
 220   bool m_allowed;
 221   /* True if the macro is variadic.  */
 222   bool m_variadic;
 223
 224   /* The state variable:
 225      0 means not parsing
 226      1 means __VA_OPT__ seen, looking for "("
 227      2 means "(" seen (so the next token can't be "##")
 228      >= 3 means looking for ")", the number encodes the paren depth.  */
 229   int m_state;
 230
 231   /* If true, the previous token was ##.  This is used to detect when
 232      a paste occurs at the end of the sequence.  */
 233   bool m_last_was_paste;
 234   /* The location of the paste token.  */
 235   source_location m_paste_location;
 236
 237   /* Location of the __VA_OPT__ token.  */
 238   source_location m_location;
 239 };
 240
 241 /* Macro expansion.  */
 242
 243 static int enter_macro_context (cpp_reader *, cpp_hashnode *,
 244                                 const cpp_token *, source_location);
 245 static int builtin_macro (cpp_reader *, cpp_hashnode *,
 246                           source_location, source_location);
 247 static void push_ptoken_context (cpp_reader *, cpp_hashnode *, _cpp_buff *,
 248                                  const cpp_token **, unsigned int);
 249 static void push_extended_tokens_context (cpp_reader *, cpp_hashnode *,
 250                                           _cpp_buff *, source_location *,
 251                                           const cpp_token **, unsigned int);
 252 static _cpp_buff *collect_args (cpp_reader *, const cpp_hashnode *,
 253                                 _cpp_buff **, unsigned *);
 254 static cpp_context *next_context (cpp_reader *);
 255 static const cpp_token *padding_token (cpp_reader *, const cpp_token *);
 256 static void expand_arg (cpp_reader *, macro_arg *);
 257 static const cpp_token *new_string_token (cpp_reader *, uchar *, unsigned int);
 258 static const cpp_token *stringify_arg (cpp_reader *, macro_arg *);
 259 static void paste_all_tokens (cpp_reader *, const cpp_token *);
 260 static bool paste_tokens (cpp_reader *, source_location,
 261                           const cpp_token **, const cpp_token *);
 262 static void alloc_expanded_arg_mem (cpp_reader *, macro_arg *, size_t);
 263 static void ensure_expanded_arg_room (cpp_reader *, macro_arg *, size_t, size_t *);
 264 static void delete_macro_args (_cpp_buff*, unsigned num_args);
 265 static void set_arg_token (macro_arg *, const cpp_token *,
 266                            source_location, size_t,
 267                            enum macro_arg_token_kind,
 268                            bool);
 269 static const source_location *get_arg_token_location (const macro_arg *,
 270                                                       enum macro_arg_token_kind);
 271 static const cpp_token **arg_token_ptr_at (const macro_arg *,
 272                                            size_t,
 273                                            enum macro_arg_token_kind,
 274                                            source_location **virt_location);
 275
 276 static void macro_arg_token_iter_init (macro_arg_token_iter *, bool,
 277                                        enum macro_arg_token_kind,
 278                                        const macro_arg *,
 279                                        const cpp_token **);
 280 static const cpp_token *macro_arg_token_iter_get_token
 281 (const macro_arg_token_iter *it);
 282 static source_location macro_arg_token_iter_get_location
 283 (const macro_arg_token_iter *);
 284 static void macro_arg_token_iter_forward (macro_arg_token_iter *);
 285 static _cpp_buff *tokens_buff_new (cpp_reader *, size_t,
 286                                    source_location **);
 287 static size_t tokens_buff_count (_cpp_buff *);
 288 static const cpp_token **tokens_buff_last_token_ptr (_cpp_buff *);
 289 static inline const cpp_token **tokens_buff_put_token_to (const cpp_token **,
 290                                                           source_location *,
 291                                                           const cpp_token *,
 292                                                           source_location,
 293                                                           source_location,
 294                                                           const line_map_macro *,
 295                                                           unsigned int);
 296
 297 static const cpp_token **tokens_buff_add_token (_cpp_buff *,
 298                                                 source_location *,
 299                                                 const cpp_token *,
 300                                                 source_location,
 301                                                 source_location,
 302                                                 const line_map_macro *,
 303                                                 unsigned int);
 304 static inline void tokens_buff_remove_last_token (_cpp_buff *);
 305 static void replace_args (cpp_reader *, cpp_hashnode *, cpp_macro *,
 306                           macro_arg *, source_location);
 307 static _cpp_buff *funlike_invocation_p (cpp_reader *, cpp_hashnode *,
 308                                         _cpp_buff **, unsigned *);
 309 static bool create_iso_definition (cpp_reader *, cpp_macro *);
 310
 311 /* #define directive parsing and handling.  */
 312
 313 static cpp_token *alloc_expansion_token (cpp_reader *, cpp_macro *);
 314 static cpp_token *lex_expansion_token (cpp_reader *, cpp_macro *);
 315 static bool warn_of_redefinition (cpp_reader *, cpp_hashnode *,
 316                                   const cpp_macro *);
 317 static bool parse_params (cpp_reader *, cpp_macro *);
 318 static void check_trad_stringification (cpp_reader *, const cpp_macro *,
 319                                         const cpp_string *);
 320 static bool reached_end_of_context (cpp_context *);
 321 static void consume_next_token_from_context (cpp_reader *pfile,
 322                                              const cpp_token **,
 323                                              source_location *);
 324 static const cpp_token* cpp_get_token_1 (cpp_reader *, source_location *);
 325
 326 static cpp_hashnode* macro_of_context (cpp_context *context);
 327
 328 static bool in_macro_expansion_p (cpp_reader *pfile);
 329
 330 /* Statistical counter tracking the number of macros that got
 331    expanded.  */
 332 unsigned num_expanded_macros_counter = 0;
 333 /* Statistical counter tracking the total number tokens resulting
 334    from macro expansion.  */
 335 unsigned num_macro_tokens_counter = 0;
 336
 337 /* Emits a warning if NODE is a macro defined in the main file that
 338    has not been used.  */
 339 int
 340 _cpp_warn_if_unused_macro (cpp_reader *pfile, cpp_hashnode *node,
 341                            void *v ATTRIBUTE_UNUSED)
 342 {
 343   if (node->type == NT_MACRO && !(node->flags & NODE_BUILTIN))
 344     {
 345       cpp_macro *macro = node->value.macro;
 346
 347       if (!macro->used
 348           && MAIN_FILE_P (linemap_check_ordinary
 349                             (linemap_lookup (pfile->line_table,
 350                                              macro->line))))
 351         cpp_warning_with_line (pfile, CPP_W_UNUSED_MACROS, macro->line, 0,
 352                                "macro \"%s\" is not used", NODE_NAME (node));
 353     }
 354
 355   return 1;
 356 }
 357
 358 /* Allocates and returns a CPP_STRING token, containing TEXT of length
 359    LEN, after null-terminating it.  TEXT must be in permanent storage.  */
 360 static const cpp_token *
 361 new_string_token (cpp_reader *pfile, unsigned char *text, unsigned int len)
 362 {
 363   cpp_token *token = _cpp_temp_token (pfile);
 364
 365   text[len] = '\0';
 366   token->type = CPP_STRING;
 367   token->val.str.len = len;
 368   token->val.str.text = text;
 369   token->flags = 0;
 370   return token;
 371 }
 372
 373 static const char * const monthnames[] =
 374 {
 375   "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 376   "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 377 };
 378
 379 /* Helper function for builtin_macro.  Returns the text generated by
 380    a builtin macro. */
 381 const uchar *
 382 _cpp_builtin_macro_text (cpp_reader *pfile, cpp_hashnode *node,
 383                          source_location loc)
 384 {
 385   const uchar *result = NULL;
 386   linenum_type number = 1;
 387
 388   switch (node->value.builtin)
 389     {
 390     default:
 391       cpp_error (pfile, CPP_DL_ICE, "invalid built-in macro \"%s\"",
 392                  NODE_NAME (node));
 393       break;
 394
 395     case BT_TIMESTAMP:
 396       {
 397         if (CPP_OPTION (pfile, warn_date_time))
 398           cpp_warning (pfile, CPP_W_DATE_TIME, "macro \"%s\" might prevent "
 399                        "reproducible builds", NODE_NAME (node));
 400
 401         cpp_buffer *pbuffer = cpp_get_buffer (pfile);
 402         if (pbuffer->timestamp == NULL)
 403           {
 404             /* Initialize timestamp value of the assotiated file. */
 405             struct _cpp_file *file = cpp_get_file (pbuffer);
 406             if (file)
 407               {
 408                 /* Generate __TIMESTAMP__ string, that represents
 409                    the date and time of the last modification
 410                    of the current source file. The string constant
 411                    looks like "Sun Sep 16 01:03:52 1973".  */
 412                 struct tm *tb = NULL;
 413                 struct stat *st = _cpp_get_file_stat (file);
 414                 if (st)
 415                   tb = localtime (&st->st_mtime);
 416                 if (tb)
 417                   {
 418                     char *str = asctime (tb);
 419                     size_t len = strlen (str);
 420                     unsigned char *buf = _cpp_unaligned_alloc (pfile, len + 2);
 421                     buf[0] = '"';
 422                     strcpy ((char *) buf + 1, str);
 423                     buf[len] = '"';
 424                     pbuffer->timestamp = buf;
 425                   }
 426                 else
 427                   {
 428                     cpp_errno (pfile, CPP_DL_WARNING,
 429                         "could not determine file timestamp");
 430                     pbuffer->timestamp = UC"\"??? ??? ?? ??:??:?? ????\"";
 431                   }
 432               }
 433           }
 434         result = pbuffer->timestamp;
 435       }
 436       break;
 437     case BT_FILE:
 438     case BT_BASE_FILE:
 439       {
 440         unsigned int len;
 441         const char *name;
 442         uchar *buf;
 443
 444         if (node->value.builtin == BT_FILE)
 445           name = linemap_get_expansion_filename (pfile->line_table,
 446                                                  pfile->line_table->highest_line);
 447         else
 448           {
 449             name = _cpp_get_file_name (pfile->main_file);
 450             if (!name)
 451               abort ();
 452           }
 453         if (pfile->cb.remap_filename)
 454           name = pfile->cb.remap_filename (name);
 455         len = strlen (name);
 456         buf = _cpp_unaligned_alloc (pfile, len * 2 + 3);
 457         result = buf;
 458         *buf = '"';
 459         buf = cpp_quote_string (buf + 1, (const unsigned char *) name, len);
 460         *buf++ = '"';
 461         *buf = '\0';
 462       }
 463       break;
 464
 465     case BT_INCLUDE_LEVEL:
 466       /* The line map depth counts the primary source as level 1, but
 467          historically __INCLUDE_DEPTH__ has called the primary source
 468          level 0.  */
 469       number = pfile->line_table->depth - 1;
 470       break;
 471
 472     case BT_SPECLINE:
 473       /* If __LINE__ is embedded in a macro, it must expand to the
 474          line of the macro's invocation, not its definition.
 475          Otherwise things like assert() will not work properly.
 476          See WG14 N1911, WG21 N4220 sec 6.5, and PR 61861.  */
 477       if (CPP_OPTION (pfile, traditional))
 478         loc = pfile->line_table->highest_line;
 479       else
 480         loc = linemap_resolve_location (pfile->line_table, loc,
 481                                         LRK_MACRO_EXPANSION_POINT, NULL);
 482       number = linemap_get_expansion_line (pfile->line_table, loc);
 483       break;
 484
 485       /* __STDC__ has the value 1 under normal circumstances.
 486          However, if (a) we are in a system header, (b) the option
 487          stdc_0_in_system_headers is true (set by target config), and
 488          (c) we are not in strictly conforming mode, then it has the
 489          value 0.  (b) and (c) are already checked in cpp_init_builtins.  */
 490     case BT_STDC:
 491       if (cpp_in_system_header (pfile))
 492         number = 0;
 493       else
 494         number = 1;
 495       break;
 496
 497     case BT_DATE:
 498     case BT_TIME:
 499       if (CPP_OPTION (pfile, warn_date_time))
 500         cpp_warning (pfile, CPP_W_DATE_TIME, "macro \"%s\" might prevent "
 501                      "reproducible builds", NODE_NAME (node));
 502       if (pfile->date == NULL)
 503         {
 504           /* Allocate __DATE__ and __TIME__ strings from permanent
 505              storage.  We only do this once, and don't generate them
 506              at init time, because time() and localtime() are very
 507              slow on some systems.  */
 508           time_t tt;
 509           struct tm *tb = NULL;
 510
 511           /* Set a reproducible timestamp for __DATE__ and __TIME__ macro
 512              if SOURCE_DATE_EPOCH is defined.  */
 513           if (pfile->source_date_epoch == (time_t) -2
 514               && pfile->cb.get_source_date_epoch != NULL)
 515             pfile->source_date_epoch = pfile->cb.get_source_date_epoch (pfile);
 516
 517           if (pfile->source_date_epoch >= (time_t) 0)
 518             tb = gmtime (&pfile->source_date_epoch);
 519           else
 520             {
 521               /* (time_t) -1 is a legitimate value for "number of seconds
 522                  since the Epoch", so we have to do a little dance to
 523                  distinguish that from a genuine error.  */
 524               errno = 0;
 525               tt = time (NULL);
 526               if (tt != (time_t)-1 || errno == 0)
 527                 tb = localtime (&tt);
 528             }
 529
 530           if (tb)
 531             {
 532               pfile->date = _cpp_unaligned_alloc (pfile,
 533                                                   sizeof ("\"Oct 11 1347\""));
 534               sprintf ((char *) pfile->date, "\"%s %2d %4d\"",
 535                        monthnames[tb->tm_mon], tb->tm_mday,
 536                        tb->tm_year + 1900);
 537
 538               pfile->time = _cpp_unaligned_alloc (pfile,
 539                                                   sizeof ("\"12:34:56\""));
 540               sprintf ((char *) pfile->time, "\"%02d:%02d:%02d\"",
 541                        tb->tm_hour, tb->tm_min, tb->tm_sec);
 542             }
 543           else
 544             {
 545               cpp_errno (pfile, CPP_DL_WARNING,
 546                          "could not determine date and time");
 547
 548               pfile->date = UC"\"??? ?? ????\"";
 549               pfile->time = UC"\"??:??:??\"";
 550             }
 551         }
 552
 553       if (node->value.builtin == BT_DATE)
 554         result = pfile->date;
 555       else
 556         result = pfile->time;
 557       break;
 558
 559     case BT_COUNTER:
 560       if (CPP_OPTION (pfile, directives_only) && pfile->state.in_directive)
 561         cpp_error (pfile, CPP_DL_ERROR,
 562             "__COUNTER__ expanded inside directive with -fdirectives-only");
 563       number = pfile->counter++;
 564       break;
 565
 566     case BT_HAS_ATTRIBUTE:
 567       number = pfile->cb.has_attribute (pfile);
 568       break;
 569     }
 570
 571   if (result == NULL)
 572     {
 573       /* 21 bytes holds all NUL-terminated unsigned 64-bit numbers.  */
 574       result = _cpp_unaligned_alloc (pfile, 21);
 575       sprintf ((char *) result, "%u", number);
 576     }
 577
 578   return result;
 579 }
 580
 581 /* Convert builtin macros like __FILE__ to a token and push it on the
 582    context stack.  Also handles _Pragma, for which a new token may not
 583    be created.  Returns 1 if it generates a new token context, 0 to
 584    return the token to the caller.  LOC is the location of the expansion
 585    point of the macro.  */
 586 static int
 587 builtin_macro (cpp_reader *pfile, cpp_hashnode *node,
 588                source_location loc, source_location expand_loc)
 589 {
 590   const uchar *buf;
 591   size_t len;
 592   char *nbuf;
 593
 594   if (node->value.builtin == BT_PRAGMA)
 595     {
 596       /* Don't interpret _Pragma within directives.  The standard is
 597          not clear on this, but to me this makes most sense.  */
 598       if (pfile->state.in_directive)
 599         return 0;
 600
 601       return _cpp_do__Pragma (pfile, loc);
 602     }
 603
 604   buf = _cpp_builtin_macro_text (pfile, node, expand_loc);
 605   len = ustrlen (buf);
 606   nbuf = (char *) alloca (len + 1);
 607   memcpy (nbuf, buf, len);
 608   nbuf[len]='\n';
 609
 610   cpp_push_buffer (pfile, (uchar *) nbuf, len, /* from_stage3 */ true);
 611   _cpp_clean_line (pfile);
 612
 613   /* Set pfile->cur_token as required by _cpp_lex_direct.  */
 614   pfile->cur_token = _cpp_temp_token (pfile);
 615   cpp_token *token = _cpp_lex_direct (pfile);
 616   /* We should point to the expansion point of the builtin macro.  */
 617   token->src_loc = loc;
 618   if (pfile->context->tokens_kind == TOKENS_KIND_EXTENDED)
 619     {
 620       /* We are tracking tokens resulting from macro expansion.
 621          Create a macro line map and generate a virtual location for
 622          the token resulting from the expansion of the built-in
 623          macro.  */
 624       source_location *virt_locs = NULL;
 625       _cpp_buff *token_buf = tokens_buff_new (pfile, 1, &virt_locs);
 626       const line_map_macro * map =
 627         linemap_enter_macro (pfile->line_table, node, loc, 1);
 628       tokens_buff_add_token (token_buf, virt_locs, token,
 629                              pfile->line_table->builtin_location,
 630                              pfile->line_table->builtin_location,
 631                             map, /*macro_token_index=*/0);
 632       push_extended_tokens_context (pfile, node, token_buf, virt_locs,
 633                                     (const cpp_token **)token_buf->base,
 634                                     1);
 635     }
 636   else
 637     _cpp_push_token_context (pfile, NULL, token, 1);
 638   if (pfile->buffer->cur != pfile->buffer->rlimit)
 639     cpp_error (pfile, CPP_DL_ICE, "invalid built-in macro \"%s\"",
 640                NODE_NAME (node));
 641   _cpp_pop_buffer (pfile);
 642
 643   return 1;
 644 }
 645
 646 /* Copies SRC, of length LEN, to DEST, adding backslashes before all
 647    backslashes and double quotes. DEST must be of sufficient size.
 648    Returns a pointer to the end of the string.  */
 649 uchar *
 650 cpp_quote_string (uchar *dest, const uchar *src, unsigned int len)
 651 {
 652   while (len--)
 653     {
 654       uchar c = *src++;
 655
 656       switch (c)
 657         {
 658         case '\n':
 659           /* Naked LF can appear in raw string literals  */
 660           c = 'n';
 661           /* FALLTHROUGH */
 662
 663         case '\\':
 664         case '"':
 665           *dest++ = '\\';
 666           /* FALLTHROUGH */
 667
 668         default:
 669           *dest++ = c;
 670         }
 671     }
 672
 673   return dest;
 674 }
 675
 676 /* Convert a token sequence ARG to a single string token according to
 677    the rules of the ISO C #-operator.  */
 678 static const cpp_token *
 679 stringify_arg (cpp_reader *pfile, macro_arg *arg)
 680 {
 681   unsigned char *dest;
 682   unsigned int i, escape_it, backslash_count = 0;
 683   const cpp_token *source = NULL;
 684   size_t len;
 685
 686   if (BUFF_ROOM (pfile->u_buff) < 3)
 687     _cpp_extend_buff (pfile, &pfile->u_buff, 3);
 688   dest = BUFF_FRONT (pfile->u_buff);
 689   *dest++ = '"';
 690
 691   /* Loop, reading in the argument's tokens.  */
 692   for (i = 0; i < arg->count; i++)
 693     {
 694       const cpp_token *token = arg->first[i];
 695
 696       if (token->type == CPP_PADDING)
 697         {
 698           if (source == NULL
 699               || (!(source->flags & PREV_WHITE)
 700                   && token->val.source == NULL))
 701             source = token->val.source;
 702           continue;
 703         }
 704
 705       escape_it = (token->type == CPP_STRING || token->type == CPP_CHAR
 706                    || token->type == CPP_WSTRING || token->type == CPP_WCHAR
 707                    || token->type == CPP_STRING32 || token->type == CPP_CHAR32
 708                    || token->type == CPP_STRING16 || token->type == CPP_CHAR16
 709                    || token->type == CPP_UTF8STRING || token->type == CPP_UTF8CHAR
 710                    || cpp_userdef_string_p (token->type)
 711                    || cpp_userdef_char_p (token->type));
 712
 713       /* Room for each char being written in octal, initial space and
 714          final quote and NUL.  */
 715       len = cpp_token_len (token);
 716       if (escape_it)
 717         len *= 4;
 718       len += 3;
 719
 720       if ((size_t) (BUFF_LIMIT (pfile->u_buff) - dest) < len)
 721         {
 722           size_t len_so_far = dest - BUFF_FRONT (pfile->u_buff);
 723           _cpp_extend_buff (pfile, &pfile->u_buff, len);
 724           dest = BUFF_FRONT (pfile->u_buff) + len_so_far;
 725         }
 726
 727       /* Leading white space?  */
 728       if (dest - 1 != BUFF_FRONT (pfile->u_buff))
 729         {
 730           if (source == NULL)
 731             source = token;
 732           if (source->flags & PREV_WHITE)
 733             *dest++ = ' ';
 734         }
 735       source = NULL;
 736
 737       if (escape_it)
 738         {
 739           _cpp_buff *buff = _cpp_get_buff (pfile, len);
 740           unsigned char *buf = BUFF_FRONT (buff);
 741           len = cpp_spell_token (pfile, token, buf, true) - buf;
 742           dest = cpp_quote_string (dest, buf, len);
 743           _cpp_release_buff (pfile, buff);
 744         }
 745       else
 746         dest = cpp_spell_token (pfile, token, dest, true);
 747
 748       if (token->type == CPP_OTHER && token->val.str.text[0] == '\\')
 749         backslash_count++;
 750       else
 751         backslash_count = 0;
 752     }
 753
 754   /* Ignore the final \ of invalid string literals.  */
 755   if (backslash_count & 1)
 756     {
 757       cpp_error (pfile, CPP_DL_WARNING,
 758                  "invalid string literal, ignoring final '\\'");
 759       dest--;
 760     }
 761
 762   /* Commit the memory, including NUL, and return the token.  */
 763   *dest++ = '"';
 764   len = dest - BUFF_FRONT (pfile->u_buff);
 765   BUFF_FRONT (pfile->u_buff) = dest + 1;
 766   return new_string_token (pfile, dest - len, len);
 767 }
 768
 769 /* Try to paste two tokens.  On success, return nonzero.  In any
 770    case, PLHS is updated to point to the pasted token, which is
 771    guaranteed to not have the PASTE_LEFT flag set.  LOCATION is
 772    the virtual location used for error reporting.  */
 773 static bool
 774 paste_tokens (cpp_reader *pfile, source_location location,
 775               const cpp_token **plhs, const cpp_token *rhs)
 776 {
 777   unsigned char *buf, *end, *lhsend;
 778   cpp_token *lhs;
 779   unsigned int len;
 780
 781   len = cpp_token_len (*plhs) + cpp_token_len (rhs) + 1;
 782   buf = (unsigned char *) alloca (len);
 783   end = lhsend = cpp_spell_token (pfile, *plhs, buf, true);
 784
 785   /* Avoid comment headers, since they are still processed in stage 3.
 786      It is simpler to insert a space here, rather than modifying the
 787      lexer to ignore comments in some circumstances.  Simply returning
 788      false doesn't work, since we want to clear the PASTE_LEFT flag.  */
 789   if ((*plhs)->type == CPP_DIV && rhs->type != CPP_EQ)
 790     *end++ = ' ';
 791   /* In one obscure case we might see padding here.  */
 792   if (rhs->type != CPP_PADDING)
 793     end = cpp_spell_token (pfile, rhs, end, true);
 794   *end = '\n';
 795
 796   cpp_push_buffer (pfile, buf, end - buf, /* from_stage3 */ true);
 797   _cpp_clean_line (pfile);
 798
 799   /* Set pfile->cur_token as required by _cpp_lex_direct.  */
 800   pfile->cur_token = _cpp_temp_token (pfile);
 801   lhs = _cpp_lex_direct (pfile);
 802   if (pfile->buffer->cur != pfile->buffer->rlimit)
 803     {
 804       source_location saved_loc = lhs->src_loc;
 805
 806       _cpp_pop_buffer (pfile);
 807       _cpp_backup_tokens (pfile, 1);
 808       *lhsend = '\0';
 809
 810       /* We have to remove the PASTE_LEFT flag from the old lhs, but
 811          we want to keep the new location.  */
 812       *lhs = **plhs;
 813       *plhs = lhs;
 814       lhs->src_loc = saved_loc;
 815       lhs->flags &= ~PASTE_LEFT;
 816
 817       /* Mandatory error for all apart from assembler.  */
 818       if (CPP_OPTION (pfile, lang) != CLK_ASM)
 819         cpp_error_with_line (pfile, CPP_DL_ERROR, location, 0,
 820          "pasting \"%s\" and \"%s\" does not give a valid preprocessing token",
 821                    buf, cpp_token_as_text (pfile, rhs));
 822       return false;
 823     }
 824
 825   *plhs = lhs;
 826   _cpp_pop_buffer (pfile);
 827   return true;
 828 }
 829
 830 /* Handles an arbitrarily long sequence of ## operators, with initial
 831    operand LHS.  This implementation is left-associative,
 832    non-recursive, and finishes a paste before handling succeeding
 833    ones.  If a paste fails, we back up to the RHS of the failing ##
 834    operator before pushing the context containing the result of prior
 835    successful pastes, with the effect that the RHS appears in the
 836    output stream after the pasted LHS normally.  */
 837 static void
 838 paste_all_tokens (cpp_reader *pfile, const cpp_token *lhs)
 839 {
 840   const cpp_token *rhs = NULL;
 841   cpp_context *context = pfile->context;
 842   source_location virt_loc = 0;
 843
 844   /* We are expanding a macro and we must have been called on a token
 845      that appears at the left hand side of a ## operator.  */
 846   if (macro_of_context (pfile->context) == NULL
 847       || (!(lhs->flags & PASTE_LEFT)))
 848     abort ();
 849
 850   if (context->tokens_kind == TOKENS_KIND_EXTENDED)
 851     /* The caller must have called consume_next_token_from_context
 852        right before calling us.  That has incremented the pointer to
 853        the current virtual location.  So it now points to the location
 854        of the token that comes right after *LHS.  We want the
 855        resulting pasted token to have the location of the current
 856        *LHS, though.  */
 857     virt_loc = context->c.mc->cur_virt_loc[-1];
 858   else
 859     /* We are not tracking macro expansion.  So the best virtual
 860        location we can get here is the expansion point of the macro we
 861        are currently expanding.  */
 862     virt_loc = pfile->invocation_location;
 863
 864   do
 865     {
 866       /* Take the token directly from the current context.  We can do
 867          this, because we are in the replacement list of either an
 868          object-like macro, or a function-like macro with arguments
 869          inserted.  In either case, the constraints to #define
 870          guarantee we have at least one more token.  */
 871       if (context->tokens_kind == TOKENS_KIND_DIRECT)
 872         rhs = FIRST (context).token++;
 873       else if (context->tokens_kind == TOKENS_KIND_INDIRECT)
 874         rhs = *FIRST (context).ptoken++;
 875       else if (context->tokens_kind == TOKENS_KIND_EXTENDED)
 876         {
 877           /* So we are in presence of an extended token context, which
 878              means that each token in this context has a virtual
 879              location attached to it.  So let's not forget to update
 880              the pointer to the current virtual location of the
 881              current token when we update the pointer to the current
 882              token */
 883
 884           rhs = *FIRST (context).ptoken++;
 885           /* context->c.mc must be non-null, as if we were not in a
 886              macro context, context->tokens_kind could not be equal to
 887              TOKENS_KIND_EXTENDED.  */
 888           context->c.mc->cur_virt_loc++;
 889         }
 890
 891       if (rhs->type == CPP_PADDING)
 892         {
 893           if (rhs->flags & PASTE_LEFT)
 894             abort ();
 895         }
 896       if (!paste_tokens (pfile, virt_loc, &lhs, rhs))
 897         break;
 898     }
 899   while (rhs->flags & PASTE_LEFT);
 900
 901   /* Put the resulting token in its own context.  */
 902   if (context->tokens_kind == TOKENS_KIND_EXTENDED)
 903     {
 904       source_location *virt_locs = NULL;
 905       _cpp_buff *token_buf = tokens_buff_new (pfile, 1, &virt_locs);
 906       tokens_buff_add_token (token_buf, virt_locs, lhs,
 907                              virt_loc, 0, NULL, 0);
 908       push_extended_tokens_context (pfile, context->c.mc->macro_node,
 909                                     token_buf, virt_locs,
 910                                     (const cpp_token **)token_buf->base, 1);
 911     }
 912   else
 913     _cpp_push_token_context (pfile, NULL, lhs, 1);
 914 }
 915
 916 /* Returns TRUE if the number of arguments ARGC supplied in an
 917    invocation of the MACRO referenced by NODE is valid.  An empty
 918    invocation to a macro with no parameters should pass ARGC as zero.
 919
 920    Note that MACRO cannot necessarily be deduced from NODE, in case
 921    NODE was redefined whilst collecting arguments.  */
 922 bool
 923 _cpp_arguments_ok (cpp_reader *pfile, cpp_macro *macro, const cpp_hashnode *node, unsigned int argc)
 924 {
 925   if (argc == macro->paramc)
 926     return true;
 927
 928   if (argc < macro->paramc)
 929     {
 930       /* In C++2a (here the va_opt flag is used), and also as a GNU
 931          extension, variadic arguments are allowed to not appear in
 932          the invocation at all.
 933          e.g. #define debug(format, args...) something
 934          debug("string");
 935
 936          This is exactly the same as if an empty variadic list had been
 937          supplied - debug("string", ).  */
 938
 939       if (argc + 1 == macro->paramc && macro->variadic)
 940         {
 941           if (CPP_PEDANTIC (pfile) && ! macro->syshdr
 942               && ! CPP_OPTION (pfile, va_opt))
 943             {
 944               if (CPP_OPTION (pfile, cplusplus))
 945                 cpp_error (pfile, CPP_DL_PEDWARN,
 946                            "ISO C++11 requires at least one argument "
 947                            "for the \"...\" in a variadic macro");
 948               else
 949                 cpp_error (pfile, CPP_DL_PEDWARN,
 950                            "ISO C99 requires at least one argument "
 951                            "for the \"...\" in a variadic macro");
 952             }
 953           return true;
 954         }
 955
 956       cpp_error (pfile, CPP_DL_ERROR,
 957                  "macro \"%s\" requires %u arguments, but only %u given",
 958                  NODE_NAME (node), macro->paramc, argc);
 959     }
 960   else
 961     cpp_error (pfile, CPP_DL_ERROR,
 962                "macro \"%s\" passed %u arguments, but takes just %u",
 963                NODE_NAME (node), argc, macro->paramc);
 964
 965   return false;
 966 }
 967
 968 /* Reads and returns the arguments to a function-like macro
 969    invocation.  Assumes the opening parenthesis has been processed.
 970    If there is an error, emits an appropriate diagnostic and returns
 971    NULL.  Each argument is terminated by a CPP_EOF token, for the
 972    future benefit of expand_arg().  If there are any deferred
 973    #pragma directives among macro arguments, store pointers to the
 974    CPP_PRAGMA ... CPP_PRAGMA_EOL tokens into *PRAGMA_BUFF buffer.
 975
 976    What is returned is the buffer that contains the memory allocated
 977    to hold the macro arguments.  NODE is the name of the macro this
 978    function is dealing with.  If NUM_ARGS is non-NULL, *NUM_ARGS is
 979    set to the actual number of macro arguments allocated in the
 980    returned buffer.  */
 981 static _cpp_buff *
 982 collect_args (cpp_reader *pfile, const cpp_hashnode *node,
 983               _cpp_buff **pragma_buff, unsigned *num_args)
 984 {
 985   _cpp_buff *buff, *base_buff;
 986   cpp_macro *macro;
 987   macro_arg *args, *arg;
 988   const cpp_token *token;
 989   unsigned int argc;
 990   source_location virt_loc;
 991   bool track_macro_expansion_p = CPP_OPTION (pfile, track_macro_expansion);
 992   unsigned num_args_alloced = 0;
 993
 994   macro = node->value.macro;
 995   if (macro->paramc)
 996     argc = macro->paramc;
 997   else
 998     argc = 1;
 999
1000 #define DEFAULT_NUM_TOKENS_PER_MACRO_ARG 50
1001 #define ARG_TOKENS_EXTENT 1000
1002
1003   buff = _cpp_get_buff (pfile, argc * (DEFAULT_NUM_TOKENS_PER_MACRO_ARG
1004                                        * sizeof (cpp_token *)
1005                                        + sizeof (macro_arg)));
1006   base_buff = buff;
1007   args = (macro_arg *) buff->base;
1008   memset (args, 0, argc * sizeof (macro_arg));
1009   buff->cur = (unsigned char *) &args[argc];
1010   arg = args, argc = 0;
1011
1012   /* Collect the tokens making up each argument.  We don't yet know
1013      how many arguments have been supplied, whether too many or too
1014      few.  Hence the slightly bizarre usage of "argc" and "arg".  */
1015   do
1016     {
1017       unsigned int paren_depth = 0;
1018       unsigned int ntokens = 0;
1019       unsigned virt_locs_capacity = DEFAULT_NUM_TOKENS_PER_MACRO_ARG;
1020       num_args_alloced++;
1021
1022       argc++;
1023       arg->first = (const cpp_token **) buff->cur;
1024       if (track_macro_expansion_p)
1025         {
1026           virt_locs_capacity = DEFAULT_NUM_TOKENS_PER_MACRO_ARG;
1027           arg->virt_locs = XNEWVEC (source_location,
1028                                     virt_locs_capacity);
1029         }
1030
1031       for (;;)
1032         {
1033           /* Require space for 2 new tokens (including a CPP_EOF).  */
1034           if ((unsigned char *) &arg->first[ntokens + 2] > buff->limit)
1035             {
1036               buff = _cpp_append_extend_buff (pfile, buff,
1037                                               ARG_TOKENS_EXTENT
1038                                               * sizeof (cpp_token *));
1039               arg->first = (const cpp_token **) buff->cur;
1040             }
1041           if (track_macro_expansion_p
1042               && (ntokens + 2 > virt_locs_capacity))
1043             {
1044               virt_locs_capacity += ARG_TOKENS_EXTENT;
1045               arg->virt_locs = XRESIZEVEC (source_location,
1046                                            arg->virt_locs,
1047                                            virt_locs_capacity);
1048             }
1049
1050           token = cpp_get_token_1 (pfile, &virt_loc);
1051
1052           if (token->type == CPP_PADDING)
1053             {
1054               /* Drop leading padding.  */
1055               if (ntokens == 0)
1056                 continue;
1057             }
1058           else if (token->type == CPP_OPEN_PAREN)
1059             paren_depth++;
1060           else if (token->type == CPP_CLOSE_PAREN)
1061             {
1062               if (paren_depth-- == 0)
1063                 break;
1064             }
1065           else if (token->type == CPP_COMMA)
1066             {
1067               /* A comma does not terminate an argument within
1068                  parentheses or as part of a variable argument.  */
1069               if (paren_depth == 0
1070                   && ! (macro->variadic && argc == macro->paramc))
1071                 break;
1072             }
1073           else if (token->type == CPP_EOF
1074                    || (token->type == CPP_HASH && token->flags & BOL))
1075             break;
1076           else if (token->type == CPP_PRAGMA)
1077             {
1078               cpp_token *newtok = _cpp_temp_token (pfile);
1079
1080               /* CPP_PRAGMA token lives in directive_result, which will
1081                  be overwritten on the next directive.  */
1082               *newtok = *token;
1083               token = newtok;
1084               do
1085                 {
1086                   if (*pragma_buff == NULL
1087                       || BUFF_ROOM (*pragma_buff) < sizeof (cpp_token *))
1088                     {
1089                       _cpp_buff *next;
1090                       if (*pragma_buff == NULL)
1091                         *pragma_buff
1092                           = _cpp_get_buff (pfile, 32 * sizeof (cpp_token *));
1093                       else
1094                         {
1095                           next = *pragma_buff;
1096                           *pragma_buff
1097                             = _cpp_get_buff (pfile,
1098                                              (BUFF_FRONT (*pragma_buff)
1099                                               - (*pragma_buff)->base) * 2);
1100                           (*pragma_buff)->next = next;
1101                         }
1102                     }
1103                   *(const cpp_token **) BUFF_FRONT (*pragma_buff) = token;
1104                   BUFF_FRONT (*pragma_buff) += sizeof (cpp_token *);
1105                   if (token->type == CPP_PRAGMA_EOL)
1106                     break;
1107                   token = cpp_get_token_1 (pfile, &virt_loc);
1108                 }
1109               while (token->type != CPP_EOF);
1110
1111               /* In deferred pragmas parsing_args and prevent_expansion
1112                  had been changed, reset it.  */
1113               pfile->state.parsing_args = 2;
1114               pfile->state.prevent_expansion = 1;
1115
1116               if (token->type == CPP_EOF)
1117                 break;
1118               else
1119                 continue;
1120             }
1121           set_arg_token (arg, token, virt_loc,
1122                          ntokens, MACRO_ARG_TOKEN_NORMAL,
1123                          CPP_OPTION (pfile, track_macro_expansion));
1124           ntokens++;
1125         }
1126
1127       /* Drop trailing padding.  */
1128       while (ntokens > 0 && arg->first[ntokens - 1]->type == CPP_PADDING)
1129         ntokens--;
1130
1131       arg->count = ntokens;
1132       set_arg_token (arg, &pfile->eof, pfile->eof.src_loc,
1133                      ntokens, MACRO_ARG_TOKEN_NORMAL,
1134                      CPP_OPTION (pfile, track_macro_expansion));
1135
1136       /* Terminate the argument.  Excess arguments loop back and
1137          overwrite the final legitimate argument, before failing.  */
1138       if (argc <= macro->paramc)
1139         {
1140           buff->cur = (unsigned char *) &arg->first[ntokens + 1];
1141           if (argc != macro->paramc)
1142             arg++;
1143         }
1144     }
1145   while (token->type != CPP_CLOSE_PAREN && token->type != CPP_EOF);
1146
1147   if (token->type == CPP_EOF)
1148     {
1149       /* We still need the CPP_EOF to end directives, and to end
1150          pre-expansion of a macro argument.  Step back is not
1151          unconditional, since we don't want to return a CPP_EOF to our
1152          callers at the end of an -include-d file.  */
1153       if (pfile->context->prev || pfile->state.in_directive)
1154         _cpp_backup_tokens (pfile, 1);
1155       cpp_error (pfile, CPP_DL_ERROR,
1156                  "unterminated argument list invoking macro \"%s\"",
1157                  NODE_NAME (node));
1158     }
1159   else
1160     {
1161       /* A single empty argument is counted as no argument.  */
1162       if (argc == 1 && macro->paramc == 0 && args[0].count == 0)
1163         argc = 0;
1164       if (_cpp_arguments_ok (pfile, macro, node, argc))
1165         {
1166           /* GCC has special semantics for , ## b where b is a varargs
1167              parameter: we remove the comma if b was omitted entirely.
1168              If b was merely an empty argument, the comma is retained.
1169              If the macro takes just one (varargs) parameter, then we
1170              retain the comma only if we are standards conforming.
1171
1172              If FIRST is NULL replace_args () swallows the comma.  */
1173           if (macro->variadic && (argc < macro->paramc
1174                                   || (argc == 1 && args[0].count == 0
1175                                       && !CPP_OPTION (pfile, std))))
1176             args[macro->paramc - 1].first = NULL;
1177           if (num_args)
1178             *num_args = num_args_alloced;
1179           return base_buff;
1180         }
1181     }
1182
1183   /* An error occurred.  */
1184   _cpp_release_buff (pfile, base_buff);
1185   return NULL;
1186 }
1187
1188 /* Search for an opening parenthesis to the macro of NODE, in such a
1189    way that, if none is found, we don't lose the information in any
1190    intervening padding tokens.  If we find the parenthesis, collect
1191    the arguments and return the buffer containing them.  PRAGMA_BUFF
1192    argument is the same as in collect_args.  If NUM_ARGS is non-NULL,
1193    *NUM_ARGS is set to the number of arguments contained in the
1194    returned buffer.  */
1195 static _cpp_buff *
1196 funlike_invocation_p (cpp_reader *pfile, cpp_hashnode *node,
1197                       _cpp_buff **pragma_buff, unsigned *num_args)
1198 {
1199   const cpp_token *token, *padding = NULL;
1200
1201   for (;;)
1202     {
1203       token = cpp_get_token (pfile);
1204       if (token->type != CPP_PADDING)
1205         break;
1206       if (padding == NULL
1207           || (!(padding->flags & PREV_WHITE) && token->val.source == NULL))
1208         padding = token;
1209     }
1210
1211   if (token->type == CPP_OPEN_PAREN)
1212     {
1213       pfile->state.parsing_args = 2;
1214       return collect_args (pfile, node, pragma_buff, num_args);
1215     }
1216
1217   /* CPP_EOF can be the end of macro arguments, or the end of the
1218      file.  We mustn't back up over the latter.  Ugh.  */
1219   if (token->type != CPP_EOF || token == &pfile->eof)
1220     {
1221       /* Back up.  We may have skipped padding, in which case backing
1222          up more than one token when expanding macros is in general
1223          too difficult.  We re-insert it in its own context.  */
1224       _cpp_backup_tokens (pfile, 1);
1225       if (padding)
1226         _cpp_push_token_context (pfile, NULL, padding, 1);
1227     }
1228
1229   return NULL;
1230 }
1231
1232 /* Return the real number of tokens in the expansion of MACRO.  */
1233 static inline unsigned int
1234 macro_real_token_count (const cpp_macro *macro)
1235 {
1236   unsigned int i;
1237   if (__builtin_expect (!macro->extra_tokens, true))
1238     return macro->count;
1239   for (i = 0; i < macro->count; i++)
1240     if (macro->exp.tokens[i].type == CPP_PASTE)
1241       return i;
1242   abort ();
1243 }
1244
1245 /* Push the context of a macro with hash entry NODE onto the context
1246    stack.  If we can successfully expand the macro, we push a context
1247    containing its yet-to-be-rescanned replacement list and return one.
1248    If there were additionally any unexpanded deferred #pragma
1249    directives among macro arguments, push another context containing
1250    the pragma tokens before the yet-to-be-rescanned replacement list
1251    and return two.  Otherwise, we don't push a context and return
1252    zero. LOCATION is the location of the expansion point of the
1253    macro.  */
1254 static int
1255 enter_macro_context (cpp_reader *pfile, cpp_hashnode *node,
1256                      const cpp_token *result, source_location location)
1257 {
1258   /* The presence of a macro invalidates a file's controlling macro.  */
1259   pfile->mi_valid = false;
1260
1261   pfile->state.angled_headers = false;
1262
1263   /* From here to when we push the context for the macro later down
1264      this function, we need to flag the fact that we are about to
1265      expand a macro.  This is useful when -ftrack-macro-expansion is
1266      turned off.  In that case, we need to record the location of the
1267      expansion point of the top-most macro we are about to to expand,
1268      into pfile->invocation_location.  But we must not record any such
1269      location once the process of expanding the macro starts; that is,
1270      we must not do that recording between now and later down this
1271      function where set this flag to FALSE.  */
1272   pfile->about_to_expand_macro_p = true;
1273
1274   if ((node->flags & NODE_BUILTIN) && !(node->flags & NODE_USED))
1275     {
1276       node->flags |= NODE_USED;
1277       if ((!pfile->cb.user_builtin_macro
1278            || !pfile->cb.user_builtin_macro (pfile, node))
1279           && pfile->cb.used_define)
1280         pfile->cb.used_define (pfile, pfile->directive_line, node);
1281     }
1282
1283   /* Handle standard macros.  */
1284   if (! (node->flags & NODE_BUILTIN))
1285     {
1286       cpp_macro *macro = node->value.macro;
1287       _cpp_buff *pragma_buff = NULL;
1288
1289       if (macro->fun_like)
1290         {
1291           _cpp_buff *buff;
1292           unsigned num_args = 0;
1293
1294           pfile->state.prevent_expansion++;
1295           pfile->keep_tokens++;
1296           pfile->state.parsing_args = 1;
1297           buff = funlike_invocation_p (pfile, node, &pragma_buff,
1298                                        &num_args);
1299           pfile->state.parsing_args = 0;
1300           pfile->keep_tokens--;
1301           pfile->state.prevent_expansion--;
1302
1303           if (buff == NULL)
1304             {
1305               if (CPP_WTRADITIONAL (pfile) && ! node->value.macro->syshdr)
1306                 cpp_warning (pfile, CPP_W_TRADITIONAL,
1307  "function-like macro \"%s\" must be used with arguments in traditional C",
1308                              NODE_NAME (node));
1309
1310               if (pragma_buff)
1311                 _cpp_release_buff (pfile, pragma_buff);
1312
1313               pfile->about_to_expand_macro_p = false;
1314               return 0;
1315             }
1316
1317           if (macro->paramc > 0)
1318             replace_args (pfile, node, macro,
1319                           (macro_arg *) buff->base,
1320                           location);
1321           /* Free the memory used by the arguments of this
1322              function-like macro.  This memory has been allocated by
1323              funlike_invocation_p and by replace_args.  */
1324           delete_macro_args (buff, num_args);
1325         }
1326
1327       /* Disable the macro within its expansion.  */
1328       node->flags |= NODE_DISABLED;
1329
1330       if (!(node->flags & NODE_USED))
1331         {
1332           node->flags |= NODE_USED;
1333           if (pfile->cb.used_define)
1334             pfile->cb.used_define (pfile, pfile->directive_line, node);
1335         }
1336
1337       if (pfile->cb.used)
1338         pfile->cb.used (pfile, location, node);
1339
1340       macro->used = 1;
1341
1342       if (macro->paramc == 0)
1343         {
1344           unsigned tokens_count = macro_real_token_count (macro);
1345           if (CPP_OPTION (pfile, track_macro_expansion))
1346             {
1347               unsigned int i;
1348               const cpp_token *src = macro->exp.tokens;
1349               const line_map_macro *map;
1350               source_location *virt_locs = NULL;
1351               _cpp_buff *macro_tokens
1352                 = tokens_buff_new (pfile, tokens_count, &virt_locs);
1353
1354               /* Create a macro map to record the locations of the
1355                  tokens that are involved in the expansion. LOCATION
1356                  is the location of the macro expansion point.  */
1357               map = linemap_enter_macro (pfile->line_table,
1358                                          node, location, tokens_count);
1359               for (i = 0; i < tokens_count; ++i)
1360                 {
1361                   tokens_buff_add_token (macro_tokens, virt_locs,
1362                                          src, src->src_loc,
1363                                          src->src_loc, map, i);
1364                   ++src;
1365                 }
1366               push_extended_tokens_context (pfile, node,
1367                                             macro_tokens,
1368                                             virt_locs,
1369                                             (const cpp_token **)
1370                                             macro_tokens->base,
1371                                             tokens_count);
1372             }
1373           else
1374             _cpp_push_token_context (pfile, node, macro->exp.tokens,
1375                                      tokens_count);
1376           num_macro_tokens_counter += tokens_count;
1377         }
1378
1379       if (pragma_buff)
1380         {
1381           if (!pfile->state.in_directive)
1382             _cpp_push_token_context (pfile, NULL,
1383                                      padding_token (pfile, result), 1);
1384           do
1385             {
1386               unsigned tokens_count;
1387               _cpp_buff *tail = pragma_buff->next;
1388               pragma_buff->next = NULL;
1389               tokens_count = ((const cpp_token **) BUFF_FRONT (pragma_buff)
1390                               - (const cpp_token **) pragma_buff->base);
1391               push_ptoken_context (pfile, NULL, pragma_buff,
1392                                    (const cpp_token **) pragma_buff->base,
1393                                    tokens_count);
1394               pragma_buff = tail;
1395               if (!CPP_OPTION (pfile, track_macro_expansion))
1396                 num_macro_tokens_counter += tokens_count;
1397
1398             }
1399           while (pragma_buff != NULL);
1400           pfile->about_to_expand_macro_p = false;
1401           return 2;
1402         }
1403
1404       pfile->about_to_expand_macro_p = false;
1405       return 1;
1406     }
1407
1408   pfile->about_to_expand_macro_p = false;
1409   /* Handle built-in macros and the _Pragma operator.  */
1410   {
1411     source_location loc, expand_loc;
1412
1413     if (/* The top-level macro invocation that triggered the expansion
1414            we are looking at is with a standard macro ...*/
1415         !(pfile->top_most_macro_node->flags & NODE_BUILTIN)
1416         /* ... and it's a function-like macro invocation.  */
1417         && pfile->top_most_macro_node->value.macro->fun_like)
1418       {
1419         /* Then the location of the end of the macro invocation is the
1420            location of the closing parenthesis.  */
1421         loc = pfile->cur_token[-1].src_loc;
1422         expand_loc = loc;
1423       }
1424     else
1425       {
1426         /* Otherwise, the location of the end of the macro invocation is
1427            the location of the expansion point of that top-level macro
1428            invocation.  */
1429         loc = location;
1430         expand_loc = pfile->invocation_location;
1431       }
1432
1433     return builtin_macro (pfile, node, loc, expand_loc);
1434   }
1435 }
1436
1437 /* De-allocate the memory used by BUFF which is an array of instances
1438    of macro_arg.  NUM_ARGS is the number of instances of macro_arg
1439    present in BUFF.  */
1440 static void
1441 delete_macro_args (_cpp_buff *buff, unsigned num_args)
1442 {
1443   macro_arg *macro_args;
1444   unsigned i;
1445
1446   if (buff == NULL)
1447     return;
1448
1449   macro_args = (macro_arg *) buff->base;
1450
1451   /* Walk instances of macro_arg to free their expanded tokens as well
1452      as their macro_arg::virt_locs members.  */
1453   for (i = 0; i < num_args; ++i)
1454     {
1455       if (macro_args[i].expanded)
1456         {
1457           free (macro_args[i].expanded);
1458           macro_args[i].expanded = NULL;
1459         }
1460       if (macro_args[i].virt_locs)
1461         {
1462           free (macro_args[i].virt_locs);
1463           macro_args[i].virt_locs = NULL;
1464         }
1465       if (macro_args[i].expanded_virt_locs)
1466         {
1467           free (macro_args[i].expanded_virt_locs);
1468           macro_args[i].expanded_virt_locs = NULL;
1469         }
1470     }
1471   _cpp_free_buff (buff);
1472 }
1473
1474 /* Set the INDEXth token of the macro argument ARG. TOKEN is the token
1475    to set, LOCATION is its virtual location.  "Virtual" location means
1476    the location that encodes loci across macro expansion. Otherwise
1477    it has to be TOKEN->SRC_LOC.  KIND is the kind of tokens the
1478    argument ARG is supposed to contain.  Note that ARG must be
1479    tailored so that it has enough room to contain INDEX + 1 numbers of
1480    tokens, at least.  */
1481 static void
1482 set_arg_token (macro_arg *arg, const cpp_token *token,
1483                source_location location, size_t index,
1484                enum macro_arg_token_kind kind,
1485                bool track_macro_exp_p)
1486 {
1487   const cpp_token **token_ptr;
1488   source_location *loc = NULL;
1489
1490   token_ptr =
1491     arg_token_ptr_at (arg, index, kind,
1492                       track_macro_exp_p ? &loc : NULL);
1493   *token_ptr = token;
1494
1495   if (loc != NULL)
1496     {
1497       /* We can't set the location of a stringified argument
1498          token and we can't set any location if we aren't tracking
1499          macro expansion locations.   */
1500       gcc_checking_assert (kind != MACRO_ARG_TOKEN_STRINGIFIED
1501                            && track_macro_exp_p);
1502       *loc = location;
1503     }
1504 }
1505
1506 /* Get the pointer to the location of the argument token of the
1507    function-like macro argument ARG.  This function must be called
1508    only when we -ftrack-macro-expansion is on.  */
1509 static const source_location *
1510 get_arg_token_location (const macro_arg *arg,
1511                         enum macro_arg_token_kind kind)
1512 {
1513   const source_location *loc = NULL;
1514   const cpp_token **token_ptr =
1515     arg_token_ptr_at (arg, 0, kind, (source_location **) &loc);
1516
1517   if (token_ptr == NULL)
1518     return NULL;
1519
1520   return loc;
1521 }
1522
1523 /* Return the pointer to the INDEXth token of the macro argument ARG.
1524    KIND specifies the kind of token the macro argument ARG contains.
1525    If VIRT_LOCATION is non NULL, *VIRT_LOCATION is set to the address
1526    of the virtual location of the returned token if the
1527    -ftrack-macro-expansion flag is on; otherwise, it's set to the
1528    spelling location of the returned token.  */
1529 static const cpp_token **
1530 arg_token_ptr_at (const macro_arg *arg, size_t index,
1531                   enum macro_arg_token_kind kind,
1532                   source_location **virt_location)
1533 {
1534   const cpp_token **tokens_ptr = NULL;
1535
1536   switch (kind)
1537     {
1538     case MACRO_ARG_TOKEN_NORMAL:
1539       tokens_ptr = arg->first;
1540       break;
1541     case MACRO_ARG_TOKEN_STRINGIFIED:
1542       tokens_ptr = (const cpp_token **) &arg->stringified;
1543       break;
1544     case MACRO_ARG_TOKEN_EXPANDED:
1545         tokens_ptr = arg->expanded;
1546       break;
1547     }
1548
1549   if (tokens_ptr == NULL)
1550     /* This can happen for e.g, an empty token argument to a
1551        funtion-like macro.  */
1552     return tokens_ptr;
1553
1554   if (virt_location)
1555     {
1556       if (kind == MACRO_ARG_TOKEN_NORMAL)
1557         *virt_location = &arg->virt_locs[index];
1558       else if (kind == MACRO_ARG_TOKEN_EXPANDED)
1559         *virt_location = &arg->expanded_virt_locs[index];
1560       else if (kind == MACRO_ARG_TOKEN_STRINGIFIED)
1561         *virt_location =
1562           (source_location *) &tokens_ptr[index]->src_loc;
1563     }
1564   return &tokens_ptr[index];
1565 }
1566
1567 /* Initialize an iterator so that it iterates over the tokens of a
1568    function-like macro argument.  KIND is the kind of tokens we want
1569    ITER to iterate over. TOKEN_PTR points the first token ITER will
1570    iterate over.  */
1571 static void
1572 macro_arg_token_iter_init (macro_arg_token_iter *iter,
1573                            bool track_macro_exp_p,
1574                            enum macro_arg_token_kind kind,
1575                            const macro_arg *arg,
1576                            const cpp_token **token_ptr)
1577 {
1578   iter->track_macro_exp_p = track_macro_exp_p;
1579   iter->kind = kind;
1580   iter->token_ptr = token_ptr;
1581   /* Unconditionally initialize this so that the compiler doesn't warn
1582      about iter->location_ptr being possibly uninitialized later after
1583      this code has been inlined somewhere.  */
1584   iter->location_ptr = NULL;
1585   if (track_macro_exp_p)
1586     iter->location_ptr = get_arg_token_location (arg, kind);
1587 #if CHECKING_P
1588   iter->num_forwards = 0;
1589   if (track_macro_exp_p
1590       && token_ptr != NULL
1591       && iter->location_ptr == NULL)
1592     abort ();
1593 #endif
1594 }
1595
1596 /* Move the iterator one token forward. Note that if IT was
1597    initialized on an argument that has a stringified token, moving it
1598    forward doesn't make sense as a stringified token is essentially one
1599    string.  */
1600 static void
1601 macro_arg_token_iter_forward (macro_arg_token_iter *it)
1602 {
1603   switch (it->kind)
1604     {
1605     case MACRO_ARG_TOKEN_NORMAL:
1606     case MACRO_ARG_TOKEN_EXPANDED:
1607       it->token_ptr++;
1608       if (it->track_macro_exp_p)
1609         it->location_ptr++;
1610       break;
1611     case MACRO_ARG_TOKEN_STRINGIFIED:
1612 #if CHECKING_P
1613       if (it->num_forwards > 0)
1614         abort ();
1615 #endif
1616       break;
1617     }
1618
1619 #if CHECKING_P
1620   it->num_forwards++;
1621 #endif
1622 }
1623
1624 /* Return the token pointed to by the iterator.  */
1625 static const cpp_token *
1626 macro_arg_token_iter_get_token (const macro_arg_token_iter *it)
1627 {
1628 #if CHECKING_P
1629   if (it->kind == MACRO_ARG_TOKEN_STRINGIFIED
1630       && it->num_forwards > 0)
1631     abort ();
1632 #endif
1633   if (it->token_ptr == NULL)
1634     return NULL;
1635   return *it->token_ptr;
1636 }
1637
1638 /* Return the location of the token pointed to by the iterator.*/
1639 static source_location
1640 macro_arg_token_iter_get_location (const macro_arg_token_iter *it)
1641 {
1642 #if CHECKING_P
1643   if (it->kind == MACRO_ARG_TOKEN_STRINGIFIED
1644       && it->num_forwards > 0)
1645     abort ();
1646 #endif
1647   if (it->track_macro_exp_p)
1648     return *it->location_ptr;
1649   else
1650     return (*it->token_ptr)->src_loc;
1651 }
1652
1653 /* Return the index of a token [resulting from macro expansion] inside
1654    the total list of tokens resulting from a given macro
1655    expansion. The index can be different depending on whether if we
1656    want each tokens resulting from function-like macro arguments
1657    expansion to have a different location or not.
1658
1659    E.g, consider this function-like macro:
1660
1661         #define M(x) x - 3
1662
1663    Then consider us "calling" it (and thus expanding it) like:
1664
1665        M(1+4)
1666
1667    It will be expanded into:
1668
1669        1+4-3
1670
1671    Let's consider the case of the token '4'.
1672
1673    Its index can be 2 (it's the third token of the set of tokens
1674    resulting from the expansion) or it can be 0 if we consider that
1675    all tokens resulting from the expansion of the argument "1+2" have
1676    the same index, which is 0. In this later case, the index of token
1677    '-' would then be 1 and the index of token '3' would be 2.
1678
1679    The later case is useful to use less memory e.g, for the case of
1680    the user using the option -ftrack-macro-expansion=1.
1681
1682    ABSOLUTE_TOKEN_INDEX is the index of the macro argument token we
1683    are interested in.  CUR_REPLACEMENT_TOKEN is the token of the macro
1684    parameter (inside the macro replacement list) that corresponds to
1685    the macro argument for which ABSOLUTE_TOKEN_INDEX is a token index
1686    of.
1687
1688    If we refer to the example above, for the '4' argument token,
1689    ABSOLUTE_TOKEN_INDEX would be set to 2, and CUR_REPLACEMENT_TOKEN
1690    would be set to the token 'x', in the replacement list "x - 3" of
1691    macro M.
1692
1693    This is a subroutine of replace_args.  */
1694 inline static unsigned
1695 expanded_token_index (cpp_reader *pfile, cpp_macro *macro,
1696                       const cpp_token *cur_replacement_token,
1697                       unsigned absolute_token_index)
1698 {
1699   if (CPP_OPTION (pfile, track_macro_expansion) > 1)
1700     return absolute_token_index;
1701   return cur_replacement_token - macro->exp.tokens;
1702 }
1703
1704 /* Replace the parameters in a function-like macro of NODE with the
1705    actual ARGS, and place the result in a newly pushed token context.
1706    Expand each argument before replacing, unless it is operated upon
1707    by the # or ## operators. EXPANSION_POINT_LOC is the location of
1708    the expansion point of the macro. E.g, the location of the
1709    function-like macro invocation.  */
1710 static void
1711 replace_args (cpp_reader *pfile, cpp_hashnode *node, cpp_macro *macro,
1712               macro_arg *args, source_location expansion_point_loc)
1713 {
1714   unsigned int i, total;
1715   const cpp_token *src, *limit;
1716   const cpp_token **first = NULL;
1717   macro_arg *arg;
1718   _cpp_buff *buff = NULL;
1719   source_location *virt_locs = NULL;
1720   unsigned int exp_count;
1721   const line_map_macro *map = NULL;
1722   int track_macro_exp;
1723
1724   /* First, fully macro-expand arguments, calculating the number of
1725      tokens in the final expansion as we go.  The ordering of the if
1726      statements below is subtle; we must handle stringification before
1727      pasting.  */
1728
1729   /* EXP_COUNT is the number of tokens in the macro replacement
1730      list.  TOTAL is the number of tokens /after/ macro parameters
1731      have been replaced by their arguments.   */
1732   exp_count = macro_real_token_count (macro);
1733   total = exp_count;
1734   limit = macro->exp.tokens + exp_count;
1735
1736   for (src = macro->exp.tokens; src < limit; src++)
1737     if (src->type == CPP_MACRO_ARG)
1738       {
1739         /* Leading and trailing padding tokens.  */
1740         total += 2;
1741         /* Account for leading and padding tokens in exp_count too.
1742            This is going to be important later down this function,
1743            when we want to handle the case of (track_macro_exp <
1744            2).  */
1745         exp_count += 2;
1746
1747         /* We have an argument.  If it is not being stringified or
1748            pasted it is macro-replaced before insertion.  */
1749         arg = &args[src->val.macro_arg.arg_no - 1];
1750
1751         if (src->flags & STRINGIFY_ARG)
1752           {
1753             if (!arg->stringified)
1754               arg->stringified = stringify_arg (pfile, arg);
1755           }
1756         else if ((src->flags & PASTE_LEFT)
1757                  || (src > macro->exp.tokens && (src[-1].flags & PASTE_LEFT)))
1758           total += arg->count - 1;
1759         else
1760           {
1761             if (!arg->expanded)
1762               expand_arg (pfile, arg);
1763             total += arg->expanded_count - 1;
1764           }
1765       }
1766
1767   /* When the compiler is called with the -ftrack-macro-expansion
1768      flag, we need to keep track of the location of each token that
1769      results from macro expansion.
1770
1771      A token resulting from macro expansion is not a new token. It is
1772      simply the same token as the token coming from the macro
1773      definition.  The new things that are allocated are the buffer
1774      that holds the tokens resulting from macro expansion and a new
1775      location that records many things like the locus of the expansion
1776      point as well as the original locus inside the definition of the
1777      macro.  This location is called a virtual location.
1778
1779      So the buffer BUFF holds a set of cpp_token*, and the buffer
1780      VIRT_LOCS holds the virtual locations of the tokens held by BUFF.
1781
1782      Both of these two buffers are going to be hung off of the macro
1783      context, when the latter is pushed.  The memory allocated to
1784      store the tokens and their locations is going to be freed once
1785      the context of macro expansion is popped.
1786
1787      As far as tokens are concerned, the memory overhead of
1788      -ftrack-macro-expansion is proportional to the number of
1789      macros that get expanded multiplied by sizeof (source_location).
1790      The good news is that extra memory gets freed when the macro
1791      context is freed, i.e shortly after the macro got expanded.  */
1792
1793   /* Is the -ftrack-macro-expansion flag in effect?  */
1794   track_macro_exp = CPP_OPTION (pfile, track_macro_expansion);
1795
1796   /* Now allocate memory space for tokens and locations resulting from
1797      the macro expansion, copy the tokens and replace the arguments.
1798      This memory must be freed when the context of the macro MACRO is
1799      popped.  */
1800   buff = tokens_buff_new (pfile, total, track_macro_exp ? &virt_locs : NULL);
1801
1802   first = (const cpp_token **) buff->base;
1803
1804   /* Create a macro map to record the locations of the tokens that are
1805      involved in the expansion.  Note that the expansion point is set
1806      to the location of the closing parenthesis.  Otherwise, the
1807      subsequent map created for the first token that comes after the
1808      macro map might have a wrong line number.  That would lead to
1809      tokens with wrong line numbers after the macro expansion.  This
1810      adds up to the memory overhead of the -ftrack-macro-expansion
1811      flag; for every macro that is expanded, a "macro map" is
1812      created.  */
1813   if (track_macro_exp)
1814     {
1815       int num_macro_tokens = total;
1816       if (track_macro_exp < 2)
1817         /* Then the number of macro tokens won't take in account the
1818            fact that function-like macro arguments can expand to
1819            multiple tokens. This is to save memory at the expense of
1820            accuracy.
1821
1822            Suppose we have #define SQARE(A) A * A
1823
1824            And then we do SQARE(2+3)
1825
1826            Then the tokens 2, +, 3, will have the same location,
1827            saying they come from the expansion of the argument A.  */
1828         num_macro_tokens = exp_count;
1829       map = linemap_enter_macro (pfile->line_table, node,
1830                                  expansion_point_loc,
1831                                  num_macro_tokens);
1832     }
1833   i = 0;
1834   vaopt_state vaopt_tracker (pfile, macro->variadic,
1835                              args[macro->paramc - 1].count > 0);
1836   for (src = macro->exp.tokens; src < limit; src++)
1837     {
1838       unsigned int arg_tokens_count;
1839       macro_arg_token_iter from;
1840       const cpp_token **paste_flag = NULL;
1841       const cpp_token **tmp_token_ptr;
1842
1843       /* __VA_OPT__ handling.  */
1844       if (vaopt_tracker.update (src) != vaopt_state::INCLUDE)
1845         continue;
1846
1847       if (src->type != CPP_MACRO_ARG)
1848         {
1849           /* Allocate a virtual location for token SRC, and add that
1850              token and its virtual location into the buffers BUFF and
1851              VIRT_LOCS.  */
1852           unsigned index = expanded_token_index (pfile, macro, src, i);
1853           tokens_buff_add_token (buff, virt_locs, src,
1854                                  src->src_loc, src->src_loc,
1855                                  map, index);
1856           i += 1;
1857           continue;
1858         }
1859
1860       paste_flag = 0;
1861       arg = &args[src->val.macro_arg.arg_no - 1];
1862       /* SRC is a macro parameter that we need to replace with its
1863          corresponding argument.  So at some point we'll need to
1864          iterate over the tokens of the macro argument and copy them
1865          into the "place" now holding the correspondig macro
1866          parameter.  We are going to use the iterator type
1867          macro_argo_token_iter to handle that iterating.  The 'if'
1868          below is to initialize the iterator depending on the type of
1869          tokens the macro argument has.  It also does some adjustment
1870          related to padding tokens and some pasting corner cases.  */
1871       if (src->flags & STRINGIFY_ARG)
1872         {
1873           arg_tokens_count = 1;
1874           macro_arg_token_iter_init (&from,
1875                                      CPP_OPTION (pfile,
1876                                                  track_macro_expansion),
1877                                      MACRO_ARG_TOKEN_STRINGIFIED,
1878                                      arg, &arg->stringified);
1879         }
1880       else if (src->flags & PASTE_LEFT)
1881         {
1882           arg_tokens_count = arg->count;
1883           macro_arg_token_iter_init (&from,
1884                                      CPP_OPTION (pfile,
1885                                                  track_macro_expansion),
1886                                      MACRO_ARG_TOKEN_NORMAL,
1887                                      arg, arg->first);
1888         }
1889       else if (src != macro->exp.tokens && (src[-1].flags & PASTE_LEFT))
1890         {
1891           int num_toks;
1892           arg_tokens_count = arg->count;
1893           macro_arg_token_iter_init (&from,
1894                                      CPP_OPTION (pfile,
1895                                                  track_macro_expansion),
1896                                      MACRO_ARG_TOKEN_NORMAL,
1897                                      arg, arg->first);
1898
1899           num_toks = tokens_buff_count (buff);
1900
1901           if (num_toks != 0)
1902             {
1903               /* So the current parameter token is pasted to the previous
1904                  token in the replacement list.  Let's look at what
1905                  we have as previous and current arguments.  */
1906
1907               /* This is the previous argument's token ...  */
1908               tmp_token_ptr = tokens_buff_last_token_ptr (buff);
1909
1910               if ((*tmp_token_ptr)->type == CPP_COMMA
1911                   && macro->variadic
1912                   && src->val.macro_arg.arg_no == macro->paramc)
1913                 {
1914                   /* ... which is a comma; and the current parameter
1915                      is the last parameter of a variadic function-like
1916                      macro.  If the argument to the current last
1917                      parameter is NULL, then swallow the comma,
1918                      otherwise drop the paste flag.  */
1919                   if (macro_arg_token_iter_get_token (&from) == NULL)
1920                     tokens_buff_remove_last_token (buff);
1921                   else
1922                     paste_flag = tmp_token_ptr;
1923                 }
1924               /* Remove the paste flag if the RHS is a placemarker.  */
1925               else if (arg_tokens_count == 0)
1926                 paste_flag = tmp_token_ptr;
1927             }
1928         }
1929       else
1930         {
1931           arg_tokens_count = arg->expanded_count;
1932           macro_arg_token_iter_init (&from,
1933                                      CPP_OPTION (pfile,
1934                                                  track_macro_expansion),
1935                                      MACRO_ARG_TOKEN_EXPANDED,
1936                                      arg, arg->expanded);
1937         }
1938
1939       /* Padding on the left of an argument (unless RHS of ##).  */
1940       if ((!pfile->state.in_directive || pfile->state.directive_wants_padding)
1941           && src != macro->exp.tokens && !(src[-1].flags & PASTE_LEFT))
1942         {
1943           const cpp_token *t = padding_token (pfile, src);
1944           unsigned index = expanded_token_index (pfile, macro, src, i);
1945           /* Allocate a virtual location for the padding token and
1946              append the token and its location to BUFF and
1947              VIRT_LOCS.   */
1948           tokens_buff_add_token (buff, virt_locs, t,
1949                                  t->src_loc, t->src_loc,
1950                                  map, index);
1951         }
1952
1953       if (arg_tokens_count)
1954         {
1955           /* So now we've got the number of tokens that make up the
1956              argument that is going to replace the current parameter
1957              in the macro's replacement list.  */
1958           unsigned int j;
1959           for (j = 0; j < arg_tokens_count; ++j)
1960             {
1961               /* So if track_macro_exp is < 2, the user wants to
1962                  save extra memory while tracking macro expansion
1963                  locations.  So in that case here is what we do:
1964
1965                  Suppose we have #define SQARE(A) A * A
1966
1967                  And then we do SQARE(2+3)
1968
1969                  Then the tokens 2, +, 3, will have the same location,
1970                  saying they come from the expansion of the argument
1971                  A.
1972
1973               So that means we are going to ignore the COUNT tokens
1974               resulting from the expansion of the current macro
1975               arugment. In other words all the ARG_TOKENS_COUNT tokens
1976               resulting from the expansion of the macro argument will
1977               have the index I. Normally, each of those token should
1978               have index I+J.  */
1979               unsigned token_index = i;
1980               unsigned index;
1981               if (track_macro_exp > 1)
1982                 token_index += j;
1983
1984               index = expanded_token_index (pfile, macro, src, token_index);
1985               tokens_buff_add_token (buff, virt_locs,
1986                                      macro_arg_token_iter_get_token (&from),
1987                                      macro_arg_token_iter_get_location (&from),
1988                                      src->src_loc, map, index);
1989               macro_arg_token_iter_forward (&from);
1990             }
1991
1992           /* With a non-empty argument on the LHS of ##, the last
1993              token should be flagged PASTE_LEFT.  */
1994           if (src->flags & PASTE_LEFT)
1995             paste_flag =
1996               (const cpp_token **) tokens_buff_last_token_ptr (buff);
1997         }
1998       else if (CPP_PEDANTIC (pfile) && ! CPP_OPTION (pfile, c99)
1999                && ! macro->syshdr && ! cpp_in_system_header (pfile))
2000         {
2001           if (CPP_OPTION (pfile, cplusplus))
2002             cpp_pedwarning (pfile, CPP_W_PEDANTIC,
2003                             "invoking macro %s argument %d: "
2004                             "empty macro arguments are undefined"
2005                             " in ISO C++98",
2006                             NODE_NAME (node), src->val.macro_arg.arg_no);
2007           else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat))
2008             cpp_pedwarning (pfile,
2009                             CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
2010                             ? CPP_W_C90_C99_COMPAT : CPP_W_PEDANTIC,
2011                             "invoking macro %s argument %d: "
2012                             "empty macro arguments are undefined"
2013                             " in ISO C90",
2014                             NODE_NAME (node), src->val.macro_arg.arg_no);
2015         }
2016       else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
2017                && ! CPP_OPTION (pfile, cplusplus)
2018                && ! macro->syshdr && ! cpp_in_system_header (pfile))
2019         cpp_warning (pfile, CPP_W_C90_C99_COMPAT,
2020                      "invoking macro %s argument %d: "
2021                      "empty macro arguments are undefined"
2022                      " in ISO C90",
2023                      NODE_NAME (node), src->val.macro_arg.arg_no);
2024
2025       /* Avoid paste on RHS (even case count == 0).  */
2026       if (!pfile->state.in_directive && !(src->flags & PASTE_LEFT))
2027         {
2028           const cpp_token *t = &pfile->avoid_paste;
2029           tokens_buff_add_token (buff, virt_locs,
2030                                  t, t->src_loc, t->src_loc,
2031                                  NULL, 0);
2032         }
2033
2034       /* Add a new paste flag, or remove an unwanted one.  */
2035       if (paste_flag)
2036         {
2037           cpp_token *token = _cpp_temp_token (pfile);
2038           token->type = (*paste_flag)->type;
2039           token->val = (*paste_flag)->val;
2040           if (src->flags & PASTE_LEFT)
2041             token->flags = (*paste_flag)->flags | PASTE_LEFT;
2042           else
2043             token->flags = (*paste_flag)->flags & ~PASTE_LEFT;
2044           *paste_flag = token;
2045         }
2046
2047       i += arg_tokens_count;
2048     }
2049
2050   if (track_macro_exp)
2051     push_extended_tokens_context (pfile, node, buff, virt_locs, first,
2052                                   tokens_buff_count (buff));
2053   else
2054     push_ptoken_context (pfile, node, buff, first,
2055                          tokens_buff_count (buff));
2056
2057   num_macro_tokens_counter += tokens_buff_count (buff);
2058 }
2059
2060 /* Return a special padding token, with padding inherited from SOURCE.  */
2061 static const cpp_token *
2062 padding_token (cpp_reader *pfile, const cpp_token *source)
2063 {
2064   cpp_token *result = _cpp_temp_token (pfile);
2065
2066   result->type = CPP_PADDING;
2067
2068   /* Data in GCed data structures cannot be made const so far, so we
2069      need a cast here.  */
2070   result->val.source = (cpp_token *) source;
2071   result->flags = 0;
2072   return result;
2073 }
2074
2075 /* Get a new uninitialized context.  Create a new one if we cannot
2076    re-use an old one.  */
2077 static cpp_context *
2078 next_context (cpp_reader *pfile)
2079 {
2080   cpp_context *result = pfile->context->next;
2081
2082   if (result == 0)
2083     {
2084       result = XNEW (cpp_context);
2085       memset (result, 0, sizeof (cpp_context));
2086       result->prev = pfile->context;
2087       result->next = 0;
2088       pfile->context->next = result;
2089     }
2090
2091   pfile->context = result;
2092   return result;
2093 }
2094
2095 /* Push a list of pointers to tokens.  */
2096 static void
2097 push_ptoken_context (cpp_reader *pfile, cpp_hashnode *macro, _cpp_buff *buff,
2098                      const cpp_token **first, unsigned int count)
2099 {
2100   cpp_context *context = next_context (pfile);
2101
2102   context->tokens_kind = TOKENS_KIND_INDIRECT;
2103   context->c.macro = macro;
2104   context->buff = buff;
2105   FIRST (context).ptoken = first;
2106   LAST (context).ptoken = first + count;
2107 }
2108
2109 /* Push a list of tokens.
2110
2111    A NULL macro means that we should continue the current macro
2112    expansion, in essence.  That means that if we are currently in a
2113    macro expansion context, we'll make the new pfile->context refer to
2114    the current macro.  */
2115 void
2116 _cpp_push_token_context (cpp_reader *pfile, cpp_hashnode *macro,
2117                          const cpp_token *first, unsigned int count)
2118 {
2119   cpp_context *context;
2120
2121    if (macro == NULL)
2122      macro = macro_of_context (pfile->context);
2123
2124    context = next_context (pfile);
2125    context->tokens_kind = TOKENS_KIND_DIRECT;
2126    context->c.macro = macro;
2127    context->buff = NULL;
2128    FIRST (context).token = first;
2129    LAST (context).token = first + count;
2130 }
2131
2132 /* Build a context containing a list of tokens as well as their
2133    virtual locations and push it.  TOKENS_BUFF is the buffer that
2134    contains the tokens pointed to by FIRST.  If TOKENS_BUFF is
2135    non-NULL, it means that the context owns it, meaning that
2136    _cpp_pop_context will free it as well as VIRT_LOCS_BUFF that
2137    contains the virtual locations.
2138
2139    A NULL macro means that we should continue the current macro
2140    expansion, in essence.  That means that if we are currently in a
2141    macro expansion context, we'll make the new pfile->context refer to
2142    the current macro.  */
2143 static void
2144 push_extended_tokens_context (cpp_reader *pfile,
2145                               cpp_hashnode *macro,
2146                               _cpp_buff *token_buff,
2147                               source_location *virt_locs,
2148                               const cpp_token **first,
2149                               unsigned int count)
2150 {
2151   cpp_context *context;
2152   macro_context *m;
2153
2154   if (macro == NULL)
2155     macro = macro_of_context (pfile->context);
2156
2157   context = next_context (pfile);
2158   context->tokens_kind = TOKENS_KIND_EXTENDED;
2159   context->buff = token_buff;
2160
2161   m = XNEW (macro_context);
2162   m->macro_node = macro;
2163   m->virt_locs = virt_locs;
2164   m->cur_virt_loc = virt_locs;
2165   context->c.mc = m;
2166   FIRST (context).ptoken = first;
2167   LAST (context).ptoken = first + count;
2168 }
2169
2170 /* Push a traditional macro's replacement text.  */
2171 void
2172 _cpp_push_text_context (cpp_reader *pfile, cpp_hashnode *macro,
2173                         const uchar *start, size_t len)
2174 {
2175   cpp_context *context = next_context (pfile);
2176
2177   context->tokens_kind = TOKENS_KIND_DIRECT;
2178   context->c.macro = macro;
2179   context->buff = NULL;
2180   CUR (context) = start;
2181   RLIMIT (context) = start + len;
2182   macro->flags |= NODE_DISABLED;
2183 }
2184
2185 /* Creates a buffer that holds tokens a.k.a "token buffer", usually
2186    for the purpose of storing them on a cpp_context. If VIRT_LOCS is
2187    non-null (which means that -ftrack-macro-expansion is on),
2188    *VIRT_LOCS is set to a newly allocated buffer that is supposed to
2189    hold the virtual locations of the tokens resulting from macro
2190    expansion.  */
2191 static _cpp_buff*
2192 tokens_buff_new (cpp_reader *pfile, size_t len,
2193                  source_location **virt_locs)
2194 {
2195   size_t tokens_size = len * sizeof (cpp_token *);
2196   size_t locs_size = len * sizeof (source_location);
2197
2198   if (virt_locs != NULL)
2199     *virt_locs = XNEWVEC (source_location, locs_size);
2200   return _cpp_get_buff (pfile, tokens_size);
2201 }
2202
2203 /* Returns the number of tokens contained in a token buffer.  The
2204    buffer holds a set of cpp_token*.  */
2205 static size_t
2206 tokens_buff_count (_cpp_buff *buff)
2207 {
2208   return (BUFF_FRONT (buff) - buff->base) / sizeof (cpp_token *);
2209 }
2210
2211 /* Return a pointer to the last token contained in the token buffer
2212    BUFF.  */
2213 static const cpp_token **
2214 tokens_buff_last_token_ptr (_cpp_buff *buff)
2215 {
2216   return &((const cpp_token **) BUFF_FRONT (buff))[-1];
2217 }
2218
2219 /* Remove the last token contained in the token buffer TOKENS_BUFF.
2220    If VIRT_LOCS_BUFF is non-NULL,  it should point at the buffer
2221    containing the virtual locations of the tokens in TOKENS_BUFF; in
2222    which case the function updates that buffer as well.   */
2223 static inline void
2224 tokens_buff_remove_last_token (_cpp_buff *tokens_buff)
2225
2226 {
2227   if (BUFF_FRONT (tokens_buff) > tokens_buff->base)
2228     BUFF_FRONT (tokens_buff) =
2229       (unsigned char *) &((cpp_token **) BUFF_FRONT (tokens_buff))[-1];
2230 }
2231
2232 /* Insert a token into the token buffer at the position pointed to by
2233    DEST.  Note that the buffer is not enlarged so the previous token
2234    that was at *DEST is overwritten.  VIRT_LOC_DEST, if non-null,
2235    means -ftrack-macro-expansion is effect; it then points to where to
2236    insert the virtual location of TOKEN.  TOKEN is the token to
2237    insert.  VIRT_LOC is the virtual location of the token, i.e, the
2238    location possibly encoding its locus across macro expansion.  If
2239    TOKEN is an argument of a function-like macro (inside a macro
2240    replacement list), PARM_DEF_LOC is the spelling location of the
2241    macro parameter that TOKEN is replacing, in the replacement list of
2242    the macro.  If TOKEN is not an argument of a function-like macro or
2243    if it doesn't come from a macro expansion, then VIRT_LOC can just
2244    be set to the same value as PARM_DEF_LOC.  If MAP is non null, it
2245    means TOKEN comes from a macro expansion and MAP is the macro map
2246    associated to the macro.  MACRO_TOKEN_INDEX points to the index of
2247    the token in the macro map; it is not considered if MAP is NULL.
2248
2249    Upon successful completion this function returns the a pointer to
2250    the position of the token coming right after the insertion
2251    point.  */
2252 static inline const cpp_token **
2253 tokens_buff_put_token_to (const cpp_token **dest,
2254                           source_location *virt_loc_dest,
2255                           const cpp_token *token,
2256                           source_location virt_loc,
2257                           source_location parm_def_loc,
2258                           const line_map_macro *map,
2259                           unsigned int macro_token_index)
2260 {
2261   source_location macro_loc = virt_loc;
2262   const cpp_token **result;
2263
2264   if (virt_loc_dest)
2265     {
2266       /* -ftrack-macro-expansion is on.  */
2267       if (map)
2268         macro_loc = linemap_add_macro_token (map, macro_token_index,
2269                                              virt_loc, parm_def_loc);
2270       *virt_loc_dest = macro_loc;
2271     }
2272   *dest = token;
2273   result = &dest[1];
2274
2275   return result;
2276 }
2277
2278 /* Adds a token at the end of the tokens contained in BUFFER.  Note
2279    that this function doesn't enlarge BUFFER when the number of tokens
2280    reaches BUFFER's size; it aborts in that situation.
2281
2282    TOKEN is the token to append. VIRT_LOC is the virtual location of
2283    the token, i.e, the location possibly encoding its locus across
2284    macro expansion. If TOKEN is an argument of a function-like macro
2285    (inside a macro replacement list), PARM_DEF_LOC is the location of
2286    the macro parameter that TOKEN is replacing.  If TOKEN doesn't come
2287    from a macro expansion, then VIRT_LOC can just be set to the same
2288    value as PARM_DEF_LOC.  If MAP is non null, it means TOKEN comes
2289    from a macro expansion and MAP is the macro map associated to the
2290    macro.  MACRO_TOKEN_INDEX points to the index of the token in the
2291    macro map; It is not considered if MAP is NULL.  If VIRT_LOCS is
2292    non-null, it means -ftrack-macro-expansion is on; in which case
2293    this function adds the virtual location DEF_LOC to the VIRT_LOCS
2294    array, at the same index as the one of TOKEN in BUFFER.  Upon
2295    successful completion this function returns the a pointer to the
2296    position of the token coming right after the insertion point.  */
2297 static const cpp_token **
2298 tokens_buff_add_token (_cpp_buff *buffer,
2299                        source_location *virt_locs,
2300                        const cpp_token *token,
2301                        source_location virt_loc,
2302                        source_location parm_def_loc,
2303                        const line_map_macro *map,
2304                        unsigned int macro_token_index)
2305 {
2306   const cpp_token **result;
2307   source_location *virt_loc_dest = NULL;
2308   unsigned token_index =
2309     (BUFF_FRONT (buffer) - buffer->base) / sizeof (cpp_token *);
2310
2311   /* Abort if we pass the end the buffer.  */
2312   if (BUFF_FRONT (buffer) > BUFF_LIMIT (buffer))
2313     abort ();
2314
2315   if (virt_locs != NULL)
2316     virt_loc_dest = &virt_locs[token_index];
2317
2318   result =
2319     tokens_buff_put_token_to ((const cpp_token **) BUFF_FRONT (buffer),
2320                               virt_loc_dest, token, virt_loc, parm_def_loc,
2321                               map, macro_token_index);
2322
2323   BUFF_FRONT (buffer) = (unsigned char *) result;
2324   return result;
2325 }
2326
2327 /* Allocate space for the function-like macro argument ARG to store
2328    the tokens resulting from the macro-expansion of the tokens that
2329    make up ARG itself. That space is allocated in ARG->expanded and
2330    needs to be freed using free.  */
2331 static void
2332 alloc_expanded_arg_mem (cpp_reader *pfile, macro_arg *arg, size_t capacity)
2333 {
2334   gcc_checking_assert (arg->expanded == NULL
2335                        && arg->expanded_virt_locs == NULL);
2336
2337   arg->expanded = XNEWVEC (const cpp_token *, capacity);
2338   if (CPP_OPTION (pfile, track_macro_expansion))
2339     arg->expanded_virt_locs = XNEWVEC (source_location, capacity);
2340
2341 }
2342
2343 /* If necessary, enlarge ARG->expanded to so that it can contain SIZE
2344    tokens.  */
2345 static void
2346 ensure_expanded_arg_room (cpp_reader *pfile, macro_arg *arg,
2347                           size_t size, size_t *expanded_capacity)
2348 {
2349   if (size <= *expanded_capacity)
2350     return;
2351
2352   size *= 2;
2353
2354   arg->expanded =
2355     XRESIZEVEC (const cpp_token *, arg->expanded, size);
2356   *expanded_capacity = size;
2357
2358   if (CPP_OPTION (pfile, track_macro_expansion))
2359     {
2360       if (arg->expanded_virt_locs == NULL)
2361         arg->expanded_virt_locs = XNEWVEC (source_location, size);
2362       else
2363         arg->expanded_virt_locs = XRESIZEVEC (source_location,
2364                                               arg->expanded_virt_locs,
2365                                               size);
2366     }
2367 }
2368
2369 /* Expand an argument ARG before replacing parameters in a
2370    function-like macro.  This works by pushing a context with the
2371    argument's tokens, and then expanding that into a temporary buffer
2372    as if it were a normal part of the token stream.  collect_args()
2373    has terminated the argument's tokens with a CPP_EOF so that we know
2374    when we have fully expanded the argument.  */
2375 static void
2376 expand_arg (cpp_reader *pfile, macro_arg *arg)
2377 {
2378   size_t capacity;
2379   bool saved_warn_trad;
2380   bool track_macro_exp_p = CPP_OPTION (pfile, track_macro_expansion);
2381
2382   if (arg->count == 0
2383       || arg->expanded != NULL)
2384     return;
2385
2386   /* Don't warn about funlike macros when pre-expanding.  */
2387   saved_warn_trad = CPP_WTRADITIONAL (pfile);
2388   CPP_WTRADITIONAL (pfile) = 0;
2389
2390   /* Loop, reading in the tokens of the argument.  */
2391   capacity = 256;
2392   alloc_expanded_arg_mem (pfile, arg, capacity);
2393
2394   if (track_macro_exp_p)
2395     push_extended_tokens_context (pfile, NULL, NULL,
2396                                   arg->virt_locs,
2397                                   arg->first,
2398                                   arg->count + 1);
2399   else
2400     push_ptoken_context (pfile, NULL, NULL,
2401                          arg->first, arg->count + 1);
2402
2403   for (;;)
2404     {
2405       const cpp_token *token;
2406       source_location location;
2407
2408       ensure_expanded_arg_room (pfile, arg, arg->expanded_count + 1,
2409                                 &capacity);
2410
2411       token = cpp_get_token_1 (pfile, &location);
2412
2413       if (token->type == CPP_EOF)
2414         break;
2415
2416       set_arg_token (arg, token, location,
2417                      arg->expanded_count, MACRO_ARG_TOKEN_EXPANDED,
2418                      CPP_OPTION (pfile, track_macro_expansion));
2419       arg->expanded_count++;
2420     }
2421
2422   _cpp_pop_context (pfile);
2423
2424   CPP_WTRADITIONAL (pfile) = saved_warn_trad;
2425 }
2426
2427 /* Returns the macro associated to the current context if we are in
2428    the context a macro expansion, NULL otherwise.  */
2429 static cpp_hashnode*
2430 macro_of_context (cpp_context *context)
2431 {
2432   if (context == NULL)
2433     return NULL;
2434
2435   return (context->tokens_kind == TOKENS_KIND_EXTENDED)
2436     ? context->c.mc->macro_node
2437     : context->c.macro;
2438 }
2439
2440 /* Return TRUE iff we are expanding a macro or are about to start
2441    expanding one.  If we are effectively expanding a macro, the
2442    function macro_of_context returns a pointer to the macro being
2443    expanded.  */
2444 static bool
2445 in_macro_expansion_p (cpp_reader *pfile)
2446 {
2447   if (pfile == NULL)
2448     return false;
2449
2450   return (pfile->about_to_expand_macro_p
2451           || macro_of_context (pfile->context));
2452 }
2453
2454 /* Pop the current context off the stack, re-enabling the macro if the
2455    context represented a macro's replacement list.  Initially the
2456    context structure was not freed so that we can re-use it later, but
2457    now we do free it to reduce peak memory consumption.  */
2458 void
2459 _cpp_pop_context (cpp_reader *pfile)
2460 {
2461   cpp_context *context = pfile->context;
2462
2463   /* We should not be popping the base context.  */
2464   if (context == &pfile->base_context)
2465     abort ();
2466
2467   if (context->c.macro)
2468     {
2469       cpp_hashnode *macro;
2470       if (context->tokens_kind == TOKENS_KIND_EXTENDED)
2471         {
2472           macro_context *mc = context->c.mc;
2473           macro = mc->macro_node;
2474           /* If context->buff is set, it means the life time of tokens
2475              is bound to the life time of this context; so we must
2476              free the tokens; that means we must free the virtual
2477              locations of these tokens too.  */
2478           if (context->buff && mc->virt_locs)
2479             {
2480               free (mc->virt_locs);
2481               mc->virt_locs = NULL;
2482             }
2483           free (mc);
2484           context->c.mc = NULL;
2485         }
2486       else
2487         macro = context->c.macro;
2488
2489       /* Beware that MACRO can be NULL in cases like when we are
2490          called from expand_arg.  In those cases, a dummy context with
2491          tokens is pushed just for the purpose of walking them using
2492          cpp_get_token_1.  In that case, no 'macro' field is set into
2493          the dummy context.  */
2494       if (macro != NULL
2495           /* Several contiguous macro expansion contexts can be
2496              associated to the same macro; that means it's the same
2497              macro expansion that spans across all these (sub)
2498              contexts.  So we should re-enable an expansion-disabled
2499              macro only when we are sure we are really out of that
2500              macro expansion.  */
2501           && macro_of_context (context->prev) != macro)
2502         macro->flags &= ~NODE_DISABLED;
2503
2504       if (macro == pfile->top_most_macro_node && context->prev == NULL)
2505         /* We are popping the context of the top-most macro node.  */
2506         pfile->top_most_macro_node = NULL;
2507     }
2508
2509   if (context->buff)
2510     {
2511       /* Decrease memory peak consumption by freeing the memory used
2512          by the context.  */
2513       _cpp_free_buff (context->buff);
2514     }
2515
2516   pfile->context = context->prev;
2517   /* decrease peak memory consumption by feeing the context.  */
2518   pfile->context->next = NULL;
2519   free (context);
2520 }
2521
2522 /* Return TRUE if we reached the end of the set of tokens stored in
2523    CONTEXT, FALSE otherwise.  */
2524 static inline bool
2525 reached_end_of_context (cpp_context *context)
2526 {
2527   if (context->tokens_kind == TOKENS_KIND_DIRECT)
2528       return FIRST (context).token == LAST (context).token;
2529   else if (context->tokens_kind == TOKENS_KIND_INDIRECT
2530            || context->tokens_kind == TOKENS_KIND_EXTENDED)
2531     return FIRST (context).ptoken == LAST (context).ptoken;
2532   else
2533     abort ();
2534 }
2535
2536 /* Consume the next token contained in the current context of PFILE,
2537    and return it in *TOKEN. It's "full location" is returned in
2538    *LOCATION. If -ftrack-macro-location is in effeect, fFull location"
2539    means the location encoding the locus of the token across macro
2540    expansion; otherwise it's just is the "normal" location of the
2541    token which (*TOKEN)->src_loc.  */
2542 static inline void
2543 consume_next_token_from_context (cpp_reader *pfile,
2544                                  const cpp_token ** token,
2545                                  source_location *location)
2546 {
2547   cpp_context *c = pfile->context;
2548
2549   if ((c)->tokens_kind == TOKENS_KIND_DIRECT)
2550     {
2551       *token = FIRST (c).token;
2552       *location = (*token)->src_loc;
2553       FIRST (c).token++;
2554     }
2555   else if ((c)->tokens_kind == TOKENS_KIND_INDIRECT)
2556     {
2557       *token = *FIRST (c).ptoken;
2558       *location = (*token)->src_loc;
2559       FIRST (c).ptoken++;
2560     }
2561   else if ((c)->tokens_kind == TOKENS_KIND_EXTENDED)
2562     {
2563       macro_context *m = c->c.mc;
2564       *token = *FIRST (c).ptoken;
2565       if (m->virt_locs)
2566         {
2567           *location = *m->cur_virt_loc;
2568           m->cur_virt_loc++;
2569         }
2570       else
2571         *location = (*token)->src_loc;
2572       FIRST (c).ptoken++;
2573     }
2574   else
2575     abort ();
2576 }
2577
2578 /* In the traditional mode of the preprocessor, if we are currently in
2579    a directive, the location of a token must be the location of the
2580    start of the directive line.  This function returns the proper
2581    location if we are in the traditional mode, and just returns
2582    LOCATION otherwise.  */
2583
2584 static inline source_location
2585 maybe_adjust_loc_for_trad_cpp (cpp_reader *pfile, source_location location)
2586 {
2587   if (CPP_OPTION (pfile, traditional))
2588     {
2589       if (pfile->state.in_directive)
2590         return pfile->directive_line;
2591     }
2592   return location;
2593 }
2594
2595 /* Routine to get a token as well as its location.
2596
2597    Macro expansions and directives are transparently handled,
2598    including entering included files.  Thus tokens are post-macro
2599    expansion, and after any intervening directives.  External callers
2600    see CPP_EOF only at EOF.  Internal callers also see it when meeting
2601    a directive inside a macro call, when at the end of a directive and
2602    state.in_directive is still 1, and at the end of argument
2603    pre-expansion.
2604
2605    LOC is an out parameter; *LOC is set to the location "as expected
2606    by the user".  Please read the comment of
2607    cpp_get_token_with_location to learn more about the meaning of this
2608    location.  */
2609 static const cpp_token*
2610 cpp_get_token_1 (cpp_reader *pfile, source_location *location)
2611 {
2612   const cpp_token *result;
2613   /* This token is a virtual token that either encodes a location
2614      related to macro expansion or a spelling location.  */
2615   source_location virt_loc = 0;
2616   /* pfile->about_to_expand_macro_p can be overriden by indirect calls
2617      to functions that push macro contexts.  So let's save it so that
2618      we can restore it when we are about to leave this routine.  */
2619   bool saved_about_to_expand_macro = pfile->about_to_expand_macro_p;
2620
2621   for (;;)
2622     {
2623       cpp_hashnode *node;
2624       cpp_context *context = pfile->context;
2625
2626       /* Context->prev == 0 <=> base context.  */
2627       if (!context->prev)
2628         {
2629           result = _cpp_lex_token (pfile);
2630           virt_loc = result->src_loc;
2631         }
2632       else if (!reached_end_of_context (context))
2633         {
2634           consume_next_token_from_context (pfile, &result,
2635                                            &virt_loc);
2636           if (result->flags & PASTE_LEFT)
2637             {
2638               paste_all_tokens (pfile, result);
2639               if (pfile->state.in_directive)
2640                 continue;
2641               result = padding_token (pfile, result);
2642               goto out;
2643             }
2644         }
2645       else
2646         {
2647           if (pfile->context->c.macro)
2648             ++num_expanded_macros_counter;
2649           _cpp_pop_context (pfile);
2650           if (pfile->state.in_directive)
2651             continue;
2652           result = &pfile->avoid_paste;
2653           goto out;
2654         }
2655
2656       if (pfile->state.in_directive && result->type == CPP_COMMENT)
2657         continue;
2658
2659       if (result->type != CPP_NAME)
2660         break;
2661
2662       node = result->val.node.node;
2663
2664       if (node->type != NT_MACRO || (result->flags & NO_EXPAND))
2665         break;
2666
2667       if (!(node->flags & NODE_DISABLED))
2668         {
2669           int ret = 0;
2670           /* If not in a macro context, and we're going to start an
2671              expansion, record the location and the top level macro
2672              about to be expanded.  */
2673           if (!in_macro_expansion_p (pfile))
2674             {
2675               pfile->invocation_location = result->src_loc;
2676               pfile->top_most_macro_node = node;
2677             }
2678           if (pfile->state.prevent_expansion)
2679             break;
2680
2681           /* Conditional macros require that a predicate be evaluated
2682              first.  */
2683           if ((node->flags & NODE_CONDITIONAL) != 0)
2684             {
2685               if (pfile->cb.macro_to_expand)
2686                 {
2687                   bool whitespace_after;
2688                   const cpp_token *peek_tok = cpp_peek_token (pfile, 0);
2689
2690                   whitespace_after = (peek_tok->type == CPP_PADDING
2691                                       || (peek_tok->flags & PREV_WHITE));
2692                   node = pfile->cb.macro_to_expand (pfile, result);
2693                   if (node)
2694                     ret = enter_macro_context (pfile, node, result,
2695                                                virt_loc);
2696                   else if (whitespace_after)
2697                     {
2698                       /* If macro_to_expand hook returned NULL and it
2699                          ate some tokens, see if we don't need to add
2700                          a padding token in between this and the
2701                          next token.  */
2702                       peek_tok = cpp_peek_token (pfile, 0);
2703                       if (peek_tok->type != CPP_PADDING
2704                           && (peek_tok->flags & PREV_WHITE) == 0)
2705                         _cpp_push_token_context (pfile, NULL,
2706                                                  padding_token (pfile,
2707                                                                 peek_tok), 1);
2708                     }
2709                 }
2710             }
2711           else
2712             ret = enter_macro_context (pfile, node, result,
2713                                        virt_loc);
2714           if (ret)
2715             {
2716               if (pfile->state.in_directive || ret == 2)
2717                 continue;
2718               result = padding_token (pfile, result);
2719               goto out;
2720             }
2721         }
2722       else
2723         {
2724           /* Flag this token as always unexpandable.  FIXME: move this
2725              to collect_args()?.  */
2726           cpp_token *t = _cpp_temp_token (pfile);
2727           t->type = result->type;
2728           t->flags = result->flags | NO_EXPAND;
2729           t->val = result->val;
2730           result = t;
2731         }
2732
2733       break;
2734     }
2735
2736  out:
2737   if (location != NULL)
2738     {
2739       if (virt_loc == 0)
2740         virt_loc = result->src_loc;
2741       *location = virt_loc;
2742
2743       if (!CPP_OPTION (pfile, track_macro_expansion)
2744           && macro_of_context (pfile->context) != NULL)
2745         /* We are in a macro expansion context, are not tracking
2746            virtual location, but were asked to report the location
2747            of the expansion point of the macro being expanded.  */
2748         *location = pfile->invocation_location;
2749
2750       *location = maybe_adjust_loc_for_trad_cpp (pfile, *location);
2751     }
2752
2753   pfile->about_to_expand_macro_p = saved_about_to_expand_macro;
2754   return result;
2755 }
2756
2757 /* External routine to get a token.  Also used nearly everywhere
2758    internally, except for places where we know we can safely call
2759    _cpp_lex_token directly, such as lexing a directive name.
2760
2761    Macro expansions and directives are transparently handled,
2762    including entering included files.  Thus tokens are post-macro
2763    expansion, and after any intervening directives.  External callers
2764    see CPP_EOF only at EOF.  Internal callers also see it when meeting
2765    a directive inside a macro call, when at the end of a directive and
2766    state.in_directive is still 1, and at the end of argument
2767    pre-expansion.  */
2768 const cpp_token *
2769 cpp_get_token (cpp_reader *pfile)
2770 {
2771   return cpp_get_token_1 (pfile, NULL);
2772 }
2773
2774 /* Like cpp_get_token, but also returns a virtual token location
2775    separate from the spelling location carried by the returned token.
2776
2777    LOC is an out parameter; *LOC is set to the location "as expected
2778    by the user".  This matters when a token results from macro
2779    expansion; in that case the token's spelling location indicates the
2780    locus of the token in the definition of the macro but *LOC
2781    virtually encodes all the other meaningful locuses associated to
2782    the token.
2783
2784    What? virtual location? Yes, virtual location.
2785
2786    If the token results from macro expansion and if macro expansion
2787    location tracking is enabled its virtual location encodes (at the
2788    same time):
2789
2790    - the spelling location of the token
2791
2792    - the locus of the macro expansion point
2793
2794    - the locus of the point where the token got instantiated as part
2795      of the macro expansion process.
2796
2797    You have to use the linemap API to get the locus you are interested
2798    in from a given virtual location.
2799
2800    Note however that virtual locations are not necessarily ordered for
2801    relations '<' and '>'.  One must use the function
2802    linemap_location_before_p instead of using the relational operator
2803    '<'.
2804
2805    If macro expansion tracking is off and if the token results from
2806    macro expansion the virtual location is the expansion point of the
2807    macro that got expanded.
2808
2809    When the token doesn't result from macro expansion, the virtual
2810    location is just the same thing as its spelling location.  */
2811
2812 const cpp_token *
2813 cpp_get_token_with_location (cpp_reader *pfile, source_location *loc)
2814 {
2815   return cpp_get_token_1 (pfile, loc);
2816 }
2817
2818 /* Returns true if we're expanding an object-like macro that was
2819    defined in a system header.  Just checks the macro at the top of
2820    the stack.  Used for diagnostic suppression.  */
2821 int
2822 cpp_sys_macro_p (cpp_reader *pfile)
2823 {
2824   cpp_hashnode *node = NULL;
2825
2826   if (pfile->context->tokens_kind == TOKENS_KIND_EXTENDED)
2827     node = pfile->context->c.mc->macro_node;
2828   else
2829     node = pfile->context->c.macro;
2830
2831   return node && node->value.macro && node->value.macro->syshdr;
2832 }
2833
2834 /* Read each token in, until end of the current file.  Directives are
2835    transparently processed.  */
2836 void
2837 cpp_scan_nooutput (cpp_reader *pfile)
2838 {
2839   /* Request a CPP_EOF token at the end of this file, rather than
2840      transparently continuing with the including file.  */
2841   pfile->buffer->return_at_eof = true;
2842
2843   pfile->state.discarding_output++;
2844   pfile->state.prevent_expansion++;
2845
2846   if (CPP_OPTION (pfile, traditional))
2847     while (_cpp_read_logical_line_trad (pfile))
2848       ;
2849   else
2850     while (cpp_get_token (pfile)->type != CPP_EOF)
2851       ;
2852
2853   pfile->state.discarding_output--;
2854   pfile->state.prevent_expansion--;
2855 }
2856
2857 /* Step back one or more tokens obtained from the lexer.  */
2858 void
2859 _cpp_backup_tokens_direct (cpp_reader *pfile, unsigned int count)
2860 {
2861   pfile->lookaheads += count;
2862   while (count--)
2863     {
2864       pfile->cur_token--;
2865       if (pfile->cur_token == pfile->cur_run->base
2866           /* Possible with -fpreprocessed and no leading #line.  */
2867           && pfile->cur_run->prev != NULL)
2868         {
2869           pfile->cur_run = pfile->cur_run->prev;
2870           pfile->cur_token = pfile->cur_run->limit;
2871         }
2872     }
2873 }
2874
2875 /* Step back one (or more) tokens.  Can only step back more than 1 if
2876    they are from the lexer, and not from macro expansion.  */
2877 void
2878 _cpp_backup_tokens (cpp_reader *pfile, unsigned int count)
2879 {
2880   if (pfile->context->prev == NULL)
2881     _cpp_backup_tokens_direct (pfile, count);
2882   else
2883     {
2884       if (count != 1)
2885         abort ();
2886       if (pfile->context->tokens_kind == TOKENS_KIND_DIRECT)
2887         FIRST (pfile->context).token--;
2888       else if (pfile->context->tokens_kind == TOKENS_KIND_INDIRECT)
2889         FIRST (pfile->context).ptoken--;
2890       else if (pfile->context->tokens_kind == TOKENS_KIND_EXTENDED)
2891         {
2892           FIRST (pfile->context).ptoken--;
2893           if (pfile->context->c.macro)
2894             {
2895               macro_context *m = pfile->context->c.mc;
2896               m->cur_virt_loc--;
2897               gcc_checking_assert (m->cur_virt_loc >= m->virt_locs);
2898             }
2899           else
2900             abort ();
2901         }
2902       else
2903         abort ();
2904     }
2905 }
2906
2907 /* #define directive parsing and handling.  */
2908
2909 /* Returns nonzero if a macro redefinition warning is required.  */
2910 static bool
2911 warn_of_redefinition (cpp_reader *pfile, cpp_hashnode *node,
2912                       const cpp_macro *macro2)
2913 {
2914   const cpp_macro *macro1;
2915   unsigned int i;
2916
2917   /* Some redefinitions need to be warned about regardless.  */
2918   if (node->flags & NODE_WARN)
2919     return true;
2920
2921   /* Suppress warnings for builtins that lack the NODE_WARN flag,
2922      unless Wbuiltin-macro-redefined.  */
2923   if (node->flags & NODE_BUILTIN
2924       && (!pfile->cb.user_builtin_macro
2925           || !pfile->cb.user_builtin_macro (pfile, node)))
2926     return CPP_OPTION (pfile, warn_builtin_macro_redefined);
2927
2928   /* Redefinitions of conditional (context-sensitive) macros, on
2929      the other hand, must be allowed silently.  */
2930   if (node->flags & NODE_CONDITIONAL)
2931     return false;
2932
2933   /* Redefinition of a macro is allowed if and only if the old and new
2934      definitions are the same.  (6.10.3 paragraph 2).  */
2935   macro1 = node->value.macro;
2936
2937   /* Don't check count here as it can be different in valid
2938      traditional redefinitions with just whitespace differences.  */
2939   if (macro1->paramc != macro2->paramc
2940       || macro1->fun_like != macro2->fun_like
2941       || macro1->variadic != macro2->variadic)
2942     return true;
2943
2944   /* Check parameter spellings.  */
2945   for (i = 0; i < macro1->paramc; i++)
2946     if (macro1->params[i] != macro2->params[i])
2947       return true;
2948
2949   /* Check the replacement text or tokens.  */
2950   if (CPP_OPTION (pfile, traditional))
2951     return _cpp_expansions_different_trad (macro1, macro2);
2952
2953   if (macro1->count != macro2->count)
2954     return true;
2955
2956   for (i = 0; i < macro1->count; i++)
2957     if (!_cpp_equiv_tokens (&macro1->exp.tokens[i], &macro2->exp.tokens[i]))
2958       return true;
2959
2960   return false;
2961 }
2962
2963 /* Free the definition of hashnode H.  */
2964 void
2965 _cpp_free_definition (cpp_hashnode *h)
2966 {
2967   /* Macros and assertions no longer have anything to free.  */
2968   h->type = NT_VOID;
2969   /* Clear builtin flag in case of redefinition.  */
2970   h->flags &= ~(NODE_BUILTIN | NODE_DISABLED | NODE_USED);
2971 }
2972
2973 /* Save parameter NODE (spelling SPELLING) to the parameter list of
2974    macro MACRO.  Returns zero on success, nonzero if the parameter is
2975    a duplicate.  */
2976 bool
2977 _cpp_save_parameter (cpp_reader *pfile, cpp_macro *macro, cpp_hashnode *node,
2978                      cpp_hashnode *spelling)
2979 {
2980   unsigned int len;
2981   /* Constraint 6.10.3.6 - duplicate parameter names.  */
2982   if (node->flags & NODE_MACRO_ARG)
2983     {
2984       cpp_error (pfile, CPP_DL_ERROR, "duplicate macro parameter \"%s\"",
2985                  NODE_NAME (node));
2986       return true;
2987     }
2988
2989   if (BUFF_ROOM (pfile->a_buff)
2990       < (macro->paramc + 1) * sizeof (cpp_hashnode *))
2991     _cpp_extend_buff (pfile, &pfile->a_buff, sizeof (cpp_hashnode *));
2992
2993   ((cpp_hashnode **) BUFF_FRONT (pfile->a_buff))[macro->paramc++] = spelling;
2994   node->flags |= NODE_MACRO_ARG;
2995   len = macro->paramc * sizeof (struct macro_arg_saved_data);
2996   if (len > pfile->macro_buffer_len)
2997     {
2998       pfile->macro_buffer = XRESIZEVEC (unsigned char, pfile->macro_buffer,
2999                                         len);
3000       pfile->macro_buffer_len = len;
3001     }
3002   struct macro_arg_saved_data save;
3003   save.value = node->value;
3004   save.canonical_node = node;
3005   ((struct macro_arg_saved_data *) pfile->macro_buffer)[macro->paramc - 1]
3006     = save;
3007
3008   node->value.arg_index  = macro->paramc;
3009   return false;
3010 }
3011
3012 /* Check the syntax of the parameters in a MACRO definition.  Returns
3013    false if an error occurs.  */
3014 static bool
3015 parse_params (cpp_reader *pfile, cpp_macro *macro)
3016 {
3017   unsigned int prev_ident = 0;
3018
3019   for (;;)
3020     {
3021       const cpp_token *token = _cpp_lex_token (pfile);
3022
3023       switch (token->type)
3024         {
3025         default:
3026           /* Allow/ignore comments in parameter lists if we are
3027              preserving comments in macro expansions.  */
3028           if (token->type == CPP_COMMENT
3029               && ! CPP_OPTION (pfile, discard_comments_in_macro_exp))
3030             continue;
3031
3032           cpp_error (pfile, CPP_DL_ERROR,
3033                      "\"%s\" may not appear in macro parameter list",
3034                      cpp_token_as_text (pfile, token));
3035           return false;
3036
3037         case CPP_NAME:
3038           if (prev_ident)
3039             {
3040               cpp_error (pfile, CPP_DL_ERROR,
3041                          "macro parameters must be comma-separated");
3042               return false;
3043             }
3044           prev_ident = 1;
3045
3046           if (_cpp_save_parameter (pfile, macro, token->val.node.node,
3047                                    token->val.node.spelling))
3048             return false;
3049           continue;
3050
3051         case CPP_CLOSE_PAREN:
3052           if (prev_ident || macro->paramc == 0)
3053             return true;
3054
3055           /* Fall through to pick up the error.  */
3056           /* FALLTHRU */
3057         case CPP_COMMA:
3058           if (!prev_ident)
3059             {
3060               cpp_error (pfile, CPP_DL_ERROR, "parameter name missing");
3061               return false;
3062             }
3063           prev_ident = 0;
3064           continue;
3065
3066         case CPP_ELLIPSIS:
3067           macro->variadic = 1;
3068           if (!prev_ident)
3069             {
3070               _cpp_save_parameter (pfile, macro,
3071                                    pfile->spec_nodes.n__VA_ARGS__,
3072                                    pfile->spec_nodes.n__VA_ARGS__);
3073               pfile->state.va_args_ok = 1;
3074               if (! CPP_OPTION (pfile, c99)
3075                   && CPP_OPTION (pfile, cpp_pedantic)
3076                   && CPP_OPTION (pfile, warn_variadic_macros))
3077                 {
3078                   if (CPP_OPTION (pfile, cplusplus))
3079                     cpp_pedwarning
3080                         (pfile, CPP_W_VARIADIC_MACROS,
3081                         "anonymous variadic macros were introduced in C++11");
3082                   else
3083                     cpp_pedwarning
3084                         (pfile, CPP_W_VARIADIC_MACROS,
3085                         "anonymous variadic macros were introduced in C99");
3086                 }
3087               else if (CPP_OPTION (pfile, cpp_warn_c90_c99_compat) > 0
3088                        && ! CPP_OPTION (pfile, cplusplus))
3089                 cpp_error (pfile, CPP_DL_WARNING,
3090                            "anonymous variadic macros were introduced in C99");
3091             }
3092           else if (CPP_OPTION (pfile, cpp_pedantic)
3093                    && CPP_OPTION (pfile, warn_variadic_macros))
3094             {
3095               if (CPP_OPTION (pfile, cplusplus))
3096                 cpp_pedwarning (pfile, CPP_W_VARIADIC_MACROS,
3097                             "ISO C++ does not permit named variadic macros");
3098               else
3099                 cpp_pedwarning (pfile, CPP_W_VARIADIC_MACROS,
3100                             "ISO C does not permit named variadic macros");
3101             }
3102
3103           /* We're at the end, and just expect a closing parenthesis.  */
3104           token = _cpp_lex_token (pfile);
3105           if (token->type == CPP_CLOSE_PAREN)
3106             return true;
3107           /* Fall through.  */
3108
3109         case CPP_EOF:
3110           cpp_error (pfile, CPP_DL_ERROR, "missing ')' in macro parameter list");
3111           return false;
3112         }
3113     }
3114 }
3115
3116 /* Allocate room for a token from a macro's replacement list.  */
3117 static cpp_token *
3118 alloc_expansion_token (cpp_reader *pfile, cpp_macro *macro)
3119 {
3120   if (BUFF_ROOM (pfile->a_buff) < (macro->count + 1) * sizeof (cpp_token))
3121     _cpp_extend_buff (pfile, &pfile->a_buff, sizeof (cpp_token));
3122
3123   return &((cpp_token *) BUFF_FRONT (pfile->a_buff))[macro->count++];
3124 }
3125
3126 /* Lex a token from the expansion of MACRO, but mark parameters as we
3127    find them and warn of traditional stringification.  */
3128 static cpp_token *
3129 lex_expansion_token (cpp_reader *pfile, cpp_macro *macro)
3130 {
3131   cpp_token *token, *saved_cur_token;
3132
3133   saved_cur_token = pfile->cur_token;
3134   pfile->cur_token = alloc_expansion_token (pfile, macro);
3135   token = _cpp_lex_direct (pfile);
3136   pfile->cur_token = saved_cur_token;
3137
3138   /* Is this a parameter?  */
3139   if (token->type == CPP_NAME
3140       && (token->val.node.node->flags & NODE_MACRO_ARG) != 0)
3141     {
3142       cpp_hashnode *spelling = token->val.node.spelling;
3143       token->type = CPP_MACRO_ARG;
3144       token->val.macro_arg.arg_no = token->val.node.node->value.arg_index;
3145       token->val.macro_arg.spelling = spelling;
3146     }
3147   else if (CPP_WTRADITIONAL (pfile) && macro->paramc > 0
3148            && (token->type == CPP_STRING || token->type == CPP_CHAR))
3149     check_trad_stringification (pfile, macro, &token->val.str);
3150
3151   return token;
3152 }
3153
3154 static bool
3155 create_iso_definition (cpp_reader *pfile, cpp_macro *macro)
3156 {
3157   cpp_token *token;
3158   const cpp_token *ctoken;
3159   bool following_paste_op = false;
3160   const char *paste_op_error_msg =
3161     N_("'##' cannot appear at either end of a macro expansion");
3162   unsigned int num_extra_tokens = 0;
3163
3164   /* Get the first token of the expansion (or the '(' of a
3165      function-like macro).  */
3166   ctoken = _cpp_lex_token (pfile);
3167
3168   if (ctoken->type == CPP_OPEN_PAREN && !(ctoken->flags & PREV_WHITE))
3169     {
3170       bool ok = parse_params (pfile, macro);
3171       macro->params = (cpp_hashnode **) BUFF_FRONT (pfile->a_buff);
3172       if (!ok)
3173         return false;
3174
3175       /* Success.  Commit or allocate the parameter array.  */
3176       if (pfile->hash_table->alloc_subobject)
3177         {
3178           cpp_hashnode **params =
3179             (cpp_hashnode **) pfile->hash_table->alloc_subobject
3180             (sizeof (cpp_hashnode *) * macro->paramc);
3181           memcpy (params, macro->params,
3182                   sizeof (cpp_hashnode *) * macro->paramc);
3183           macro->params = params;
3184         }
3185       else
3186         BUFF_FRONT (pfile->a_buff) = (uchar *) &macro->params[macro->paramc];
3187       macro->fun_like = 1;
3188     }
3189   else if (ctoken->type != CPP_EOF && !(ctoken->flags & PREV_WHITE))
3190     {
3191       /* While ISO C99 requires whitespace before replacement text
3192          in a macro definition, ISO C90 with TC1 allows characters
3193          from the basic source character set there.  */
3194       if (CPP_OPTION (pfile, c99))
3195         {
3196           if (CPP_OPTION (pfile, cplusplus))
3197             cpp_error (pfile, CPP_DL_PEDWARN,
3198                        "ISO C++11 requires whitespace after the macro name");
3199           else
3200             cpp_error (pfile, CPP_DL_PEDWARN,
3201                        "ISO C99 requires whitespace after the macro name");
3202         }
3203       else
3204         {
3205           int warntype = CPP_DL_WARNING;
3206           switch (ctoken->type)
3207             {
3208             case CPP_ATSIGN:
3209             case CPP_AT_NAME:
3210             case CPP_OBJC_STRING:
3211               /* '@' is not in basic character set.  */
3212               warntype = CPP_DL_PEDWARN;
3213               break;
3214             case CPP_OTHER:
3215               /* Basic character set sans letters, digits and _.  */
3216               if (strchr ("!\"#%&'()*+,-./:;<=>?[\\]^{|}~",
3217                           ctoken->val.str.text[0]) == NULL)
3218                 warntype = CPP_DL_PEDWARN;
3219               break;
3220             default:
3221               /* All other tokens start with a character from basic
3222                  character set.  */
3223               break;
3224             }
3225           cpp_error (pfile, warntype,
3226                      "missing whitespace after the macro name");
3227         }
3228     }
3229
3230   if (macro->fun_like)
3231     token = lex_expansion_token (pfile, macro);
3232   else
3233     {
3234       token = alloc_expansion_token (pfile, macro);
3235       *token = *ctoken;
3236     }
3237
3238   /* The argument doesn't matter here.  */
3239   vaopt_state vaopt_tracker (pfile, macro->variadic, true);
3240
3241   for (;;)
3242     {
3243       /* Check the stringifying # constraint 6.10.3.2.1 of
3244          function-like macros when lexing the subsequent token.  */
3245       if (macro->count > 1 && token[-1].type == CPP_HASH && macro->fun_like)
3246         {
3247           if (token->type == CPP_MACRO_ARG)
3248             {
3249               if (token->flags & PREV_WHITE)
3250                 token->flags |= SP_PREV_WHITE;
3251               if (token[-1].flags & DIGRAPH)
3252                 token->flags |= SP_DIGRAPH;
3253               token->flags &= ~PREV_WHITE;
3254               token->flags |= STRINGIFY_ARG;
3255               token->flags |= token[-1].flags & PREV_WHITE;
3256               token[-1] = token[0];
3257               macro->count--;
3258             }
3259           /* Let assembler get away with murder.  */
3260           else if (CPP_OPTION (pfile, lang) != CLK_ASM)
3261             {
3262               cpp_error (pfile, CPP_DL_ERROR,
3263                          "'#' is not followed by a macro parameter");
3264               return false;
3265             }
3266         }
3267
3268       if (token->type == CPP_EOF)
3269         {
3270           /* Paste operator constraint 6.10.3.3.1:
3271              Token-paste ##, can appear in both object-like and
3272              function-like macros, but not at the end.  */
3273           if (following_paste_op)
3274             {
3275               cpp_error (pfile, CPP_DL_ERROR, paste_op_error_msg);
3276               return false;
3277             }
3278           break;
3279         }
3280
3281       /* Paste operator constraint 6.10.3.3.1.  */
3282       if (token->type == CPP_PASTE)
3283         {
3284           /* Token-paste ##, can appear in both object-like and
3285              function-like macros, but not at the beginning.  */
3286           if (macro->count == 1)
3287             {
3288               cpp_error (pfile, CPP_DL_ERROR, paste_op_error_msg);
3289               return false;
3290             }
3291
3292           if (token[-1].flags & PASTE_LEFT)
3293             {
3294               macro->extra_tokens = 1;
3295               num_extra_tokens++;
3296               token->val.token_no = macro->count - 1;
3297             }
3298           else
3299             {
3300               --macro->count;
3301               token[-1].flags |= PASTE_LEFT;
3302               if (token->flags & DIGRAPH)
3303                 token[-1].flags |= SP_DIGRAPH;
3304               if (token->flags & PREV_WHITE)
3305                 token[-1].flags |= SP_PREV_WHITE;
3306             }
3307         }
3308
3309       if (vaopt_tracker.update (token) == vaopt_state::ERROR)
3310         return false;
3311
3312       following_paste_op = (token->type == CPP_PASTE);
3313       token = lex_expansion_token (pfile, macro);
3314     }
3315
3316   if (!vaopt_tracker.completed ())
3317     return false;
3318
3319   macro->exp.tokens = (cpp_token *) BUFF_FRONT (pfile->a_buff);
3320   macro->traditional = 0;
3321
3322   /* Don't count the CPP_EOF.  */
3323   macro->count--;
3324
3325   /* Clear whitespace on first token for warn_of_redefinition().  */
3326   if (macro->count)
3327     macro->exp.tokens[0].flags &= ~PREV_WHITE;
3328
3329   /* Commit or allocate the memory.  */
3330   if (pfile->hash_table->alloc_subobject)
3331     {
3332       cpp_token *tokns =
3333         (cpp_token *) pfile->hash_table->alloc_subobject (sizeof (cpp_token)
3334                                                           * macro->count);
3335       if (num_extra_tokens)
3336         {
3337           /* Place second and subsequent ## or %:%: tokens in
3338              sequences of consecutive such tokens at the end of the
3339              list to preserve information about where they appear, how
3340              they are spelt and whether they are preceded by
3341              whitespace without otherwise interfering with macro
3342              expansion.  */
3343           cpp_token *normal_dest = tokns;
3344           cpp_token *extra_dest = tokns + macro->count - num_extra_tokens;
3345           unsigned int i;
3346           for (i = 0; i < macro->count; i++)
3347             {
3348               if (macro->exp.tokens[i].type == CPP_PASTE)
3349                 *extra_dest++ = macro->exp.tokens[i];
3350               else
3351                 *normal_dest++ = macro->exp.tokens[i];
3352             }
3353         }
3354       else
3355         memcpy (tokns, macro->exp.tokens, sizeof (cpp_token) * macro->count);
3356       macro->exp.tokens = tokns;
3357     }
3358   else
3359     BUFF_FRONT (pfile->a_buff) = (uchar *) &macro->exp.tokens[macro->count];
3360
3361   return true;
3362 }
3363
3364 /* Parse a macro and save its expansion.  Returns nonzero on success.  */
3365 bool
3366 _cpp_create_definition (cpp_reader *pfile, cpp_hashnode *node)
3367 {
3368   cpp_macro *macro;
3369   unsigned int i;
3370   bool ok;
3371
3372   if (pfile->hash_table->alloc_subobject)
3373     macro = (cpp_macro *) pfile->hash_table->alloc_subobject
3374       (sizeof (cpp_macro));
3375   else
3376     macro = (cpp_macro *) _cpp_aligned_alloc (pfile, sizeof (cpp_macro));
3377   macro->line = pfile->directive_line;
3378   macro->params = 0;
3379   macro->paramc = 0;
3380   macro->variadic = 0;
3381   macro->used = !CPP_OPTION (pfile, warn_unused_macros);
3382   macro->count = 0;
3383   macro->fun_like = 0;
3384   macro->extra_tokens = 0;
3385   /* To suppress some diagnostics.  */
3386   macro->syshdr = pfile->buffer && pfile->buffer->sysp != 0;
3387
3388   if (CPP_OPTION (pfile, traditional))
3389     ok = _cpp_create_trad_definition (pfile, macro);
3390   else
3391     {
3392       ok = create_iso_definition (pfile, macro);
3393
3394       /* We set the type for SEEN_EOL() in directives.c.
3395
3396          Longer term we should lex the whole line before coming here,
3397          and just copy the expansion.  */
3398
3399       /* Stop the lexer accepting __VA_ARGS__.  */
3400       pfile->state.va_args_ok = 0;
3401     }
3402
3403   /* Clear the fast argument lookup indices.  */
3404   for (i = macro->paramc; i-- > 0; )
3405     {
3406       struct macro_arg_saved_data *save =
3407         &((struct macro_arg_saved_data *) pfile->macro_buffer)[i];
3408       struct cpp_hashnode *node = save->canonical_node;
3409       node->flags &= ~ NODE_MACRO_ARG;
3410       node->value = save->value;
3411     }
3412
3413   if (!ok)
3414     return ok;
3415
3416   if (node->type == NT_MACRO)
3417     {
3418       if (CPP_OPTION (pfile, warn_unused_macros))
3419         _cpp_warn_if_unused_macro (pfile, node, NULL);
3420
3421       if (warn_of_redefinition (pfile, node, macro))
3422         {
3423           const int reason = ((node->flags & NODE_BUILTIN)
3424                               && !(node->flags & NODE_WARN))
3425                              ? CPP_W_BUILTIN_MACRO_REDEFINED : CPP_W_NONE;
3426
3427           bool warned =
3428             cpp_pedwarning_with_line (pfile, reason,
3429                                       pfile->directive_line, 0,
3430                                       "\"%s\" redefined", NODE_NAME (node));
3431
3432           if (warned && node->type == NT_MACRO && !(node->flags & NODE_BUILTIN))
3433             cpp_error_with_line (pfile, CPP_DL_NOTE,
3434                                  node->value.macro->line, 0,
3435                          "this is the location of the previous definition");
3436         }
3437     }
3438
3439   if (node->type != NT_VOID)
3440     _cpp_free_definition (node);
3441
3442   /* Enter definition in hash table.  */
3443   node->type = NT_MACRO;
3444   node->value.macro = macro;
3445   if (! ustrncmp (NODE_NAME (node), DSC ("__STDC_"))
3446       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_FORMAT_MACROS")
3447       /* __STDC_LIMIT_MACROS and __STDC_CONSTANT_MACROS are mentioned
3448          in the C standard, as something that one must use in C++.
3449          However DR#593 and C++11 indicate that they play no role in C++.
3450          We special-case them anyway.  */
3451       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_LIMIT_MACROS")
3452       && ustrcmp (NODE_NAME (node), (const uchar *) "__STDC_CONSTANT_MACROS"))
3453     node->flags |= NODE_WARN;
3454
3455   /* If user defines one of the conditional macros, remove the
3456      conditional flag */
3457   node->flags &= ~NODE_CONDITIONAL;
3458
3459   return ok;
3460 }
3461
3462 /* Warn if a token in STRING matches one of a function-like MACRO's
3463    parameters.  */
3464 static void
3465 check_trad_stringification (cpp_reader *pfile, const cpp_macro *macro,
3466                             const cpp_string *string)
3467 {
3468   unsigned int i, len;
3469   const uchar *p, *q, *limit;
3470
3471   /* Loop over the string.  */
3472   limit = string->text + string->len - 1;
3473   for (p = string->text + 1; p < limit; p = q)
3474     {
3475       /* Find the start of an identifier.  */
3476       while (p < limit && !is_idstart (*p))
3477         p++;
3478
3479       /* Find the end of the identifier.  */
3480       q = p;
3481       while (q < limit && is_idchar (*q))
3482         q++;
3483
3484       len = q - p;
3485
3486       /* Loop over the function macro arguments to see if the
3487          identifier inside the string matches one of them.  */
3488       for (i = 0; i < macro->paramc; i++)
3489         {
3490           const cpp_hashnode *node = macro->params[i];
3491
3492           if (NODE_LEN (node) == len
3493               && !memcmp (p, NODE_NAME (node), len))
3494             {
3495               cpp_warning (pfile, CPP_W_TRADITIONAL,
3496            "macro argument \"%s\" would be stringified in traditional C",
3497                          NODE_NAME (node));
3498               break;
3499             }
3500         }
3501     }
3502 }
3503
3504 /* Returns true of NODE is a function-like macro.  */
3505 bool
3506 cpp_fun_like_macro_p (cpp_hashnode *node)
3507 {
3508   return (node->type == NT_MACRO
3509           && (node->flags & (NODE_BUILTIN | NODE_MACRO_ARG)) == 0
3510           && node->value.macro->fun_like);
3511 }
3512
3513 /* Returns the name, arguments and expansion of a macro, in a format
3514    suitable to be read back in again, and therefore also for DWARF 2
3515    debugging info.  e.g. "PASTE(X, Y) X ## Y", or "MACNAME EXPANSION".
3516    Caller is expected to generate the "#define" bit if needed.  The
3517    returned text is temporary, and automatically freed later.  */
3518 const unsigned char *
3519 cpp_macro_definition (cpp_reader *pfile, cpp_hashnode *node)
3520 {
3521   unsigned int i, len;
3522   const cpp_macro *macro;
3523   unsigned char *buffer;
3524
3525   if (node->type != NT_MACRO || (node->flags & NODE_BUILTIN))
3526     {
3527       if (node->type != NT_MACRO
3528           || !pfile->cb.user_builtin_macro
3529           || !pfile->cb.user_builtin_macro (pfile, node))
3530         {
3531           cpp_error (pfile, CPP_DL_ICE,
3532                      "invalid hash type %d in cpp_macro_definition",
3533                      node->type);
3534           return 0;
3535         }
3536     }
3537
3538   macro = node->value.macro;
3539   /* Calculate length.  */
3540   len = NODE_LEN (node) * 10 + 2;               /* ' ' and NUL.  */
3541   if (macro->fun_like)
3542     {
3543       len += 4;         /* "()" plus possible final ".." of named
3544                            varargs (we have + 1 below).  */
3545       for (i = 0; i < macro->paramc; i++)
3546         len += NODE_LEN (macro->params[i]) + 1; /* "," */
3547     }
3548
3549   /* This should match below where we fill in the buffer.  */
3550   if (CPP_OPTION (pfile, traditional))
3551     len += _cpp_replacement_text_len (macro);
3552   else
3553     {
3554       unsigned int count = macro_real_token_count (macro);
3555       for (i = 0; i < count; i++)
3556         {
3557           cpp_token *token = &macro->exp.tokens[i];
3558
3559           if (token->type == CPP_MACRO_ARG)
3560             len += NODE_LEN (token->val.macro_arg.spelling);
3561           else
3562             len += cpp_token_len (token);
3563
3564           if (token->flags & STRINGIFY_ARG)
3565             len++;                      /* "#" */
3566           if (token->flags & PASTE_LEFT)
3567             len += 3;           /* " ##" */
3568           if (token->flags & PREV_WHITE)
3569             len++;              /* " " */
3570         }
3571     }
3572
3573   if (len > pfile->macro_buffer_len)
3574     {
3575       pfile->macro_buffer = XRESIZEVEC (unsigned char,
3576                                         pfile->macro_buffer, len);
3577       pfile->macro_buffer_len = len;
3578     }
3579
3580   /* Fill in the buffer.  Start with the macro name.  */
3581   buffer = pfile->macro_buffer;
3582   buffer = _cpp_spell_ident_ucns (buffer, node);
3583
3584   /* Parameter names.  */
3585   if (macro->fun_like)
3586     {
3587       *buffer++ = '(';
3588       for (i = 0; i < macro->paramc; i++)
3589         {
3590           cpp_hashnode *param = macro->params[i];
3591
3592           if (param != pfile->spec_nodes.n__VA_ARGS__)
3593             {
3594               memcpy (buffer, NODE_NAME (param), NODE_LEN (param));
3595               buffer += NODE_LEN (param);
3596             }
3597
3598           if (i + 1 < macro->paramc)
3599             /* Don't emit a space after the comma here; we're trying
3600                to emit a Dwarf-friendly definition, and the Dwarf spec
3601                forbids spaces in the argument list.  */
3602             *buffer++ = ',';
3603           else if (macro->variadic)
3604             *buffer++ = '.', *buffer++ = '.', *buffer++ = '.';
3605         }
3606       *buffer++ = ')';
3607     }
3608
3609   /* The Dwarf spec requires a space after the macro name, even if the
3610      definition is the empty string.  */
3611   *buffer++ = ' ';
3612
3613   if (CPP_OPTION (pfile, traditional))
3614     buffer = _cpp_copy_replacement_text (macro, buffer);
3615   else if (macro->count)
3616   /* Expansion tokens.  */
3617     {
3618       unsigned int count = macro_real_token_count (macro);
3619       for (i = 0; i < count; i++)
3620         {
3621           cpp_token *token = &macro->exp.tokens[i];
3622
3623           if (token->flags & PREV_WHITE)
3624             *buffer++ = ' ';
3625           if (token->flags & STRINGIFY_ARG)
3626             *buffer++ = '#';
3627
3628           if (token->type == CPP_MACRO_ARG)
3629             {
3630               memcpy (buffer,
3631                       NODE_NAME (token->val.macro_arg.spelling),
3632                       NODE_LEN (token->val.macro_arg.spelling));
3633               buffer += NODE_LEN (token->val.macro_arg.spelling);
3634             }
3635           else
3636             buffer = cpp_spell_token (pfile, token, buffer, true);
3637
3638           if (token->flags & PASTE_LEFT)
3639             {
3640               *buffer++ = ' ';
3641               *buffer++ = '#';
3642               *buffer++ = '#';
3643               /* Next has PREV_WHITE; see _cpp_create_definition.  */
3644             }
3645         }
3646     }
3647
3648   *buffer = '\0';
3649   return pfile->macro_buffer;
3650 }
3651
3652 /* Get the line at which the macro was defined.  */
3653
3654 source_location
3655 cpp_macro_definition_location (cpp_hashnode *node)
3656 {
3657   return node->value.macro->line;
3658 }