gcc/tree-inline.c

   1 /* Tree inlining.
   2    Copyright 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
   3    Free Software Foundation, Inc.
   4    Contributed by Alexandre Oliva <aoliva@redhat.com>
   5
   6 This file is part of GCC.
   7
   8 GCC is free software; you can redistribute it and/or modify
   9 it under the terms of the GNU General Public License as published by
  10 the Free Software Foundation; either version 3, or (at your option)
  11 any later version.
  12
  13 GCC is distributed in the hope that it will be useful,
  14 but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 GNU General Public License for more details.
  17
  18 You should have received a copy of the GNU General Public License
  19 along with GCC; see the file COPYING3.  If not see
  20 <http://www.gnu.org/licenses/>.  */
  21
  22 #include "config.h"
  23 #include "system.h"
  24 #include "coretypes.h"
  25 #include "tm.h"
  26 #include "toplev.h"
  27 #include "tree.h"
  28 #include "tree-inline.h"
  29 #include "rtl.h"
  30 #include "expr.h"
  31 #include "flags.h"
  32 #include "params.h"
  33 #include "input.h"
  34 #include "insn-config.h"
  35 #include "hashtab.h"
  36 #include "langhooks.h"
  37 #include "basic-block.h"
  38 #include "tree-iterator.h"
  39 #include "cgraph.h"
  40 #include "intl.h"
  41 #include "tree-mudflap.h"
  42 #include "tree-flow.h"
  43 #include "function.h"
  44 #include "ggc.h"
  45 #include "tree-flow.h"
  46 #include "diagnostic.h"
  47 #include "except.h"
  48 #include "debug.h"
  49 #include "pointer-set.h"
  50 #include "ipa-prop.h"
  51 #include "value-prof.h"
  52 #include "tree-pass.h"
  53 #include "target.h"
  54 #include "integrate.h"
  55
  56 /* I'm not real happy about this, but we need to handle gimple and
  57    non-gimple trees.  */
  58 #include "gimple.h"
  59
  60 /* Inlining, Cloning, Versioning, Parallelization
  61
  62    Inlining: a function body is duplicated, but the PARM_DECLs are
  63    remapped into VAR_DECLs, and non-void RETURN_EXPRs become
  64    MODIFY_EXPRs that store to a dedicated returned-value variable.
  65    The duplicated eh_region info of the copy will later be appended
  66    to the info for the caller; the eh_region info in copied throwing
  67    statements and RESX_EXPRs is adjusted accordingly.
  68
  69    Cloning: (only in C++) We have one body for a con/de/structor, and
  70    multiple function decls, each with a unique parameter list.
  71    Duplicate the body, using the given splay tree; some parameters
  72    will become constants (like 0 or 1).
  73
  74    Versioning: a function body is duplicated and the result is a new
  75    function rather than into blocks of an existing function as with
  76    inlining.  Some parameters will become constants.
  77
  78    Parallelization: a region of a function is duplicated resulting in
  79    a new function.  Variables may be replaced with complex expressions
  80    to enable shared variable semantics.
  81
  82    All of these will simultaneously lookup any callgraph edges.  If
  83    we're going to inline the duplicated function body, and the given
  84    function has some cloned callgraph nodes (one for each place this
  85    function will be inlined) those callgraph edges will be duplicated.
  86    If we're cloning the body, those callgraph edges will be
  87    updated to point into the new body.  (Note that the original
  88    callgraph node and edge list will not be altered.)
  89
  90    See the CALL_EXPR handling case in copy_tree_body_r ().  */
  91
  92 /* To Do:
  93
  94    o In order to make inlining-on-trees work, we pessimized
  95      function-local static constants.  In particular, they are now
  96      always output, even when not addressed.  Fix this by treating
  97      function-local static constants just like global static
  98      constants; the back-end already knows not to output them if they
  99      are not needed.
 100
 101    o Provide heuristics to clamp inlining of recursive template
 102      calls?  */
 103
 104
 105 /* Weights that estimate_num_insns uses for heuristics in inlining.  */
 106
 107 eni_weights eni_inlining_weights;
 108
 109 /* Weights that estimate_num_insns uses to estimate the size of the
 110    produced code.  */
 111
 112 eni_weights eni_size_weights;
 113
 114 /* Weights that estimate_num_insns uses to estimate the time necessary
 115    to execute the produced code.  */
 116
 117 eni_weights eni_time_weights;
 118
 119 /* Prototypes.  */
 120
 121 static tree declare_return_variable (copy_body_data *, tree, tree, tree *);
 122 static bool inlinable_function_p (tree);
 123 static void remap_block (tree *, copy_body_data *);
 124 static void copy_bind_expr (tree *, int *, copy_body_data *);
 125 static tree mark_local_for_remap_r (tree *, int *, void *);
 126 static void unsave_expr_1 (tree);
 127 static tree unsave_r (tree *, int *, void *);
 128 static void declare_inline_vars (tree, tree);
 129 static void remap_save_expr (tree *, void *, int *);
 130 static void prepend_lexical_block (tree current_block, tree new_block);
 131 static tree copy_decl_to_var (tree, copy_body_data *);
 132 static tree copy_result_decl_to_var (tree, copy_body_data *);
 133 static tree copy_decl_maybe_to_var (tree, copy_body_data *);
 134 static gimple remap_gimple_stmt (gimple, copy_body_data *);
 135 static bool delete_unreachable_blocks_update_callgraph (copy_body_data *id);
 136
 137 /* Insert a tree->tree mapping for ID.  Despite the name suggests
 138    that the trees should be variables, it is used for more than that.  */
 139
 140 void
 141 insert_decl_map (copy_body_data *id, tree key, tree value)
 142 {
 143   *pointer_map_insert (id->decl_map, key) = value;
 144
 145   /* Always insert an identity map as well.  If we see this same new
 146      node again, we won't want to duplicate it a second time.  */
 147   if (key != value)
 148     *pointer_map_insert (id->decl_map, value) = value;
 149 }
 150
 151 /* Construct new SSA name for old NAME. ID is the inline context.  */
 152
 153 static tree
 154 remap_ssa_name (tree name, copy_body_data *id)
 155 {
 156   tree new_tree;
 157   tree *n;
 158
 159   gcc_assert (TREE_CODE (name) == SSA_NAME);
 160
 161   n = (tree *) pointer_map_contains (id->decl_map, name);
 162   if (n)
 163     return unshare_expr (*n);
 164
 165   /* Do not set DEF_STMT yet as statement is not copied yet. We do that
 166      in copy_bb.  */
 167   new_tree = remap_decl (SSA_NAME_VAR (name), id);
 168
 169   /* We might've substituted constant or another SSA_NAME for
 170      the variable.
 171
 172      Replace the SSA name representing RESULT_DECL by variable during
 173      inlining:  this saves us from need to introduce PHI node in a case
 174      return value is just partly initialized.  */
 175   if ((TREE_CODE (new_tree) == VAR_DECL || TREE_CODE (new_tree) == PARM_DECL)
 176       && (TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL
 177           || !id->transform_return_to_modify))
 178     {
 179       new_tree = make_ssa_name (new_tree, NULL);
 180       insert_decl_map (id, name, new_tree);
 181       SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
 182         = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
 183       TREE_TYPE (new_tree) = TREE_TYPE (SSA_NAME_VAR (new_tree));
 184       if (gimple_nop_p (SSA_NAME_DEF_STMT (name)))
 185         {
 186           /* By inlining function having uninitialized variable, we might
 187              extend the lifetime (variable might get reused).  This cause
 188              ICE in the case we end up extending lifetime of SSA name across
 189              abnormal edge, but also increase register pressure.
 190
 191              We simply initialize all uninitialized vars by 0 except
 192              for case we are inlining to very first BB.  We can avoid
 193              this for all BBs that are not inside strongly connected
 194              regions of the CFG, but this is expensive to test.  */
 195           if (id->entry_bb
 196               && is_gimple_reg (SSA_NAME_VAR (name))
 197               && TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL
 198               && (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR, 0)->dest
 199                   || EDGE_COUNT (id->entry_bb->preds) != 1))
 200             {
 201               gimple_stmt_iterator gsi = gsi_last_bb (id->entry_bb);
 202               gimple init_stmt;
 203
 204               init_stmt = gimple_build_assign (new_tree,
 205                                                fold_convert (TREE_TYPE (new_tree),
 206                                                             integer_zero_node));
 207               gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT);
 208               SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0;
 209             }
 210           else
 211             {
 212               SSA_NAME_DEF_STMT (new_tree) = gimple_build_nop ();
 213               if (gimple_default_def (id->src_cfun, SSA_NAME_VAR (name))
 214                   == name)
 215                 set_default_def (SSA_NAME_VAR (new_tree), new_tree);
 216             }
 217         }
 218     }
 219   else
 220     insert_decl_map (id, name, new_tree);
 221   return new_tree;
 222 }
 223
 224 /* Remap DECL during the copying of the BLOCK tree for the function.  */
 225
 226 tree
 227 remap_decl (tree decl, copy_body_data *id)
 228 {
 229   tree *n;
 230   tree fn;
 231
 232   /* We only remap local variables in the current function.  */
 233   fn = id->src_fn;
 234
 235   /* See if we have remapped this declaration.  */
 236
 237   n = (tree *) pointer_map_contains (id->decl_map, decl);
 238
 239   /* If we didn't already have an equivalent for this declaration,
 240      create one now.  */
 241   if (!n)
 242     {
 243       /* Make a copy of the variable or label.  */
 244       tree t = id->copy_decl (decl, id);
 245
 246       /* Remember it, so that if we encounter this local entity again
 247          we can reuse this copy.  Do this early because remap_type may
 248          need this decl for TYPE_STUB_DECL.  */
 249       insert_decl_map (id, decl, t);
 250
 251       if (!DECL_P (t))
 252         return t;
 253
 254       /* Remap types, if necessary.  */
 255       TREE_TYPE (t) = remap_type (TREE_TYPE (t), id);
 256       if (TREE_CODE (t) == TYPE_DECL)
 257         DECL_ORIGINAL_TYPE (t) = remap_type (DECL_ORIGINAL_TYPE (t), id);
 258
 259       /* Remap sizes as necessary.  */
 260       walk_tree (&DECL_SIZE (t), copy_tree_body_r, id, NULL);
 261       walk_tree (&DECL_SIZE_UNIT (t), copy_tree_body_r, id, NULL);
 262
 263       /* If fields, do likewise for offset and qualifier.  */
 264       if (TREE_CODE (t) == FIELD_DECL)
 265         {
 266           walk_tree (&DECL_FIELD_OFFSET (t), copy_tree_body_r, id, NULL);
 267           if (TREE_CODE (DECL_CONTEXT (t)) == QUAL_UNION_TYPE)
 268             walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL);
 269         }
 270
 271       if (cfun && gimple_in_ssa_p (cfun)
 272           && (TREE_CODE (t) == VAR_DECL
 273               || TREE_CODE (t) == RESULT_DECL || TREE_CODE (t) == PARM_DECL))
 274         {
 275           tree def = gimple_default_def (id->src_cfun, decl);
 276           get_var_ann (t);
 277           if (TREE_CODE (decl) != PARM_DECL && def)
 278             {
 279               tree map = remap_ssa_name (def, id);
 280               /* Watch out RESULT_DECLs whose SSA names map directly
 281                  to them.  */
 282               if (TREE_CODE (map) == SSA_NAME
 283                   && gimple_nop_p (SSA_NAME_DEF_STMT (map)))
 284                 set_default_def (t, map);
 285             }
 286           add_referenced_var (t);
 287         }
 288       return t;
 289     }
 290
 291   return unshare_expr (*n);
 292 }
 293
 294 static tree
 295 remap_type_1 (tree type, copy_body_data *id)
 296 {
 297   tree new_tree, t;
 298
 299   /* We do need a copy.  build and register it now.  If this is a pointer or
 300      reference type, remap the designated type and make a new pointer or
 301      reference type.  */
 302   if (TREE_CODE (type) == POINTER_TYPE)
 303     {
 304       new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id),
 305                                          TYPE_MODE (type),
 306                                          TYPE_REF_CAN_ALIAS_ALL (type));
 307       insert_decl_map (id, type, new_tree);
 308       return new_tree;
 309     }
 310   else if (TREE_CODE (type) == REFERENCE_TYPE)
 311     {
 312       new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id),
 313                                             TYPE_MODE (type),
 314                                             TYPE_REF_CAN_ALIAS_ALL (type));
 315       insert_decl_map (id, type, new_tree);
 316       return new_tree;
 317     }
 318   else
 319     new_tree = copy_node (type);
 320
 321   insert_decl_map (id, type, new_tree);
 322
 323   /* This is a new type, not a copy of an old type.  Need to reassociate
 324      variants.  We can handle everything except the main variant lazily.  */
 325   t = TYPE_MAIN_VARIANT (type);
 326   if (type != t)
 327     {
 328       t = remap_type (t, id);
 329       TYPE_MAIN_VARIANT (new_tree) = t;
 330       TYPE_NEXT_VARIANT (new_tree) = TYPE_NEXT_VARIANT (t);
 331       TYPE_NEXT_VARIANT (t) = new_tree;
 332     }
 333   else
 334     {
 335       TYPE_MAIN_VARIANT (new_tree) = new_tree;
 336       TYPE_NEXT_VARIANT (new_tree) = NULL;
 337     }
 338
 339   if (TYPE_STUB_DECL (type))
 340     TYPE_STUB_DECL (new_tree) = remap_decl (TYPE_STUB_DECL (type), id);
 341
 342   /* Lazily create pointer and reference types.  */
 343   TYPE_POINTER_TO (new_tree) = NULL;
 344   TYPE_REFERENCE_TO (new_tree) = NULL;
 345
 346   switch (TREE_CODE (new_tree))
 347     {
 348     case INTEGER_TYPE:
 349     case REAL_TYPE:
 350     case FIXED_POINT_TYPE:
 351     case ENUMERAL_TYPE:
 352     case BOOLEAN_TYPE:
 353       t = TYPE_MIN_VALUE (new_tree);
 354       if (t && TREE_CODE (t) != INTEGER_CST)
 355         walk_tree (&TYPE_MIN_VALUE (new_tree), copy_tree_body_r, id, NULL);
 356
 357       t = TYPE_MAX_VALUE (new_tree);
 358       if (t && TREE_CODE (t) != INTEGER_CST)
 359         walk_tree (&TYPE_MAX_VALUE (new_tree), copy_tree_body_r, id, NULL);
 360       return new_tree;
 361
 362     case FUNCTION_TYPE:
 363       TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
 364       walk_tree (&TYPE_ARG_TYPES (new_tree), copy_tree_body_r, id, NULL);
 365       return new_tree;
 366
 367     case ARRAY_TYPE:
 368       TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
 369       TYPE_DOMAIN (new_tree) = remap_type (TYPE_DOMAIN (new_tree), id);
 370       break;
 371
 372     case RECORD_TYPE:
 373     case UNION_TYPE:
 374     case QUAL_UNION_TYPE:
 375       {
 376         tree f, nf = NULL;
 377
 378         for (f = TYPE_FIELDS (new_tree); f ; f = TREE_CHAIN (f))
 379           {
 380             t = remap_decl (f, id);
 381             DECL_CONTEXT (t) = new_tree;
 382             TREE_CHAIN (t) = nf;
 383             nf = t;
 384           }
 385         TYPE_FIELDS (new_tree) = nreverse (nf);
 386       }
 387       break;
 388
 389     case OFFSET_TYPE:
 390     default:
 391       /* Shouldn't have been thought variable sized.  */
 392       gcc_unreachable ();
 393     }
 394
 395   walk_tree (&TYPE_SIZE (new_tree), copy_tree_body_r, id, NULL);
 396   walk_tree (&TYPE_SIZE_UNIT (new_tree), copy_tree_body_r, id, NULL);
 397
 398   return new_tree;
 399 }
 400
 401 tree
 402 remap_type (tree type, copy_body_data *id)
 403 {
 404   tree *node;
 405   tree tmp;
 406
 407   if (type == NULL)
 408     return type;
 409
 410   /* See if we have remapped this type.  */
 411   node = (tree *) pointer_map_contains (id->decl_map, type);
 412   if (node)
 413     return *node;
 414
 415   /* The type only needs remapping if it's variably modified.  */
 416   if (! variably_modified_type_p (type, id->src_fn))
 417     {
 418       insert_decl_map (id, type, type);
 419       return type;
 420     }
 421
 422   id->remapping_type_depth++;
 423   tmp = remap_type_1 (type, id);
 424   id->remapping_type_depth--;
 425
 426   return tmp;
 427 }
 428
 429 /* Return previously remapped type of TYPE in ID.  Return NULL if TYPE
 430    is NULL or TYPE has not been remapped before.  */
 431
 432 static tree
 433 remapped_type (tree type, copy_body_data *id)
 434 {
 435   tree *node;
 436
 437   if (type == NULL)
 438     return type;
 439
 440   /* See if we have remapped this type.  */
 441   node = (tree *) pointer_map_contains (id->decl_map, type);
 442   if (node)
 443     return *node;
 444   else
 445     return NULL;
 446 }
 447
 448   /* The type only needs remapping if it's variably modified.  */
 449 /* Decide if DECL can be put into BLOCK_NONLOCAL_VARs.  */
 450
 451 static bool
 452 can_be_nonlocal (tree decl, copy_body_data *id)
 453 {
 454   /* We can not duplicate function decls.  */
 455   if (TREE_CODE (decl) == FUNCTION_DECL)
 456     return true;
 457
 458   /* Local static vars must be non-local or we get multiple declaration
 459      problems.  */
 460   if (TREE_CODE (decl) == VAR_DECL
 461       && !auto_var_in_fn_p (decl, id->src_fn))
 462     return true;
 463
 464   /* At the moment dwarf2out can handle only these types of nodes.  We
 465      can support more later.  */
 466   if (TREE_CODE (decl) != VAR_DECL && TREE_CODE (decl) != PARM_DECL)
 467     return false;
 468
 469   /* We must use global type.  We call remapped_type instead of
 470      remap_type since we don't want to remap this type here if it
 471      hasn't been remapped before.  */
 472   if (TREE_TYPE (decl) != remapped_type (TREE_TYPE (decl), id))
 473     return false;
 474
 475   /* Wihtout SSA we can't tell if variable is used.  */
 476   if (!gimple_in_ssa_p (cfun))
 477     return false;
 478
 479   /* Live variables must be copied so we can attach DECL_RTL.  */
 480   if (var_ann (decl))
 481     return false;
 482
 483   return true;
 484 }
 485
 486 static tree
 487 remap_decls (tree decls, VEC(tree,gc) **nonlocalized_list, copy_body_data *id)
 488 {
 489   tree old_var;
 490   tree new_decls = NULL_TREE;
 491
 492   /* Remap its variables.  */
 493   for (old_var = decls; old_var; old_var = TREE_CHAIN (old_var))
 494     {
 495       tree new_var;
 496       tree origin_var = DECL_ORIGIN (old_var);
 497
 498       if (can_be_nonlocal (old_var, id))
 499         {
 500           if (TREE_CODE (old_var) == VAR_DECL
 501               && (var_ann (old_var) || !gimple_in_ssa_p (cfun)))
 502             cfun->local_decls = tree_cons (NULL_TREE, old_var,
 503                                                    cfun->local_decls);
 504           if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
 505               && !DECL_IGNORED_P (old_var)
 506               && nonlocalized_list)
 507             VEC_safe_push (tree, gc, *nonlocalized_list, origin_var);
 508           continue;
 509         }
 510
 511       /* Remap the variable.  */
 512       new_var = remap_decl (old_var, id);
 513
 514       /* If we didn't remap this variable, we can't mess with its
 515          TREE_CHAIN.  If we remapped this variable to the return slot, it's
 516          already declared somewhere else, so don't declare it here.  */
 517
 518       if (new_var == id->retvar)
 519         ;
 520       else if (!new_var)
 521         {
 522           if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
 523               && !DECL_IGNORED_P (old_var)
 524               && nonlocalized_list)
 525             VEC_safe_push (tree, gc, *nonlocalized_list, origin_var);
 526         }
 527       else
 528         {
 529           gcc_assert (DECL_P (new_var));
 530           TREE_CHAIN (new_var) = new_decls;
 531           new_decls = new_var;
 532         }
 533     }
 534
 535   return nreverse (new_decls);
 536 }
 537
 538 /* Copy the BLOCK to contain remapped versions of the variables
 539    therein.  And hook the new block into the block-tree.  */
 540
 541 static void
 542 remap_block (tree *block, copy_body_data *id)
 543 {
 544   tree old_block;
 545   tree new_block;
 546   tree fn;
 547
 548   /* Make the new block.  */
 549   old_block = *block;
 550   new_block = make_node (BLOCK);
 551   TREE_USED (new_block) = TREE_USED (old_block);
 552   BLOCK_ABSTRACT_ORIGIN (new_block) = old_block;
 553   BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block);
 554   BLOCK_NONLOCALIZED_VARS (new_block)
 555     = VEC_copy (tree, gc, BLOCK_NONLOCALIZED_VARS (old_block));
 556   *block = new_block;
 557
 558   /* Remap its variables.  */
 559   BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block),
 560                                         &BLOCK_NONLOCALIZED_VARS (new_block),
 561                                         id);
 562
 563   fn = id->dst_fn;
 564
 565   if (id->transform_lang_insert_block)
 566     id->transform_lang_insert_block (new_block);
 567
 568   /* Remember the remapped block.  */
 569   insert_decl_map (id, old_block, new_block);
 570 }
 571
 572 /* Copy the whole block tree and root it in id->block.  */
 573 static tree
 574 remap_blocks (tree block, copy_body_data *id)
 575 {
 576   tree t;
 577   tree new_tree = block;
 578
 579   if (!block)
 580     return NULL;
 581
 582   remap_block (&new_tree, id);
 583   gcc_assert (new_tree != block);
 584   for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
 585     prepend_lexical_block (new_tree, remap_blocks (t, id));
 586   /* Blocks are in arbitrary order, but make things slightly prettier and do
 587      not swap order when producing a copy.  */
 588   BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree));
 589   return new_tree;
 590 }
 591
 592 static void
 593 copy_statement_list (tree *tp)
 594 {
 595   tree_stmt_iterator oi, ni;
 596   tree new_tree;
 597
 598   new_tree = alloc_stmt_list ();
 599   ni = tsi_start (new_tree);
 600   oi = tsi_start (*tp);
 601   *tp = new_tree;
 602
 603   for (; !tsi_end_p (oi); tsi_next (&oi))
 604     tsi_link_after (&ni, tsi_stmt (oi), TSI_NEW_STMT);
 605 }
 606
 607 static void
 608 copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id)
 609 {
 610   tree block = BIND_EXPR_BLOCK (*tp);
 611   /* Copy (and replace) the statement.  */
 612   copy_tree_r (tp, walk_subtrees, NULL);
 613   if (block)
 614     {
 615       remap_block (&block, id);
 616       BIND_EXPR_BLOCK (*tp) = block;
 617     }
 618
 619   if (BIND_EXPR_VARS (*tp))
 620     /* This will remap a lot of the same decls again, but this should be
 621        harmless.  */
 622     BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id);
 623 }
 624
 625
 626 /* Create a new gimple_seq by remapping all the statements in BODY
 627    using the inlining information in ID.  */
 628
 629 gimple_seq
 630 remap_gimple_seq (gimple_seq body, copy_body_data *id)
 631 {
 632   gimple_stmt_iterator si;
 633   gimple_seq new_body = NULL;
 634
 635   for (si = gsi_start (body); !gsi_end_p (si); gsi_next (&si))
 636     {
 637       gimple new_stmt = remap_gimple_stmt (gsi_stmt (si), id);
 638       gimple_seq_add_stmt (&new_body, new_stmt);
 639     }
 640
 641   return new_body;
 642 }
 643
 644
 645 /* Copy a GIMPLE_BIND statement STMT, remapping all the symbols in its
 646    block using the mapping information in ID.  */
 647
 648 static gimple
 649 copy_gimple_bind (gimple stmt, copy_body_data *id)
 650 {
 651   gimple new_bind;
 652   tree new_block, new_vars;
 653   gimple_seq body, new_body;
 654
 655   /* Copy the statement.  Note that we purposely don't use copy_stmt
 656      here because we need to remap statements as we copy.  */
 657   body = gimple_bind_body (stmt);
 658   new_body = remap_gimple_seq (body, id);
 659
 660   new_block = gimple_bind_block (stmt);
 661   if (new_block)
 662     remap_block (&new_block, id);
 663
 664   /* This will remap a lot of the same decls again, but this should be
 665      harmless.  */
 666   new_vars = gimple_bind_vars (stmt);
 667   if (new_vars)
 668     new_vars = remap_decls (new_vars, NULL, id);
 669
 670   new_bind = gimple_build_bind (new_vars, new_body, new_block);
 671
 672   return new_bind;
 673 }
 674
 675
 676 /* Remap the GIMPLE operand pointed to by *TP.  DATA is really a
 677    'struct walk_stmt_info *'.  DATA->INFO is a 'copy_body_data *'.
 678    WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
 679    recursing into the children nodes of *TP.  */
 680
 681 static tree
 682 remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
 683 {
 684   struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
 685   copy_body_data *id = (copy_body_data *) wi_p->info;
 686   tree fn = id->src_fn;
 687
 688   if (TREE_CODE (*tp) == SSA_NAME)
 689     {
 690       *tp = remap_ssa_name (*tp, id);
 691       *walk_subtrees = 0;
 692       return NULL;
 693     }
 694   else if (auto_var_in_fn_p (*tp, fn))
 695     {
 696       /* Local variables and labels need to be replaced by equivalent
 697          variables.  We don't want to copy static variables; there's
 698          only one of those, no matter how many times we inline the
 699          containing function.  Similarly for globals from an outer
 700          function.  */
 701       tree new_decl;
 702
 703       /* Remap the declaration.  */
 704       new_decl = remap_decl (*tp, id);
 705       gcc_assert (new_decl);
 706       /* Replace this variable with the copy.  */
 707       STRIP_TYPE_NOPS (new_decl);
 708       /* ???  The C++ frontend uses void * pointer zero to initialize
 709          any other type.  This confuses the middle-end type verification.
 710          As cloned bodies do not go through gimplification again the fixup
 711          there doesn't trigger.  */
 712       if (TREE_CODE (new_decl) == INTEGER_CST
 713           && !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
 714         new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
 715       *tp = new_decl;
 716       *walk_subtrees = 0;
 717     }
 718   else if (TREE_CODE (*tp) == STATEMENT_LIST)
 719     gcc_unreachable ();
 720   else if (TREE_CODE (*tp) == SAVE_EXPR)
 721     gcc_unreachable ();
 722   else if (TREE_CODE (*tp) == LABEL_DECL
 723            && (!DECL_CONTEXT (*tp)
 724                || decl_function_context (*tp) == id->src_fn))
 725     /* These may need to be remapped for EH handling.  */
 726     *tp = remap_decl (*tp, id);
 727   else if (TYPE_P (*tp))
 728     /* Types may need remapping as well.  */
 729     *tp = remap_type (*tp, id);
 730   else if (CONSTANT_CLASS_P (*tp))
 731     {
 732       /* If this is a constant, we have to copy the node iff the type
 733          will be remapped.  copy_tree_r will not copy a constant.  */
 734       tree new_type = remap_type (TREE_TYPE (*tp), id);
 735
 736       if (new_type == TREE_TYPE (*tp))
 737         *walk_subtrees = 0;
 738
 739       else if (TREE_CODE (*tp) == INTEGER_CST)
 740         *tp = build_int_cst_wide (new_type, TREE_INT_CST_LOW (*tp),
 741                                   TREE_INT_CST_HIGH (*tp));
 742       else
 743         {
 744           *tp = copy_node (*tp);
 745           TREE_TYPE (*tp) = new_type;
 746         }
 747     }
 748   else
 749     {
 750       /* Otherwise, just copy the node.  Note that copy_tree_r already
 751          knows not to copy VAR_DECLs, etc., so this is safe.  */
 752       if (TREE_CODE (*tp) == INDIRECT_REF)
 753         {
 754           /* Get rid of *& from inline substitutions that can happen when a
 755              pointer argument is an ADDR_EXPR.  */
 756           tree decl = TREE_OPERAND (*tp, 0);
 757           tree *n;
 758
 759           n = (tree *) pointer_map_contains (id->decl_map, decl);
 760           if (n)
 761             {
 762               tree type, new_tree, old;
 763
 764               /* If we happen to get an ADDR_EXPR in n->value, strip
 765                  it manually here as we'll eventually get ADDR_EXPRs
 766                  which lie about their types pointed to.  In this case
 767                  build_fold_indirect_ref wouldn't strip the
 768                  INDIRECT_REF, but we absolutely rely on that.  As
 769                  fold_indirect_ref does other useful transformations,
 770                  try that first, though.  */
 771               type = TREE_TYPE (TREE_TYPE (*n));
 772               new_tree = unshare_expr (*n);
 773               old = *tp;
 774               *tp = gimple_fold_indirect_ref (new_tree);
 775               if (!*tp)
 776                 {
 777                   if (TREE_CODE (new_tree) == ADDR_EXPR)
 778                     {
 779                       *tp = fold_indirect_ref_1 (type, new_tree);
 780                       /* ???  We should either assert here or build
 781                          a VIEW_CONVERT_EXPR instead of blindly leaking
 782                          incompatible types to our IL.  */
 783                       if (! *tp)
 784                         *tp = TREE_OPERAND (new_tree, 0);
 785                     }
 786                   else
 787                     {
 788                       *tp = build1 (INDIRECT_REF, type, new_tree);
 789                       TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
 790                       TREE_NO_WARNING (*tp) = TREE_NO_WARNING (old);
 791                     }
 792                 }
 793               *walk_subtrees = 0;
 794               return NULL;
 795             }
 796         }
 797
 798       /* Here is the "usual case".  Copy this tree node, and then
 799          tweak some special cases.  */
 800       copy_tree_r (tp, walk_subtrees, NULL);
 801
 802       /* Global variables we haven't seen yet need to go into referenced
 803          vars.  If not referenced from types only.  */
 804       if (gimple_in_ssa_p (cfun)
 805           && TREE_CODE (*tp) == VAR_DECL
 806           && id->remapping_type_depth == 0)
 807         add_referenced_var (*tp);
 808
 809       /* We should never have TREE_BLOCK set on non-statements.  */
 810       if (EXPR_P (*tp))
 811         gcc_assert (!TREE_BLOCK (*tp));
 812
 813       if (TREE_CODE (*tp) != OMP_CLAUSE)
 814         TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
 815
 816       if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
 817         {
 818           /* The copied TARGET_EXPR has never been expanded, even if the
 819              original node was expanded already.  */
 820           TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
 821           TREE_OPERAND (*tp, 3) = NULL_TREE;
 822         }
 823       else if (TREE_CODE (*tp) == ADDR_EXPR)
 824         {
 825           /* Variable substitution need not be simple.  In particular,
 826              the INDIRECT_REF substitution above.  Make sure that
 827              TREE_CONSTANT and friends are up-to-date.  But make sure
 828              to not improperly set TREE_BLOCK on some sub-expressions.  */
 829           int invariant = is_gimple_min_invariant (*tp);
 830           tree block = id->block;
 831           id->block = NULL_TREE;
 832           walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
 833           id->block = block;
 834
 835           /* Handle the case where we substituted an INDIRECT_REF
 836              into the operand of the ADDR_EXPR.  */
 837           if (TREE_CODE (TREE_OPERAND (*tp, 0)) == INDIRECT_REF)
 838             *tp = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
 839           else
 840             recompute_tree_invariant_for_addr_expr (*tp);
 841
 842           /* If this used to be invariant, but is not any longer,
 843              then regimplification is probably needed.  */
 844           if (invariant && !is_gimple_min_invariant (*tp))
 845             id->regimplify = true;
 846
 847           *walk_subtrees = 0;
 848         }
 849     }
 850
 851   /* Keep iterating.  */
 852   return NULL_TREE;
 853 }
 854
 855
 856 /* Called from copy_body_id via walk_tree.  DATA is really a
 857    `copy_body_data *'.  */
 858
 859 tree
 860 copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
 861 {
 862   copy_body_data *id = (copy_body_data *) data;
 863   tree fn = id->src_fn;
 864   tree new_block;
 865
 866   /* Begin by recognizing trees that we'll completely rewrite for the
 867      inlining context.  Our output for these trees is completely
 868      different from out input (e.g. RETURN_EXPR is deleted, and morphs
 869      into an edge).  Further down, we'll handle trees that get
 870      duplicated and/or tweaked.  */
 871
 872   /* When requested, RETURN_EXPRs should be transformed to just the
 873      contained MODIFY_EXPR.  The branch semantics of the return will
 874      be handled elsewhere by manipulating the CFG rather than a statement.  */
 875   if (TREE_CODE (*tp) == RETURN_EXPR && id->transform_return_to_modify)
 876     {
 877       tree assignment = TREE_OPERAND (*tp, 0);
 878
 879       /* If we're returning something, just turn that into an
 880          assignment into the equivalent of the original RESULT_DECL.
 881          If the "assignment" is just the result decl, the result
 882          decl has already been set (e.g. a recent "foo (&result_decl,
 883          ...)"); just toss the entire RETURN_EXPR.  */
 884       if (assignment && TREE_CODE (assignment) == MODIFY_EXPR)
 885         {
 886           /* Replace the RETURN_EXPR with (a copy of) the
 887              MODIFY_EXPR hanging underneath.  */
 888           *tp = copy_node (assignment);
 889         }
 890       else /* Else the RETURN_EXPR returns no value.  */
 891         {
 892           *tp = NULL;
 893           return (tree) (void *)1;
 894         }
 895     }
 896   else if (TREE_CODE (*tp) == SSA_NAME)
 897     {
 898       *tp = remap_ssa_name (*tp, id);
 899       *walk_subtrees = 0;
 900       return NULL;
 901     }
 902
 903   /* Local variables and labels need to be replaced by equivalent
 904      variables.  We don't want to copy static variables; there's only
 905      one of those, no matter how many times we inline the containing
 906      function.  Similarly for globals from an outer function.  */
 907   else if (auto_var_in_fn_p (*tp, fn))
 908     {
 909       tree new_decl;
 910
 911       /* Remap the declaration.  */
 912       new_decl = remap_decl (*tp, id);
 913       gcc_assert (new_decl);
 914       /* Replace this variable with the copy.  */
 915       STRIP_TYPE_NOPS (new_decl);
 916       *tp = new_decl;
 917       *walk_subtrees = 0;
 918     }
 919   else if (TREE_CODE (*tp) == STATEMENT_LIST)
 920     copy_statement_list (tp);
 921   else if (TREE_CODE (*tp) == SAVE_EXPR)
 922     remap_save_expr (tp, id->decl_map, walk_subtrees);
 923   else if (TREE_CODE (*tp) == LABEL_DECL
 924            && (! DECL_CONTEXT (*tp)
 925                || decl_function_context (*tp) == id->src_fn))
 926     /* These may need to be remapped for EH handling.  */
 927     *tp = remap_decl (*tp, id);
 928   else if (TREE_CODE (*tp) == BIND_EXPR)
 929     copy_bind_expr (tp, walk_subtrees, id);
 930   /* Types may need remapping as well.  */
 931   else if (TYPE_P (*tp))
 932     *tp = remap_type (*tp, id);
 933
 934   /* If this is a constant, we have to copy the node iff the type will be
 935      remapped.  copy_tree_r will not copy a constant.  */
 936   else if (CONSTANT_CLASS_P (*tp))
 937     {
 938       tree new_type = remap_type (TREE_TYPE (*tp), id);
 939
 940       if (new_type == TREE_TYPE (*tp))
 941         *walk_subtrees = 0;
 942
 943       else if (TREE_CODE (*tp) == INTEGER_CST)
 944         *tp = build_int_cst_wide (new_type, TREE_INT_CST_LOW (*tp),
 945                                   TREE_INT_CST_HIGH (*tp));
 946       else
 947         {
 948           *tp = copy_node (*tp);
 949           TREE_TYPE (*tp) = new_type;
 950         }
 951     }
 952
 953   /* Otherwise, just copy the node.  Note that copy_tree_r already
 954      knows not to copy VAR_DECLs, etc., so this is safe.  */
 955   else
 956     {
 957       /* Here we handle trees that are not completely rewritten.
 958          First we detect some inlining-induced bogosities for
 959          discarding.  */
 960       if (TREE_CODE (*tp) == MODIFY_EXPR
 961           && TREE_OPERAND (*tp, 0) == TREE_OPERAND (*tp, 1)
 962           && (auto_var_in_fn_p (TREE_OPERAND (*tp, 0), fn)))
 963         {
 964           /* Some assignments VAR = VAR; don't generate any rtl code
 965              and thus don't count as variable modification.  Avoid
 966              keeping bogosities like 0 = 0.  */
 967           tree decl = TREE_OPERAND (*tp, 0), value;
 968           tree *n;
 969
 970           n = (tree *) pointer_map_contains (id->decl_map, decl);
 971           if (n)
 972             {
 973               value = *n;
 974               STRIP_TYPE_NOPS (value);
 975               if (TREE_CONSTANT (value) || TREE_READONLY (value))
 976                 {
 977                   *tp = build_empty_stmt (EXPR_LOCATION (*tp));
 978                   return copy_tree_body_r (tp, walk_subtrees, data);
 979                 }
 980             }
 981         }
 982       else if (TREE_CODE (*tp) == INDIRECT_REF)
 983         {
 984           /* Get rid of *& from inline substitutions that can happen when a
 985              pointer argument is an ADDR_EXPR.  */
 986           tree decl = TREE_OPERAND (*tp, 0);
 987           tree *n;
 988
 989           n = (tree *) pointer_map_contains (id->decl_map, decl);
 990           if (n)
 991             {
 992               tree new_tree;
 993               tree old;
 994               /* If we happen to get an ADDR_EXPR in n->value, strip
 995                  it manually here as we'll eventually get ADDR_EXPRs
 996                  which lie about their types pointed to.  In this case
 997                  build_fold_indirect_ref wouldn't strip the INDIRECT_REF,
 998                  but we absolutely rely on that.  As fold_indirect_ref
 999                  does other useful transformations, try that first, though.  */
1000               tree type = TREE_TYPE (TREE_TYPE (*n));
1001               new_tree = unshare_expr (*n);
1002               old = *tp;
1003               *tp = gimple_fold_indirect_ref (new_tree);
1004               if (! *tp)
1005                 {
1006                   if (TREE_CODE (new_tree) == ADDR_EXPR)
1007                     {
1008                       *tp = fold_indirect_ref_1 (type, new_tree);
1009                       /* ???  We should either assert here or build
1010                          a VIEW_CONVERT_EXPR instead of blindly leaking
1011                          incompatible types to our IL.  */
1012                       if (! *tp)
1013                         *tp = TREE_OPERAND (new_tree, 0);
1014                     }
1015                   else
1016                     {
1017                       *tp = build1 (INDIRECT_REF, type, new_tree);
1018                       TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1019                       TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1020                     }
1021                 }
1022               *walk_subtrees = 0;
1023               return NULL;
1024             }
1025         }
1026
1027       /* Here is the "usual case".  Copy this tree node, and then
1028          tweak some special cases.  */
1029       copy_tree_r (tp, walk_subtrees, NULL);
1030
1031       /* Global variables we haven't seen yet needs to go into referenced
1032          vars.  If not referenced from types only.  */
1033       if (gimple_in_ssa_p (cfun)
1034           && TREE_CODE (*tp) == VAR_DECL
1035           && id->remapping_type_depth == 0)
1036         add_referenced_var (*tp);
1037
1038       /* If EXPR has block defined, map it to newly constructed block.
1039          When inlining we want EXPRs without block appear in the block
1040          of function call.  */
1041       if (EXPR_P (*tp))
1042         {
1043           new_block = id->block;
1044           if (TREE_BLOCK (*tp))
1045             {
1046               tree *n;
1047               n = (tree *) pointer_map_contains (id->decl_map,
1048                                                  TREE_BLOCK (*tp));
1049               gcc_assert (n);
1050               new_block = *n;
1051             }
1052           TREE_BLOCK (*tp) = new_block;
1053         }
1054
1055       if (TREE_CODE (*tp) == RESX_EXPR && id->eh_region_offset)
1056         TREE_OPERAND (*tp, 0) =
1057           build_int_cst (NULL_TREE,
1058                          id->eh_region_offset
1059                          + TREE_INT_CST_LOW (TREE_OPERAND (*tp, 0)));
1060
1061       if (TREE_CODE (*tp) != OMP_CLAUSE)
1062         TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1063
1064       /* The copied TARGET_EXPR has never been expanded, even if the
1065          original node was expanded already.  */
1066       if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1067         {
1068           TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1069           TREE_OPERAND (*tp, 3) = NULL_TREE;
1070         }
1071
1072       /* Variable substitution need not be simple.  In particular, the
1073          INDIRECT_REF substitution above.  Make sure that TREE_CONSTANT
1074          and friends are up-to-date.  */
1075       else if (TREE_CODE (*tp) == ADDR_EXPR)
1076         {
1077           int invariant = is_gimple_min_invariant (*tp);
1078           walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
1079
1080           /* Handle the case where we substituted an INDIRECT_REF
1081              into the operand of the ADDR_EXPR.  */
1082           if (TREE_CODE (TREE_OPERAND (*tp, 0)) == INDIRECT_REF)
1083             *tp = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
1084           else
1085             recompute_tree_invariant_for_addr_expr (*tp);
1086
1087           /* If this used to be invariant, but is not any longer,
1088              then regimplification is probably needed.  */
1089           if (invariant && !is_gimple_min_invariant (*tp))
1090             id->regimplify = true;
1091
1092           *walk_subtrees = 0;
1093         }
1094     }
1095
1096   /* Keep iterating.  */
1097   return NULL_TREE;
1098 }
1099
1100
1101 /* Helper for copy_bb.  Remap statement STMT using the inlining
1102    information in ID.  Return the new statement copy.  */
1103
1104 static gimple
1105 remap_gimple_stmt (gimple stmt, copy_body_data *id)
1106 {
1107   gimple copy = NULL;
1108   struct walk_stmt_info wi;
1109   tree new_block;
1110   bool skip_first = false;
1111
1112   /* Begin by recognizing trees that we'll completely rewrite for the
1113      inlining context.  Our output for these trees is completely
1114      different from out input (e.g. RETURN_EXPR is deleted, and morphs
1115      into an edge).  Further down, we'll handle trees that get
1116      duplicated and/or tweaked.  */
1117
1118   /* When requested, GIMPLE_RETURNs should be transformed to just the
1119      contained GIMPLE_ASSIGN.  The branch semantics of the return will
1120      be handled elsewhere by manipulating the CFG rather than the
1121      statement.  */
1122   if (gimple_code (stmt) == GIMPLE_RETURN && id->transform_return_to_modify)
1123     {
1124       tree retval = gimple_return_retval (stmt);
1125
1126       /* If we're returning something, just turn that into an
1127          assignment into the equivalent of the original RESULT_DECL.
1128          If RETVAL is just the result decl, the result decl has
1129          already been set (e.g. a recent "foo (&result_decl, ...)");
1130          just toss the entire GIMPLE_RETURN.  */
1131       if (retval && TREE_CODE (retval) != RESULT_DECL)
1132         {
1133           copy = gimple_build_assign (id->retvar, retval);
1134           /* id->retvar is already substituted.  Skip it on later remapping.  */
1135           skip_first = true;
1136         }
1137       else
1138         return gimple_build_nop ();
1139     }
1140   else if (gimple_has_substatements (stmt))
1141     {
1142       gimple_seq s1, s2;
1143
1144       /* When cloning bodies from the C++ front end, we will be handed bodies
1145          in High GIMPLE form.  Handle here all the High GIMPLE statements that
1146          have embedded statements.  */
1147       switch (gimple_code (stmt))
1148         {
1149         case GIMPLE_BIND:
1150           copy = copy_gimple_bind (stmt, id);
1151           break;
1152
1153         case GIMPLE_CATCH:
1154           s1 = remap_gimple_seq (gimple_catch_handler (stmt), id);
1155           copy = gimple_build_catch (gimple_catch_types (stmt), s1);
1156           break;
1157
1158         case GIMPLE_EH_FILTER:
1159           s1 = remap_gimple_seq (gimple_eh_filter_failure (stmt), id);
1160           copy = gimple_build_eh_filter (gimple_eh_filter_types (stmt), s1);
1161           break;
1162
1163         case GIMPLE_TRY:
1164           s1 = remap_gimple_seq (gimple_try_eval (stmt), id);
1165           s2 = remap_gimple_seq (gimple_try_cleanup (stmt), id);
1166           copy = gimple_build_try (s1, s2, gimple_try_kind (stmt));
1167           break;
1168
1169         case GIMPLE_WITH_CLEANUP_EXPR:
1170           s1 = remap_gimple_seq (gimple_wce_cleanup (stmt), id);
1171           copy = gimple_build_wce (s1);
1172           break;
1173
1174         case GIMPLE_OMP_PARALLEL:
1175           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1176           copy = gimple_build_omp_parallel
1177                    (s1,
1178                     gimple_omp_parallel_clauses (stmt),
1179                     gimple_omp_parallel_child_fn (stmt),
1180                     gimple_omp_parallel_data_arg (stmt));
1181           break;
1182
1183         case GIMPLE_OMP_TASK:
1184           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1185           copy = gimple_build_omp_task
1186                    (s1,
1187                     gimple_omp_task_clauses (stmt),
1188                     gimple_omp_task_child_fn (stmt),
1189                     gimple_omp_task_data_arg (stmt),
1190                     gimple_omp_task_copy_fn (stmt),
1191                     gimple_omp_task_arg_size (stmt),
1192                     gimple_omp_task_arg_align (stmt));
1193           break;
1194
1195         case GIMPLE_OMP_FOR:
1196           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1197           s2 = remap_gimple_seq (gimple_omp_for_pre_body (stmt), id);
1198           copy = gimple_build_omp_for (s1, gimple_omp_for_clauses (stmt),
1199                                        gimple_omp_for_collapse (stmt), s2);
1200           {
1201             size_t i;
1202             for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
1203               {
1204                 gimple_omp_for_set_index (copy, i,
1205                                           gimple_omp_for_index (stmt, i));
1206                 gimple_omp_for_set_initial (copy, i,
1207                                             gimple_omp_for_initial (stmt, i));
1208                 gimple_omp_for_set_final (copy, i,
1209                                           gimple_omp_for_final (stmt, i));
1210                 gimple_omp_for_set_incr (copy, i,
1211                                          gimple_omp_for_incr (stmt, i));
1212                 gimple_omp_for_set_cond (copy, i,
1213                                          gimple_omp_for_cond (stmt, i));
1214               }
1215           }
1216           break;
1217
1218         case GIMPLE_OMP_MASTER:
1219           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1220           copy = gimple_build_omp_master (s1);
1221           break;
1222
1223         case GIMPLE_OMP_ORDERED:
1224           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1225           copy = gimple_build_omp_ordered (s1);
1226           break;
1227
1228         case GIMPLE_OMP_SECTION:
1229           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1230           copy = gimple_build_omp_section (s1);
1231           break;
1232
1233         case GIMPLE_OMP_SECTIONS:
1234           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1235           copy = gimple_build_omp_sections
1236                    (s1, gimple_omp_sections_clauses (stmt));
1237           break;
1238
1239         case GIMPLE_OMP_SINGLE:
1240           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1241           copy = gimple_build_omp_single
1242                    (s1, gimple_omp_single_clauses (stmt));
1243           break;
1244
1245         case GIMPLE_OMP_CRITICAL:
1246           s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1247           copy
1248             = gimple_build_omp_critical (s1, gimple_omp_critical_name (stmt));
1249           break;
1250
1251         default:
1252           gcc_unreachable ();
1253         }
1254     }
1255   else
1256     {
1257       if (gimple_assign_copy_p (stmt)
1258           && gimple_assign_lhs (stmt) == gimple_assign_rhs1 (stmt)
1259           && auto_var_in_fn_p (gimple_assign_lhs (stmt), id->src_fn))
1260         {
1261           /* Here we handle statements that are not completely rewritten.
1262              First we detect some inlining-induced bogosities for
1263              discarding.  */
1264
1265           /* Some assignments VAR = VAR; don't generate any rtl code
1266              and thus don't count as variable modification.  Avoid
1267              keeping bogosities like 0 = 0.  */
1268           tree decl = gimple_assign_lhs (stmt), value;
1269           tree *n;
1270
1271           n = (tree *) pointer_map_contains (id->decl_map, decl);
1272           if (n)
1273             {
1274               value = *n;
1275               STRIP_TYPE_NOPS (value);
1276               if (TREE_CONSTANT (value) || TREE_READONLY (value))
1277                 return gimple_build_nop ();
1278             }
1279         }
1280
1281       /* Create a new deep copy of the statement.  */
1282       copy = gimple_copy (stmt);
1283     }
1284
1285   /* If STMT has a block defined, map it to the newly constructed
1286      block.  When inlining we want statements without a block to
1287      appear in the block of the function call.  */
1288   new_block = id->block;
1289   if (gimple_block (copy))
1290     {
1291       tree *n;
1292       n = (tree *) pointer_map_contains (id->decl_map, gimple_block (copy));
1293       gcc_assert (n);
1294       new_block = *n;
1295     }
1296
1297   gimple_set_block (copy, new_block);
1298
1299   /* Remap all the operands in COPY.  */
1300   memset (&wi, 0, sizeof (wi));
1301   wi.info = id;
1302   if (skip_first)
1303     walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL);
1304   else
1305     walk_gimple_op (copy, remap_gimple_op_r, &wi);
1306
1307   /* Clear the copied virtual operands.  We are not remapping them here
1308      but are going to recreate them from scratch.  */
1309   if (gimple_has_mem_ops (copy))
1310     {
1311       gimple_set_vdef (copy, NULL_TREE);
1312       gimple_set_vuse (copy, NULL_TREE);
1313     }
1314
1315   /* We have to handle EH region remapping of GIMPLE_RESX specially because
1316      the region number is not an operand.  */
1317   if (gimple_code (stmt) == GIMPLE_RESX && id->eh_region_offset)
1318     {
1319       gimple_resx_set_region (copy, gimple_resx_region (stmt) + id->eh_region_offset);
1320     }
1321   return copy;
1322 }
1323
1324
1325 /* Copy basic block, scale profile accordingly.  Edges will be taken care of
1326    later  */
1327
1328 static basic_block
1329 copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
1330          gcov_type count_scale)
1331 {
1332   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
1333   basic_block copy_basic_block;
1334   tree decl;
1335
1336   /* create_basic_block() will append every new block to
1337      basic_block_info automatically.  */
1338   copy_basic_block = create_basic_block (NULL, (void *) 0,
1339                                          (basic_block) bb->prev_bb->aux);
1340   copy_basic_block->count = bb->count * count_scale / REG_BR_PROB_BASE;
1341
1342   /* We are going to rebuild frequencies from scratch.  These values
1343      have just small importance to drive canonicalize_loop_headers.  */
1344   copy_basic_block->frequency = ((gcov_type)bb->frequency
1345                                  * frequency_scale / REG_BR_PROB_BASE);
1346
1347   if (copy_basic_block->frequency > BB_FREQ_MAX)
1348     copy_basic_block->frequency = BB_FREQ_MAX;
1349
1350   copy_gsi = gsi_start_bb (copy_basic_block);
1351
1352   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1353     {
1354       gimple stmt = gsi_stmt (gsi);
1355       gimple orig_stmt = stmt;
1356
1357       id->regimplify = false;
1358       stmt = remap_gimple_stmt (stmt, id);
1359       if (gimple_nop_p (stmt))
1360         continue;
1361
1362       gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun, orig_stmt);
1363       seq_gsi = copy_gsi;
1364
1365       /* With return slot optimization we can end up with
1366          non-gimple (foo *)&this->m, fix that here.  */
1367       if (is_gimple_assign (stmt)
1368           && gimple_assign_rhs_code (stmt) == NOP_EXPR
1369           && !is_gimple_val (gimple_assign_rhs1 (stmt)))
1370         {
1371           tree new_rhs;
1372           new_rhs = force_gimple_operand_gsi (&seq_gsi,
1373                                               gimple_assign_rhs1 (stmt),
1374                                               true, NULL, true, GSI_SAME_STMT);
1375           gimple_assign_set_rhs1 (stmt, new_rhs);
1376           id->regimplify = false;
1377         }
1378
1379       gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT);
1380
1381       if (id->regimplify)
1382         gimple_regimplify_operands (stmt, &seq_gsi);
1383
1384       /* If copy_basic_block has been empty at the start of this iteration,
1385          call gsi_start_bb again to get at the newly added statements.  */
1386       if (gsi_end_p (copy_gsi))
1387         copy_gsi = gsi_start_bb (copy_basic_block);
1388       else
1389         gsi_next (&copy_gsi);
1390
1391       /* Process the new statement.  The call to gimple_regimplify_operands
1392          possibly turned the statement into multiple statements, we
1393          need to process all of them.  */
1394       do
1395         {
1396           tree fn;
1397
1398           stmt = gsi_stmt (copy_gsi);
1399           if (is_gimple_call (stmt)
1400               && gimple_call_va_arg_pack_p (stmt)
1401               && id->gimple_call)
1402             {
1403               /* __builtin_va_arg_pack () should be replaced by
1404                  all arguments corresponding to ... in the caller.  */
1405               tree p;
1406               gimple new_call;
1407               VEC(tree, heap) *argarray;
1408               size_t nargs = gimple_call_num_args (id->gimple_call);
1409               size_t n;
1410
1411               for (p = DECL_ARGUMENTS (id->src_fn); p; p = TREE_CHAIN (p))
1412                 nargs--;
1413
1414               /* Create the new array of arguments.  */
1415               n = nargs + gimple_call_num_args (stmt);
1416               argarray = VEC_alloc (tree, heap, n);
1417               VEC_safe_grow (tree, heap, argarray, n);
1418
1419               /* Copy all the arguments before '...'  */
1420               memcpy (VEC_address (tree, argarray),
1421                       gimple_call_arg_ptr (stmt, 0),
1422                       gimple_call_num_args (stmt) * sizeof (tree));
1423
1424               /* Append the arguments passed in '...'  */
1425               memcpy (VEC_address(tree, argarray) + gimple_call_num_args (stmt),
1426                       gimple_call_arg_ptr (id->gimple_call, 0)
1427                         + (gimple_call_num_args (id->gimple_call) - nargs),
1428                       nargs * sizeof (tree));
1429
1430               new_call = gimple_build_call_vec (gimple_call_fn (stmt),
1431                                                 argarray);
1432
1433               VEC_free (tree, heap, argarray);
1434
1435               /* Copy all GIMPLE_CALL flags, location and block, except
1436                  GF_CALL_VA_ARG_PACK.  */
1437               gimple_call_copy_flags (new_call, stmt);
1438               gimple_call_set_va_arg_pack (new_call, false);
1439               gimple_set_location (new_call, gimple_location (stmt));
1440               gimple_set_block (new_call, gimple_block (stmt));
1441               gimple_call_set_lhs (new_call, gimple_call_lhs (stmt));
1442
1443               gsi_replace (&copy_gsi, new_call, false);
1444               gimple_set_bb (stmt, NULL);
1445               stmt = new_call;
1446             }
1447           else if (is_gimple_call (stmt)
1448                    && id->gimple_call
1449                    && (decl = gimple_call_fndecl (stmt))
1450                    && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL
1451                    && DECL_FUNCTION_CODE (decl) == BUILT_IN_VA_ARG_PACK_LEN)
1452             {
1453               /* __builtin_va_arg_pack_len () should be replaced by
1454                  the number of anonymous arguments.  */
1455               size_t nargs = gimple_call_num_args (id->gimple_call);
1456               tree count, p;
1457               gimple new_stmt;
1458
1459               for (p = DECL_ARGUMENTS (id->src_fn); p; p = TREE_CHAIN (p))
1460                 nargs--;
1461
1462               count = build_int_cst (integer_type_node, nargs);
1463               new_stmt = gimple_build_assign (gimple_call_lhs (stmt), count);
1464               gsi_replace (&copy_gsi, new_stmt, false);
1465               stmt = new_stmt;
1466             }
1467
1468           /* Statements produced by inlining can be unfolded, especially
1469              when we constant propagated some operands.  We can't fold
1470              them right now for two reasons:
1471              1) folding require SSA_NAME_DEF_STMTs to be correct
1472              2) we can't change function calls to builtins.
1473              So we just mark statement for later folding.  We mark
1474              all new statements, instead just statements that has changed
1475              by some nontrivial substitution so even statements made
1476              foldable indirectly are updated.  If this turns out to be
1477              expensive, copy_body can be told to watch for nontrivial
1478              changes.  */
1479           if (id->statements_to_fold)
1480             pointer_set_insert (id->statements_to_fold, stmt);
1481
1482           /* We're duplicating a CALL_EXPR.  Find any corresponding
1483              callgraph edges and update or duplicate them.  */
1484           if (is_gimple_call (stmt))
1485             {
1486               struct cgraph_edge *edge = cgraph_edge (id->src_node, orig_stmt);
1487               int flags;
1488
1489               switch (id->transform_call_graph_edges)
1490                 {
1491               case CB_CGE_DUPLICATE:
1492                 if (edge)
1493                   cgraph_clone_edge (edge, id->dst_node, stmt,
1494                                            REG_BR_PROB_BASE, 1,
1495                                            edge->frequency, true);
1496                 break;
1497
1498               case CB_CGE_MOVE_CLONES:
1499                 cgraph_set_call_stmt_including_clones (id->dst_node, orig_stmt, stmt);
1500                 break;
1501
1502               case CB_CGE_MOVE:
1503                 if (edge)
1504                   cgraph_set_call_stmt (edge, stmt);
1505                 break;
1506
1507               default:
1508                 gcc_unreachable ();
1509                 }
1510
1511             edge = cgraph_edge (id->src_node, orig_stmt);
1512             /* Constant propagation on argument done during inlining
1513                may create new direct call.  Produce an edge for it.  */
1514             if ((!edge
1515                  || (edge->indirect_call
1516                      && id->transform_call_graph_edges == CB_CGE_MOVE_CLONES))
1517                 && is_gimple_call (stmt)
1518                 && (fn = gimple_call_fndecl (stmt)) != NULL)
1519               {
1520                 struct cgraph_node *dest = cgraph_node (fn);
1521
1522                 /* We have missing edge in the callgraph.  This can happen in one case
1523                    where previous inlining turned indirect call into direct call by
1524                    constant propagating arguments.  In all other cases we hit a bug
1525                    (incorrect node sharing is most common reason for missing edges.  */
1526                 gcc_assert (dest->needed || !dest->analyzed);
1527                 if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
1528                   cgraph_create_edge_including_clones (id->dst_node, dest, stmt,
1529                                                        bb->count,
1530                                                        compute_call_stmt_bb_frequency (id->dst_node->decl, bb),
1531                                                        bb->loop_depth,
1532                                                        CIF_ORIGINALLY_INDIRECT_CALL);
1533                 else
1534                   cgraph_create_edge (id->dst_node, dest, stmt,
1535                                       bb->count, CGRAPH_FREQ_BASE,
1536                                       bb->loop_depth)->inline_failed
1537                     = CIF_ORIGINALLY_INDIRECT_CALL;
1538                 if (dump_file)
1539                   {
1540                      fprintf (dump_file, "Created new direct edge to %s",
1541                               cgraph_node_name (dest));
1542                   }
1543               }
1544
1545               flags = gimple_call_flags (stmt);
1546
1547               if (flags & ECF_MAY_BE_ALLOCA)
1548                 cfun->calls_alloca = true;
1549               if (flags & ECF_RETURNS_TWICE)
1550                 cfun->calls_setjmp = true;
1551             }
1552
1553           /* If you think we can abort here, you are wrong.
1554              There is no region 0 in gimple.  */
1555           gcc_assert (lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt) != 0);
1556
1557           if (stmt_could_throw_p (stmt)
1558               /* When we are cloning for inlining, we are supposed to
1559                  construct a clone that calls precisely the same functions
1560                  as original.  However IPA optimizers might've proved
1561                  earlier some function calls as non-trapping that might
1562                  render some basic blocks dead that might become
1563                  unreachable.
1564
1565                  We can't update SSA with unreachable blocks in CFG and thus
1566                  we prevent the scenario by preserving even the "dead" eh
1567                  edges until the point they are later removed by
1568                  fixup_cfg pass.  */
1569               || (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
1570                   && lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt) > 0))
1571             {
1572               int region = lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt);
1573
1574               /* Add an entry for the copied tree in the EH hashtable.
1575                  When cloning or versioning, use the hashtable in
1576                  cfun, and just copy the EH number.  When inlining, use the
1577                  hashtable in the caller, and adjust the region number.  */
1578               if (region > 0)
1579                 add_stmt_to_eh_region (stmt, region + id->eh_region_offset);
1580
1581               /* If this tree doesn't have a region associated with it,
1582                  and there is a "current region,"
1583                  then associate this tree with the current region
1584                  and add edges associated with this region.  */
1585               if (lookup_stmt_eh_region_fn (id->src_cfun, orig_stmt) <= 0
1586                   && id->eh_region > 0
1587                   && stmt_could_throw_p (stmt))
1588                 add_stmt_to_eh_region (stmt, id->eh_region);
1589             }
1590
1591           if (gimple_in_ssa_p (cfun))
1592             {
1593               ssa_op_iter i;
1594               tree def;
1595
1596               find_new_referenced_vars (gsi_stmt (copy_gsi));
1597               FOR_EACH_SSA_TREE_OPERAND (def, stmt, i, SSA_OP_DEF)
1598                 if (TREE_CODE (def) == SSA_NAME)
1599                   SSA_NAME_DEF_STMT (def) = stmt;
1600             }
1601
1602           gsi_next (&copy_gsi);
1603         }
1604       while (!gsi_end_p (copy_gsi));
1605
1606       copy_gsi = gsi_last_bb (copy_basic_block);
1607     }
1608
1609   return copy_basic_block;
1610 }
1611
1612 /* Inserting Single Entry Multiple Exit region in SSA form into code in SSA
1613    form is quite easy, since dominator relationship for old basic blocks does
1614    not change.
1615
1616    There is however exception where inlining might change dominator relation
1617    across EH edges from basic block within inlined functions destinating
1618    to landing pads in function we inline into.
1619
1620    The function fills in PHI_RESULTs of such PHI nodes if they refer
1621    to gimple regs.  Otherwise, the function mark PHI_RESULT of such
1622    PHI nodes for renaming.  For non-gimple regs, renaming is safe: the
1623    EH edges are abnormal and SSA_NAME_OCCURS_IN_ABNORMAL_PHI must be
1624    set, and this means that there will be no overlapping live ranges
1625    for the underlying symbol.
1626
1627    This might change in future if we allow redirecting of EH edges and
1628    we might want to change way build CFG pre-inlining to include
1629    all the possible edges then.  */
1630 static void
1631 update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
1632                                   bool can_throw, bool nonlocal_goto)
1633 {
1634   edge e;
1635   edge_iterator ei;
1636
1637   FOR_EACH_EDGE (e, ei, bb->succs)
1638     if (!e->dest->aux
1639         || ((basic_block)e->dest->aux)->index == ENTRY_BLOCK)
1640       {
1641         gimple phi;
1642         gimple_stmt_iterator si;
1643
1644         if (!nonlocal_goto)
1645           gcc_assert (e->flags & EDGE_EH);
1646
1647         if (!can_throw)
1648           gcc_assert (!(e->flags & EDGE_EH));
1649
1650         for (si = gsi_start_phis (e->dest); !gsi_end_p (si); gsi_next (&si))
1651           {
1652             edge re;
1653
1654             phi = gsi_stmt (si);
1655
1656             /* There shouldn't be any PHI nodes in the ENTRY_BLOCK.  */
1657             gcc_assert (!e->dest->aux);
1658
1659             gcc_assert ((e->flags & EDGE_EH)
1660                         || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)));
1661
1662             if (!is_gimple_reg (PHI_RESULT (phi)))
1663               {
1664                 mark_sym_for_renaming (SSA_NAME_VAR (PHI_RESULT (phi)));
1665                 continue;
1666               }
1667
1668             re = find_edge (ret_bb, e->dest);
1669             gcc_assert (re);
1670             gcc_assert ((re->flags & (EDGE_EH | EDGE_ABNORMAL))
1671                         == (e->flags & (EDGE_EH | EDGE_ABNORMAL)));
1672
1673             SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
1674                      USE_FROM_PTR (PHI_ARG_DEF_PTR_FROM_EDGE (phi, re)));
1675           }
1676       }
1677 }
1678
1679
1680 /* Copy edges from BB into its copy constructed earlier, scale profile
1681    accordingly.  Edges will be taken care of later.  Assume aux
1682    pointers to point to the copies of each BB.  */
1683
1684 static void
1685 copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb)
1686 {
1687   basic_block new_bb = (basic_block) bb->aux;
1688   edge_iterator ei;
1689   edge old_edge;
1690   gimple_stmt_iterator si;
1691   int flags;
1692
1693   /* Use the indices from the original blocks to create edges for the
1694      new ones.  */
1695   FOR_EACH_EDGE (old_edge, ei, bb->succs)
1696     if (!(old_edge->flags & EDGE_EH))
1697       {
1698         edge new_edge;
1699
1700         flags = old_edge->flags;
1701
1702         /* Return edges do get a FALLTHRU flag when the get inlined.  */
1703         if (old_edge->dest->index == EXIT_BLOCK && !old_edge->flags
1704             && old_edge->dest->aux != EXIT_BLOCK_PTR)
1705           flags |= EDGE_FALLTHRU;
1706         new_edge = make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
1707         new_edge->count = old_edge->count * count_scale / REG_BR_PROB_BASE;
1708         new_edge->probability = old_edge->probability;
1709       }
1710
1711   if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK)
1712     return;
1713
1714   for (si = gsi_start_bb (new_bb); !gsi_end_p (si);)
1715     {
1716       gimple copy_stmt;
1717       bool can_throw, nonlocal_goto;
1718
1719       copy_stmt = gsi_stmt (si);
1720       update_stmt (copy_stmt);
1721       if (gimple_in_ssa_p (cfun))
1722         mark_symbols_for_renaming (copy_stmt);
1723
1724       /* Do this before the possible split_block.  */
1725       gsi_next (&si);
1726
1727       /* If this tree could throw an exception, there are two
1728          cases where we need to add abnormal edge(s): the
1729          tree wasn't in a region and there is a "current
1730          region" in the caller; or the original tree had
1731          EH edges.  In both cases split the block after the tree,
1732          and add abnormal edge(s) as needed; we need both
1733          those from the callee and the caller.
1734          We check whether the copy can throw, because the const
1735          propagation can change an INDIRECT_REF which throws
1736          into a COMPONENT_REF which doesn't.  If the copy
1737          can throw, the original could also throw.  */
1738       can_throw = stmt_can_throw_internal (copy_stmt);
1739       nonlocal_goto = stmt_can_make_abnormal_goto (copy_stmt);
1740
1741       if (can_throw || nonlocal_goto)
1742         {
1743           if (!gsi_end_p (si))
1744             /* Note that bb's predecessor edges aren't necessarily
1745                right at this point; split_block doesn't care.  */
1746             {
1747               edge e = split_block (new_bb, copy_stmt);
1748
1749               new_bb = e->dest;
1750               new_bb->aux = e->src->aux;
1751               si = gsi_start_bb (new_bb);
1752             }
1753         }
1754
1755       if (can_throw)
1756         make_eh_edges (copy_stmt);
1757
1758       if (nonlocal_goto)
1759         make_abnormal_goto_edges (gimple_bb (copy_stmt), true);
1760
1761       if ((can_throw || nonlocal_goto)
1762           && gimple_in_ssa_p (cfun))
1763         update_ssa_across_abnormal_edges (gimple_bb (copy_stmt), ret_bb,
1764                                           can_throw, nonlocal_goto);
1765     }
1766 }
1767
1768 /* Copy the PHIs.  All blocks and edges are copied, some blocks
1769    was possibly split and new outgoing EH edges inserted.
1770    BB points to the block of original function and AUX pointers links
1771    the original and newly copied blocks.  */
1772
1773 static void
1774 copy_phis_for_bb (basic_block bb, copy_body_data *id)
1775 {
1776   basic_block const new_bb = (basic_block) bb->aux;
1777   edge_iterator ei;
1778   gimple phi;
1779   gimple_stmt_iterator si;
1780
1781   for (si = gsi_start (phi_nodes (bb)); !gsi_end_p (si); gsi_next (&si))
1782     {
1783       tree res, new_res;
1784       gimple new_phi;
1785       edge new_edge;
1786
1787       phi = gsi_stmt (si);
1788       res = PHI_RESULT (phi);
1789       new_res = res;
1790       if (is_gimple_reg (res))
1791         {
1792           walk_tree (&new_res, copy_tree_body_r, id, NULL);
1793           SSA_NAME_DEF_STMT (new_res)
1794             = new_phi = create_phi_node (new_res, new_bb);
1795           FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
1796             {
1797               edge const old_edge
1798                 = find_edge ((basic_block) new_edge->src->aux, bb);
1799               tree arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge);
1800               tree new_arg = arg;
1801               tree block = id->block;
1802               id->block = NULL_TREE;
1803               walk_tree (&new_arg, copy_tree_body_r, id, NULL);
1804               id->block = block;
1805               gcc_assert (new_arg);
1806               /* With return slot optimization we can end up with
1807                  non-gimple (foo *)&this->m, fix that here.  */
1808               if (TREE_CODE (new_arg) != SSA_NAME
1809                   && TREE_CODE (new_arg) != FUNCTION_DECL
1810                   && !is_gimple_val (new_arg))
1811                 {
1812                   gimple_seq stmts = NULL;
1813                   new_arg = force_gimple_operand (new_arg, &stmts, true, NULL);
1814                   gsi_insert_seq_on_edge_immediate (new_edge, stmts);
1815                 }
1816               add_phi_arg (new_phi, new_arg, new_edge);
1817             }
1818         }
1819     }
1820 }
1821
1822
1823 /* Wrapper for remap_decl so it can be used as a callback.  */
1824
1825 static tree
1826 remap_decl_1 (tree decl, void *data)
1827 {
1828   return remap_decl (decl, (copy_body_data *) data);
1829 }
1830
1831 /* Build struct function and associated datastructures for the new clone
1832    NEW_FNDECL to be build.  CALLEE_FNDECL is the original */
1833
1834 static void
1835 initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
1836                  int frequency)
1837 {
1838   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
1839   gcov_type count_scale, frequency_scale;
1840
1841   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
1842     count_scale = (REG_BR_PROB_BASE * count
1843                    / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
1844   else
1845     count_scale = 1;
1846
1847   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
1848     frequency_scale = (REG_BR_PROB_BASE * frequency
1849                        /
1850                        ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
1851   else
1852     frequency_scale = count_scale;
1853
1854   /* Register specific tree functions.  */
1855   gimple_register_cfg_hooks ();
1856
1857   /* Get clean struct function.  */
1858   push_struct_function (new_fndecl);
1859
1860   /* We will rebuild these, so just sanity check that they are empty.  */
1861   gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL);
1862   gcc_assert (cfun->local_decls == NULL);
1863   gcc_assert (cfun->cfg == NULL);
1864   gcc_assert (cfun->decl == new_fndecl);
1865
1866   /* Copy items we preserve during clonning.  */
1867   cfun->static_chain_decl = src_cfun->static_chain_decl;
1868   cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area;
1869   cfun->function_end_locus = src_cfun->function_end_locus;
1870   cfun->curr_properties = src_cfun->curr_properties;
1871   cfun->last_verified = src_cfun->last_verified;
1872   if (src_cfun->ipa_transforms_to_apply)
1873     cfun->ipa_transforms_to_apply = VEC_copy (ipa_opt_pass, heap,
1874                                               src_cfun->ipa_transforms_to_apply);
1875   cfun->va_list_gpr_size = src_cfun->va_list_gpr_size;
1876   cfun->va_list_fpr_size = src_cfun->va_list_fpr_size;
1877   cfun->function_frequency = src_cfun->function_frequency;
1878   cfun->has_nonlocal_label = src_cfun->has_nonlocal_label;
1879   cfun->stdarg = src_cfun->stdarg;
1880   cfun->dont_save_pending_sizes_p = src_cfun->dont_save_pending_sizes_p;
1881   cfun->after_inlining = src_cfun->after_inlining;
1882   cfun->returns_struct = src_cfun->returns_struct;
1883   cfun->returns_pcc_struct = src_cfun->returns_pcc_struct;
1884   cfun->after_tree_profile = src_cfun->after_tree_profile;
1885
1886   init_empty_tree_cfg ();
1887
1888   ENTRY_BLOCK_PTR->count =
1889     (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
1890      REG_BR_PROB_BASE);
1891   ENTRY_BLOCK_PTR->frequency =
1892     (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
1893      frequency_scale / REG_BR_PROB_BASE);
1894   EXIT_BLOCK_PTR->count =
1895     (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
1896      REG_BR_PROB_BASE);
1897   EXIT_BLOCK_PTR->frequency =
1898     (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
1899      frequency_scale / REG_BR_PROB_BASE);
1900   if (src_cfun->eh)
1901     init_eh_for_function ();
1902
1903   if (src_cfun->gimple_df)
1904     {
1905       init_tree_ssa (cfun);
1906       cfun->gimple_df->in_ssa_p = true;
1907       init_ssa_operands ();
1908     }
1909   pop_cfun ();
1910 }
1911
1912 /* Make a copy of the body of FN so that it can be inserted inline in
1913    another function.  Walks FN via CFG, returns new fndecl.  */
1914
1915 static tree
1916 copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
1917                basic_block entry_block_map, basic_block exit_block_map)
1918 {
1919   tree callee_fndecl = id->src_fn;
1920   /* Original cfun for the callee, doesn't change.  */
1921   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
1922   struct function *cfun_to_copy;
1923   basic_block bb;
1924   tree new_fndecl = NULL;
1925   gcov_type count_scale, frequency_scale;
1926   int last;
1927
1928   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
1929     count_scale = (REG_BR_PROB_BASE * count
1930                    / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
1931   else
1932     count_scale = 1;
1933
1934   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
1935     frequency_scale = (REG_BR_PROB_BASE * frequency
1936                        /
1937                        ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
1938   else
1939     frequency_scale = count_scale;
1940
1941   /* Register specific tree functions.  */
1942   gimple_register_cfg_hooks ();
1943
1944   /* Must have a CFG here at this point.  */
1945   gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION
1946               (DECL_STRUCT_FUNCTION (callee_fndecl)));
1947
1948   cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
1949
1950   ENTRY_BLOCK_PTR_FOR_FUNCTION (cfun_to_copy)->aux = entry_block_map;
1951   EXIT_BLOCK_PTR_FOR_FUNCTION (cfun_to_copy)->aux = exit_block_map;
1952   entry_block_map->aux = ENTRY_BLOCK_PTR_FOR_FUNCTION (cfun_to_copy);
1953   exit_block_map->aux = EXIT_BLOCK_PTR_FOR_FUNCTION (cfun_to_copy);
1954
1955   /* Duplicate any exception-handling regions.  */
1956   if (cfun->eh)
1957     {
1958       id->eh_region_offset
1959         = duplicate_eh_regions (cfun_to_copy, remap_decl_1, id,
1960                                 0, id->eh_region);
1961     }
1962
1963   /* Use aux pointers to map the original blocks to copy.  */
1964   FOR_EACH_BB_FN (bb, cfun_to_copy)
1965     {
1966       basic_block new_bb = copy_bb (id, bb, frequency_scale, count_scale);
1967       bb->aux = new_bb;
1968       new_bb->aux = bb;
1969     }
1970
1971   last = last_basic_block;
1972
1973   /* Now that we've duplicated the blocks, duplicate their edges.  */
1974   FOR_ALL_BB_FN (bb, cfun_to_copy)
1975     copy_edges_for_bb (bb, count_scale, exit_block_map);
1976
1977   if (gimple_in_ssa_p (cfun))
1978     FOR_ALL_BB_FN (bb, cfun_to_copy)
1979       copy_phis_for_bb (bb, id);
1980
1981   FOR_ALL_BB_FN (bb, cfun_to_copy)
1982     {
1983       ((basic_block)bb->aux)->aux = NULL;
1984       bb->aux = NULL;
1985     }
1986
1987   /* Zero out AUX fields of newly created block during EH edge
1988      insertion. */
1989   for (; last < last_basic_block; last++)
1990     BASIC_BLOCK (last)->aux = NULL;
1991   entry_block_map->aux = NULL;
1992   exit_block_map->aux = NULL;
1993
1994   return new_fndecl;
1995 }
1996
1997 static tree
1998 copy_body (copy_body_data *id, gcov_type count, int frequency,
1999            basic_block entry_block_map, basic_block exit_block_map)
2000 {
2001   tree fndecl = id->src_fn;
2002   tree body;
2003
2004   /* If this body has a CFG, walk CFG and copy.  */
2005   gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl)));
2006   body = copy_cfg_body (id, count, frequency, entry_block_map, exit_block_map);
2007
2008   return body;
2009 }
2010
2011 /* Return true if VALUE is an ADDR_EXPR of an automatic variable
2012    defined in function FN, or of a data member thereof.  */
2013
2014 static bool
2015 self_inlining_addr_expr (tree value, tree fn)
2016 {
2017   tree var;
2018
2019   if (TREE_CODE (value) != ADDR_EXPR)
2020     return false;
2021
2022   var = get_base_address (TREE_OPERAND (value, 0));
2023
2024   return var && auto_var_in_fn_p (var, fn);
2025 }
2026
2027 static void
2028 insert_init_stmt (basic_block bb, gimple init_stmt)
2029 {
2030   /* If VAR represents a zero-sized variable, it's possible that the
2031      assignment statement may result in no gimple statements.  */
2032   if (init_stmt)
2033     {
2034       gimple_stmt_iterator si = gsi_last_bb (bb);
2035
2036       /* We can end up with init statements that store to a non-register
2037          from a rhs with a conversion.  Handle that here by forcing the
2038          rhs into a temporary.  gimple_regimplify_operands is not
2039          prepared to do this for us.  */
2040       if (!is_gimple_reg (gimple_assign_lhs (init_stmt))
2041           && is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt)))
2042           && gimple_assign_rhs_class (init_stmt) == GIMPLE_UNARY_RHS)
2043         {
2044           tree rhs = build1 (gimple_assign_rhs_code (init_stmt),
2045                              gimple_expr_type (init_stmt),
2046                              gimple_assign_rhs1 (init_stmt));
2047           rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false,
2048                                           GSI_NEW_STMT);
2049           gimple_assign_set_rhs_code (init_stmt, TREE_CODE (rhs));
2050           gimple_assign_set_rhs1 (init_stmt, rhs);
2051         }
2052       gsi_insert_after (&si, init_stmt, GSI_NEW_STMT);
2053       gimple_regimplify_operands (init_stmt, &si);
2054       mark_symbols_for_renaming (init_stmt);
2055     }
2056 }
2057
2058 /* Initialize parameter P with VALUE.  If needed, produce init statement
2059    at the end of BB.  When BB is NULL, we return init statement to be
2060    output later.  */
2061 static gimple
2062 setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn,
2063                      basic_block bb, tree *vars)
2064 {
2065   gimple init_stmt = NULL;
2066   tree var;
2067   tree rhs = value;
2068   tree def = (gimple_in_ssa_p (cfun)
2069               ? gimple_default_def (id->src_cfun, p) : NULL);
2070
2071   if (value
2072       && value != error_mark_node
2073       && !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value)))
2074     {
2075       if (fold_convertible_p (TREE_TYPE (p), value))
2076         rhs = fold_build1 (NOP_EXPR, TREE_TYPE (p), value);
2077       else
2078         /* ???  For valid (GIMPLE) programs we should not end up here.
2079            Still if something has gone wrong and we end up with truly
2080            mismatched types here, fall back to using a VIEW_CONVERT_EXPR
2081            to not leak invalid GIMPLE to the following passes.  */
2082         rhs = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (p), value);
2083     }
2084
2085   /* If the parameter is never assigned to, has no SSA_NAMEs created,
2086      we may not need to create a new variable here at all.  Instead, we may
2087      be able to just use the argument value.  */
2088   if (TREE_READONLY (p)
2089       && !TREE_ADDRESSABLE (p)
2090       && value && !TREE_SIDE_EFFECTS (value)
2091       && !def)
2092     {
2093       /* We may produce non-gimple trees by adding NOPs or introduce
2094          invalid sharing when operand is not really constant.
2095          It is not big deal to prohibit constant propagation here as
2096          we will constant propagate in DOM1 pass anyway.  */
2097       if (is_gimple_min_invariant (value)
2098           && useless_type_conversion_p (TREE_TYPE (p),
2099                                                  TREE_TYPE (value))
2100           /* We have to be very careful about ADDR_EXPR.  Make sure
2101              the base variable isn't a local variable of the inlined
2102              function, e.g., when doing recursive inlining, direct or
2103              mutually-recursive or whatever, which is why we don't
2104              just test whether fn == current_function_decl.  */
2105           && ! self_inlining_addr_expr (value, fn))
2106         {
2107           insert_decl_map (id, p, value);
2108           return NULL;
2109         }
2110     }
2111
2112   /* Make an equivalent VAR_DECL.  Note that we must NOT remap the type
2113      here since the type of this decl must be visible to the calling
2114      function.  */
2115   var = copy_decl_to_var (p, id);
2116   if (gimple_in_ssa_p (cfun) && TREE_CODE (var) == VAR_DECL)
2117     {
2118       get_var_ann (var);
2119       add_referenced_var (var);
2120     }
2121
2122   /* Register the VAR_DECL as the equivalent for the PARM_DECL;
2123      that way, when the PARM_DECL is encountered, it will be
2124      automatically replaced by the VAR_DECL.  */
2125   insert_decl_map (id, p, var);
2126
2127   /* Declare this new variable.  */
2128   TREE_CHAIN (var) = *vars;
2129   *vars = var;
2130
2131   /* Make gimplifier happy about this variable.  */
2132   DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
2133
2134   /* Even if P was TREE_READONLY, the new VAR should not be.
2135      In the original code, we would have constructed a
2136      temporary, and then the function body would have never
2137      changed the value of P.  However, now, we will be
2138      constructing VAR directly.  The constructor body may
2139      change its value multiple times as it is being
2140      constructed.  Therefore, it must not be TREE_READONLY;
2141      the back-end assumes that TREE_READONLY variable is
2142      assigned to only once.  */
2143   if (TYPE_NEEDS_CONSTRUCTING (TREE_TYPE (p)))
2144     TREE_READONLY (var) = 0;
2145
2146   /* If there is no setup required and we are in SSA, take the easy route
2147      replacing all SSA names representing the function parameter by the
2148      SSA name passed to function.
2149
2150      We need to construct map for the variable anyway as it might be used
2151      in different SSA names when parameter is set in function.
2152
2153      Do replacement at -O0 for const arguments replaced by constant.
2154      This is important for builtin_constant_p and other construct requiring
2155      constant argument to be visible in inlined function body.
2156
2157      FIXME: This usually kills the last connection in between inlined
2158      function parameter and the actual value in debug info.  Can we do
2159      better here?  If we just inserted the statement, copy propagation
2160      would kill it anyway as it always did in older versions of GCC.
2161
2162      We might want to introduce a notion that single SSA_NAME might
2163      represent multiple variables for purposes of debugging. */
2164   if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p)
2165       && (optimize
2166           || (TREE_READONLY (p)
2167               && is_gimple_min_invariant (rhs)))
2168       && (TREE_CODE (rhs) == SSA_NAME
2169           || is_gimple_min_invariant (rhs))
2170       && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
2171     {
2172       insert_decl_map (id, def, rhs);
2173       return NULL;
2174     }
2175
2176   /* If the value of argument is never used, don't care about initializing
2177      it.  */
2178   if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p))
2179     {
2180       gcc_assert (!value || !TREE_SIDE_EFFECTS (value));
2181       return NULL;
2182     }
2183
2184   /* Initialize this VAR_DECL from the equivalent argument.  Convert
2185      the argument to the proper type in case it was promoted.  */
2186   if (value)
2187     {
2188       if (rhs == error_mark_node)
2189         {
2190           insert_decl_map (id, p, var);
2191           return NULL;
2192         }
2193
2194       STRIP_USELESS_TYPE_CONVERSION (rhs);
2195
2196       /* We want to use MODIFY_EXPR, not INIT_EXPR here so that we
2197          keep our trees in gimple form.  */
2198       if (def && gimple_in_ssa_p (cfun) && is_gimple_reg (p))
2199         {
2200           def = remap_ssa_name (def, id);
2201           init_stmt = gimple_build_assign (def, rhs);
2202           SSA_NAME_IS_DEFAULT_DEF (def) = 0;
2203           set_default_def (var, NULL);
2204         }
2205       else
2206         init_stmt = gimple_build_assign (var, rhs);
2207
2208       if (bb && init_stmt)
2209         insert_init_stmt (bb, init_stmt);
2210     }
2211   return init_stmt;
2212 }
2213
2214 /* Generate code to initialize the parameters of the function at the
2215    top of the stack in ID from the GIMPLE_CALL STMT.  */
2216
2217 static void
2218 initialize_inlined_parameters (copy_body_data *id, gimple stmt,
2219                                tree fn, basic_block bb)
2220 {
2221   tree parms;
2222   size_t i;
2223   tree p;
2224   tree vars = NULL_TREE;
2225   tree static_chain = gimple_call_chain (stmt);
2226
2227   /* Figure out what the parameters are.  */
2228   parms = DECL_ARGUMENTS (fn);
2229
2230   /* Loop through the parameter declarations, replacing each with an
2231      equivalent VAR_DECL, appropriately initialized.  */
2232   for (p = parms, i = 0; p; p = TREE_CHAIN (p), i++)
2233     {
2234       tree val;
2235       val = i < gimple_call_num_args (stmt) ? gimple_call_arg (stmt, i) : NULL;
2236       setup_one_parameter (id, p, val, fn, bb, &vars);
2237     }
2238
2239   /* Initialize the static chain.  */
2240   p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl;
2241   gcc_assert (fn != current_function_decl);
2242   if (p)
2243     {
2244       /* No static chain?  Seems like a bug in tree-nested.c.  */
2245       gcc_assert (static_chain);
2246
2247       setup_one_parameter (id, p, static_chain, fn, bb, &vars);
2248     }
2249
2250   declare_inline_vars (id->block, vars);
2251 }
2252
2253
2254 /* Declare a return variable to replace the RESULT_DECL for the
2255    function we are calling.  An appropriate DECL_STMT is returned.
2256    The USE_STMT is filled to contain a use of the declaration to
2257    indicate the return value of the function.
2258
2259    RETURN_SLOT, if non-null is place where to store the result.  It
2260    is set only for CALL_EXPR_RETURN_SLOT_OPT.  MODIFY_DEST, if non-null,
2261    was the LHS of the MODIFY_EXPR to which this call is the RHS.
2262
2263    The return value is a (possibly null) value that is the result of the
2264    function as seen by the callee.  *USE_P is a (possibly null) value that
2265    holds the result as seen by the caller.  */
2266
2267 static tree
2268 declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest,
2269                          tree *use_p)
2270 {
2271   tree callee = id->src_fn;
2272   tree caller = id->dst_fn;
2273   tree result = DECL_RESULT (callee);
2274   tree callee_type = TREE_TYPE (result);
2275   tree caller_type = TREE_TYPE (TREE_TYPE (callee));
2276   tree var, use;
2277
2278   /* We don't need to do anything for functions that don't return
2279      anything.  */
2280   if (!result || VOID_TYPE_P (callee_type))
2281     {
2282       *use_p = NULL_TREE;
2283       return NULL_TREE;
2284     }
2285
2286   /* If there was a return slot, then the return value is the
2287      dereferenced address of that object.  */
2288   if (return_slot)
2289     {
2290       /* The front end shouldn't have used both return_slot and
2291          a modify expression.  */
2292       gcc_assert (!modify_dest);
2293       if (DECL_BY_REFERENCE (result))
2294         {
2295           tree return_slot_addr = build_fold_addr_expr (return_slot);
2296           STRIP_USELESS_TYPE_CONVERSION (return_slot_addr);
2297
2298           /* We are going to construct *&return_slot and we can't do that
2299              for variables believed to be not addressable.
2300
2301              FIXME: This check possibly can match, because values returned
2302              via return slot optimization are not believed to have address
2303              taken by alias analysis.  */
2304           gcc_assert (TREE_CODE (return_slot) != SSA_NAME);
2305           if (gimple_in_ssa_p (cfun))
2306             {
2307               HOST_WIDE_INT bitsize;
2308               HOST_WIDE_INT bitpos;
2309               tree offset;
2310               enum machine_mode mode;
2311               int unsignedp;
2312               int volatilep;
2313               tree base;
2314               base = get_inner_reference (return_slot, &bitsize, &bitpos,
2315                                           &offset,
2316                                           &mode, &unsignedp, &volatilep,
2317                                           false);
2318               if (TREE_CODE (base) == INDIRECT_REF)
2319                 base = TREE_OPERAND (base, 0);
2320               if (TREE_CODE (base) == SSA_NAME)
2321                 base = SSA_NAME_VAR (base);
2322               mark_sym_for_renaming (base);
2323             }
2324           var = return_slot_addr;
2325         }
2326       else
2327         {
2328           var = return_slot;
2329           gcc_assert (TREE_CODE (var) != SSA_NAME);
2330           TREE_ADDRESSABLE (var) |= TREE_ADDRESSABLE (result);
2331         }
2332       if ((TREE_CODE (TREE_TYPE (result)) == COMPLEX_TYPE
2333            || TREE_CODE (TREE_TYPE (result)) == VECTOR_TYPE)
2334           && !DECL_GIMPLE_REG_P (result)
2335           && DECL_P (var))
2336         DECL_GIMPLE_REG_P (var) = 0;
2337       use = NULL;
2338       goto done;
2339     }
2340
2341   /* All types requiring non-trivial constructors should have been handled.  */
2342   gcc_assert (!TREE_ADDRESSABLE (callee_type));
2343
2344   /* Attempt to avoid creating a new temporary variable.  */
2345   if (modify_dest
2346       && TREE_CODE (modify_dest) != SSA_NAME)
2347     {
2348       bool use_it = false;
2349
2350       /* We can't use MODIFY_DEST if there's type promotion involved.  */
2351       if (!useless_type_conversion_p (callee_type, caller_type))
2352         use_it = false;
2353
2354       /* ??? If we're assigning to a variable sized type, then we must
2355          reuse the destination variable, because we've no good way to
2356          create variable sized temporaries at this point.  */
2357       else if (TREE_CODE (TYPE_SIZE_UNIT (caller_type)) != INTEGER_CST)
2358         use_it = true;
2359
2360       /* If the callee cannot possibly modify MODIFY_DEST, then we can
2361          reuse it as the result of the call directly.  Don't do this if
2362          it would promote MODIFY_DEST to addressable.  */
2363       else if (TREE_ADDRESSABLE (result))
2364         use_it = false;
2365       else
2366         {
2367           tree base_m = get_base_address (modify_dest);
2368
2369           /* If the base isn't a decl, then it's a pointer, and we don't
2370              know where that's going to go.  */
2371           if (!DECL_P (base_m))
2372             use_it = false;
2373           else if (is_global_var (base_m))
2374             use_it = false;
2375           else if ((TREE_CODE (TREE_TYPE (result)) == COMPLEX_TYPE
2376                     || TREE_CODE (TREE_TYPE (result)) == VECTOR_TYPE)
2377                    && !DECL_GIMPLE_REG_P (result)
2378                    && DECL_GIMPLE_REG_P (base_m))
2379             use_it = false;
2380           else if (!TREE_ADDRESSABLE (base_m))
2381             use_it = true;
2382         }
2383
2384       if (use_it)
2385         {
2386           var = modify_dest;
2387           use = NULL;
2388           goto done;
2389         }
2390     }
2391
2392   gcc_assert (TREE_CODE (TYPE_SIZE_UNIT (callee_type)) == INTEGER_CST);
2393
2394   var = copy_result_decl_to_var (result, id);
2395   if (gimple_in_ssa_p (cfun))
2396     {
2397       get_var_ann (var);
2398       add_referenced_var (var);
2399     }
2400
2401   DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
2402   DECL_STRUCT_FUNCTION (caller)->local_decls
2403     = tree_cons (NULL_TREE, var,
2404                  DECL_STRUCT_FUNCTION (caller)->local_decls);
2405
2406   /* Do not have the rest of GCC warn about this variable as it should
2407      not be visible to the user.  */
2408   TREE_NO_WARNING (var) = 1;
2409
2410   declare_inline_vars (id->block, var);
2411
2412   /* Build the use expr.  If the return type of the function was
2413      promoted, convert it back to the expected type.  */
2414   use = var;
2415   if (!useless_type_conversion_p (caller_type, TREE_TYPE (var)))
2416     use = fold_convert (caller_type, var);
2417
2418   STRIP_USELESS_TYPE_CONVERSION (use);
2419
2420   if (DECL_BY_REFERENCE (result))
2421     {
2422       TREE_ADDRESSABLE (var) = 1;
2423       var = build_fold_addr_expr (var);
2424     }
2425
2426  done:
2427   /* Register the VAR_DECL as the equivalent for the RESULT_DECL; that
2428      way, when the RESULT_DECL is encountered, it will be
2429      automatically replaced by the VAR_DECL.  */
2430   insert_decl_map (id, result, var);
2431
2432   /* Remember this so we can ignore it in remap_decls.  */
2433   id->retvar = var;
2434
2435   *use_p = use;
2436   return var;
2437 }
2438
2439 /* Returns nonzero if a function can be inlined as a tree.  */
2440
2441 bool
2442 tree_inlinable_function_p (tree fn)
2443 {
2444   return inlinable_function_p (fn);
2445 }
2446
2447 static const char *inline_forbidden_reason;
2448
2449 /* A callback for walk_gimple_seq to handle tree operands.  Returns
2450    NULL_TREE if a function can be inlined, otherwise sets the reason
2451    why not and returns a tree representing the offending operand. */
2452
2453 static tree
2454 inline_forbidden_p_op (tree *nodep, int *walk_subtrees ATTRIBUTE_UNUSED,
2455                          void *fnp ATTRIBUTE_UNUSED)
2456 {
2457   tree node = *nodep;
2458   tree t;
2459
2460   if (TREE_CODE (node) == RECORD_TYPE || TREE_CODE (node) == UNION_TYPE)
2461     {
2462       /* We cannot inline a function of the form
2463
2464            void F (int i) { struct S { int ar[i]; } s; }
2465
2466          Attempting to do so produces a catch-22.
2467          If walk_tree examines the TYPE_FIELDS chain of RECORD_TYPE/
2468          UNION_TYPE nodes, then it goes into infinite recursion on a
2469          structure containing a pointer to its own type.  If it doesn't,
2470          then the type node for S doesn't get adjusted properly when
2471          F is inlined.
2472
2473          ??? This is likely no longer true, but it's too late in the 4.0
2474          cycle to try to find out.  This should be checked for 4.1.  */
2475       for (t = TYPE_FIELDS (node); t; t = TREE_CHAIN (t))
2476         if (variably_modified_type_p (TREE_TYPE (t), NULL))
2477           {
2478             inline_forbidden_reason
2479               = G_("function %q+F can never be inlined "
2480                    "because it uses variable sized variables");
2481             return node;
2482           }
2483     }
2484
2485   return NULL_TREE;
2486 }
2487
2488
2489 /* A callback for walk_gimple_seq to handle statements.  Returns
2490    non-NULL iff a function can not be inlined.  Also sets the reason
2491    why. */
2492
2493 static tree
2494 inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
2495                          struct walk_stmt_info *wip)
2496 {
2497   tree fn = (tree) wip->info;
2498   tree t;
2499   gimple stmt = gsi_stmt (*gsi);
2500
2501   switch (gimple_code (stmt))
2502     {
2503     case GIMPLE_CALL:
2504       /* Refuse to inline alloca call unless user explicitly forced so as
2505          this may change program's memory overhead drastically when the
2506          function using alloca is called in loop.  In GCC present in
2507          SPEC2000 inlining into schedule_block cause it to require 2GB of
2508          RAM instead of 256MB.  */
2509       if (gimple_alloca_call_p (stmt)
2510           && !lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn)))
2511         {
2512           inline_forbidden_reason
2513             = G_("function %q+F can never be inlined because it uses "
2514                  "alloca (override using the always_inline attribute)");
2515           *handled_ops_p = true;
2516           return fn;
2517         }
2518
2519       t = gimple_call_fndecl (stmt);
2520       if (t == NULL_TREE)
2521         break;
2522
2523       /* We cannot inline functions that call setjmp.  */
2524       if (setjmp_call_p (t))
2525         {
2526           inline_forbidden_reason
2527             = G_("function %q+F can never be inlined because it uses setjmp");
2528           *handled_ops_p = true;
2529           return t;
2530         }
2531
2532       if (DECL_BUILT_IN_CLASS (t) == BUILT_IN_NORMAL)
2533         switch (DECL_FUNCTION_CODE (t))
2534           {
2535             /* We cannot inline functions that take a variable number of
2536                arguments.  */
2537           case BUILT_IN_VA_START:
2538           case BUILT_IN_NEXT_ARG:
2539           case BUILT_IN_VA_END:
2540             inline_forbidden_reason
2541               = G_("function %q+F can never be inlined because it "
2542                    "uses variable argument lists");
2543             *handled_ops_p = true;
2544             return t;
2545
2546           case BUILT_IN_LONGJMP:
2547             /* We can't inline functions that call __builtin_longjmp at
2548                all.  The non-local goto machinery really requires the
2549                destination be in a different function.  If we allow the
2550                function calling __builtin_longjmp to be inlined into the
2551                function calling __builtin_setjmp, Things will Go Awry.  */
2552             inline_forbidden_reason
2553               = G_("function %q+F can never be inlined because "
2554                    "it uses setjmp-longjmp exception handling");
2555             *handled_ops_p = true;
2556             return t;
2557
2558           case BUILT_IN_NONLOCAL_GOTO:
2559             /* Similarly.  */
2560             inline_forbidden_reason
2561               = G_("function %q+F can never be inlined because "
2562                    "it uses non-local goto");
2563             *handled_ops_p = true;
2564             return t;
2565
2566           case BUILT_IN_RETURN:
2567           case BUILT_IN_APPLY_ARGS:
2568             /* If a __builtin_apply_args caller would be inlined,
2569                it would be saving arguments of the function it has
2570                been inlined into.  Similarly __builtin_return would
2571                return from the function the inline has been inlined into.  */
2572             inline_forbidden_reason
2573               = G_("function %q+F can never be inlined because "
2574                    "it uses __builtin_return or __builtin_apply_args");
2575             *handled_ops_p = true;
2576             return t;
2577
2578           default:
2579             break;
2580           }
2581       break;
2582
2583     case GIMPLE_GOTO:
2584       t = gimple_goto_dest (stmt);
2585
2586       /* We will not inline a function which uses computed goto.  The
2587          addresses of its local labels, which may be tucked into
2588          global storage, are of course not constant across
2589          instantiations, which causes unexpected behavior.  */
2590       if (TREE_CODE (t) != LABEL_DECL)
2591         {
2592           inline_forbidden_reason
2593             = G_("function %q+F can never be inlined "
2594                  "because it contains a computed goto");
2595           *handled_ops_p = true;
2596           return t;
2597         }
2598       break;
2599
2600     case GIMPLE_LABEL:
2601       t = gimple_label_label (stmt);
2602       if (DECL_NONLOCAL (t))
2603         {
2604           /* We cannot inline a function that receives a non-local goto
2605              because we cannot remap the destination label used in the
2606              function that is performing the non-local goto.  */
2607           inline_forbidden_reason
2608             = G_("function %q+F can never be inlined "
2609                  "because it receives a non-local goto");
2610           *handled_ops_p = true;
2611           return t;
2612         }
2613       break;
2614
2615     default:
2616       break;
2617     }
2618
2619   *handled_ops_p = false;
2620   return NULL_TREE;
2621 }
2622
2623
2624 static tree
2625 inline_forbidden_p_2 (tree *nodep, int *walk_subtrees,
2626                       void *fnp)
2627 {
2628   tree node = *nodep;
2629   tree fn = (tree) fnp;
2630
2631   if (TREE_CODE (node) == LABEL_DECL && DECL_CONTEXT (node) == fn)
2632     {
2633       inline_forbidden_reason
2634         = G_("function %q+F can never be inlined "
2635              "because it saves address of local label in a static variable");
2636       return node;
2637     }
2638
2639   if (TYPE_P (node))
2640     *walk_subtrees = 0;
2641
2642   return NULL_TREE;
2643 }
2644
2645 /* Return true if FNDECL is a function that cannot be inlined into
2646    another one.  */
2647
2648 static bool
2649 inline_forbidden_p (tree fndecl)
2650 {
2651   struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
2652   tree step;
2653   struct walk_stmt_info wi;
2654   struct pointer_set_t *visited_nodes;
2655   basic_block bb;
2656   bool forbidden_p = false;
2657
2658   visited_nodes = pointer_set_create ();
2659   memset (&wi, 0, sizeof (wi));
2660   wi.info = (void *) fndecl;
2661   wi.pset = visited_nodes;
2662
2663   FOR_EACH_BB_FN (bb, fun)
2664     {
2665       gimple ret;
2666       gimple_seq seq = bb_seq (bb);
2667       ret = walk_gimple_seq (seq, inline_forbidden_p_stmt,
2668                              inline_forbidden_p_op, &wi);
2669       forbidden_p = (ret != NULL);
2670       if (forbidden_p)
2671         goto egress;
2672     }
2673
2674   for (step = fun->local_decls; step; step = TREE_CHAIN (step))
2675     {
2676       tree decl = TREE_VALUE (step);
2677       if (TREE_CODE (decl) == VAR_DECL
2678           && TREE_STATIC (decl)
2679           && !DECL_EXTERNAL (decl)
2680           && DECL_INITIAL (decl))
2681         {
2682           tree ret;
2683           ret = walk_tree_without_duplicates (&DECL_INITIAL (decl),
2684                                               inline_forbidden_p_2, fndecl);
2685           forbidden_p = (ret != NULL);
2686           if (forbidden_p)
2687             goto egress;
2688         }
2689     }
2690
2691 egress:
2692   pointer_set_destroy (visited_nodes);
2693   return forbidden_p;
2694 }
2695
2696 /* Returns nonzero if FN is a function that does not have any
2697    fundamental inline blocking properties.  */
2698
2699 static bool
2700 inlinable_function_p (tree fn)
2701 {
2702   bool inlinable = true;
2703   bool do_warning;
2704   tree always_inline;
2705
2706   /* If we've already decided this function shouldn't be inlined,
2707      there's no need to check again.  */
2708   if (DECL_UNINLINABLE (fn))
2709     return false;
2710
2711   /* We only warn for functions declared `inline' by the user.  */
2712   do_warning = (warn_inline
2713                 && DECL_DECLARED_INLINE_P (fn)
2714                 && !DECL_NO_INLINE_WARNING_P (fn)
2715                 && !DECL_IN_SYSTEM_HEADER (fn));
2716
2717   always_inline = lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn));
2718
2719   if (flag_no_inline
2720       && always_inline == NULL)
2721     {
2722       if (do_warning)
2723         warning (OPT_Winline, "function %q+F can never be inlined because it "
2724                  "is suppressed using -fno-inline", fn);
2725       inlinable = false;
2726     }
2727
2728   /* Don't auto-inline anything that might not be bound within
2729      this unit of translation.  */
2730   else if (!DECL_DECLARED_INLINE_P (fn)
2731            && DECL_REPLACEABLE_P (fn))
2732     inlinable = false;
2733
2734   else if (!function_attribute_inlinable_p (fn))
2735     {
2736       if (do_warning)
2737         warning (OPT_Winline, "function %q+F can never be inlined because it "
2738                  "uses attributes conflicting with inlining", fn);
2739       inlinable = false;
2740     }
2741
2742   else if (inline_forbidden_p (fn))
2743     {
2744       /* See if we should warn about uninlinable functions.  Previously,
2745          some of these warnings would be issued while trying to expand
2746          the function inline, but that would cause multiple warnings
2747          about functions that would for example call alloca.  But since
2748          this a property of the function, just one warning is enough.
2749          As a bonus we can now give more details about the reason why a
2750          function is not inlinable.  */
2751       if (always_inline)
2752         sorry (inline_forbidden_reason, fn);
2753       else if (do_warning)
2754         warning (OPT_Winline, inline_forbidden_reason, fn);
2755
2756       inlinable = false;
2757     }
2758
2759   /* Squirrel away the result so that we don't have to check again.  */
2760   DECL_UNINLINABLE (fn) = !inlinable;
2761
2762   return inlinable;
2763 }
2764
2765 /* Estimate the cost of a memory move.  Use machine dependent
2766    word size and take possible memcpy call into account.  */
2767
2768 int
2769 estimate_move_cost (tree type)
2770 {
2771   HOST_WIDE_INT size;
2772
2773   gcc_assert (!VOID_TYPE_P (type));
2774
2775   size = int_size_in_bytes (type);
2776
2777   if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (!optimize_size))
2778     /* Cost of a memcpy call, 3 arguments and the call.  */
2779     return 4;
2780   else
2781     return ((size + MOVE_MAX_PIECES - 1) / MOVE_MAX_PIECES);
2782 }
2783
2784 /* Returns cost of operation CODE, according to WEIGHTS  */
2785
2786 static int
2787 estimate_operator_cost (enum tree_code code, eni_weights *weights,
2788                         tree op1 ATTRIBUTE_UNUSED, tree op2)
2789 {
2790   switch (code)
2791     {
2792     /* These are "free" conversions, or their presumed cost
2793        is folded into other operations.  */
2794     case RANGE_EXPR:
2795     CASE_CONVERT:
2796     case COMPLEX_EXPR:
2797     case PAREN_EXPR:
2798       return 0;
2799
2800     /* Assign cost of 1 to usual operations.
2801        ??? We may consider mapping RTL costs to this.  */
2802     case COND_EXPR:
2803     case VEC_COND_EXPR:
2804
2805     case PLUS_EXPR:
2806     case POINTER_PLUS_EXPR:
2807     case MINUS_EXPR:
2808     case MULT_EXPR:
2809
2810     case FIXED_CONVERT_EXPR:
2811     case FIX_TRUNC_EXPR:
2812
2813     case NEGATE_EXPR:
2814     case FLOAT_EXPR:
2815     case MIN_EXPR:
2816     case MAX_EXPR:
2817     case ABS_EXPR:
2818
2819     case LSHIFT_EXPR:
2820     case RSHIFT_EXPR:
2821     case LROTATE_EXPR:
2822     case RROTATE_EXPR:
2823     case VEC_LSHIFT_EXPR:
2824     case VEC_RSHIFT_EXPR:
2825
2826     case BIT_IOR_EXPR:
2827     case BIT_XOR_EXPR:
2828     case BIT_AND_EXPR:
2829     case BIT_NOT_EXPR:
2830
2831     case TRUTH_ANDIF_EXPR:
2832     case TRUTH_ORIF_EXPR:
2833     case TRUTH_AND_EXPR:
2834     case TRUTH_OR_EXPR:
2835     case TRUTH_XOR_EXPR:
2836     case TRUTH_NOT_EXPR:
2837
2838     case LT_EXPR:
2839     case LE_EXPR:
2840     case GT_EXPR:
2841     case GE_EXPR:
2842     case EQ_EXPR:
2843     case NE_EXPR:
2844     case ORDERED_EXPR:
2845     case UNORDERED_EXPR:
2846
2847     case UNLT_EXPR:
2848     case UNLE_EXPR:
2849     case UNGT_EXPR:
2850     case UNGE_EXPR:
2851     case UNEQ_EXPR:
2852     case LTGT_EXPR:
2853
2854     case CONJ_EXPR:
2855
2856     case PREDECREMENT_EXPR:
2857     case PREINCREMENT_EXPR:
2858     case POSTDECREMENT_EXPR:
2859     case POSTINCREMENT_EXPR:
2860
2861     case REALIGN_LOAD_EXPR:
2862
2863     case REDUC_MAX_EXPR:
2864     case REDUC_MIN_EXPR:
2865     case REDUC_PLUS_EXPR:
2866     case WIDEN_SUM_EXPR:
2867     case WIDEN_MULT_EXPR:
2868     case DOT_PROD_EXPR:
2869
2870     case VEC_WIDEN_MULT_HI_EXPR:
2871     case VEC_WIDEN_MULT_LO_EXPR:
2872     case VEC_UNPACK_HI_EXPR:
2873     case VEC_UNPACK_LO_EXPR:
2874     case VEC_UNPACK_FLOAT_HI_EXPR:
2875     case VEC_UNPACK_FLOAT_LO_EXPR:
2876     case VEC_PACK_TRUNC_EXPR:
2877     case VEC_PACK_SAT_EXPR:
2878     case VEC_PACK_FIX_TRUNC_EXPR:
2879     case VEC_EXTRACT_EVEN_EXPR:
2880     case VEC_EXTRACT_ODD_EXPR:
2881     case VEC_INTERLEAVE_HIGH_EXPR:
2882     case VEC_INTERLEAVE_LOW_EXPR:
2883
2884       return 1;
2885
2886     /* Few special cases of expensive operations.  This is useful
2887        to avoid inlining on functions having too many of these.  */
2888     case TRUNC_DIV_EXPR:
2889     case CEIL_DIV_EXPR:
2890     case FLOOR_DIV_EXPR:
2891     case ROUND_DIV_EXPR:
2892     case EXACT_DIV_EXPR:
2893     case TRUNC_MOD_EXPR:
2894     case CEIL_MOD_EXPR:
2895     case FLOOR_MOD_EXPR:
2896     case ROUND_MOD_EXPR:
2897     case RDIV_EXPR:
2898       if (TREE_CODE (op2) != INTEGER_CST)
2899         return weights->div_mod_cost;
2900       return 1;
2901
2902     default:
2903       /* We expect a copy assignment with no operator.  */
2904       gcc_assert (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS);
2905       return 0;
2906     }
2907 }
2908
2909
2910 /* Estimate number of instructions that will be created by expanding
2911    the statements in the statement sequence STMTS.
2912    WEIGHTS contains weights attributed to various constructs.  */
2913
2914 static
2915 int estimate_num_insns_seq (gimple_seq stmts, eni_weights *weights)
2916 {
2917   int cost;
2918   gimple_stmt_iterator gsi;
2919
2920   cost = 0;
2921   for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
2922     cost += estimate_num_insns (gsi_stmt (gsi), weights);
2923
2924   return cost;
2925 }
2926
2927
2928 /* Estimate number of instructions that will be created by expanding STMT.
2929    WEIGHTS contains weights attributed to various constructs.  */
2930
2931 int
2932 estimate_num_insns (gimple stmt, eni_weights *weights)
2933 {
2934   unsigned cost, i;
2935   enum gimple_code code = gimple_code (stmt);
2936   tree lhs;
2937   tree rhs;
2938
2939   switch (code)
2940     {
2941     case GIMPLE_ASSIGN:
2942       /* Try to estimate the cost of assignments.  We have three cases to
2943          deal with:
2944          1) Simple assignments to registers;
2945          2) Stores to things that must live in memory.  This includes
2946             "normal" stores to scalars, but also assignments of large
2947             structures, or constructors of big arrays;
2948
2949          Let us look at the first two cases, assuming we have "a = b + C":
2950          <GIMPLE_ASSIGN <var_decl "a">
2951                 <plus_expr <var_decl "b"> <constant C>>
2952          If "a" is a GIMPLE register, the assignment to it is free on almost
2953          any target, because "a" usually ends up in a real register.  Hence
2954          the only cost of this expression comes from the PLUS_EXPR, and we
2955          can ignore the GIMPLE_ASSIGN.
2956          If "a" is not a GIMPLE register, the assignment to "a" will most
2957          likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost
2958          of moving something into "a", which we compute using the function
2959          estimate_move_cost.  */
2960       lhs = gimple_assign_lhs (stmt);
2961       rhs = gimple_assign_rhs1 (stmt);
2962
2963       /* EH magic stuff is most probably going to be optimized out.
2964          We rarely really need to save EH info for unwinding
2965          nested exceptions.  */
2966       if (TREE_CODE (lhs) == FILTER_EXPR
2967           || TREE_CODE (lhs) == EXC_PTR_EXPR
2968           || TREE_CODE (rhs) == FILTER_EXPR
2969           || TREE_CODE (rhs) == EXC_PTR_EXPR)
2970         return 0;
2971       if (is_gimple_reg (lhs))
2972         cost = 0;
2973       else
2974         cost = estimate_move_cost (TREE_TYPE (lhs));
2975
2976       if (!is_gimple_reg (rhs) && !is_gimple_min_invariant (rhs))
2977         cost += estimate_move_cost (TREE_TYPE (rhs));
2978
2979       cost += estimate_operator_cost (gimple_assign_rhs_code (stmt), weights,
2980                                       gimple_assign_rhs1 (stmt),
2981                                       get_gimple_rhs_class (gimple_assign_rhs_code (stmt))
2982                                       == GIMPLE_BINARY_RHS
2983                                       ? gimple_assign_rhs2 (stmt) : NULL);
2984       break;
2985
2986     case GIMPLE_COND:
2987       cost = 1 + estimate_operator_cost (gimple_cond_code (stmt), weights,
2988                                          gimple_op (stmt, 0),
2989                                          gimple_op (stmt, 1));
2990       break;
2991
2992     case GIMPLE_SWITCH:
2993       /* Take into account cost of the switch + guess 2 conditional jumps for
2994          each case label.
2995
2996          TODO: once the switch expansion logic is sufficiently separated, we can
2997          do better job on estimating cost of the switch.  */
2998       if (weights->time_based)
2999         cost = floor_log2 (gimple_switch_num_labels (stmt)) * 2;
3000       else
3001         cost = gimple_switch_num_labels (stmt) * 2;
3002       break;
3003
3004     case GIMPLE_CALL:
3005       {
3006         tree decl = gimple_call_fndecl (stmt);
3007         tree addr = gimple_call_fn (stmt);
3008         tree funtype = TREE_TYPE (addr);
3009
3010         if (POINTER_TYPE_P (funtype))
3011           funtype = TREE_TYPE (funtype);
3012
3013         if (decl && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_MD)
3014           cost = weights->target_builtin_call_cost;
3015         else
3016           cost = weights->call_cost;
3017
3018         if (decl && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
3019           switch (DECL_FUNCTION_CODE (decl))
3020             {
3021             case BUILT_IN_CONSTANT_P:
3022               return 0;
3023             case BUILT_IN_EXPECT:
3024               return 0;
3025
3026             /* Prefetch instruction is not expensive.  */
3027             case BUILT_IN_PREFETCH:
3028               cost = weights->target_builtin_call_cost;
3029               break;
3030
3031             default:
3032               break;
3033             }
3034
3035         if (decl)
3036           funtype = TREE_TYPE (decl);
3037
3038         if (!VOID_TYPE_P (TREE_TYPE (funtype)))
3039           cost += estimate_move_cost (TREE_TYPE (funtype));
3040         /* Our cost must be kept in sync with
3041            cgraph_estimate_size_after_inlining that does use function
3042            declaration to figure out the arguments.  */
3043         if (decl && DECL_ARGUMENTS (decl))
3044           {
3045             tree arg;
3046             for (arg = DECL_ARGUMENTS (decl); arg; arg = TREE_CHAIN (arg))
3047               if (!VOID_TYPE_P (TREE_TYPE (arg)))
3048                 cost += estimate_move_cost (TREE_TYPE (arg));
3049           }
3050         else if (funtype && prototype_p (funtype))
3051           {
3052             tree t;
3053             for (t = TYPE_ARG_TYPES (funtype); t && t != void_list_node;
3054                  t = TREE_CHAIN (t))
3055               if (!VOID_TYPE_P (TREE_VALUE (t)))
3056                 cost += estimate_move_cost (TREE_VALUE (t));
3057           }
3058         else
3059           {
3060             for (i = 0; i < gimple_call_num_args (stmt); i++)
3061               {
3062                 tree arg = gimple_call_arg (stmt, i);
3063                 if (!VOID_TYPE_P (TREE_TYPE (arg)))
3064                   cost += estimate_move_cost (TREE_TYPE (arg));
3065               }
3066           }
3067
3068         break;
3069       }
3070
3071     case GIMPLE_GOTO:
3072     case GIMPLE_LABEL:
3073     case GIMPLE_NOP:
3074     case GIMPLE_PHI:
3075     case GIMPLE_RETURN:
3076     case GIMPLE_PREDICT:
3077       return 0;
3078
3079     case GIMPLE_ASM:
3080     case GIMPLE_RESX:
3081       return 1;
3082
3083     case GIMPLE_BIND:
3084       return estimate_num_insns_seq (gimple_bind_body (stmt), weights);
3085
3086     case GIMPLE_EH_FILTER:
3087       return estimate_num_insns_seq (gimple_eh_filter_failure (stmt), weights);
3088
3089     case GIMPLE_CATCH:
3090       return estimate_num_insns_seq (gimple_catch_handler (stmt), weights);
3091
3092     case GIMPLE_TRY:
3093       return (estimate_num_insns_seq (gimple_try_eval (stmt), weights)
3094               + estimate_num_insns_seq (gimple_try_cleanup (stmt), weights));
3095
3096     /* OpenMP directives are generally very expensive.  */
3097
3098     case GIMPLE_OMP_RETURN:
3099     case GIMPLE_OMP_SECTIONS_SWITCH:
3100     case GIMPLE_OMP_ATOMIC_STORE:
3101     case GIMPLE_OMP_CONTINUE:
3102       /* ...except these, which are cheap.  */
3103       return 0;
3104
3105     case GIMPLE_OMP_ATOMIC_LOAD:
3106       return weights->omp_cost;
3107
3108     case GIMPLE_OMP_FOR:
3109       return (weights->omp_cost
3110               + estimate_num_insns_seq (gimple_omp_body (stmt), weights)
3111               + estimate_num_insns_seq (gimple_omp_for_pre_body (stmt), weights));
3112
3113     case GIMPLE_OMP_PARALLEL:
3114     case GIMPLE_OMP_TASK:
3115     case GIMPLE_OMP_CRITICAL:
3116     case GIMPLE_OMP_MASTER:
3117     case GIMPLE_OMP_ORDERED:
3118     case GIMPLE_OMP_SECTION:
3119     case GIMPLE_OMP_SECTIONS:
3120     case GIMPLE_OMP_SINGLE:
3121       return (weights->omp_cost
3122               + estimate_num_insns_seq (gimple_omp_body (stmt), weights));
3123
3124     default:
3125       gcc_unreachable ();
3126     }
3127
3128   return cost;
3129 }
3130
3131 /* Estimate number of instructions that will be created by expanding
3132    function FNDECL.  WEIGHTS contains weights attributed to various
3133    constructs.  */
3134
3135 int
3136 estimate_num_insns_fn (tree fndecl, eni_weights *weights)
3137 {
3138   struct function *my_function = DECL_STRUCT_FUNCTION (fndecl);
3139   gimple_stmt_iterator bsi;
3140   basic_block bb;
3141   int n = 0;
3142
3143   gcc_assert (my_function && my_function->cfg);
3144   FOR_EACH_BB_FN (bb, my_function)
3145     {
3146       for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
3147         n += estimate_num_insns (gsi_stmt (bsi), weights);
3148     }
3149
3150   return n;
3151 }
3152
3153
3154 /* Initializes weights used by estimate_num_insns.  */
3155
3156 void
3157 init_inline_once (void)
3158 {
3159   eni_size_weights.call_cost = 1;
3160   eni_size_weights.target_builtin_call_cost = 1;
3161   eni_size_weights.div_mod_cost = 1;
3162   eni_size_weights.omp_cost = 40;
3163   eni_size_weights.time_based = false;
3164
3165   /* Estimating time for call is difficult, since we have no idea what the
3166      called function does.  In the current uses of eni_time_weights,
3167      underestimating the cost does less harm than overestimating it, so
3168      we choose a rather small value here.  */
3169   eni_time_weights.call_cost = 10;
3170   eni_time_weights.target_builtin_call_cost = 10;
3171   eni_time_weights.div_mod_cost = 10;
3172   eni_time_weights.omp_cost = 40;
3173   eni_time_weights.time_based = true;
3174 }
3175
3176 /* Estimate the number of instructions in a gimple_seq. */
3177
3178 int
3179 count_insns_seq (gimple_seq seq, eni_weights *weights)
3180 {
3181   gimple_stmt_iterator gsi;
3182   int n = 0;
3183   for (gsi = gsi_start (seq); !gsi_end_p (gsi); gsi_next (&gsi))
3184     n += estimate_num_insns (gsi_stmt (gsi), weights);
3185
3186   return n;
3187 }
3188
3189
3190 /* Install new lexical TREE_BLOCK underneath 'current_block'.  */
3191
3192 static void
3193 prepend_lexical_block (tree current_block, tree new_block)
3194 {
3195   BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block);
3196   BLOCK_SUBBLOCKS (current_block) = new_block;
3197   BLOCK_SUPERCONTEXT (new_block) = current_block;
3198 }
3199
3200 /* Fetch callee declaration from the call graph edge going from NODE and
3201    associated with STMR call statement.  Return NULL_TREE if not found.  */
3202 static tree
3203 get_indirect_callee_fndecl (struct cgraph_node *node, gimple stmt)
3204 {
3205   struct cgraph_edge *cs;
3206
3207   cs = cgraph_edge (node, stmt);
3208   if (cs)
3209     return cs->callee->decl;
3210
3211   return NULL_TREE;
3212 }
3213
3214 /* If STMT is a GIMPLE_CALL, replace it with its inline expansion.  */
3215
3216 static bool
3217 expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id)
3218 {
3219   tree retvar, use_retvar;
3220   tree fn;
3221   struct pointer_map_t *st;
3222   tree return_slot;
3223   tree modify_dest;
3224   location_t saved_location;
3225   struct cgraph_edge *cg_edge;
3226   cgraph_inline_failed_t reason;
3227   basic_block return_block;
3228   edge e;
3229   gimple_stmt_iterator gsi, stmt_gsi;
3230   bool successfully_inlined = FALSE;
3231   bool purge_dead_abnormal_edges;
3232   tree t_step;
3233   tree var;
3234
3235   /* Set input_location here so we get the right instantiation context
3236      if we call instantiate_decl from inlinable_function_p.  */
3237   saved_location = input_location;
3238   if (gimple_has_location (stmt))
3239     input_location = gimple_location (stmt);
3240
3241   /* From here on, we're only interested in CALL_EXPRs.  */
3242   if (gimple_code (stmt) != GIMPLE_CALL)
3243     goto egress;
3244
3245   /* First, see if we can figure out what function is being called.
3246      If we cannot, then there is no hope of inlining the function.  */
3247   fn = gimple_call_fndecl (stmt);
3248   if (!fn)
3249     {
3250       fn = get_indirect_callee_fndecl (id->dst_node, stmt);
3251       if (!fn)
3252         goto egress;
3253     }
3254
3255   /* Turn forward declarations into real ones.  */
3256   fn = cgraph_node (fn)->decl;
3257
3258   /* If FN is a declaration of a function in a nested scope that was
3259      globally declared inline, we don't set its DECL_INITIAL.
3260      However, we can't blindly follow DECL_ABSTRACT_ORIGIN because the
3261      C++ front-end uses it for cdtors to refer to their internal
3262      declarations, that are not real functions.  Fortunately those
3263      don't have trees to be saved, so we can tell by checking their
3264      gimple_body.  */
3265   if (!DECL_INITIAL (fn)
3266       && DECL_ABSTRACT_ORIGIN (fn)
3267       && gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn)))
3268     fn = DECL_ABSTRACT_ORIGIN (fn);
3269
3270   /* Objective C and fortran still calls tree_rest_of_compilation directly.
3271      Kill this check once this is fixed.  */
3272   if (!id->dst_node->analyzed)
3273     goto egress;
3274
3275   cg_edge = cgraph_edge (id->dst_node, stmt);
3276
3277   /* Don't try to inline functions that are not well-suited to
3278      inlining.  */
3279   if (!cgraph_inline_p (cg_edge, &reason))
3280     {
3281       /* If this call was originally indirect, we do not want to emit any
3282          inlining related warnings or sorry messages because there are no
3283          guarantees regarding those.  */
3284       if (cg_edge->indirect_call)
3285         goto egress;
3286
3287       if (lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn))
3288           /* Avoid warnings during early inline pass. */
3289           && cgraph_global_info_ready)
3290         {
3291           sorry ("inlining failed in call to %q+F: %s", fn,
3292                  cgraph_inline_failed_string (reason));
3293           sorry ("called from here");
3294         }
3295       else if (warn_inline && DECL_DECLARED_INLINE_P (fn)
3296                && !DECL_IN_SYSTEM_HEADER (fn)
3297                && reason != CIF_UNSPECIFIED
3298                && !lookup_attribute ("noinline", DECL_ATTRIBUTES (fn))
3299                /* Avoid warnings during early inline pass. */
3300                && cgraph_global_info_ready)
3301         {
3302           warning (OPT_Winline, "inlining failed in call to %q+F: %s",
3303                    fn, cgraph_inline_failed_string (reason));
3304           warning (OPT_Winline, "called from here");
3305         }
3306       goto egress;
3307     }
3308   fn = cg_edge->callee->decl;
3309
3310 #ifdef ENABLE_CHECKING
3311   if (cg_edge->callee->decl != id->dst_node->decl)
3312     verify_cgraph_node (cg_edge->callee);
3313 #endif
3314
3315   /* We will be inlining this callee.  */
3316   id->eh_region = lookup_stmt_eh_region (stmt);
3317
3318   /* Split the block holding the GIMPLE_CALL.  */
3319   e = split_block (bb, stmt);
3320   bb = e->src;
3321   return_block = e->dest;
3322   remove_edge (e);
3323
3324   /* split_block splits after the statement; work around this by
3325      moving the call into the second block manually.  Not pretty,
3326      but seems easier than doing the CFG manipulation by hand
3327      when the GIMPLE_CALL is in the last statement of BB.  */
3328   stmt_gsi = gsi_last_bb (bb);
3329   gsi_remove (&stmt_gsi, false);
3330
3331   /* If the GIMPLE_CALL was in the last statement of BB, it may have
3332      been the source of abnormal edges.  In this case, schedule
3333      the removal of dead abnormal edges.  */
3334   gsi = gsi_start_bb (return_block);
3335   if (gsi_end_p (gsi))
3336     {
3337       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
3338       purge_dead_abnormal_edges = true;
3339     }
3340   else
3341     {
3342       gsi_insert_before (&gsi, stmt, GSI_NEW_STMT);
3343       purge_dead_abnormal_edges = false;
3344     }
3345
3346   stmt_gsi = gsi_start_bb (return_block);
3347
3348   /* Build a block containing code to initialize the arguments, the
3349      actual inline expansion of the body, and a label for the return
3350      statements within the function to jump to.  The type of the
3351      statement expression is the return type of the function call.  */
3352   id->block = make_node (BLOCK);
3353   BLOCK_ABSTRACT_ORIGIN (id->block) = fn;
3354   BLOCK_SOURCE_LOCATION (id->block) = input_location;
3355   prepend_lexical_block (gimple_block (stmt), id->block);
3356
3357   /* Local declarations will be replaced by their equivalents in this
3358      map.  */
3359   st = id->decl_map;
3360   id->decl_map = pointer_map_create ();
3361
3362   /* Record the function we are about to inline.  */
3363   id->src_fn = fn;
3364   id->src_node = cg_edge->callee;
3365   id->src_cfun = DECL_STRUCT_FUNCTION (fn);
3366   id->gimple_call = stmt;
3367
3368   gcc_assert (!id->src_cfun->after_inlining);
3369
3370   id->entry_bb = bb;
3371   if (lookup_attribute ("cold", DECL_ATTRIBUTES (fn)))
3372     {
3373       gimple_stmt_iterator si = gsi_last_bb (bb);
3374       gsi_insert_after (&si, gimple_build_predict (PRED_COLD_FUNCTION,
3375                                                    NOT_TAKEN),
3376                         GSI_NEW_STMT);
3377     }
3378   initialize_inlined_parameters (id, stmt, fn, bb);
3379
3380   if (DECL_INITIAL (fn))
3381     prepend_lexical_block (id->block, remap_blocks (DECL_INITIAL (fn), id));
3382
3383   /* Return statements in the function body will be replaced by jumps
3384      to the RET_LABEL.  */
3385   gcc_assert (DECL_INITIAL (fn));
3386   gcc_assert (TREE_CODE (DECL_INITIAL (fn)) == BLOCK);
3387
3388   /* Find the LHS to which the result of this call is assigned.  */
3389   return_slot = NULL;
3390   if (gimple_call_lhs (stmt))
3391     {
3392       modify_dest = gimple_call_lhs (stmt);
3393
3394       /* The function which we are inlining might not return a value,
3395          in which case we should issue a warning that the function
3396          does not return a value.  In that case the optimizers will
3397          see that the variable to which the value is assigned was not
3398          initialized.  We do not want to issue a warning about that
3399          uninitialized variable.  */
3400       if (DECL_P (modify_dest))
3401         TREE_NO_WARNING (modify_dest) = 1;
3402
3403       if (gimple_call_return_slot_opt_p (stmt))
3404         {
3405           return_slot = modify_dest;
3406           modify_dest = NULL;
3407         }
3408     }
3409   else
3410     modify_dest = NULL;
3411
3412   /* If we are inlining a call to the C++ operator new, we don't want
3413      to use type based alias analysis on the return value.  Otherwise
3414      we may get confused if the compiler sees that the inlined new
3415      function returns a pointer which was just deleted.  See bug
3416      33407.  */
3417   if (DECL_IS_OPERATOR_NEW (fn))
3418     {
3419       return_slot = NULL;
3420       modify_dest = NULL;
3421     }
3422
3423   /* Declare the return variable for the function.  */
3424   retvar = declare_return_variable (id, return_slot, modify_dest, &use_retvar);
3425
3426   /* Add local vars in this inlined callee to caller.  */
3427   t_step = id->src_cfun->local_decls;
3428   for (; t_step; t_step = TREE_CHAIN (t_step))
3429     {
3430       var = TREE_VALUE (t_step);
3431       if (TREE_STATIC (var) && !TREE_ASM_WRITTEN (var))
3432         {
3433           if (var_ann (var) && add_referenced_var (var))
3434             cfun->local_decls = tree_cons (NULL_TREE, var,
3435                                            cfun->local_decls);
3436         }
3437       else if (!can_be_nonlocal (var, id))
3438         cfun->local_decls = tree_cons (NULL_TREE, remap_decl (var, id),
3439                                        cfun->local_decls);
3440     }
3441
3442   /* This is it.  Duplicate the callee body.  Assume callee is
3443      pre-gimplified.  Note that we must not alter the caller
3444      function in any way before this point, as this CALL_EXPR may be
3445      a self-referential call; if we're calling ourselves, we need to
3446      duplicate our body before altering anything.  */
3447   copy_body (id, bb->count, bb->frequency, bb, return_block);
3448
3449   /* Reset the escaped and callused solutions.  */
3450   if (cfun->gimple_df)
3451     {
3452       pt_solution_reset (&cfun->gimple_df->escaped);
3453       pt_solution_reset (&cfun->gimple_df->callused);
3454     }
3455
3456   /* Clean up.  */
3457   pointer_map_destroy (id->decl_map);
3458   id->decl_map = st;
3459
3460   /* Unlink the calls virtual operands before replacing it.  */
3461   unlink_stmt_vdef (stmt);
3462
3463   /* If the inlined function returns a result that we care about,
3464      substitute the GIMPLE_CALL with an assignment of the return
3465      variable to the LHS of the call.  That is, if STMT was
3466      'a = foo (...)', substitute the call with 'a = USE_RETVAR'.  */
3467   if (use_retvar && gimple_call_lhs (stmt))
3468     {
3469       gimple old_stmt = stmt;
3470       stmt = gimple_build_assign (gimple_call_lhs (stmt), use_retvar);
3471       gsi_replace (&stmt_gsi, stmt, false);
3472       if (gimple_in_ssa_p (cfun))
3473         mark_symbols_for_renaming (stmt);
3474       maybe_clean_or_replace_eh_stmt (old_stmt, stmt);
3475     }
3476   else
3477     {
3478       /* Handle the case of inlining a function with no return
3479          statement, which causes the return value to become undefined.  */
3480       if (gimple_call_lhs (stmt)
3481           && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
3482         {
3483           tree name = gimple_call_lhs (stmt);
3484           tree var = SSA_NAME_VAR (name);
3485           tree def = gimple_default_def (cfun, var);
3486
3487           if (def)
3488             {
3489               /* If the variable is used undefined, make this name
3490                  undefined via a move.  */
3491               stmt = gimple_build_assign (gimple_call_lhs (stmt), def);
3492               gsi_replace (&stmt_gsi, stmt, true);
3493             }
3494           else
3495             {
3496               /* Otherwise make this variable undefined.  */
3497               gsi_remove (&stmt_gsi, true);
3498               set_default_def (var, name);
3499               SSA_NAME_DEF_STMT (name) = gimple_build_nop ();
3500             }
3501         }
3502       else
3503         gsi_remove (&stmt_gsi, true);
3504     }
3505
3506   if (purge_dead_abnormal_edges)
3507     gimple_purge_dead_abnormal_call_edges (return_block);
3508
3509   /* If the value of the new expression is ignored, that's OK.  We
3510      don't warn about this for CALL_EXPRs, so we shouldn't warn about
3511      the equivalent inlined version either.  */
3512   if (is_gimple_assign (stmt))
3513     {
3514       gcc_assert (gimple_assign_single_p (stmt)
3515                   || CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)));
3516       TREE_USED (gimple_assign_rhs1 (stmt)) = 1;
3517     }
3518
3519   /* Output the inlining info for this abstract function, since it has been
3520      inlined.  If we don't do this now, we can lose the information about the
3521      variables in the function when the blocks get blown away as soon as we
3522      remove the cgraph node.  */
3523   (*debug_hooks->outlining_inline_function) (cg_edge->callee->decl);
3524
3525   /* Update callgraph if needed.  */
3526   cgraph_remove_node (cg_edge->callee);
3527
3528   id->block = NULL_TREE;
3529   successfully_inlined = TRUE;
3530
3531  egress:
3532   input_location = saved_location;
3533   return successfully_inlined;
3534 }
3535
3536 /* Expand call statements reachable from STMT_P.
3537    We can only have CALL_EXPRs as the "toplevel" tree code or nested
3538    in a MODIFY_EXPR.  See tree-gimple.c:get_call_expr_in().  We can
3539    unfortunately not use that function here because we need a pointer
3540    to the CALL_EXPR, not the tree itself.  */
3541
3542 static bool
3543 gimple_expand_calls_inline (basic_block bb, copy_body_data *id)
3544 {
3545   gimple_stmt_iterator gsi;
3546
3547   for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3548     {
3549       gimple stmt = gsi_stmt (gsi);
3550
3551       if (is_gimple_call (stmt)
3552           && expand_call_inline (bb, stmt, id))
3553         return true;
3554     }
3555
3556   return false;
3557 }
3558
3559
3560 /* Walk all basic blocks created after FIRST and try to fold every statement
3561    in the STATEMENTS pointer set.  */
3562
3563 static void
3564 fold_marked_statements (int first, struct pointer_set_t *statements)
3565 {
3566   for (; first < n_basic_blocks; first++)
3567     if (BASIC_BLOCK (first))
3568       {
3569         gimple_stmt_iterator gsi;
3570
3571         for (gsi = gsi_start_bb (BASIC_BLOCK (first));
3572              !gsi_end_p (gsi);
3573              gsi_next (&gsi))
3574           if (pointer_set_contains (statements, gsi_stmt (gsi)))
3575             {
3576               gimple old_stmt = gsi_stmt (gsi);
3577               tree old_decl = is_gimple_call (old_stmt) ? gimple_call_fndecl (old_stmt) : 0;
3578
3579               if (fold_stmt (&gsi))
3580                 {
3581                   /* Re-read the statement from GSI as fold_stmt() may
3582                      have changed it.  */
3583                   gimple new_stmt = gsi_stmt (gsi);
3584                   update_stmt (new_stmt);
3585
3586                   if (is_gimple_call (old_stmt)
3587                       || is_gimple_call (new_stmt))
3588                     cgraph_update_edges_for_call_stmt (old_stmt, old_decl, new_stmt);
3589
3590                   if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt))
3591                     gimple_purge_dead_eh_edges (BASIC_BLOCK (first));
3592                 }
3593             }
3594       }
3595 }
3596
3597 /* Return true if BB has at least one abnormal outgoing edge.  */
3598
3599 static inline bool
3600 has_abnormal_outgoing_edge_p (basic_block bb)
3601 {
3602   edge e;
3603   edge_iterator ei;
3604
3605   FOR_EACH_EDGE (e, ei, bb->succs)
3606     if (e->flags & EDGE_ABNORMAL)
3607       return true;
3608
3609   return false;
3610 }
3611
3612 /* Expand calls to inline functions in the body of FN.  */
3613
3614 unsigned int
3615 optimize_inline_calls (tree fn)
3616 {
3617   copy_body_data id;
3618   tree prev_fn;
3619   basic_block bb;
3620   int last = n_basic_blocks;
3621   struct gimplify_ctx gctx;
3622
3623   /* There is no point in performing inlining if errors have already
3624      occurred -- and we might crash if we try to inline invalid
3625      code.  */
3626   if (errorcount || sorrycount)
3627     return 0;
3628
3629   /* Clear out ID.  */
3630   memset (&id, 0, sizeof (id));
3631
3632   id.src_node = id.dst_node = cgraph_node (fn);
3633   id.dst_fn = fn;
3634   /* Or any functions that aren't finished yet.  */
3635   prev_fn = NULL_TREE;
3636   if (current_function_decl)
3637     {
3638       id.dst_fn = current_function_decl;
3639       prev_fn = current_function_decl;
3640     }
3641
3642   id.copy_decl = copy_decl_maybe_to_var;
3643   id.transform_call_graph_edges = CB_CGE_DUPLICATE;
3644   id.transform_new_cfg = false;
3645   id.transform_return_to_modify = true;
3646   id.transform_lang_insert_block = NULL;
3647   id.statements_to_fold = pointer_set_create ();
3648
3649   push_gimplify_context (&gctx);
3650
3651   /* We make no attempts to keep dominance info up-to-date.  */
3652   free_dominance_info (CDI_DOMINATORS);
3653   free_dominance_info (CDI_POST_DOMINATORS);
3654
3655   /* Register specific gimple functions.  */
3656   gimple_register_cfg_hooks ();
3657
3658   /* Reach the trees by walking over the CFG, and note the
3659      enclosing basic-blocks in the call edges.  */
3660   /* We walk the blocks going forward, because inlined function bodies
3661      will split id->current_basic_block, and the new blocks will
3662      follow it; we'll trudge through them, processing their CALL_EXPRs
3663      along the way.  */
3664   FOR_EACH_BB (bb)
3665     gimple_expand_calls_inline (bb, &id);
3666
3667   pop_gimplify_context (NULL);
3668
3669 #ifdef ENABLE_CHECKING
3670     {
3671       struct cgraph_edge *e;
3672
3673       verify_cgraph_node (id.dst_node);
3674
3675       /* Double check that we inlined everything we are supposed to inline.  */
3676       for (e = id.dst_node->callees; e; e = e->next_callee)
3677         gcc_assert (e->inline_failed);
3678     }
3679 #endif
3680
3681   /* Fold the statements before compacting/renumbering the basic blocks.  */
3682   fold_marked_statements (last, id.statements_to_fold);
3683   pointer_set_destroy (id.statements_to_fold);
3684
3685   /* Renumber the (code) basic_blocks consecutively.  */
3686   compact_blocks ();
3687   /* Renumber the lexical scoping (non-code) blocks consecutively.  */
3688   number_blocks (fn);
3689
3690   fold_cond_expr_cond ();
3691   delete_unreachable_blocks_update_callgraph (&id);
3692 #ifdef ENABLE_CHECKING
3693   verify_cgraph_node (id.dst_node);
3694 #endif
3695
3696   /* It would be nice to check SSA/CFG/statement consistency here, but it is
3697      not possible yet - the IPA passes might make various functions to not
3698      throw and they don't care to proactively update local EH info.  This is
3699      done later in fixup_cfg pass that also execute the verification.  */
3700   return (TODO_update_ssa
3701           | TODO_cleanup_cfg
3702           | (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0)
3703           | (profile_status != PROFILE_ABSENT ? TODO_rebuild_frequencies : 0));
3704 }
3705
3706 /* Passed to walk_tree.  Copies the node pointed to, if appropriate.  */
3707
3708 tree
3709 copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
3710 {
3711   enum tree_code code = TREE_CODE (*tp);
3712   enum tree_code_class cl = TREE_CODE_CLASS (code);
3713
3714   /* We make copies of most nodes.  */
3715   if (IS_EXPR_CODE_CLASS (cl)
3716       || code == TREE_LIST
3717       || code == TREE_VEC
3718       || code == TYPE_DECL
3719       || code == OMP_CLAUSE)
3720     {
3721       /* Because the chain gets clobbered when we make a copy, we save it
3722          here.  */
3723       tree chain = NULL_TREE, new_tree;
3724
3725       chain = TREE_CHAIN (*tp);
3726
3727       /* Copy the node.  */
3728       new_tree = copy_node (*tp);
3729
3730       /* Propagate mudflap marked-ness.  */
3731       if (flag_mudflap && mf_marked_p (*tp))
3732         mf_mark (new_tree);
3733
3734       *tp = new_tree;
3735
3736       /* Now, restore the chain, if appropriate.  That will cause
3737          walk_tree to walk into the chain as well.  */
3738       if (code == PARM_DECL
3739           || code == TREE_LIST
3740           || code == OMP_CLAUSE)
3741         TREE_CHAIN (*tp) = chain;
3742
3743       /* For now, we don't update BLOCKs when we make copies.  So, we
3744          have to nullify all BIND_EXPRs.  */
3745       if (TREE_CODE (*tp) == BIND_EXPR)
3746         BIND_EXPR_BLOCK (*tp) = NULL_TREE;
3747     }
3748   else if (code == CONSTRUCTOR)
3749     {
3750       /* CONSTRUCTOR nodes need special handling because
3751          we need to duplicate the vector of elements.  */
3752       tree new_tree;
3753
3754       new_tree = copy_node (*tp);
3755
3756       /* Propagate mudflap marked-ness.  */
3757       if (flag_mudflap && mf_marked_p (*tp))
3758         mf_mark (new_tree);
3759
3760       CONSTRUCTOR_ELTS (new_tree) = VEC_copy (constructor_elt, gc,
3761                                          CONSTRUCTOR_ELTS (*tp));
3762       *tp = new_tree;
3763     }
3764   else if (TREE_CODE_CLASS (code) == tcc_type)
3765     *walk_subtrees = 0;
3766   else if (TREE_CODE_CLASS (code) == tcc_declaration)
3767     *walk_subtrees = 0;
3768   else if (TREE_CODE_CLASS (code) == tcc_constant)
3769     *walk_subtrees = 0;
3770   else
3771     gcc_assert (code != STATEMENT_LIST);
3772   return NULL_TREE;
3773 }
3774
3775 /* The SAVE_EXPR pointed to by TP is being copied.  If ST contains
3776    information indicating to what new SAVE_EXPR this one should be mapped,
3777    use that one.  Otherwise, create a new node and enter it in ST.  FN is
3778    the function into which the copy will be placed.  */
3779
3780 static void
3781 remap_save_expr (tree *tp, void *st_, int *walk_subtrees)
3782 {
3783   struct pointer_map_t *st = (struct pointer_map_t *) st_;
3784   tree *n;
3785   tree t;
3786
3787   /* See if we already encountered this SAVE_EXPR.  */
3788   n = (tree *) pointer_map_contains (st, *tp);
3789
3790   /* If we didn't already remap this SAVE_EXPR, do so now.  */
3791   if (!n)
3792     {
3793       t = copy_node (*tp);
3794
3795       /* Remember this SAVE_EXPR.  */
3796       *pointer_map_insert (st, *tp) = t;
3797       /* Make sure we don't remap an already-remapped SAVE_EXPR.  */
3798       *pointer_map_insert (st, t) = t;
3799     }
3800   else
3801     {
3802       /* We've already walked into this SAVE_EXPR; don't do it again.  */
3803       *walk_subtrees = 0;
3804       t = *n;
3805     }
3806
3807   /* Replace this SAVE_EXPR with the copy.  */
3808   *tp = t;
3809 }
3810
3811 /* Called via walk_tree.  If *TP points to a DECL_STMT for a local label,
3812    copies the declaration and enters it in the splay_tree in DATA (which is
3813    really an `copy_body_data *').  */
3814
3815 static tree
3816 mark_local_for_remap_r (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED,
3817                         void *data)
3818 {
3819   copy_body_data *id = (copy_body_data *) data;
3820
3821   /* Don't walk into types.  */
3822   if (TYPE_P (*tp))
3823     *walk_subtrees = 0;
3824
3825   else if (TREE_CODE (*tp) == LABEL_EXPR)
3826     {
3827       tree decl = TREE_OPERAND (*tp, 0);
3828
3829       /* Copy the decl and remember the copy.  */
3830       insert_decl_map (id, decl, id->copy_decl (decl, id));
3831     }
3832
3833   return NULL_TREE;
3834 }
3835
3836 /* Perform any modifications to EXPR required when it is unsaved.  Does
3837    not recurse into EXPR's subtrees.  */
3838
3839 static void
3840 unsave_expr_1 (tree expr)
3841 {
3842   switch (TREE_CODE (expr))
3843     {
3844     case TARGET_EXPR:
3845       /* Don't mess with a TARGET_EXPR that hasn't been expanded.
3846          It's OK for this to happen if it was part of a subtree that
3847          isn't immediately expanded, such as operand 2 of another
3848          TARGET_EXPR.  */
3849       if (TREE_OPERAND (expr, 1))
3850         break;
3851
3852       TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
3853       TREE_OPERAND (expr, 3) = NULL_TREE;
3854       break;
3855
3856     default:
3857       break;
3858     }
3859 }
3860
3861 /* Called via walk_tree when an expression is unsaved.  Using the
3862    splay_tree pointed to by ST (which is really a `splay_tree'),
3863    remaps all local declarations to appropriate replacements.  */
3864
3865 static tree
3866 unsave_r (tree *tp, int *walk_subtrees, void *data)
3867 {
3868   copy_body_data *id = (copy_body_data *) data;
3869   struct pointer_map_t *st = id->decl_map;
3870   tree *n;
3871
3872   /* Only a local declaration (variable or label).  */
3873   if ((TREE_CODE (*tp) == VAR_DECL && !TREE_STATIC (*tp))
3874       || TREE_CODE (*tp) == LABEL_DECL)
3875     {
3876       /* Lookup the declaration.  */
3877       n = (tree *) pointer_map_contains (st, *tp);
3878
3879       /* If it's there, remap it.  */
3880       if (n)
3881         *tp = *n;
3882     }
3883
3884   else if (TREE_CODE (*tp) == STATEMENT_LIST)
3885     gcc_unreachable ();
3886   else if (TREE_CODE (*tp) == BIND_EXPR)
3887     copy_bind_expr (tp, walk_subtrees, id);
3888   else if (TREE_CODE (*tp) == SAVE_EXPR)
3889     remap_save_expr (tp, st, walk_subtrees);
3890   else
3891     {
3892       copy_tree_r (tp, walk_subtrees, NULL);
3893
3894       /* Do whatever unsaving is required.  */
3895       unsave_expr_1 (*tp);
3896     }
3897
3898   /* Keep iterating.  */
3899   return NULL_TREE;
3900 }
3901
3902 /* Copies everything in EXPR and replaces variables, labels
3903    and SAVE_EXPRs local to EXPR.  */
3904
3905 tree
3906 unsave_expr_now (tree expr)
3907 {
3908   copy_body_data id;
3909
3910   /* There's nothing to do for NULL_TREE.  */
3911   if (expr == 0)
3912     return expr;
3913
3914   /* Set up ID.  */
3915   memset (&id, 0, sizeof (id));
3916   id.src_fn = current_function_decl;
3917   id.dst_fn = current_function_decl;
3918   id.decl_map = pointer_map_create ();
3919
3920   id.copy_decl = copy_decl_no_change;
3921   id.transform_call_graph_edges = CB_CGE_DUPLICATE;
3922   id.transform_new_cfg = false;
3923   id.transform_return_to_modify = false;
3924   id.transform_lang_insert_block = NULL;
3925
3926   /* Walk the tree once to find local labels.  */
3927   walk_tree_without_duplicates (&expr, mark_local_for_remap_r, &id);
3928
3929   /* Walk the tree again, copying, remapping, and unsaving.  */
3930   walk_tree (&expr, unsave_r, &id, NULL);
3931
3932   /* Clean up.  */
3933   pointer_map_destroy (id.decl_map);
3934
3935   return expr;
3936 }
3937
3938 /* Called via walk_gimple_seq.  If *GSIP points to a GIMPLE_LABEL for a local
3939    label, copies the declaration and enters it in the splay_tree in DATA (which
3940    is really a 'copy_body_data *'.  */
3941
3942 static tree
3943 mark_local_labels_stmt (gimple_stmt_iterator *gsip,
3944                         bool *handled_ops_p ATTRIBUTE_UNUSED,
3945                         struct walk_stmt_info *wi)
3946 {
3947   copy_body_data *id = (copy_body_data *) wi->info;
3948   gimple stmt = gsi_stmt (*gsip);
3949
3950   if (gimple_code (stmt) == GIMPLE_LABEL)
3951     {
3952       tree decl = gimple_label_label (stmt);
3953
3954       /* Copy the decl and remember the copy.  */
3955       insert_decl_map (id, decl, id->copy_decl (decl, id));
3956     }
3957
3958   return NULL_TREE;
3959 }
3960
3961
3962 /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
3963    Using the splay_tree pointed to by ST (which is really a `splay_tree'),
3964    remaps all local declarations to appropriate replacements in gimple
3965    operands. */
3966
3967 static tree
3968 replace_locals_op (tree *tp, int *walk_subtrees, void *data)
3969 {
3970   struct walk_stmt_info *wi = (struct walk_stmt_info*) data;
3971   copy_body_data *id = (copy_body_data *) wi->info;
3972   struct pointer_map_t *st = id->decl_map;
3973   tree *n;
3974   tree expr = *tp;
3975
3976   /* Only a local declaration (variable or label).  */
3977   if ((TREE_CODE (expr) == VAR_DECL
3978        && !TREE_STATIC (expr))
3979       || TREE_CODE (expr) == LABEL_DECL)
3980     {
3981       /* Lookup the declaration.  */
3982       n = (tree *) pointer_map_contains (st, expr);
3983
3984       /* If it's there, remap it.  */
3985       if (n)
3986         *tp = *n;
3987       *walk_subtrees = 0;
3988     }
3989   else if (TREE_CODE (expr) == STATEMENT_LIST
3990            || TREE_CODE (expr) == BIND_EXPR
3991            || TREE_CODE (expr) == SAVE_EXPR)
3992     gcc_unreachable ();
3993   else if (TREE_CODE (expr) == TARGET_EXPR)
3994     {
3995       /* Don't mess with a TARGET_EXPR that hasn't been expanded.
3996          It's OK for this to happen if it was part of a subtree that
3997          isn't immediately expanded, such as operand 2 of another
3998          TARGET_EXPR.  */
3999       if (!TREE_OPERAND (expr, 1))
4000         {
4001           TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
4002           TREE_OPERAND (expr, 3) = NULL_TREE;
4003         }
4004     }
4005
4006   /* Keep iterating.  */
4007   return NULL_TREE;
4008 }
4009
4010
4011 /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
4012    Using the splay_tree pointed to by ST (which is really a `splay_tree'),
4013    remaps all local declarations to appropriate replacements in gimple
4014    statements. */
4015
4016 static tree
4017 replace_locals_stmt (gimple_stmt_iterator *gsip,
4018                      bool *handled_ops_p ATTRIBUTE_UNUSED,
4019                      struct walk_stmt_info *wi)
4020 {
4021   copy_body_data *id = (copy_body_data *) wi->info;
4022   gimple stmt = gsi_stmt (*gsip);
4023
4024   if (gimple_code (stmt) == GIMPLE_BIND)
4025     {
4026       tree block = gimple_bind_block (stmt);
4027
4028       if (block)
4029         {
4030           remap_block (&block, id);
4031           gimple_bind_set_block (stmt, block);
4032         }
4033
4034       /* This will remap a lot of the same decls again, but this should be
4035          harmless.  */
4036       if (gimple_bind_vars (stmt))
4037         gimple_bind_set_vars (stmt, remap_decls (gimple_bind_vars (stmt), NULL, id));
4038     }
4039
4040   /* Keep iterating.  */
4041   return NULL_TREE;
4042 }
4043
4044
4045 /* Copies everything in SEQ and replaces variables and labels local to
4046    current_function_decl.  */
4047
4048 gimple_seq
4049 copy_gimple_seq_and_replace_locals (gimple_seq seq)
4050 {
4051   copy_body_data id;
4052   struct walk_stmt_info wi;
4053   struct pointer_set_t *visited;
4054   gimple_seq copy;
4055
4056   /* There's nothing to do for NULL_TREE.  */
4057   if (seq == NULL)
4058     return seq;
4059
4060   /* Set up ID.  */
4061   memset (&id, 0, sizeof (id));
4062   id.src_fn = current_function_decl;
4063   id.dst_fn = current_function_decl;
4064   id.decl_map = pointer_map_create ();
4065
4066   id.copy_decl = copy_decl_no_change;
4067   id.transform_call_graph_edges = CB_CGE_DUPLICATE;
4068   id.transform_new_cfg = false;
4069   id.transform_return_to_modify = false;
4070   id.transform_lang_insert_block = NULL;
4071
4072   /* Walk the tree once to find local labels.  */
4073   memset (&wi, 0, sizeof (wi));
4074   visited = pointer_set_create ();
4075   wi.info = &id;
4076   wi.pset = visited;
4077   walk_gimple_seq (seq, mark_local_labels_stmt, NULL, &wi);
4078   pointer_set_destroy (visited);
4079
4080   copy = gimple_seq_copy (seq);
4081
4082   /* Walk the copy, remapping decls.  */
4083   memset (&wi, 0, sizeof (wi));
4084   wi.info = &id;
4085   walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, &wi);
4086
4087   /* Clean up.  */
4088   pointer_map_destroy (id.decl_map);
4089
4090   return copy;
4091 }
4092
4093
4094 /* Allow someone to determine if SEARCH is a child of TOP from gdb.  */
4095
4096 static tree
4097 debug_find_tree_1 (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED, void *data)
4098 {
4099   if (*tp == data)
4100     return (tree) data;
4101   else
4102     return NULL;
4103 }
4104
4105 bool
4106 debug_find_tree (tree top, tree search)
4107 {
4108   return walk_tree_without_duplicates (&top, debug_find_tree_1, search) != 0;
4109 }
4110
4111
4112 /* Declare the variables created by the inliner.  Add all the variables in
4113    VARS to BIND_EXPR.  */
4114
4115 static void
4116 declare_inline_vars (tree block, tree vars)
4117 {
4118   tree t;
4119   for (t = vars; t; t = TREE_CHAIN (t))
4120     {
4121       DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
4122       gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t));
4123       cfun->local_decls = tree_cons (NULL_TREE, t, cfun->local_decls);
4124     }
4125
4126   if (block)
4127     BLOCK_VARS (block) = chainon (BLOCK_VARS (block), vars);
4128 }
4129
4130 /* Copy NODE (which must be a DECL).  The DECL originally was in the FROM_FN,
4131    but now it will be in the TO_FN.  PARM_TO_VAR means enable PARM_DECL to
4132    VAR_DECL translation.  */
4133
4134 static tree
4135 copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy)
4136 {
4137   /* Don't generate debug information for the copy if we wouldn't have
4138      generated it for the copy either.  */
4139   DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (decl);
4140   DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl);
4141
4142   /* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what
4143      declaration inspired this copy.  */
4144   DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl);
4145
4146   /* The new variable/label has no RTL, yet.  */
4147   if (CODE_CONTAINS_STRUCT (TREE_CODE (copy), TS_DECL_WRTL)
4148       && !TREE_STATIC (copy) && !DECL_EXTERNAL (copy))
4149     SET_DECL_RTL (copy, NULL_RTX);
4150
4151   /* These args would always appear unused, if not for this.  */
4152   TREE_USED (copy) = 1;
4153
4154   /* Set the context for the new declaration.  */
4155   if (!DECL_CONTEXT (decl))
4156     /* Globals stay global.  */
4157     ;
4158   else if (DECL_CONTEXT (decl) != id->src_fn)
4159     /* Things that weren't in the scope of the function we're inlining
4160        from aren't in the scope we're inlining to, either.  */
4161     ;
4162   else if (TREE_STATIC (decl))
4163     /* Function-scoped static variables should stay in the original
4164        function.  */
4165     ;
4166   else
4167     /* Ordinary automatic local variables are now in the scope of the
4168        new function.  */
4169     DECL_CONTEXT (copy) = id->dst_fn;
4170
4171   return copy;
4172 }
4173
4174 static tree
4175 copy_decl_to_var (tree decl, copy_body_data *id)
4176 {
4177   tree copy, type;
4178
4179   gcc_assert (TREE_CODE (decl) == PARM_DECL
4180               || TREE_CODE (decl) == RESULT_DECL);
4181
4182   type = TREE_TYPE (decl);
4183
4184   copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
4185                      VAR_DECL, DECL_NAME (decl), type);
4186   TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
4187   TREE_READONLY (copy) = TREE_READONLY (decl);
4188   TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
4189   DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (decl);
4190
4191   return copy_decl_for_dup_finish (id, decl, copy);
4192 }
4193
4194 /* Like copy_decl_to_var, but create a return slot object instead of a
4195    pointer variable for return by invisible reference.  */
4196
4197 static tree
4198 copy_result_decl_to_var (tree decl, copy_body_data *id)
4199 {
4200   tree copy, type;
4201
4202   gcc_assert (TREE_CODE (decl) == PARM_DECL
4203               || TREE_CODE (decl) == RESULT_DECL);
4204
4205   type = TREE_TYPE (decl);
4206   if (DECL_BY_REFERENCE (decl))
4207     type = TREE_TYPE (type);
4208
4209   copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
4210                      VAR_DECL, DECL_NAME (decl), type);
4211   TREE_READONLY (copy) = TREE_READONLY (decl);
4212   TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
4213   if (!DECL_BY_REFERENCE (decl))
4214     {
4215       TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
4216       DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (decl);
4217     }
4218
4219   return copy_decl_for_dup_finish (id, decl, copy);
4220 }
4221
4222 tree
4223 copy_decl_no_change (tree decl, copy_body_data *id)
4224 {
4225   tree copy;
4226
4227   copy = copy_node (decl);
4228
4229   /* The COPY is not abstract; it will be generated in DST_FN.  */
4230   DECL_ABSTRACT (copy) = 0;
4231   lang_hooks.dup_lang_specific_decl (copy);
4232
4233   /* TREE_ADDRESSABLE isn't used to indicate that a label's address has
4234      been taken; it's for internal bookkeeping in expand_goto_internal.  */
4235   if (TREE_CODE (copy) == LABEL_DECL)
4236     {
4237       TREE_ADDRESSABLE (copy) = 0;
4238       LABEL_DECL_UID (copy) = -1;
4239     }
4240
4241   return copy_decl_for_dup_finish (id, decl, copy);
4242 }
4243
4244 static tree
4245 copy_decl_maybe_to_var (tree decl, copy_body_data *id)
4246 {
4247   if (TREE_CODE (decl) == PARM_DECL || TREE_CODE (decl) == RESULT_DECL)
4248     return copy_decl_to_var (decl, id);
4249   else
4250     return copy_decl_no_change (decl, id);
4251 }
4252
4253 /* Return a copy of the function's argument tree.  */
4254 static tree
4255 copy_arguments_for_versioning (tree orig_parm, copy_body_data * id,
4256                                bitmap args_to_skip, tree *vars)
4257 {
4258   tree arg, *parg;
4259   tree new_parm = NULL;
4260   int i = 0;
4261
4262   parg = &new_parm;
4263
4264   for (arg = orig_parm; arg; arg = TREE_CHAIN (arg), i++)
4265     if (!args_to_skip || !bitmap_bit_p (args_to_skip, i))
4266       {
4267         tree new_tree = remap_decl (arg, id);
4268         lang_hooks.dup_lang_specific_decl (new_tree);
4269         *parg = new_tree;
4270         parg = &TREE_CHAIN (new_tree);
4271       }
4272     else if (!pointer_map_contains (id->decl_map, arg))
4273       {
4274         /* Make an equivalent VAR_DECL.  If the argument was used
4275            as temporary variable later in function, the uses will be
4276            replaced by local variable.  */
4277         tree var = copy_decl_to_var (arg, id);
4278         get_var_ann (var);
4279         add_referenced_var (var);
4280         insert_decl_map (id, arg, var);
4281         /* Declare this new variable.  */
4282         TREE_CHAIN (var) = *vars;
4283         *vars = var;
4284       }
4285   return new_parm;
4286 }
4287
4288 /* Return a copy of the function's static chain.  */
4289 static tree
4290 copy_static_chain (tree static_chain, copy_body_data * id)
4291 {
4292   tree *chain_copy, *pvar;
4293
4294   chain_copy = &static_chain;
4295   for (pvar = chain_copy; *pvar; pvar = &TREE_CHAIN (*pvar))
4296     {
4297       tree new_tree = remap_decl (*pvar, id);
4298       lang_hooks.dup_lang_specific_decl (new_tree);
4299       TREE_CHAIN (new_tree) = TREE_CHAIN (*pvar);
4300       *pvar = new_tree;
4301     }
4302   return static_chain;
4303 }
4304
4305 /* Return true if the function is allowed to be versioned.
4306    This is a guard for the versioning functionality.  */
4307 bool
4308 tree_versionable_function_p (tree fndecl)
4309 {
4310   if (fndecl == NULL_TREE)
4311     return false;
4312   /* ??? There are cases where a function is
4313      uninlinable but can be versioned.  */
4314   if (!tree_inlinable_function_p (fndecl))
4315     return false;
4316
4317   return true;
4318 }
4319
4320 /* Delete all unreachable basic blocks and update callgraph.
4321    Doing so is somewhat nontrivial because we need to update all clones and
4322    remove inline function that become unreachable.  */
4323
4324 static bool
4325 delete_unreachable_blocks_update_callgraph (copy_body_data *id)
4326 {
4327   bool changed = false;
4328   basic_block b, next_bb;
4329
4330   find_unreachable_blocks ();
4331
4332   /* Delete all unreachable basic blocks.  */
4333
4334   for (b = ENTRY_BLOCK_PTR->next_bb; b != EXIT_BLOCK_PTR; b = next_bb)
4335     {
4336       next_bb = b->next_bb;
4337
4338       if (!(b->flags & BB_REACHABLE))
4339         {
4340           gimple_stmt_iterator bsi;
4341
4342           for (bsi = gsi_start_bb (b); !gsi_end_p (bsi); gsi_next (&bsi))
4343             if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL)
4344               {
4345                 struct cgraph_edge *e;
4346                 struct cgraph_node *node;
4347
4348                 if ((e = cgraph_edge (id->dst_node, gsi_stmt (bsi))) != NULL)
4349                   {
4350                     if (!e->inline_failed)
4351                       cgraph_remove_node_and_inline_clones (e->callee);
4352                     else
4353                       cgraph_remove_edge (e);
4354                   }
4355                 if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
4356                     && id->dst_node->clones)
4357                   for (node = id->dst_node->clones; node != id->dst_node;)
4358                     {
4359                       if ((e = cgraph_edge (node, gsi_stmt (bsi))) != NULL)
4360                         {
4361                           if (!e->inline_failed)
4362                             cgraph_remove_node_and_inline_clones (e->callee);
4363                           else
4364                             cgraph_remove_edge (e);
4365                         }
4366
4367                       if (node->clones)
4368                         node = node->clones;
4369                       else if (node->next_sibling_clone)
4370                         node = node->next_sibling_clone;
4371                       else
4372                         {
4373                           while (node != id->dst_node && !node->next_sibling_clone)
4374                             node = node->clone_of;
4375                           if (node != id->dst_node)
4376                             node = node->next_sibling_clone;
4377                         }
4378                     }
4379               }
4380           delete_basic_block (b);
4381           changed = true;
4382         }
4383     }
4384
4385   if (changed)
4386     tidy_fallthru_edges ();
4387 #ifdef ENABLE_CHECKING0
4388   verify_cgraph_node (id->dst_node);
4389   if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
4390       && id->dst_node->clones)
4391     {
4392       struct cgraph_node *node;
4393       for (node = id->dst_node->clones; node != id->dst_node;)
4394         {
4395           verify_cgraph_node (node);
4396
4397           if (node->clones)
4398             node = node->clones;
4399           else if (node->next_sibling_clone)
4400             node = node->next_sibling_clone;
4401           else
4402             {
4403               while (node != id->dst_node && !node->next_sibling_clone)
4404                 node = node->clone_of;
4405               if (node != id->dst_node)
4406                 node = node->next_sibling_clone;
4407             }
4408         }
4409      }
4410 #endif
4411   return changed;
4412 }
4413
4414 /* Create a copy of a function's tree.
4415    OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes
4416    of the original function and the new copied function
4417    respectively.  In case we want to replace a DECL
4418    tree with another tree while duplicating the function's
4419    body, TREE_MAP represents the mapping between these
4420    trees. If UPDATE_CLONES is set, the call_stmt fields
4421    of edges of clones of the function will be updated.  */
4422 void
4423 tree_function_versioning (tree old_decl, tree new_decl, VEC(ipa_replace_map_p,gc)* tree_map,
4424                           bool update_clones, bitmap args_to_skip)
4425 {
4426   struct cgraph_node *old_version_node;
4427   struct cgraph_node *new_version_node;
4428   copy_body_data id;
4429   tree p;
4430   unsigned i;
4431   struct ipa_replace_map *replace_info;
4432   basic_block old_entry_block;
4433   VEC (gimple, heap) *init_stmts = VEC_alloc (gimple, heap, 10);
4434
4435   tree t_step;
4436   tree old_current_function_decl = current_function_decl;
4437   tree vars = NULL_TREE;
4438
4439   gcc_assert (TREE_CODE (old_decl) == FUNCTION_DECL
4440               && TREE_CODE (new_decl) == FUNCTION_DECL);
4441   DECL_POSSIBLY_INLINED (old_decl) = 1;
4442
4443   old_version_node = cgraph_node (old_decl);
4444   new_version_node = cgraph_node (new_decl);
4445
4446   /* Output the inlining info for this abstract function, since it has been
4447      inlined.  If we don't do this now, we can lose the information about the
4448      variables in the function when the blocks get blown away as soon as we
4449      remove the cgraph node.  */
4450   (*debug_hooks->outlining_inline_function) (old_decl);
4451
4452   DECL_ARTIFICIAL (new_decl) = 1;
4453   DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl);
4454
4455   /* Prepare the data structures for the tree copy.  */
4456   memset (&id, 0, sizeof (id));
4457
4458   /* Generate a new name for the new version. */
4459   id.statements_to_fold = pointer_set_create ();
4460
4461   id.decl_map = pointer_map_create ();
4462   id.src_fn = old_decl;
4463   id.dst_fn = new_decl;
4464   id.src_node = old_version_node;
4465   id.dst_node = new_version_node;
4466   id.src_cfun = DECL_STRUCT_FUNCTION (old_decl);
4467
4468   id.copy_decl = copy_decl_no_change;
4469   id.transform_call_graph_edges
4470     = update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE;
4471   id.transform_new_cfg = true;
4472   id.transform_return_to_modify = false;
4473   id.transform_lang_insert_block = NULL;
4474
4475   current_function_decl = new_decl;
4476   old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION
4477     (DECL_STRUCT_FUNCTION (old_decl));
4478   initialize_cfun (new_decl, old_decl,
4479                    old_entry_block->count,
4480                    old_entry_block->frequency);
4481   push_cfun (DECL_STRUCT_FUNCTION (new_decl));
4482
4483   /* Copy the function's static chain.  */
4484   p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl;
4485   if (p)
4486     DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl =
4487       copy_static_chain (DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl,
4488                          &id);
4489
4490   /* If there's a tree_map, prepare for substitution.  */
4491   if (tree_map)
4492     for (i = 0; i < VEC_length (ipa_replace_map_p, tree_map); i++)
4493       {
4494         gimple init;
4495         replace_info = VEC_index (ipa_replace_map_p, tree_map, i);
4496         if (replace_info->replace_p)
4497           {
4498             tree op = replace_info->new_tree;
4499
4500             STRIP_NOPS (op);
4501
4502             if (TREE_CODE (op) == VIEW_CONVERT_EXPR)
4503               op = TREE_OPERAND (op, 0);
4504
4505             if (TREE_CODE (op) == ADDR_EXPR)
4506               {
4507                 op = TREE_OPERAND (op, 0);
4508                 while (handled_component_p (op))
4509                   op = TREE_OPERAND (op, 0);
4510                 if (TREE_CODE (op) == VAR_DECL)
4511                   add_referenced_var (op);
4512               }
4513             gcc_assert (TREE_CODE (replace_info->old_tree) == PARM_DECL);
4514             init = setup_one_parameter (&id, replace_info->old_tree,
4515                                         replace_info->new_tree, id.src_fn,
4516                                         NULL,
4517                                         &vars);
4518             if (init)
4519               VEC_safe_push (gimple, heap, init_stmts, init);
4520           }
4521       }
4522   /* Copy the function's arguments.  */
4523   if (DECL_ARGUMENTS (old_decl) != NULL_TREE)
4524     DECL_ARGUMENTS (new_decl) =
4525       copy_arguments_for_versioning (DECL_ARGUMENTS (old_decl), &id,
4526                                      args_to_skip, &vars);
4527
4528   DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), &id);
4529
4530   /* Renumber the lexical scoping (non-code) blocks consecutively.  */
4531   number_blocks (id.dst_fn);
4532
4533   declare_inline_vars (DECL_INITIAL (new_decl), vars);
4534
4535   if (DECL_STRUCT_FUNCTION (old_decl)->local_decls != NULL_TREE)
4536     /* Add local vars.  */
4537     for (t_step = DECL_STRUCT_FUNCTION (old_decl)->local_decls;
4538          t_step; t_step = TREE_CHAIN (t_step))
4539       {
4540         tree var = TREE_VALUE (t_step);
4541         if (TREE_STATIC (var) && !TREE_ASM_WRITTEN (var))
4542           cfun->local_decls = tree_cons (NULL_TREE, var, cfun->local_decls);
4543         else if (!can_be_nonlocal (var, &id))
4544           cfun->local_decls =
4545             tree_cons (NULL_TREE, remap_decl (var, &id),
4546                        cfun->local_decls);
4547       }
4548
4549   /* Copy the Function's body.  */
4550   copy_body (&id, old_entry_block->count, old_entry_block->frequency, ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR);
4551
4552   if (DECL_RESULT (old_decl) != NULL_TREE)
4553     {
4554       tree *res_decl = &DECL_RESULT (old_decl);
4555       DECL_RESULT (new_decl) = remap_decl (*res_decl, &id);
4556       lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl));
4557     }
4558
4559   /* Renumber the lexical scoping (non-code) blocks consecutively.  */
4560   number_blocks (new_decl);
4561
4562   if (VEC_length (gimple, init_stmts))
4563     {
4564       basic_block bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR));
4565       while (VEC_length (gimple, init_stmts))
4566         insert_init_stmt (bb, VEC_pop (gimple, init_stmts));
4567     }
4568
4569   /* Clean up.  */
4570   pointer_map_destroy (id.decl_map);
4571   free_dominance_info (CDI_DOMINATORS);
4572   free_dominance_info (CDI_POST_DOMINATORS);
4573
4574   fold_marked_statements (0, id.statements_to_fold);
4575   pointer_set_destroy (id.statements_to_fold);
4576   fold_cond_expr_cond ();
4577   delete_unreachable_blocks_update_callgraph (&id);
4578   update_ssa (TODO_update_ssa);
4579   free_dominance_info (CDI_DOMINATORS);
4580   free_dominance_info (CDI_POST_DOMINATORS);
4581
4582   VEC_free (gimple, heap, init_stmts);
4583   pop_cfun ();
4584   current_function_decl = old_current_function_decl;
4585   gcc_assert (!current_function_decl
4586               || DECL_STRUCT_FUNCTION (current_function_decl) == cfun);
4587   return;
4588 }
4589
4590 /* Duplicate a type, fields and all.  */
4591
4592 tree
4593 build_duplicate_type (tree type)
4594 {
4595   struct copy_body_data id;
4596
4597   memset (&id, 0, sizeof (id));
4598   id.src_fn = current_function_decl;
4599   id.dst_fn = current_function_decl;
4600   id.src_cfun = cfun;
4601   id.decl_map = pointer_map_create ();
4602   id.copy_decl = copy_decl_no_change;
4603
4604   type = remap_type_1 (type, &id);
4605
4606   pointer_map_destroy (id.decl_map);
4607
4608   TYPE_CANONICAL (type) = type;
4609
4610   return type;
4611 }
4612
4613 /* Return whether it is safe to inline a function because it used different
4614    target specific options or different optimization options.  */
4615 bool
4616 tree_can_inline_p (tree caller, tree callee)
4617 {
4618 #if 0
4619   /* This causes a regression in SPEC in that it prevents a cold function from
4620      inlining a hot function.  Perhaps this should only apply to functions
4621      that the user declares hot/cold/optimize explicitly.  */
4622
4623   /* Don't inline a function with a higher optimization level than the
4624      caller, or with different space constraints (hot/cold functions).  */
4625   tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (caller);
4626   tree callee_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee);
4627
4628   if (caller_tree != callee_tree)
4629     {
4630       struct cl_optimization *caller_opt
4631         = TREE_OPTIMIZATION ((caller_tree)
4632                              ? caller_tree
4633                              : optimization_default_node);
4634
4635       struct cl_optimization *callee_opt
4636         = TREE_OPTIMIZATION ((callee_tree)
4637                              ? callee_tree
4638                              : optimization_default_node);
4639
4640       if ((caller_opt->optimize > callee_opt->optimize)
4641           || (caller_opt->optimize_size != callee_opt->optimize_size))
4642         return false;
4643     }
4644 #endif
4645
4646   /* Allow the backend to decide if inlining is ok.  */
4647   return targetm.target_option.can_inline_p (caller, callee);
4648 }