1 /* Function splitting pass
2 Copyright (C) 2010-2019 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka <jh@suse.cz>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* The purpose of this pass is to split function bodies to improve
22 inlining. I.e. for function of the form:
47 When func becomes inlinable and when cheap_test is often true, inlining func,
48 but not fund.part leads to performance improvement similar as inlining
49 original func while the code size growth is smaller.
51 The pass is organized in three stages:
52 1) Collect local info about basic block into BB_INFO structure and
53 compute function body estimated size and time.
54 2) Via DFS walk find all possible basic blocks where we can split
56 3) If split point is found, split at the specified BB by creating a clone
57 and updating function to call it.
59 The decisions what functions to split are in execute_split_functions
62 There are several possible future improvements for this pass including:
64 1) Splitting to break up large functions
65 2) Splitting to reduce stack frame usage
66 3) Allow split part of function to use values computed in the header part.
67 The values needs to be passed to split function, perhaps via same
68 interface as for nested functions or as argument.
69 4) Support for simple rematerialization. I.e. when split part use
70 value computed in header from function parameter in very cheap way, we
71 can just recompute it.
72 5) Support splitting of nested functions.
73 6) Support non-SSA arguments.
74 7) There is nothing preventing us from producing multiple parts of single function
75 when needed or splitting also the parts. */
79 #include "coretypes.h"
85 #include "alloc-pool.h"
86 #include "tree-pass.h"
89 #include "diagnostic.h"
90 #include "fold-const.h"
94 #include "gimple-iterator.h"
95 #include "gimplify-me.h"
96 #include "gimple-walk.h"
97 #include "symbol-summary.h"
100 #include "tree-into-ssa.h"
101 #include "tree-dfa.h"
102 #include "tree-inline.h"
104 #include "gimple-pretty-print.h"
105 #include "ipa-fnsummary.h"
109 /* Per basic block info. */
118 static vec
<split_bb_info
> bb_info_vec
;
120 /* Description of split point. */
125 /* Size of the partitions. */
126 sreal header_time
, split_time
;
127 unsigned int header_size
, split_size
;
129 /* SSA names that need to be passed into spit function. */
130 bitmap ssa_names_to_pass
;
132 /* Basic block where we split (that will become entry point of new function. */
133 basic_block entry_bb
;
135 /* Count for entering the split part.
136 This is not count of the entry_bb because it may be in loop. */
139 /* Basic blocks we are splitting away. */
142 /* True when return value is computed on split part and thus it needs
144 bool split_part_set_retval
;
147 /* Best split point found. */
149 class split_point best_split_point
;
151 /* Set of basic blocks that are not allowed to dominate a split point. */
153 static bitmap forbidden_dominators
;
155 static tree
find_retval (basic_block return_bb
);
157 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
158 variable, check it if it is present in bitmap passed via DATA. */
161 test_nonssa_use (gimple
*, tree t
, tree
, void *data
)
163 t
= get_base_address (t
);
165 if (!t
|| is_gimple_reg (t
))
168 if (TREE_CODE (t
) == PARM_DECL
170 && auto_var_in_fn_p (t
, current_function_decl
))
171 || TREE_CODE (t
) == RESULT_DECL
172 /* Normal labels are part of CFG and will be handled gratefuly.
173 Forced labels however can be used directly by statements and
174 need to stay in one partition along with their uses. */
175 || (TREE_CODE (t
) == LABEL_DECL
176 && FORCED_LABEL (t
)))
177 return bitmap_bit_p ((bitmap
)data
, DECL_UID (t
));
179 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
180 to pretend that the value pointed to is actual result decl. */
181 if ((TREE_CODE (t
) == MEM_REF
|| INDIRECT_REF_P (t
))
182 && TREE_CODE (TREE_OPERAND (t
, 0)) == SSA_NAME
183 && SSA_NAME_VAR (TREE_OPERAND (t
, 0))
184 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t
, 0))) == RESULT_DECL
185 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
187 bitmap_bit_p ((bitmap
)data
,
188 DECL_UID (DECL_RESULT (current_function_decl
)));
193 /* Dump split point CURRENT. */
196 dump_split_point (FILE * file
, class split_point
*current
)
199 "Split point at BB %i\n"
200 " header time: %f header size: %i\n"
201 " split time: %f split size: %i\n bbs: ",
202 current
->entry_bb
->index
, current
->header_time
.to_double (),
203 current
->header_size
, current
->split_time
.to_double (),
204 current
->split_size
);
205 dump_bitmap (file
, current
->split_bbs
);
206 fprintf (file
, " SSA names to pass: ");
207 dump_bitmap (file
, current
->ssa_names_to_pass
);
210 /* Look for all BBs in header that might lead to the split part and verify
211 that they are not defining any non-SSA var used by the split part.
212 Parameters are the same as for consider_split. */
215 verify_non_ssa_vars (class split_point
*current
, bitmap non_ssa_vars
,
216 basic_block return_bb
)
218 bitmap seen
= BITMAP_ALLOC (NULL
);
219 vec
<basic_block
> worklist
= vNULL
;
225 FOR_EACH_EDGE (e
, ei
, current
->entry_bb
->preds
)
226 if (e
->src
!= ENTRY_BLOCK_PTR_FOR_FN (cfun
)
227 && !bitmap_bit_p (current
->split_bbs
, e
->src
->index
))
229 worklist
.safe_push (e
->src
);
230 bitmap_set_bit (seen
, e
->src
->index
);
233 while (!worklist
.is_empty ())
235 bb
= worklist
.pop ();
236 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
237 if (e
->src
!= ENTRY_BLOCK_PTR_FOR_FN (cfun
)
238 && bitmap_set_bit (seen
, e
->src
->index
))
240 gcc_checking_assert (!bitmap_bit_p (current
->split_bbs
,
242 worklist
.safe_push (e
->src
);
244 for (gimple_stmt_iterator bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
);
247 gimple
*stmt
= gsi_stmt (bsi
);
248 if (is_gimple_debug (stmt
))
250 if (walk_stmt_load_store_addr_ops
251 (stmt
, non_ssa_vars
, test_nonssa_use
, test_nonssa_use
,
257 if (glabel
*label_stmt
= dyn_cast
<glabel
*> (stmt
))
258 if (test_nonssa_use (stmt
, gimple_label_label (label_stmt
),
259 NULL_TREE
, non_ssa_vars
))
265 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
268 if (walk_stmt_load_store_addr_ops
269 (gsi_stmt (bsi
), non_ssa_vars
, test_nonssa_use
, test_nonssa_use
,
276 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
278 if (e
->dest
!= return_bb
)
280 for (gphi_iterator bsi
= gsi_start_phis (return_bb
);
284 gphi
*stmt
= bsi
.phi ();
285 tree op
= gimple_phi_arg_def (stmt
, e
->dest_idx
);
287 if (virtual_operand_p (gimple_phi_result (stmt
)))
289 if (TREE_CODE (op
) != SSA_NAME
290 && test_nonssa_use (stmt
, op
, op
, non_ssa_vars
))
299 /* Verify that the rest of function does not define any label
300 used by the split part. */
301 FOR_EACH_BB_FN (bb
, cfun
)
302 if (!bitmap_bit_p (current
->split_bbs
, bb
->index
)
303 && !bitmap_bit_p (seen
, bb
->index
))
305 gimple_stmt_iterator bsi
;
306 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
307 if (glabel
*label_stmt
= dyn_cast
<glabel
*> (gsi_stmt (bsi
)))
309 if (test_nonssa_use (label_stmt
,
310 gimple_label_label (label_stmt
),
311 NULL_TREE
, non_ssa_vars
))
327 /* If STMT is a call, check the callee against a list of forbidden
328 predicate functions. If a match is found, look for uses of the
329 call result in condition statements that compare against zero.
330 For each such use, find the block targeted by the condition
331 statement for the nonzero result, and set the bit for this block
332 in the forbidden dominators bitmap. The purpose of this is to avoid
333 selecting a split point where we are likely to lose the chance
334 to optimize away an unused function call. */
337 check_forbidden_calls (gimple
*stmt
)
339 imm_use_iterator use_iter
;
343 /* At the moment, __builtin_constant_p is the only forbidden
344 predicate function call (see PR49642). */
345 if (!gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
))
348 lhs
= gimple_call_lhs (stmt
);
350 if (!lhs
|| TREE_CODE (lhs
) != SSA_NAME
)
353 FOR_EACH_IMM_USE_FAST (use_p
, use_iter
, lhs
)
356 basic_block use_bb
, forbidden_bb
;
358 edge true_edge
, false_edge
;
361 use_stmt
= dyn_cast
<gcond
*> (USE_STMT (use_p
));
365 /* Assuming canonical form for GIMPLE_COND here, with constant
366 in second position. */
367 op1
= gimple_cond_rhs (use_stmt
);
368 code
= gimple_cond_code (use_stmt
);
369 use_bb
= gimple_bb (use_stmt
);
371 extract_true_false_edges_from_block (use_bb
, &true_edge
, &false_edge
);
373 /* We're only interested in comparisons that distinguish
374 unambiguously from zero. */
375 if (!integer_zerop (op1
) || code
== LE_EXPR
|| code
== GE_EXPR
)
379 forbidden_bb
= false_edge
->dest
;
381 forbidden_bb
= true_edge
->dest
;
383 bitmap_set_bit (forbidden_dominators
, forbidden_bb
->index
);
387 /* If BB is dominated by any block in the forbidden dominators set,
388 return TRUE; else FALSE. */
391 dominated_by_forbidden (basic_block bb
)
396 EXECUTE_IF_SET_IN_BITMAP (forbidden_dominators
, 1, dom_bb
, bi
)
398 if (dominated_by_p (CDI_DOMINATORS
, bb
,
399 BASIC_BLOCK_FOR_FN (cfun
, dom_bb
)))
406 /* For give split point CURRENT and return block RETURN_BB return 1
407 if ssa name VAL is set by split part and 0 otherwise. */
409 split_part_set_ssa_name_p (tree val
, class split_point
*current
,
410 basic_block return_bb
)
412 if (TREE_CODE (val
) != SSA_NAME
)
415 return (!SSA_NAME_IS_DEFAULT_DEF (val
)
416 && (bitmap_bit_p (current
->split_bbs
,
417 gimple_bb (SSA_NAME_DEF_STMT (val
))->index
)
418 || gimple_bb (SSA_NAME_DEF_STMT (val
)) == return_bb
));
421 /* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa
422 variables used and RETURN_BB is return basic block.
423 See if we can split function here. */
426 consider_split (class split_point
*current
, bitmap non_ssa_vars
,
427 basic_block return_bb
)
430 unsigned int num_args
= 0;
431 unsigned int call_overhead
;
437 bool back_edge
= false;
439 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
440 dump_split_point (dump_file
, current
);
442 current
->count
= profile_count::zero ();
443 FOR_EACH_EDGE (e
, ei
, current
->entry_bb
->preds
)
445 if (e
->flags
& EDGE_DFS_BACK
)
447 if (!bitmap_bit_p (current
->split_bbs
, e
->src
->index
))
448 current
->count
+= e
->count ();
451 /* Do not split when we would end up calling function anyway.
452 Compares are three state, use !(...<...) to also give up when outcome
455 < (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.apply_scale
456 (PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY
), 100))))
458 /* When profile is guessed, we cannot expect it to give us
459 realistic estimate on likelyness of function taking the
460 complex path. As a special case, when tail of the function is
461 a loop, enable splitting since inlining code skipping the loop
462 is likely noticeable win. */
464 && profile_status_for_fn (cfun
) != PROFILE_READ
466 < ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
)
468 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
471 " Split before loop, accepting despite low counts");
472 current
->count
.dump (dump_file
);
473 fprintf (dump_file
, " ");
474 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.dump (dump_file
);
479 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
481 " Refused: incoming frequency is too large.\n");
486 if (!current
->header_size
)
488 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
489 fprintf (dump_file
, " Refused: header empty\n");
493 /* Verify that PHI args on entry are either virtual or all their operands
494 incoming from header are the same. */
495 for (bsi
= gsi_start_phis (current
->entry_bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
497 gphi
*stmt
= bsi
.phi ();
500 if (virtual_operand_p (gimple_phi_result (stmt
)))
502 for (i
= 0; i
< gimple_phi_num_args (stmt
); i
++)
504 edge e
= gimple_phi_arg_edge (stmt
, i
);
505 if (!bitmap_bit_p (current
->split_bbs
, e
->src
->index
))
507 tree edge_val
= gimple_phi_arg_def (stmt
, i
);
508 if (val
&& edge_val
!= val
)
510 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
512 " Refused: entry BB has PHI with multiple variants\n");
521 /* See what argument we will pass to the split function and compute
523 call_overhead
= eni_size_weights
.call_cost
;
524 for (parm
= DECL_ARGUMENTS (current_function_decl
); parm
;
525 parm
= DECL_CHAIN (parm
))
527 if (!is_gimple_reg (parm
))
529 if (bitmap_bit_p (non_ssa_vars
, DECL_UID (parm
)))
531 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
533 " Refused: need to pass non-ssa param values\n");
539 tree ddef
= ssa_default_def (cfun
, parm
);
541 && bitmap_bit_p (current
->ssa_names_to_pass
,
542 SSA_NAME_VERSION (ddef
)))
544 if (!VOID_TYPE_P (TREE_TYPE (parm
)))
545 call_overhead
+= estimate_move_cost (TREE_TYPE (parm
), false);
550 if (!VOID_TYPE_P (TREE_TYPE (current_function_decl
)))
551 call_overhead
+= estimate_move_cost (TREE_TYPE (current_function_decl
),
554 if (current
->split_size
<= call_overhead
)
556 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
558 " Refused: split size is smaller than call overhead\n");
561 /* FIXME: The logic here is not very precise, because inliner does use
562 inline predicates to reduce function body size. We add 10 to anticipate
563 that. Next stage1 we should try to be more meaningful here. */
564 if (current
->header_size
+ call_overhead
565 >= (unsigned int)(DECL_DECLARED_INLINE_P (current_function_decl
)
566 ? MAX_INLINE_INSNS_SINGLE
567 : MAX_INLINE_INSNS_AUTO
) + 10)
569 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
571 " Refused: header size is too large for inline candidate\n");
575 /* Splitting functions brings the target out of comdat group; this will
576 lead to code duplication if the function is reused by other unit.
577 Limit this duplication. This is consistent with limit in tree-sra.c
578 FIXME: with LTO we ought to be able to do better! */
579 if (DECL_ONE_ONLY (current_function_decl
)
580 && current
->split_size
>= (unsigned int) MAX_INLINE_INSNS_AUTO
+ 10)
582 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
584 " Refused: function is COMDAT and tail is too large\n");
587 /* For comdat functions also reject very small tails; those will likely get
588 inlined back and we do not want to risk the duplication overhead.
589 FIXME: with LTO we ought to be able to do better! */
590 if (DECL_ONE_ONLY (current_function_decl
)
591 && current
->split_size
592 <= (unsigned int) PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
) / 2)
594 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
596 " Refused: function is COMDAT and tail is too small\n");
600 /* FIXME: we currently can pass only SSA function parameters to the split
601 arguments. Once parm_adjustment infrastructure is supported by cloning,
602 we can pass more than that. */
603 if (num_args
!= bitmap_count_bits (current
->ssa_names_to_pass
))
606 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
608 " Refused: need to pass non-param values\n");
612 /* When there are non-ssa vars used in the split region, see if they
613 are used in the header region. If so, reject the split.
614 FIXME: we can use nested function support to access both. */
615 if (!bitmap_empty_p (non_ssa_vars
)
616 && !verify_non_ssa_vars (current
, non_ssa_vars
, return_bb
))
618 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
620 " Refused: split part has non-ssa uses\n");
624 /* If the split point is dominated by a forbidden block, reject
626 if (!bitmap_empty_p (forbidden_dominators
)
627 && dominated_by_forbidden (current
->entry_bb
))
629 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
631 " Refused: split point dominated by forbidden block\n");
635 /* See if retval used by return bb is computed by header or split part.
636 When it is computed by split part, we need to produce return statement
637 in the split part and add code to header to pass it around.
639 This is bit tricky to test:
640 1) When there is no return_bb or no return value, we always pass
642 2) Invariants are always computed by caller.
643 3) For SSA we need to look if defining statement is in header or split part
644 4) For non-SSA we need to look where the var is computed. */
645 retval
= find_retval (return_bb
);
648 /* If there is a return_bb with no return value in function returning
649 value by reference, also make the split part return void, otherwise
650 we expansion would try to create a non-POD temporary, which is
652 if (return_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
653 && DECL_RESULT (current_function_decl
)
654 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
655 current
->split_part_set_retval
= false;
657 current
->split_part_set_retval
= true;
659 else if (is_gimple_min_invariant (retval
))
660 current
->split_part_set_retval
= false;
661 /* Special case is value returned by reference we record as if it was non-ssa
662 set to result_decl. */
663 else if (TREE_CODE (retval
) == SSA_NAME
664 && SSA_NAME_VAR (retval
)
665 && TREE_CODE (SSA_NAME_VAR (retval
)) == RESULT_DECL
666 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
667 current
->split_part_set_retval
668 = bitmap_bit_p (non_ssa_vars
, DECL_UID (SSA_NAME_VAR (retval
)));
669 else if (TREE_CODE (retval
) == SSA_NAME
)
670 current
->split_part_set_retval
671 = split_part_set_ssa_name_p (retval
, current
, return_bb
);
672 else if (TREE_CODE (retval
) == PARM_DECL
)
673 current
->split_part_set_retval
= false;
674 else if (VAR_P (retval
)
675 || TREE_CODE (retval
) == RESULT_DECL
)
676 current
->split_part_set_retval
677 = bitmap_bit_p (non_ssa_vars
, DECL_UID (retval
));
679 current
->split_part_set_retval
= true;
681 /* split_function fixes up at most one PHI non-virtual PHI node in return_bb,
682 for the return value. If there are other PHIs, give up. */
683 if (return_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
687 for (psi
= gsi_start_phis (return_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
688 if (!virtual_operand_p (gimple_phi_result (psi
.phi ()))
690 && current
->split_part_set_retval
691 && TREE_CODE (retval
) == SSA_NAME
692 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
))
693 && SSA_NAME_DEF_STMT (retval
) == psi
.phi ()))
695 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
697 " Refused: return bb has extra PHIs\n");
702 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
703 fprintf (dump_file
, " Accepted!\n");
705 /* At the moment chose split point with lowest count and that leaves
706 out smallest size of header.
707 In future we might re-consider this heuristics. */
708 if (!best_split_point
.split_bbs
709 || best_split_point
.count
711 || (best_split_point
.count
== current
->count
712 && best_split_point
.split_size
< current
->split_size
))
715 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
716 fprintf (dump_file
, " New best split point!\n");
717 if (best_split_point
.ssa_names_to_pass
)
719 BITMAP_FREE (best_split_point
.ssa_names_to_pass
);
720 BITMAP_FREE (best_split_point
.split_bbs
);
722 best_split_point
= *current
;
723 best_split_point
.ssa_names_to_pass
= BITMAP_ALLOC (NULL
);
724 bitmap_copy (best_split_point
.ssa_names_to_pass
,
725 current
->ssa_names_to_pass
);
726 best_split_point
.split_bbs
= BITMAP_ALLOC (NULL
);
727 bitmap_copy (best_split_point
.split_bbs
, current
->split_bbs
);
731 /* Return basic block containing RETURN statement. We allow basic blocks
735 but return_bb cannot be more complex than this (except for
736 -fsanitize=thread we allow TSAN_FUNC_EXIT () internal call in there).
737 If nothing is found, return the exit block.
739 When there are multiple RETURN statement, chose one with return value,
740 since that one is more likely shared by multiple code paths.
742 Return BB is special, because for function splitting it is the only
743 basic block that is duplicated in between header and split part of the
746 TODO: We might support multiple return blocks. */
749 find_return_bb (void)
752 basic_block return_bb
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
753 gimple_stmt_iterator bsi
;
754 bool found_return
= false;
755 tree retval
= NULL_TREE
;
757 if (!single_pred_p (EXIT_BLOCK_PTR_FOR_FN (cfun
)))
760 e
= single_pred_edge (EXIT_BLOCK_PTR_FOR_FN (cfun
));
761 for (bsi
= gsi_last_bb (e
->src
); !gsi_end_p (bsi
); gsi_prev (&bsi
))
763 gimple
*stmt
= gsi_stmt (bsi
);
764 if (gimple_code (stmt
) == GIMPLE_LABEL
765 || is_gimple_debug (stmt
)
766 || gimple_clobber_p (stmt
))
768 else if (gimple_code (stmt
) == GIMPLE_ASSIGN
770 && gimple_assign_single_p (stmt
)
771 && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt
),
772 current_function_decl
)
773 || is_gimple_min_invariant (gimple_assign_rhs1 (stmt
)))
774 && retval
== gimple_assign_lhs (stmt
))
776 else if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
779 retval
= gimple_return_retval (return_stmt
);
781 /* For -fsanitize=thread, allow also TSAN_FUNC_EXIT () in the return
783 else if ((flag_sanitize
& SANITIZE_THREAD
)
784 && gimple_call_internal_p (stmt
, IFN_TSAN_FUNC_EXIT
))
789 if (gsi_end_p (bsi
) && found_return
)
795 /* Given return basic block RETURN_BB, see where return value is really
798 find_retval (basic_block return_bb
)
800 gimple_stmt_iterator bsi
;
801 for (bsi
= gsi_start_bb (return_bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
802 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (gsi_stmt (bsi
)))
803 return gimple_return_retval (return_stmt
);
804 else if (gimple_code (gsi_stmt (bsi
)) == GIMPLE_ASSIGN
805 && !gimple_clobber_p (gsi_stmt (bsi
)))
806 return gimple_assign_rhs1 (gsi_stmt (bsi
));
810 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
811 variable, mark it as used in bitmap passed via DATA.
812 Return true when access to T prevents splitting the function. */
815 mark_nonssa_use (gimple
*, tree t
, tree
, void *data
)
817 t
= get_base_address (t
);
819 if (!t
|| is_gimple_reg (t
))
822 /* At present we can't pass non-SSA arguments to split function.
823 FIXME: this can be relaxed by passing references to arguments. */
824 if (TREE_CODE (t
) == PARM_DECL
)
826 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
828 "Cannot split: use of non-ssa function parameter.\n");
832 if ((VAR_P (t
) && auto_var_in_fn_p (t
, current_function_decl
))
833 || TREE_CODE (t
) == RESULT_DECL
834 || (TREE_CODE (t
) == LABEL_DECL
&& FORCED_LABEL (t
)))
835 bitmap_set_bit ((bitmap
)data
, DECL_UID (t
));
837 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
838 to pretend that the value pointed to is actual result decl. */
839 if ((TREE_CODE (t
) == MEM_REF
|| INDIRECT_REF_P (t
))
840 && TREE_CODE (TREE_OPERAND (t
, 0)) == SSA_NAME
841 && SSA_NAME_VAR (TREE_OPERAND (t
, 0))
842 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t
, 0))) == RESULT_DECL
843 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
845 bitmap_bit_p ((bitmap
)data
,
846 DECL_UID (DECL_RESULT (current_function_decl
)));
851 /* Compute local properties of basic block BB we collect when looking for
852 split points. We look for ssa defs and store them in SET_SSA_NAMES,
853 for ssa uses and store them in USED_SSA_NAMES and for any non-SSA automatic
854 vars stored in NON_SSA_VARS.
856 When BB has edge to RETURN_BB, collect uses in RETURN_BB too.
858 Return false when BB contains something that prevents it from being put into
862 visit_bb (basic_block bb
, basic_block return_bb
,
863 bitmap set_ssa_names
, bitmap used_ssa_names
,
868 bool can_split
= true;
870 for (gimple_stmt_iterator bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
);
873 gimple
*stmt
= gsi_stmt (bsi
);
878 if (is_gimple_debug (stmt
))
881 if (gimple_clobber_p (stmt
))
884 /* FIXME: We can split regions containing EH. We cannot however
885 split RESX, EH_DISPATCH and EH_POINTER referring to same region
886 into different partitions. This would require tracking of
887 EH regions and checking in consider_split_point if they
888 are not used elsewhere. */
889 if (gimple_code (stmt
) == GIMPLE_RESX
)
891 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
892 fprintf (dump_file
, "Cannot split: resx.\n");
895 if (gimple_code (stmt
) == GIMPLE_EH_DISPATCH
)
897 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
898 fprintf (dump_file
, "Cannot split: eh dispatch.\n");
902 /* Check builtins that prevent splitting. */
903 if (gimple_code (stmt
) == GIMPLE_CALL
904 && (decl
= gimple_call_fndecl (stmt
)) != NULL_TREE
905 && fndecl_built_in_p (decl
, BUILT_IN_NORMAL
))
906 switch (DECL_FUNCTION_CODE (decl
))
908 /* FIXME: once we will allow passing non-parm values to split part,
909 we need to be sure to handle correct builtin_stack_save and
910 builtin_stack_restore. At the moment we are safe; there is no
911 way to store builtin_stack_save result in non-SSA variable
912 since all calls to those are compiler generated. */
914 case BUILT_IN_APPLY_ARGS
:
915 case BUILT_IN_VA_START
:
916 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
918 "Cannot split: builtin_apply and va_start.\n");
921 case BUILT_IN_EH_POINTER
:
922 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
923 fprintf (dump_file
, "Cannot split: builtin_eh_pointer.\n");
930 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_DEF
)
931 bitmap_set_bit (set_ssa_names
, SSA_NAME_VERSION (op
));
932 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_USE
)
933 bitmap_set_bit (used_ssa_names
, SSA_NAME_VERSION (op
));
934 can_split
&= !walk_stmt_load_store_addr_ops (stmt
, non_ssa_vars
,
939 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
942 gphi
*stmt
= bsi
.phi ();
945 if (virtual_operand_p (gimple_phi_result (stmt
)))
947 bitmap_set_bit (set_ssa_names
,
948 SSA_NAME_VERSION (gimple_phi_result (stmt
)));
949 for (i
= 0; i
< gimple_phi_num_args (stmt
); i
++)
951 tree op
= gimple_phi_arg_def (stmt
, i
);
952 if (TREE_CODE (op
) == SSA_NAME
)
953 bitmap_set_bit (used_ssa_names
, SSA_NAME_VERSION (op
));
955 can_split
&= !walk_stmt_load_store_addr_ops (stmt
, non_ssa_vars
,
960 /* Record also uses coming from PHI operand in return BB. */
961 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
962 if (e
->dest
== return_bb
)
964 for (gphi_iterator bsi
= gsi_start_phis (return_bb
);
968 gphi
*stmt
= bsi
.phi ();
969 tree op
= gimple_phi_arg_def (stmt
, e
->dest_idx
);
971 if (virtual_operand_p (gimple_phi_result (stmt
)))
973 if (TREE_CODE (op
) == SSA_NAME
)
974 bitmap_set_bit (used_ssa_names
, SSA_NAME_VERSION (op
));
976 can_split
&= !mark_nonssa_use (stmt
, op
, op
, non_ssa_vars
);
982 /* Stack entry for recursive DFS walk in find_split_point. */
987 /* Basic block we are examining. */
990 /* SSA names set and used by the BB and all BBs reachable
991 from it via DFS walk. */
992 bitmap set_ssa_names
, used_ssa_names
;
995 /* All BBS visited from this BB via DFS walk. */
998 /* Last examined edge in DFS walk. Since we walk unoriented graph,
999 the value is up to sum of incoming and outgoing edges of BB. */
1000 unsigned int edge_num
;
1002 /* Stack entry index of earliest BB reachable from current BB
1003 or any BB visited later in DFS walk. */
1006 /* Overall time and size of all BBs reached from this BB in DFS walk. */
1010 /* When false we cannot split on this BB. */
1015 /* Find all articulations and call consider_split on them.
1016 OVERALL_TIME and OVERALL_SIZE is time and size of the function.
1018 We perform basic algorithm for finding an articulation in a graph
1019 created from CFG by considering it to be an unoriented graph.
1021 The articulation is discovered via DFS walk. We collect earliest
1022 basic block on stack that is reachable via backward edge. Articulation
1023 is any basic block such that there is no backward edge bypassing it.
1024 To reduce stack usage we maintain heap allocated stack in STACK vector.
1025 AUX pointer of BB is set to index it appears in the stack or -1 once
1026 it is visited and popped off the stack.
1028 The algorithm finds articulation after visiting the whole component
1029 reachable by it. This makes it convenient to collect information about
1030 the component used by consider_split. */
1033 find_split_points (basic_block return_bb
, sreal overall_time
, int overall_size
)
1036 vec
<stack_entry
> stack
= vNULL
;
1038 class split_point current
;
1040 current
.header_time
= overall_time
;
1041 current
.header_size
= overall_size
;
1042 current
.split_time
= 0;
1043 current
.split_size
= 0;
1044 current
.ssa_names_to_pass
= BITMAP_ALLOC (NULL
);
1046 first
.bb
= ENTRY_BLOCK_PTR_FOR_FN (cfun
);
1048 first
.overall_time
= 0;
1049 first
.overall_size
= 0;
1050 first
.earliest
= INT_MAX
;
1051 first
.set_ssa_names
= 0;
1052 first
.used_ssa_names
= 0;
1053 first
.non_ssa_vars
= 0;
1054 first
.bbs_visited
= 0;
1055 first
.can_split
= false;
1056 stack
.safe_push (first
);
1057 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->aux
= (void *)(intptr_t)-1;
1059 while (!stack
.is_empty ())
1061 stack_entry
*entry
= &stack
.last ();
1063 /* We are walking an acyclic graph, so edge_num counts
1064 succ and pred edges together. However when considering
1065 articulation, we want to have processed everything reachable
1066 from articulation but nothing that reaches into it. */
1067 if (entry
->edge_num
== EDGE_COUNT (entry
->bb
->succs
)
1068 && entry
->bb
!= ENTRY_BLOCK_PTR_FOR_FN (cfun
))
1070 int pos
= stack
.length ();
1071 entry
->can_split
&= visit_bb (entry
->bb
, return_bb
,
1072 entry
->set_ssa_names
,
1073 entry
->used_ssa_names
,
1074 entry
->non_ssa_vars
);
1075 if (pos
<= entry
->earliest
&& !entry
->can_split
1076 && dump_file
&& (dump_flags
& TDF_DETAILS
))
1078 "found articulation at bb %i but cannot split\n",
1080 if (pos
<= entry
->earliest
&& entry
->can_split
)
1082 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1083 fprintf (dump_file
, "found articulation at bb %i\n",
1085 current
.entry_bb
= entry
->bb
;
1086 current
.ssa_names_to_pass
= BITMAP_ALLOC (NULL
);
1087 bitmap_and_compl (current
.ssa_names_to_pass
,
1088 entry
->used_ssa_names
, entry
->set_ssa_names
);
1089 current
.header_time
= overall_time
- entry
->overall_time
;
1090 current
.header_size
= overall_size
- entry
->overall_size
;
1091 current
.split_time
= entry
->overall_time
;
1092 current
.split_size
= entry
->overall_size
;
1093 current
.split_bbs
= entry
->bbs_visited
;
1094 consider_split (¤t
, entry
->non_ssa_vars
, return_bb
);
1095 BITMAP_FREE (current
.ssa_names_to_pass
);
1098 /* Do actual DFS walk. */
1100 < (EDGE_COUNT (entry
->bb
->succs
)
1101 + EDGE_COUNT (entry
->bb
->preds
)))
1105 if (entry
->edge_num
< EDGE_COUNT (entry
->bb
->succs
))
1107 e
= EDGE_SUCC (entry
->bb
, entry
->edge_num
);
1112 e
= EDGE_PRED (entry
->bb
, entry
->edge_num
1113 - EDGE_COUNT (entry
->bb
->succs
));
1119 /* New BB to visit, push it to the stack. */
1120 if (dest
!= return_bb
&& dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
1123 stack_entry new_entry
;
1125 new_entry
.bb
= dest
;
1126 new_entry
.edge_num
= 0;
1127 new_entry
.overall_time
1128 = bb_info_vec
[dest
->index
].time
;
1129 new_entry
.overall_size
1130 = bb_info_vec
[dest
->index
].size
;
1131 new_entry
.earliest
= INT_MAX
;
1132 new_entry
.set_ssa_names
= BITMAP_ALLOC (NULL
);
1133 new_entry
.used_ssa_names
= BITMAP_ALLOC (NULL
);
1134 new_entry
.bbs_visited
= BITMAP_ALLOC (NULL
);
1135 new_entry
.non_ssa_vars
= BITMAP_ALLOC (NULL
);
1136 new_entry
.can_split
= true;
1137 bitmap_set_bit (new_entry
.bbs_visited
, dest
->index
);
1138 stack
.safe_push (new_entry
);
1139 dest
->aux
= (void *)(intptr_t)stack
.length ();
1141 /* Back edge found, record the earliest point. */
1142 else if ((intptr_t)dest
->aux
> 0
1143 && (intptr_t)dest
->aux
< entry
->earliest
)
1144 entry
->earliest
= (intptr_t)dest
->aux
;
1146 /* We are done with examining the edges. Pop off the value from stack
1147 and merge stuff we accumulate during the walk. */
1148 else if (entry
->bb
!= ENTRY_BLOCK_PTR_FOR_FN (cfun
))
1150 stack_entry
*prev
= &stack
[stack
.length () - 2];
1152 entry
->bb
->aux
= (void *)(intptr_t)-1;
1153 prev
->can_split
&= entry
->can_split
;
1154 if (prev
->set_ssa_names
)
1156 bitmap_ior_into (prev
->set_ssa_names
, entry
->set_ssa_names
);
1157 bitmap_ior_into (prev
->used_ssa_names
, entry
->used_ssa_names
);
1158 bitmap_ior_into (prev
->bbs_visited
, entry
->bbs_visited
);
1159 bitmap_ior_into (prev
->non_ssa_vars
, entry
->non_ssa_vars
);
1161 if (prev
->earliest
> entry
->earliest
)
1162 prev
->earliest
= entry
->earliest
;
1163 prev
->overall_time
+= entry
->overall_time
;
1164 prev
->overall_size
+= entry
->overall_size
;
1165 BITMAP_FREE (entry
->set_ssa_names
);
1166 BITMAP_FREE (entry
->used_ssa_names
);
1167 BITMAP_FREE (entry
->bbs_visited
);
1168 BITMAP_FREE (entry
->non_ssa_vars
);
1174 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->aux
= NULL
;
1175 FOR_EACH_BB_FN (bb
, cfun
)
1178 BITMAP_FREE (current
.ssa_names_to_pass
);
1181 /* Split function at SPLIT_POINT. */
1184 split_function (basic_block return_bb
, class split_point
*split_point
,
1185 bool add_tsan_func_exit
)
1187 vec
<tree
> args_to_pass
= vNULL
;
1188 bitmap args_to_skip
;
1191 cgraph_node
*node
, *cur_node
= cgraph_node::get (current_function_decl
);
1192 basic_block call_bb
;
1193 gcall
*call
, *tsan_func_exit_call
= NULL
;
1196 tree retval
= NULL
, real_retval
= NULL
;
1197 gimple
*last_stmt
= NULL
;
1203 fprintf (dump_file
, "\n\nSplitting function at:\n");
1204 dump_split_point (dump_file
, split_point
);
1207 if (cur_node
->local
.can_change_signature
)
1208 args_to_skip
= BITMAP_ALLOC (NULL
);
1210 args_to_skip
= NULL
;
1212 /* Collect the parameters of new function and args_to_skip bitmap. */
1213 for (parm
= DECL_ARGUMENTS (current_function_decl
);
1214 parm
; parm
= DECL_CHAIN (parm
), num
++)
1216 && (!is_gimple_reg (parm
)
1217 || (ddef
= ssa_default_def (cfun
, parm
)) == NULL_TREE
1218 || !bitmap_bit_p (split_point
->ssa_names_to_pass
,
1219 SSA_NAME_VERSION (ddef
))))
1220 bitmap_set_bit (args_to_skip
, num
);
1223 /* This parm might not have been used up to now, but is going to be
1224 used, hence register it. */
1225 if (is_gimple_reg (parm
))
1226 arg
= get_or_create_ssa_default_def (cfun
, parm
);
1230 if (!useless_type_conversion_p (DECL_ARG_TYPE (parm
), TREE_TYPE (arg
)))
1231 arg
= fold_convert (DECL_ARG_TYPE (parm
), arg
);
1232 args_to_pass
.safe_push (arg
);
1235 /* See if the split function will return. */
1236 bool split_part_return_p
= false;
1237 FOR_EACH_EDGE (e
, ei
, return_bb
->preds
)
1239 if (bitmap_bit_p (split_point
->split_bbs
, e
->src
->index
))
1240 split_part_return_p
= true;
1243 /* Add return block to what will become the split function.
1244 We do not return; no return block is needed. */
1245 if (!split_part_return_p
)
1247 /* We have no return block, so nothing is needed. */
1248 else if (return_bb
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1250 /* When we do not want to return value, we need to construct
1251 new return block with empty return statement.
1252 FIXME: Once we are able to change return type, we should change function
1253 to return void instead of just outputting function with undefined return
1254 value. For structures this affects quality of codegen. */
1255 else if ((retval
= find_retval (return_bb
))
1256 && !split_point
->split_part_set_retval
)
1258 bool redirected
= true;
1259 basic_block new_return_bb
= create_basic_block (NULL
, 0, return_bb
);
1260 gimple_stmt_iterator gsi
= gsi_start_bb (new_return_bb
);
1261 gsi_insert_after (&gsi
, gimple_build_return (NULL
), GSI_NEW_STMT
);
1262 new_return_bb
->count
= profile_count::zero ();
1266 FOR_EACH_EDGE (e
, ei
, return_bb
->preds
)
1267 if (bitmap_bit_p (split_point
->split_bbs
, e
->src
->index
))
1269 new_return_bb
->count
+= e
->count ();
1270 redirect_edge_and_branch (e
, new_return_bb
);
1275 e
= make_single_succ_edge (new_return_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1276 add_bb_to_loop (new_return_bb
, current_loops
->tree_root
);
1277 bitmap_set_bit (split_point
->split_bbs
, new_return_bb
->index
);
1279 /* When we pass around the value, use existing return block. */
1281 bitmap_set_bit (split_point
->split_bbs
, return_bb
->index
);
1283 /* If RETURN_BB has virtual operand PHIs, they must be removed and the
1284 virtual operand marked for renaming as we change the CFG in a way that
1285 tree-inline is not able to compensate for.
1287 Note this can happen whether or not we have a return value. If we have
1288 a return value, then RETURN_BB may have PHIs for real operands too. */
1289 if (return_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
1292 for (gphi_iterator gsi
= gsi_start_phis (return_bb
);
1295 gphi
*stmt
= gsi
.phi ();
1296 if (!virtual_operand_p (gimple_phi_result (stmt
)))
1301 mark_virtual_phi_result_for_renaming (stmt
);
1302 remove_phi_node (&gsi
, true);
1305 /* In reality we have to rename the reaching definition of the
1306 virtual operand at return_bb as we will eventually release it
1307 when we remove the code region we outlined.
1308 So we have to rename all immediate virtual uses of that region
1309 if we didn't see a PHI definition yet. */
1310 /* ??? In real reality we want to set the reaching vdef of the
1311 entry of the SESE region as the vuse of the call and the reaching
1312 vdef of the exit of the SESE region as the vdef of the call. */
1314 for (gimple_stmt_iterator gsi
= gsi_start_bb (return_bb
);
1318 gimple
*stmt
= gsi_stmt (gsi
);
1319 if (gimple_vuse (stmt
))
1321 gimple_set_vuse (stmt
, NULL_TREE
);
1324 if (gimple_vdef (stmt
))
1329 /* Now create the actual clone. */
1330 cgraph_edge::rebuild_edges ();
1331 node
= cur_node
->create_version_clone_with_body
1332 (vNULL
, NULL
, args_to_skip
,
1333 !split_part_return_p
|| !split_point
->split_part_set_retval
,
1334 split_point
->split_bbs
, split_point
->entry_bb
, "part");
1336 node
->split_part
= true;
1338 if (cur_node
->same_comdat_group
)
1340 /* TODO: call is versionable if we make sure that all
1341 callers are inside of a comdat group. */
1342 cur_node
->calls_comdat_local
= 1;
1343 node
->add_to_same_comdat_group (cur_node
);
1347 /* Let's take a time profile for splitted function. */
1348 node
->tp_first_run
= cur_node
->tp_first_run
+ 1;
1350 /* For usual cloning it is enough to clear builtin only when signature
1351 changes. For partial inlining we however cannot expect the part
1352 of builtin implementation to have same semantic as the whole. */
1353 if (fndecl_built_in_p (node
->decl
))
1355 DECL_BUILT_IN_CLASS (node
->decl
) = NOT_BUILT_IN
;
1356 DECL_FUNCTION_CODE (node
->decl
) = (enum built_in_function
) 0;
1359 /* If return_bb contains any clobbers that refer to SSA_NAMEs
1360 set in the split part, remove them. Also reset debug stmts that
1361 refer to SSA_NAMEs set in the split part. */
1362 if (return_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
1364 gimple_stmt_iterator gsi
= gsi_start_bb (return_bb
);
1365 while (!gsi_end_p (gsi
))
1369 gimple
*stmt
= gsi_stmt (gsi
);
1370 bool remove
= false;
1371 if (gimple_clobber_p (stmt
) || is_gimple_debug (stmt
))
1372 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_USE
)
1374 basic_block bb
= gimple_bb (SSA_NAME_DEF_STMT (op
));
1378 && bitmap_bit_p (split_point
->split_bbs
, bb
->index
))
1380 if (is_gimple_debug (stmt
))
1382 gimple_debug_bind_reset_value (stmt
);
1391 gsi_remove (&gsi
, true);
1397 /* If the original function is declared inline, there is no point in issuing
1398 a warning for the non-inlinable part. */
1399 DECL_NO_INLINE_WARNING_P (node
->decl
) = 1;
1400 cur_node
->remove_callees ();
1401 cur_node
->remove_all_references ();
1402 if (!split_part_return_p
)
1403 TREE_THIS_VOLATILE (node
->decl
) = 1;
1405 dump_function_to_file (node
->decl
, dump_file
, dump_flags
);
1407 /* Create the basic block we place call into. It is the entry basic block
1408 split after last label. */
1409 call_bb
= split_point
->entry_bb
;
1410 for (gimple_stmt_iterator gsi
= gsi_start_bb (call_bb
); !gsi_end_p (gsi
);)
1411 if (gimple_code (gsi_stmt (gsi
)) == GIMPLE_LABEL
)
1413 last_stmt
= gsi_stmt (gsi
);
1418 call_bb
->count
= split_point
->count
;
1419 e
= split_block (split_point
->entry_bb
, last_stmt
);
1422 /* Produce the call statement. */
1423 gimple_stmt_iterator gsi
= gsi_last_bb (call_bb
);
1424 FOR_EACH_VEC_ELT (args_to_pass
, i
, arg
)
1425 if (!is_gimple_val (arg
))
1427 arg
= force_gimple_operand_gsi (&gsi
, arg
, true, NULL_TREE
,
1428 false, GSI_CONTINUE_LINKING
);
1429 args_to_pass
[i
] = arg
;
1431 call
= gimple_build_call_vec (node
->decl
, args_to_pass
);
1432 gimple_set_block (call
, DECL_INITIAL (current_function_decl
));
1433 args_to_pass
.release ();
1435 /* For optimized away parameters, add on the caller side
1437 DEBUG D#X => parm_Y(D)
1438 stmts and associate D#X with parm in decl_debug_args_lookup
1439 vector to say for debug info that if parameter parm had been passed,
1440 it would have value parm_Y(D). */
1443 vec
<tree
, va_gc
> **debug_args
= NULL
;
1444 unsigned i
= 0, len
= 0;
1445 if (MAY_HAVE_DEBUG_BIND_STMTS
)
1447 debug_args
= decl_debug_args_lookup (node
->decl
);
1449 len
= vec_safe_length (*debug_args
);
1451 for (parm
= DECL_ARGUMENTS (current_function_decl
), num
= 0;
1452 parm
; parm
= DECL_CHAIN (parm
), num
++)
1453 if (bitmap_bit_p (args_to_skip
, num
) && is_gimple_reg (parm
))
1458 /* This needs to be done even without
1459 MAY_HAVE_DEBUG_BIND_STMTS, otherwise if it didn't exist
1460 before, we'd end up with different SSA_NAME_VERSIONs
1461 between -g and -g0. */
1462 arg
= get_or_create_ssa_default_def (cfun
, parm
);
1463 if (!MAY_HAVE_DEBUG_BIND_STMTS
|| debug_args
== NULL
)
1466 while (i
< len
&& (**debug_args
)[i
] != DECL_ORIGIN (parm
))
1470 ddecl
= (**debug_args
)[i
+ 1];
1472 = gimple_build_debug_bind (ddecl
, unshare_expr (arg
), call
);
1473 gsi_insert_after (&gsi
, def_temp
, GSI_NEW_STMT
);
1477 /* We avoid address being taken on any variable used by split part,
1478 so return slot optimization is always possible. Moreover this is
1479 required to make DECL_BY_REFERENCE work. */
1480 if (aggregate_value_p (DECL_RESULT (current_function_decl
),
1481 TREE_TYPE (current_function_decl
))
1482 && (!is_gimple_reg_type (TREE_TYPE (DECL_RESULT (current_function_decl
)))
1483 || DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
))))
1484 gimple_call_set_return_slot_opt (call
, true);
1486 if (add_tsan_func_exit
)
1487 tsan_func_exit_call
= gimple_build_call_internal (IFN_TSAN_FUNC_EXIT
, 0);
1489 /* Update return value. This is bit tricky. When we do not return,
1490 do nothing. When we return we might need to update return_bb
1491 or produce a new return statement. */
1492 if (!split_part_return_p
)
1494 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1495 if (tsan_func_exit_call
)
1496 gsi_insert_after (&gsi
, tsan_func_exit_call
, GSI_NEW_STMT
);
1500 e
= make_single_succ_edge (call_bb
, return_bb
,
1501 return_bb
== EXIT_BLOCK_PTR_FOR_FN (cfun
)
1502 ? 0 : EDGE_FALLTHRU
);
1504 /* If there is return basic block, see what value we need to store
1505 return value into and put call just before it. */
1506 if (return_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
1508 real_retval
= retval
;
1509 if (real_retval
&& split_point
->split_part_set_retval
)
1513 /* See if we need new SSA_NAME for the result.
1514 When DECL_BY_REFERENCE is true, retval is actually pointer to
1515 return value and it is constant in whole function. */
1516 if (TREE_CODE (retval
) == SSA_NAME
1517 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
1519 retval
= copy_ssa_name (retval
, call
);
1521 /* See if there is PHI defining return value. */
1522 for (psi
= gsi_start_phis (return_bb
);
1523 !gsi_end_p (psi
); gsi_next (&psi
))
1524 if (!virtual_operand_p (gimple_phi_result (psi
.phi ())))
1527 /* When there is PHI, just update its value. */
1528 if (TREE_CODE (retval
) == SSA_NAME
1529 && !gsi_end_p (psi
))
1530 add_phi_arg (psi
.phi (), retval
, e
, UNKNOWN_LOCATION
);
1531 /* Otherwise update the return BB itself.
1532 find_return_bb allows at most one assignment to return value,
1533 so update first statement. */
1536 gimple_stmt_iterator bsi
;
1537 for (bsi
= gsi_start_bb (return_bb
); !gsi_end_p (bsi
);
1539 if (greturn
*return_stmt
1540 = dyn_cast
<greturn
*> (gsi_stmt (bsi
)))
1542 gimple_return_set_retval (return_stmt
, retval
);
1545 else if (gimple_code (gsi_stmt (bsi
)) == GIMPLE_ASSIGN
1546 && !gimple_clobber_p (gsi_stmt (bsi
)))
1548 gimple_assign_set_rhs1 (gsi_stmt (bsi
), retval
);
1551 update_stmt (gsi_stmt (bsi
));
1552 /* Also adjust clobbers and debug stmts in return_bb. */
1553 for (bsi
= gsi_start_bb (return_bb
); !gsi_end_p (bsi
);
1556 gimple
*stmt
= gsi_stmt (bsi
);
1557 if (gimple_clobber_p (stmt
)
1558 || is_gimple_debug (stmt
))
1561 use_operand_p use_p
;
1562 bool update
= false;
1563 FOR_EACH_SSA_USE_OPERAND (use_p
, stmt
, iter
,
1565 if (USE_FROM_PTR (use_p
) == real_retval
)
1567 SET_USE (use_p
, retval
);
1576 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
1578 gimple_call_set_lhs (call
, build_simple_mem_ref (retval
));
1579 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1584 restype
= TREE_TYPE (DECL_RESULT (current_function_decl
));
1585 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1586 if (!useless_type_conversion_p (TREE_TYPE (retval
), restype
))
1589 tree tem
= create_tmp_reg (restype
);
1590 tem
= make_ssa_name (tem
, call
);
1591 cpy
= gimple_build_assign (retval
, NOP_EXPR
, tem
);
1592 gsi_insert_after (&gsi
, cpy
, GSI_NEW_STMT
);
1595 gimple_call_set_lhs (call
, retval
);
1600 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1601 if (tsan_func_exit_call
)
1602 gsi_insert_after (&gsi
, tsan_func_exit_call
, GSI_NEW_STMT
);
1604 /* We don't use return block (there is either no return in function or
1605 multiple of them). So create new basic block with return statement.
1610 if (split_point
->split_part_set_retval
1611 && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl
))))
1613 retval
= DECL_RESULT (current_function_decl
);
1615 /* We use temporary register to hold value when aggregate_value_p
1616 is false. Similarly for DECL_BY_REFERENCE we must avoid extra
1618 if (!aggregate_value_p (retval
, TREE_TYPE (current_function_decl
))
1619 && !DECL_BY_REFERENCE (retval
))
1620 retval
= create_tmp_reg (TREE_TYPE (retval
));
1621 if (is_gimple_reg (retval
))
1623 /* When returning by reference, there is only one SSA name
1624 assigned to RESULT_DECL (that is pointer to return value).
1625 Look it up or create new one if it is missing. */
1626 if (DECL_BY_REFERENCE (retval
))
1627 retval
= get_or_create_ssa_default_def (cfun
, retval
);
1628 /* Otherwise produce new SSA name for return value. */
1630 retval
= make_ssa_name (retval
, call
);
1632 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
1633 gimple_call_set_lhs (call
, build_simple_mem_ref (retval
));
1635 gimple_call_set_lhs (call
, retval
);
1636 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1640 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1642 && is_gimple_reg_type (TREE_TYPE (retval
))
1643 && !is_gimple_val (retval
))
1646 = gimple_build_assign (make_ssa_name (TREE_TYPE (retval
)),
1648 retval
= gimple_assign_lhs (g
);
1649 gsi_insert_after (&gsi
, g
, GSI_NEW_STMT
);
1652 if (tsan_func_exit_call
)
1653 gsi_insert_after (&gsi
, tsan_func_exit_call
, GSI_NEW_STMT
);
1654 ret
= gimple_build_return (retval
);
1655 gsi_insert_after (&gsi
, ret
, GSI_NEW_STMT
);
1658 free_dominance_info (CDI_DOMINATORS
);
1659 free_dominance_info (CDI_POST_DOMINATORS
);
1660 compute_fn_summary (node
, true);
1663 /* Execute function splitting pass. */
1666 execute_split_functions (void)
1668 gimple_stmt_iterator bsi
;
1670 sreal overall_time
= 0;
1671 int overall_size
= 0;
1673 struct cgraph_node
*node
= cgraph_node::get (current_function_decl
);
1675 if (flags_from_decl_or_type (current_function_decl
)
1676 & (ECF_NORETURN
|ECF_MALLOC
))
1679 fprintf (dump_file
, "Not splitting: noreturn/malloc function.\n");
1682 if (MAIN_NAME_P (DECL_NAME (current_function_decl
)))
1685 fprintf (dump_file
, "Not splitting: main function.\n");
1688 if (node
->frequency
== NODE_FREQUENCY_UNLIKELY_EXECUTED
)
1691 fprintf (dump_file
, "Not splitting: function is unlikely executed.\n");
1694 /* This can be relaxed; function might become inlinable after splitting
1695 away the uninlinable part. */
1696 if (ipa_fn_summaries
1697 && ipa_fn_summaries
->get (node
)
1698 && !ipa_fn_summaries
->get (node
)->inlinable
)
1701 fprintf (dump_file
, "Not splitting: not inlinable.\n");
1704 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
1707 fprintf (dump_file
, "Not splitting: disregarding inline limits.\n");
1710 /* This can be relaxed; most of versioning tests actually prevents
1712 if (!tree_versionable_function_p (current_function_decl
))
1715 fprintf (dump_file
, "Not splitting: not versionable.\n");
1718 /* FIXME: we could support this. */
1719 if (DECL_STRUCT_FUNCTION (current_function_decl
)->static_chain_decl
)
1722 fprintf (dump_file
, "Not splitting: nested function.\n");
1726 /* See if it makes sense to try to split.
1727 It makes sense to split if we inline, that is if we have direct calls to
1728 handle or direct calls are possibly going to appear as result of indirect
1729 inlining or LTO. Also handle -fprofile-generate as LTO to allow non-LTO
1730 training for LTO -fprofile-use build.
1732 Note that we are not completely conservative about disqualifying functions
1733 called once. It is possible that the caller is called more then once and
1734 then inlining would still benefit. */
1736 /* Local functions called once will be completely inlined most of time. */
1737 || (!node
->callers
->next_caller
&& node
->local
.local
))
1738 && !node
->address_taken
1739 && !node
->has_aliases_p ()
1740 && (!flag_lto
|| !node
->externally_visible
))
1743 fprintf (dump_file
, "Not splitting: not called directly "
1744 "or called once.\n");
1748 /* FIXME: We can actually split if splitting reduces call overhead. */
1749 if (!flag_inline_small_functions
1750 && !DECL_DECLARED_INLINE_P (current_function_decl
))
1753 fprintf (dump_file
, "Not splitting: not autoinlining and function"
1754 " is not inline.\n");
1758 if (lookup_attribute ("noinline", DECL_ATTRIBUTES (current_function_decl
)))
1761 fprintf (dump_file
, "Not splitting: function is noinline.\n");
1764 if (lookup_attribute ("section", DECL_ATTRIBUTES (current_function_decl
)))
1767 fprintf (dump_file
, "Not splitting: function is in user defined "
1772 /* We enforce splitting after loop headers when profile info is not
1774 if (profile_status_for_fn (cfun
) != PROFILE_READ
)
1775 mark_dfs_back_edges ();
1777 /* Initialize bitmap to track forbidden calls. */
1778 forbidden_dominators
= BITMAP_ALLOC (NULL
);
1779 calculate_dominance_info (CDI_DOMINATORS
);
1781 /* Compute local info about basic blocks and determine function size/time. */
1782 bb_info_vec
.safe_grow_cleared (last_basic_block_for_fn (cfun
) + 1);
1783 best_split_point
.split_bbs
= NULL
;
1784 basic_block return_bb
= find_return_bb ();
1785 int tsan_exit_found
= -1;
1786 FOR_EACH_BB_FN (bb
, cfun
)
1790 sreal freq
= bb
->count
.to_sreal_scale
1791 (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
);
1793 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1794 fprintf (dump_file
, "Basic block %i\n", bb
->index
);
1796 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1800 gimple
*stmt
= gsi_stmt (bsi
);
1802 this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
1803 this_time
= (sreal
)estimate_num_insns (stmt
, &eni_time_weights
)
1807 check_forbidden_calls (stmt
);
1809 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1811 fprintf (dump_file
, " freq:%4.2f size:%3i time:%4.2f ",
1812 freq
.to_double (), this_size
, this_time
.to_double ());
1813 print_gimple_stmt (dump_file
, stmt
, 0);
1816 if ((flag_sanitize
& SANITIZE_THREAD
)
1817 && gimple_call_internal_p (stmt
, IFN_TSAN_FUNC_EXIT
))
1819 /* We handle TSAN_FUNC_EXIT for splitting either in the
1820 return_bb, or in its immediate predecessors. */
1821 if ((bb
!= return_bb
&& !find_edge (bb
, return_bb
))
1822 || (tsan_exit_found
!= -1
1823 && tsan_exit_found
!= (bb
!= return_bb
)))
1826 fprintf (dump_file
, "Not splitting: TSAN_FUNC_EXIT"
1827 " in unexpected basic block.\n");
1828 BITMAP_FREE (forbidden_dominators
);
1829 bb_info_vec
.release ();
1832 tsan_exit_found
= bb
!= return_bb
;
1835 overall_time
+= time
;
1836 overall_size
+= size
;
1837 bb_info_vec
[bb
->index
].time
= time
;
1838 bb_info_vec
[bb
->index
].size
= size
;
1840 find_split_points (return_bb
, overall_time
, overall_size
);
1841 if (best_split_point
.split_bbs
)
1843 split_function (return_bb
, &best_split_point
, tsan_exit_found
== 1);
1844 BITMAP_FREE (best_split_point
.ssa_names_to_pass
);
1845 BITMAP_FREE (best_split_point
.split_bbs
);
1846 todo
= TODO_update_ssa
| TODO_cleanup_cfg
;
1848 BITMAP_FREE (forbidden_dominators
);
1849 bb_info_vec
.release ();
1855 const pass_data pass_data_split_functions
=
1857 GIMPLE_PASS
, /* type */
1858 "fnsplit", /* name */
1859 OPTGROUP_NONE
, /* optinfo_flags */
1860 TV_IPA_FNSPLIT
, /* tv_id */
1861 PROP_cfg
, /* properties_required */
1862 0, /* properties_provided */
1863 0, /* properties_destroyed */
1864 0, /* todo_flags_start */
1865 0, /* todo_flags_finish */
1868 class pass_split_functions
: public gimple_opt_pass
1871 pass_split_functions (gcc::context
*ctxt
)
1872 : gimple_opt_pass (pass_data_split_functions
, ctxt
)
1875 /* opt_pass methods: */
1876 virtual bool gate (function
*);
1877 virtual unsigned int execute (function
*)
1879 return execute_split_functions ();
1882 }; // class pass_split_functions
1885 pass_split_functions::gate (function
*)
1887 /* When doing profile feedback, we want to execute the pass after profiling
1888 is read. So disable one in early optimization. */
1889 return (flag_partial_inlining
1890 && !profile_arc_flag
&& !flag_branch_probabilities
);
1896 make_pass_split_functions (gcc::context
*ctxt
)
1898 return new pass_split_functions (ctxt
);
1901 /* Execute function splitting pass. */
1904 execute_feedback_split_functions (void)
1906 unsigned int retval
= execute_split_functions ();
1908 retval
|= TODO_rebuild_cgraph_edges
;
1914 const pass_data pass_data_feedback_split_functions
=
1916 GIMPLE_PASS
, /* type */
1917 "feedback_fnsplit", /* name */
1918 OPTGROUP_NONE
, /* optinfo_flags */
1919 TV_IPA_FNSPLIT
, /* tv_id */
1920 PROP_cfg
, /* properties_required */
1921 0, /* properties_provided */
1922 0, /* properties_destroyed */
1923 0, /* todo_flags_start */
1924 0, /* todo_flags_finish */
1927 class pass_feedback_split_functions
: public gimple_opt_pass
1930 pass_feedback_split_functions (gcc::context
*ctxt
)
1931 : gimple_opt_pass (pass_data_feedback_split_functions
, ctxt
)
1934 /* opt_pass methods: */
1935 virtual bool gate (function
*);
1936 virtual unsigned int execute (function
*)
1938 return execute_feedback_split_functions ();
1941 }; // class pass_feedback_split_functions
1944 pass_feedback_split_functions::gate (function
*)
1946 /* We don't need to split when profiling at all, we are producing
1947 lousy code anyway. */
1948 return (flag_partial_inlining
1949 && flag_branch_probabilities
);
1955 make_pass_feedback_split_functions (gcc::context
*ctxt
)
1957 return new pass_feedback_split_functions (ctxt
);