1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 The implementation of inliner is organized as follows:
25 inlining heuristics limits
27 can_inline_edge_p allow to check that particular inlining is allowed
28 by the limits specified by user (allowed function growth, growth and so
31 Functions are inlined when it is obvious the result is profitable (such
32 as functions called once or when inlining reduce code size).
33 In addition to that we perform inlining of small functions and recursive
38 The inliner itself is split into two passes:
42 Simple local inlining pass inlining callees into current function.
43 This pass makes no use of whole unit analysis and thus it can do only
44 very simple decisions based on local properties.
46 The strength of the pass is that it is run in topological order
47 (reverse postorder) on the callgraph. Functions are converted into SSA
48 form just before this pass and optimized subsequently. As a result, the
49 callees of the function seen by the early inliner was already optimized
50 and results of early inlining adds a lot of optimization opportunities
51 for the local optimization.
53 The pass handle the obvious inlining decisions within the compilation
54 unit - inlining auto inline functions, inlining for size and
57 main strength of the pass is the ability to eliminate abstraction
58 penalty in C++ code (via combination of inlining and early
59 optimization) and thus improve quality of analysis done by real IPA
62 Because of lack of whole unit knowledge, the pass can not really make
63 good code size/performance tradeoffs. It however does very simple
64 speculative inlining allowing code size to grow by
65 EARLY_INLINING_INSNS when callee is leaf function. In this case the
66 optimizations performed later are very likely to eliminate the cost.
70 This is the real inliner able to handle inlining with whole program
71 knowledge. It performs following steps:
73 1) inlining of small functions. This is implemented by greedy
74 algorithm ordering all inlinable cgraph edges by their badness and
75 inlining them in this order as long as inline limits allows doing so.
77 This heuristics is not very good on inlining recursive calls. Recursive
78 calls can be inlined with results similar to loop unrolling. To do so,
79 special purpose recursive inliner is executed on function when
80 recursive edge is met as viable candidate.
82 2) Unreachable functions are removed from callgraph. Inlining leads
83 to devirtualization and other modification of callgraph so functions
84 may become unreachable during the process. Also functions declared as
85 extern inline or virtual functions are removed, since after inlining
86 we no longer need the offline bodies.
88 3) Functions called once and not exported from the unit are inlined.
89 This should almost always lead to reduction of code size by eliminating
90 the need for offline copy of the function. */
94 #include "coretypes.h"
97 #include "tree-inline.h"
98 #include "langhooks.h"
101 #include "diagnostic.h"
102 #include "gimple-pretty-print.h"
106 #include "tree-pass.h"
107 #include "coverage.h"
110 #include "tree-ssa.h"
111 #include "ipa-prop.h"
114 #include "ipa-inline.h"
115 #include "ipa-utils.h"
118 /* Statistics we collect about inlining algorithm. */
119 static int overall_size
;
120 static gcov_type max_count
;
121 static sreal max_count_real
, max_relbenefit_real
, half_int_min_real
;
123 /* Return false when inlining edge E would lead to violating
124 limits on function unit growth or stack usage growth.
126 The relative function body growth limit is present generally
127 to avoid problems with non-linear behavior of the compiler.
128 To allow inlining huge functions into tiny wrapper, the limit
129 is always based on the bigger of the two functions considered.
131 For stack growth limits we always base the growth in stack usage
132 of the callers. We want to prevent applications from segfaulting
133 on stack overflow when functions with huge stack frames gets
137 caller_growth_limits (struct cgraph_edge
*e
)
139 struct cgraph_node
*to
= e
->caller
;
140 struct cgraph_node
*what
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
143 HOST_WIDE_INT stack_size_limit
= 0, inlined_stack
;
144 struct inline_summary
*info
, *what_info
, *outer_info
= inline_summary (to
);
146 /* Look for function e->caller is inlined to. While doing
147 so work out the largest function body on the way. As
148 described above, we want to base our function growth
149 limits based on that. Not on the self size of the
150 outer function, not on the self size of inline code
151 we immediately inline to. This is the most relaxed
152 interpretation of the rule "do not grow large functions
153 too much in order to prevent compiler from exploding". */
156 info
= inline_summary (to
);
157 if (limit
< info
->self_size
)
158 limit
= info
->self_size
;
159 if (stack_size_limit
< info
->estimated_self_stack_size
)
160 stack_size_limit
= info
->estimated_self_stack_size
;
161 if (to
->global
.inlined_to
)
162 to
= to
->callers
->caller
;
167 what_info
= inline_summary (what
);
169 if (limit
< what_info
->self_size
)
170 limit
= what_info
->self_size
;
172 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
174 /* Check the size after inlining against the function limits. But allow
175 the function to shrink if it went over the limits by forced inlining. */
176 newsize
= estimate_size_after_inlining (to
, e
);
177 if (newsize
>= info
->size
178 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
181 e
->inline_failed
= CIF_LARGE_FUNCTION_GROWTH_LIMIT
;
185 if (!what_info
->estimated_stack_size
)
188 /* FIXME: Stack size limit often prevents inlining in Fortran programs
189 due to large i/o datastructures used by the Fortran front-end.
190 We ought to ignore this limit when we know that the edge is executed
191 on every invocation of the caller (i.e. its call statement dominates
192 exit block). We do not track this information, yet. */
193 stack_size_limit
+= ((gcov_type
)stack_size_limit
194 * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100);
196 inlined_stack
= (outer_info
->stack_frame_offset
197 + outer_info
->estimated_self_stack_size
198 + what_info
->estimated_stack_size
);
199 /* Check new stack consumption with stack consumption at the place
201 if (inlined_stack
> stack_size_limit
202 /* If function already has large stack usage from sibling
203 inline call, we can inline, too.
204 This bit overoptimistically assume that we are good at stack
206 && inlined_stack
> info
->estimated_stack_size
207 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
209 e
->inline_failed
= CIF_LARGE_STACK_FRAME_GROWTH_LIMIT
;
215 /* Dump info about why inlining has failed. */
218 report_inline_failed_reason (struct cgraph_edge
*e
)
222 fprintf (dump_file
, " not inlinable: %s/%i -> %s/%i, %s\n",
223 xstrdup (cgraph_node_name (e
->caller
)), e
->caller
->symbol
.order
,
224 xstrdup (cgraph_node_name (e
->callee
)), e
->callee
->symbol
.order
,
225 cgraph_inline_failed_string (e
->inline_failed
));
229 /* Decide if we can inline the edge and possibly update
230 inline_failed reason.
231 We check whether inlining is possible at all and whether
232 caller growth limits allow doing so.
234 if REPORT is true, output reason to the dump file.
236 if DISREGARD_LIMITES is true, ignore size limits.*/
239 can_inline_edge_p (struct cgraph_edge
*e
, bool report
,
240 bool disregard_limits
= false)
242 bool inlinable
= true;
243 enum availability avail
;
244 struct cgraph_node
*callee
245 = cgraph_function_or_thunk_node (e
->callee
, &avail
);
246 tree caller_tree
= DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e
->caller
->symbol
.decl
);
248 = callee
? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee
->symbol
.decl
) : NULL
;
249 struct function
*caller_cfun
= DECL_STRUCT_FUNCTION (e
->caller
->symbol
.decl
);
250 struct function
*callee_cfun
251 = callee
? DECL_STRUCT_FUNCTION (callee
->symbol
.decl
) : NULL
;
253 if (!caller_cfun
&& e
->caller
->clone_of
)
254 caller_cfun
= DECL_STRUCT_FUNCTION (e
->caller
->clone_of
->symbol
.decl
);
256 if (!callee_cfun
&& callee
&& callee
->clone_of
)
257 callee_cfun
= DECL_STRUCT_FUNCTION (callee
->clone_of
->symbol
.decl
);
259 gcc_assert (e
->inline_failed
);
261 if (!callee
|| !callee
->symbol
.definition
)
263 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
266 else if (!inline_summary (callee
)->inlinable
)
268 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
271 else if (avail
<= AVAIL_OVERWRITABLE
)
273 e
->inline_failed
= CIF_OVERWRITABLE
;
276 else if (e
->call_stmt_cannot_inline_p
)
278 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
281 /* Don't inline if the functions have different EH personalities. */
282 else if (DECL_FUNCTION_PERSONALITY (e
->caller
->symbol
.decl
)
283 && DECL_FUNCTION_PERSONALITY (callee
->symbol
.decl
)
284 && (DECL_FUNCTION_PERSONALITY (e
->caller
->symbol
.decl
)
285 != DECL_FUNCTION_PERSONALITY (callee
->symbol
.decl
)))
287 e
->inline_failed
= CIF_EH_PERSONALITY
;
290 /* TM pure functions should not be inlined into non-TM_pure
292 else if (is_tm_pure (callee
->symbol
.decl
)
293 && !is_tm_pure (e
->caller
->symbol
.decl
))
295 e
->inline_failed
= CIF_UNSPECIFIED
;
298 /* Don't inline if the callee can throw non-call exceptions but the
300 FIXME: this is obviously wrong for LTO where STRUCT_FUNCTION is missing.
301 Move the flag into cgraph node or mirror it in the inline summary. */
302 else if (callee_cfun
&& callee_cfun
->can_throw_non_call_exceptions
303 && !(caller_cfun
&& caller_cfun
->can_throw_non_call_exceptions
))
305 e
->inline_failed
= CIF_NON_CALL_EXCEPTIONS
;
308 /* Check compatibility of target optimization options. */
309 else if (!targetm
.target_option
.can_inline_p (e
->caller
->symbol
.decl
,
310 callee
->symbol
.decl
))
312 e
->inline_failed
= CIF_TARGET_OPTION_MISMATCH
;
315 /* Check if caller growth allows the inlining. */
316 else if (!DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
)
318 && !lookup_attribute ("flatten",
320 (e
->caller
->global
.inlined_to
321 ? e
->caller
->global
.inlined_to
->symbol
.decl
322 : e
->caller
->symbol
.decl
))
323 && !caller_growth_limits (e
))
325 /* Don't inline a function with a higher optimization level than the
326 caller. FIXME: this is really just tip of iceberg of handling
327 optimization attribute. */
328 else if (caller_tree
!= callee_tree
)
330 struct cl_optimization
*caller_opt
331 = TREE_OPTIMIZATION ((caller_tree
)
333 : optimization_default_node
);
335 struct cl_optimization
*callee_opt
336 = TREE_OPTIMIZATION ((callee_tree
)
338 : optimization_default_node
);
340 if (((caller_opt
->x_optimize
> callee_opt
->x_optimize
)
341 || (caller_opt
->x_optimize_size
!= callee_opt
->x_optimize_size
))
342 /* gcc.dg/pr43564.c. Look at forced inline even in -O0. */
343 && !DECL_DISREGARD_INLINE_LIMITS (e
->callee
->symbol
.decl
))
345 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
350 if (!inlinable
&& report
)
351 report_inline_failed_reason (e
);
356 /* Return true if the edge E is inlinable during early inlining. */
359 can_early_inline_edge_p (struct cgraph_edge
*e
)
361 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
,
363 /* Early inliner might get called at WPA stage when IPA pass adds new
364 function. In this case we can not really do any of early inlining
365 because function bodies are missing. */
366 if (!gimple_has_body_p (callee
->symbol
.decl
))
368 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
371 /* In early inliner some of callees may not be in SSA form yet
372 (i.e. the callgraph is cyclic and we did not process
373 the callee by early inliner, yet). We don't have CIF code for this
374 case; later we will re-do the decision in the real inliner. */
375 if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->caller
->symbol
.decl
))
376 || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->symbol
.decl
)))
379 fprintf (dump_file
, " edge not inlinable: not in SSA form\n");
382 if (!can_inline_edge_p (e
, true))
388 /* Return number of calls in N. Ignore cheap builtins. */
391 num_calls (struct cgraph_node
*n
)
393 struct cgraph_edge
*e
;
396 for (e
= n
->callees
; e
; e
= e
->next_callee
)
397 if (!is_inexpensive_builtin (e
->callee
->symbol
.decl
))
403 /* Return true if we are interested in inlining small function. */
406 want_early_inline_function_p (struct cgraph_edge
*e
)
408 bool want_inline
= true;
409 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
411 if (DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
413 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
414 && !flag_inline_small_functions
)
416 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
417 report_inline_failed_reason (e
);
422 int growth
= estimate_edge_growth (e
);
427 else if (!cgraph_maybe_hot_edge_p (e
)
431 fprintf (dump_file
, " will not early inline: %s/%i->%s/%i, "
432 "call is cold and code would grow by %i\n",
433 xstrdup (cgraph_node_name (e
->caller
)),
434 e
->caller
->symbol
.order
,
435 xstrdup (cgraph_node_name (callee
)), callee
->symbol
.order
,
439 else if (growth
> PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
442 fprintf (dump_file
, " will not early inline: %s/%i->%s/%i, "
443 "growth %i exceeds --param early-inlining-insns\n",
444 xstrdup (cgraph_node_name (e
->caller
)),
445 e
->caller
->symbol
.order
,
446 xstrdup (cgraph_node_name (callee
)), callee
->symbol
.order
,
450 else if ((n
= num_calls (callee
)) != 0
451 && growth
* (n
+ 1) > PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
454 fprintf (dump_file
, " will not early inline: %s/%i->%s/%i, "
455 "growth %i exceeds --param early-inlining-insns "
456 "divided by number of calls\n",
457 xstrdup (cgraph_node_name (e
->caller
)),
458 e
->caller
->symbol
.order
,
459 xstrdup (cgraph_node_name (callee
)), callee
->symbol
.order
,
467 /* Compute time of the edge->caller + edge->callee execution when inlining
471 compute_uninlined_call_time (struct inline_summary
*callee_info
,
472 struct cgraph_edge
*edge
)
474 gcov_type uninlined_call_time
=
475 RDIV ((gcov_type
)callee_info
->time
* MAX (edge
->frequency
, 1),
477 gcov_type caller_time
= inline_summary (edge
->caller
->global
.inlined_to
478 ? edge
->caller
->global
.inlined_to
479 : edge
->caller
)->time
;
480 return uninlined_call_time
+ caller_time
;
483 /* Same as compute_uinlined_call_time but compute time when inlining
487 compute_inlined_call_time (struct cgraph_edge
*edge
,
490 gcov_type caller_time
= inline_summary (edge
->caller
->global
.inlined_to
491 ? edge
->caller
->global
.inlined_to
492 : edge
->caller
)->time
;
493 gcov_type time
= (caller_time
494 + RDIV (((gcov_type
) edge_time
495 - inline_edge_summary (edge
)->call_stmt_time
)
496 * MAX (edge
->frequency
, 1), CGRAPH_FREQ_BASE
));
497 /* Possible one roundoff error, but watch for overflows. */
498 gcc_checking_assert (time
>= INT_MIN
/ 2);
504 /* Return true if the speedup for inlining E is bigger than
505 PARAM_MAX_INLINE_MIN_SPEEDUP. */
508 big_speedup_p (struct cgraph_edge
*e
)
510 gcov_type time
= compute_uninlined_call_time (inline_summary (e
->callee
),
512 gcov_type inlined_time
= compute_inlined_call_time (e
,
513 estimate_edge_time (e
));
514 if (time
- inlined_time
515 > RDIV (time
* PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP
), 100))
520 /* Return true if we are interested in inlining small function.
521 When REPORT is true, report reason to dump file. */
524 want_inline_small_function_p (struct cgraph_edge
*e
, bool report
)
526 bool want_inline
= true;
527 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
529 if (DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
531 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
532 && !flag_inline_small_functions
)
534 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
539 int growth
= estimate_edge_growth (e
);
540 inline_hints hints
= estimate_edge_hints (e
);
541 bool big_speedup
= big_speedup_p (e
);
545 /* Apply MAX_INLINE_INSNS_SINGLE limit. Do not do so when
546 hints suggests that inlining given function is very profitable. */
547 else if (DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
548 && growth
>= MAX_INLINE_INSNS_SINGLE
550 && !(hints
& (INLINE_HINT_indirect_call
551 | INLINE_HINT_loop_iterations
552 | INLINE_HINT_array_index
553 | INLINE_HINT_loop_stride
)))
555 e
->inline_failed
= CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
;
558 /* Before giving up based on fact that caller size will grow, allow
559 functions that are called few times and eliminating the offline
560 copy will lead to overall code size reduction.
561 Not all of these will be handled by subsequent inlining of functions
562 called once: in particular weak functions are not handled or funcitons
563 that inline to multiple calls but a lot of bodies is optimized out.
564 Finally we want to inline earlier to allow inlining of callbacks.
566 This is slightly wrong on aggressive side: it is entirely possible
567 that function is called many times with a context where inlining
568 reduces code size and few times with a context where inlining increase
569 code size. Resoluting growth estimate will be negative even if it
570 would make more sense to keep offline copy and do not inline into the
571 call sites that makes the code size grow.
573 When badness orders the calls in a way that code reducing calls come
574 first, this situation is not a problem at all: after inlining all
575 "good" calls, we will realize that keeping the function around is
577 else if (growth
<= MAX_INLINE_INSNS_SINGLE
578 /* Unlike for functions called once, we play unsafe with
579 COMDATs. We can allow that since we know functions
580 in consideration are small (and thus risk is small) and
581 moreover grow estimates already accounts that COMDAT
582 functions may or may not disappear when eliminated from
583 current unit. With good probability making aggressive
584 choice in all units is going to make overall program
587 Consequently we ask cgraph_can_remove_if_no_direct_calls_p
589 cgraph_will_be_removed_from_program_if_no_direct_calls */
590 && !DECL_EXTERNAL (callee
->symbol
.decl
)
591 && cgraph_can_remove_if_no_direct_calls_p (callee
)
592 && estimate_growth (callee
) <= 0)
594 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
595 && !flag_inline_functions
)
597 e
->inline_failed
= CIF_NOT_DECLARED_INLINED
;
600 /* Apply MAX_INLINE_INSNS_AUTO limit for functions not declared inline
601 Upgrade it to MAX_INLINE_INSNS_SINGLE when hints suggests that
602 inlining given function is very profitable. */
603 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
605 && growth
>= ((hints
& (INLINE_HINT_indirect_call
606 | INLINE_HINT_loop_iterations
607 | INLINE_HINT_array_index
608 | INLINE_HINT_loop_stride
))
609 ? MAX (MAX_INLINE_INSNS_AUTO
,
610 MAX_INLINE_INSNS_SINGLE
)
611 : MAX_INLINE_INSNS_AUTO
))
613 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
616 /* If call is cold, do not inline when function body would grow. */
617 else if (!cgraph_maybe_hot_edge_p (e
))
619 e
->inline_failed
= CIF_UNLIKELY_CALL
;
623 if (!want_inline
&& report
)
624 report_inline_failed_reason (e
);
628 /* EDGE is self recursive edge.
629 We hand two cases - when function A is inlining into itself
630 or when function A is being inlined into another inliner copy of function
633 In first case OUTER_NODE points to the toplevel copy of A, while
634 in the second case OUTER_NODE points to the outermost copy of A in B.
636 In both cases we want to be extra selective since
637 inlining the call will just introduce new recursive calls to appear. */
640 want_inline_self_recursive_call_p (struct cgraph_edge
*edge
,
641 struct cgraph_node
*outer_node
,
645 char const *reason
= NULL
;
646 bool want_inline
= true;
647 int caller_freq
= CGRAPH_FREQ_BASE
;
648 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
650 if (DECL_DECLARED_INLINE_P (edge
->caller
->symbol
.decl
))
651 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
653 if (!cgraph_maybe_hot_edge_p (edge
))
655 reason
= "recursive call is cold";
658 else if (max_count
&& !outer_node
->count
)
660 reason
= "not executed in profile";
663 else if (depth
> max_depth
)
665 reason
= "--param max-inline-recursive-depth exceeded.";
669 if (outer_node
->global
.inlined_to
)
670 caller_freq
= outer_node
->callers
->frequency
;
674 /* Inlining of self recursive function into copy of itself within other function
675 is transformation similar to loop peeling.
677 Peeling is profitable if we can inline enough copies to make probability
678 of actual call to the self recursive function very small. Be sure that
679 the probability of recursion is small.
681 We ensure that the frequency of recursing is at most 1 - (1/max_depth).
682 This way the expected number of recision is at most max_depth. */
685 int max_prob
= CGRAPH_FREQ_BASE
- ((CGRAPH_FREQ_BASE
+ max_depth
- 1)
688 for (i
= 1; i
< depth
; i
++)
689 max_prob
= max_prob
* max_prob
/ CGRAPH_FREQ_BASE
;
691 && (edge
->count
* CGRAPH_FREQ_BASE
/ outer_node
->count
694 reason
= "profile of recursive call is too large";
698 && (edge
->frequency
* CGRAPH_FREQ_BASE
/ caller_freq
701 reason
= "frequency of recursive call is too large";
705 /* Recursive inlining, i.e. equivalent of unrolling, is profitable if recursion
706 depth is large. We reduce function call overhead and increase chances that
707 things fit in hardware return predictor.
709 Recursive inlining might however increase cost of stack frame setup
710 actually slowing down functions whose recursion tree is wide rather than
713 Deciding reliably on when to do recursive inlining without profile feedback
714 is tricky. For now we disable recursive inlining when probability of self
717 Recursive inlining of self recursive call within loop also results in large loop
718 depths that generally optimize badly. We may want to throttle down inlining
719 in those cases. In particular this seems to happen in one of libstdc++ rb tree
724 && (edge
->count
* 100 / outer_node
->count
725 <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
)))
727 reason
= "profile of recursive call is too small";
731 && (edge
->frequency
* 100 / caller_freq
732 <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
)))
734 reason
= "frequency of recursive call is too small";
738 if (!want_inline
&& dump_file
)
739 fprintf (dump_file
, " not inlining recursively: %s\n", reason
);
743 /* Return true when NODE has uninlinable caller;
744 set HAS_HOT_CALL if it has hot call.
745 Worker for cgraph_for_node_and_aliases. */
748 check_callers (struct cgraph_node
*node
, void *has_hot_call
)
750 struct cgraph_edge
*e
;
751 for (e
= node
->callers
; e
; e
= e
->next_caller
)
753 if (!can_inline_edge_p (e
, true))
755 if (!has_hot_call
&& cgraph_maybe_hot_edge_p (e
))
756 *(bool *)has_hot_call
= true;
761 /* If NODE has a caller, return true. */
764 has_caller_p (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
771 /* Decide if inlining NODE would reduce unit size by eliminating
772 the offline copy of function.
773 When COLD is true the cold calls are considered, too. */
776 want_inline_function_to_all_callers_p (struct cgraph_node
*node
, bool cold
)
778 struct cgraph_node
*function
= cgraph_function_or_thunk_node (node
, NULL
);
779 bool has_hot_call
= false;
781 /* Does it have callers? */
782 if (!cgraph_for_node_and_aliases (node
, has_caller_p
, NULL
, true))
784 /* Already inlined? */
785 if (function
->global
.inlined_to
)
787 if (cgraph_function_or_thunk_node (node
, NULL
) != node
)
789 /* Inlining into all callers would increase size? */
790 if (estimate_growth (node
) > 0)
792 /* All inlines must be possible. */
793 if (cgraph_for_node_and_aliases (node
, check_callers
, &has_hot_call
, true))
795 if (!cold
&& !has_hot_call
)
800 #define RELATIVE_TIME_BENEFIT_RANGE (INT_MAX / 64)
802 /* Return relative time improvement for inlining EDGE in range
803 1...RELATIVE_TIME_BENEFIT_RANGE */
806 relative_time_benefit (struct inline_summary
*callee_info
,
807 struct cgraph_edge
*edge
,
810 gcov_type relbenefit
;
811 gcov_type uninlined_call_time
= compute_uninlined_call_time (callee_info
, edge
);
812 gcov_type inlined_call_time
= compute_inlined_call_time (edge
, edge_time
);
814 /* Inlining into extern inline function is not a win. */
815 if (DECL_EXTERNAL (edge
->caller
->global
.inlined_to
816 ? edge
->caller
->global
.inlined_to
->symbol
.decl
817 : edge
->caller
->symbol
.decl
))
820 /* Watch overflows. */
821 gcc_checking_assert (uninlined_call_time
>= 0);
822 gcc_checking_assert (inlined_call_time
>= 0);
823 gcc_checking_assert (uninlined_call_time
>= inlined_call_time
);
825 /* Compute relative time benefit, i.e. how much the call becomes faster.
826 ??? perhaps computing how much the caller+calle together become faster
827 would lead to more realistic results. */
828 if (!uninlined_call_time
)
829 uninlined_call_time
= 1;
831 RDIV (((gcov_type
)uninlined_call_time
- inlined_call_time
) * RELATIVE_TIME_BENEFIT_RANGE
,
832 uninlined_call_time
);
833 relbenefit
= MIN (relbenefit
, RELATIVE_TIME_BENEFIT_RANGE
);
834 gcc_checking_assert (relbenefit
>= 0);
835 relbenefit
= MAX (relbenefit
, 1);
840 /* A cost model driving the inlining heuristics in a way so the edges with
841 smallest badness are inlined first. After each inlining is performed
842 the costs of all caller edges of nodes affected are recomputed so the
843 metrics may accurately depend on values such as number of inlinable callers
844 of the function or function body size. */
847 edge_badness (struct cgraph_edge
*edge
, bool dump
)
850 int growth
, edge_time
;
851 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (edge
->callee
,
853 struct inline_summary
*callee_info
= inline_summary (callee
);
856 if (DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
859 growth
= estimate_edge_growth (edge
);
860 edge_time
= estimate_edge_time (edge
);
861 hints
= estimate_edge_hints (edge
);
862 gcc_checking_assert (edge_time
>= 0);
863 gcc_checking_assert (edge_time
<= callee_info
->time
);
864 gcc_checking_assert (growth
<= callee_info
->size
);
868 fprintf (dump_file
, " Badness calculation for %s/%i -> %s/%i\n",
869 xstrdup (cgraph_node_name (edge
->caller
)),
870 edge
->caller
->symbol
.order
,
871 xstrdup (cgraph_node_name (callee
)),
872 edge
->callee
->symbol
.order
);
873 fprintf (dump_file
, " size growth %i, time %i ",
876 dump_inline_hints (dump_file
, hints
);
877 if (big_speedup_p (edge
))
878 fprintf (dump_file
, " big_speedup");
879 fprintf (dump_file
, "\n");
882 /* Always prefer inlining saving code size. */
885 badness
= INT_MIN
/ 2 + growth
;
887 fprintf (dump_file
, " %i: Growth %i <= 0\n", (int) badness
,
891 /* When profiling is available, compute badness as:
893 relative_edge_count * relative_time_benefit
894 goodness = -------------------------------------------
898 The fraction is upside down, because on edge counts and time beneits
899 the bounds are known. Edge growth is essentially unlimited. */
903 sreal tmp
, relbenefit_real
, growth_real
;
904 int relbenefit
= relative_time_benefit (callee_info
, edge
, edge_time
);
906 sreal_init(&relbenefit_real
, relbenefit
, 0);
907 sreal_init(&growth_real
, growth
, 0);
909 /* relative_edge_count. */
910 sreal_init (&tmp
, edge
->count
, 0);
911 sreal_div (&tmp
, &tmp
, &max_count_real
);
913 /* relative_time_benefit. */
914 sreal_mul (&tmp
, &tmp
, &relbenefit_real
);
915 sreal_div (&tmp
, &tmp
, &max_relbenefit_real
);
917 /* growth_f_caller. */
918 sreal_mul (&tmp
, &tmp
, &half_int_min_real
);
919 sreal_div (&tmp
, &tmp
, &growth_real
);
921 badness
= -1 * sreal_to_int (&tmp
);
923 /* Be sure that insanity of the profile won't lead to increasing counts
924 in the scalling and thus to overflow in the computation above. */
925 gcc_assert (max_count
>= edge
->count
);
929 " %i (relative %f): profile info. Relative count %f"
930 " * Relative benefit %f\n",
931 (int) badness
, (double) badness
/ INT_MIN
,
932 (double) edge
->count
/ max_count
,
933 relbenefit
* 100.0 / RELATIVE_TIME_BENEFIT_RANGE
);
937 /* When function local profile is available. Compute badness as:
939 relative_time_benefit
940 goodness = ---------------------------------
941 growth_of_caller * overall_growth
945 compensated by the inline hints.
947 else if (flag_guess_branch_prob
)
949 badness
= (relative_time_benefit (callee_info
, edge
, edge_time
)
950 * (INT_MIN
/ 16 / RELATIVE_TIME_BENEFIT_RANGE
));
951 badness
/= (MIN (65536/2, growth
) * MIN (65536/2, MAX (1, callee_info
->growth
)));
952 gcc_checking_assert (badness
<=0 && badness
>= INT_MIN
/ 16);
953 if ((hints
& (INLINE_HINT_indirect_call
954 | INLINE_HINT_loop_iterations
955 | INLINE_HINT_array_index
956 | INLINE_HINT_loop_stride
))
957 || callee_info
->growth
<= 0)
959 if (hints
& (INLINE_HINT_same_scc
))
961 else if (hints
& (INLINE_HINT_in_scc
))
963 else if (hints
& (INLINE_HINT_cross_module
))
965 gcc_checking_assert (badness
<= 0 && badness
>= INT_MIN
/ 2);
966 if ((hints
& INLINE_HINT_declared_inline
) && badness
>= INT_MIN
/ 32)
971 " %i: guessed profile. frequency %f,"
972 " benefit %f%%, time w/o inlining %i, time w inlining %i"
973 " overall growth %i (current) %i (original)\n",
974 (int) badness
, (double)edge
->frequency
/ CGRAPH_FREQ_BASE
,
975 relative_time_benefit (callee_info
, edge
, edge_time
) * 100.0
976 / RELATIVE_TIME_BENEFIT_RANGE
,
977 (int)compute_uninlined_call_time (callee_info
, edge
),
978 (int)compute_inlined_call_time (edge
, edge_time
),
979 estimate_growth (callee
),
980 callee_info
->growth
);
983 /* When function local profile is not available or it does not give
984 useful information (ie frequency is zero), base the cost on
985 loop nest and overall size growth, so we optimize for overall number
986 of functions fully inlined in program. */
989 int nest
= MIN (inline_edge_summary (edge
)->loop_depth
, 8);
990 badness
= growth
* 256;
992 /* Decrease badness if call is nested. */
1000 fprintf (dump_file
, " %i: no profile. nest %i\n", (int) badness
,
1004 /* Ensure that we did not overflow in all the fixed point math above. */
1005 gcc_assert (badness
>= INT_MIN
);
1006 gcc_assert (badness
<= INT_MAX
- 1);
1007 /* Make recursive inlining happen always after other inlining is done. */
1008 if (cgraph_edge_recursive_p (edge
))
1014 /* Recompute badness of EDGE and update its key in HEAP if needed. */
1016 update_edge_key (fibheap_t heap
, struct cgraph_edge
*edge
)
1018 int badness
= edge_badness (edge
, false);
1021 fibnode_t n
= (fibnode_t
) edge
->aux
;
1022 gcc_checking_assert (n
->data
== edge
);
1024 /* fibheap_replace_key only decrease the keys.
1025 When we increase the key we do not update heap
1026 and instead re-insert the element once it becomes
1027 a minimum of heap. */
1028 if (badness
< n
->key
)
1030 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1033 " decreasing badness %s/%i -> %s/%i, %i to %i\n",
1034 xstrdup (cgraph_node_name (edge
->caller
)),
1035 edge
->caller
->symbol
.order
,
1036 xstrdup (cgraph_node_name (edge
->callee
)),
1037 edge
->callee
->symbol
.order
,
1041 fibheap_replace_key (heap
, n
, badness
);
1042 gcc_checking_assert (n
->key
== badness
);
1047 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1050 " enqueuing call %s/%i -> %s/%i, badness %i\n",
1051 xstrdup (cgraph_node_name (edge
->caller
)),
1052 edge
->caller
->symbol
.order
,
1053 xstrdup (cgraph_node_name (edge
->callee
)),
1054 edge
->callee
->symbol
.order
,
1057 edge
->aux
= fibheap_insert (heap
, badness
, edge
);
1062 /* NODE was inlined.
1063 All caller edges needs to be resetted because
1064 size estimates change. Similarly callees needs reset
1065 because better context may be known. */
1068 reset_edge_caches (struct cgraph_node
*node
)
1070 struct cgraph_edge
*edge
;
1071 struct cgraph_edge
*e
= node
->callees
;
1072 struct cgraph_node
*where
= node
;
1074 struct ipa_ref
*ref
;
1076 if (where
->global
.inlined_to
)
1077 where
= where
->global
.inlined_to
;
1079 /* WHERE body size has changed, the cached growth is invalid. */
1080 reset_node_growth_cache (where
);
1082 for (edge
= where
->callers
; edge
; edge
= edge
->next_caller
)
1083 if (edge
->inline_failed
)
1084 reset_edge_growth_cache (edge
);
1085 for (i
= 0; ipa_ref_list_referring_iterate (&where
->symbol
.ref_list
,
1087 if (ref
->use
== IPA_REF_ALIAS
)
1088 reset_edge_caches (ipa_ref_referring_node (ref
));
1094 if (!e
->inline_failed
&& e
->callee
->callees
)
1095 e
= e
->callee
->callees
;
1098 if (e
->inline_failed
)
1099 reset_edge_growth_cache (e
);
1106 if (e
->caller
== node
)
1108 e
= e
->caller
->callers
;
1110 while (!e
->next_callee
);
1116 /* Recompute HEAP nodes for each of caller of NODE.
1117 UPDATED_NODES track nodes we already visited, to avoid redundant work.
1118 When CHECK_INLINABLITY_FOR is set, re-check for specified edge that
1119 it is inlinable. Otherwise check all edges. */
1122 update_caller_keys (fibheap_t heap
, struct cgraph_node
*node
,
1123 bitmap updated_nodes
,
1124 struct cgraph_edge
*check_inlinablity_for
)
1126 struct cgraph_edge
*edge
;
1128 struct ipa_ref
*ref
;
1130 if ((!node
->symbol
.alias
&& !inline_summary (node
)->inlinable
)
1131 || node
->global
.inlined_to
)
1133 if (!bitmap_set_bit (updated_nodes
, node
->uid
))
1136 for (i
= 0; ipa_ref_list_referring_iterate (&node
->symbol
.ref_list
,
1138 if (ref
->use
== IPA_REF_ALIAS
)
1140 struct cgraph_node
*alias
= ipa_ref_referring_node (ref
);
1141 update_caller_keys (heap
, alias
, updated_nodes
, check_inlinablity_for
);
1144 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1145 if (edge
->inline_failed
)
1147 if (!check_inlinablity_for
1148 || check_inlinablity_for
== edge
)
1150 if (can_inline_edge_p (edge
, false)
1151 && want_inline_small_function_p (edge
, false))
1152 update_edge_key (heap
, edge
);
1155 report_inline_failed_reason (edge
);
1156 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
1161 update_edge_key (heap
, edge
);
1165 /* Recompute HEAP nodes for each uninlined call in NODE.
1166 This is used when we know that edge badnesses are going only to increase
1167 (we introduced new call site) and thus all we need is to insert newly
1168 created edges into heap. */
1171 update_callee_keys (fibheap_t heap
, struct cgraph_node
*node
,
1172 bitmap updated_nodes
)
1174 struct cgraph_edge
*e
= node
->callees
;
1179 if (!e
->inline_failed
&& e
->callee
->callees
)
1180 e
= e
->callee
->callees
;
1183 enum availability avail
;
1184 struct cgraph_node
*callee
;
1185 /* We do not reset callee growth cache here. Since we added a new call,
1186 growth chould have just increased and consequentely badness metric
1187 don't need updating. */
1188 if (e
->inline_failed
1189 && (callee
= cgraph_function_or_thunk_node (e
->callee
, &avail
))
1190 && inline_summary (callee
)->inlinable
1191 && avail
>= AVAIL_AVAILABLE
1192 && !bitmap_bit_p (updated_nodes
, callee
->uid
))
1194 if (can_inline_edge_p (e
, false)
1195 && want_inline_small_function_p (e
, false))
1196 update_edge_key (heap
, e
);
1199 report_inline_failed_reason (e
);
1200 fibheap_delete_node (heap
, (fibnode_t
) e
->aux
);
1210 if (e
->caller
== node
)
1212 e
= e
->caller
->callers
;
1214 while (!e
->next_callee
);
1220 /* Enqueue all recursive calls from NODE into priority queue depending on
1221 how likely we want to recursively inline the call. */
1224 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
1227 struct cgraph_edge
*e
;
1228 enum availability avail
;
1230 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1231 if (e
->callee
== node
1232 || (cgraph_function_or_thunk_node (e
->callee
, &avail
) == node
1233 && avail
> AVAIL_OVERWRITABLE
))
1235 /* When profile feedback is available, prioritize by expected number
1237 fibheap_insert (heap
,
1238 !max_count
? -e
->frequency
1239 : -(e
->count
/ ((max_count
+ (1<<24) - 1) / (1<<24))),
1242 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1243 if (!e
->inline_failed
)
1244 lookup_recursive_calls (node
, e
->callee
, heap
);
1247 /* Decide on recursive inlining: in the case function has recursive calls,
1248 inline until body size reaches given argument. If any new indirect edges
1249 are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES
1253 recursive_inlining (struct cgraph_edge
*edge
,
1254 vec
<cgraph_edge_p
> *new_edges
)
1256 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
1258 struct cgraph_node
*node
;
1259 struct cgraph_edge
*e
;
1260 struct cgraph_node
*master_clone
= NULL
, *next
;
1264 node
= edge
->caller
;
1265 if (node
->global
.inlined_to
)
1266 node
= node
->global
.inlined_to
;
1268 if (DECL_DECLARED_INLINE_P (node
->symbol
.decl
))
1269 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
1271 /* Make sure that function is small enough to be considered for inlining. */
1272 if (estimate_size_after_inlining (node
, edge
) >= limit
)
1274 heap
= fibheap_new ();
1275 lookup_recursive_calls (node
, node
, heap
);
1276 if (fibheap_empty (heap
))
1278 fibheap_delete (heap
);
1284 " Performing recursive inlining on %s\n",
1285 cgraph_node_name (node
));
1287 /* Do the inlining and update list of recursive call during process. */
1288 while (!fibheap_empty (heap
))
1290 struct cgraph_edge
*curr
1291 = (struct cgraph_edge
*) fibheap_extract_min (heap
);
1292 struct cgraph_node
*cnode
, *dest
= curr
->callee
;
1294 if (!can_inline_edge_p (curr
, true))
1297 /* MASTER_CLONE is produced in the case we already started modified
1298 the function. Be sure to redirect edge to the original body before
1299 estimating growths otherwise we will be seeing growths after inlining
1300 the already modified body. */
1303 cgraph_redirect_edge_callee (curr
, master_clone
);
1304 reset_edge_growth_cache (curr
);
1307 if (estimate_size_after_inlining (node
, curr
) > limit
)
1309 cgraph_redirect_edge_callee (curr
, dest
);
1310 reset_edge_growth_cache (curr
);
1315 for (cnode
= curr
->caller
;
1316 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
1317 if (node
->symbol
.decl
1318 == cgraph_function_or_thunk_node (curr
->callee
, NULL
)->symbol
.decl
)
1321 if (!want_inline_self_recursive_call_p (curr
, node
, false, depth
))
1323 cgraph_redirect_edge_callee (curr
, dest
);
1324 reset_edge_growth_cache (curr
);
1331 " Inlining call of depth %i", depth
);
1334 fprintf (dump_file
, " called approx. %.2f times per call",
1335 (double)curr
->count
/ node
->count
);
1337 fprintf (dump_file
, "\n");
1341 /* We need original clone to copy around. */
1342 master_clone
= cgraph_clone_node (node
, node
->symbol
.decl
,
1343 node
->count
, CGRAPH_FREQ_BASE
,
1344 false, vNULL
, true, NULL
);
1345 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
1346 if (!e
->inline_failed
)
1347 clone_inlined_nodes (e
, true, false, NULL
);
1348 cgraph_redirect_edge_callee (curr
, master_clone
);
1349 reset_edge_growth_cache (curr
);
1352 inline_call (curr
, false, new_edges
, &overall_size
, true);
1353 lookup_recursive_calls (node
, curr
->callee
, heap
);
1357 if (!fibheap_empty (heap
) && dump_file
)
1358 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
1359 fibheap_delete (heap
);
1366 "\n Inlined %i times, "
1367 "body grown from size %i to %i, time %i to %i\n", n
,
1368 inline_summary (master_clone
)->size
, inline_summary (node
)->size
,
1369 inline_summary (master_clone
)->time
, inline_summary (node
)->time
);
1371 /* Remove master clone we used for inlining. We rely that clones inlined
1372 into master clone gets queued just before master clone so we don't
1374 for (node
= cgraph_first_function (); node
!= master_clone
;
1377 next
= cgraph_next_function (node
);
1378 if (node
->global
.inlined_to
== master_clone
)
1379 cgraph_remove_node (node
);
1381 cgraph_remove_node (master_clone
);
1386 /* Given whole compilation unit estimate of INSNS, compute how large we can
1387 allow the unit to grow. */
1390 compute_max_insns (int insns
)
1392 int max_insns
= insns
;
1393 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
1394 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
1396 return ((HOST_WIDEST_INT
) max_insns
1397 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
1401 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
1404 add_new_edges_to_heap (fibheap_t heap
, vec
<cgraph_edge_p
> new_edges
)
1406 while (new_edges
.length () > 0)
1408 struct cgraph_edge
*edge
= new_edges
.pop ();
1410 gcc_assert (!edge
->aux
);
1411 if (edge
->inline_failed
1412 && can_inline_edge_p (edge
, true)
1413 && want_inline_small_function_p (edge
, true))
1414 edge
->aux
= fibheap_insert (heap
, edge_badness (edge
, false), edge
);
1418 /* Remove EDGE from the fibheap. */
1421 heap_edge_removal_hook (struct cgraph_edge
*e
, void *data
)
1424 reset_node_growth_cache (e
->callee
);
1427 fibheap_delete_node ((fibheap_t
)data
, (fibnode_t
)e
->aux
);
1432 /* Return true if speculation of edge E seems useful.
1433 If ANTICIPATE_INLINING is true, be conservative and hope that E
1437 speculation_useful_p (struct cgraph_edge
*e
, bool anticipate_inlining
)
1439 enum availability avail
;
1440 struct cgraph_node
*target
= cgraph_function_or_thunk_node (e
->callee
, &avail
);
1441 struct cgraph_edge
*direct
, *indirect
;
1442 struct ipa_ref
*ref
;
1444 gcc_assert (e
->speculative
&& !e
->indirect_unknown_callee
);
1446 if (!cgraph_maybe_hot_edge_p (e
))
1449 /* See if IP optimizations found something potentially useful about the
1450 function. For now we look only for CONST/PURE flags. Almost everything
1451 else we propagate is useless. */
1452 if (avail
>= AVAIL_AVAILABLE
)
1454 int ecf_flags
= flags_from_decl_or_type (target
->symbol
.decl
);
1455 if (ecf_flags
& ECF_CONST
)
1457 cgraph_speculative_call_info (e
, direct
, indirect
, ref
);
1458 if (!(indirect
->indirect_info
->ecf_flags
& ECF_CONST
))
1461 else if (ecf_flags
& ECF_PURE
)
1463 cgraph_speculative_call_info (e
, direct
, indirect
, ref
);
1464 if (!(indirect
->indirect_info
->ecf_flags
& ECF_PURE
))
1468 /* If we did not managed to inline the function nor redirect
1469 to an ipa-cp clone (that are seen by having local flag set),
1470 it is probably pointless to inline it unless hardware is missing
1471 indirect call predictor. */
1472 if (!anticipate_inlining
&& e
->inline_failed
&& !target
->local
.local
)
1474 /* For overwritable targets there is not much to do. */
1475 if (e
->inline_failed
&& !can_inline_edge_p (e
, false, true))
1477 /* OK, speculation seems interesting. */
1481 /* We know that EDGE is not going to be inlined.
1482 See if we can remove speculation. */
1485 resolve_noninline_speculation (fibheap_t edge_heap
, struct cgraph_edge
*edge
)
1487 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
1489 struct cgraph_node
*node
= edge
->caller
;
1490 struct cgraph_node
*where
= node
->global
.inlined_to
1491 ? node
->global
.inlined_to
: node
;
1492 bitmap updated_nodes
= BITMAP_ALLOC (NULL
);
1494 cgraph_resolve_speculation (edge
, NULL
);
1495 reset_edge_caches (where
);
1496 inline_update_overall_summary (where
);
1497 update_caller_keys (edge_heap
, where
,
1498 updated_nodes
, NULL
);
1499 update_callee_keys (edge_heap
, where
,
1501 BITMAP_FREE (updated_nodes
);
1505 /* We use greedy algorithm for inlining of small functions:
1506 All inline candidates are put into prioritized heap ordered in
1509 The inlining of small functions is bounded by unit growth parameters. */
1512 inline_small_functions (void)
1514 struct cgraph_node
*node
;
1515 struct cgraph_edge
*edge
;
1516 fibheap_t edge_heap
= fibheap_new ();
1517 bitmap updated_nodes
= BITMAP_ALLOC (NULL
);
1518 int min_size
, max_size
;
1519 vec
<cgraph_edge_p
> new_indirect_edges
= vNULL
;
1520 int initial_size
= 0;
1521 struct cgraph_node
**order
= XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1522 struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
1524 if (flag_indirect_inlining
)
1525 new_indirect_edges
.create (8);
1527 edge_removal_hook_holder
1528 = cgraph_add_edge_removal_hook (&heap_edge_removal_hook
, edge_heap
);
1530 /* Compute overall unit size and other global parameters used by badness
1534 ipa_reduced_postorder (order
, true, true, NULL
);
1537 FOR_EACH_DEFINED_FUNCTION (node
)
1538 if (!node
->global
.inlined_to
)
1540 if (cgraph_function_with_gimple_body_p (node
)
1541 || node
->thunk
.thunk_p
)
1543 struct inline_summary
*info
= inline_summary (node
);
1544 struct ipa_dfs_info
*dfs
= (struct ipa_dfs_info
*) node
->symbol
.aux
;
1546 if (!DECL_EXTERNAL (node
->symbol
.decl
))
1547 initial_size
+= info
->size
;
1548 info
->growth
= estimate_growth (node
);
1549 if (dfs
&& dfs
->next_cycle
)
1551 struct cgraph_node
*n2
;
1552 int id
= dfs
->scc_no
+ 1;
1554 n2
= ((struct ipa_dfs_info
*) node
->symbol
.aux
)->next_cycle
)
1556 struct inline_summary
*info2
= inline_summary (n2
);
1564 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1565 if (max_count
< edge
->count
)
1566 max_count
= edge
->count
;
1568 sreal_init (&max_count_real
, max_count
, 0);
1569 sreal_init (&max_relbenefit_real
, RELATIVE_TIME_BENEFIT_RANGE
, 0);
1570 sreal_init (&half_int_min_real
, INT_MAX
/ 2, 0);
1571 ipa_free_postorder_info ();
1572 initialize_growth_caches ();
1576 "\nDeciding on inlining of small functions. Starting with size %i.\n",
1579 overall_size
= initial_size
;
1580 max_size
= compute_max_insns (overall_size
);
1581 min_size
= overall_size
;
1583 /* Populate the heeap with all edges we might inline. */
1585 FOR_EACH_DEFINED_FUNCTION (node
)
1587 bool update
= false;
1588 struct cgraph_edge
*next
;
1591 fprintf (dump_file
, "Enqueueing calls in %s/%i.\n",
1592 cgraph_node_name (node
), node
->symbol
.order
);
1594 for (edge
= node
->callees
; edge
; edge
= next
)
1596 next
= edge
->next_callee
;
1597 if (edge
->inline_failed
1599 && can_inline_edge_p (edge
, true)
1600 && want_inline_small_function_p (edge
, true)
1601 && edge
->inline_failed
)
1603 gcc_assert (!edge
->aux
);
1604 update_edge_key (edge_heap
, edge
);
1606 if (edge
->speculative
&& !speculation_useful_p (edge
, edge
->aux
!= NULL
))
1608 cgraph_resolve_speculation (edge
, NULL
);
1614 struct cgraph_node
*where
= node
->global
.inlined_to
1615 ? node
->global
.inlined_to
: node
;
1616 inline_update_overall_summary (where
);
1617 reset_node_growth_cache (where
);
1618 reset_edge_caches (where
);
1619 update_caller_keys (edge_heap
, where
,
1620 updated_nodes
, NULL
);
1621 bitmap_clear (updated_nodes
);
1625 gcc_assert (in_lto_p
1627 || (profile_info
&& flag_branch_probabilities
));
1629 while (!fibheap_empty (edge_heap
))
1631 int old_size
= overall_size
;
1632 struct cgraph_node
*where
, *callee
;
1633 int badness
= fibheap_min_key (edge_heap
);
1634 int current_badness
;
1638 edge
= (struct cgraph_edge
*) fibheap_extract_min (edge_heap
);
1639 gcc_assert (edge
->aux
);
1641 if (!edge
->inline_failed
)
1644 /* Be sure that caches are maintained consistent.
1645 We can not make this ENABLE_CHECKING only because it cause different
1646 updates of the fibheap queue. */
1647 cached_badness
= edge_badness (edge
, false);
1648 reset_edge_growth_cache (edge
);
1649 reset_node_growth_cache (edge
->callee
);
1651 /* When updating the edge costs, we only decrease badness in the keys.
1652 Increases of badness are handled lazilly; when we see key with out
1653 of date value on it, we re-insert it now. */
1654 current_badness
= edge_badness (edge
, false);
1655 gcc_assert (cached_badness
== current_badness
);
1656 gcc_assert (current_badness
>= badness
);
1657 if (current_badness
!= badness
)
1659 edge
->aux
= fibheap_insert (edge_heap
, current_badness
, edge
);
1663 if (!can_inline_edge_p (edge
, true))
1665 resolve_noninline_speculation (edge_heap
, edge
);
1669 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1670 growth
= estimate_edge_growth (edge
);
1674 "\nConsidering %s/%i with %i size\n",
1675 cgraph_node_name (callee
), callee
->symbol
.order
,
1676 inline_summary (callee
)->size
);
1678 " to be inlined into %s/%i in %s:%i\n"
1679 " Estimated growth after inlined into all is %+i insns.\n"
1680 " Estimated badness is %i, frequency %.2f.\n",
1681 cgraph_node_name (edge
->caller
), edge
->caller
->symbol
.order
,
1682 flag_wpa
? "unknown"
1683 : gimple_filename ((const_gimple
) edge
->call_stmt
),
1685 : gimple_lineno ((const_gimple
) edge
->call_stmt
),
1686 estimate_growth (callee
),
1688 edge
->frequency
/ (double)CGRAPH_FREQ_BASE
);
1690 fprintf (dump_file
," Called "HOST_WIDEST_INT_PRINT_DEC
"x\n",
1692 if (dump_flags
& TDF_DETAILS
)
1693 edge_badness (edge
, true);
1696 if (overall_size
+ growth
> max_size
1697 && !DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
1699 edge
->inline_failed
= CIF_INLINE_UNIT_GROWTH_LIMIT
;
1700 report_inline_failed_reason (edge
);
1701 resolve_noninline_speculation (edge_heap
, edge
);
1705 if (!want_inline_small_function_p (edge
, true))
1707 resolve_noninline_speculation (edge_heap
, edge
);
1711 /* Heuristics for inlining small functions works poorly for
1712 recursive calls where we do efect similar to loop unrolling.
1713 When inliing such edge seems profitable, leave decision on
1714 specific inliner. */
1715 if (cgraph_edge_recursive_p (edge
))
1717 where
= edge
->caller
;
1718 if (where
->global
.inlined_to
)
1719 where
= where
->global
.inlined_to
;
1720 if (!recursive_inlining (edge
,
1721 flag_indirect_inlining
1722 ? &new_indirect_edges
: NULL
))
1724 edge
->inline_failed
= CIF_RECURSIVE_INLINING
;
1725 resolve_noninline_speculation (edge_heap
, edge
);
1728 reset_edge_caches (where
);
1729 /* Recursive inliner inlines all recursive calls of the function
1730 at once. Consequently we need to update all callee keys. */
1731 if (flag_indirect_inlining
)
1732 add_new_edges_to_heap (edge_heap
, new_indirect_edges
);
1733 update_callee_keys (edge_heap
, where
, updated_nodes
);
1734 bitmap_clear (updated_nodes
);
1738 struct cgraph_node
*outer_node
= NULL
;
1741 /* Consider the case where self recursive function A is inlined into B.
1742 This is desired optimization in some cases, since it leads to effect
1743 similar of loop peeling and we might completely optimize out the
1744 recursive call. However we must be extra selective. */
1746 where
= edge
->caller
;
1747 while (where
->global
.inlined_to
)
1749 if (where
->symbol
.decl
== callee
->symbol
.decl
)
1750 outer_node
= where
, depth
++;
1751 where
= where
->callers
->caller
;
1754 && !want_inline_self_recursive_call_p (edge
, outer_node
,
1758 = (DECL_DISREGARD_INLINE_LIMITS (edge
->callee
->symbol
.decl
)
1759 ? CIF_RECURSIVE_INLINING
: CIF_UNSPECIFIED
);
1760 resolve_noninline_speculation (edge_heap
, edge
);
1763 else if (depth
&& dump_file
)
1764 fprintf (dump_file
, " Peeling recursion with depth %i\n", depth
);
1766 gcc_checking_assert (!callee
->global
.inlined_to
);
1767 inline_call (edge
, true, &new_indirect_edges
, &overall_size
, true);
1768 if (flag_indirect_inlining
)
1769 add_new_edges_to_heap (edge_heap
, new_indirect_edges
);
1771 reset_edge_caches (edge
->callee
);
1772 reset_node_growth_cache (callee
);
1774 update_callee_keys (edge_heap
, where
, updated_nodes
);
1776 where
= edge
->caller
;
1777 if (where
->global
.inlined_to
)
1778 where
= where
->global
.inlined_to
;
1780 /* Our profitability metric can depend on local properties
1781 such as number of inlinable calls and size of the function body.
1782 After inlining these properties might change for the function we
1783 inlined into (since it's body size changed) and for the functions
1784 called by function we inlined (since number of it inlinable callers
1786 update_caller_keys (edge_heap
, where
, updated_nodes
, NULL
);
1787 bitmap_clear (updated_nodes
);
1792 " Inlined into %s which now has time %i and size %i,"
1793 "net change of %+i.\n",
1794 cgraph_node_name (edge
->caller
),
1795 inline_summary (edge
->caller
)->time
,
1796 inline_summary (edge
->caller
)->size
,
1797 overall_size
- old_size
);
1799 if (min_size
> overall_size
)
1801 min_size
= overall_size
;
1802 max_size
= compute_max_insns (min_size
);
1805 fprintf (dump_file
, "New minimal size reached: %i\n", min_size
);
1809 free_growth_caches ();
1810 new_indirect_edges
.release ();
1811 fibheap_delete (edge_heap
);
1814 "Unit growth for small function inlining: %i->%i (%i%%)\n",
1815 initial_size
, overall_size
,
1816 initial_size
? overall_size
* 100 / (initial_size
) - 100: 0);
1817 BITMAP_FREE (updated_nodes
);
1818 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
1821 /* Flatten NODE. Performed both during early inlining and
1822 at IPA inlining time. */
1825 flatten_function (struct cgraph_node
*node
, bool early
)
1827 struct cgraph_edge
*e
;
1829 /* We shouldn't be called recursively when we are being processed. */
1830 gcc_assert (node
->symbol
.aux
== NULL
);
1832 node
->symbol
.aux
= (void *) node
;
1834 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1836 struct cgraph_node
*orig_callee
;
1837 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
1839 /* We've hit cycle? It is time to give up. */
1840 if (callee
->symbol
.aux
)
1844 "Not inlining %s into %s to avoid cycle.\n",
1845 xstrdup (cgraph_node_name (callee
)),
1846 xstrdup (cgraph_node_name (e
->caller
)));
1847 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
1851 /* When the edge is already inlined, we just need to recurse into
1852 it in order to fully flatten the leaves. */
1853 if (!e
->inline_failed
)
1855 flatten_function (callee
, early
);
1859 /* Flatten attribute needs to be processed during late inlining. For
1860 extra code quality we however do flattening during early optimization,
1863 ? !can_inline_edge_p (e
, true)
1864 : !can_early_inline_edge_p (e
))
1867 if (cgraph_edge_recursive_p (e
))
1870 fprintf (dump_file
, "Not inlining: recursive call.\n");
1874 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->symbol
.decl
))
1875 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->symbol
.decl
)))
1878 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1882 /* Inline the edge and flatten the inline clone. Avoid
1883 recursing through the original node if the node was cloned. */
1885 fprintf (dump_file
, " Inlining %s into %s.\n",
1886 xstrdup (cgraph_node_name (callee
)),
1887 xstrdup (cgraph_node_name (e
->caller
)));
1888 orig_callee
= callee
;
1889 inline_call (e
, true, NULL
, NULL
, false);
1890 if (e
->callee
!= orig_callee
)
1891 orig_callee
->symbol
.aux
= (void *) node
;
1892 flatten_function (e
->callee
, early
);
1893 if (e
->callee
!= orig_callee
)
1894 orig_callee
->symbol
.aux
= NULL
;
1897 node
->symbol
.aux
= NULL
;
1898 if (!node
->global
.inlined_to
)
1899 inline_update_overall_summary (node
);
1902 /* Count number of callers of NODE and store it into DATA (that
1903 points to int. Worker for cgraph_for_node_and_aliases. */
1906 sum_callers (struct cgraph_node
*node
, void *data
)
1908 struct cgraph_edge
*e
;
1909 int *num_calls
= (int *)data
;
1911 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1916 /* Inline NODE to all callers. Worker for cgraph_for_node_and_aliases.
1917 DATA points to number of calls originally found so we avoid infinite
1921 inline_to_all_callers (struct cgraph_node
*node
, void *data
)
1923 int *num_calls
= (int *)data
;
1924 while (node
->callers
&& !node
->global
.inlined_to
)
1926 struct cgraph_node
*caller
= node
->callers
->caller
;
1931 "\nInlining %s size %i.\n",
1932 cgraph_node_name (node
),
1933 inline_summary (node
)->size
);
1935 " Called once from %s %i insns.\n",
1936 cgraph_node_name (node
->callers
->caller
),
1937 inline_summary (node
->callers
->caller
)->size
);
1940 inline_call (node
->callers
, true, NULL
, NULL
, true);
1943 " Inlined into %s which now has %i size\n",
1944 cgraph_node_name (caller
),
1945 inline_summary (caller
)->size
);
1946 if (!(*num_calls
)--)
1949 fprintf (dump_file
, "New calls found; giving up.\n");
1956 /* Decide on the inlining. We do so in the topological order to avoid
1957 expenses on updating data structures. */
1962 struct cgraph_node
*node
;
1964 struct cgraph_node
**order
=
1965 XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1968 bool remove_functions
= false;
1973 if (in_lto_p
&& optimize
)
1974 ipa_update_after_lto_read ();
1977 dump_inline_summaries (dump_file
);
1979 nnodes
= ipa_reverse_postorder (order
);
1981 FOR_EACH_FUNCTION (node
)
1982 node
->symbol
.aux
= 0;
1985 fprintf (dump_file
, "\nFlattening functions:\n");
1987 /* In the first pass handle functions to be flattened. Do this with
1988 a priority so none of our later choices will make this impossible. */
1989 for (i
= nnodes
- 1; i
>= 0; i
--)
1993 /* Handle nodes to be flattened.
1994 Ideally when processing callees we stop inlining at the
1995 entry of cycles, possibly cloning that entry point and
1996 try to flatten itself turning it into a self-recursive
1998 if (lookup_attribute ("flatten",
1999 DECL_ATTRIBUTES (node
->symbol
.decl
)) != NULL
)
2003 "Flattening %s\n", cgraph_node_name (node
));
2004 flatten_function (node
, false);
2008 inline_small_functions ();
2010 /* Do first after-inlining removal. We want to remove all "stale" extern inline
2011 functions and virtual functions so we really know what is called once. */
2012 symtab_remove_unreachable_nodes (false, dump_file
);
2015 /* Inline functions with a property that after inlining into all callers the
2016 code size will shrink because the out-of-line copy is eliminated.
2017 We do this regardless on the callee size as long as function growth limits
2021 "\nDeciding on functions to be inlined into all callers and removing useless speculations:\n");
2023 /* Inlining one function called once has good chance of preventing
2024 inlining other function into the same callee. Ideally we should
2025 work in priority order, but probably inlining hot functions first
2026 is good cut without the extra pain of maintaining the queue.
2028 ??? this is not really fitting the bill perfectly: inlining function
2029 into callee often leads to better optimization of callee due to
2030 increased context for optimization.
2031 For example if main() function calls a function that outputs help
2032 and then function that does the main optmization, we should inline
2033 the second with priority even if both calls are cold by themselves.
2035 We probably want to implement new predicate replacing our use of
2036 maybe_hot_edge interpreted as maybe_hot_edge || callee is known
2038 for (cold
= 0; cold
<= 1; cold
++)
2040 FOR_EACH_DEFINED_FUNCTION (node
)
2042 struct cgraph_edge
*edge
, *next
;
2045 for (edge
= node
->callees
; edge
; edge
= next
)
2047 next
= edge
->next_callee
;
2048 if (edge
->speculative
&& !speculation_useful_p (edge
, false))
2050 cgraph_resolve_speculation (edge
, NULL
);
2052 remove_functions
= true;
2057 struct cgraph_node
*where
= node
->global
.inlined_to
2058 ? node
->global
.inlined_to
: node
;
2059 reset_node_growth_cache (where
);
2060 reset_edge_caches (where
);
2061 inline_update_overall_summary (where
);
2063 if (flag_inline_functions_called_once
2064 && want_inline_function_to_all_callers_p (node
, cold
))
2067 cgraph_for_node_and_aliases (node
, sum_callers
,
2069 cgraph_for_node_and_aliases (node
, inline_to_all_callers
,
2071 remove_functions
= true;
2076 /* Free ipa-prop structures if they are no longer needed. */
2078 ipa_free_all_structures_after_iinln ();
2082 "\nInlined %i calls, eliminated %i functions\n\n",
2083 ncalls_inlined
, nfunctions_inlined
);
2086 dump_inline_summaries (dump_file
);
2087 /* In WPA we use inline summaries for partitioning process. */
2089 inline_free_summary ();
2090 return remove_functions
? TODO_remove_functions
: 0;
2093 /* Inline always-inline function calls in NODE. */
2096 inline_always_inline_functions (struct cgraph_node
*node
)
2098 struct cgraph_edge
*e
;
2099 bool inlined
= false;
2101 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2103 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
2104 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
2107 if (cgraph_edge_recursive_p (e
))
2110 fprintf (dump_file
, " Not inlining recursive call to %s.\n",
2111 cgraph_node_name (e
->callee
));
2112 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2116 if (!can_early_inline_edge_p (e
))
2118 /* Set inlined to true if the callee is marked "always_inline" but
2119 is not inlinable. This will allow flagging an error later in
2120 expand_call_inline in tree-inline.c. */
2121 if (lookup_attribute ("always_inline",
2122 DECL_ATTRIBUTES (callee
->symbol
.decl
)) != NULL
)
2128 fprintf (dump_file
, " Inlining %s into %s (always_inline).\n",
2129 xstrdup (cgraph_node_name (e
->callee
)),
2130 xstrdup (cgraph_node_name (e
->caller
)));
2131 inline_call (e
, true, NULL
, NULL
, false);
2135 inline_update_overall_summary (node
);
2140 /* Decide on the inlining. We do so in the topological order to avoid
2141 expenses on updating data structures. */
2144 early_inline_small_functions (struct cgraph_node
*node
)
2146 struct cgraph_edge
*e
;
2147 bool inlined
= false;
2149 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2151 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
2152 if (!inline_summary (callee
)->inlinable
2153 || !e
->inline_failed
)
2156 /* Do not consider functions not declared inline. */
2157 if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
2158 && !flag_inline_small_functions
2159 && !flag_inline_functions
)
2163 fprintf (dump_file
, "Considering inline candidate %s.\n",
2164 cgraph_node_name (callee
));
2166 if (!can_early_inline_edge_p (e
))
2169 if (cgraph_edge_recursive_p (e
))
2172 fprintf (dump_file
, " Not inlining: recursive call.\n");
2176 if (!want_early_inline_function_p (e
))
2180 fprintf (dump_file
, " Inlining %s into %s.\n",
2181 xstrdup (cgraph_node_name (callee
)),
2182 xstrdup (cgraph_node_name (e
->caller
)));
2183 inline_call (e
, true, NULL
, NULL
, true);
2190 /* Do inlining of small functions. Doing so early helps profiling and other
2191 passes to be somewhat more effective and avoids some code duplication in
2192 later real inlining pass for testcases with very many function calls. */
2194 early_inliner (void)
2196 struct cgraph_node
*node
= cgraph_get_node (current_function_decl
);
2197 struct cgraph_edge
*edge
;
2198 unsigned int todo
= 0;
2200 bool inlined
= false;
2205 /* Do nothing if datastructures for ipa-inliner are already computed. This
2206 happens when some pass decides to construct new function and
2207 cgraph_add_new_function calls lowering passes and early optimization on
2208 it. This may confuse ourself when early inliner decide to inline call to
2209 function clone, because function clones don't have parameter list in
2210 ipa-prop matching their signature. */
2211 if (ipa_node_params_vector
.exists ())
2214 #ifdef ENABLE_CHECKING
2215 verify_cgraph_node (node
);
2217 ipa_remove_all_references (&node
->symbol
.ref_list
);
2219 /* Even when not optimizing or not inlining inline always-inline
2221 inlined
= inline_always_inline_functions (node
);
2225 || !flag_early_inlining
2226 /* Never inline regular functions into always-inline functions
2227 during incremental inlining. This sucks as functions calling
2228 always inline functions will get less optimized, but at the
2229 same time inlining of functions calling always inline
2230 function into an always inline function might introduce
2231 cycles of edges to be always inlined in the callgraph.
2233 We might want to be smarter and just avoid this type of inlining. */
2234 || DECL_DISREGARD_INLINE_LIMITS (node
->symbol
.decl
))
2236 else if (lookup_attribute ("flatten",
2237 DECL_ATTRIBUTES (node
->symbol
.decl
)) != NULL
)
2239 /* When the function is marked to be flattened, recursively inline
2243 "Flattening %s\n", cgraph_node_name (node
));
2244 flatten_function (node
, true);
2249 /* We iterate incremental inlining to get trivial cases of indirect
2251 while (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
)
2252 && early_inline_small_functions (node
))
2254 timevar_push (TV_INTEGRATION
);
2255 todo
|= optimize_inline_calls (current_function_decl
);
2257 /* Technically we ought to recompute inline parameters so the new
2258 iteration of early inliner works as expected. We however have
2259 values approximately right and thus we only need to update edge
2260 info that might be cleared out for newly discovered edges. */
2261 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2263 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2265 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2267 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2268 if (edge
->callee
->symbol
.decl
2269 && !gimple_check_call_matching_types (
2270 edge
->call_stmt
, edge
->callee
->symbol
.decl
, false))
2271 edge
->call_stmt_cannot_inline_p
= true;
2273 timevar_pop (TV_INTEGRATION
);
2278 fprintf (dump_file
, "Iterations: %i\n", iterations
);
2283 timevar_push (TV_INTEGRATION
);
2284 todo
|= optimize_inline_calls (current_function_decl
);
2285 timevar_pop (TV_INTEGRATION
);
2288 cfun
->always_inline_functions_inlined
= true;
2295 const pass_data pass_data_early_inline
=
2297 GIMPLE_PASS
, /* type */
2298 "einline", /* name */
2299 OPTGROUP_INLINE
, /* optinfo_flags */
2300 false, /* has_gate */
2301 true, /* has_execute */
2302 TV_EARLY_INLINING
, /* tv_id */
2303 PROP_ssa
, /* properties_required */
2304 0, /* properties_provided */
2305 0, /* properties_destroyed */
2306 0, /* todo_flags_start */
2307 0, /* todo_flags_finish */
2310 class pass_early_inline
: public gimple_opt_pass
2313 pass_early_inline(gcc::context
*ctxt
)
2314 : gimple_opt_pass(pass_data_early_inline
, ctxt
)
2317 /* opt_pass methods: */
2318 unsigned int execute () { return early_inliner (); }
2320 }; // class pass_early_inline
2325 make_pass_early_inline (gcc::context
*ctxt
)
2327 return new pass_early_inline (ctxt
);
2331 /* When to run IPA inlining. Inlining of always-inline functions
2332 happens during early inlining.
2334 Enable inlining unconditoinally, because callgraph redirection
2338 gate_ipa_inline (void)
2345 const pass_data pass_data_ipa_inline
=
2347 IPA_PASS
, /* type */
2348 "inline", /* name */
2349 OPTGROUP_INLINE
, /* optinfo_flags */
2350 true, /* has_gate */
2351 true, /* has_execute */
2352 TV_IPA_INLINING
, /* tv_id */
2353 0, /* properties_required */
2354 0, /* properties_provided */
2355 0, /* properties_destroyed */
2356 TODO_remove_functions
, /* todo_flags_start */
2357 ( TODO_dump_symtab
), /* todo_flags_finish */
2360 class pass_ipa_inline
: public ipa_opt_pass_d
2363 pass_ipa_inline(gcc::context
*ctxt
)
2364 : ipa_opt_pass_d(pass_data_ipa_inline
, ctxt
,
2365 inline_generate_summary
, /* generate_summary */
2366 inline_write_summary
, /* write_summary */
2367 inline_read_summary
, /* read_summary */
2368 NULL
, /* write_optimization_summary */
2369 NULL
, /* read_optimization_summary */
2370 NULL
, /* stmt_fixup */
2371 0, /* function_transform_todo_flags_start */
2372 inline_transform
, /* function_transform */
2373 NULL
) /* variable_transform */
2376 /* opt_pass methods: */
2377 bool gate () { return gate_ipa_inline (); }
2378 unsigned int execute () { return ipa_inline (); }
2380 }; // class pass_ipa_inline
2385 make_pass_ipa_inline (gcc::context
*ctxt
)
2387 return new pass_ipa_inline (ctxt
);