1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 The implementation of inliner is organized as follows:
25 inlining heuristics limits
27 can_inline_edge_p allow to check that particular inlining is allowed
28 by the limits specified by user (allowed function growth, growth and so
31 Functions are inlined when it is obvious the result is profitable (such
32 as functions called once or when inlining reduce code size).
33 In addition to that we perform inlining of small functions and recursive
38 The inliner itself is split into two passes:
42 Simple local inlining pass inlining callees into current function.
43 This pass makes no use of whole unit analysis and thus it can do only
44 very simple decisions based on local properties.
46 The strength of the pass is that it is run in topological order
47 (reverse postorder) on the callgraph. Functions are converted into SSA
48 form just before this pass and optimized subsequently. As a result, the
49 callees of the function seen by the early inliner was already optimized
50 and results of early inlining adds a lot of optimization opportunities
51 for the local optimization.
53 The pass handle the obvious inlining decisions within the compilation
54 unit - inlining auto inline functions, inlining for size and
57 main strength of the pass is the ability to eliminate abstraction
58 penalty in C++ code (via combination of inlining and early
59 optimization) and thus improve quality of analysis done by real IPA
62 Because of lack of whole unit knowledge, the pass can not really make
63 good code size/performance tradeoffs. It however does very simple
64 speculative inlining allowing code size to grow by
65 EARLY_INLINING_INSNS when callee is leaf function. In this case the
66 optimizations performed later are very likely to eliminate the cost.
70 This is the real inliner able to handle inlining with whole program
71 knowledge. It performs following steps:
73 1) inlining of small functions. This is implemented by greedy
74 algorithm ordering all inlinable cgraph edges by their badness and
75 inlining them in this order as long as inline limits allows doing so.
77 This heuristics is not very good on inlining recursive calls. Recursive
78 calls can be inlined with results similar to loop unrolling. To do so,
79 special purpose recursive inliner is executed on function when
80 recursive edge is met as viable candidate.
82 2) Unreachable functions are removed from callgraph. Inlining leads
83 to devirtualization and other modification of callgraph so functions
84 may become unreachable during the process. Also functions declared as
85 extern inline or virtual functions are removed, since after inlining
86 we no longer need the offline bodies.
88 3) Functions called once and not exported from the unit are inlined.
89 This should almost always lead to reduction of code size by eliminating
90 the need for offline copy of the function. */
94 #include "coretypes.h"
97 #include "tree-inline.h"
98 #include "langhooks.h"
101 #include "diagnostic.h"
102 #include "gimple-pretty-print.h"
106 #include "tree-pass.h"
107 #include "coverage.h"
110 #include "tree-flow.h"
111 #include "ipa-prop.h"
112 #include "basic-block.h"
118 #include "ipa-inline.h"
119 #include "ipa-utils.h"
121 /* Statistics we collect about inlining algorithm. */
122 static int overall_size
;
123 static gcov_type max_count
;
125 /* Return false when inlining edge E would lead to violating
126 limits on function unit growth or stack usage growth.
128 The relative function body growth limit is present generally
129 to avoid problems with non-linear behavior of the compiler.
130 To allow inlining huge functions into tiny wrapper, the limit
131 is always based on the bigger of the two functions considered.
133 For stack growth limits we always base the growth in stack usage
134 of the callers. We want to prevent applications from segfaulting
135 on stack overflow when functions with huge stack frames gets
139 caller_growth_limits (struct cgraph_edge
*e
)
141 struct cgraph_node
*to
= e
->caller
;
142 struct cgraph_node
*what
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
145 HOST_WIDE_INT stack_size_limit
= 0, inlined_stack
;
146 struct inline_summary
*info
, *what_info
, *outer_info
= inline_summary (to
);
148 /* Look for function e->caller is inlined to. While doing
149 so work out the largest function body on the way. As
150 described above, we want to base our function growth
151 limits based on that. Not on the self size of the
152 outer function, not on the self size of inline code
153 we immediately inline to. This is the most relaxed
154 interpretation of the rule "do not grow large functions
155 too much in order to prevent compiler from exploding". */
158 info
= inline_summary (to
);
159 if (limit
< info
->self_size
)
160 limit
= info
->self_size
;
161 if (stack_size_limit
< info
->estimated_self_stack_size
)
162 stack_size_limit
= info
->estimated_self_stack_size
;
163 if (to
->global
.inlined_to
)
164 to
= to
->callers
->caller
;
169 what_info
= inline_summary (what
);
171 if (limit
< what_info
->self_size
)
172 limit
= what_info
->self_size
;
174 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
176 /* Check the size after inlining against the function limits. But allow
177 the function to shrink if it went over the limits by forced inlining. */
178 newsize
= estimate_size_after_inlining (to
, e
);
179 if (newsize
>= info
->size
180 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
183 e
->inline_failed
= CIF_LARGE_FUNCTION_GROWTH_LIMIT
;
187 if (!what_info
->estimated_stack_size
)
190 /* FIXME: Stack size limit often prevents inlining in Fortran programs
191 due to large i/o datastructures used by the Fortran front-end.
192 We ought to ignore this limit when we know that the edge is executed
193 on every invocation of the caller (i.e. its call statement dominates
194 exit block). We do not track this information, yet. */
195 stack_size_limit
+= ((gcov_type
)stack_size_limit
196 * PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100);
198 inlined_stack
= (outer_info
->estimated_stack_size
199 + what_info
->estimated_stack_size
);
200 /* Check new stack consumption with stack consumption at the place
202 if (inlined_stack
> stack_size_limit
203 /* If function already has large stack usage from sibling
204 inline call, we can inline, too.
205 This bit overoptimistically assume that we are good at stack
207 && inlined_stack
> info
->estimated_stack_size
208 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
210 e
->inline_failed
= CIF_LARGE_STACK_FRAME_GROWTH_LIMIT
;
216 /* Dump info about why inlining has failed. */
219 report_inline_failed_reason (struct cgraph_edge
*e
)
223 fprintf (dump_file
, " not inlinable: %s/%i -> %s/%i, %s\n",
224 xstrdup (cgraph_node_name (e
->caller
)), e
->caller
->uid
,
225 xstrdup (cgraph_node_name (e
->callee
)), e
->callee
->uid
,
226 cgraph_inline_failed_string (e
->inline_failed
));
230 /* Decide if we can inline the edge and possibly update
231 inline_failed reason.
232 We check whether inlining is possible at all and whether
233 caller growth limits allow doing so.
235 if REPORT is true, output reason to the dump file. */
238 can_inline_edge_p (struct cgraph_edge
*e
, bool report
)
240 bool inlinable
= true;
241 enum availability avail
;
242 struct cgraph_node
*callee
243 = cgraph_function_or_thunk_node (e
->callee
, &avail
);
244 tree caller_tree
= DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e
->caller
->symbol
.decl
);
246 = callee
? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee
->symbol
.decl
) : NULL
;
247 struct function
*caller_cfun
= DECL_STRUCT_FUNCTION (e
->caller
->symbol
.decl
);
248 struct function
*callee_cfun
249 = callee
? DECL_STRUCT_FUNCTION (callee
->symbol
.decl
) : NULL
;
251 if (!caller_cfun
&& e
->caller
->clone_of
)
252 caller_cfun
= DECL_STRUCT_FUNCTION (e
->caller
->clone_of
->symbol
.decl
);
254 if (!callee_cfun
&& callee
&& callee
->clone_of
)
255 callee_cfun
= DECL_STRUCT_FUNCTION (callee
->clone_of
->symbol
.decl
);
257 gcc_assert (e
->inline_failed
);
259 if (!callee
|| !callee
->analyzed
)
261 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
264 else if (!inline_summary (callee
)->inlinable
)
266 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
269 else if (avail
<= AVAIL_OVERWRITABLE
)
271 e
->inline_failed
= CIF_OVERWRITABLE
;
274 else if (e
->call_stmt_cannot_inline_p
)
276 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
279 /* Don't inline if the functions have different EH personalities. */
280 else if (DECL_FUNCTION_PERSONALITY (e
->caller
->symbol
.decl
)
281 && DECL_FUNCTION_PERSONALITY (callee
->symbol
.decl
)
282 && (DECL_FUNCTION_PERSONALITY (e
->caller
->symbol
.decl
)
283 != DECL_FUNCTION_PERSONALITY (callee
->symbol
.decl
)))
285 e
->inline_failed
= CIF_EH_PERSONALITY
;
288 /* TM pure functions should not be inlined into non-TM_pure
290 else if (is_tm_pure (callee
->symbol
.decl
)
291 && !is_tm_pure (e
->caller
->symbol
.decl
))
293 e
->inline_failed
= CIF_UNSPECIFIED
;
296 /* Don't inline if the callee can throw non-call exceptions but the
298 FIXME: this is obviously wrong for LTO where STRUCT_FUNCTION is missing.
299 Move the flag into cgraph node or mirror it in the inline summary. */
300 else if (callee_cfun
&& callee_cfun
->can_throw_non_call_exceptions
301 && !(caller_cfun
&& caller_cfun
->can_throw_non_call_exceptions
))
303 e
->inline_failed
= CIF_NON_CALL_EXCEPTIONS
;
306 /* Check compatibility of target optimization options. */
307 else if (!targetm
.target_option
.can_inline_p (e
->caller
->symbol
.decl
,
308 callee
->symbol
.decl
))
310 e
->inline_failed
= CIF_TARGET_OPTION_MISMATCH
;
313 /* Check if caller growth allows the inlining. */
314 else if (!DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
)
315 && !lookup_attribute ("flatten",
317 (e
->caller
->global
.inlined_to
318 ? e
->caller
->global
.inlined_to
->symbol
.decl
319 : e
->caller
->symbol
.decl
))
320 && !caller_growth_limits (e
))
322 /* Don't inline a function with a higher optimization level than the
323 caller. FIXME: this is really just tip of iceberg of handling
324 optimization attribute. */
325 else if (caller_tree
!= callee_tree
)
327 struct cl_optimization
*caller_opt
328 = TREE_OPTIMIZATION ((caller_tree
)
330 : optimization_default_node
);
332 struct cl_optimization
*callee_opt
333 = TREE_OPTIMIZATION ((callee_tree
)
335 : optimization_default_node
);
337 if (((caller_opt
->x_optimize
> callee_opt
->x_optimize
)
338 || (caller_opt
->x_optimize_size
!= callee_opt
->x_optimize_size
))
339 /* gcc.dg/pr43564.c. Look at forced inline even in -O0. */
340 && !DECL_DISREGARD_INLINE_LIMITS (e
->callee
->symbol
.decl
))
342 e
->inline_failed
= CIF_OPTIMIZATION_MISMATCH
;
347 if (!inlinable
&& report
)
348 report_inline_failed_reason (e
);
353 /* Return true if the edge E is inlinable during early inlining. */
356 can_early_inline_edge_p (struct cgraph_edge
*e
)
358 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
,
360 /* Early inliner might get called at WPA stage when IPA pass adds new
361 function. In this case we can not really do any of early inlining
362 because function bodies are missing. */
363 if (!gimple_has_body_p (callee
->symbol
.decl
))
365 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
369 /* Skip fake edges */
370 if (L_IPO_COMP_MODE
&& !e
->call_stmt
)
373 /* In early inliner some of callees may not be in SSA form yet
374 (i.e. the callgraph is cyclic and we did not process
375 the callee by early inliner, yet). We don't have CIF code for this
376 case; later we will re-do the decision in the real inliner. */
377 if (!gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->caller
->symbol
.decl
))
378 || !gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->symbol
.decl
)))
381 fprintf (dump_file
, " edge not inlinable: not in SSA form\n");
384 if (!can_inline_edge_p (e
, true))
390 /* Return number of calls in N. Ignore cheap builtins. */
393 num_calls (struct cgraph_node
*n
)
395 struct cgraph_edge
*e
;
396 /* The following is buggy -- indirect call is not considered. */
399 for (e
= n
->callees
; e
; e
= e
->next_callee
)
400 if (e
->call_stmt
/* Only exist in profile use pass in LIPO */
401 && !is_inexpensive_builtin (e
->callee
->symbol
.decl
))
406 /* Return true if we are interested in inlining small function. */
409 want_early_inline_function_p (struct cgraph_edge
*e
)
411 bool want_inline
= true;
412 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
414 if (DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
416 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
417 && !flag_inline_small_functions
)
419 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
420 report_inline_failed_reason (e
);
425 int growth
= estimate_edge_growth (e
);
426 struct cgraph_node
*callee
= e
->callee
;
429 if (growth
<= PARAM_VALUE (PARAM_EARLY_INLINING_INSNS_ANY
))
431 else if (!cgraph_maybe_hot_edge_p (e
))
434 fprintf (dump_file
, " will not early inline: %s/%i->%s/%i, "
435 "call is cold and code would grow by %i\n",
436 xstrdup (cgraph_node_name (e
->caller
)), e
->caller
->uid
,
437 xstrdup (cgraph_node_name (callee
)), callee
->uid
,
441 else if (growth
> PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
444 fprintf (dump_file
, " will not early inline: %s/%i->%s/%i, "
445 "growth %i exceeds --param early-inlining-insns\n",
446 xstrdup (cgraph_node_name (e
->caller
)), e
->caller
->uid
,
447 xstrdup (cgraph_node_name (callee
)), callee
->uid
,
451 else if (DECL_COMDAT (callee
->symbol
.decl
)
452 && growth
<= PARAM_VALUE (PARAM_EARLY_INLINING_INSNS_COMDAT
))
454 else if ((n
= num_calls (callee
)) != 0
455 && growth
* (n
+ 1) > PARAM_VALUE (PARAM_EARLY_INLINING_INSNS
))
458 fprintf (dump_file
, " will not early inline: %s/%i->%s/%i, "
459 "growth %i exceeds --param early-inlining-insns "
460 "divided by number of calls\n",
461 xstrdup (cgraph_node_name (e
->caller
)), e
->caller
->uid
,
462 xstrdup (cgraph_node_name (callee
)), callee
->uid
,
470 /* Compute time of the edge->caller + edge->callee execution when inlining
474 compute_uninlined_call_time (struct inline_summary
*callee_info
,
475 struct cgraph_edge
*edge
)
477 gcov_type uninlined_call_time
=
478 RDIV ((gcov_type
)callee_info
->time
* MAX (edge
->frequency
, 1),
480 gcov_type caller_time
= inline_summary (edge
->caller
->global
.inlined_to
481 ? edge
->caller
->global
.inlined_to
482 : edge
->caller
)->time
;
483 return uninlined_call_time
+ caller_time
;
486 /* Same as compute_uinlined_call_time but compute time when inlining
490 compute_inlined_call_time (struct cgraph_edge
*edge
,
493 gcov_type caller_time
= inline_summary (edge
->caller
->global
.inlined_to
494 ? edge
->caller
->global
.inlined_to
495 : edge
->caller
)->time
;
496 gcov_type time
= (caller_time
497 + RDIV (((gcov_type
) edge_time
498 - inline_edge_summary (edge
)->call_stmt_time
)
499 * MAX (edge
->frequency
, 1), CGRAPH_FREQ_BASE
));
500 /* Possible one roundoff error, but watch for overflows. */
501 gcc_checking_assert (time
>= INT_MIN
/ 2);
507 /* Return true if the speedup for inlining E is bigger than
508 PARAM_MAX_INLINE_MIN_SPEEDUP. */
511 big_speedup_p (struct cgraph_edge
*e
)
513 gcov_type time
= compute_uninlined_call_time (inline_summary (e
->callee
),
515 gcov_type inlined_time
= compute_inlined_call_time (e
,
516 estimate_edge_time (e
));
517 if (time
- inlined_time
518 > RDIV (time
* PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP
), 100))
523 /* Returns true if callee of edge E is considered useful to inline
524 even if it is cold. A callee is considered useful if there is at
525 least one argument of pointer type with IPA_JF_KNOWN_TYPE or
526 IPA_JF_UNKNOWN as the jump function. The reasoniong here is that
527 it is often beneficial to inline bar into foo in the following
528 code even if the callsite is cold:
535 This exposes accesses to the 'a' object. The jump function of &a
536 is either IPA_JF_KNOWN_TYPE or IPA_JF_UNKNOWN (depending on
537 intervening code). */
540 useful_cold_callee (struct cgraph_edge
*e
)
542 gimple call
= e
->call_stmt
;
543 int n
, arg_num
= gimple_call_num_args (call
);
544 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
546 if (ipa_node_params_vector
.exists ())
548 for (n
= 0; n
< arg_num
; n
++)
550 tree arg
= gimple_call_arg (call
, n
);
551 if (POINTER_TYPE_P (TREE_TYPE (arg
)))
553 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, n
);
554 if (jfunc
->type
== IPA_JF_KNOWN_TYPE
555 || jfunc
->type
== IPA_JF_UNKNOWN
)
563 /* Returns true if hot caller heuristic should be used. */
566 enable_hot_caller_heuristic (void)
569 gcov_working_set_t
*ws
= NULL
;
570 int size_threshold
= PARAM_VALUE (PARAM_HOT_CALLER_CODESIZE_THRESHOLD
);
571 int num_counters
= 0;
572 int param_inline_hot_caller
= PARAM_VALUE (PARAM_INLINE_HOT_CALLER
);
574 if (param_inline_hot_caller
== 0)
576 else if (param_inline_hot_caller
== 1)
579 ws
= find_working_set(PARAM_VALUE (HOT_BB_COUNT_WS_PERMILLE
));
582 num_counters
= ws
->num_counters
;
583 return num_counters
<= size_threshold
;
586 /* Returns true if an edge or its caller are hot enough to
587 be considered for inlining. */
590 edge_hot_enough_p (struct cgraph_edge
*edge
)
592 static bool use_hot_caller_heuristic
= enable_hot_caller_heuristic ();
593 if (cgraph_maybe_hot_edge_p (edge
))
596 if (use_hot_caller_heuristic
)
598 struct cgraph_node
*where
= edge
->caller
;
599 if (maybe_hot_count_p (NULL
, where
->max_bb_count
))
601 if (PARAM_VALUE (PARAM_INLINE_USEFUL_COLD_CALLEE
))
602 return useful_cold_callee (edge
);
611 /* Return true if we are interested in inlining small function.
612 When REPORT is true, report reason to dump file. */
615 want_inline_small_function_p (struct cgraph_edge
*e
, bool report
)
617 bool want_inline
= true;
618 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
620 if (DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
622 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
623 && !flag_inline_small_functions
)
625 e
->inline_failed
= CIF_FUNCTION_NOT_INLINE_CANDIDATE
;
630 int growth
= estimate_edge_growth (e
);
631 inline_hints hints
= estimate_edge_hints (e
);
632 bool big_speedup
= big_speedup_p (e
);
636 /* Apply MAX_INLINE_INSNS_SINGLE limit. Do not do so when
637 hints suggests that inlining given function is very profitable. */
638 else if (DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
639 && growth
>= MAX_INLINE_INSNS_SINGLE
641 && !(hints
& (INLINE_HINT_indirect_call
642 | INLINE_HINT_loop_iterations
643 | INLINE_HINT_array_index
644 | INLINE_HINT_loop_stride
)))
646 e
->inline_failed
= CIF_MAX_INLINE_INSNS_SINGLE_LIMIT
;
649 /* Before giving up based on fact that caller size will grow, allow
650 functions that are called few times and eliminating the offline
651 copy will lead to overall code size reduction.
652 Not all of these will be handled by subsequent inlining of functions
653 called once: in particular weak functions are not handled or funcitons
654 that inline to multiple calls but a lot of bodies is optimized out.
655 Finally we want to inline earlier to allow inlining of callbacks.
657 This is slightly wrong on aggressive side: it is entirely possible
658 that function is called many times with a context where inlining
659 reduces code size and few times with a context where inlining increase
660 code size. Resoluting growth estimate will be negative even if it
661 would make more sense to keep offline copy and do not inline into the
662 call sites that makes the code size grow.
664 When badness orders the calls in a way that code reducing calls come
665 first, this situation is not a problem at all: after inlining all
666 "good" calls, we will realize that keeping the function around is
668 else if (growth
<= MAX_INLINE_INSNS_SINGLE
669 /* Unlike for functions called once, we play unsafe with
670 COMDATs. We can allow that since we know functions
671 in consideration are small (and thus risk is small) and
672 moreover grow estimates already accounts that COMDAT
673 functions may or may not disappear when eliminated from
674 current unit. With good probability making aggressive
675 choice in all units is going to make overall program
678 Consequently we ask cgraph_can_remove_if_no_direct_calls_p
680 cgraph_will_be_removed_from_program_if_no_direct_calls */
681 && !DECL_EXTERNAL (callee
->symbol
.decl
)
682 && cgraph_can_remove_if_no_direct_calls_p (callee
)
683 && estimate_growth (callee
) <= 0)
685 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
686 && !flag_inline_functions
)
688 e
->inline_failed
= CIF_NOT_DECLARED_INLINED
;
691 /* Apply MAX_INLINE_INSNS_AUTO limit for functions not declared inline
692 Upgrade it to MAX_INLINE_INSNS_SINGLE when hints suggests that
693 inlining given function is very profitable. */
694 else if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
696 && growth
>= ((hints
& (INLINE_HINT_indirect_call
697 | INLINE_HINT_loop_iterations
698 | INLINE_HINT_array_index
699 | INLINE_HINT_loop_stride
))
700 ? MAX (MAX_INLINE_INSNS_AUTO
,
701 MAX_INLINE_INSNS_SINGLE
)
702 : MAX_INLINE_INSNS_AUTO
))
704 e
->inline_failed
= CIF_MAX_INLINE_INSNS_AUTO_LIMIT
;
707 /* If call is cold, do not inline when function body would grow. */
708 else if (!edge_hot_enough_p (e
))
710 e
->inline_failed
= CIF_UNLIKELY_CALL
;
714 if (!want_inline
&& report
)
715 report_inline_failed_reason (e
);
719 /* EDGE is self recursive edge.
720 We hand two cases - when function A is inlining into itself
721 or when function A is being inlined into another inliner copy of function
724 In first case OUTER_NODE points to the toplevel copy of A, while
725 in the second case OUTER_NODE points to the outermost copy of A in B.
727 In both cases we want to be extra selective since
728 inlining the call will just introduce new recursive calls to appear. */
731 want_inline_self_recursive_call_p (struct cgraph_edge
*edge
,
732 struct cgraph_node
*outer_node
,
736 char const *reason
= NULL
;
737 bool want_inline
= true;
738 int caller_freq
= CGRAPH_FREQ_BASE
;
739 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
741 if (DECL_DECLARED_INLINE_P (edge
->caller
->symbol
.decl
))
742 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
744 if (!cgraph_maybe_hot_edge_p (edge
))
746 reason
= "recursive call is cold";
749 else if (max_count
&& !outer_node
->count
)
751 reason
= "not executed in profile";
754 else if (depth
> max_depth
)
756 reason
= "--param max-inline-recursive-depth exceeded.";
760 if (outer_node
->global
.inlined_to
)
761 caller_freq
= outer_node
->callers
->frequency
;
765 /* Inlining of self recursive function into copy of itself within other function
766 is transformation similar to loop peeling.
768 Peeling is profitable if we can inline enough copies to make probability
769 of actual call to the self recursive function very small. Be sure that
770 the probability of recursion is small.
772 We ensure that the frequency of recursing is at most 1 - (1/max_depth).
773 This way the expected number of recision is at most max_depth. */
776 int max_prob
= CGRAPH_FREQ_BASE
- ((CGRAPH_FREQ_BASE
+ max_depth
- 1)
779 for (i
= 1; i
< depth
; i
++)
780 max_prob
= max_prob
* max_prob
/ CGRAPH_FREQ_BASE
;
782 && (edge
->count
* CGRAPH_FREQ_BASE
/ outer_node
->count
785 reason
= "profile of recursive call is too large";
789 && (edge
->frequency
* CGRAPH_FREQ_BASE
/ caller_freq
792 reason
= "frequency of recursive call is too large";
796 /* Recursive inlining, i.e. equivalent of unrolling, is profitable if recursion
797 depth is large. We reduce function call overhead and increase chances that
798 things fit in hardware return predictor.
800 Recursive inlining might however increase cost of stack frame setup
801 actually slowing down functions whose recursion tree is wide rather than
804 Deciding reliably on when to do recursive inlining without profile feedback
805 is tricky. For now we disable recursive inlining when probability of self
808 Recursive inlining of self recursive call within loop also results in large loop
809 depths that generally optimize badly. We may want to throttle down inlining
810 in those cases. In particular this seems to happen in one of libstdc++ rb tree
815 && (edge
->count
* 100 / outer_node
->count
816 <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
)))
818 reason
= "profile of recursive call is too small";
822 && (edge
->frequency
* 100 / caller_freq
823 <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
)))
825 reason
= "frequency of recursive call is too small";
829 if (!want_inline
&& dump_file
)
830 fprintf (dump_file
, " not inlining recursively: %s\n", reason
);
834 /* Return true when NODE has caller other than EDGE.
835 Worker for cgraph_for_node_and_aliases. */
838 check_caller_edge (struct cgraph_node
*node
, void *edge
)
840 return (node
->callers
841 && node
->callers
!= edge
);
845 /* Decide if inlining NODE would reduce unit size by eliminating
846 the offline copy of function.
847 When COLD is true the cold calls are considered, too. */
850 want_inline_function_to_all_callers_p (struct cgraph_node
*node
, bool cold
)
852 struct cgraph_node
*function
= cgraph_function_or_thunk_node (node
, NULL
);
853 struct cgraph_edge
*e
;
854 bool has_hot_call
= false;
856 /* Does it have callers? */
859 /* Already inlined? */
860 if (function
->global
.inlined_to
)
862 if (cgraph_function_or_thunk_node (node
, NULL
) != node
)
864 /* Inlining into all callers would increase size? */
865 if (estimate_growth (node
) > 0)
867 /* Maybe other aliases has more direct calls. */
868 if (cgraph_for_node_and_aliases (node
, check_caller_edge
, node
->callers
, true))
870 /* All inlines must be possible. */
871 for (e
= node
->callers
; e
; e
= e
->next_caller
)
873 if (!can_inline_edge_p (e
, true))
875 if (!has_hot_call
&& cgraph_maybe_hot_edge_p (e
))
879 if (!cold
&& !has_hot_call
)
884 #define RELATIVE_TIME_BENEFIT_RANGE (INT_MAX / 64)
886 /* Return true if FUNCDECL is a function with fixed
890 fixed_arg_function_p (tree fndecl
)
892 tree fntype
= TREE_TYPE (fndecl
);
893 return (TYPE_ARG_TYPES (fntype
) == 0
894 || (TREE_VALUE (tree_last (TYPE_ARG_TYPES (fntype
)))
898 /* For profile collection with flag_dyn_ipa (LIPO), we always
899 want to inline comdat functions for the following reasons:
900 1) Functions in comdat may be actually defined in a different
901 module (depending on how linker picks). This results in a edge
902 from one module to another module in the dynamic callgraph.
903 The edge is false and result in unnecessary module grouping.
904 2) The profile counters in comdat functions are not 'comdated'
905 -- which means each copy of the same comdat function has its
906 own set of counters. With inlining, we are actually splitting
907 the counters and make the profile information 'context sensitive',
908 which is a good thing.
909 3) During profile-use pass of LIPO (flag_dyn_ipa == 1),
910 the pre-tree_profile inline decisions have to be the same as the
911 profile-gen pass (otherwise coverage mismatch will occur). Due to
912 this reason, it is better for each module to 'use' the comdat copy
913 of its own. The only way to get profile data for the copy is to
914 inline the copy in profile-gen phase.
915 TODO: For indirectly called comdat functions, the above issues
919 better_inline_comdat_function_p (struct cgraph_node
*node
)
921 return (profile_arc_flag
&& flag_dyn_ipa
922 && DECL_COMDAT (node
->symbol
.decl
)
923 && inline_summary (node
)->size
924 <= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_SINGLE
)
925 && fixed_arg_function_p (node
->symbol
.decl
));
929 /* Return relative time improvement for inlining EDGE in range
930 1...RELATIVE_TIME_BENEFIT_RANGE */
933 relative_time_benefit (struct inline_summary
*callee_info
,
934 struct cgraph_edge
*edge
,
937 gcov_type relbenefit
;
938 gcov_type uninlined_call_time
= compute_uninlined_call_time (callee_info
, edge
);
939 gcov_type inlined_call_time
= compute_inlined_call_time (edge
, edge_time
);
941 /* Inlining into extern inline function is not a win. */
942 if (DECL_EXTERNAL (edge
->caller
->global
.inlined_to
943 ? edge
->caller
->global
.inlined_to
->symbol
.decl
944 : edge
->caller
->symbol
.decl
))
947 /* Watch overflows. */
948 gcc_checking_assert (uninlined_call_time
>= 0);
949 gcc_checking_assert (inlined_call_time
>= 0);
950 gcc_checking_assert (uninlined_call_time
>= inlined_call_time
);
952 /* Compute relative time benefit, i.e. how much the call becomes faster.
953 ??? perhaps computing how much the caller+calle together become faster
954 would lead to more realistic results. */
955 if (!uninlined_call_time
)
956 uninlined_call_time
= 1;
958 RDIV (((gcov_type
)uninlined_call_time
- inlined_call_time
) * RELATIVE_TIME_BENEFIT_RANGE
,
959 uninlined_call_time
);
960 relbenefit
= MIN (relbenefit
, RELATIVE_TIME_BENEFIT_RANGE
);
961 gcc_checking_assert (relbenefit
>= 0);
962 relbenefit
= MAX (relbenefit
, 1);
967 /* A cost model driving the inlining heuristics in a way so the edges with
968 smallest badness are inlined first. After each inlining is performed
969 the costs of all caller edges of nodes affected are recomputed so the
970 metrics may accurately depend on values such as number of inlinable callers
971 of the function or function body size. */
974 edge_badness (struct cgraph_edge
*edge
, bool dump
)
977 int growth
, edge_time
;
978 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (edge
->callee
,
980 struct inline_summary
*callee_info
= inline_summary (callee
);
983 if (DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
986 growth
= estimate_edge_growth (edge
);
987 edge_time
= estimate_edge_time (edge
);
988 hints
= estimate_edge_hints (edge
);
989 gcc_checking_assert (edge_time
>= 0);
990 gcc_checking_assert (edge_time
<= callee_info
->time
);
991 gcc_checking_assert (growth
<= callee_info
->size
);
995 fprintf (dump_file
, " Badness calculation for %s/%i -> %s/%i\n",
996 xstrdup (cgraph_node_name (edge
->caller
)),
998 xstrdup (cgraph_node_name (callee
)),
1000 fprintf (dump_file
, " size growth %i, time %i ",
1003 dump_inline_hints (dump_file
, hints
);
1004 if (big_speedup_p (edge
))
1005 fprintf (dump_file
, " big_speedup");
1006 fprintf (dump_file
, "\n");
1009 /* Always prefer inlining saving code size. */
1012 badness
= INT_MIN
/ 2 + growth
;
1014 fprintf (dump_file
, " %i: Growth %i <= 0\n", (int) badness
,
1018 /* When profiling is available, compute badness as:
1020 relative_edge_count * relative_time_benefit
1021 goodness = -------------------------------------------
1025 The fraction is upside down, because on edge counts and time beneits
1026 the bounds are known. Edge growth is essentially unlimited. */
1030 int relbenefit
= relative_time_benefit (callee_info
, edge
, edge_time
);
1033 ((double) edge
->count
* INT_MIN
/ 2 / max_count
/ RELATIVE_TIME_BENEFIT_RANGE
) *
1034 relbenefit
) / growth
;
1036 /* Be sure that insanity of the profile won't lead to increasing counts
1037 in the scalling and thus to overflow in the computation above. */
1038 gcc_assert (max_count
>= edge
->count
);
1042 " %i (relative %f): profile info. Relative count %f"
1043 " * Relative benefit %f\n",
1044 (int) badness
, (double) badness
/ INT_MIN
,
1045 (double) edge
->count
/ max_count
,
1046 relbenefit
* 100.0 / RELATIVE_TIME_BENEFIT_RANGE
);
1050 /* When function local profile is available. Compute badness as:
1052 relative_time_benefit
1053 goodness = ---------------------------------
1054 growth_of_caller * overall_growth
1056 badness = - goodness
1058 compensated by the inline hints.
1060 else if (flag_guess_branch_prob
)
1062 badness
= (relative_time_benefit (callee_info
, edge
, edge_time
)
1063 * (INT_MIN
/ 16 / RELATIVE_TIME_BENEFIT_RANGE
));
1064 badness
/= (MIN (65536/2, growth
) * MIN (65536/2, MAX (1, callee_info
->growth
)));
1065 gcc_checking_assert (badness
<=0 && badness
>= INT_MIN
/ 16);
1066 if ((hints
& (INLINE_HINT_indirect_call
1067 | INLINE_HINT_loop_iterations
1068 | INLINE_HINT_array_index
1069 | INLINE_HINT_loop_stride
))
1070 || callee_info
->growth
<= 0)
1072 if (hints
& (INLINE_HINT_same_scc
))
1074 else if (hints
& (INLINE_HINT_in_scc
))
1076 else if (hints
& (INLINE_HINT_cross_module
))
1078 gcc_checking_assert (badness
<= 0 && badness
>= INT_MIN
/ 2);
1079 if ((hints
& INLINE_HINT_declared_inline
) && badness
>= INT_MIN
/ 32)
1084 " %i: guessed profile. frequency %f,"
1085 " benefit %f%%, time w/o inlining %i, time w inlining %i"
1086 " overall growth %i (current) %i (original)\n",
1087 (int) badness
, (double)edge
->frequency
/ CGRAPH_FREQ_BASE
,
1088 relative_time_benefit (callee_info
, edge
, edge_time
) * 100.0
1089 / RELATIVE_TIME_BENEFIT_RANGE
,
1090 (int)compute_uninlined_call_time (callee_info
, edge
),
1091 (int)compute_inlined_call_time (edge
, edge_time
),
1092 estimate_growth (callee
),
1093 callee_info
->growth
);
1096 /* When function local profile is not available or it does not give
1097 useful information (ie frequency is zero), base the cost on
1098 loop nest and overall size growth, so we optimize for overall number
1099 of functions fully inlined in program. */
1102 int nest
= MIN (inline_edge_summary (edge
)->loop_depth
, 8);
1103 badness
= growth
* 256;
1105 /* Decrease badness if call is nested. */
1113 fprintf (dump_file
, " %i: no profile. nest %i\n", (int) badness
,
1117 /* Ensure that we did not overflow in all the fixed point math above. */
1118 gcc_assert (badness
>= INT_MIN
);
1119 gcc_assert (badness
<= INT_MAX
- 1);
1120 /* Make recursive inlining happen always after other inlining is done. */
1121 if (cgraph_edge_recursive_p (edge
))
1125 if (better_inline_comdat_function_p (edge
->callee
))
1132 /* Recompute badness of EDGE and update its key in HEAP if needed. */
1134 update_edge_key (fibheap_t heap
, struct cgraph_edge
*edge
)
1136 int badness
= edge_badness (edge
, false);
1139 fibnode_t n
= (fibnode_t
) edge
->aux
;
1140 gcc_checking_assert (n
->data
== edge
);
1142 /* fibheap_replace_key only decrease the keys.
1143 When we increase the key we do not update heap
1144 and instead re-insert the element once it becomes
1145 a minimum of heap. */
1146 if (badness
< n
->key
)
1148 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1151 " decreasing badness %s/%i -> %s/%i, %i to %i\n",
1152 xstrdup (cgraph_node_name (edge
->caller
)),
1154 xstrdup (cgraph_node_name (edge
->callee
)),
1159 fibheap_replace_key (heap
, n
, badness
);
1160 gcc_checking_assert (n
->key
== badness
);
1165 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1168 " enqueuing call %s/%i -> %s/%i, badness %i\n",
1169 xstrdup (cgraph_node_name (edge
->caller
)),
1171 xstrdup (cgraph_node_name (edge
->callee
)),
1175 edge
->aux
= fibheap_insert (heap
, badness
, edge
);
1180 /* NODE was inlined.
1181 All caller edges needs to be resetted because
1182 size estimates change. Similarly callees needs reset
1183 because better context may be known. */
1186 reset_edge_caches (struct cgraph_node
*node
)
1188 struct cgraph_edge
*edge
;
1189 struct cgraph_edge
*e
= node
->callees
;
1190 struct cgraph_node
*where
= node
;
1192 struct ipa_ref
*ref
;
1194 if (where
->global
.inlined_to
)
1195 where
= where
->global
.inlined_to
;
1197 /* WHERE body size has changed, the cached growth is invalid. */
1198 reset_node_growth_cache (where
);
1200 for (edge
= where
->callers
; edge
; edge
= edge
->next_caller
)
1201 if (edge
->inline_failed
)
1202 reset_edge_growth_cache (edge
);
1203 for (i
= 0; ipa_ref_list_referring_iterate (&where
->symbol
.ref_list
,
1205 if (ref
->use
== IPA_REF_ALIAS
)
1206 reset_edge_caches (ipa_ref_referring_node (ref
));
1212 if (!e
->inline_failed
&& e
->callee
->callees
)
1213 e
= e
->callee
->callees
;
1216 if (e
->inline_failed
)
1217 reset_edge_growth_cache (e
);
1224 if (e
->caller
== node
)
1226 e
= e
->caller
->callers
;
1228 while (!e
->next_callee
);
1234 /* Recompute HEAP nodes for each of caller of NODE.
1235 UPDATED_NODES track nodes we already visited, to avoid redundant work.
1236 When CHECK_INLINABLITY_FOR is set, re-check for specified edge that
1237 it is inlinable. Otherwise check all edges. */
1240 update_caller_keys (fibheap_t heap
, struct cgraph_node
*node
,
1241 bitmap updated_nodes
,
1242 struct cgraph_edge
*check_inlinablity_for
)
1244 struct cgraph_edge
*edge
;
1246 struct ipa_ref
*ref
;
1248 if ((!node
->alias
&& !inline_summary (node
)->inlinable
)
1249 || cgraph_function_body_availability (node
) <= AVAIL_OVERWRITABLE
1250 || node
->global
.inlined_to
)
1252 if (!bitmap_set_bit (updated_nodes
, node
->uid
))
1255 for (i
= 0; ipa_ref_list_referring_iterate (&node
->symbol
.ref_list
,
1257 if (ref
->use
== IPA_REF_ALIAS
)
1259 struct cgraph_node
*alias
= ipa_ref_referring_node (ref
);
1260 update_caller_keys (heap
, alias
, updated_nodes
, check_inlinablity_for
);
1263 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1264 if (edge
->inline_failed
)
1266 if (!check_inlinablity_for
1267 || check_inlinablity_for
== edge
)
1269 if (can_inline_edge_p (edge
, false)
1270 && (want_inline_small_function_p (edge
, false)
1271 || better_inline_comdat_function_p (node
)))
1272 update_edge_key (heap
, edge
);
1275 report_inline_failed_reason (edge
);
1276 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
1281 update_edge_key (heap
, edge
);
1285 /* Recompute HEAP nodes for each uninlined call in NODE.
1286 This is used when we know that edge badnesses are going only to increase
1287 (we introduced new call site) and thus all we need is to insert newly
1288 created edges into heap. */
1291 update_callee_keys (fibheap_t heap
, struct cgraph_node
*node
,
1292 bitmap updated_nodes
)
1294 struct cgraph_edge
*e
= node
->callees
;
1299 if (!e
->inline_failed
&& e
->callee
->callees
)
1300 e
= e
->callee
->callees
;
1303 enum availability avail
;
1304 struct cgraph_node
*callee
;
1305 /* We do not reset callee growth cache here. Since we added a new call,
1306 growth chould have just increased and consequentely badness metric
1307 don't need updating. */
1308 if (e
->inline_failed
1309 && (callee
= cgraph_function_or_thunk_node (e
->callee
, &avail
))
1310 && inline_summary (callee
)->inlinable
1311 && cgraph_function_body_availability (callee
) >= AVAIL_AVAILABLE
1312 && !bitmap_bit_p (updated_nodes
, callee
->uid
))
1314 if (can_inline_edge_p (e
, false)
1315 && want_inline_small_function_p (e
, false))
1316 update_edge_key (heap
, e
);
1319 report_inline_failed_reason (e
);
1320 fibheap_delete_node (heap
, (fibnode_t
) e
->aux
);
1330 if (e
->caller
== node
)
1332 e
= e
->caller
->callers
;
1334 while (!e
->next_callee
);
1340 /* Enqueue all recursive calls from NODE into priority queue depending on
1341 how likely we want to recursively inline the call. */
1344 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
1347 struct cgraph_edge
*e
;
1348 enum availability avail
;
1350 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1351 if (e
->callee
== node
1352 || (cgraph_function_or_thunk_node (e
->callee
, &avail
) == node
1353 && avail
> AVAIL_OVERWRITABLE
))
1355 /* When profile feedback is available, prioritize by expected number
1357 fibheap_insert (heap
,
1358 !max_count
? -e
->frequency
1359 : -(e
->count
/ ((max_count
+ (1<<24) - 1) / (1<<24))),
1362 for (e
= where
->callees
; e
; e
= e
->next_callee
)
1363 if (!e
->inline_failed
)
1364 lookup_recursive_calls (node
, e
->callee
, heap
);
1367 /* Decide on recursive inlining: in the case function has recursive calls,
1368 inline until body size reaches given argument. If any new indirect edges
1369 are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES
1373 recursive_inlining (struct cgraph_edge
*edge
,
1374 vec
<cgraph_edge_p
> *new_edges
)
1376 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
1377 int probability
= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
);
1379 struct cgraph_node
*node
;
1380 struct cgraph_edge
*e
;
1381 struct cgraph_node
*master_clone
= NULL
, *next
;
1385 node
= edge
->caller
;
1386 if (node
->global
.inlined_to
)
1387 node
= node
->global
.inlined_to
;
1389 if (DECL_DECLARED_INLINE_P (node
->symbol
.decl
))
1390 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
1392 /* Make sure that function is small enough to be considered for inlining. */
1393 if (estimate_size_after_inlining (node
, edge
) >= limit
)
1395 heap
= fibheap_new ();
1396 lookup_recursive_calls (node
, node
, heap
);
1397 if (fibheap_empty (heap
))
1399 fibheap_delete (heap
);
1405 " Performing recursive inlining on %s\n",
1406 cgraph_node_name (node
));
1408 /* Do the inlining and update list of recursive call during process. */
1409 while (!fibheap_empty (heap
))
1411 struct cgraph_edge
*curr
1412 = (struct cgraph_edge
*) fibheap_extract_min (heap
);
1413 struct cgraph_node
*cnode
, *dest
= curr
->callee
;
1415 if (!can_inline_edge_p (curr
, true))
1418 /* MASTER_CLONE is produced in the case we already started modified
1419 the function. Be sure to redirect edge to the original body before
1420 estimating growths otherwise we will be seeing growths after inlining
1421 the already modified body. */
1424 cgraph_redirect_edge_callee (curr
, master_clone
);
1425 reset_edge_growth_cache (curr
);
1428 if (estimate_size_after_inlining (node
, curr
) > limit
)
1430 cgraph_redirect_edge_callee (curr
, dest
);
1431 reset_edge_growth_cache (curr
);
1436 for (cnode
= curr
->caller
;
1437 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
1438 if (node
->symbol
.decl
1439 == cgraph_function_or_thunk_node (curr
->callee
, NULL
)->symbol
.decl
)
1444 if (!cgraph_maybe_hot_edge_p (curr
))
1447 fprintf (dump_file
, " Not inlining cold call\n");
1449 cgraph_redirect_edge_callee (curr
, dest
);
1450 reset_edge_growth_cache (curr
);
1453 if (node
->count
== 0 || curr
->count
* 100 / node
->count
< probability
)
1457 " Probability of edge is too small\n");
1459 cgraph_redirect_edge_callee (curr
, dest
);
1460 reset_edge_growth_cache (curr
);
1465 if (!want_inline_self_recursive_call_p (curr
, node
, false, depth
))
1467 cgraph_redirect_edge_callee (curr
, dest
);
1468 reset_edge_growth_cache (curr
);
1478 " Inlining call of depth %i", depth
);
1481 fprintf (dump_file
, " called approx. %.2f times per call",
1482 (double)curr
->count
/ node
->count
);
1484 fprintf (dump_file
, "\n");
1488 /* We need original clone to copy around. */
1489 master_clone
= cgraph_clone_node (node
, node
->symbol
.decl
,
1490 node
->count
, CGRAPH_FREQ_BASE
,
1491 false, vNULL
, true);
1492 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
1493 if (!e
->inline_failed
)
1494 clone_inlined_nodes (e
, true, false, NULL
);
1495 cgraph_redirect_edge_callee (curr
, master_clone
);
1496 reset_edge_growth_cache (curr
);
1499 inline_call (curr
, false, new_edges
, &overall_size
, true);
1500 lookup_recursive_calls (node
, curr
->callee
, heap
);
1504 if (!fibheap_empty (heap
) && dump_file
)
1505 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
1506 fibheap_delete (heap
);
1513 "\n Inlined %i times, "
1514 "body grown from size %i to %i, time %i to %i\n", n
,
1515 inline_summary (master_clone
)->size
, inline_summary (node
)->size
,
1516 inline_summary (master_clone
)->time
, inline_summary (node
)->time
);
1518 /* Remove master clone we used for inlining. We rely that clones inlined
1519 into master clone gets queued just before master clone so we don't
1521 for (node
= cgraph_first_function (); node
!= master_clone
;
1524 next
= cgraph_next_function (node
);
1525 if (node
->global
.inlined_to
== master_clone
)
1526 cgraph_remove_node (node
);
1528 cgraph_remove_node (master_clone
);
1533 /* Given whole compilation unit estimate of INSNS, compute how large we can
1534 allow the unit to grow. */
1537 compute_max_insns (int insns
)
1539 int max_insns
= insns
;
1540 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
1541 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
1543 return ((HOST_WIDEST_INT
) max_insns
1544 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
1548 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
1551 add_new_edges_to_heap (fibheap_t heap
, vec
<cgraph_edge_p
> new_edges
)
1553 while (new_edges
.length () > 0)
1555 struct cgraph_edge
*edge
= new_edges
.pop ();
1557 gcc_assert (!edge
->aux
);
1558 if (edge
->inline_failed
1559 && can_inline_edge_p (edge
, true)
1560 && want_inline_small_function_p (edge
, true))
1561 edge
->aux
= fibheap_insert (heap
, edge_badness (edge
, false), edge
);
1566 /* We use greedy algorithm for inlining of small functions:
1567 All inline candidates are put into prioritized heap ordered in
1570 The inlining of small functions is bounded by unit growth parameters. */
1573 inline_small_functions (void)
1575 struct cgraph_node
*node
;
1576 struct cgraph_edge
*edge
;
1577 fibheap_t edge_heap
= fibheap_new ();
1578 bitmap updated_nodes
= BITMAP_ALLOC (NULL
);
1579 int min_size
, max_size
;
1580 vec
<cgraph_edge_p
> new_indirect_edges
= vNULL
;
1581 int initial_size
= 0;
1582 struct cgraph_node
**order
= XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1584 if (flag_indirect_inlining
)
1585 new_indirect_edges
.create (8);
1587 /* Compute overall unit size and other global parameters used by badness
1591 ipa_reduced_postorder (order
, true, true, NULL
);
1594 FOR_EACH_DEFINED_FUNCTION (node
)
1595 if (!node
->global
.inlined_to
)
1597 if (cgraph_function_with_gimple_body_p (node
)
1598 || node
->thunk
.thunk_p
)
1600 struct inline_summary
*info
= inline_summary (node
);
1601 struct ipa_dfs_info
*dfs
= (struct ipa_dfs_info
*) node
->symbol
.aux
;
1603 if (!DECL_EXTERNAL (node
->symbol
.decl
))
1604 initial_size
+= info
->size
;
1605 info
->growth
= estimate_growth (node
);
1606 if (dfs
&& dfs
->next_cycle
)
1608 struct cgraph_node
*n2
;
1609 int id
= dfs
->scc_no
+ 1;
1611 n2
= ((struct ipa_dfs_info
*) node
->symbol
.aux
)->next_cycle
)
1613 struct inline_summary
*info2
= inline_summary (n2
);
1621 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1622 if (max_count
< edge
->count
)
1623 max_count
= edge
->count
;
1624 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1625 if (max_count
< edge
->count
)
1626 max_count
= edge
->count
;
1628 ipa_free_postorder_info ();
1629 initialize_growth_caches ();
1633 "\nDeciding on inlining of small functions. Starting with size %i.\n",
1636 overall_size
= initial_size
;
1637 max_size
= compute_max_insns (overall_size
);
1638 min_size
= overall_size
;
1640 /* Populate the heeap with all edges we might inline. */
1642 FOR_EACH_DEFINED_FUNCTION (node
)
1643 if (!node
->global
.inlined_to
)
1646 fprintf (dump_file
, "Enqueueing calls of %s/%i.\n",
1647 cgraph_node_name (node
), node
->uid
);
1649 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
1650 if (edge
->inline_failed
1651 && can_inline_edge_p (edge
, true)
1652 && (want_inline_small_function_p (edge
, true)
1653 || better_inline_comdat_function_p (node
))
1654 && edge
->inline_failed
)
1656 gcc_assert (!edge
->aux
);
1657 update_edge_key (edge_heap
, edge
);
1661 gcc_assert (in_lto_p
1663 || (profile_info
&& flag_branch_probabilities
));
1665 while (!fibheap_empty (edge_heap
))
1667 int old_size
= overall_size
;
1668 struct cgraph_node
*where
, *callee
;
1669 int badness
= fibheap_min_key (edge_heap
);
1670 int current_badness
;
1674 edge
= (struct cgraph_edge
*) fibheap_extract_min (edge_heap
);
1675 gcc_assert (edge
->aux
);
1677 if (!edge
->inline_failed
)
1680 if (L_IPO_COMP_MODE
&& !edge
->call_stmt
)
1683 /* Be sure that caches are maintained consistent.
1684 We can not make this ENABLE_CHECKING only because it cause different
1685 updates of the fibheap queue. */
1686 cached_badness
= edge_badness (edge
, false);
1687 reset_edge_growth_cache (edge
);
1688 reset_node_growth_cache (edge
->callee
);
1690 /* When updating the edge costs, we only decrease badness in the keys.
1691 Increases of badness are handled lazilly; when we see key with out
1692 of date value on it, we re-insert it now. */
1693 current_badness
= edge_badness (edge
, false);
1694 gcc_assert (cached_badness
== current_badness
);
1695 gcc_assert (current_badness
>= badness
);
1696 if (current_badness
!= badness
)
1698 edge
->aux
= fibheap_insert (edge_heap
, current_badness
, edge
);
1702 if (!can_inline_edge_p (edge
, true))
1705 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1706 growth
= estimate_edge_growth (edge
);
1710 "\nConsidering %s with %i size\n",
1711 cgraph_node_name (callee
),
1712 inline_summary (callee
)->size
);
1714 " to be inlined into %s in %s:%i\n"
1715 " Estimated growth after inlined into all is %+i insns.\n"
1716 " Estimated badness is %i, frequency %.2f.\n",
1717 cgraph_node_name (edge
->caller
),
1718 flag_wpa
? "unknown"
1719 : gimple_filename ((const_gimple
) edge
->call_stmt
),
1721 : gimple_lineno ((const_gimple
) edge
->call_stmt
),
1722 estimate_growth (callee
),
1724 edge
->frequency
/ (double)CGRAPH_FREQ_BASE
);
1726 fprintf (dump_file
," Called "HOST_WIDEST_INT_PRINT_DEC
"x\n",
1728 if (dump_flags
& TDF_DETAILS
)
1729 edge_badness (edge
, true);
1732 if (overall_size
+ growth
> max_size
1733 && !DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
1735 edge
->inline_failed
= CIF_INLINE_UNIT_GROWTH_LIMIT
;
1736 report_inline_failed_reason (edge
);
1740 if (!want_inline_small_function_p (edge
, true)
1741 && !better_inline_comdat_function_p (edge
->callee
))
1744 /* Heuristics for inlining small functions works poorly for
1745 recursive calls where we do efect similar to loop unrolling.
1746 When inliing such edge seems profitable, leave decision on
1747 specific inliner. */
1748 if (cgraph_edge_recursive_p (edge
))
1750 where
= edge
->caller
;
1751 if (where
->global
.inlined_to
)
1752 where
= where
->global
.inlined_to
;
1753 if (!recursive_inlining (edge
,
1754 flag_indirect_inlining
1755 ? &new_indirect_edges
: NULL
))
1757 edge
->inline_failed
= CIF_RECURSIVE_INLINING
;
1760 reset_edge_caches (where
);
1761 /* Recursive inliner inlines all recursive calls of the function
1762 at once. Consequently we need to update all callee keys. */
1763 if (flag_indirect_inlining
)
1764 add_new_edges_to_heap (edge_heap
, new_indirect_edges
);
1765 update_callee_keys (edge_heap
, where
, updated_nodes
);
1769 struct cgraph_node
*outer_node
= NULL
;
1775 /* Consider the case where self recursive function A is inlined into B.
1776 This is desired optimization in some cases, since it leads to effect
1777 similar of loop peeling and we might completely optimize out the
1778 recursive call. However we must be extra selective. */
1780 where
= edge
->caller
;
1781 while (where
->global
.inlined_to
)
1783 if (where
->symbol
.decl
== callee
->symbol
.decl
)
1784 outer_node
= where
, depth
++;
1785 where
= where
->callers
->caller
;
1788 && !want_inline_self_recursive_call_p (edge
, outer_node
,
1792 = (DECL_DISREGARD_INLINE_LIMITS (edge
->callee
->symbol
.decl
)
1793 ? CIF_RECURSIVE_INLINING
: CIF_UNSPECIFIED
);
1796 else if (depth
&& dump_file
)
1797 fprintf (dump_file
, " Peeling recursion with depth %i\n", depth
);
1799 gcc_checking_assert (!callee
->global
.inlined_to
);
1800 inline_call (edge
, true, &new_indirect_edges
, &overall_size
, true);
1801 if (flag_indirect_inlining
)
1802 add_new_edges_to_heap (edge_heap
, new_indirect_edges
);
1804 reset_edge_caches (edge
->callee
);
1805 reset_node_growth_cache (callee
);
1807 update_callee_keys (edge_heap
, where
, updated_nodes
);
1809 where
= edge
->caller
;
1810 if (where
->global
.inlined_to
)
1811 where
= where
->global
.inlined_to
;
1813 /* Our profitability metric can depend on local properties
1814 such as number of inlinable calls and size of the function body.
1815 After inlining these properties might change for the function we
1816 inlined into (since it's body size changed) and for the functions
1817 called by function we inlined (since number of it inlinable callers
1819 update_caller_keys (edge_heap
, where
, updated_nodes
, NULL
);
1820 bitmap_clear (updated_nodes
);
1825 "INFO: %s Inlined into %s which now has time %i and size %i,"
1826 "net change of %+i.\n",
1827 cgraph_node_name (edge
->callee
),
1828 cgraph_node_name (edge
->caller
),
1829 inline_summary (edge
->caller
)->time
,
1830 inline_summary (edge
->caller
)->size
,
1831 overall_size
- old_size
);
1833 if (min_size
> overall_size
)
1835 min_size
= overall_size
;
1836 max_size
= compute_max_insns (min_size
);
1839 fprintf (dump_file
, "New minimal size reached: %i\n", min_size
);
1843 free_growth_caches ();
1844 new_indirect_edges
.release ();
1845 fibheap_delete (edge_heap
);
1848 "Unit growth for small function inlining: %i->%i (%i%%)\n",
1849 initial_size
, overall_size
,
1850 initial_size
? overall_size
* 100 / (initial_size
) - 100: 0);
1851 BITMAP_FREE (updated_nodes
);
1854 /* Flatten NODE. Performed both during early inlining and
1855 at IPA inlining time. */
1858 flatten_function (struct cgraph_node
*node
, bool early
)
1860 struct cgraph_edge
*e
;
1862 /* We shouldn't be called recursively when we are being processed. */
1863 gcc_assert (node
->symbol
.aux
== NULL
);
1865 node
->symbol
.aux
= (void *) node
;
1867 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1869 struct cgraph_node
*orig_callee
;
1870 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
1872 /* We've hit cycle? It is time to give up. */
1873 if (callee
->symbol
.aux
)
1877 "Not inlining %s into %s to avoid cycle.\n",
1878 xstrdup (cgraph_node_name (callee
)),
1879 xstrdup (cgraph_node_name (e
->caller
)));
1880 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
1884 /* When the edge is already inlined, we just need to recurse into
1885 it in order to fully flatten the leaves. */
1886 if (!e
->inline_failed
)
1888 flatten_function (callee
, early
);
1892 /* Flatten attribute needs to be processed during late inlining. For
1893 extra code quality we however do flattening during early optimization,
1896 ? !can_inline_edge_p (e
, true)
1897 : !can_early_inline_edge_p (e
))
1900 if (cgraph_edge_recursive_p (e
))
1903 fprintf (dump_file
, "Not inlining: recursive call.\n");
1907 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->symbol
.decl
))
1908 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (callee
->symbol
.decl
)))
1911 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1915 /* Inline the edge and flatten the inline clone. Avoid
1916 recursing through the original node if the node was cloned. */
1918 fprintf (dump_file
, " Inlining %s into %s.\n",
1919 xstrdup (cgraph_node_name (callee
)),
1920 xstrdup (cgraph_node_name (e
->caller
)));
1921 orig_callee
= callee
;
1922 inline_call (e
, true, NULL
, NULL
, false);
1923 if (e
->callee
!= orig_callee
)
1924 orig_callee
->symbol
.aux
= (void *) node
;
1925 flatten_function (e
->callee
, early
);
1926 if (e
->callee
!= orig_callee
)
1927 orig_callee
->symbol
.aux
= NULL
;
1930 node
->symbol
.aux
= NULL
;
1931 if (!node
->global
.inlined_to
)
1932 inline_update_overall_summary (node
);
1935 /* Decide on the inlining. We do so in the topological order to avoid
1936 expenses on updating data structures. */
1941 struct cgraph_node
*node
;
1943 struct cgraph_node
**order
=
1944 XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1947 if (in_lto_p
&& optimize
)
1948 ipa_update_after_lto_read ();
1951 dump_inline_summaries (dump_file
);
1953 nnodes
= ipa_reverse_postorder (order
);
1955 FOR_EACH_FUNCTION (node
)
1956 node
->symbol
.aux
= 0;
1959 fprintf (dump_file
, "\nFlattening functions:\n");
1961 /* In the first pass handle functions to be flattened. Do this with
1962 a priority so none of our later choices will make this impossible. */
1963 for (i
= nnodes
- 1; i
>= 0; i
--)
1967 /* Handle nodes to be flattened.
1968 Ideally when processing callees we stop inlining at the
1969 entry of cycles, possibly cloning that entry point and
1970 try to flatten itself turning it into a self-recursive
1972 if (lookup_attribute ("flatten",
1973 DECL_ATTRIBUTES (node
->symbol
.decl
)) != NULL
)
1977 "Flattening %s\n", cgraph_node_name (node
));
1978 flatten_function (node
, false);
1982 inline_small_functions ();
1983 symtab_remove_unreachable_nodes (false, dump_file
);
1986 /* Inline functions with a property that after inlining into all callers the
1987 code size will shrink because the out-of-line copy is eliminated.
1988 We do this regardless on the callee size as long as function growth limits
1990 if (flag_inline_functions_called_once
)
1995 "\nDeciding on functions to be inlined into all callers:\n");
1997 /* Inlining one function called once has good chance of preventing
1998 inlining other function into the same callee. Ideally we should
1999 work in priority order, but probably inlining hot functions first
2000 is good cut without the extra pain of maintaining the queue.
2002 ??? this is not really fitting the bill perfectly: inlining function
2003 into callee often leads to better optimization of callee due to
2004 increased context for optimization.
2005 For example if main() function calls a function that outputs help
2006 and then function that does the main optmization, we should inline
2007 the second with priority even if both calls are cold by themselves.
2009 We probably want to implement new predicate replacing our use of
2010 maybe_hot_edge interpreted as maybe_hot_edge || callee is known
2012 for (cold
= 0; cold
<= 1; cold
++)
2014 FOR_EACH_DEFINED_FUNCTION (node
)
2016 if (want_inline_function_to_all_callers_p (node
, cold
))
2019 struct cgraph_edge
*e
;
2020 for (e
= node
->callers
; e
; e
= e
->next_caller
)
2022 while (node
->callers
&& !node
->global
.inlined_to
)
2024 struct cgraph_node
*caller
= node
->callers
->caller
;
2029 "\nInlining %s size %i.\n",
2030 cgraph_node_name (node
),
2031 inline_summary (node
)->size
);
2033 " Called once from %s %i insns.\n",
2034 cgraph_node_name (node
->callers
->caller
),
2035 inline_summary (node
->callers
->caller
)->size
);
2038 inline_call (node
->callers
, true, NULL
, NULL
, true);
2041 " Inlined into %s which now has %i size\n",
2042 cgraph_node_name (caller
),
2043 inline_summary (caller
)->size
);
2047 fprintf (dump_file
, "New calls found; giving up.\n");
2056 /* Free ipa-prop structures if they are no longer needed. */
2058 ipa_free_all_structures_after_iinln ();
2062 "\nInlined %i calls, eliminated %i functions\n\n",
2063 ncalls_inlined
, nfunctions_inlined
);
2066 dump_inline_summaries (dump_file
);
2067 /* In WPA we use inline summaries for partitioning process. */
2069 inline_free_summary ();
2073 /* Inline always-inline function calls in NODE. */
2076 inline_always_inline_functions (struct cgraph_node
*node
)
2078 struct cgraph_edge
*e
;
2079 bool inlined
= false;
2081 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2083 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
2084 if (!DECL_DISREGARD_INLINE_LIMITS (callee
->symbol
.decl
))
2087 if (cgraph_edge_recursive_p (e
))
2090 fprintf (dump_file
, " Not inlining recursive call to %s.\n",
2091 cgraph_node_name (e
->callee
));
2092 e
->inline_failed
= CIF_RECURSIVE_INLINING
;
2096 if (!can_early_inline_edge_p (e
))
2100 fprintf (dump_file
, " Inlining %s into %s (always_inline).\n",
2101 xstrdup (cgraph_node_name (e
->callee
)),
2102 xstrdup (cgraph_node_name (e
->caller
)));
2103 inline_call (e
, true, NULL
, NULL
, false);
2107 inline_update_overall_summary (node
);
2112 /* Decide on the inlining. We do so in the topological order to avoid
2113 expenses on updating data structures. */
2116 early_inline_small_functions (struct cgraph_node
*node
)
2118 struct cgraph_edge
*e
;
2119 bool inlined
= false;
2121 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2123 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
2124 if (!inline_summary (callee
)->inlinable
2125 || !e
->inline_failed
)
2128 /* Do not consider functions not declared inline. */
2129 if (!DECL_DECLARED_INLINE_P (callee
->symbol
.decl
)
2130 && !flag_inline_small_functions
2131 && !flag_inline_functions
)
2135 fprintf (dump_file
, "Considering inline candidate %s.\n",
2136 cgraph_node_name (callee
));
2138 if (!can_early_inline_edge_p (e
))
2141 if (cgraph_edge_recursive_p (e
))
2145 fprintf (dump_file
, " Not inlining: recursive call.\n");
2149 if (!want_early_inline_function_p (e
))
2153 fprintf (dump_file
, " Inlining %s into %s.\n",
2154 xstrdup (cgraph_node_name (callee
)),
2155 xstrdup (cgraph_node_name (e
->caller
)));
2156 inline_call (e
, true, NULL
, NULL
, true);
2163 /* Do inlining of small functions. Doing so early helps profiling and other
2164 passes to be somewhat more effective and avoids some code duplication in
2165 later real inlining pass for testcases with very many function calls. */
2167 early_inliner (void)
2169 struct cgraph_node
*node
= cgraph_get_node (current_function_decl
);
2170 struct cgraph_edge
*edge
;
2171 unsigned int todo
= 0;
2173 bool inlined
= false;
2178 /* Do nothing if datastructures for ipa-inliner are already computed. This
2179 happens when some pass decides to construct new function and
2180 cgraph_add_new_function calls lowering passes and early optimization on
2181 it. This may confuse ourself when early inliner decide to inline call to
2182 function clone, because function clones don't have parameter list in
2183 ipa-prop matching their signature. */
2184 if (ipa_node_params_vector
.exists ())
2187 #ifdef ENABLE_CHECKING
2188 verify_cgraph_node (node
);
2191 /* Even when not optimizing or not inlining inline always-inline
2193 inlined
= inline_always_inline_functions (node
);
2197 || !flag_early_inlining
2198 /* Never inline regular functions into always-inline functions
2199 during incremental inlining. This sucks as functions calling
2200 always inline functions will get less optimized, but at the
2201 same time inlining of functions calling always inline
2202 function into an always inline function might introduce
2203 cycles of edges to be always inlined in the callgraph.
2205 We might want to be smarter and just avoid this type of inlining. */
2206 || DECL_DISREGARD_INLINE_LIMITS (node
->symbol
.decl
))
2208 else if (lookup_attribute ("flatten",
2209 DECL_ATTRIBUTES (node
->symbol
.decl
)) != NULL
)
2211 /* When the function is marked to be flattened, recursively inline
2215 "Flattening %s\n", cgraph_node_name (node
));
2216 flatten_function (node
, true);
2221 /* We iterate incremental inlining to get trivial cases of indirect
2223 while (iterations
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
)
2224 && early_inline_small_functions (node
))
2226 timevar_push (TV_INTEGRATION
);
2227 todo
|= optimize_inline_calls (current_function_decl
);
2229 /* Technically we ought to recompute inline parameters so the new
2230 iteration of early inliner works as expected. We however have
2231 values approximately right and thus we only need to update edge
2232 info that might be cleared out for newly discovered edges. */
2233 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2235 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2237 if (!edge
->call_stmt
)
2240 = estimate_num_insns (edge
->call_stmt
, &eni_size_weights
);
2242 = estimate_num_insns (edge
->call_stmt
, &eni_time_weights
);
2243 if (edge
->callee
->symbol
.decl
2244 && !gimple_check_call_matching_types (edge
->call_stmt
,
2245 edge
->callee
->symbol
.decl
))
2246 edge
->call_stmt_cannot_inline_p
= true;
2248 timevar_pop (TV_INTEGRATION
);
2253 fprintf (dump_file
, "Iterations: %i\n", iterations
);
2258 timevar_push (TV_INTEGRATION
);
2259 todo
|= optimize_inline_calls (current_function_decl
);
2260 timevar_pop (TV_INTEGRATION
);
2263 cfun
->always_inline_functions_inlined
= true;
2268 struct gimple_opt_pass pass_early_inline
=
2272 "einline", /* name */
2273 OPTGROUP_INLINE
, /* optinfo_flags */
2275 early_inliner
, /* execute */
2278 0, /* static_pass_number */
2279 TV_EARLY_INLINING
, /* tv_id */
2280 PROP_ssa
, /* properties_required */
2281 0, /* properties_provided */
2282 0, /* properties_destroyed */
2283 0, /* todo_flags_start */
2284 0 /* todo_flags_finish */
2288 /* When to run IPA inlining. Inlining of always-inline functions
2289 happens during early inlining.
2291 Enable inlining unconditoinally at -flto. We need size estimates to
2292 drive partitioning. */
2295 gate_ipa_inline (void)
2297 return optimize
|| flag_lto
|| flag_wpa
;
2300 struct ipa_opt_pass_d pass_ipa_inline
=
2304 "inline", /* name */
2305 OPTGROUP_INLINE
, /* optinfo_flags */
2306 gate_ipa_inline
, /* gate */
2307 ipa_inline
, /* execute */
2310 0, /* static_pass_number */
2311 TV_IPA_INLINING
, /* tv_id */
2312 0, /* properties_required */
2313 0, /* properties_provided */
2314 0, /* properties_destroyed */
2315 TODO_remove_functions
, /* todo_flags_finish */
2317 | TODO_remove_functions
| TODO_ggc_collect
/* todo_flags_finish */
2319 inline_generate_summary
, /* generate_summary */
2320 inline_write_summary
, /* write_summary */
2321 inline_read_summary
, /* read_summary */
2322 NULL
, /* write_optimization_summary */
2323 NULL
, /* read_optimization_summary */
2324 NULL
, /* stmt_fixup */
2326 inline_transform
, /* function_transform */
2327 NULL
, /* variable_transform */