1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis used by the inliner and other passes limiting code size growth.
23 We estimate for each function
25 - average function execution time
26 - inlining size benefit (that is how much of function body size
27 and its call sequence is expected to disappear by inlining)
28 - inlining time benefit
31 - call statement size and time
33 inlinie_summary datastructures store above information locally (i.e.
34 parameters of the function itself) and globally (i.e. parameters of
35 the function created by applying all the inline decisions already
36 present in the callgraph).
38 We provide accestor to the inline_summary datastructure and
39 basic logic updating the parameters when inlining is performed.
41 The summaries are context sensitive. Context means
42 1) partial assignment of known constant values of operands
43 2) whether function is inlined into the call or not.
44 It is easy to add more variants. To represent function size and time
45 that depends on context (i.e. it is known to be optimized away when
46 context is known either by inlining or from IP-CP and clonning),
47 we use predicates. Predicates are logical formulas in
48 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
49 specifying what conditions must be true. Conditions are simple test
50 of the form described above.
52 In order to make predicate (possibly) true, all of its clauses must
53 be (possibly) true. To make clause (possibly) true, one of conditions
54 it mentions must be (possibly) true. There are fixed bounds on
55 number of clauses and conditions and all the manipulation functions
56 are conservative in positive direction. I.e. we may lose precision
57 by thinking that predicate may be true even when it is not.
59 estimate_edge_size and estimate_edge_growth can be used to query
60 function size/time in the given context. inline_merge_summary merges
61 properties of caller and callee after inlining.
63 Finally pass_inline_parameters is exported. This is used to drive
64 computation of function parameters used by the early inliner. IPA
65 inlined performs analysis via its analyze_function method. */
69 #include "coretypes.h"
72 #include "tree-inline.h"
73 #include "langhooks.h"
75 #include "diagnostic.h"
76 #include "gimple-pretty-print.h"
78 #include "tree-pass.h"
82 #include "gimple-ssa.h"
84 #include "tree-phinodes.h"
85 #include "ssa-iterators.h"
86 #include "tree-ssanames.h"
87 #include "tree-ssa-loop-niter.h"
88 #include "tree-ssa-loop.h"
90 #include "lto-streamer.h"
91 #include "data-streamer.h"
92 #include "tree-streamer.h"
93 #include "ipa-inline.h"
94 #include "alloc-pool.h"
96 #include "tree-scalar-evolution.h"
97 #include "ipa-utils.h"
99 /* Estimate runtime of function can easilly run into huge numbers with many
100 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
101 integer. For anything larger we use gcov_type. */
102 #define MAX_TIME 500000
104 /* Number of bits in integer, but we really want to be stable across different
106 #define NUM_CONDITIONS 32
108 enum predicate_conditions
110 predicate_false_condition
= 0,
111 predicate_not_inlined_condition
= 1,
112 predicate_first_dynamic_condition
= 2
115 /* Special condition code we use to represent test that operand is compile time
117 #define IS_NOT_CONSTANT ERROR_MARK
118 /* Special condition code we use to represent test that operand is not changed
119 across invocation of the function. When operand IS_NOT_CONSTANT it is always
120 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
121 of executions even when they are not compile time constants. */
122 #define CHANGED IDENTIFIER_NODE
124 /* Holders of ipa cgraph hooks: */
125 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
126 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
127 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
128 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
129 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
130 static void inline_node_removal_hook (struct cgraph_node
*, void *);
131 static void inline_node_duplication_hook (struct cgraph_node
*,
132 struct cgraph_node
*, void *);
133 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
134 static void inline_edge_duplication_hook (struct cgraph_edge
*,
135 struct cgraph_edge
*, void *);
137 /* VECtor holding inline summaries.
138 In GGC memory because conditions might point to constant trees. */
139 vec
<inline_summary_t
, va_gc
> *inline_summary_vec
;
140 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
142 /* Cached node/edge growths. */
143 vec
<int> node_growth_cache
;
144 vec
<edge_growth_cache_entry
> edge_growth_cache
;
146 /* Edge predicates goes here. */
147 static alloc_pool edge_predicate_pool
;
149 /* Return true predicate (tautology).
150 We represent it by empty list of clauses. */
152 static inline struct predicate
153 true_predicate (void)
161 /* Return predicate testing single condition number COND. */
163 static inline struct predicate
164 single_cond_predicate (int cond
)
167 p
.clause
[0] = 1 << cond
;
173 /* Return false predicate. First clause require false condition. */
175 static inline struct predicate
176 false_predicate (void)
178 return single_cond_predicate (predicate_false_condition
);
182 /* Return true if P is (false). */
185 true_predicate_p (struct predicate
*p
)
187 return !p
->clause
[0];
191 /* Return true if P is (false). */
194 false_predicate_p (struct predicate
*p
)
196 if (p
->clause
[0] == (1 << predicate_false_condition
))
198 gcc_checking_assert (!p
->clause
[1]
199 && p
->clause
[0] == 1 << predicate_false_condition
);
206 /* Return predicate that is set true when function is not inlined. */
208 static inline struct predicate
209 not_inlined_predicate (void)
211 return single_cond_predicate (predicate_not_inlined_condition
);
214 /* Simple description of whether a memory load or a condition refers to a load
215 from an aggregate and if so, how and where from in the aggregate.
216 Individual fields have the same meaning like fields with the same name in
219 struct agg_position_info
221 HOST_WIDE_INT offset
;
226 /* Add condition to condition list CONDS. AGGPOS describes whether the used
227 oprand is loaded from an aggregate and where in the aggregate it is. It can
228 be NULL, which means this not a load from an aggregate. */
230 static struct predicate
231 add_condition (struct inline_summary
*summary
, int operand_num
,
232 struct agg_position_info
*aggpos
,
233 enum tree_code code
, tree val
)
237 struct condition new_cond
;
238 HOST_WIDE_INT offset
;
239 bool agg_contents
, by_ref
;
243 offset
= aggpos
->offset
;
244 agg_contents
= aggpos
->agg_contents
;
245 by_ref
= aggpos
->by_ref
;
250 agg_contents
= false;
254 gcc_checking_assert (operand_num
>= 0);
255 for (i
= 0; vec_safe_iterate (summary
->conds
, i
, &c
); i
++)
257 if (c
->operand_num
== operand_num
260 && c
->agg_contents
== agg_contents
261 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
262 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
264 /* Too many conditions. Give up and return constant true. */
265 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
266 return true_predicate ();
268 new_cond
.operand_num
= operand_num
;
269 new_cond
.code
= code
;
271 new_cond
.agg_contents
= agg_contents
;
272 new_cond
.by_ref
= by_ref
;
273 new_cond
.offset
= offset
;
274 vec_safe_push (summary
->conds
, new_cond
);
275 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
279 /* Add clause CLAUSE into the predicate P. */
282 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
286 int insert_here
= -1;
293 /* False clause makes the whole predicate false. Kill the other variants. */
294 if (clause
== (1 << predicate_false_condition
))
296 p
->clause
[0] = (1 << predicate_false_condition
);
300 if (false_predicate_p (p
))
303 /* No one should be sily enough to add false into nontrivial clauses. */
304 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
306 /* Look where to insert the clause. At the same time prune out
307 clauses of P that are implied by the new clause and thus
309 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
311 p
->clause
[i2
] = p
->clause
[i
];
316 /* If p->clause[i] implies clause, there is nothing to add. */
317 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
319 /* We had nothing to add, none of clauses should've become
321 gcc_checking_assert (i
== i2
);
325 if (p
->clause
[i
] < clause
&& insert_here
< 0)
328 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
329 Otherwise the p->clause[i] has to stay. */
330 if ((p
->clause
[i
] & clause
) != clause
)
334 /* Look for clauses that are obviously true. I.e.
335 op0 == 5 || op0 != 5. */
336 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
339 if (!(clause
& (1 << c1
)))
341 cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
342 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
343 and thus there is no point for looking for them. */
344 if (cc1
->code
== CHANGED
|| cc1
->code
== IS_NOT_CONSTANT
)
346 for (c2
= c1
+ 1; c2
< NUM_CONDITIONS
; c2
++)
347 if (clause
& (1 << c2
))
350 &(*conditions
)[c1
- predicate_first_dynamic_condition
];
352 &(*conditions
)[c2
- predicate_first_dynamic_condition
];
353 if (cc1
->operand_num
== cc2
->operand_num
354 && cc1
->val
== cc2
->val
355 && cc2
->code
!= IS_NOT_CONSTANT
356 && cc2
->code
!= CHANGED
357 && cc1
->code
== invert_tree_comparison
359 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
365 /* We run out of variants. Be conservative in positive direction. */
366 if (i2
== MAX_CLAUSES
)
368 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
369 p
->clause
[i2
+ 1] = 0;
370 if (insert_here
>= 0)
371 for (; i2
> insert_here
; i2
--)
372 p
->clause
[i2
] = p
->clause
[i2
- 1];
375 p
->clause
[insert_here
] = clause
;
381 static struct predicate
382 and_predicates (conditions conditions
,
383 struct predicate
*p
, struct predicate
*p2
)
385 struct predicate out
= *p
;
388 /* Avoid busy work. */
389 if (false_predicate_p (p2
) || true_predicate_p (p
))
391 if (false_predicate_p (p
) || true_predicate_p (p2
))
394 /* See how far predicates match. */
395 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
397 gcc_checking_assert (i
< MAX_CLAUSES
);
400 /* Combine the predicates rest. */
401 for (; p2
->clause
[i
]; i
++)
403 gcc_checking_assert (i
< MAX_CLAUSES
);
404 add_clause (conditions
, &out
, p2
->clause
[i
]);
410 /* Return true if predicates are obviously equal. */
413 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
416 for (i
= 0; p
->clause
[i
]; i
++)
418 gcc_checking_assert (i
< MAX_CLAUSES
);
419 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
420 gcc_checking_assert (!p2
->clause
[i
]
421 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
422 if (p
->clause
[i
] != p2
->clause
[i
])
425 return !p2
->clause
[i
];
431 static struct predicate
432 or_predicates (conditions conditions
,
433 struct predicate
*p
, struct predicate
*p2
)
435 struct predicate out
= true_predicate ();
438 /* Avoid busy work. */
439 if (false_predicate_p (p2
) || true_predicate_p (p
))
441 if (false_predicate_p (p
) || true_predicate_p (p2
))
443 if (predicates_equal_p (p
, p2
))
446 /* OK, combine the predicates. */
447 for (i
= 0; p
->clause
[i
]; i
++)
448 for (j
= 0; p2
->clause
[j
]; j
++)
450 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
451 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
457 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
458 if predicate P is known to be false. */
461 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
465 /* True remains true. */
466 if (true_predicate_p (p
))
469 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
471 /* See if we can find clause we can disprove. */
472 for (i
= 0; p
->clause
[i
]; i
++)
474 gcc_checking_assert (i
< MAX_CLAUSES
);
475 if (!(p
->clause
[i
] & possible_truths
))
481 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
482 instruction will be recomputed per invocation of the inlined call. */
485 predicate_probability (conditions conds
,
486 struct predicate
*p
, clause_t possible_truths
,
487 vec
<inline_param_summary_t
> inline_param_summary
)
490 int combined_prob
= REG_BR_PROB_BASE
;
492 /* True remains true. */
493 if (true_predicate_p (p
))
494 return REG_BR_PROB_BASE
;
496 if (false_predicate_p (p
))
499 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
501 /* See if we can find clause we can disprove. */
502 for (i
= 0; p
->clause
[i
]; i
++)
504 gcc_checking_assert (i
< MAX_CLAUSES
);
505 if (!(p
->clause
[i
] & possible_truths
))
511 if (!inline_param_summary
.exists ())
512 return REG_BR_PROB_BASE
;
513 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
514 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
516 if (i2
>= predicate_first_dynamic_condition
)
519 &(*conds
)[i2
- predicate_first_dynamic_condition
];
520 if (c
->code
== CHANGED
522 (int) inline_param_summary
.length ()))
525 inline_param_summary
[c
->operand_num
].change_prob
;
526 this_prob
= MAX (this_prob
, iprob
);
529 this_prob
= REG_BR_PROB_BASE
;
532 this_prob
= REG_BR_PROB_BASE
;
534 combined_prob
= MIN (this_prob
, combined_prob
);
539 return combined_prob
;
543 /* Dump conditional COND. */
546 dump_condition (FILE *f
, conditions conditions
, int cond
)
549 if (cond
== predicate_false_condition
)
550 fprintf (f
, "false");
551 else if (cond
== predicate_not_inlined_condition
)
552 fprintf (f
, "not inlined");
555 c
= &(*conditions
)[cond
- predicate_first_dynamic_condition
];
556 fprintf (f
, "op%i", c
->operand_num
);
558 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
559 c
->by_ref
? "ref " : "", c
->offset
);
560 if (c
->code
== IS_NOT_CONSTANT
)
562 fprintf (f
, " not constant");
565 if (c
->code
== CHANGED
)
567 fprintf (f
, " changed");
570 fprintf (f
, " %s ", op_symbol_code (c
->code
));
571 print_generic_expr (f
, c
->val
, 1);
576 /* Dump clause CLAUSE. */
579 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
586 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
587 if (clause
& (1 << i
))
592 dump_condition (f
, conds
, i
);
598 /* Dump predicate PREDICATE. */
601 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
604 if (true_predicate_p (pred
))
605 dump_clause (f
, conds
, 0);
607 for (i
= 0; pred
->clause
[i
]; i
++)
611 dump_clause (f
, conds
, pred
->clause
[i
]);
617 /* Dump inline hints. */
619 dump_inline_hints (FILE *f
, inline_hints hints
)
623 fprintf (f
, "inline hints:");
624 if (hints
& INLINE_HINT_indirect_call
)
626 hints
&= ~INLINE_HINT_indirect_call
;
627 fprintf (f
, " indirect_call");
629 if (hints
& INLINE_HINT_loop_iterations
)
631 hints
&= ~INLINE_HINT_loop_iterations
;
632 fprintf (f
, " loop_iterations");
634 if (hints
& INLINE_HINT_loop_stride
)
636 hints
&= ~INLINE_HINT_loop_stride
;
637 fprintf (f
, " loop_stride");
639 if (hints
& INLINE_HINT_same_scc
)
641 hints
&= ~INLINE_HINT_same_scc
;
642 fprintf (f
, " same_scc");
644 if (hints
& INLINE_HINT_in_scc
)
646 hints
&= ~INLINE_HINT_in_scc
;
647 fprintf (f
, " in_scc");
649 if (hints
& INLINE_HINT_cross_module
)
651 hints
&= ~INLINE_HINT_cross_module
;
652 fprintf (f
, " cross_module");
654 if (hints
& INLINE_HINT_declared_inline
)
656 hints
&= ~INLINE_HINT_declared_inline
;
657 fprintf (f
, " declared_inline");
659 if (hints
& INLINE_HINT_array_index
)
661 hints
&= ~INLINE_HINT_array_index
;
662 fprintf (f
, " array_index");
668 /* Record SIZE and TIME under condition PRED into the inline summary. */
671 account_size_time (struct inline_summary
*summary
, int size
, int time
,
672 struct predicate
*pred
)
678 if (false_predicate_p (pred
))
681 /* We need to create initial empty unconitional clause, but otherwie
682 we don't need to account empty times and sizes. */
683 if (!size
&& !time
&& summary
->entry
)
686 /* Watch overflow that might result from insane profiles. */
687 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
688 time
= MAX_TIME
* INLINE_TIME_SCALE
;
689 gcc_assert (time
>= 0);
691 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
692 if (predicates_equal_p (&e
->predicate
, pred
))
701 e
= &(*summary
->entry
)[0];
702 gcc_assert (!e
->predicate
.clause
[0]);
703 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
705 "\t\tReached limit on number of entries, "
706 "ignoring the predicate.");
708 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
711 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
712 ((double) size
) / INLINE_SIZE_SCALE
,
713 ((double) time
) / INLINE_TIME_SCALE
, found
? "" : "new ");
714 dump_predicate (dump_file
, summary
->conds
, pred
);
718 struct size_time_entry new_entry
;
719 new_entry
.size
= size
;
720 new_entry
.time
= time
;
721 new_entry
.predicate
= *pred
;
722 vec_safe_push (summary
->entry
, new_entry
);
728 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
729 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
733 /* Set predicate for edge E. */
736 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
738 struct inline_edge_summary
*es
= inline_edge_summary (e
);
739 if (predicate
&& !true_predicate_p (predicate
))
742 es
->predicate
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
743 *es
->predicate
= *predicate
;
748 pool_free (edge_predicate_pool
, es
->predicate
);
749 es
->predicate
= NULL
;
753 /* Set predicate for hint *P. */
756 set_hint_predicate (struct predicate
**p
, struct predicate new_predicate
)
758 if (false_predicate_p (&new_predicate
) || true_predicate_p (&new_predicate
))
761 pool_free (edge_predicate_pool
, *p
);
767 *p
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
773 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
774 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
775 Return clause of possible truths. When INLINE_P is true, assume that we are
778 ERROR_MARK means compile time invariant. */
781 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
783 vec
<tree
> known_vals
,
784 vec
<ipa_agg_jump_function_p
>
787 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
788 struct inline_summary
*info
= inline_summary (node
);
792 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
797 /* We allow call stmt to have fewer arguments than the callee function
798 (especially for K&R style programs). So bound check here (we assume
799 known_aggs vector, if non-NULL, has the same length as
801 gcc_checking_assert (!known_aggs
.exists ()
802 || (known_vals
.length () == known_aggs
.length ()));
803 if (c
->operand_num
>= (int) known_vals
.length ())
805 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
811 struct ipa_agg_jump_function
*agg
;
813 if (c
->code
== CHANGED
815 && (known_vals
[c
->operand_num
] == error_mark_node
))
818 if (known_aggs
.exists ())
820 agg
= known_aggs
[c
->operand_num
];
821 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
828 val
= known_vals
[c
->operand_num
];
829 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
835 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
838 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
840 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
841 if (res
&& integer_zerop (res
))
843 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
849 /* Work out what conditions might be true at invocation of E. */
852 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
853 clause_t
*clause_ptr
,
854 vec
<tree
> *known_vals_ptr
,
855 vec
<tree
> *known_binfos_ptr
,
856 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
858 struct cgraph_node
*callee
=
859 cgraph_function_or_thunk_node (e
->callee
, NULL
);
860 struct inline_summary
*info
= inline_summary (callee
);
861 vec
<tree
> known_vals
= vNULL
;
862 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
865 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
867 known_vals_ptr
->create (0);
868 if (known_binfos_ptr
)
869 known_binfos_ptr
->create (0);
871 if (ipa_node_params_vector
.exists ()
872 && !e
->call_stmt_cannot_inline_p
873 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_binfos_ptr
))
875 struct ipa_node_params
*parms_info
;
876 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
877 struct inline_edge_summary
*es
= inline_edge_summary (e
);
878 int i
, count
= ipa_get_cs_argument_count (args
);
880 if (e
->caller
->global
.inlined_to
)
881 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
883 parms_info
= IPA_NODE_REF (e
->caller
);
885 if (count
&& (info
->conds
|| known_vals_ptr
))
886 known_vals
.safe_grow_cleared (count
);
887 if (count
&& (info
->conds
|| known_aggs_ptr
))
888 known_aggs
.safe_grow_cleared (count
);
889 if (count
&& known_binfos_ptr
)
890 known_binfos_ptr
->safe_grow_cleared (count
);
892 for (i
= 0; i
< count
; i
++)
894 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
895 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
898 if (known_vals
.exists () && TREE_CODE (cst
) != TREE_BINFO
)
900 else if (known_binfos_ptr
!= NULL
901 && TREE_CODE (cst
) == TREE_BINFO
)
902 (*known_binfos_ptr
)[i
] = cst
;
904 else if (inline_p
&& !es
->param
[i
].change_prob
)
905 known_vals
[i
] = error_mark_node
;
906 /* TODO: When IPA-CP starts propagating and merging aggregate jump
907 functions, use its knowledge of the caller too, just like the
908 scalar case above. */
909 known_aggs
[i
] = &jf
->agg
;
914 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
915 known_vals
, known_aggs
);
918 *known_vals_ptr
= known_vals
;
920 known_vals
.release ();
923 *known_aggs_ptr
= known_aggs
;
925 known_aggs
.release ();
929 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
932 inline_summary_alloc (void)
934 if (!node_removal_hook_holder
)
935 node_removal_hook_holder
=
936 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
937 if (!edge_removal_hook_holder
)
938 edge_removal_hook_holder
=
939 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
940 if (!node_duplication_hook_holder
)
941 node_duplication_hook_holder
=
942 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
943 if (!edge_duplication_hook_holder
)
944 edge_duplication_hook_holder
=
945 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
947 if (vec_safe_length (inline_summary_vec
) <= (unsigned) cgraph_max_uid
)
948 vec_safe_grow_cleared (inline_summary_vec
, cgraph_max_uid
+ 1);
949 if (inline_edge_summary_vec
.length () <= (unsigned) cgraph_edge_max_uid
)
950 inline_edge_summary_vec
.safe_grow_cleared (cgraph_edge_max_uid
+ 1);
951 if (!edge_predicate_pool
)
952 edge_predicate_pool
= create_alloc_pool ("edge predicates",
953 sizeof (struct predicate
), 10);
956 /* We are called multiple time for given function; clear
957 data from previous run so they are not cumulated. */
960 reset_inline_edge_summary (struct cgraph_edge
*e
)
962 if (e
->uid
< (int) inline_edge_summary_vec
.length ())
964 struct inline_edge_summary
*es
= inline_edge_summary (e
);
966 es
->call_stmt_size
= es
->call_stmt_time
= 0;
968 pool_free (edge_predicate_pool
, es
->predicate
);
969 es
->predicate
= NULL
;
970 es
->param
.release ();
974 /* We are called multiple time for given function; clear
975 data from previous run so they are not cumulated. */
978 reset_inline_summary (struct cgraph_node
*node
)
980 struct inline_summary
*info
= inline_summary (node
);
981 struct cgraph_edge
*e
;
983 info
->self_size
= info
->self_time
= 0;
984 info
->estimated_stack_size
= 0;
985 info
->estimated_self_stack_size
= 0;
986 info
->stack_frame_offset
= 0;
991 if (info
->loop_iterations
)
993 pool_free (edge_predicate_pool
, info
->loop_iterations
);
994 info
->loop_iterations
= NULL
;
996 if (info
->loop_stride
)
998 pool_free (edge_predicate_pool
, info
->loop_stride
);
999 info
->loop_stride
= NULL
;
1001 if (info
->array_index
)
1003 pool_free (edge_predicate_pool
, info
->array_index
);
1004 info
->array_index
= NULL
;
1006 vec_free (info
->conds
);
1007 vec_free (info
->entry
);
1008 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1009 reset_inline_edge_summary (e
);
1010 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1011 reset_inline_edge_summary (e
);
1014 /* Hook that is called by cgraph.c when a node is removed. */
1017 inline_node_removal_hook (struct cgraph_node
*node
,
1018 void *data ATTRIBUTE_UNUSED
)
1020 struct inline_summary
*info
;
1021 if (vec_safe_length (inline_summary_vec
) <= (unsigned) node
->uid
)
1023 info
= inline_summary (node
);
1024 reset_inline_summary (node
);
1025 memset (info
, 0, sizeof (inline_summary_t
));
1028 /* Remap predicate P of former function to be predicate of duplicated functoin.
1029 POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
1030 INFO is inline summary of the duplicated node. */
1032 static struct predicate
1033 remap_predicate_after_duplication (struct predicate
*p
,
1034 clause_t possible_truths
,
1035 struct inline_summary
*info
)
1037 struct predicate new_predicate
= true_predicate ();
1039 for (j
= 0; p
->clause
[j
]; j
++)
1040 if (!(possible_truths
& p
->clause
[j
]))
1042 new_predicate
= false_predicate ();
1046 add_clause (info
->conds
, &new_predicate
,
1047 possible_truths
& p
->clause
[j
]);
1048 return new_predicate
;
1051 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
1052 Additionally care about allocating new memory slot for updated predicate
1053 and set it to NULL when it becomes true or false (and thus uninteresting).
1057 remap_hint_predicate_after_duplication (struct predicate
**p
,
1058 clause_t possible_truths
,
1059 struct inline_summary
*info
)
1061 struct predicate new_predicate
;
1066 new_predicate
= remap_predicate_after_duplication (*p
,
1067 possible_truths
, info
);
1068 /* We do not want to free previous predicate; it is used by node origin. */
1070 set_hint_predicate (p
, new_predicate
);
1074 /* Hook that is called by cgraph.c when a node is duplicated. */
1077 inline_node_duplication_hook (struct cgraph_node
*src
,
1078 struct cgraph_node
*dst
,
1079 ATTRIBUTE_UNUSED
void *data
)
1081 struct inline_summary
*info
;
1082 inline_summary_alloc ();
1083 info
= inline_summary (dst
);
1084 memcpy (info
, inline_summary (src
), sizeof (struct inline_summary
));
1085 /* TODO: as an optimization, we may avoid copying conditions
1086 that are known to be false or true. */
1087 info
->conds
= vec_safe_copy (info
->conds
);
1089 /* When there are any replacements in the function body, see if we can figure
1090 out that something was optimized out. */
1091 if (ipa_node_params_vector
.exists () && dst
->clone
.tree_map
)
1093 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
1094 /* Use SRC parm info since it may not be copied yet. */
1095 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1096 vec
<tree
> known_vals
= vNULL
;
1097 int count
= ipa_get_param_count (parms_info
);
1099 clause_t possible_truths
;
1100 struct predicate true_pred
= true_predicate ();
1102 int optimized_out_size
= 0;
1103 bool inlined_to_p
= false;
1104 struct cgraph_edge
*edge
;
1107 known_vals
.safe_grow_cleared (count
);
1108 for (i
= 0; i
< count
; i
++)
1110 struct ipa_replace_map
*r
;
1112 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
1114 if (((!r
->old_tree
&& r
->parm_num
== i
)
1115 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
1116 && r
->replace_p
&& !r
->ref_p
)
1118 known_vals
[i
] = r
->new_tree
;
1123 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1126 known_vals
.release ();
1128 account_size_time (info
, 0, 0, &true_pred
);
1130 /* Remap size_time vectors.
1131 Simplify the predicate by prunning out alternatives that are known
1133 TODO: as on optimization, we can also eliminate conditions known
1135 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
1137 struct predicate new_predicate
;
1138 new_predicate
= remap_predicate_after_duplication (&e
->predicate
,
1141 if (false_predicate_p (&new_predicate
))
1142 optimized_out_size
+= e
->size
;
1144 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1147 /* Remap edge predicates with the same simplification as above.
1148 Also copy constantness arrays. */
1149 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1151 struct predicate new_predicate
;
1152 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1154 if (!edge
->inline_failed
)
1155 inlined_to_p
= true;
1158 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1161 if (false_predicate_p (&new_predicate
)
1162 && !false_predicate_p (es
->predicate
))
1164 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1165 edge
->frequency
= 0;
1167 edge_set_predicate (edge
, &new_predicate
);
1170 /* Remap indirect edge predicates with the same simplificaiton as above.
1171 Also copy constantness arrays. */
1172 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1174 struct predicate new_predicate
;
1175 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1177 gcc_checking_assert (edge
->inline_failed
);
1180 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1183 if (false_predicate_p (&new_predicate
)
1184 && !false_predicate_p (es
->predicate
))
1186 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1187 edge
->frequency
= 0;
1189 edge_set_predicate (edge
, &new_predicate
);
1191 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
1192 possible_truths
, info
);
1193 remap_hint_predicate_after_duplication (&info
->loop_stride
,
1194 possible_truths
, info
);
1195 remap_hint_predicate_after_duplication (&info
->array_index
,
1196 possible_truths
, info
);
1198 /* If inliner or someone after inliner will ever start producing
1199 non-trivial clones, we will get trouble with lack of information
1200 about updating self sizes, because size vectors already contains
1201 sizes of the calees. */
1202 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
1206 info
->entry
= vec_safe_copy (info
->entry
);
1207 if (info
->loop_iterations
)
1209 predicate p
= *info
->loop_iterations
;
1210 info
->loop_iterations
= NULL
;
1211 set_hint_predicate (&info
->loop_iterations
, p
);
1213 if (info
->loop_stride
)
1215 predicate p
= *info
->loop_stride
;
1216 info
->loop_stride
= NULL
;
1217 set_hint_predicate (&info
->loop_stride
, p
);
1219 if (info
->array_index
)
1221 predicate p
= *info
->array_index
;
1222 info
->array_index
= NULL
;
1223 set_hint_predicate (&info
->array_index
, p
);
1226 inline_update_overall_summary (dst
);
1230 /* Hook that is called by cgraph.c when a node is duplicated. */
1233 inline_edge_duplication_hook (struct cgraph_edge
*src
,
1234 struct cgraph_edge
*dst
,
1235 ATTRIBUTE_UNUSED
void *data
)
1237 struct inline_edge_summary
*info
;
1238 struct inline_edge_summary
*srcinfo
;
1239 inline_summary_alloc ();
1240 info
= inline_edge_summary (dst
);
1241 srcinfo
= inline_edge_summary (src
);
1242 memcpy (info
, srcinfo
, sizeof (struct inline_edge_summary
));
1243 info
->predicate
= NULL
;
1244 edge_set_predicate (dst
, srcinfo
->predicate
);
1245 info
->param
= srcinfo
->param
.copy ();
1249 /* Keep edge cache consistent across edge removal. */
1252 inline_edge_removal_hook (struct cgraph_edge
*edge
,
1253 void *data ATTRIBUTE_UNUSED
)
1255 if (edge_growth_cache
.exists ())
1256 reset_edge_growth_cache (edge
);
1257 reset_inline_edge_summary (edge
);
1261 /* Initialize growth caches. */
1264 initialize_growth_caches (void)
1266 if (cgraph_edge_max_uid
)
1267 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
1269 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
1273 /* Free growth caches. */
1276 free_growth_caches (void)
1278 edge_growth_cache
.release ();
1279 node_growth_cache
.release ();
1283 /* Dump edge summaries associated to NODE and recursively to all clones.
1284 Indent by INDENT. */
1287 dump_inline_edge_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
1288 struct inline_summary
*info
)
1290 struct cgraph_edge
*edge
;
1291 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1293 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1294 struct cgraph_node
*callee
=
1295 cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1299 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
1300 " time: %2i callee size:%2i stack:%2i",
1301 indent
, "", cgraph_node_name (callee
), callee
->symbol
.order
,
1302 !edge
->inline_failed
1303 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
1304 indent
, "", es
->loop_depth
, edge
->frequency
,
1305 es
->call_stmt_size
, es
->call_stmt_time
,
1306 (int) inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1307 (int) inline_summary (callee
)->estimated_stack_size
);
1311 fprintf (f
, " predicate: ");
1312 dump_predicate (f
, info
->conds
, es
->predicate
);
1316 if (es
->param
.exists ())
1317 for (i
= 0; i
< (int) es
->param
.length (); i
++)
1319 int prob
= es
->param
[i
].change_prob
;
1322 fprintf (f
, "%*s op%i is compile time invariant\n",
1324 else if (prob
!= REG_BR_PROB_BASE
)
1325 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1326 prob
* 100.0 / REG_BR_PROB_BASE
);
1328 if (!edge
->inline_failed
)
1330 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1331 " callee size %i\n",
1333 (int) inline_summary (callee
)->stack_frame_offset
,
1334 (int) inline_summary (callee
)->estimated_self_stack_size
,
1335 (int) inline_summary (callee
)->estimated_stack_size
);
1336 dump_inline_edge_summary (f
, indent
+ 2, callee
, info
);
1339 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1341 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1342 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1346 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
1349 fprintf (f
, "predicate: ");
1350 dump_predicate (f
, info
->conds
, es
->predicate
);
1359 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
1361 if (node
->symbol
.definition
)
1363 struct inline_summary
*s
= inline_summary (node
);
1366 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
1367 node
->symbol
.order
);
1368 if (DECL_DISREGARD_INLINE_LIMITS (node
->symbol
.decl
))
1369 fprintf (f
, " always_inline");
1371 fprintf (f
, " inlinable");
1372 fprintf (f
, "\n self time: %i\n", s
->self_time
);
1373 fprintf (f
, " global time: %i\n", s
->time
);
1374 fprintf (f
, " self size: %i\n", s
->self_size
);
1375 fprintf (f
, " global size: %i\n", s
->size
);
1376 fprintf (f
, " self stack: %i\n",
1377 (int) s
->estimated_self_stack_size
);
1378 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
1380 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
1382 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
1383 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
1385 fprintf (f
, " size:%f, time:%f, predicate:",
1386 (double) e
->size
/ INLINE_SIZE_SCALE
,
1387 (double) e
->time
/ INLINE_TIME_SCALE
);
1388 dump_predicate (f
, s
->conds
, &e
->predicate
);
1390 if (s
->loop_iterations
)
1392 fprintf (f
, " loop iterations:");
1393 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1397 fprintf (f
, " loop stride:");
1398 dump_predicate (f
, s
->conds
, s
->loop_stride
);
1402 fprintf (f
, " array index:");
1403 dump_predicate (f
, s
->conds
, s
->array_index
);
1405 fprintf (f
, " calls:\n");
1406 dump_inline_edge_summary (f
, 4, node
, s
);
1412 debug_inline_summary (struct cgraph_node
*node
)
1414 dump_inline_summary (stderr
, node
);
1418 dump_inline_summaries (FILE *f
)
1420 struct cgraph_node
*node
;
1422 FOR_EACH_DEFINED_FUNCTION (node
)
1423 if (!node
->global
.inlined_to
)
1424 dump_inline_summary (f
, node
);
1427 /* Give initial reasons why inlining would fail on EDGE. This gets either
1428 nullified or usually overwritten by more precise reasons later. */
1431 initialize_inline_failed (struct cgraph_edge
*e
)
1433 struct cgraph_node
*callee
= e
->callee
;
1435 if (e
->indirect_unknown_callee
)
1436 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1437 else if (!callee
->symbol
.definition
)
1438 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1439 else if (callee
->local
.redefined_extern_inline
)
1440 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1441 else if (e
->call_stmt_cannot_inline_p
)
1442 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1444 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1447 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1448 boolean variable pointed to by DATA. */
1451 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1454 bool *b
= (bool *) data
;
1459 /* If OP refers to value of function parameter, return the corresponding
1463 unmodified_parm_1 (gimple stmt
, tree op
)
1465 /* SSA_NAME referring to parm default def? */
1466 if (TREE_CODE (op
) == SSA_NAME
1467 && SSA_NAME_IS_DEFAULT_DEF (op
)
1468 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1469 return SSA_NAME_VAR (op
);
1470 /* Non-SSA parm reference? */
1471 if (TREE_CODE (op
) == PARM_DECL
)
1473 bool modified
= false;
1476 ao_ref_init (&refd
, op
);
1477 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1485 /* If OP refers to value of function parameter, return the corresponding
1486 parameter. Also traverse chains of SSA register assignments. */
1489 unmodified_parm (gimple stmt
, tree op
)
1491 tree res
= unmodified_parm_1 (stmt
, op
);
1495 if (TREE_CODE (op
) == SSA_NAME
1496 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1497 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1498 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1499 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1503 /* If OP refers to a value of a function parameter or value loaded from an
1504 aggregate passed to a parameter (either by value or reference), return TRUE
1505 and store the number of the parameter to *INDEX_P and information whether
1506 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1507 the function parameters, STMT is the statement in which OP is used or
1511 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1512 gimple stmt
, tree op
, int *index_p
,
1513 struct agg_position_info
*aggpos
)
1515 tree res
= unmodified_parm_1 (stmt
, op
);
1517 gcc_checking_assert (aggpos
);
1520 *index_p
= ipa_get_param_decl_index (info
, res
);
1523 aggpos
->agg_contents
= false;
1524 aggpos
->by_ref
= false;
1528 if (TREE_CODE (op
) == SSA_NAME
)
1530 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1531 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1533 stmt
= SSA_NAME_DEF_STMT (op
);
1534 op
= gimple_assign_rhs1 (stmt
);
1535 if (!REFERENCE_CLASS_P (op
))
1536 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1540 aggpos
->agg_contents
= true;
1541 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1545 /* See if statement might disappear after inlining.
1546 0 - means not eliminated
1547 1 - half of statements goes away
1548 2 - for sure it is eliminated.
1549 We are not terribly sophisticated, basically looking for simple abstraction
1550 penalty wrappers. */
1553 eliminated_by_inlining_prob (gimple stmt
)
1555 enum gimple_code code
= gimple_code (stmt
);
1556 enum tree_code rhs_code
;
1566 if (gimple_num_ops (stmt
) != 2)
1569 rhs_code
= gimple_assign_rhs_code (stmt
);
1571 /* Casts of parameters, loads from parameters passed by reference
1572 and stores to return value or parameters are often free after
1573 inlining dua to SRA and further combining.
1574 Assume that half of statements goes away. */
1575 if (rhs_code
== CONVERT_EXPR
1576 || rhs_code
== NOP_EXPR
1577 || rhs_code
== VIEW_CONVERT_EXPR
1578 || rhs_code
== ADDR_EXPR
1579 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1581 tree rhs
= gimple_assign_rhs1 (stmt
);
1582 tree lhs
= gimple_assign_lhs (stmt
);
1583 tree inner_rhs
= get_base_address (rhs
);
1584 tree inner_lhs
= get_base_address (lhs
);
1585 bool rhs_free
= false;
1586 bool lhs_free
= false;
1593 /* Reads of parameter are expected to be free. */
1594 if (unmodified_parm (stmt
, inner_rhs
))
1596 /* Match expressions of form &this->field. Those will most likely
1597 combine with something upstream after inlining. */
1598 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1600 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1601 if (TREE_CODE (op
) == PARM_DECL
)
1603 else if (TREE_CODE (op
) == MEM_REF
1604 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0)))
1608 /* When parameter is not SSA register because its address is taken
1609 and it is just copied into one, the statement will be completely
1610 free after inlining (we will copy propagate backward). */
1611 if (rhs_free
&& is_gimple_reg (lhs
))
1614 /* Reads of parameters passed by reference
1615 expected to be free (i.e. optimized out after inlining). */
1616 if (TREE_CODE (inner_rhs
) == MEM_REF
1617 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1620 /* Copying parameter passed by reference into gimple register is
1621 probably also going to copy propagate, but we can't be quite
1623 if (rhs_free
&& is_gimple_reg (lhs
))
1626 /* Writes to parameters, parameters passed by value and return value
1627 (either dirrectly or passed via invisible reference) are free.
1629 TODO: We ought to handle testcase like
1630 struct a {int a,b;};
1632 retrurnsturct (void)
1638 This translate into:
1653 For that we either need to copy ipa-split logic detecting writes
1655 if (TREE_CODE (inner_lhs
) == PARM_DECL
1656 || TREE_CODE (inner_lhs
) == RESULT_DECL
1657 || (TREE_CODE (inner_lhs
) == MEM_REF
1658 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1659 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1660 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1661 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1663 0))) == RESULT_DECL
))))
1666 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1668 if (lhs_free
&& rhs_free
)
1678 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1679 predicates to the CFG edges. */
1682 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1683 struct inline_summary
*summary
,
1689 struct agg_position_info aggpos
;
1690 enum tree_code code
, inverted_code
;
1696 last
= last_stmt (bb
);
1697 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1699 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1701 op
= gimple_cond_lhs (last
);
1702 /* TODO: handle conditionals like
1705 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1707 code
= gimple_cond_code (last
);
1709 = invert_tree_comparison (code
,
1710 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1712 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1714 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1715 e
->flags
& EDGE_TRUE_VALUE
1716 ? code
: inverted_code
,
1717 gimple_cond_rhs (last
));
1718 e
->aux
= pool_alloc (edge_predicate_pool
);
1719 *(struct predicate
*) e
->aux
= p
;
1723 if (TREE_CODE (op
) != SSA_NAME
)
1726 if (builtin_constant_p (op))
1730 Here we can predicate nonconstant_code. We can't
1731 really handle constant_code since we have no predicate
1732 for this and also the constant code is not known to be
1733 optimized away when inliner doen't see operand is constant.
1734 Other optimizers might think otherwise. */
1735 if (gimple_cond_code (last
) != NE_EXPR
1736 || !integer_zerop (gimple_cond_rhs (last
)))
1738 set_stmt
= SSA_NAME_DEF_STMT (op
);
1739 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1740 || gimple_call_num_args (set_stmt
) != 1)
1742 op2
= gimple_call_arg (set_stmt
, 0);
1743 if (!unmodified_parm_or_parm_agg_item
1744 (info
, set_stmt
, op2
, &index
, &aggpos
))
1746 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1748 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1749 IS_NOT_CONSTANT
, NULL_TREE
);
1750 e
->aux
= pool_alloc (edge_predicate_pool
);
1751 *(struct predicate
*) e
->aux
= p
;
1756 /* If BB ends by a switch we can turn into predicates, attach corresponding
1757 predicates to the CFG edges. */
1760 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1761 struct inline_summary
*summary
,
1767 struct agg_position_info aggpos
;
1773 last
= last_stmt (bb
);
1774 if (!last
|| gimple_code (last
) != GIMPLE_SWITCH
)
1776 op
= gimple_switch_index (last
);
1777 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1780 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1782 e
->aux
= pool_alloc (edge_predicate_pool
);
1783 *(struct predicate
*) e
->aux
= false_predicate ();
1785 n
= gimple_switch_num_labels (last
);
1786 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1788 tree cl
= gimple_switch_label (last
, case_idx
);
1792 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1793 min
= CASE_LOW (cl
);
1794 max
= CASE_HIGH (cl
);
1796 /* For default we might want to construct predicate that none
1797 of cases is met, but it is bit hard to do not having negations
1798 of conditionals handy. */
1800 p
= true_predicate ();
1802 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1805 struct predicate p1
, p2
;
1806 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1807 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1808 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1810 *(struct predicate
*) e
->aux
1811 = or_predicates (summary
->conds
, &p
, (struct predicate
*) e
->aux
);
1816 /* For each BB in NODE attach to its AUX pointer predicate under
1817 which it is executable. */
1820 compute_bb_predicates (struct cgraph_node
*node
,
1821 struct ipa_node_params
*parms_info
,
1822 struct inline_summary
*summary
)
1824 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
1828 FOR_EACH_BB_FN (bb
, my_function
)
1830 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1831 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1834 /* Entry block is always executable. */
1835 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1836 = pool_alloc (edge_predicate_pool
);
1837 *(struct predicate
*) ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1838 = true_predicate ();
1840 /* A simple dataflow propagation of predicates forward in the CFG.
1841 TODO: work in reverse postorder. */
1845 FOR_EACH_BB_FN (bb
, my_function
)
1847 struct predicate p
= false_predicate ();
1850 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1854 struct predicate this_bb_predicate
1855 = *(struct predicate
*) e
->src
->aux
;
1858 = and_predicates (summary
->conds
, &this_bb_predicate
,
1859 (struct predicate
*) e
->aux
);
1860 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1861 if (true_predicate_p (&p
))
1865 if (false_predicate_p (&p
))
1866 gcc_assert (!bb
->aux
);
1872 bb
->aux
= pool_alloc (edge_predicate_pool
);
1873 *((struct predicate
*) bb
->aux
) = p
;
1875 else if (!predicates_equal_p (&p
, (struct predicate
*) bb
->aux
))
1878 *((struct predicate
*) bb
->aux
) = p
;
1886 /* We keep info about constantness of SSA names. */
1888 typedef struct predicate predicate_t
;
1889 /* Return predicate specifying when the STMT might have result that is not
1890 a compile time constant. */
1892 static struct predicate
1893 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1894 struct inline_summary
*summary
,
1896 vec
<predicate_t
> nonconstant_names
)
1901 while (UNARY_CLASS_P (expr
))
1902 expr
= TREE_OPERAND (expr
, 0);
1904 parm
= unmodified_parm (NULL
, expr
);
1905 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1906 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1907 if (is_gimple_min_invariant (expr
))
1908 return false_predicate ();
1909 if (TREE_CODE (expr
) == SSA_NAME
)
1910 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1911 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1913 struct predicate p1
= will_be_nonconstant_expr_predicate
1914 (info
, summary
, TREE_OPERAND (expr
, 0),
1916 struct predicate p2
;
1917 if (true_predicate_p (&p1
))
1919 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1920 TREE_OPERAND (expr
, 1),
1922 return or_predicates (summary
->conds
, &p1
, &p2
);
1924 else if (TREE_CODE (expr
) == COND_EXPR
)
1926 struct predicate p1
= will_be_nonconstant_expr_predicate
1927 (info
, summary
, TREE_OPERAND (expr
, 0),
1929 struct predicate p2
;
1930 if (true_predicate_p (&p1
))
1932 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1933 TREE_OPERAND (expr
, 1),
1935 if (true_predicate_p (&p2
))
1937 p1
= or_predicates (summary
->conds
, &p1
, &p2
);
1938 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1939 TREE_OPERAND (expr
, 2),
1941 return or_predicates (summary
->conds
, &p1
, &p2
);
1948 return false_predicate ();
1952 /* Return predicate specifying when the STMT might have result that is not
1953 a compile time constant. */
1955 static struct predicate
1956 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1957 struct inline_summary
*summary
,
1959 vec
<predicate_t
> nonconstant_names
)
1961 struct predicate p
= true_predicate ();
1964 struct predicate op_non_const
;
1967 struct agg_position_info aggpos
;
1969 /* What statments might be optimized away
1970 when their arguments are constant
1971 TODO: also trivial builtins.
1972 builtin_constant_p is already handled later. */
1973 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1974 && gimple_code (stmt
) != GIMPLE_COND
1975 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1978 /* Stores will stay anyway. */
1979 if (gimple_store_p (stmt
))
1982 is_load
= gimple_assign_load_p (stmt
);
1984 /* Loads can be optimized when the value is known. */
1988 gcc_assert (gimple_assign_single_p (stmt
));
1989 op
= gimple_assign_rhs1 (stmt
);
1990 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
1997 /* See if we understand all operands before we start
1998 adding conditionals. */
1999 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2001 tree parm
= unmodified_parm (stmt
, use
);
2002 /* For arguments we can build a condition. */
2003 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
2005 if (TREE_CODE (use
) != SSA_NAME
)
2007 /* If we know when operand is constant,
2008 we still can say something useful. */
2009 if (!true_predicate_p (&nonconstant_names
[SSA_NAME_VERSION (use
)]))
2016 add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
2018 op_non_const
= false_predicate ();
2019 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2021 tree parm
= unmodified_parm (stmt
, use
);
2024 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
2026 if (index
!= base_index
)
2027 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
2032 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2033 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
2035 if (gimple_code (stmt
) == GIMPLE_ASSIGN
2036 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
2037 nonconstant_names
[SSA_NAME_VERSION (gimple_assign_lhs (stmt
))]
2039 return op_non_const
;
2042 struct record_modified_bb_info
2048 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2049 set except for info->stmt. */
2052 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
2054 struct record_modified_bb_info
*info
=
2055 (struct record_modified_bb_info
*) data
;
2056 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2058 bitmap_set_bit (info
->bb_set
,
2059 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2060 ? ENTRY_BLOCK_PTR
->index
2061 : gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
2065 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2066 will change since last invocation of STMT.
2068 Value 0 is reserved for compile time invariants.
2069 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2070 ought to be REG_BR_PROB_BASE / estimated_iters. */
2073 param_change_prob (gimple stmt
, int i
)
2075 tree op
= gimple_call_arg (stmt
, i
);
2076 basic_block bb
= gimple_bb (stmt
);
2079 /* Global invariants neve change. */
2080 if (is_gimple_min_invariant (op
))
2082 /* We would have to do non-trivial analysis to really work out what
2083 is the probability of value to change (i.e. when init statement
2084 is in a sibling loop of the call).
2086 We do an conservative estimate: when call is executed N times more often
2087 than the statement defining value, we take the frequency 1/N. */
2088 if (TREE_CODE (op
) == SSA_NAME
)
2093 return REG_BR_PROB_BASE
;
2095 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2096 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2098 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2102 if (init_freq
< bb
->frequency
)
2103 return MAX (GCOV_COMPUTE_SCALE (init_freq
, bb
->frequency
), 1);
2105 return REG_BR_PROB_BASE
;
2108 base
= get_base_address (op
);
2113 struct record_modified_bb_info info
;
2116 tree init
= ctor_for_folding (base
);
2118 if (init
!= error_mark_node
)
2121 return REG_BR_PROB_BASE
;
2122 ao_ref_init (&refd
, op
);
2124 info
.bb_set
= BITMAP_ALLOC (NULL
);
2125 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2127 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2129 BITMAP_FREE (info
.bb_set
);
2130 return REG_BR_PROB_BASE
;
2133 /* Assume that every memory is initialized at entry.
2134 TODO: Can we easilly determine if value is always defined
2135 and thus we may skip entry block? */
2136 if (ENTRY_BLOCK_PTR
->frequency
)
2137 max
= ENTRY_BLOCK_PTR
->frequency
;
2141 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2142 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2144 BITMAP_FREE (info
.bb_set
);
2145 if (max
< bb
->frequency
)
2146 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->frequency
), 1);
2148 return REG_BR_PROB_BASE
;
2150 return REG_BR_PROB_BASE
;
2153 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2154 sub-graph and if the predicate the condition depends on is known. If so,
2155 return true and store the pointer the predicate in *P. */
2158 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2159 struct inline_summary
*summary
, basic_block bb
,
2160 struct predicate
*p
,
2161 vec
<predicate_t
> nonconstant_names
)
2165 basic_block first_bb
= NULL
;
2168 if (single_pred_p (bb
))
2170 *p
= false_predicate ();
2174 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2176 if (single_succ_p (e
->src
))
2178 if (!single_pred_p (e
->src
))
2181 first_bb
= single_pred (e
->src
);
2182 else if (single_pred (e
->src
) != first_bb
)
2189 else if (e
->src
!= first_bb
)
2197 stmt
= last_stmt (first_bb
);
2199 || gimple_code (stmt
) != GIMPLE_COND
2200 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2203 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2204 gimple_cond_lhs (stmt
),
2206 if (true_predicate_p (p
))
2212 /* Given a PHI statement in a function described by inline properties SUMMARY
2213 and *P being the predicate describing whether the selected PHI argument is
2214 known, store a predicate for the result of the PHI statement into
2215 NONCONSTANT_NAMES, if possible. */
2218 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2219 struct predicate
*p
,
2220 vec
<predicate_t
> nonconstant_names
)
2224 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2226 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2227 if (!is_gimple_min_invariant (arg
))
2229 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2230 *p
= or_predicates (summary
->conds
, p
,
2231 &nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2232 if (true_predicate_p (p
))
2237 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2239 fprintf (dump_file
, "\t\tphi predicate: ");
2240 dump_predicate (dump_file
, summary
->conds
, p
);
2242 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2245 /* Return predicate specifying when array index in access OP becomes non-constant. */
2247 static struct predicate
2248 array_index_predicate (struct inline_summary
*info
,
2249 vec
< predicate_t
> nonconstant_names
, tree op
)
2251 struct predicate p
= false_predicate ();
2252 while (handled_component_p (op
))
2254 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
2256 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2257 p
= or_predicates (info
->conds
, &p
,
2258 &nonconstant_names
[SSA_NAME_VERSION
2259 (TREE_OPERAND (op
, 1))]);
2261 op
= TREE_OPERAND (op
, 0);
2266 /* For a typical usage of __builtin_expect (a<b, 1), we
2267 may introduce an extra relation stmt:
2268 With the builtin, we have
2271 t3 = __builtin_expect (t2, 1);
2274 Without the builtin, we have
2277 This affects the size/time estimation and may have
2278 an impact on the earlier inlining.
2279 Here find this pattern and fix it up later. */
2282 find_foldable_builtin_expect (basic_block bb
)
2284 gimple_stmt_iterator bsi
;
2286 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2288 gimple stmt
= gsi_stmt (bsi
);
2289 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
))
2291 tree var
= gimple_call_lhs (stmt
);
2292 tree arg
= gimple_call_arg (stmt
, 0);
2293 use_operand_p use_p
;
2300 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2302 while (TREE_CODE (arg
) == SSA_NAME
)
2304 gimple stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2305 if (!is_gimple_assign (stmt_tmp
))
2307 switch (gimple_assign_rhs_code (stmt_tmp
))
2326 arg
= gimple_assign_rhs1 (stmt_tmp
);
2329 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2330 && gimple_code (use_stmt
) == GIMPLE_COND
)
2337 /* Compute function body size parameters for NODE.
2338 When EARLY is true, we compute only simple summaries without
2339 non-trivial predicates to drive the early inliner. */
2342 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2345 /* Estimate static overhead for function prologue/epilogue and alignment. */
2347 /* Benefits are scaled by probability of elimination that is in range
2350 gimple_stmt_iterator bsi
;
2351 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
2353 struct inline_summary
*info
= inline_summary (node
);
2354 struct predicate bb_predicate
;
2355 struct ipa_node_params
*parms_info
= NULL
;
2356 vec
<predicate_t
> nonconstant_names
= vNULL
;
2359 predicate array_index
= true_predicate ();
2360 gimple fix_builtin_expect_stmt
;
2365 if (optimize
&& !early
)
2367 calculate_dominance_info (CDI_DOMINATORS
);
2368 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2370 if (ipa_node_params_vector
.exists ())
2372 parms_info
= IPA_NODE_REF (node
);
2373 nonconstant_names
.safe_grow_cleared
2374 (SSANAMES (my_function
)->length ());
2379 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2380 cgraph_node_name (node
));
2382 /* When we run into maximal number of entries, we assign everything to the
2383 constant truth case. Be sure to have it in list. */
2384 bb_predicate
= true_predicate ();
2385 account_size_time (info
, 0, 0, &bb_predicate
);
2387 bb_predicate
= not_inlined_predicate ();
2388 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2390 gcc_assert (my_function
&& my_function
->cfg
);
2392 compute_bb_predicates (node
, parms_info
, info
);
2393 gcc_assert (cfun
== my_function
);
2394 order
= XNEWVEC (int, n_basic_blocks
);
2395 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2396 for (n
= 0; n
< nblocks
; n
++)
2398 bb
= BASIC_BLOCK (order
[n
]);
2399 freq
= compute_call_stmt_bb_frequency (node
->symbol
.decl
, bb
);
2401 /* TODO: Obviously predicates can be propagated down across CFG. */
2405 bb_predicate
= *(struct predicate
*) bb
->aux
;
2407 bb_predicate
= false_predicate ();
2410 bb_predicate
= true_predicate ();
2412 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2414 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2415 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2418 if (parms_info
&& nonconstant_names
.exists ())
2420 struct predicate phi_predicate
;
2421 bool first_phi
= true;
2423 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2426 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2431 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2433 fprintf (dump_file
, " ");
2434 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2436 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2441 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2443 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2445 gimple stmt
= gsi_stmt (bsi
);
2446 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2447 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2449 struct predicate will_be_nonconstant
;
2451 /* This relation stmt should be folded after we remove
2452 buildin_expect call. Adjust the cost here. */
2453 if (stmt
== fix_builtin_expect_stmt
)
2459 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2461 fprintf (dump_file
, " ");
2462 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2463 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2464 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2468 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2470 struct predicate this_array_index
;
2472 array_index_predicate (info
, nonconstant_names
,
2473 gimple_assign_rhs1 (stmt
));
2474 if (!false_predicate_p (&this_array_index
))
2476 and_predicates (info
->conds
, &array_index
,
2479 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2481 struct predicate this_array_index
;
2483 array_index_predicate (info
, nonconstant_names
,
2484 gimple_get_lhs (stmt
));
2485 if (!false_predicate_p (&this_array_index
))
2487 and_predicates (info
->conds
, &array_index
,
2492 if (is_gimple_call (stmt
))
2494 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2495 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2497 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2498 resolved as constant. We however don't want to optimize
2499 out the cgraph edges. */
2500 if (nonconstant_names
.exists ()
2501 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2502 && gimple_call_lhs (stmt
)
2503 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2505 struct predicate false_p
= false_predicate ();
2506 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2509 if (ipa_node_params_vector
.exists ())
2511 int count
= gimple_call_num_args (stmt
);
2515 es
->param
.safe_grow_cleared (count
);
2516 for (i
= 0; i
< count
; i
++)
2518 int prob
= param_change_prob (stmt
, i
);
2519 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2520 es
->param
[i
].change_prob
= prob
;
2524 es
->call_stmt_size
= this_size
;
2525 es
->call_stmt_time
= this_time
;
2526 es
->loop_depth
= bb_loop_depth (bb
);
2527 edge_set_predicate (edge
, &bb_predicate
);
2530 /* TODO: When conditional jump or swithc is known to be constant, but
2531 we did not translate it into the predicates, we really can account
2532 just maximum of the possible paths. */
2535 = will_be_nonconstant_predicate (parms_info
, info
,
2536 stmt
, nonconstant_names
);
2537 if (this_time
|| this_size
)
2543 prob
= eliminated_by_inlining_prob (stmt
);
2544 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2546 "\t\t50%% will be eliminated by inlining\n");
2547 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2548 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2551 p
= and_predicates (info
->conds
, &bb_predicate
,
2552 &will_be_nonconstant
);
2554 p
= true_predicate ();
2556 if (!false_predicate_p (&p
))
2560 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2561 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2564 /* We account everything but the calls. Calls have their own
2565 size/time info attached to cgraph edges. This is necessary
2566 in order to make the cost disappear after inlining. */
2567 if (!is_gimple_call (stmt
))
2571 struct predicate ip
= not_inlined_predicate ();
2572 ip
= and_predicates (info
->conds
, &ip
, &p
);
2573 account_size_time (info
, this_size
* prob
,
2574 this_time
* prob
, &ip
);
2577 account_size_time (info
, this_size
* (2 - prob
),
2578 this_time
* (2 - prob
), &p
);
2581 gcc_assert (time
>= 0);
2582 gcc_assert (size
>= 0);
2586 set_hint_predicate (&inline_summary (node
)->array_index
, array_index
);
2587 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2588 if (time
> MAX_TIME
)
2592 if (!early
&& nonconstant_names
.exists ())
2596 predicate loop_iterations
= true_predicate ();
2597 predicate loop_stride
= true_predicate ();
2599 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2600 flow_loops_dump (dump_file
, NULL
, 0);
2602 FOR_EACH_LOOP (li
, loop
, 0)
2607 struct tree_niter_desc niter_desc
;
2608 basic_block
*body
= get_loop_body (loop
);
2609 bb_predicate
= *(struct predicate
*) loop
->header
->aux
;
2611 exits
= get_loop_exit_edges (loop
);
2612 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2613 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2614 && !is_gimple_min_invariant (niter_desc
.niter
))
2616 predicate will_be_nonconstant
2617 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2620 if (!true_predicate_p (&will_be_nonconstant
))
2621 will_be_nonconstant
= and_predicates (info
->conds
,
2623 &will_be_nonconstant
);
2624 if (!true_predicate_p (&will_be_nonconstant
)
2625 && !false_predicate_p (&will_be_nonconstant
))
2626 /* This is slightly inprecise. We may want to represent each
2627 loop with independent predicate. */
2629 and_predicates (info
->conds
, &loop_iterations
,
2630 &will_be_nonconstant
);
2634 for (i
= 0; i
< loop
->num_nodes
; i
++)
2636 gimple_stmt_iterator gsi
;
2637 bb_predicate
= *(struct predicate
*) body
[i
]->aux
;
2638 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2641 gimple stmt
= gsi_stmt (gsi
);
2646 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2648 predicate will_be_nonconstant
;
2651 (loop
, loop_containing_stmt (stmt
), use
, &iv
, true)
2652 || is_gimple_min_invariant (iv
.step
))
2655 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2658 if (!true_predicate_p (&will_be_nonconstant
))
2660 = and_predicates (info
->conds
,
2662 &will_be_nonconstant
);
2663 if (!true_predicate_p (&will_be_nonconstant
)
2664 && !false_predicate_p (&will_be_nonconstant
))
2665 /* This is slightly inprecise. We may want to represent
2666 each loop with independent predicate. */
2668 and_predicates (info
->conds
, &loop_stride
,
2669 &will_be_nonconstant
);
2675 set_hint_predicate (&inline_summary (node
)->loop_iterations
,
2677 set_hint_predicate (&inline_summary (node
)->loop_stride
, loop_stride
);
2680 FOR_ALL_BB_FN (bb
, my_function
)
2686 pool_free (edge_predicate_pool
, bb
->aux
);
2688 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2691 pool_free (edge_predicate_pool
, e
->aux
);
2695 inline_summary (node
)->self_time
= time
;
2696 inline_summary (node
)->self_size
= size
;
2697 nonconstant_names
.release ();
2698 if (optimize
&& !early
)
2700 loop_optimizer_finalize ();
2701 free_dominance_info (CDI_DOMINATORS
);
2705 fprintf (dump_file
, "\n");
2706 dump_inline_summary (dump_file
, node
);
2711 /* Compute parameters of functions used by inliner.
2712 EARLY is true when we compute parameters for the early inliner */
2715 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2717 HOST_WIDE_INT self_stack_size
;
2718 struct cgraph_edge
*e
;
2719 struct inline_summary
*info
;
2721 gcc_assert (!node
->global
.inlined_to
);
2723 inline_summary_alloc ();
2725 info
= inline_summary (node
);
2726 reset_inline_summary (node
);
2728 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2729 Once this happen, we will need to more curefully predict call
2731 if (node
->thunk
.thunk_p
)
2733 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2734 struct predicate t
= true_predicate ();
2736 info
->inlinable
= 0;
2737 node
->callees
->call_stmt_cannot_inline_p
= true;
2738 node
->local
.can_change_signature
= false;
2739 es
->call_stmt_time
= 1;
2740 es
->call_stmt_size
= 1;
2741 account_size_time (info
, 0, 0, &t
);
2745 /* Even is_gimple_min_invariant rely on current_function_decl. */
2746 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
2748 /* Estimate the stack size for the function if we're optimizing. */
2749 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2750 info
->estimated_self_stack_size
= self_stack_size
;
2751 info
->estimated_stack_size
= self_stack_size
;
2752 info
->stack_frame_offset
= 0;
2754 /* Can this function be inlined at all? */
2755 if (!optimize
&& !lookup_attribute ("always_inline",
2756 DECL_ATTRIBUTES (node
->symbol
.decl
)))
2757 info
->inlinable
= false;
2759 info
->inlinable
= tree_inlinable_function_p (node
->symbol
.decl
);
2761 /* Type attributes can use parameter indices to describe them. */
2762 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->symbol
.decl
)))
2763 node
->local
.can_change_signature
= false;
2766 /* Otherwise, inlinable functions always can change signature. */
2767 if (info
->inlinable
)
2768 node
->local
.can_change_signature
= true;
2771 /* Functions calling builtin_apply can not change signature. */
2772 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2774 tree
cdecl = e
->callee
->symbol
.decl
;
2775 if (DECL_BUILT_IN (cdecl)
2776 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2777 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2778 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2781 node
->local
.can_change_signature
= !e
;
2784 estimate_function_body_sizes (node
, early
);
2786 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2787 info
->time
= info
->self_time
;
2788 info
->size
= info
->self_size
;
2789 info
->stack_frame_offset
= 0;
2790 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2791 #ifdef ENABLE_CHECKING
2792 inline_update_overall_summary (node
);
2793 gcc_assert (info
->time
== info
->self_time
&& info
->size
== info
->self_size
);
2800 /* Compute parameters of functions used by inliner using
2801 current_function_decl. */
2804 compute_inline_parameters_for_current (void)
2806 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2812 const pass_data pass_data_inline_parameters
=
2814 GIMPLE_PASS
, /* type */
2815 "inline_param", /* name */
2816 OPTGROUP_INLINE
, /* optinfo_flags */
2817 false, /* has_gate */
2818 true, /* has_execute */
2819 TV_INLINE_PARAMETERS
, /* tv_id */
2820 0, /* properties_required */
2821 0, /* properties_provided */
2822 0, /* properties_destroyed */
2823 0, /* todo_flags_start */
2824 0, /* todo_flags_finish */
2827 class pass_inline_parameters
: public gimple_opt_pass
2830 pass_inline_parameters (gcc::context
*ctxt
)
2831 : gimple_opt_pass (pass_data_inline_parameters
, ctxt
)
2834 /* opt_pass methods: */
2835 opt_pass
* clone () { return new pass_inline_parameters (m_ctxt
); }
2836 unsigned int execute () {
2837 return compute_inline_parameters_for_current ();
2840 }; // class pass_inline_parameters
2845 make_pass_inline_parameters (gcc::context
*ctxt
)
2847 return new pass_inline_parameters (ctxt
);
2851 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2855 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2856 int *size
, int *time
,
2857 vec
<tree
> known_vals
,
2858 vec
<tree
> known_binfos
,
2859 vec
<ipa_agg_jump_function_p
> known_aggs
)
2862 struct cgraph_node
*callee
;
2863 struct inline_summary
*isummary
;
2865 if (!known_vals
.exists () && !known_binfos
.exists ())
2867 if (!flag_indirect_inlining
)
2870 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2875 /* Account for difference in cost between indirect and direct calls. */
2876 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2877 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2878 gcc_checking_assert (*time
>= 0);
2879 gcc_checking_assert (*size
>= 0);
2881 callee
= cgraph_get_node (target
);
2882 if (!callee
|| !callee
->symbol
.definition
)
2884 isummary
= inline_summary (callee
);
2885 return isummary
->inlinable
;
2888 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2891 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2893 vec
<tree
> known_vals
,
2894 vec
<tree
> known_binfos
,
2895 vec
<ipa_agg_jump_function_p
> known_aggs
,
2896 inline_hints
*hints
)
2898 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2899 int call_size
= es
->call_stmt_size
;
2900 int call_time
= es
->call_stmt_time
;
2902 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2903 known_vals
, known_binfos
, known_aggs
)
2904 && hints
&& cgraph_maybe_hot_edge_p (e
))
2905 *hints
|= INLINE_HINT_indirect_call
;
2906 *size
+= call_size
* INLINE_SIZE_SCALE
;
2907 *time
+= apply_probability ((gcov_type
) call_time
, prob
)
2908 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
);
2909 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2910 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2915 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2916 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2920 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2921 inline_hints
*hints
,
2922 clause_t possible_truths
,
2923 vec
<tree
> known_vals
,
2924 vec
<tree
> known_binfos
,
2925 vec
<ipa_agg_jump_function_p
> known_aggs
)
2927 struct cgraph_edge
*e
;
2928 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2930 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2932 || evaluate_predicate (es
->predicate
, possible_truths
))
2934 if (e
->inline_failed
)
2936 /* Predicates of calls shall not use NOT_CHANGED codes,
2937 sowe do not need to compute probabilities. */
2938 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2939 known_vals
, known_binfos
,
2943 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2945 known_vals
, known_binfos
,
2949 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2951 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2953 || evaluate_predicate (es
->predicate
, possible_truths
))
2954 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2955 known_vals
, known_binfos
, known_aggs
,
2961 /* Estimate size and time needed to execute NODE assuming
2962 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2963 about NODE's arguments. */
2966 estimate_node_size_and_time (struct cgraph_node
*node
,
2967 clause_t possible_truths
,
2968 vec
<tree
> known_vals
,
2969 vec
<tree
> known_binfos
,
2970 vec
<ipa_agg_jump_function_p
> known_aggs
,
2971 int *ret_size
, int *ret_time
,
2972 inline_hints
*ret_hints
,
2973 vec
<inline_param_summary_t
>
2974 inline_param_summary
)
2976 struct inline_summary
*info
= inline_summary (node
);
2980 inline_hints hints
= 0;
2983 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2986 fprintf (dump_file
, " Estimating body: %s/%i\n"
2987 " Known to be false: ", cgraph_node_name (node
),
2988 node
->symbol
.order
);
2990 for (i
= predicate_not_inlined_condition
;
2991 i
< (predicate_first_dynamic_condition
2992 + (int) vec_safe_length (info
->conds
)); i
++)
2993 if (!(possible_truths
& (1 << i
)))
2996 fprintf (dump_file
, ", ");
2998 dump_condition (dump_file
, info
->conds
, i
);
3002 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3003 if (evaluate_predicate (&e
->predicate
, possible_truths
))
3006 gcc_checking_assert (e
->time
>= 0);
3007 gcc_checking_assert (time
>= 0);
3008 if (!inline_param_summary
.exists ())
3012 int prob
= predicate_probability (info
->conds
,
3015 inline_param_summary
);
3016 gcc_checking_assert (prob
>= 0);
3017 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3018 time
+= apply_probability ((gcov_type
) e
->time
, prob
);
3020 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
3021 time
= MAX_TIME
* INLINE_TIME_SCALE
;
3022 gcc_checking_assert (time
>= 0);
3025 gcc_checking_assert (size
>= 0);
3026 gcc_checking_assert (time
>= 0);
3028 if (info
->loop_iterations
3029 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
3030 hints
|= INLINE_HINT_loop_iterations
;
3031 if (info
->loop_stride
3032 && !evaluate_predicate (info
->loop_stride
, possible_truths
))
3033 hints
|= INLINE_HINT_loop_stride
;
3034 if (info
->array_index
3035 && !evaluate_predicate (info
->array_index
, possible_truths
))
3036 hints
|= INLINE_HINT_array_index
;
3038 hints
|= INLINE_HINT_in_scc
;
3039 if (DECL_DECLARED_INLINE_P (node
->symbol
.decl
))
3040 hints
|= INLINE_HINT_declared_inline
;
3042 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
3043 known_vals
, known_binfos
, known_aggs
);
3044 gcc_checking_assert (size
>= 0);
3045 gcc_checking_assert (time
>= 0);
3046 time
= RDIV (time
, INLINE_TIME_SCALE
);
3047 size
= RDIV (size
, INLINE_SIZE_SCALE
);
3049 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3050 fprintf (dump_file
, "\n size:%i time:%i\n", (int) size
, (int) time
);
3061 /* Estimate size and time needed to execute callee of EDGE assuming that
3062 parameters known to be constant at caller of EDGE are propagated.
3063 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
3064 and types for parameters. */
3067 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
3068 vec
<tree
> known_vals
,
3069 vec
<tree
> known_binfos
,
3070 vec
<ipa_agg_jump_function_p
> known_aggs
,
3071 int *ret_size
, int *ret_time
,
3072 inline_hints
*hints
)
3076 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
,
3078 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
,
3079 known_aggs
, ret_size
, ret_time
, hints
, vNULL
);
3082 /* Translate all conditions from callee representation into caller
3083 representation and symbolically evaluate predicate P into new predicate.
3085 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
3086 is summary of function predicate P is from. OPERAND_MAP is array giving
3087 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
3088 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
3089 predicate under which callee is executed. OFFSET_MAP is an array of of
3090 offsets that need to be added to conditions, negative offset means that
3091 conditions relying on values passed by reference have to be discarded
3092 because they might not be preserved (and should be considered offset zero
3093 for other purposes). */
3095 static struct predicate
3096 remap_predicate (struct inline_summary
*info
,
3097 struct inline_summary
*callee_info
,
3098 struct predicate
*p
,
3099 vec
<int> operand_map
,
3100 vec
<int> offset_map
,
3101 clause_t possible_truths
, struct predicate
*toplev_predicate
)
3104 struct predicate out
= true_predicate ();
3106 /* True predicate is easy. */
3107 if (true_predicate_p (p
))
3108 return *toplev_predicate
;
3109 for (i
= 0; p
->clause
[i
]; i
++)
3111 clause_t clause
= p
->clause
[i
];
3113 struct predicate clause_predicate
= false_predicate ();
3115 gcc_assert (i
< MAX_CLAUSES
);
3117 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
3118 /* Do we have condition we can't disprove? */
3119 if (clause
& possible_truths
& (1 << cond
))
3121 struct predicate cond_predicate
;
3122 /* Work out if the condition can translate to predicate in the
3123 inlined function. */
3124 if (cond
>= predicate_first_dynamic_condition
)
3126 struct condition
*c
;
3128 c
= &(*callee_info
->conds
)[cond
3130 predicate_first_dynamic_condition
];
3131 /* See if we can remap condition operand to caller's operand.
3132 Otherwise give up. */
3133 if (!operand_map
.exists ()
3134 || (int) operand_map
.length () <= c
->operand_num
3135 || operand_map
[c
->operand_num
] == -1
3136 /* TODO: For non-aggregate conditions, adding an offset is
3137 basically an arithmetic jump function processing which
3138 we should support in future. */
3139 || ((!c
->agg_contents
|| !c
->by_ref
)
3140 && offset_map
[c
->operand_num
] > 0)
3141 || (c
->agg_contents
&& c
->by_ref
3142 && offset_map
[c
->operand_num
] < 0))
3143 cond_predicate
= true_predicate ();
3146 struct agg_position_info ap
;
3147 HOST_WIDE_INT offset_delta
= offset_map
[c
->operand_num
];
3148 if (offset_delta
< 0)
3150 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
3153 gcc_assert (!c
->agg_contents
3154 || c
->by_ref
|| offset_delta
== 0);
3155 ap
.offset
= c
->offset
+ offset_delta
;
3156 ap
.agg_contents
= c
->agg_contents
;
3157 ap
.by_ref
= c
->by_ref
;
3158 cond_predicate
= add_condition (info
,
3159 operand_map
[c
->operand_num
],
3160 &ap
, c
->code
, c
->val
);
3163 /* Fixed conditions remains same, construct single
3164 condition predicate. */
3167 cond_predicate
.clause
[0] = 1 << cond
;
3168 cond_predicate
.clause
[1] = 0;
3170 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
3173 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
3175 return and_predicates (info
->conds
, &out
, toplev_predicate
);
3179 /* Update summary information of inline clones after inlining.
3180 Compute peak stack usage. */
3183 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3185 struct cgraph_edge
*e
;
3186 struct inline_summary
*callee_info
= inline_summary (node
);
3187 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
3190 callee_info
->stack_frame_offset
3191 = caller_info
->stack_frame_offset
3192 + caller_info
->estimated_self_stack_size
;
3193 peak
= callee_info
->stack_frame_offset
3194 + callee_info
->estimated_self_stack_size
;
3195 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
3196 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
3197 ipa_propagate_frequency (node
);
3198 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3200 if (!e
->inline_failed
)
3201 inline_update_callee_summaries (e
->callee
, depth
);
3202 inline_edge_summary (e
)->loop_depth
+= depth
;
3204 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3205 inline_edge_summary (e
)->loop_depth
+= depth
;
3208 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3209 When functoin A is inlined in B and A calls C with parameter that
3210 changes with probability PROB1 and C is known to be passthroug
3211 of argument if B that change with probability PROB2, the probability
3212 of change is now PROB1*PROB2. */
3215 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3216 struct cgraph_edge
*edge
)
3218 if (ipa_node_params_vector
.exists ())
3221 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3222 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3223 struct inline_edge_summary
*inlined_es
3224 = inline_edge_summary (inlined_edge
);
3226 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3228 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3229 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3230 && (ipa_get_jf_pass_through_formal_id (jfunc
)
3231 < (int) inlined_es
->param
.length ()))
3233 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
3234 int prob1
= es
->param
[i
].change_prob
;
3235 int prob2
= inlined_es
->param
[jf_formal_id
].change_prob
;
3236 int prob
= combine_probabilities (prob1
, prob2
);
3238 if (prob1
&& prob2
&& !prob
)
3241 es
->param
[i
].change_prob
= prob
;
3247 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3249 Remap predicates of callees of NODE. Rest of arguments match
3252 Also update change probabilities. */
3255 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3256 struct cgraph_node
*node
,
3257 struct inline_summary
*info
,
3258 struct inline_summary
*callee_info
,
3259 vec
<int> operand_map
,
3260 vec
<int> offset_map
,
3261 clause_t possible_truths
,
3262 struct predicate
*toplev_predicate
)
3264 struct cgraph_edge
*e
;
3265 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3267 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3270 if (e
->inline_failed
)
3272 remap_edge_change_prob (inlined_edge
, e
);
3276 p
= remap_predicate (info
, callee_info
,
3277 es
->predicate
, operand_map
, offset_map
,
3278 possible_truths
, toplev_predicate
);
3279 edge_set_predicate (e
, &p
);
3280 /* TODO: We should remove the edge for code that will be
3281 optimized out, but we need to keep verifiers and tree-inline
3282 happy. Make it cold for now. */
3283 if (false_predicate_p (&p
))
3290 edge_set_predicate (e
, toplev_predicate
);
3293 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3294 operand_map
, offset_map
, possible_truths
,
3297 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3299 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3302 remap_edge_change_prob (inlined_edge
, e
);
3305 p
= remap_predicate (info
, callee_info
,
3306 es
->predicate
, operand_map
, offset_map
,
3307 possible_truths
, toplev_predicate
);
3308 edge_set_predicate (e
, &p
);
3309 /* TODO: We should remove the edge for code that will be optimized
3310 out, but we need to keep verifiers and tree-inline happy.
3311 Make it cold for now. */
3312 if (false_predicate_p (&p
))
3319 edge_set_predicate (e
, toplev_predicate
);
3323 /* Same as remap_predicate, but set result into hint *HINT. */
3326 remap_hint_predicate (struct inline_summary
*info
,
3327 struct inline_summary
*callee_info
,
3328 struct predicate
**hint
,
3329 vec
<int> operand_map
,
3330 vec
<int> offset_map
,
3331 clause_t possible_truths
,
3332 struct predicate
*toplev_predicate
)
3338 p
= remap_predicate (info
, callee_info
,
3340 operand_map
, offset_map
,
3341 possible_truths
, toplev_predicate
);
3342 if (!false_predicate_p (&p
) && !true_predicate_p (&p
))
3345 set_hint_predicate (hint
, p
);
3347 **hint
= and_predicates (info
->conds
, *hint
, &p
);
3351 /* We inlined EDGE. Update summary of the function we inlined into. */
3354 inline_merge_summary (struct cgraph_edge
*edge
)
3356 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3357 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3358 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3359 struct inline_summary
*info
= inline_summary (to
);
3360 clause_t clause
= 0; /* not_inline is known to be false. */
3362 vec
<int> operand_map
= vNULL
;
3363 vec
<int> offset_map
= vNULL
;
3365 struct predicate toplev_predicate
;
3366 struct predicate true_p
= true_predicate ();
3367 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3370 toplev_predicate
= *es
->predicate
;
3372 toplev_predicate
= true_predicate ();
3374 if (ipa_node_params_vector
.exists () && callee_info
->conds
)
3376 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3377 int count
= ipa_get_cs_argument_count (args
);
3380 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3383 operand_map
.safe_grow_cleared (count
);
3384 offset_map
.safe_grow_cleared (count
);
3386 for (i
= 0; i
< count
; i
++)
3388 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3391 /* TODO: handle non-NOPs when merging. */
3392 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3394 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3395 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3396 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3399 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3401 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3402 if (offset
>= 0 && offset
< INT_MAX
)
3404 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3405 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3407 offset_map
[i
] = offset
;
3410 operand_map
[i
] = map
;
3411 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3414 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3416 struct predicate p
= remap_predicate (info
, callee_info
,
3417 &e
->predicate
, operand_map
,
3420 if (!false_predicate_p (&p
))
3422 gcov_type add_time
= ((gcov_type
) e
->time
* edge
->frequency
3423 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3424 int prob
= predicate_probability (callee_info
->conds
,
3427 add_time
= apply_probability ((gcov_type
) add_time
, prob
);
3428 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3429 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3430 if (prob
!= REG_BR_PROB_BASE
3431 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3433 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3434 (double) prob
/ REG_BR_PROB_BASE
);
3436 account_size_time (info
, e
->size
, add_time
, &p
);
3439 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3440 offset_map
, clause
, &toplev_predicate
);
3441 remap_hint_predicate (info
, callee_info
,
3442 &callee_info
->loop_iterations
,
3443 operand_map
, offset_map
, clause
, &toplev_predicate
);
3444 remap_hint_predicate (info
, callee_info
,
3445 &callee_info
->loop_stride
,
3446 operand_map
, offset_map
, clause
, &toplev_predicate
);
3447 remap_hint_predicate (info
, callee_info
,
3448 &callee_info
->array_index
,
3449 operand_map
, offset_map
, clause
, &toplev_predicate
);
3451 inline_update_callee_summaries (edge
->callee
,
3452 inline_edge_summary (edge
)->loop_depth
);
3454 /* We do not maintain predicates of inlined edges, free it. */
3455 edge_set_predicate (edge
, &true_p
);
3456 /* Similarly remove param summaries. */
3457 es
->param
.release ();
3458 operand_map
.release ();
3459 offset_map
.release ();
3462 /* For performance reasons inline_merge_summary is not updating overall size
3463 and time. Recompute it. */
3466 inline_update_overall_summary (struct cgraph_node
*node
)
3468 struct inline_summary
*info
= inline_summary (node
);
3474 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3476 info
->size
+= e
->size
, info
->time
+= e
->time
;
3477 if (info
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
3478 info
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
3480 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3481 ~(clause_t
) (1 << predicate_false_condition
),
3482 vNULL
, vNULL
, vNULL
);
3483 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3484 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3487 /* Return hints derrived from EDGE. */
3489 simple_edge_hints (struct cgraph_edge
*edge
)
3492 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3493 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3494 if (inline_summary (to
)->scc_no
3495 && inline_summary (to
)->scc_no
== inline_summary (edge
->callee
)->scc_no
3496 && !cgraph_edge_recursive_p (edge
))
3497 hints
|= INLINE_HINT_same_scc
;
3499 if (to
->symbol
.lto_file_data
&& edge
->callee
->symbol
.lto_file_data
3500 && to
->symbol
.lto_file_data
!= edge
->callee
->symbol
.lto_file_data
)
3501 hints
|= INLINE_HINT_cross_module
;
3506 /* Estimate the time cost for the caller when inlining EDGE.
3507 Only to be called via estimate_edge_time, that handles the
3510 When caching, also update the cache entry. Compute both time and
3511 size, since we always need both metrics eventually. */
3514 do_estimate_edge_time (struct cgraph_edge
*edge
)
3519 struct cgraph_node
*callee
;
3521 vec
<tree
> known_vals
;
3522 vec
<tree
> known_binfos
;
3523 vec
<ipa_agg_jump_function_p
> known_aggs
;
3524 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3526 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3528 gcc_checking_assert (edge
->inline_failed
);
3529 evaluate_properties_for_edge (edge
, true,
3530 &clause
, &known_vals
, &known_binfos
,
3532 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3533 known_aggs
, &size
, &time
, &hints
, es
->param
);
3534 known_vals
.release ();
3535 known_binfos
.release ();
3536 known_aggs
.release ();
3537 gcc_checking_assert (size
>= 0);
3538 gcc_checking_assert (time
>= 0);
3540 /* When caching, update the cache entry. */
3541 if (edge_growth_cache
.exists ())
3543 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3544 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
3545 edge_growth_cache
[edge
->uid
].time
= time
+ (time
>= 0);
3547 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3548 hints
|= simple_edge_hints (edge
);
3549 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3555 /* Return estimated callee growth after inlining EDGE.
3556 Only to be called via estimate_edge_size. */
3559 do_estimate_edge_size (struct cgraph_edge
*edge
)
3562 struct cgraph_node
*callee
;
3564 vec
<tree
> known_vals
;
3565 vec
<tree
> known_binfos
;
3566 vec
<ipa_agg_jump_function_p
> known_aggs
;
3568 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3570 if (edge_growth_cache
.exists ())
3572 do_estimate_edge_time (edge
);
3573 size
= edge_growth_cache
[edge
->uid
].size
;
3574 gcc_checking_assert (size
);
3575 return size
- (size
> 0);
3578 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3580 /* Early inliner runs without caching, go ahead and do the dirty work. */
3581 gcc_checking_assert (edge
->inline_failed
);
3582 evaluate_properties_for_edge (edge
, true,
3583 &clause
, &known_vals
, &known_binfos
,
3585 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3586 known_aggs
, &size
, NULL
, NULL
, vNULL
);
3587 known_vals
.release ();
3588 known_binfos
.release ();
3589 known_aggs
.release ();
3594 /* Estimate the growth of the caller when inlining EDGE.
3595 Only to be called via estimate_edge_size. */
3598 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3601 struct cgraph_node
*callee
;
3603 vec
<tree
> known_vals
;
3604 vec
<tree
> known_binfos
;
3605 vec
<ipa_agg_jump_function_p
> known_aggs
;
3607 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3609 if (edge_growth_cache
.exists ())
3611 do_estimate_edge_time (edge
);
3612 hints
= edge_growth_cache
[edge
->uid
].hints
;
3613 gcc_checking_assert (hints
);
3617 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3619 /* Early inliner runs without caching, go ahead and do the dirty work. */
3620 gcc_checking_assert (edge
->inline_failed
);
3621 evaluate_properties_for_edge (edge
, true,
3622 &clause
, &known_vals
, &known_binfos
,
3624 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3625 known_aggs
, NULL
, NULL
, &hints
, vNULL
);
3626 known_vals
.release ();
3627 known_binfos
.release ();
3628 known_aggs
.release ();
3629 hints
|= simple_edge_hints (edge
);
3634 /* Estimate self time of the function NODE after inlining EDGE. */
3637 estimate_time_after_inlining (struct cgraph_node
*node
,
3638 struct cgraph_edge
*edge
)
3640 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3641 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3644 inline_summary (node
)->time
+ estimate_edge_time (edge
);
3647 if (time
> MAX_TIME
)
3651 return inline_summary (node
)->time
;
3655 /* Estimate the size of NODE after inlining EDGE which should be an
3656 edge to either NODE or a call inlined into NODE. */
3659 estimate_size_after_inlining (struct cgraph_node
*node
,
3660 struct cgraph_edge
*edge
)
3662 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3663 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3665 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3666 gcc_assert (size
>= 0);
3669 return inline_summary (node
)->size
;
3675 struct cgraph_node
*node
;
3676 bool self_recursive
;
3681 /* Worker for do_estimate_growth. Collect growth for all callers. */
3684 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3686 struct cgraph_edge
*e
;
3687 struct growth_data
*d
= (struct growth_data
*) data
;
3689 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3691 gcc_checking_assert (e
->inline_failed
);
3693 if (e
->caller
== d
->node
3694 || (e
->caller
->global
.inlined_to
3695 && e
->caller
->global
.inlined_to
== d
->node
))
3696 d
->self_recursive
= true;
3697 d
->growth
+= estimate_edge_growth (e
);
3703 /* Estimate the growth caused by inlining NODE into all callees. */
3706 do_estimate_growth (struct cgraph_node
*node
)
3708 struct growth_data d
= { node
, 0, false };
3709 struct inline_summary
*info
= inline_summary (node
);
3711 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3713 /* For self recursive functions the growth estimation really should be
3714 infinity. We don't want to return very large values because the growth
3715 plays various roles in badness computation fractions. Be sure to not
3716 return zero or negative growths. */
3717 if (d
.self_recursive
)
3718 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3719 else if (DECL_EXTERNAL (node
->symbol
.decl
))
3723 if (cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3724 d
.growth
-= info
->size
;
3725 /* COMDAT functions are very often not shared across multiple units
3726 since they come from various template instantiations.
3727 Take this into account. */
3728 else if (DECL_COMDAT (node
->symbol
.decl
)
3729 && cgraph_can_remove_if_no_direct_calls_p (node
))
3730 d
.growth
-= (info
->size
3731 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3735 if (node_growth_cache
.exists ())
3737 if ((int) node_growth_cache
.length () <= node
->uid
)
3738 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
3739 node_growth_cache
[node
->uid
] = d
.growth
+ (d
.growth
>= 0);
3745 /* This function performs intraprocedural analysis in NODE that is required to
3746 inline indirect calls. */
3749 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3751 ipa_analyze_node (node
);
3752 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3754 ipa_print_node_params (dump_file
, node
);
3755 ipa_print_node_jump_functions (dump_file
, node
);
3760 /* Note function body size. */
3763 inline_analyze_function (struct cgraph_node
*node
)
3765 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
3768 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3769 cgraph_node_name (node
), node
->symbol
.order
);
3770 if (optimize
&& !node
->thunk
.thunk_p
)
3771 inline_indirect_intraprocedural_analysis (node
);
3772 compute_inline_parameters (node
, false);
3775 struct cgraph_edge
*e
;
3776 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3778 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3779 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3780 e
->call_stmt_cannot_inline_p
= true;
3782 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3784 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3785 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3786 e
->call_stmt_cannot_inline_p
= true;
3794 /* Called when new function is inserted to callgraph late. */
3797 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3799 inline_analyze_function (node
);
3803 /* Note function body size. */
3806 inline_generate_summary (void)
3808 struct cgraph_node
*node
;
3810 /* When not optimizing, do not bother to analyze. Inlining is still done
3811 because edge redirection needs to happen there. */
3812 if (!optimize
&& !flag_lto
&& !flag_wpa
)
3815 function_insertion_hook_holder
=
3816 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3818 ipa_register_cgraph_hooks ();
3819 inline_free_summary ();
3821 FOR_EACH_DEFINED_FUNCTION (node
)
3822 if (!node
->symbol
.alias
)
3823 inline_analyze_function (node
);
3827 /* Read predicate from IB. */
3829 static struct predicate
3830 read_predicate (struct lto_input_block
*ib
)
3832 struct predicate out
;
3838 gcc_assert (k
<= MAX_CLAUSES
);
3839 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3843 /* Zero-initialize the remaining clauses in OUT. */
3844 while (k
<= MAX_CLAUSES
)
3845 out
.clause
[k
++] = 0;
3851 /* Write inline summary for edge E to OB. */
3854 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3856 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3860 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3861 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3862 es
->loop_depth
= streamer_read_uhwi (ib
);
3863 p
= read_predicate (ib
);
3864 edge_set_predicate (e
, &p
);
3865 length
= streamer_read_uhwi (ib
);
3868 es
->param
.safe_grow_cleared (length
);
3869 for (i
= 0; i
< length
; i
++)
3870 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3875 /* Stream in inline summaries from the section. */
3878 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3881 const struct lto_function_header
*header
=
3882 (const struct lto_function_header
*) data
;
3883 const int cfg_offset
= sizeof (struct lto_function_header
);
3884 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3885 const int string_offset
= main_offset
+ header
->main_size
;
3886 struct data_in
*data_in
;
3887 struct lto_input_block ib
;
3888 unsigned int i
, count2
, j
;
3889 unsigned int f_count
;
3891 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3895 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3896 header
->string_size
, vNULL
);
3897 f_count
= streamer_read_uhwi (&ib
);
3898 for (i
= 0; i
< f_count
; i
++)
3901 struct cgraph_node
*node
;
3902 struct inline_summary
*info
;
3903 lto_symtab_encoder_t encoder
;
3904 struct bitpack_d bp
;
3905 struct cgraph_edge
*e
;
3908 index
= streamer_read_uhwi (&ib
);
3909 encoder
= file_data
->symtab_node_encoder
;
3910 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3911 info
= inline_summary (node
);
3913 info
->estimated_stack_size
3914 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3915 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3916 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3918 bp
= streamer_read_bitpack (&ib
);
3919 info
->inlinable
= bp_unpack_value (&bp
, 1);
3921 count2
= streamer_read_uhwi (&ib
);
3922 gcc_assert (!info
->conds
);
3923 for (j
= 0; j
< count2
; j
++)
3926 c
.operand_num
= streamer_read_uhwi (&ib
);
3927 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3928 c
.val
= stream_read_tree (&ib
, data_in
);
3929 bp
= streamer_read_bitpack (&ib
);
3930 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3931 c
.by_ref
= bp_unpack_value (&bp
, 1);
3933 c
.offset
= streamer_read_uhwi (&ib
);
3934 vec_safe_push (info
->conds
, c
);
3936 count2
= streamer_read_uhwi (&ib
);
3937 gcc_assert (!info
->entry
);
3938 for (j
= 0; j
< count2
; j
++)
3940 struct size_time_entry e
;
3942 e
.size
= streamer_read_uhwi (&ib
);
3943 e
.time
= streamer_read_uhwi (&ib
);
3944 e
.predicate
= read_predicate (&ib
);
3946 vec_safe_push (info
->entry
, e
);
3949 p
= read_predicate (&ib
);
3950 set_hint_predicate (&info
->loop_iterations
, p
);
3951 p
= read_predicate (&ib
);
3952 set_hint_predicate (&info
->loop_stride
, p
);
3953 p
= read_predicate (&ib
);
3954 set_hint_predicate (&info
->array_index
, p
);
3955 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3956 read_inline_edge_summary (&ib
, e
);
3957 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3958 read_inline_edge_summary (&ib
, e
);
3961 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3963 lto_data_in_delete (data_in
);
3967 /* Read inline summary. Jump functions are shared among ipa-cp
3968 and inliner, so when ipa-cp is active, we don't need to write them
3972 inline_read_summary (void)
3974 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3975 struct lto_file_decl_data
*file_data
;
3978 inline_summary_alloc ();
3980 while ((file_data
= file_data_vec
[j
++]))
3983 const char *data
= lto_get_section_data (file_data
,
3984 LTO_section_inline_summary
,
3987 inline_read_section (file_data
, data
, len
);
3989 /* Fatal error here. We do not want to support compiling ltrans units
3990 with different version of compiler or different flags than the WPA
3991 unit, so this should never happen. */
3992 fatal_error ("ipa inline summary is missing in input file");
3996 ipa_register_cgraph_hooks ();
3998 ipa_prop_read_jump_functions ();
4000 function_insertion_hook_holder
=
4001 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
4005 /* Write predicate P to OB. */
4008 write_predicate (struct output_block
*ob
, struct predicate
*p
)
4012 for (j
= 0; p
->clause
[j
]; j
++)
4014 gcc_assert (j
< MAX_CLAUSES
);
4015 streamer_write_uhwi (ob
, p
->clause
[j
]);
4017 streamer_write_uhwi (ob
, 0);
4021 /* Write inline summary for edge E to OB. */
4024 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4026 struct inline_edge_summary
*es
= inline_edge_summary (e
);
4029 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4030 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4031 streamer_write_uhwi (ob
, es
->loop_depth
);
4032 write_predicate (ob
, es
->predicate
);
4033 streamer_write_uhwi (ob
, es
->param
.length ());
4034 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4035 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4039 /* Write inline summary for node in SET.
4040 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4041 active, we don't need to write them twice. */
4044 inline_write_summary (void)
4046 struct cgraph_node
*node
;
4047 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
4048 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4049 unsigned int count
= 0;
4052 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4054 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
4055 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4056 if (cnode
&& cnode
->symbol
.definition
&& !cnode
->symbol
.alias
)
4059 streamer_write_uhwi (ob
, count
);
4061 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4063 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
4064 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4065 if (cnode
&& (node
= cnode
)->symbol
.definition
&& !node
->symbol
.alias
)
4067 struct inline_summary
*info
= inline_summary (node
);
4068 struct bitpack_d bp
;
4069 struct cgraph_edge
*edge
;
4072 struct condition
*c
;
4074 streamer_write_uhwi (ob
,
4075 lto_symtab_encoder_encode (encoder
,
4078 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
4079 streamer_write_hwi (ob
, info
->self_size
);
4080 streamer_write_hwi (ob
, info
->self_time
);
4081 bp
= bitpack_create (ob
->main_stream
);
4082 bp_pack_value (&bp
, info
->inlinable
, 1);
4083 streamer_write_bitpack (&bp
);
4084 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4085 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4087 streamer_write_uhwi (ob
, c
->operand_num
);
4088 streamer_write_uhwi (ob
, c
->code
);
4089 stream_write_tree (ob
, c
->val
, true);
4090 bp
= bitpack_create (ob
->main_stream
);
4091 bp_pack_value (&bp
, c
->agg_contents
, 1);
4092 bp_pack_value (&bp
, c
->by_ref
, 1);
4093 streamer_write_bitpack (&bp
);
4094 if (c
->agg_contents
)
4095 streamer_write_uhwi (ob
, c
->offset
);
4097 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
4098 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
4100 streamer_write_uhwi (ob
, e
->size
);
4101 streamer_write_uhwi (ob
, e
->time
);
4102 write_predicate (ob
, &e
->predicate
);
4104 write_predicate (ob
, info
->loop_iterations
);
4105 write_predicate (ob
, info
->loop_stride
);
4106 write_predicate (ob
, info
->array_index
);
4107 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
4108 write_inline_edge_summary (ob
, edge
);
4109 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4110 write_inline_edge_summary (ob
, edge
);
4113 streamer_write_char_stream (ob
->main_stream
, 0);
4114 produce_asm (ob
, NULL
);
4115 destroy_output_block (ob
);
4117 if (optimize
&& !flag_ipa_cp
)
4118 ipa_prop_write_jump_functions ();
4122 /* Release inline summary. */
4125 inline_free_summary (void)
4127 struct cgraph_node
*node
;
4128 if (!inline_edge_summary_vec
.exists ())
4130 FOR_EACH_DEFINED_FUNCTION (node
)
4131 reset_inline_summary (node
);
4132 if (function_insertion_hook_holder
)
4133 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
4134 function_insertion_hook_holder
= NULL
;
4135 if (node_removal_hook_holder
)
4136 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
4137 node_removal_hook_holder
= NULL
;
4138 if (edge_removal_hook_holder
)
4139 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
4140 edge_removal_hook_holder
= NULL
;
4141 if (node_duplication_hook_holder
)
4142 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
4143 node_duplication_hook_holder
= NULL
;
4144 if (edge_duplication_hook_holder
)
4145 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
4146 edge_duplication_hook_holder
= NULL
;
4147 vec_free (inline_summary_vec
);
4148 inline_edge_summary_vec
.release ();
4149 if (edge_predicate_pool
)
4150 free_alloc_pool (edge_predicate_pool
);
4151 edge_predicate_pool
= 0;