1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011, 2012, 2013
3 Free Software Foundation, Inc.
4 Contributed by Jan Hubicka
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Analysis used by the inliner and other passes limiting code size growth.
24 We estimate for each function
26 - average function execution time
27 - inlining size benefit (that is how much of function body size
28 and its call sequence is expected to disappear by inlining)
29 - inlining time benefit
32 - call statement size and time
34 inlinie_summary datastructures store above information locally (i.e.
35 parameters of the function itself) and globally (i.e. parameters of
36 the function created by applying all the inline decisions already
37 present in the callgraph).
39 We provide accestor to the inline_summary datastructure and
40 basic logic updating the parameters when inlining is performed.
42 The summaries are context sensitive. Context means
43 1) partial assignment of known constant values of operands
44 2) whether function is inlined into the call or not.
45 It is easy to add more variants. To represent function size and time
46 that depends on context (i.e. it is known to be optimized away when
47 context is known either by inlining or from IP-CP and clonning),
48 we use predicates. Predicates are logical formulas in
49 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
50 specifying what conditions must be true. Conditions are simple test
51 of the form described above.
53 In order to make predicate (possibly) true, all of its clauses must
54 be (possibly) true. To make clause (possibly) true, one of conditions
55 it mentions must be (possibly) true. There are fixed bounds on
56 number of clauses and conditions and all the manipulation functions
57 are conservative in positive direction. I.e. we may lose precision
58 by thinking that predicate may be true even when it is not.
60 estimate_edge_size and estimate_edge_growth can be used to query
61 function size/time in the given context. inline_merge_summary merges
62 properties of caller and callee after inlining.
64 Finally pass_inline_parameters is exported. This is used to drive
65 computation of function parameters used by the early inliner. IPA
66 inlined performs analysis via its analyze_function method. */
70 #include "coretypes.h"
73 #include "tree-inline.h"
74 #include "langhooks.h"
77 #include "diagnostic.h"
78 #include "gimple-pretty-print.h"
80 #include "tree-pass.h"
83 #include "tree-flow.h"
85 #include "lto-streamer.h"
86 #include "data-streamer.h"
87 #include "tree-streamer.h"
88 #include "ipa-inline.h"
89 #include "alloc-pool.h"
92 #include "tree-scalar-evolution.h"
94 /* Estimate runtime of function can easilly run into huge numbers with many
95 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
96 integer. For anything larger we use gcov_type. */
97 #define MAX_TIME 500000
99 /* Number of bits in integer, but we really want to be stable across different
101 #define NUM_CONDITIONS 32
103 enum predicate_conditions
105 predicate_false_condition
= 0,
106 predicate_not_inlined_condition
= 1,
107 predicate_first_dynamic_condition
= 2
110 /* Special condition code we use to represent test that operand is compile time
112 #define IS_NOT_CONSTANT ERROR_MARK
113 /* Special condition code we use to represent test that operand is not changed
114 across invocation of the function. When operand IS_NOT_CONSTANT it is always
115 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
116 of executions even when they are not compile time constants. */
117 #define CHANGED IDENTIFIER_NODE
119 /* Holders of ipa cgraph hooks: */
120 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
121 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
122 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
123 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
124 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
125 static void inline_node_removal_hook (struct cgraph_node
*, void *);
126 static void inline_node_duplication_hook (struct cgraph_node
*,
127 struct cgraph_node
*, void *);
128 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
129 static void inline_edge_duplication_hook (struct cgraph_edge
*,
130 struct cgraph_edge
*, void *);
132 /* VECtor holding inline summaries.
133 In GGC memory because conditions might point to constant trees. */
134 vec
<inline_summary_t
, va_gc
> *inline_summary_vec
;
135 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
137 /* Cached node/edge growths. */
138 vec
<int> node_growth_cache
;
139 vec
<edge_growth_cache_entry
> edge_growth_cache
;
141 /* Edge predicates goes here. */
142 static alloc_pool edge_predicate_pool
;
144 /* Return true predicate (tautology).
145 We represent it by empty list of clauses. */
147 static inline struct predicate
148 true_predicate (void)
156 /* Return predicate testing single condition number COND. */
158 static inline struct predicate
159 single_cond_predicate (int cond
)
162 p
.clause
[0] = 1 << cond
;
168 /* Return false predicate. First clause require false condition. */
170 static inline struct predicate
171 false_predicate (void)
173 return single_cond_predicate (predicate_false_condition
);
177 /* Return true if P is (false). */
180 true_predicate_p (struct predicate
*p
)
182 return !p
->clause
[0];
186 /* Return true if P is (false). */
189 false_predicate_p (struct predicate
*p
)
191 if (p
->clause
[0] == (1 << predicate_false_condition
))
193 gcc_checking_assert (!p
->clause
[1]
194 && p
->clause
[0] == 1 << predicate_false_condition
);
201 /* Return predicate that is set true when function is not inlined. */
203 static inline struct predicate
204 not_inlined_predicate (void)
206 return single_cond_predicate (predicate_not_inlined_condition
);
209 /* Simple description of whether a memory load or a condition refers to a load
210 from an aggregate and if so, how and where from in the aggregate.
211 Individual fields have the same meaning like fields with the same name in
214 struct agg_position_info
216 HOST_WIDE_INT offset
;
221 /* Add condition to condition list CONDS. AGGPOS describes whether the used
222 oprand is loaded from an aggregate and where in the aggregate it is. It can
223 be NULL, which means this not a load from an aggregate. */
225 static struct predicate
226 add_condition (struct inline_summary
*summary
, int operand_num
,
227 struct agg_position_info
*aggpos
,
228 enum tree_code code
, tree val
)
232 struct condition new_cond
;
233 HOST_WIDE_INT offset
;
234 bool agg_contents
, by_ref
;
238 offset
= aggpos
->offset
;
239 agg_contents
= aggpos
->agg_contents
;
240 by_ref
= aggpos
->by_ref
;
245 agg_contents
= false;
249 gcc_checking_assert (operand_num
>= 0);
250 for (i
= 0; vec_safe_iterate (summary
->conds
, i
, &c
); i
++)
252 if (c
->operand_num
== operand_num
255 && c
->agg_contents
== agg_contents
256 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
257 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
259 /* Too many conditions. Give up and return constant true. */
260 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
261 return true_predicate ();
263 new_cond
.operand_num
= operand_num
;
264 new_cond
.code
= code
;
266 new_cond
.agg_contents
= agg_contents
;
267 new_cond
.by_ref
= by_ref
;
268 new_cond
.offset
= offset
;
269 vec_safe_push (summary
->conds
, new_cond
);
270 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
274 /* Add clause CLAUSE into the predicate P. */
277 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
281 int insert_here
= -1;
288 /* False clause makes the whole predicate false. Kill the other variants. */
289 if (clause
== (1 << predicate_false_condition
))
291 p
->clause
[0] = (1 << predicate_false_condition
);
295 if (false_predicate_p (p
))
298 /* No one should be sily enough to add false into nontrivial clauses. */
299 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
301 /* Look where to insert the clause. At the same time prune out
302 clauses of P that are implied by the new clause and thus
304 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
306 p
->clause
[i2
] = p
->clause
[i
];
311 /* If p->clause[i] implies clause, there is nothing to add. */
312 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
314 /* We had nothing to add, none of clauses should've become
316 gcc_checking_assert (i
== i2
);
320 if (p
->clause
[i
] < clause
&& insert_here
< 0)
323 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
324 Otherwise the p->clause[i] has to stay. */
325 if ((p
->clause
[i
] & clause
) != clause
)
329 /* Look for clauses that are obviously true. I.e.
330 op0 == 5 || op0 != 5. */
331 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
334 if (!(clause
& (1 << c1
)))
336 cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
337 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
338 and thus there is no point for looking for them. */
339 if (cc1
->code
== CHANGED
|| cc1
->code
== IS_NOT_CONSTANT
)
341 for (c2
= c1
+ 1; c2
<= NUM_CONDITIONS
; c2
++)
342 if (clause
& (1 << c2
))
345 &(*conditions
)[c1
- predicate_first_dynamic_condition
];
347 &(*conditions
)[c2
- predicate_first_dynamic_condition
];
348 if (cc1
->operand_num
== cc2
->operand_num
349 && cc1
->val
== cc2
->val
350 && cc2
->code
!= IS_NOT_CONSTANT
351 && cc2
->code
!= CHANGED
352 && cc1
->code
== invert_tree_comparison
354 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
360 /* We run out of variants. Be conservative in positive direction. */
361 if (i2
== MAX_CLAUSES
)
363 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
364 p
->clause
[i2
+ 1] = 0;
365 if (insert_here
>= 0)
366 for (; i2
> insert_here
; i2
--)
367 p
->clause
[i2
] = p
->clause
[i2
- 1];
370 p
->clause
[insert_here
] = clause
;
376 static struct predicate
377 and_predicates (conditions conditions
,
378 struct predicate
*p
, struct predicate
*p2
)
380 struct predicate out
= *p
;
383 /* Avoid busy work. */
384 if (false_predicate_p (p2
) || true_predicate_p (p
))
386 if (false_predicate_p (p
) || true_predicate_p (p2
))
389 /* See how far predicates match. */
390 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
392 gcc_checking_assert (i
< MAX_CLAUSES
);
395 /* Combine the predicates rest. */
396 for (; p2
->clause
[i
]; i
++)
398 gcc_checking_assert (i
< MAX_CLAUSES
);
399 add_clause (conditions
, &out
, p2
->clause
[i
]);
405 /* Return true if predicates are obviously equal. */
408 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
411 for (i
= 0; p
->clause
[i
]; i
++)
413 gcc_checking_assert (i
< MAX_CLAUSES
);
414 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
415 gcc_checking_assert (!p2
->clause
[i
]
416 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
417 if (p
->clause
[i
] != p2
->clause
[i
])
420 return !p2
->clause
[i
];
426 static struct predicate
427 or_predicates (conditions conditions
,
428 struct predicate
*p
, struct predicate
*p2
)
430 struct predicate out
= true_predicate ();
433 /* Avoid busy work. */
434 if (false_predicate_p (p2
) || true_predicate_p (p
))
436 if (false_predicate_p (p
) || true_predicate_p (p2
))
438 if (predicates_equal_p (p
, p2
))
441 /* OK, combine the predicates. */
442 for (i
= 0; p
->clause
[i
]; i
++)
443 for (j
= 0; p2
->clause
[j
]; j
++)
445 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
446 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
452 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
453 if predicate P is known to be false. */
456 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
460 /* True remains true. */
461 if (true_predicate_p (p
))
464 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
466 /* See if we can find clause we can disprove. */
467 for (i
= 0; p
->clause
[i
]; i
++)
469 gcc_checking_assert (i
< MAX_CLAUSES
);
470 if (!(p
->clause
[i
] & possible_truths
))
476 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
477 instruction will be recomputed per invocation of the inlined call. */
480 predicate_probability (conditions conds
,
481 struct predicate
*p
, clause_t possible_truths
,
482 vec
<inline_param_summary_t
> inline_param_summary
)
485 int combined_prob
= REG_BR_PROB_BASE
;
487 /* True remains true. */
488 if (true_predicate_p (p
))
489 return REG_BR_PROB_BASE
;
491 if (false_predicate_p (p
))
494 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
496 /* See if we can find clause we can disprove. */
497 for (i
= 0; p
->clause
[i
]; i
++)
499 gcc_checking_assert (i
< MAX_CLAUSES
);
500 if (!(p
->clause
[i
] & possible_truths
))
506 if (!inline_param_summary
.exists ())
507 return REG_BR_PROB_BASE
;
508 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
509 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
511 if (i2
>= predicate_first_dynamic_condition
)
514 &(*conds
)[i2
- predicate_first_dynamic_condition
];
515 if (c
->code
== CHANGED
517 (int) inline_param_summary
.length ()))
520 inline_param_summary
[c
->operand_num
].change_prob
;
521 this_prob
= MAX (this_prob
, iprob
);
524 this_prob
= REG_BR_PROB_BASE
;
527 this_prob
= REG_BR_PROB_BASE
;
529 combined_prob
= MIN (this_prob
, combined_prob
);
534 return combined_prob
;
538 /* Dump conditional COND. */
541 dump_condition (FILE *f
, conditions conditions
, int cond
)
544 if (cond
== predicate_false_condition
)
545 fprintf (f
, "false");
546 else if (cond
== predicate_not_inlined_condition
)
547 fprintf (f
, "not inlined");
550 c
= &(*conditions
)[cond
- predicate_first_dynamic_condition
];
551 fprintf (f
, "op%i", c
->operand_num
);
553 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
554 c
->by_ref
? "ref " : "", c
->offset
);
555 if (c
->code
== IS_NOT_CONSTANT
)
557 fprintf (f
, " not constant");
560 if (c
->code
== CHANGED
)
562 fprintf (f
, " changed");
565 fprintf (f
, " %s ", op_symbol_code (c
->code
));
566 print_generic_expr (f
, c
->val
, 1);
571 /* Dump clause CLAUSE. */
574 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
581 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
582 if (clause
& (1 << i
))
587 dump_condition (f
, conds
, i
);
593 /* Dump predicate PREDICATE. */
596 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
599 if (true_predicate_p (pred
))
600 dump_clause (f
, conds
, 0);
602 for (i
= 0; pred
->clause
[i
]; i
++)
606 dump_clause (f
, conds
, pred
->clause
[i
]);
612 /* Dump inline hints. */
614 dump_inline_hints (FILE *f
, inline_hints hints
)
618 fprintf (f
, "inline hints:");
619 if (hints
& INLINE_HINT_indirect_call
)
621 hints
&= ~INLINE_HINT_indirect_call
;
622 fprintf (f
, " indirect_call");
624 if (hints
& INLINE_HINT_loop_iterations
)
626 hints
&= ~INLINE_HINT_loop_iterations
;
627 fprintf (f
, " loop_iterations");
629 if (hints
& INLINE_HINT_loop_stride
)
631 hints
&= ~INLINE_HINT_loop_stride
;
632 fprintf (f
, " loop_stride");
634 if (hints
& INLINE_HINT_same_scc
)
636 hints
&= ~INLINE_HINT_same_scc
;
637 fprintf (f
, " same_scc");
639 if (hints
& INLINE_HINT_in_scc
)
641 hints
&= ~INLINE_HINT_in_scc
;
642 fprintf (f
, " in_scc");
644 if (hints
& INLINE_HINT_cross_module
)
646 hints
&= ~INLINE_HINT_cross_module
;
647 fprintf (f
, " cross_module");
649 if (hints
& INLINE_HINT_declared_inline
)
651 hints
&= ~INLINE_HINT_declared_inline
;
652 fprintf (f
, " declared_inline");
654 if (hints
& INLINE_HINT_array_index
)
656 hints
&= ~INLINE_HINT_array_index
;
657 fprintf (f
, " array_index");
663 /* Record SIZE and TIME under condition PRED into the inline summary. */
666 account_size_time (struct inline_summary
*summary
, int size
, int time
,
667 struct predicate
*pred
)
673 if (false_predicate_p (pred
))
676 /* We need to create initial empty unconitional clause, but otherwie
677 we don't need to account empty times and sizes. */
678 if (!size
&& !time
&& summary
->entry
)
681 /* Watch overflow that might result from insane profiles. */
682 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
683 time
= MAX_TIME
* INLINE_TIME_SCALE
;
684 gcc_assert (time
>= 0);
686 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
687 if (predicates_equal_p (&e
->predicate
, pred
))
696 e
= &(*summary
->entry
)[0];
697 gcc_assert (!e
->predicate
.clause
[0]);
698 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
700 "\t\tReached limit on number of entries, "
701 "ignoring the predicate.");
703 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
706 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
707 ((double) size
) / INLINE_SIZE_SCALE
,
708 ((double) time
) / INLINE_TIME_SCALE
, found
? "" : "new ");
709 dump_predicate (dump_file
, summary
->conds
, pred
);
713 struct size_time_entry new_entry
;
714 new_entry
.size
= size
;
715 new_entry
.time
= time
;
716 new_entry
.predicate
= *pred
;
717 vec_safe_push (summary
->entry
, new_entry
);
723 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
724 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
728 /* Set predicate for edge E. */
731 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
733 struct inline_edge_summary
*es
= inline_edge_summary (e
);
734 if (predicate
&& !true_predicate_p (predicate
))
737 es
->predicate
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
738 *es
->predicate
= *predicate
;
743 pool_free (edge_predicate_pool
, es
->predicate
);
744 es
->predicate
= NULL
;
748 /* Set predicate for hint *P. */
751 set_hint_predicate (struct predicate
**p
, struct predicate new_predicate
)
753 if (false_predicate_p (&new_predicate
) || true_predicate_p (&new_predicate
))
756 pool_free (edge_predicate_pool
, *p
);
762 *p
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
768 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
769 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
770 Return clause of possible truths. When INLINE_P is true, assume that we are
773 ERROR_MARK means compile time invariant. */
776 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
778 vec
<tree
> known_vals
,
779 vec
<ipa_agg_jump_function_p
>
782 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
783 struct inline_summary
*info
= inline_summary (node
);
787 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
792 /* We allow call stmt to have fewer arguments than the callee function
793 (especially for K&R style programs). So bound check here (we assume
794 known_aggs vector, if non-NULL, has the same length as
796 gcc_checking_assert (!known_aggs
.exists ()
797 || (known_vals
.length () == known_aggs
.length ()));
798 if (c
->operand_num
>= (int) known_vals
.length ())
800 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
806 struct ipa_agg_jump_function
*agg
;
808 if (c
->code
== CHANGED
810 && (known_vals
[c
->operand_num
] == error_mark_node
))
813 if (known_aggs
.exists ())
815 agg
= known_aggs
[c
->operand_num
];
816 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
823 val
= known_vals
[c
->operand_num
];
824 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
830 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
833 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
835 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
836 if (res
&& integer_zerop (res
))
838 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
844 /* Work out what conditions might be true at invocation of E. */
847 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
848 clause_t
*clause_ptr
,
849 vec
<tree
> *known_vals_ptr
,
850 vec
<tree
> *known_binfos_ptr
,
851 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
853 struct cgraph_node
*callee
=
854 cgraph_function_or_thunk_node (e
->callee
, NULL
);
855 struct inline_summary
*info
= inline_summary (callee
);
856 vec
<tree
> known_vals
= vNULL
;
857 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
860 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
862 known_vals_ptr
->create (0);
863 if (known_binfos_ptr
)
864 known_binfos_ptr
->create (0);
866 if (ipa_node_params_vector
.exists ()
867 && !e
->call_stmt_cannot_inline_p
868 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_binfos_ptr
))
870 struct ipa_node_params
*parms_info
;
871 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
872 struct inline_edge_summary
*es
= inline_edge_summary (e
);
873 int i
, count
= ipa_get_cs_argument_count (args
);
875 if (e
->caller
->global
.inlined_to
)
876 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
878 parms_info
= IPA_NODE_REF (e
->caller
);
880 if (count
&& (info
->conds
|| known_vals_ptr
))
881 known_vals
.safe_grow_cleared (count
);
882 if (count
&& (info
->conds
|| known_aggs_ptr
))
883 known_aggs
.safe_grow_cleared (count
);
884 if (count
&& known_binfos_ptr
)
885 known_binfos_ptr
->safe_grow_cleared (count
);
887 for (i
= 0; i
< count
; i
++)
889 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
890 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
893 if (known_vals
.exists () && TREE_CODE (cst
) != TREE_BINFO
)
895 else if (known_binfos_ptr
!= NULL
896 && TREE_CODE (cst
) == TREE_BINFO
)
897 (*known_binfos_ptr
)[i
] = cst
;
899 else if (inline_p
&& !es
->param
[i
].change_prob
)
900 known_vals
[i
] = error_mark_node
;
901 /* TODO: When IPA-CP starts propagating and merging aggregate jump
902 functions, use its knowledge of the caller too, just like the
903 scalar case above. */
904 known_aggs
[i
] = &jf
->agg
;
909 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
910 known_vals
, known_aggs
);
913 *known_vals_ptr
= known_vals
;
915 known_vals
.release ();
918 *known_aggs_ptr
= known_aggs
;
920 known_aggs
.release ();
924 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
927 inline_summary_alloc (void)
929 if (!node_removal_hook_holder
)
930 node_removal_hook_holder
=
931 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
932 if (!edge_removal_hook_holder
)
933 edge_removal_hook_holder
=
934 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
935 if (!node_duplication_hook_holder
)
936 node_duplication_hook_holder
=
937 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
938 if (!edge_duplication_hook_holder
)
939 edge_duplication_hook_holder
=
940 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
942 if (vec_safe_length (inline_summary_vec
) <= (unsigned) cgraph_max_uid
)
943 vec_safe_grow_cleared (inline_summary_vec
, cgraph_max_uid
+ 1);
944 if (inline_edge_summary_vec
.length () <= (unsigned) cgraph_edge_max_uid
)
945 inline_edge_summary_vec
.safe_grow_cleared (cgraph_edge_max_uid
+ 1);
946 if (!edge_predicate_pool
)
947 edge_predicate_pool
= create_alloc_pool ("edge predicates",
948 sizeof (struct predicate
), 10);
951 /* We are called multiple time for given function; clear
952 data from previous run so they are not cumulated. */
955 reset_inline_edge_summary (struct cgraph_edge
*e
)
957 if (e
->uid
< (int) inline_edge_summary_vec
.length ())
959 struct inline_edge_summary
*es
= inline_edge_summary (e
);
961 es
->call_stmt_size
= es
->call_stmt_time
= 0;
963 pool_free (edge_predicate_pool
, es
->predicate
);
964 es
->predicate
= NULL
;
965 es
->param
.release ();
969 /* We are called multiple time for given function; clear
970 data from previous run so they are not cumulated. */
973 reset_inline_summary (struct cgraph_node
*node
)
975 struct inline_summary
*info
= inline_summary (node
);
976 struct cgraph_edge
*e
;
978 info
->self_size
= info
->self_time
= 0;
979 info
->estimated_stack_size
= 0;
980 info
->estimated_self_stack_size
= 0;
981 info
->stack_frame_offset
= 0;
986 if (info
->loop_iterations
)
988 pool_free (edge_predicate_pool
, info
->loop_iterations
);
989 info
->loop_iterations
= NULL
;
991 if (info
->loop_stride
)
993 pool_free (edge_predicate_pool
, info
->loop_stride
);
994 info
->loop_stride
= NULL
;
996 if (info
->array_index
)
998 pool_free (edge_predicate_pool
, info
->array_index
);
999 info
->array_index
= NULL
;
1001 vec_free (info
->conds
);
1002 vec_free (info
->entry
);
1003 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1004 reset_inline_edge_summary (e
);
1005 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1006 reset_inline_edge_summary (e
);
1009 /* Hook that is called by cgraph.c when a node is removed. */
1012 inline_node_removal_hook (struct cgraph_node
*node
,
1013 void *data ATTRIBUTE_UNUSED
)
1015 struct inline_summary
*info
;
1016 if (vec_safe_length (inline_summary_vec
) <= (unsigned) node
->uid
)
1018 info
= inline_summary (node
);
1019 reset_inline_summary (node
);
1020 memset (info
, 0, sizeof (inline_summary_t
));
1023 /* Remap predicate P of former function to be predicate of duplicated functoin.
1024 POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
1025 INFO is inline summary of the duplicated node. */
1027 static struct predicate
1028 remap_predicate_after_duplication (struct predicate
*p
,
1029 clause_t possible_truths
,
1030 struct inline_summary
*info
)
1032 struct predicate new_predicate
= true_predicate ();
1034 for (j
= 0; p
->clause
[j
]; j
++)
1035 if (!(possible_truths
& p
->clause
[j
]))
1037 new_predicate
= false_predicate ();
1041 add_clause (info
->conds
, &new_predicate
,
1042 possible_truths
& p
->clause
[j
]);
1043 return new_predicate
;
1046 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
1047 Additionally care about allocating new memory slot for updated predicate
1048 and set it to NULL when it becomes true or false (and thus uninteresting).
1052 remap_hint_predicate_after_duplication (struct predicate
**p
,
1053 clause_t possible_truths
,
1054 struct inline_summary
*info
)
1056 struct predicate new_predicate
;
1061 new_predicate
= remap_predicate_after_duplication (*p
,
1062 possible_truths
, info
);
1063 /* We do not want to free previous predicate; it is used by node origin. */
1065 set_hint_predicate (p
, new_predicate
);
1069 /* Hook that is called by cgraph.c when a node is duplicated. */
1072 inline_node_duplication_hook (struct cgraph_node
*src
,
1073 struct cgraph_node
*dst
,
1074 ATTRIBUTE_UNUSED
void *data
)
1076 struct inline_summary
*info
;
1077 inline_summary_alloc ();
1078 info
= inline_summary (dst
);
1079 memcpy (info
, inline_summary (src
), sizeof (struct inline_summary
));
1080 /* TODO: as an optimization, we may avoid copying conditions
1081 that are known to be false or true. */
1082 info
->conds
= vec_safe_copy (info
->conds
);
1084 /* When there are any replacements in the function body, see if we can figure
1085 out that something was optimized out. */
1086 if (ipa_node_params_vector
.exists () && dst
->clone
.tree_map
)
1088 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
1089 /* Use SRC parm info since it may not be copied yet. */
1090 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1091 vec
<tree
> known_vals
= vNULL
;
1092 int count
= ipa_get_param_count (parms_info
);
1094 clause_t possible_truths
;
1095 struct predicate true_pred
= true_predicate ();
1097 int optimized_out_size
= 0;
1098 bool inlined_to_p
= false;
1099 struct cgraph_edge
*edge
;
1102 known_vals
.safe_grow_cleared (count
);
1103 for (i
= 0; i
< count
; i
++)
1105 tree t
= ipa_get_param (parms_info
, i
);
1106 struct ipa_replace_map
*r
;
1108 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
1110 if (r
->old_tree
== t
&& r
->replace_p
&& !r
->ref_p
)
1112 known_vals
[i
] = r
->new_tree
;
1117 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1120 known_vals
.release ();
1122 account_size_time (info
, 0, 0, &true_pred
);
1124 /* Remap size_time vectors.
1125 Simplify the predicate by prunning out alternatives that are known
1127 TODO: as on optimization, we can also eliminate conditions known
1129 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
1131 struct predicate new_predicate
;
1132 new_predicate
= remap_predicate_after_duplication (&e
->predicate
,
1135 if (false_predicate_p (&new_predicate
))
1136 optimized_out_size
+= e
->size
;
1138 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1141 /* Remap edge predicates with the same simplification as above.
1142 Also copy constantness arrays. */
1143 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1145 struct predicate new_predicate
;
1146 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1148 if (!edge
->inline_failed
)
1149 inlined_to_p
= true;
1152 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1155 if (false_predicate_p (&new_predicate
)
1156 && !false_predicate_p (es
->predicate
))
1158 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1159 edge
->frequency
= 0;
1161 edge_set_predicate (edge
, &new_predicate
);
1164 /* Remap indirect edge predicates with the same simplificaiton as above.
1165 Also copy constantness arrays. */
1166 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1168 struct predicate new_predicate
;
1169 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1171 gcc_checking_assert (edge
->inline_failed
);
1174 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1177 if (false_predicate_p (&new_predicate
)
1178 && !false_predicate_p (es
->predicate
))
1180 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1181 edge
->frequency
= 0;
1183 edge_set_predicate (edge
, &new_predicate
);
1185 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
1186 possible_truths
, info
);
1187 remap_hint_predicate_after_duplication (&info
->loop_stride
,
1188 possible_truths
, info
);
1189 remap_hint_predicate_after_duplication (&info
->array_index
,
1190 possible_truths
, info
);
1192 /* If inliner or someone after inliner will ever start producing
1193 non-trivial clones, we will get trouble with lack of information
1194 about updating self sizes, because size vectors already contains
1195 sizes of the calees. */
1196 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
1200 info
->entry
= vec_safe_copy (info
->entry
);
1201 if (info
->loop_iterations
)
1203 predicate p
= *info
->loop_iterations
;
1204 info
->loop_iterations
= NULL
;
1205 set_hint_predicate (&info
->loop_iterations
, p
);
1207 if (info
->loop_stride
)
1209 predicate p
= *info
->loop_stride
;
1210 info
->loop_stride
= NULL
;
1211 set_hint_predicate (&info
->loop_stride
, p
);
1213 if (info
->array_index
)
1215 predicate p
= *info
->array_index
;
1216 info
->array_index
= NULL
;
1217 set_hint_predicate (&info
->array_index
, p
);
1220 inline_update_overall_summary (dst
);
1224 /* Hook that is called by cgraph.c when a node is duplicated. */
1227 inline_edge_duplication_hook (struct cgraph_edge
*src
,
1228 struct cgraph_edge
*dst
,
1229 ATTRIBUTE_UNUSED
void *data
)
1231 struct inline_edge_summary
*info
;
1232 struct inline_edge_summary
*srcinfo
;
1233 inline_summary_alloc ();
1234 info
= inline_edge_summary (dst
);
1235 srcinfo
= inline_edge_summary (src
);
1236 memcpy (info
, srcinfo
, sizeof (struct inline_edge_summary
));
1237 info
->predicate
= NULL
;
1238 edge_set_predicate (dst
, srcinfo
->predicate
);
1239 info
->param
= srcinfo
->param
.copy ();
1243 /* Keep edge cache consistent across edge removal. */
1246 inline_edge_removal_hook (struct cgraph_edge
*edge
,
1247 void *data ATTRIBUTE_UNUSED
)
1249 if (edge_growth_cache
.exists ())
1250 reset_edge_growth_cache (edge
);
1251 reset_inline_edge_summary (edge
);
1255 /* Initialize growth caches. */
1258 initialize_growth_caches (void)
1260 if (cgraph_edge_max_uid
)
1261 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
1263 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
1267 /* Free growth caches. */
1270 free_growth_caches (void)
1272 edge_growth_cache
.release ();
1273 node_growth_cache
.release ();
1277 /* Dump edge summaries associated to NODE and recursively to all clones.
1278 Indent by INDENT. */
1281 dump_inline_edge_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
1282 struct inline_summary
*info
)
1284 struct cgraph_edge
*edge
;
1285 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1287 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1288 struct cgraph_node
*callee
=
1289 cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1293 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
1294 " time: %2i callee size:%2i stack:%2i",
1295 indent
, "", cgraph_node_name (callee
), callee
->uid
,
1296 !edge
->inline_failed
1297 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
1298 indent
, "", es
->loop_depth
, edge
->frequency
,
1299 es
->call_stmt_size
, es
->call_stmt_time
,
1300 (int) inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1301 (int) inline_summary (callee
)->estimated_stack_size
);
1305 fprintf (f
, " predicate: ");
1306 dump_predicate (f
, info
->conds
, es
->predicate
);
1310 if (es
->param
.exists ())
1311 for (i
= 0; i
< (int) es
->param
.length (); i
++)
1313 int prob
= es
->param
[i
].change_prob
;
1316 fprintf (f
, "%*s op%i is compile time invariant\n",
1318 else if (prob
!= REG_BR_PROB_BASE
)
1319 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1320 prob
* 100.0 / REG_BR_PROB_BASE
);
1322 if (!edge
->inline_failed
)
1324 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1325 " callee size %i\n",
1327 (int) inline_summary (callee
)->stack_frame_offset
,
1328 (int) inline_summary (callee
)->estimated_self_stack_size
,
1329 (int) inline_summary (callee
)->estimated_stack_size
);
1330 dump_inline_edge_summary (f
, indent
+ 2, callee
, info
);
1333 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1335 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1336 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1340 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
1343 fprintf (f
, "predicate: ");
1344 dump_predicate (f
, info
->conds
, es
->predicate
);
1353 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
1357 struct inline_summary
*s
= inline_summary (node
);
1360 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
1362 if (DECL_DISREGARD_INLINE_LIMITS (node
->symbol
.decl
))
1363 fprintf (f
, " always_inline");
1365 fprintf (f
, " inlinable");
1366 fprintf (f
, "\n self time: %i\n", s
->self_time
);
1367 fprintf (f
, " global time: %i\n", s
->time
);
1368 fprintf (f
, " self size: %i\n", s
->self_size
);
1369 fprintf (f
, " global size: %i\n", s
->size
);
1370 fprintf (f
, " self stack: %i\n",
1371 (int) s
->estimated_self_stack_size
);
1372 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
1374 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
1376 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
1377 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
1379 fprintf (f
, " size:%f, time:%f, predicate:",
1380 (double) e
->size
/ INLINE_SIZE_SCALE
,
1381 (double) e
->time
/ INLINE_TIME_SCALE
);
1382 dump_predicate (f
, s
->conds
, &e
->predicate
);
1384 if (s
->loop_iterations
)
1386 fprintf (f
, " loop iterations:");
1387 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1391 fprintf (f
, " loop stride:");
1392 dump_predicate (f
, s
->conds
, s
->loop_stride
);
1396 fprintf (f
, " array index:");
1397 dump_predicate (f
, s
->conds
, s
->array_index
);
1399 fprintf (f
, " calls:\n");
1400 dump_inline_edge_summary (f
, 4, node
, s
);
1406 debug_inline_summary (struct cgraph_node
*node
)
1408 dump_inline_summary (stderr
, node
);
1412 dump_inline_summaries (FILE *f
)
1414 struct cgraph_node
*node
;
1416 FOR_EACH_DEFINED_FUNCTION (node
)
1417 if (!node
->global
.inlined_to
)
1418 dump_inline_summary (f
, node
);
1421 /* Give initial reasons why inlining would fail on EDGE. This gets either
1422 nullified or usually overwritten by more precise reasons later. */
1425 initialize_inline_failed (struct cgraph_edge
*e
)
1427 struct cgraph_node
*callee
= e
->callee
;
1429 if (e
->indirect_unknown_callee
)
1430 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1431 else if (!callee
->analyzed
)
1432 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1433 else if (callee
->local
.redefined_extern_inline
)
1434 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1435 else if (e
->call_stmt_cannot_inline_p
)
1436 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1438 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1441 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1442 boolean variable pointed to by DATA. */
1445 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1448 bool *b
= (bool *) data
;
1453 /* If OP refers to value of function parameter, return the corresponding
1457 unmodified_parm_1 (gimple stmt
, tree op
)
1459 /* SSA_NAME referring to parm default def? */
1460 if (TREE_CODE (op
) == SSA_NAME
1461 && SSA_NAME_IS_DEFAULT_DEF (op
)
1462 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1463 return SSA_NAME_VAR (op
);
1464 /* Non-SSA parm reference? */
1465 if (TREE_CODE (op
) == PARM_DECL
)
1467 bool modified
= false;
1470 ao_ref_init (&refd
, op
);
1471 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1479 /* If OP refers to value of function parameter, return the corresponding
1480 parameter. Also traverse chains of SSA register assignments. */
1483 unmodified_parm (gimple stmt
, tree op
)
1485 tree res
= unmodified_parm_1 (stmt
, op
);
1489 if (TREE_CODE (op
) == SSA_NAME
1490 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1491 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1492 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1493 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1497 /* If OP refers to a value of a function parameter or value loaded from an
1498 aggregate passed to a parameter (either by value or reference), return TRUE
1499 and store the number of the parameter to *INDEX_P and information whether
1500 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1501 the function parameters, STMT is the statement in which OP is used or
1505 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1506 gimple stmt
, tree op
, int *index_p
,
1507 struct agg_position_info
*aggpos
)
1509 tree res
= unmodified_parm_1 (stmt
, op
);
1511 gcc_checking_assert (aggpos
);
1514 *index_p
= ipa_get_param_decl_index (info
, res
);
1517 aggpos
->agg_contents
= false;
1518 aggpos
->by_ref
= false;
1522 if (TREE_CODE (op
) == SSA_NAME
)
1524 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1525 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1527 stmt
= SSA_NAME_DEF_STMT (op
);
1528 op
= gimple_assign_rhs1 (stmt
);
1529 if (!REFERENCE_CLASS_P (op
))
1530 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1534 aggpos
->agg_contents
= true;
1535 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1539 /* See if statement might disappear after inlining.
1540 0 - means not eliminated
1541 1 - half of statements goes away
1542 2 - for sure it is eliminated.
1543 We are not terribly sophisticated, basically looking for simple abstraction
1544 penalty wrappers. */
1547 eliminated_by_inlining_prob (gimple stmt
)
1549 enum gimple_code code
= gimple_code (stmt
);
1550 enum tree_code rhs_code
;
1560 if (gimple_num_ops (stmt
) != 2)
1563 rhs_code
= gimple_assign_rhs_code (stmt
);
1565 /* Casts of parameters, loads from parameters passed by reference
1566 and stores to return value or parameters are often free after
1567 inlining dua to SRA and further combining.
1568 Assume that half of statements goes away. */
1569 if (rhs_code
== CONVERT_EXPR
1570 || rhs_code
== NOP_EXPR
1571 || rhs_code
== VIEW_CONVERT_EXPR
1572 || rhs_code
== ADDR_EXPR
1573 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1575 tree rhs
= gimple_assign_rhs1 (stmt
);
1576 tree lhs
= gimple_assign_lhs (stmt
);
1577 tree inner_rhs
= get_base_address (rhs
);
1578 tree inner_lhs
= get_base_address (lhs
);
1579 bool rhs_free
= false;
1580 bool lhs_free
= false;
1587 /* Reads of parameter are expected to be free. */
1588 if (unmodified_parm (stmt
, inner_rhs
))
1590 /* Match expressions of form &this->field. Those will most likely
1591 combine with something upstream after inlining. */
1592 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1594 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1595 if (TREE_CODE (op
) == PARM_DECL
)
1597 else if (TREE_CODE (op
) == MEM_REF
1598 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0)))
1602 /* When parameter is not SSA register because its address is taken
1603 and it is just copied into one, the statement will be completely
1604 free after inlining (we will copy propagate backward). */
1605 if (rhs_free
&& is_gimple_reg (lhs
))
1608 /* Reads of parameters passed by reference
1609 expected to be free (i.e. optimized out after inlining). */
1610 if (TREE_CODE (inner_rhs
) == MEM_REF
1611 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1614 /* Copying parameter passed by reference into gimple register is
1615 probably also going to copy propagate, but we can't be quite
1617 if (rhs_free
&& is_gimple_reg (lhs
))
1620 /* Writes to parameters, parameters passed by value and return value
1621 (either dirrectly or passed via invisible reference) are free.
1623 TODO: We ought to handle testcase like
1624 struct a {int a,b;};
1626 retrurnsturct (void)
1632 This translate into:
1647 For that we either need to copy ipa-split logic detecting writes
1649 if (TREE_CODE (inner_lhs
) == PARM_DECL
1650 || TREE_CODE (inner_lhs
) == RESULT_DECL
1651 || (TREE_CODE (inner_lhs
) == MEM_REF
1652 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1653 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1654 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1655 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1657 0))) == RESULT_DECL
))))
1660 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1662 if (lhs_free
&& rhs_free
)
1672 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1673 predicates to the CFG edges. */
1676 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1677 struct inline_summary
*summary
,
1683 struct agg_position_info aggpos
;
1684 enum tree_code code
, inverted_code
;
1690 last
= last_stmt (bb
);
1691 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1693 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1695 op
= gimple_cond_lhs (last
);
1696 /* TODO: handle conditionals like
1699 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1701 code
= gimple_cond_code (last
);
1703 = invert_tree_comparison (code
,
1704 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1706 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1708 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1709 e
->flags
& EDGE_TRUE_VALUE
1710 ? code
: inverted_code
,
1711 gimple_cond_rhs (last
));
1712 e
->aux
= pool_alloc (edge_predicate_pool
);
1713 *(struct predicate
*) e
->aux
= p
;
1717 if (TREE_CODE (op
) != SSA_NAME
)
1720 if (builtin_constant_p (op))
1724 Here we can predicate nonconstant_code. We can't
1725 really handle constant_code since we have no predicate
1726 for this and also the constant code is not known to be
1727 optimized away when inliner doen't see operand is constant.
1728 Other optimizers might think otherwise. */
1729 if (gimple_cond_code (last
) != NE_EXPR
1730 || !integer_zerop (gimple_cond_rhs (last
)))
1732 set_stmt
= SSA_NAME_DEF_STMT (op
);
1733 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1734 || gimple_call_num_args (set_stmt
) != 1)
1736 op2
= gimple_call_arg (set_stmt
, 0);
1737 if (!unmodified_parm_or_parm_agg_item
1738 (info
, set_stmt
, op2
, &index
, &aggpos
))
1740 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1742 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1743 IS_NOT_CONSTANT
, NULL_TREE
);
1744 e
->aux
= pool_alloc (edge_predicate_pool
);
1745 *(struct predicate
*) e
->aux
= p
;
1750 /* If BB ends by a switch we can turn into predicates, attach corresponding
1751 predicates to the CFG edges. */
1754 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1755 struct inline_summary
*summary
,
1761 struct agg_position_info aggpos
;
1767 last
= last_stmt (bb
);
1768 if (!last
|| gimple_code (last
) != GIMPLE_SWITCH
)
1770 op
= gimple_switch_index (last
);
1771 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1774 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1776 e
->aux
= pool_alloc (edge_predicate_pool
);
1777 *(struct predicate
*) e
->aux
= false_predicate ();
1779 n
= gimple_switch_num_labels (last
);
1780 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1782 tree cl
= gimple_switch_label (last
, case_idx
);
1786 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1787 min
= CASE_LOW (cl
);
1788 max
= CASE_HIGH (cl
);
1790 /* For default we might want to construct predicate that none
1791 of cases is met, but it is bit hard to do not having negations
1792 of conditionals handy. */
1794 p
= true_predicate ();
1796 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1799 struct predicate p1
, p2
;
1800 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1801 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1802 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1804 *(struct predicate
*) e
->aux
1805 = or_predicates (summary
->conds
, &p
, (struct predicate
*) e
->aux
);
1810 /* For each BB in NODE attach to its AUX pointer predicate under
1811 which it is executable. */
1814 compute_bb_predicates (struct cgraph_node
*node
,
1815 struct ipa_node_params
*parms_info
,
1816 struct inline_summary
*summary
)
1818 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
1822 FOR_EACH_BB_FN (bb
, my_function
)
1824 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1825 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1828 /* Entry block is always executable. */
1829 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1830 = pool_alloc (edge_predicate_pool
);
1831 *(struct predicate
*) ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1832 = true_predicate ();
1834 /* A simple dataflow propagation of predicates forward in the CFG.
1835 TODO: work in reverse postorder. */
1839 FOR_EACH_BB_FN (bb
, my_function
)
1841 struct predicate p
= false_predicate ();
1844 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1848 struct predicate this_bb_predicate
1849 = *(struct predicate
*) e
->src
->aux
;
1852 = and_predicates (summary
->conds
, &this_bb_predicate
,
1853 (struct predicate
*) e
->aux
);
1854 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1855 if (true_predicate_p (&p
))
1859 if (false_predicate_p (&p
))
1860 gcc_assert (!bb
->aux
);
1866 bb
->aux
= pool_alloc (edge_predicate_pool
);
1867 *((struct predicate
*) bb
->aux
) = p
;
1869 else if (!predicates_equal_p (&p
, (struct predicate
*) bb
->aux
))
1872 *((struct predicate
*) bb
->aux
) = p
;
1880 /* We keep info about constantness of SSA names. */
1882 typedef struct predicate predicate_t
;
1883 /* Return predicate specifying when the STMT might have result that is not
1884 a compile time constant. */
1886 static struct predicate
1887 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1888 struct inline_summary
*summary
,
1890 vec
<predicate_t
> nonconstant_names
)
1895 while (UNARY_CLASS_P (expr
))
1896 expr
= TREE_OPERAND (expr
, 0);
1898 parm
= unmodified_parm (NULL
, expr
);
1899 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1900 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1901 if (is_gimple_min_invariant (expr
))
1902 return false_predicate ();
1903 if (TREE_CODE (expr
) == SSA_NAME
)
1904 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1905 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1907 struct predicate p1
= will_be_nonconstant_expr_predicate
1908 (info
, summary
, TREE_OPERAND (expr
, 0),
1910 struct predicate p2
;
1911 if (true_predicate_p (&p1
))
1913 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1914 TREE_OPERAND (expr
, 1),
1916 return or_predicates (summary
->conds
, &p1
, &p2
);
1918 else if (TREE_CODE (expr
) == COND_EXPR
)
1920 struct predicate p1
= will_be_nonconstant_expr_predicate
1921 (info
, summary
, TREE_OPERAND (expr
, 0),
1923 struct predicate p2
;
1924 if (true_predicate_p (&p1
))
1926 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1927 TREE_OPERAND (expr
, 1),
1929 if (true_predicate_p (&p2
))
1931 p1
= or_predicates (summary
->conds
, &p1
, &p2
);
1932 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1933 TREE_OPERAND (expr
, 2),
1935 return or_predicates (summary
->conds
, &p1
, &p2
);
1942 return false_predicate ();
1946 /* Return predicate specifying when the STMT might have result that is not
1947 a compile time constant. */
1949 static struct predicate
1950 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1951 struct inline_summary
*summary
,
1953 vec
<predicate_t
> nonconstant_names
)
1955 struct predicate p
= true_predicate ();
1958 struct predicate op_non_const
;
1961 struct agg_position_info aggpos
;
1963 /* What statments might be optimized away
1964 when their arguments are constant
1965 TODO: also trivial builtins.
1966 builtin_constant_p is already handled later. */
1967 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1968 && gimple_code (stmt
) != GIMPLE_COND
1969 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1972 /* Stores will stay anyway. */
1973 if (gimple_store_p (stmt
))
1976 is_load
= gimple_assign_load_p (stmt
);
1978 /* Loads can be optimized when the value is known. */
1982 gcc_assert (gimple_assign_single_p (stmt
));
1983 op
= gimple_assign_rhs1 (stmt
);
1984 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
1991 /* See if we understand all operands before we start
1992 adding conditionals. */
1993 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1995 tree parm
= unmodified_parm (stmt
, use
);
1996 /* For arguments we can build a condition. */
1997 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
1999 if (TREE_CODE (use
) != SSA_NAME
)
2001 /* If we know when operand is constant,
2002 we still can say something useful. */
2003 if (!true_predicate_p (&nonconstant_names
[SSA_NAME_VERSION (use
)]))
2010 add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
2012 op_non_const
= false_predicate ();
2013 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2015 tree parm
= unmodified_parm (stmt
, use
);
2018 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
2020 if (index
!= base_index
)
2021 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
2026 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2027 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
2029 if (gimple_code (stmt
) == GIMPLE_ASSIGN
2030 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
2031 nonconstant_names
[SSA_NAME_VERSION (gimple_assign_lhs (stmt
))]
2033 return op_non_const
;
2036 struct record_modified_bb_info
2042 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2043 set except for info->stmt. */
2046 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
2048 struct record_modified_bb_info
*info
=
2049 (struct record_modified_bb_info
*) data
;
2050 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2052 bitmap_set_bit (info
->bb_set
,
2053 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2054 ? ENTRY_BLOCK_PTR
->index
2055 : gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
2059 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2060 will change since last invocation of STMT.
2062 Value 0 is reserved for compile time invariants.
2063 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2064 ought to be REG_BR_PROB_BASE / estimated_iters. */
2067 param_change_prob (gimple stmt
, int i
)
2069 tree op
= gimple_call_arg (stmt
, i
);
2070 basic_block bb
= gimple_bb (stmt
);
2073 /* Global invariants neve change. */
2074 if (is_gimple_min_invariant (op
))
2076 /* We would have to do non-trivial analysis to really work out what
2077 is the probability of value to change (i.e. when init statement
2078 is in a sibling loop of the call).
2080 We do an conservative estimate: when call is executed N times more often
2081 than the statement defining value, we take the frequency 1/N. */
2082 if (TREE_CODE (op
) == SSA_NAME
)
2087 return REG_BR_PROB_BASE
;
2089 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2090 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2092 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2096 if (init_freq
< bb
->frequency
)
2097 return MAX ((init_freq
* REG_BR_PROB_BASE
+
2098 bb
->frequency
/ 2) / bb
->frequency
, 1);
2100 return REG_BR_PROB_BASE
;
2103 base
= get_base_address (op
);
2108 struct record_modified_bb_info info
;
2112 if (const_value_known_p (base
))
2115 return REG_BR_PROB_BASE
;
2116 ao_ref_init (&refd
, op
);
2118 info
.bb_set
= BITMAP_ALLOC (NULL
);
2119 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2121 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2123 BITMAP_FREE (info
.bb_set
);
2124 return REG_BR_PROB_BASE
;
2127 /* Assume that every memory is initialized at entry.
2128 TODO: Can we easilly determine if value is always defined
2129 and thus we may skip entry block? */
2130 if (ENTRY_BLOCK_PTR
->frequency
)
2131 max
= ENTRY_BLOCK_PTR
->frequency
;
2135 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2136 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2138 BITMAP_FREE (info
.bb_set
);
2139 if (max
< bb
->frequency
)
2140 return MAX ((max
* REG_BR_PROB_BASE
+
2141 bb
->frequency
/ 2) / bb
->frequency
, 1);
2143 return REG_BR_PROB_BASE
;
2145 return REG_BR_PROB_BASE
;
2148 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2149 sub-graph and if the predicate the condition depends on is known. If so,
2150 return true and store the pointer the predicate in *P. */
2153 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2154 struct inline_summary
*summary
, basic_block bb
,
2155 struct predicate
*p
,
2156 vec
<predicate_t
> nonconstant_names
)
2160 basic_block first_bb
= NULL
;
2163 if (single_pred_p (bb
))
2165 *p
= false_predicate ();
2169 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2171 if (single_succ_p (e
->src
))
2173 if (!single_pred_p (e
->src
))
2176 first_bb
= single_pred (e
->src
);
2177 else if (single_pred (e
->src
) != first_bb
)
2184 else if (e
->src
!= first_bb
)
2192 stmt
= last_stmt (first_bb
);
2194 || gimple_code (stmt
) != GIMPLE_COND
2195 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2198 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2199 gimple_cond_lhs (stmt
),
2201 if (true_predicate_p (p
))
2207 /* Given a PHI statement in a function described by inline properties SUMMARY
2208 and *P being the predicate describing whether the selected PHI argument is
2209 known, store a predicate for the result of the PHI statement into
2210 NONCONSTANT_NAMES, if possible. */
2213 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2214 struct predicate
*p
,
2215 vec
<predicate_t
> nonconstant_names
)
2219 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2221 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2222 if (!is_gimple_min_invariant (arg
))
2224 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2225 *p
= or_predicates (summary
->conds
, p
,
2226 &nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2227 if (true_predicate_p (p
))
2232 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2234 fprintf (dump_file
, "\t\tphi predicate: ");
2235 dump_predicate (dump_file
, summary
->conds
, p
);
2237 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2240 /* Return predicate specifying when array index in access OP becomes non-constant. */
2242 static struct predicate
2243 array_index_predicate (struct inline_summary
*info
,
2244 vec
< predicate_t
> nonconstant_names
, tree op
)
2246 struct predicate p
= false_predicate ();
2247 while (handled_component_p (op
))
2249 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
2251 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2252 p
= or_predicates (info
->conds
, &p
,
2253 &nonconstant_names
[SSA_NAME_VERSION
2254 (TREE_OPERAND (op
, 1))]);
2256 op
= TREE_OPERAND (op
, 0);
2261 /* Compute function body size parameters for NODE.
2262 When EARLY is true, we compute only simple summaries without
2263 non-trivial predicates to drive the early inliner. */
2266 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2269 /* Estimate static overhead for function prologue/epilogue and alignment. */
2271 /* Benefits are scaled by probability of elimination that is in range
2274 gimple_stmt_iterator bsi
;
2275 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
2277 struct inline_summary
*info
= inline_summary (node
);
2278 struct predicate bb_predicate
;
2279 struct ipa_node_params
*parms_info
= NULL
;
2280 vec
<predicate_t
> nonconstant_names
= vNULL
;
2283 predicate array_index
= true_predicate ();
2288 if (optimize
&& !early
)
2290 calculate_dominance_info (CDI_DOMINATORS
);
2291 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2293 if (ipa_node_params_vector
.exists ())
2295 parms_info
= IPA_NODE_REF (node
);
2296 nonconstant_names
.safe_grow_cleared
2297 (SSANAMES (my_function
)->length ());
2302 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2303 cgraph_node_name (node
));
2305 /* When we run into maximal number of entries, we assign everything to the
2306 constant truth case. Be sure to have it in list. */
2307 bb_predicate
= true_predicate ();
2308 account_size_time (info
, 0, 0, &bb_predicate
);
2310 bb_predicate
= not_inlined_predicate ();
2311 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2313 gcc_assert (my_function
&& my_function
->cfg
);
2315 compute_bb_predicates (node
, parms_info
, info
);
2316 gcc_assert (cfun
== my_function
);
2317 order
= XNEWVEC (int, n_basic_blocks
);
2318 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2319 for (n
= 0; n
< nblocks
; n
++)
2321 bb
= BASIC_BLOCK (order
[n
]);
2322 freq
= compute_call_stmt_bb_frequency (node
->symbol
.decl
, bb
);
2324 /* TODO: Obviously predicates can be propagated down across CFG. */
2328 bb_predicate
= *(struct predicate
*) bb
->aux
;
2330 bb_predicate
= false_predicate ();
2333 bb_predicate
= true_predicate ();
2335 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2337 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2338 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2341 if (parms_info
&& nonconstant_names
.exists ())
2343 struct predicate phi_predicate
;
2344 bool first_phi
= true;
2346 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2349 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2354 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2356 fprintf (dump_file
, " ");
2357 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2359 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2364 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2366 gimple stmt
= gsi_stmt (bsi
);
2367 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2368 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2370 struct predicate will_be_nonconstant
;
2372 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2374 fprintf (dump_file
, " ");
2375 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2376 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2377 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2381 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2383 struct predicate this_array_index
;
2385 array_index_predicate (info
, nonconstant_names
,
2386 gimple_assign_rhs1 (stmt
));
2387 if (!false_predicate_p (&this_array_index
))
2389 and_predicates (info
->conds
, &array_index
,
2392 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2394 struct predicate this_array_index
;
2396 array_index_predicate (info
, nonconstant_names
,
2397 gimple_get_lhs (stmt
));
2398 if (!false_predicate_p (&this_array_index
))
2400 and_predicates (info
->conds
, &array_index
,
2405 if (is_gimple_call (stmt
))
2407 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2408 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2410 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2411 resolved as constant. We however don't want to optimize
2412 out the cgraph edges. */
2413 if (nonconstant_names
.exists ()
2414 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2415 && gimple_call_lhs (stmt
)
2416 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2418 struct predicate false_p
= false_predicate ();
2419 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2422 if (ipa_node_params_vector
.exists ())
2424 int count
= gimple_call_num_args (stmt
);
2428 es
->param
.safe_grow_cleared (count
);
2429 for (i
= 0; i
< count
; i
++)
2431 int prob
= param_change_prob (stmt
, i
);
2432 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2433 es
->param
[i
].change_prob
= prob
;
2437 es
->call_stmt_size
= this_size
;
2438 es
->call_stmt_time
= this_time
;
2439 es
->loop_depth
= bb_loop_depth (bb
);
2440 edge_set_predicate (edge
, &bb_predicate
);
2443 /* TODO: When conditional jump or swithc is known to be constant, but
2444 we did not translate it into the predicates, we really can account
2445 just maximum of the possible paths. */
2448 = will_be_nonconstant_predicate (parms_info
, info
,
2449 stmt
, nonconstant_names
);
2450 if (this_time
|| this_size
)
2456 prob
= eliminated_by_inlining_prob (stmt
);
2457 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2459 "\t\t50%% will be eliminated by inlining\n");
2460 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2461 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2464 p
= and_predicates (info
->conds
, &bb_predicate
,
2465 &will_be_nonconstant
);
2467 p
= true_predicate ();
2469 if (!false_predicate_p (&p
))
2473 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2474 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2477 /* We account everything but the calls. Calls have their own
2478 size/time info attached to cgraph edges. This is necessary
2479 in order to make the cost disappear after inlining. */
2480 if (!is_gimple_call (stmt
))
2484 struct predicate ip
= not_inlined_predicate ();
2485 ip
= and_predicates (info
->conds
, &ip
, &p
);
2486 account_size_time (info
, this_size
* prob
,
2487 this_time
* prob
, &ip
);
2490 account_size_time (info
, this_size
* (2 - prob
),
2491 this_time
* (2 - prob
), &p
);
2494 gcc_assert (time
>= 0);
2495 gcc_assert (size
>= 0);
2499 set_hint_predicate (&inline_summary (node
)->array_index
, array_index
);
2500 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2501 if (time
> MAX_TIME
)
2505 if (!early
&& nonconstant_names
.exists ())
2509 predicate loop_iterations
= true_predicate ();
2510 predicate loop_stride
= true_predicate ();
2512 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2513 flow_loops_dump (dump_file
, NULL
, 0);
2515 FOR_EACH_LOOP (li
, loop
, 0)
2520 struct tree_niter_desc niter_desc
;
2521 basic_block
*body
= get_loop_body (loop
);
2522 bb_predicate
= *(struct predicate
*) loop
->header
->aux
;
2524 exits
= get_loop_exit_edges (loop
);
2525 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2526 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2527 && !is_gimple_min_invariant (niter_desc
.niter
))
2529 predicate will_be_nonconstant
2530 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2533 if (!true_predicate_p (&will_be_nonconstant
))
2534 will_be_nonconstant
= and_predicates (info
->conds
,
2536 &will_be_nonconstant
);
2537 if (!true_predicate_p (&will_be_nonconstant
)
2538 && !false_predicate_p (&will_be_nonconstant
))
2539 /* This is slightly inprecise. We may want to represent each
2540 loop with independent predicate. */
2542 and_predicates (info
->conds
, &loop_iterations
,
2543 &will_be_nonconstant
);
2547 for (i
= 0; i
< loop
->num_nodes
; i
++)
2549 gimple_stmt_iterator gsi
;
2550 bb_predicate
= *(struct predicate
*) body
[i
]->aux
;
2551 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2554 gimple stmt
= gsi_stmt (gsi
);
2559 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2561 predicate will_be_nonconstant
;
2564 (loop
, loop_containing_stmt (stmt
), use
, &iv
, true)
2565 || is_gimple_min_invariant (iv
.step
))
2568 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2571 if (!true_predicate_p (&will_be_nonconstant
))
2573 = and_predicates (info
->conds
,
2575 &will_be_nonconstant
);
2576 if (!true_predicate_p (&will_be_nonconstant
)
2577 && !false_predicate_p (&will_be_nonconstant
))
2578 /* This is slightly inprecise. We may want to represent
2579 each loop with independent predicate. */
2581 and_predicates (info
->conds
, &loop_stride
,
2582 &will_be_nonconstant
);
2588 set_hint_predicate (&inline_summary (node
)->loop_iterations
,
2590 set_hint_predicate (&inline_summary (node
)->loop_stride
, loop_stride
);
2593 FOR_ALL_BB_FN (bb
, my_function
)
2599 pool_free (edge_predicate_pool
, bb
->aux
);
2601 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2604 pool_free (edge_predicate_pool
, e
->aux
);
2608 inline_summary (node
)->self_time
= time
;
2609 inline_summary (node
)->self_size
= size
;
2610 nonconstant_names
.release ();
2611 if (optimize
&& !early
)
2613 loop_optimizer_finalize ();
2614 free_dominance_info (CDI_DOMINATORS
);
2618 fprintf (dump_file
, "\n");
2619 dump_inline_summary (dump_file
, node
);
2624 /* Compute parameters of functions used by inliner.
2625 EARLY is true when we compute parameters for the early inliner */
2628 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2630 HOST_WIDE_INT self_stack_size
;
2631 struct cgraph_edge
*e
;
2632 struct inline_summary
*info
;
2634 gcc_assert (!node
->global
.inlined_to
);
2636 inline_summary_alloc ();
2638 info
= inline_summary (node
);
2639 reset_inline_summary (node
);
2641 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2642 Once this happen, we will need to more curefully predict call
2644 if (node
->thunk
.thunk_p
)
2646 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2647 struct predicate t
= true_predicate ();
2649 info
->inlinable
= 0;
2650 node
->callees
->call_stmt_cannot_inline_p
= true;
2651 node
->local
.can_change_signature
= false;
2652 es
->call_stmt_time
= 1;
2653 es
->call_stmt_size
= 1;
2654 account_size_time (info
, 0, 0, &t
);
2658 /* Even is_gimple_min_invariant rely on current_function_decl. */
2659 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
2661 /* Estimate the stack size for the function if we're optimizing. */
2662 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2663 info
->estimated_self_stack_size
= self_stack_size
;
2664 info
->estimated_stack_size
= self_stack_size
;
2665 info
->stack_frame_offset
= 0;
2667 /* Can this function be inlined at all? */
2668 info
->inlinable
= tree_inlinable_function_p (node
->symbol
.decl
);
2670 /* Type attributes can use parameter indices to describe them. */
2671 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->symbol
.decl
)))
2672 node
->local
.can_change_signature
= false;
2675 /* Otherwise, inlinable functions always can change signature. */
2676 if (info
->inlinable
)
2677 node
->local
.can_change_signature
= true;
2680 /* Functions calling builtin_apply can not change signature. */
2681 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2683 tree
cdecl = e
->callee
->symbol
.decl
;
2684 if (DECL_BUILT_IN (cdecl)
2685 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2686 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2687 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2690 node
->local
.can_change_signature
= !e
;
2693 estimate_function_body_sizes (node
, early
);
2695 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2696 info
->time
= info
->self_time
;
2697 info
->size
= info
->self_size
;
2698 info
->stack_frame_offset
= 0;
2699 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2700 #ifdef ENABLE_CHECKING
2701 inline_update_overall_summary (node
);
2702 gcc_assert (info
->time
== info
->self_time
&& info
->size
== info
->self_size
);
2709 /* Compute parameters of functions used by inliner using
2710 current_function_decl. */
2713 compute_inline_parameters_for_current (void)
2715 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2719 struct gimple_opt_pass pass_inline_parameters
=
2723 "inline_param", /* name */
2724 OPTGROUP_INLINE
, /* optinfo_flags */
2726 compute_inline_parameters_for_current
, /* execute */
2729 0, /* static_pass_number */
2730 TV_INLINE_PARAMETERS
, /* tv_id */
2731 0, /* properties_required */
2732 0, /* properties_provided */
2733 0, /* properties_destroyed */
2734 0, /* todo_flags_start */
2735 0 /* todo_flags_finish */
2740 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2744 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2745 int *size
, int *time
,
2746 vec
<tree
> known_vals
,
2747 vec
<tree
> known_binfos
,
2748 vec
<ipa_agg_jump_function_p
> known_aggs
)
2751 struct cgraph_node
*callee
;
2752 struct inline_summary
*isummary
;
2754 if (!known_vals
.exists () && !known_binfos
.exists ())
2756 if (!flag_indirect_inlining
)
2759 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2764 /* Account for difference in cost between indirect and direct calls. */
2765 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2766 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2767 gcc_checking_assert (*time
>= 0);
2768 gcc_checking_assert (*size
>= 0);
2770 callee
= cgraph_get_node (target
);
2771 if (!callee
|| !callee
->analyzed
)
2773 isummary
= inline_summary (callee
);
2774 return isummary
->inlinable
;
2777 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2780 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2782 vec
<tree
> known_vals
,
2783 vec
<tree
> known_binfos
,
2784 vec
<ipa_agg_jump_function_p
> known_aggs
,
2785 inline_hints
*hints
)
2787 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2788 int call_size
= es
->call_stmt_size
;
2789 int call_time
= es
->call_stmt_time
;
2791 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2792 known_vals
, known_binfos
, known_aggs
)
2793 && hints
&& cgraph_maybe_hot_edge_p (e
))
2794 *hints
|= INLINE_HINT_indirect_call
;
2795 *size
+= call_size
* INLINE_SIZE_SCALE
;
2796 *time
+= call_time
* prob
/ REG_BR_PROB_BASE
2797 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
);
2798 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2799 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2804 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2805 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2809 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2810 inline_hints
*hints
,
2811 clause_t possible_truths
,
2812 vec
<tree
> known_vals
,
2813 vec
<tree
> known_binfos
,
2814 vec
<ipa_agg_jump_function_p
> known_aggs
)
2816 struct cgraph_edge
*e
;
2817 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2819 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2821 || evaluate_predicate (es
->predicate
, possible_truths
))
2823 if (e
->inline_failed
)
2825 /* Predicates of calls shall not use NOT_CHANGED codes,
2826 sowe do not need to compute probabilities. */
2827 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2828 known_vals
, known_binfos
,
2832 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2834 known_vals
, known_binfos
,
2838 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2840 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2842 || evaluate_predicate (es
->predicate
, possible_truths
))
2843 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2844 known_vals
, known_binfos
, known_aggs
,
2850 /* Estimate size and time needed to execute NODE assuming
2851 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2852 about NODE's arguments. */
2855 estimate_node_size_and_time (struct cgraph_node
*node
,
2856 clause_t possible_truths
,
2857 vec
<tree
> known_vals
,
2858 vec
<tree
> known_binfos
,
2859 vec
<ipa_agg_jump_function_p
> known_aggs
,
2860 int *ret_size
, int *ret_time
,
2861 inline_hints
*ret_hints
,
2862 vec
<inline_param_summary_t
>
2863 inline_param_summary
)
2865 struct inline_summary
*info
= inline_summary (node
);
2869 inline_hints hints
= 0;
2872 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2875 fprintf (dump_file
, " Estimating body: %s/%i\n"
2876 " Known to be false: ", cgraph_node_name (node
), node
->uid
);
2878 for (i
= predicate_not_inlined_condition
;
2879 i
< (predicate_first_dynamic_condition
2880 + (int) vec_safe_length (info
->conds
)); i
++)
2881 if (!(possible_truths
& (1 << i
)))
2884 fprintf (dump_file
, ", ");
2886 dump_condition (dump_file
, info
->conds
, i
);
2890 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
2891 if (evaluate_predicate (&e
->predicate
, possible_truths
))
2894 gcc_checking_assert (e
->time
>= 0);
2895 gcc_checking_assert (time
>= 0);
2896 if (!inline_param_summary
.exists ())
2900 int prob
= predicate_probability (info
->conds
,
2903 inline_param_summary
);
2904 gcc_checking_assert (prob
>= 0);
2905 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
2906 time
+= ((gcov_type
) e
->time
* prob
) / REG_BR_PROB_BASE
;
2908 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2909 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2910 gcc_checking_assert (time
>= 0);
2913 gcc_checking_assert (size
>= 0);
2914 gcc_checking_assert (time
>= 0);
2916 if (info
->loop_iterations
2917 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
2918 hints
|= INLINE_HINT_loop_iterations
;
2919 if (info
->loop_stride
2920 && !evaluate_predicate (info
->loop_stride
, possible_truths
))
2921 hints
|= INLINE_HINT_loop_stride
;
2922 if (info
->array_index
2923 && !evaluate_predicate (info
->array_index
, possible_truths
))
2924 hints
|= INLINE_HINT_array_index
;
2926 hints
|= INLINE_HINT_in_scc
;
2927 if (DECL_DECLARED_INLINE_P (node
->symbol
.decl
))
2928 hints
|= INLINE_HINT_declared_inline
;
2930 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
2931 known_vals
, known_binfos
, known_aggs
);
2932 gcc_checking_assert (size
>= 0);
2933 gcc_checking_assert (time
>= 0);
2934 time
= RDIV (time
, INLINE_TIME_SCALE
);
2935 size
= RDIV (size
, INLINE_SIZE_SCALE
);
2937 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2938 fprintf (dump_file
, "\n size:%i time:%i\n", (int) size
, (int) time
);
2949 /* Estimate size and time needed to execute callee of EDGE assuming that
2950 parameters known to be constant at caller of EDGE are propagated.
2951 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
2952 and types for parameters. */
2955 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
2956 vec
<tree
> known_vals
,
2957 vec
<tree
> known_binfos
,
2958 vec
<ipa_agg_jump_function_p
> known_aggs
,
2959 int *ret_size
, int *ret_time
,
2960 inline_hints
*hints
)
2964 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
,
2966 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
,
2967 known_aggs
, ret_size
, ret_time
, hints
, vNULL
);
2970 /* Translate all conditions from callee representation into caller
2971 representation and symbolically evaluate predicate P into new predicate.
2973 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
2974 is summary of function predicate P is from. OPERAND_MAP is array giving
2975 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
2976 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
2977 predicate under which callee is executed. OFFSET_MAP is an array of of
2978 offsets that need to be added to conditions, negative offset means that
2979 conditions relying on values passed by reference have to be discarded
2980 because they might not be preserved (and should be considered offset zero
2981 for other purposes). */
2983 static struct predicate
2984 remap_predicate (struct inline_summary
*info
,
2985 struct inline_summary
*callee_info
,
2986 struct predicate
*p
,
2987 vec
<int> operand_map
,
2988 vec
<int> offset_map
,
2989 clause_t possible_truths
, struct predicate
*toplev_predicate
)
2992 struct predicate out
= true_predicate ();
2994 /* True predicate is easy. */
2995 if (true_predicate_p (p
))
2996 return *toplev_predicate
;
2997 for (i
= 0; p
->clause
[i
]; i
++)
2999 clause_t clause
= p
->clause
[i
];
3001 struct predicate clause_predicate
= false_predicate ();
3003 gcc_assert (i
< MAX_CLAUSES
);
3005 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
3006 /* Do we have condition we can't disprove? */
3007 if (clause
& possible_truths
& (1 << cond
))
3009 struct predicate cond_predicate
;
3010 /* Work out if the condition can translate to predicate in the
3011 inlined function. */
3012 if (cond
>= predicate_first_dynamic_condition
)
3014 struct condition
*c
;
3016 c
= &(*callee_info
->conds
)[cond
3018 predicate_first_dynamic_condition
];
3019 /* See if we can remap condition operand to caller's operand.
3020 Otherwise give up. */
3021 if (!operand_map
.exists ()
3022 || (int) operand_map
.length () <= c
->operand_num
3023 || operand_map
[c
->operand_num
] == -1
3024 /* TODO: For non-aggregate conditions, adding an offset is
3025 basically an arithmetic jump function processing which
3026 we should support in future. */
3027 || ((!c
->agg_contents
|| !c
->by_ref
)
3028 && offset_map
[c
->operand_num
] > 0)
3029 || (c
->agg_contents
&& c
->by_ref
3030 && offset_map
[c
->operand_num
] < 0))
3031 cond_predicate
= true_predicate ();
3034 struct agg_position_info ap
;
3035 HOST_WIDE_INT offset_delta
= offset_map
[c
->operand_num
];
3036 if (offset_delta
< 0)
3038 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
3041 gcc_assert (!c
->agg_contents
3042 || c
->by_ref
|| offset_delta
== 0);
3043 ap
.offset
= c
->offset
+ offset_delta
;
3044 ap
.agg_contents
= c
->agg_contents
;
3045 ap
.by_ref
= c
->by_ref
;
3046 cond_predicate
= add_condition (info
,
3047 operand_map
[c
->operand_num
],
3048 &ap
, c
->code
, c
->val
);
3051 /* Fixed conditions remains same, construct single
3052 condition predicate. */
3055 cond_predicate
.clause
[0] = 1 << cond
;
3056 cond_predicate
.clause
[1] = 0;
3058 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
3061 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
3063 return and_predicates (info
->conds
, &out
, toplev_predicate
);
3067 /* Update summary information of inline clones after inlining.
3068 Compute peak stack usage. */
3071 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3073 struct cgraph_edge
*e
;
3074 struct inline_summary
*callee_info
= inline_summary (node
);
3075 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
3078 callee_info
->stack_frame_offset
3079 = caller_info
->stack_frame_offset
3080 + caller_info
->estimated_self_stack_size
;
3081 peak
= callee_info
->stack_frame_offset
3082 + callee_info
->estimated_self_stack_size
;
3083 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
3084 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
3085 cgraph_propagate_frequency (node
);
3086 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3088 if (!e
->inline_failed
)
3089 inline_update_callee_summaries (e
->callee
, depth
);
3090 inline_edge_summary (e
)->loop_depth
+= depth
;
3092 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3093 inline_edge_summary (e
)->loop_depth
+= depth
;
3096 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3097 When functoin A is inlined in B and A calls C with parameter that
3098 changes with probability PROB1 and C is known to be passthroug
3099 of argument if B that change with probability PROB2, the probability
3100 of change is now PROB1*PROB2. */
3103 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3104 struct cgraph_edge
*edge
)
3106 if (ipa_node_params_vector
.exists ())
3109 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3110 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3111 struct inline_edge_summary
*inlined_es
3112 = inline_edge_summary (inlined_edge
);
3114 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3116 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3117 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3118 && (ipa_get_jf_pass_through_formal_id (jfunc
)
3119 < (int) inlined_es
->param
.length ()))
3121 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
3122 int prob1
= es
->param
[i
].change_prob
;
3123 int prob2
= inlined_es
->param
[jf_formal_id
].change_prob
;
3124 int prob
= ((prob1
* prob2
+ REG_BR_PROB_BASE
/ 2)
3125 / REG_BR_PROB_BASE
);
3127 if (prob1
&& prob2
&& !prob
)
3130 es
->param
[i
].change_prob
= prob
;
3136 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3138 Remap predicates of callees of NODE. Rest of arguments match
3141 Also update change probabilities. */
3144 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3145 struct cgraph_node
*node
,
3146 struct inline_summary
*info
,
3147 struct inline_summary
*callee_info
,
3148 vec
<int> operand_map
,
3149 vec
<int> offset_map
,
3150 clause_t possible_truths
,
3151 struct predicate
*toplev_predicate
)
3153 struct cgraph_edge
*e
;
3154 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3156 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3159 if (e
->inline_failed
)
3161 remap_edge_change_prob (inlined_edge
, e
);
3165 p
= remap_predicate (info
, callee_info
,
3166 es
->predicate
, operand_map
, offset_map
,
3167 possible_truths
, toplev_predicate
);
3168 edge_set_predicate (e
, &p
);
3169 /* TODO: We should remove the edge for code that will be
3170 optimized out, but we need to keep verifiers and tree-inline
3171 happy. Make it cold for now. */
3172 if (false_predicate_p (&p
))
3179 edge_set_predicate (e
, toplev_predicate
);
3182 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3183 operand_map
, offset_map
, possible_truths
,
3186 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3188 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3191 remap_edge_change_prob (inlined_edge
, e
);
3194 p
= remap_predicate (info
, callee_info
,
3195 es
->predicate
, operand_map
, offset_map
,
3196 possible_truths
, toplev_predicate
);
3197 edge_set_predicate (e
, &p
);
3198 /* TODO: We should remove the edge for code that will be optimized
3199 out, but we need to keep verifiers and tree-inline happy.
3200 Make it cold for now. */
3201 if (false_predicate_p (&p
))
3208 edge_set_predicate (e
, toplev_predicate
);
3212 /* Same as remap_predicate, but set result into hint *HINT. */
3215 remap_hint_predicate (struct inline_summary
*info
,
3216 struct inline_summary
*callee_info
,
3217 struct predicate
**hint
,
3218 vec
<int> operand_map
,
3219 vec
<int> offset_map
,
3220 clause_t possible_truths
,
3221 struct predicate
*toplev_predicate
)
3227 p
= remap_predicate (info
, callee_info
,
3229 operand_map
, offset_map
,
3230 possible_truths
, toplev_predicate
);
3231 if (!false_predicate_p (&p
) && !true_predicate_p (&p
))
3234 set_hint_predicate (hint
, p
);
3236 **hint
= and_predicates (info
->conds
, *hint
, &p
);
3240 /* We inlined EDGE. Update summary of the function we inlined into. */
3243 inline_merge_summary (struct cgraph_edge
*edge
)
3245 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3246 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3247 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3248 struct inline_summary
*info
= inline_summary (to
);
3249 clause_t clause
= 0; /* not_inline is known to be false. */
3251 vec
<int> operand_map
= vNULL
;
3252 vec
<int> offset_map
= vNULL
;
3254 struct predicate toplev_predicate
;
3255 struct predicate true_p
= true_predicate ();
3256 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3259 toplev_predicate
= *es
->predicate
;
3261 toplev_predicate
= true_predicate ();
3263 if (ipa_node_params_vector
.exists () && callee_info
->conds
)
3265 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3266 int count
= ipa_get_cs_argument_count (args
);
3269 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3272 operand_map
.safe_grow_cleared (count
);
3273 offset_map
.safe_grow_cleared (count
);
3275 for (i
= 0; i
< count
; i
++)
3277 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3280 /* TODO: handle non-NOPs when merging. */
3281 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3283 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3284 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3285 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3288 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3290 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3291 if (offset
>= 0 && offset
< INT_MAX
)
3293 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3294 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3296 offset_map
[i
] = offset
;
3299 operand_map
[i
] = map
;
3300 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3303 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3305 struct predicate p
= remap_predicate (info
, callee_info
,
3306 &e
->predicate
, operand_map
,
3309 if (!false_predicate_p (&p
))
3311 gcov_type add_time
= ((gcov_type
) e
->time
* edge
->frequency
3312 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3313 int prob
= predicate_probability (callee_info
->conds
,
3316 add_time
= ((gcov_type
) add_time
* prob
) / REG_BR_PROB_BASE
;
3317 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3318 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3319 if (prob
!= REG_BR_PROB_BASE
3320 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3322 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3323 (double) prob
/ REG_BR_PROB_BASE
);
3325 account_size_time (info
, e
->size
, add_time
, &p
);
3328 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3329 offset_map
, clause
, &toplev_predicate
);
3330 remap_hint_predicate (info
, callee_info
,
3331 &callee_info
->loop_iterations
,
3332 operand_map
, offset_map
, clause
, &toplev_predicate
);
3333 remap_hint_predicate (info
, callee_info
,
3334 &callee_info
->loop_stride
,
3335 operand_map
, offset_map
, clause
, &toplev_predicate
);
3336 remap_hint_predicate (info
, callee_info
,
3337 &callee_info
->array_index
,
3338 operand_map
, offset_map
, clause
, &toplev_predicate
);
3340 inline_update_callee_summaries (edge
->callee
,
3341 inline_edge_summary (edge
)->loop_depth
);
3343 /* We do not maintain predicates of inlined edges, free it. */
3344 edge_set_predicate (edge
, &true_p
);
3345 /* Similarly remove param summaries. */
3346 es
->param
.release ();
3347 operand_map
.release ();
3348 offset_map
.release ();
3351 /* For performance reasons inline_merge_summary is not updating overall size
3352 and time. Recompute it. */
3355 inline_update_overall_summary (struct cgraph_node
*node
)
3357 struct inline_summary
*info
= inline_summary (node
);
3363 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3365 info
->size
+= e
->size
, info
->time
+= e
->time
;
3366 if (info
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
3367 info
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
3369 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3370 ~(clause_t
) (1 << predicate_false_condition
),
3371 vNULL
, vNULL
, vNULL
);
3372 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3373 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3376 /* Return hints derrived from EDGE. */
3378 simple_edge_hints (struct cgraph_edge
*edge
)
3381 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3382 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3383 if (inline_summary (to
)->scc_no
3384 && inline_summary (to
)->scc_no
== inline_summary (edge
->callee
)->scc_no
3385 && !cgraph_edge_recursive_p (edge
))
3386 hints
|= INLINE_HINT_same_scc
;
3388 if (to
->symbol
.lto_file_data
&& edge
->callee
->symbol
.lto_file_data
3389 && to
->symbol
.lto_file_data
!= edge
->callee
->symbol
.lto_file_data
)
3390 hints
|= INLINE_HINT_cross_module
;
3395 /* Estimate the time cost for the caller when inlining EDGE.
3396 Only to be called via estimate_edge_time, that handles the
3399 When caching, also update the cache entry. Compute both time and
3400 size, since we always need both metrics eventually. */
3403 do_estimate_edge_time (struct cgraph_edge
*edge
)
3408 struct cgraph_node
*callee
;
3410 vec
<tree
> known_vals
;
3411 vec
<tree
> known_binfos
;
3412 vec
<ipa_agg_jump_function_p
> known_aggs
;
3413 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3415 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3417 gcc_checking_assert (edge
->inline_failed
);
3418 evaluate_properties_for_edge (edge
, true,
3419 &clause
, &known_vals
, &known_binfos
,
3421 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3422 known_aggs
, &size
, &time
, &hints
, es
->param
);
3423 known_vals
.release ();
3424 known_binfos
.release ();
3425 known_aggs
.release ();
3426 gcc_checking_assert (size
>= 0);
3427 gcc_checking_assert (time
>= 0);
3429 /* When caching, update the cache entry. */
3430 if (edge_growth_cache
.exists ())
3432 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3433 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
3434 edge_growth_cache
[edge
->uid
].time
= time
+ (time
>= 0);
3436 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3437 hints
|= simple_edge_hints (edge
);
3438 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3444 /* Return estimated callee growth after inlining EDGE.
3445 Only to be called via estimate_edge_size. */
3448 do_estimate_edge_size (struct cgraph_edge
*edge
)
3451 struct cgraph_node
*callee
;
3453 vec
<tree
> known_vals
;
3454 vec
<tree
> known_binfos
;
3455 vec
<ipa_agg_jump_function_p
> known_aggs
;
3457 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3459 if (edge_growth_cache
.exists ())
3461 do_estimate_edge_time (edge
);
3462 size
= edge_growth_cache
[edge
->uid
].size
;
3463 gcc_checking_assert (size
);
3464 return size
- (size
> 0);
3467 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3469 /* Early inliner runs without caching, go ahead and do the dirty work. */
3470 gcc_checking_assert (edge
->inline_failed
);
3471 evaluate_properties_for_edge (edge
, true,
3472 &clause
, &known_vals
, &known_binfos
,
3474 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3475 known_aggs
, &size
, NULL
, NULL
, vNULL
);
3476 known_vals
.release ();
3477 known_binfos
.release ();
3478 known_aggs
.release ();
3483 /* Estimate the growth of the caller when inlining EDGE.
3484 Only to be called via estimate_edge_size. */
3487 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3490 struct cgraph_node
*callee
;
3492 vec
<tree
> known_vals
;
3493 vec
<tree
> known_binfos
;
3494 vec
<ipa_agg_jump_function_p
> known_aggs
;
3496 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3498 if (edge_growth_cache
.exists ())
3500 do_estimate_edge_time (edge
);
3501 hints
= edge_growth_cache
[edge
->uid
].hints
;
3502 gcc_checking_assert (hints
);
3506 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3508 /* Early inliner runs without caching, go ahead and do the dirty work. */
3509 gcc_checking_assert (edge
->inline_failed
);
3510 evaluate_properties_for_edge (edge
, true,
3511 &clause
, &known_vals
, &known_binfos
,
3513 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3514 known_aggs
, NULL
, NULL
, &hints
, vNULL
);
3515 known_vals
.release ();
3516 known_binfos
.release ();
3517 known_aggs
.release ();
3518 hints
|= simple_edge_hints (edge
);
3523 /* Estimate self time of the function NODE after inlining EDGE. */
3526 estimate_time_after_inlining (struct cgraph_node
*node
,
3527 struct cgraph_edge
*edge
)
3529 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3530 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3533 inline_summary (node
)->time
+ estimate_edge_time (edge
);
3536 if (time
> MAX_TIME
)
3540 return inline_summary (node
)->time
;
3544 /* Estimate the size of NODE after inlining EDGE which should be an
3545 edge to either NODE or a call inlined into NODE. */
3548 estimate_size_after_inlining (struct cgraph_node
*node
,
3549 struct cgraph_edge
*edge
)
3551 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3552 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3554 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3555 gcc_assert (size
>= 0);
3558 return inline_summary (node
)->size
;
3564 bool self_recursive
;
3569 /* Worker for do_estimate_growth. Collect growth for all callers. */
3572 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3574 struct cgraph_edge
*e
;
3575 struct growth_data
*d
= (struct growth_data
*) data
;
3577 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3579 gcc_checking_assert (e
->inline_failed
);
3581 if (e
->caller
== node
3582 || (e
->caller
->global
.inlined_to
3583 && e
->caller
->global
.inlined_to
== node
))
3584 d
->self_recursive
= true;
3585 d
->growth
+= estimate_edge_growth (e
);
3591 /* Estimate the growth caused by inlining NODE into all callees. */
3594 do_estimate_growth (struct cgraph_node
*node
)
3596 struct growth_data d
= { 0, false };
3597 struct inline_summary
*info
= inline_summary (node
);
3599 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3601 /* For self recursive functions the growth estimation really should be
3602 infinity. We don't want to return very large values because the growth
3603 plays various roles in badness computation fractions. Be sure to not
3604 return zero or negative growths. */
3605 if (d
.self_recursive
)
3606 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3607 else if (DECL_EXTERNAL (node
->symbol
.decl
))
3611 if (cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3612 d
.growth
-= info
->size
;
3613 /* COMDAT functions are very often not shared across multiple units
3614 since they come from various template instantiations.
3615 Take this into account. */
3616 else if (DECL_COMDAT (node
->symbol
.decl
)
3617 && cgraph_can_remove_if_no_direct_calls_p (node
))
3618 d
.growth
-= (info
->size
3619 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3623 if (node_growth_cache
.exists ())
3625 if ((int) node_growth_cache
.length () <= node
->uid
)
3626 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
3627 node_growth_cache
[node
->uid
] = d
.growth
+ (d
.growth
>= 0);
3633 /* This function performs intraprocedural analysis in NODE that is required to
3634 inline indirect calls. */
3637 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3639 ipa_analyze_node (node
);
3640 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3642 ipa_print_node_params (dump_file
, node
);
3643 ipa_print_node_jump_functions (dump_file
, node
);
3648 /* Note function body size. */
3651 inline_analyze_function (struct cgraph_node
*node
)
3653 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
3656 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3657 cgraph_node_name (node
), node
->uid
);
3658 if (optimize
&& !node
->thunk
.thunk_p
)
3659 inline_indirect_intraprocedural_analysis (node
);
3660 compute_inline_parameters (node
, false);
3666 /* Called when new function is inserted to callgraph late. */
3669 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3671 inline_analyze_function (node
);
3675 /* Note function body size. */
3678 inline_generate_summary (void)
3680 struct cgraph_node
*node
;
3682 function_insertion_hook_holder
=
3683 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3685 ipa_register_cgraph_hooks ();
3686 inline_free_summary ();
3688 FOR_EACH_DEFINED_FUNCTION (node
)
3690 inline_analyze_function (node
);
3694 /* Read predicate from IB. */
3696 static struct predicate
3697 read_predicate (struct lto_input_block
*ib
)
3699 struct predicate out
;
3705 gcc_assert (k
<= MAX_CLAUSES
);
3706 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3710 /* Zero-initialize the remaining clauses in OUT. */
3711 while (k
<= MAX_CLAUSES
)
3712 out
.clause
[k
++] = 0;
3718 /* Write inline summary for edge E to OB. */
3721 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3723 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3727 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3728 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3729 es
->loop_depth
= streamer_read_uhwi (ib
);
3730 p
= read_predicate (ib
);
3731 edge_set_predicate (e
, &p
);
3732 length
= streamer_read_uhwi (ib
);
3735 es
->param
.safe_grow_cleared (length
);
3736 for (i
= 0; i
< length
; i
++)
3737 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3742 /* Stream in inline summaries from the section. */
3745 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3748 const struct lto_function_header
*header
=
3749 (const struct lto_function_header
*) data
;
3750 const int cfg_offset
= sizeof (struct lto_function_header
);
3751 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3752 const int string_offset
= main_offset
+ header
->main_size
;
3753 struct data_in
*data_in
;
3754 struct lto_input_block ib
;
3755 unsigned int i
, count2
, j
;
3756 unsigned int f_count
;
3758 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3762 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3763 header
->string_size
, vNULL
);
3764 f_count
= streamer_read_uhwi (&ib
);
3765 for (i
= 0; i
< f_count
; i
++)
3768 struct cgraph_node
*node
;
3769 struct inline_summary
*info
;
3770 lto_symtab_encoder_t encoder
;
3771 struct bitpack_d bp
;
3772 struct cgraph_edge
*e
;
3775 index
= streamer_read_uhwi (&ib
);
3776 encoder
= file_data
->symtab_node_encoder
;
3777 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3778 info
= inline_summary (node
);
3780 info
->estimated_stack_size
3781 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3782 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3783 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3785 bp
= streamer_read_bitpack (&ib
);
3786 info
->inlinable
= bp_unpack_value (&bp
, 1);
3788 count2
= streamer_read_uhwi (&ib
);
3789 gcc_assert (!info
->conds
);
3790 for (j
= 0; j
< count2
; j
++)
3793 c
.operand_num
= streamer_read_uhwi (&ib
);
3794 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3795 c
.val
= stream_read_tree (&ib
, data_in
);
3796 bp
= streamer_read_bitpack (&ib
);
3797 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3798 c
.by_ref
= bp_unpack_value (&bp
, 1);
3800 c
.offset
= streamer_read_uhwi (&ib
);
3801 vec_safe_push (info
->conds
, c
);
3803 count2
= streamer_read_uhwi (&ib
);
3804 gcc_assert (!info
->entry
);
3805 for (j
= 0; j
< count2
; j
++)
3807 struct size_time_entry e
;
3809 e
.size
= streamer_read_uhwi (&ib
);
3810 e
.time
= streamer_read_uhwi (&ib
);
3811 e
.predicate
= read_predicate (&ib
);
3813 vec_safe_push (info
->entry
, e
);
3816 p
= read_predicate (&ib
);
3817 set_hint_predicate (&info
->loop_iterations
, p
);
3818 p
= read_predicate (&ib
);
3819 set_hint_predicate (&info
->loop_stride
, p
);
3820 p
= read_predicate (&ib
);
3821 set_hint_predicate (&info
->array_index
, p
);
3822 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3823 read_inline_edge_summary (&ib
, e
);
3824 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3825 read_inline_edge_summary (&ib
, e
);
3828 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3830 lto_data_in_delete (data_in
);
3834 /* Read inline summary. Jump functions are shared among ipa-cp
3835 and inliner, so when ipa-cp is active, we don't need to write them
3839 inline_read_summary (void)
3841 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3842 struct lto_file_decl_data
*file_data
;
3845 inline_summary_alloc ();
3847 while ((file_data
= file_data_vec
[j
++]))
3850 const char *data
= lto_get_section_data (file_data
,
3851 LTO_section_inline_summary
,
3854 inline_read_section (file_data
, data
, len
);
3856 /* Fatal error here. We do not want to support compiling ltrans units
3857 with different version of compiler or different flags than the WPA
3858 unit, so this should never happen. */
3859 fatal_error ("ipa inline summary is missing in input file");
3863 ipa_register_cgraph_hooks ();
3865 ipa_prop_read_jump_functions ();
3867 function_insertion_hook_holder
=
3868 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3872 /* Write predicate P to OB. */
3875 write_predicate (struct output_block
*ob
, struct predicate
*p
)
3879 for (j
= 0; p
->clause
[j
]; j
++)
3881 gcc_assert (j
< MAX_CLAUSES
);
3882 streamer_write_uhwi (ob
, p
->clause
[j
]);
3884 streamer_write_uhwi (ob
, 0);
3888 /* Write inline summary for edge E to OB. */
3891 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
3893 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3896 streamer_write_uhwi (ob
, es
->call_stmt_size
);
3897 streamer_write_uhwi (ob
, es
->call_stmt_time
);
3898 streamer_write_uhwi (ob
, es
->loop_depth
);
3899 write_predicate (ob
, es
->predicate
);
3900 streamer_write_uhwi (ob
, es
->param
.length ());
3901 for (i
= 0; i
< (int) es
->param
.length (); i
++)
3902 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
3906 /* Write inline summary for node in SET.
3907 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
3908 active, we don't need to write them twice. */
3911 inline_write_summary (void)
3913 struct cgraph_node
*node
;
3914 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
3915 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
3916 unsigned int count
= 0;
3919 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3921 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
3922 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
3923 if (cnode
&& cnode
->analyzed
)
3926 streamer_write_uhwi (ob
, count
);
3928 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3930 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
3931 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
3932 if (cnode
&& (node
= cnode
)->analyzed
)
3934 struct inline_summary
*info
= inline_summary (node
);
3935 struct bitpack_d bp
;
3936 struct cgraph_edge
*edge
;
3939 struct condition
*c
;
3941 streamer_write_uhwi (ob
,
3942 lto_symtab_encoder_encode (encoder
,
3945 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
3946 streamer_write_hwi (ob
, info
->self_size
);
3947 streamer_write_hwi (ob
, info
->self_time
);
3948 bp
= bitpack_create (ob
->main_stream
);
3949 bp_pack_value (&bp
, info
->inlinable
, 1);
3950 streamer_write_bitpack (&bp
);
3951 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
3952 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
3954 streamer_write_uhwi (ob
, c
->operand_num
);
3955 streamer_write_uhwi (ob
, c
->code
);
3956 stream_write_tree (ob
, c
->val
, true);
3957 bp
= bitpack_create (ob
->main_stream
);
3958 bp_pack_value (&bp
, c
->agg_contents
, 1);
3959 bp_pack_value (&bp
, c
->by_ref
, 1);
3960 streamer_write_bitpack (&bp
);
3961 if (c
->agg_contents
)
3962 streamer_write_uhwi (ob
, c
->offset
);
3964 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
3965 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3967 streamer_write_uhwi (ob
, e
->size
);
3968 streamer_write_uhwi (ob
, e
->time
);
3969 write_predicate (ob
, &e
->predicate
);
3971 write_predicate (ob
, info
->loop_iterations
);
3972 write_predicate (ob
, info
->loop_stride
);
3973 write_predicate (ob
, info
->array_index
);
3974 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
3975 write_inline_edge_summary (ob
, edge
);
3976 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
3977 write_inline_edge_summary (ob
, edge
);
3980 streamer_write_char_stream (ob
->main_stream
, 0);
3981 produce_asm (ob
, NULL
);
3982 destroy_output_block (ob
);
3984 if (optimize
&& !flag_ipa_cp
)
3985 ipa_prop_write_jump_functions ();
3989 /* Release inline summary. */
3992 inline_free_summary (void)
3994 struct cgraph_node
*node
;
3995 if (!inline_edge_summary_vec
.exists ())
3997 FOR_EACH_DEFINED_FUNCTION (node
)
3998 reset_inline_summary (node
);
3999 if (function_insertion_hook_holder
)
4000 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
4001 function_insertion_hook_holder
= NULL
;
4002 if (node_removal_hook_holder
)
4003 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
4004 node_removal_hook_holder
= NULL
;
4005 if (edge_removal_hook_holder
)
4006 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
4007 edge_removal_hook_holder
= NULL
;
4008 if (node_duplication_hook_holder
)
4009 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
4010 node_duplication_hook_holder
= NULL
;
4011 if (edge_duplication_hook_holder
)
4012 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
4013 edge_duplication_hook_holder
= NULL
;
4014 vec_free (inline_summary_vec
);
4015 inline_edge_summary_vec
.release ();
4016 if (edge_predicate_pool
)
4017 free_alloc_pool (edge_predicate_pool
);
4018 edge_predicate_pool
= 0;