1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis used by the inliner and other passes limiting code size growth.
23 We estimate for each function
25 - average function execution time
26 - inlining size benefit (that is how much of function body size
27 and its call sequence is expected to disappear by inlining)
28 - inlining time benefit
31 - call statement size and time
33 inlinie_summary datastructures store above information locally (i.e.
34 parameters of the function itself) and globally (i.e. parameters of
35 the function created by applying all the inline decisions already
36 present in the callgraph).
38 We provide accestor to the inline_summary datastructure and
39 basic logic updating the parameters when inlining is performed.
41 The summaries are context sensitive. Context means
42 1) partial assignment of known constant values of operands
43 2) whether function is inlined into the call or not.
44 It is easy to add more variants. To represent function size and time
45 that depends on context (i.e. it is known to be optimized away when
46 context is known either by inlining or from IP-CP and clonning),
47 we use predicates. Predicates are logical formulas in
48 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
49 specifying what conditions must be true. Conditions are simple test
50 of the form described above.
52 In order to make predicate (possibly) true, all of its clauses must
53 be (possibly) true. To make clause (possibly) true, one of conditions
54 it mentions must be (possibly) true. There are fixed bounds on
55 number of clauses and conditions and all the manipulation functions
56 are conservative in positive direction. I.e. we may lose precision
57 by thinking that predicate may be true even when it is not.
59 estimate_edge_size and estimate_edge_growth can be used to query
60 function size/time in the given context. inline_merge_summary merges
61 properties of caller and callee after inlining.
63 Finally pass_inline_parameters is exported. This is used to drive
64 computation of function parameters used by the early inliner. IPA
65 inlined performs analysis via its analyze_function method. */
69 #include "coretypes.h"
72 #include "tree-inline.h"
73 #include "langhooks.h"
75 #include "diagnostic.h"
76 #include "gimple-pretty-print.h"
78 #include "tree-pass.h"
82 #include "gimple-ssa.h"
84 #include "tree-phinodes.h"
85 #include "ssa-iterators.h"
86 #include "tree-ssanames.h"
87 #include "tree-ssa-loop-niter.h"
88 #include "tree-ssa-loop.h"
90 #include "lto-streamer.h"
91 #include "data-streamer.h"
92 #include "tree-streamer.h"
93 #include "ipa-inline.h"
94 #include "alloc-pool.h"
96 #include "tree-scalar-evolution.h"
97 #include "ipa-utils.h"
100 /* Estimate runtime of function can easilly run into huge numbers with many
101 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
102 integer. For anything larger we use gcov_type. */
103 #define MAX_TIME 500000
105 /* Number of bits in integer, but we really want to be stable across different
107 #define NUM_CONDITIONS 32
109 enum predicate_conditions
111 predicate_false_condition
= 0,
112 predicate_not_inlined_condition
= 1,
113 predicate_first_dynamic_condition
= 2
116 /* Special condition code we use to represent test that operand is compile time
118 #define IS_NOT_CONSTANT ERROR_MARK
119 /* Special condition code we use to represent test that operand is not changed
120 across invocation of the function. When operand IS_NOT_CONSTANT it is always
121 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
122 of executions even when they are not compile time constants. */
123 #define CHANGED IDENTIFIER_NODE
125 /* Holders of ipa cgraph hooks: */
126 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
127 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
128 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
129 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
130 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
131 static void inline_node_removal_hook (struct cgraph_node
*, void *);
132 static void inline_node_duplication_hook (struct cgraph_node
*,
133 struct cgraph_node
*, void *);
134 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
135 static void inline_edge_duplication_hook (struct cgraph_edge
*,
136 struct cgraph_edge
*, void *);
138 /* VECtor holding inline summaries.
139 In GGC memory because conditions might point to constant trees. */
140 vec
<inline_summary_t
, va_gc
> *inline_summary_vec
;
141 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
143 /* Cached node/edge growths. */
144 vec
<int> node_growth_cache
;
145 vec
<edge_growth_cache_entry
> edge_growth_cache
;
147 /* Edge predicates goes here. */
148 static alloc_pool edge_predicate_pool
;
150 /* Return true predicate (tautology).
151 We represent it by empty list of clauses. */
153 static inline struct predicate
154 true_predicate (void)
162 /* Return predicate testing single condition number COND. */
164 static inline struct predicate
165 single_cond_predicate (int cond
)
168 p
.clause
[0] = 1 << cond
;
174 /* Return false predicate. First clause require false condition. */
176 static inline struct predicate
177 false_predicate (void)
179 return single_cond_predicate (predicate_false_condition
);
183 /* Return true if P is (false). */
186 true_predicate_p (struct predicate
*p
)
188 return !p
->clause
[0];
192 /* Return true if P is (false). */
195 false_predicate_p (struct predicate
*p
)
197 if (p
->clause
[0] == (1 << predicate_false_condition
))
199 gcc_checking_assert (!p
->clause
[1]
200 && p
->clause
[0] == 1 << predicate_false_condition
);
207 /* Return predicate that is set true when function is not inlined. */
209 static inline struct predicate
210 not_inlined_predicate (void)
212 return single_cond_predicate (predicate_not_inlined_condition
);
215 /* Simple description of whether a memory load or a condition refers to a load
216 from an aggregate and if so, how and where from in the aggregate.
217 Individual fields have the same meaning like fields with the same name in
220 struct agg_position_info
222 HOST_WIDE_INT offset
;
227 /* Add condition to condition list CONDS. AGGPOS describes whether the used
228 oprand is loaded from an aggregate and where in the aggregate it is. It can
229 be NULL, which means this not a load from an aggregate. */
231 static struct predicate
232 add_condition (struct inline_summary
*summary
, int operand_num
,
233 struct agg_position_info
*aggpos
,
234 enum tree_code code
, tree val
)
238 struct condition new_cond
;
239 HOST_WIDE_INT offset
;
240 bool agg_contents
, by_ref
;
244 offset
= aggpos
->offset
;
245 agg_contents
= aggpos
->agg_contents
;
246 by_ref
= aggpos
->by_ref
;
251 agg_contents
= false;
255 gcc_checking_assert (operand_num
>= 0);
256 for (i
= 0; vec_safe_iterate (summary
->conds
, i
, &c
); i
++)
258 if (c
->operand_num
== operand_num
261 && c
->agg_contents
== agg_contents
262 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
263 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
265 /* Too many conditions. Give up and return constant true. */
266 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
267 return true_predicate ();
269 new_cond
.operand_num
= operand_num
;
270 new_cond
.code
= code
;
272 new_cond
.agg_contents
= agg_contents
;
273 new_cond
.by_ref
= by_ref
;
274 new_cond
.offset
= offset
;
275 vec_safe_push (summary
->conds
, new_cond
);
276 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
280 /* Add clause CLAUSE into the predicate P. */
283 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
287 int insert_here
= -1;
294 /* False clause makes the whole predicate false. Kill the other variants. */
295 if (clause
== (1 << predicate_false_condition
))
297 p
->clause
[0] = (1 << predicate_false_condition
);
301 if (false_predicate_p (p
))
304 /* No one should be sily enough to add false into nontrivial clauses. */
305 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
307 /* Look where to insert the clause. At the same time prune out
308 clauses of P that are implied by the new clause and thus
310 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
312 p
->clause
[i2
] = p
->clause
[i
];
317 /* If p->clause[i] implies clause, there is nothing to add. */
318 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
320 /* We had nothing to add, none of clauses should've become
322 gcc_checking_assert (i
== i2
);
326 if (p
->clause
[i
] < clause
&& insert_here
< 0)
329 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
330 Otherwise the p->clause[i] has to stay. */
331 if ((p
->clause
[i
] & clause
) != clause
)
335 /* Look for clauses that are obviously true. I.e.
336 op0 == 5 || op0 != 5. */
337 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
340 if (!(clause
& (1 << c1
)))
342 cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
343 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
344 and thus there is no point for looking for them. */
345 if (cc1
->code
== CHANGED
|| cc1
->code
== IS_NOT_CONSTANT
)
347 for (c2
= c1
+ 1; c2
< NUM_CONDITIONS
; c2
++)
348 if (clause
& (1 << c2
))
351 &(*conditions
)[c1
- predicate_first_dynamic_condition
];
353 &(*conditions
)[c2
- predicate_first_dynamic_condition
];
354 if (cc1
->operand_num
== cc2
->operand_num
355 && cc1
->val
== cc2
->val
356 && cc2
->code
!= IS_NOT_CONSTANT
357 && cc2
->code
!= CHANGED
358 && cc1
->code
== invert_tree_comparison
360 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
366 /* We run out of variants. Be conservative in positive direction. */
367 if (i2
== MAX_CLAUSES
)
369 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
370 p
->clause
[i2
+ 1] = 0;
371 if (insert_here
>= 0)
372 for (; i2
> insert_here
; i2
--)
373 p
->clause
[i2
] = p
->clause
[i2
- 1];
376 p
->clause
[insert_here
] = clause
;
382 static struct predicate
383 and_predicates (conditions conditions
,
384 struct predicate
*p
, struct predicate
*p2
)
386 struct predicate out
= *p
;
389 /* Avoid busy work. */
390 if (false_predicate_p (p2
) || true_predicate_p (p
))
392 if (false_predicate_p (p
) || true_predicate_p (p2
))
395 /* See how far predicates match. */
396 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
398 gcc_checking_assert (i
< MAX_CLAUSES
);
401 /* Combine the predicates rest. */
402 for (; p2
->clause
[i
]; i
++)
404 gcc_checking_assert (i
< MAX_CLAUSES
);
405 add_clause (conditions
, &out
, p2
->clause
[i
]);
411 /* Return true if predicates are obviously equal. */
414 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
417 for (i
= 0; p
->clause
[i
]; i
++)
419 gcc_checking_assert (i
< MAX_CLAUSES
);
420 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
421 gcc_checking_assert (!p2
->clause
[i
]
422 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
423 if (p
->clause
[i
] != p2
->clause
[i
])
426 return !p2
->clause
[i
];
432 static struct predicate
433 or_predicates (conditions conditions
,
434 struct predicate
*p
, struct predicate
*p2
)
436 struct predicate out
= true_predicate ();
439 /* Avoid busy work. */
440 if (false_predicate_p (p2
) || true_predicate_p (p
))
442 if (false_predicate_p (p
) || true_predicate_p (p2
))
444 if (predicates_equal_p (p
, p2
))
447 /* OK, combine the predicates. */
448 for (i
= 0; p
->clause
[i
]; i
++)
449 for (j
= 0; p2
->clause
[j
]; j
++)
451 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
452 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
458 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
459 if predicate P is known to be false. */
462 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
466 /* True remains true. */
467 if (true_predicate_p (p
))
470 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
472 /* See if we can find clause we can disprove. */
473 for (i
= 0; p
->clause
[i
]; i
++)
475 gcc_checking_assert (i
< MAX_CLAUSES
);
476 if (!(p
->clause
[i
] & possible_truths
))
482 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
483 instruction will be recomputed per invocation of the inlined call. */
486 predicate_probability (conditions conds
,
487 struct predicate
*p
, clause_t possible_truths
,
488 vec
<inline_param_summary_t
> inline_param_summary
)
491 int combined_prob
= REG_BR_PROB_BASE
;
493 /* True remains true. */
494 if (true_predicate_p (p
))
495 return REG_BR_PROB_BASE
;
497 if (false_predicate_p (p
))
500 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
502 /* See if we can find clause we can disprove. */
503 for (i
= 0; p
->clause
[i
]; i
++)
505 gcc_checking_assert (i
< MAX_CLAUSES
);
506 if (!(p
->clause
[i
] & possible_truths
))
512 if (!inline_param_summary
.exists ())
513 return REG_BR_PROB_BASE
;
514 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
515 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
517 if (i2
>= predicate_first_dynamic_condition
)
520 &(*conds
)[i2
- predicate_first_dynamic_condition
];
521 if (c
->code
== CHANGED
523 (int) inline_param_summary
.length ()))
526 inline_param_summary
[c
->operand_num
].change_prob
;
527 this_prob
= MAX (this_prob
, iprob
);
530 this_prob
= REG_BR_PROB_BASE
;
533 this_prob
= REG_BR_PROB_BASE
;
535 combined_prob
= MIN (this_prob
, combined_prob
);
540 return combined_prob
;
544 /* Dump conditional COND. */
547 dump_condition (FILE *f
, conditions conditions
, int cond
)
550 if (cond
== predicate_false_condition
)
551 fprintf (f
, "false");
552 else if (cond
== predicate_not_inlined_condition
)
553 fprintf (f
, "not inlined");
556 c
= &(*conditions
)[cond
- predicate_first_dynamic_condition
];
557 fprintf (f
, "op%i", c
->operand_num
);
559 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
560 c
->by_ref
? "ref " : "", c
->offset
);
561 if (c
->code
== IS_NOT_CONSTANT
)
563 fprintf (f
, " not constant");
566 if (c
->code
== CHANGED
)
568 fprintf (f
, " changed");
571 fprintf (f
, " %s ", op_symbol_code (c
->code
));
572 print_generic_expr (f
, c
->val
, 1);
577 /* Dump clause CLAUSE. */
580 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
587 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
588 if (clause
& (1 << i
))
593 dump_condition (f
, conds
, i
);
599 /* Dump predicate PREDICATE. */
602 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
605 if (true_predicate_p (pred
))
606 dump_clause (f
, conds
, 0);
608 for (i
= 0; pred
->clause
[i
]; i
++)
612 dump_clause (f
, conds
, pred
->clause
[i
]);
618 /* Dump inline hints. */
620 dump_inline_hints (FILE *f
, inline_hints hints
)
624 fprintf (f
, "inline hints:");
625 if (hints
& INLINE_HINT_indirect_call
)
627 hints
&= ~INLINE_HINT_indirect_call
;
628 fprintf (f
, " indirect_call");
630 if (hints
& INLINE_HINT_loop_iterations
)
632 hints
&= ~INLINE_HINT_loop_iterations
;
633 fprintf (f
, " loop_iterations");
635 if (hints
& INLINE_HINT_loop_stride
)
637 hints
&= ~INLINE_HINT_loop_stride
;
638 fprintf (f
, " loop_stride");
640 if (hints
& INLINE_HINT_same_scc
)
642 hints
&= ~INLINE_HINT_same_scc
;
643 fprintf (f
, " same_scc");
645 if (hints
& INLINE_HINT_in_scc
)
647 hints
&= ~INLINE_HINT_in_scc
;
648 fprintf (f
, " in_scc");
650 if (hints
& INLINE_HINT_cross_module
)
652 hints
&= ~INLINE_HINT_cross_module
;
653 fprintf (f
, " cross_module");
655 if (hints
& INLINE_HINT_declared_inline
)
657 hints
&= ~INLINE_HINT_declared_inline
;
658 fprintf (f
, " declared_inline");
660 if (hints
& INLINE_HINT_array_index
)
662 hints
&= ~INLINE_HINT_array_index
;
663 fprintf (f
, " array_index");
669 /* Record SIZE and TIME under condition PRED into the inline summary. */
672 account_size_time (struct inline_summary
*summary
, int size
, int time
,
673 struct predicate
*pred
)
679 if (false_predicate_p (pred
))
682 /* We need to create initial empty unconitional clause, but otherwie
683 we don't need to account empty times and sizes. */
684 if (!size
&& !time
&& summary
->entry
)
687 /* Watch overflow that might result from insane profiles. */
688 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
689 time
= MAX_TIME
* INLINE_TIME_SCALE
;
690 gcc_assert (time
>= 0);
692 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
693 if (predicates_equal_p (&e
->predicate
, pred
))
702 e
= &(*summary
->entry
)[0];
703 gcc_assert (!e
->predicate
.clause
[0]);
704 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
706 "\t\tReached limit on number of entries, "
707 "ignoring the predicate.");
709 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
712 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
713 ((double) size
) / INLINE_SIZE_SCALE
,
714 ((double) time
) / INLINE_TIME_SCALE
, found
? "" : "new ");
715 dump_predicate (dump_file
, summary
->conds
, pred
);
719 struct size_time_entry new_entry
;
720 new_entry
.size
= size
;
721 new_entry
.time
= time
;
722 new_entry
.predicate
= *pred
;
723 vec_safe_push (summary
->entry
, new_entry
);
729 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
730 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
734 /* Set predicate for edge E. */
737 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
739 struct inline_edge_summary
*es
= inline_edge_summary (e
);
740 if (predicate
&& !true_predicate_p (predicate
))
743 es
->predicate
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
744 *es
->predicate
= *predicate
;
749 pool_free (edge_predicate_pool
, es
->predicate
);
750 es
->predicate
= NULL
;
754 /* Set predicate for hint *P. */
757 set_hint_predicate (struct predicate
**p
, struct predicate new_predicate
)
759 if (false_predicate_p (&new_predicate
) || true_predicate_p (&new_predicate
))
762 pool_free (edge_predicate_pool
, *p
);
768 *p
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
774 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
775 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
776 Return clause of possible truths. When INLINE_P is true, assume that we are
779 ERROR_MARK means compile time invariant. */
782 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
784 vec
<tree
> known_vals
,
785 vec
<ipa_agg_jump_function_p
>
788 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
789 struct inline_summary
*info
= inline_summary (node
);
793 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
798 /* We allow call stmt to have fewer arguments than the callee function
799 (especially for K&R style programs). So bound check here (we assume
800 known_aggs vector, if non-NULL, has the same length as
802 gcc_checking_assert (!known_aggs
.exists ()
803 || (known_vals
.length () == known_aggs
.length ()));
804 if (c
->operand_num
>= (int) known_vals
.length ())
806 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
812 struct ipa_agg_jump_function
*agg
;
814 if (c
->code
== CHANGED
816 && (known_vals
[c
->operand_num
] == error_mark_node
))
819 if (known_aggs
.exists ())
821 agg
= known_aggs
[c
->operand_num
];
822 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
829 val
= known_vals
[c
->operand_num
];
830 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
836 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
839 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
841 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
842 if (res
&& integer_zerop (res
))
844 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
850 /* Work out what conditions might be true at invocation of E. */
853 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
854 clause_t
*clause_ptr
,
855 vec
<tree
> *known_vals_ptr
,
856 vec
<tree
> *known_binfos_ptr
,
857 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
859 struct cgraph_node
*callee
=
860 cgraph_function_or_thunk_node (e
->callee
, NULL
);
861 struct inline_summary
*info
= inline_summary (callee
);
862 vec
<tree
> known_vals
= vNULL
;
863 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
866 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
868 known_vals_ptr
->create (0);
869 if (known_binfos_ptr
)
870 known_binfos_ptr
->create (0);
872 if (ipa_node_params_vector
.exists ()
873 && !e
->call_stmt_cannot_inline_p
874 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_binfos_ptr
))
876 struct ipa_node_params
*parms_info
;
877 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
878 struct inline_edge_summary
*es
= inline_edge_summary (e
);
879 int i
, count
= ipa_get_cs_argument_count (args
);
881 if (e
->caller
->global
.inlined_to
)
882 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
884 parms_info
= IPA_NODE_REF (e
->caller
);
886 if (count
&& (info
->conds
|| known_vals_ptr
))
887 known_vals
.safe_grow_cleared (count
);
888 if (count
&& (info
->conds
|| known_aggs_ptr
))
889 known_aggs
.safe_grow_cleared (count
);
890 if (count
&& known_binfos_ptr
)
891 known_binfos_ptr
->safe_grow_cleared (count
);
893 for (i
= 0; i
< count
; i
++)
895 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
896 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
899 if (known_vals
.exists () && TREE_CODE (cst
) != TREE_BINFO
)
901 else if (known_binfos_ptr
!= NULL
902 && TREE_CODE (cst
) == TREE_BINFO
)
903 (*known_binfos_ptr
)[i
] = cst
;
905 else if (inline_p
&& !es
->param
[i
].change_prob
)
906 known_vals
[i
] = error_mark_node
;
907 /* TODO: When IPA-CP starts propagating and merging aggregate jump
908 functions, use its knowledge of the caller too, just like the
909 scalar case above. */
910 known_aggs
[i
] = &jf
->agg
;
915 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
916 known_vals
, known_aggs
);
919 *known_vals_ptr
= known_vals
;
921 known_vals
.release ();
924 *known_aggs_ptr
= known_aggs
;
926 known_aggs
.release ();
930 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
933 inline_summary_alloc (void)
935 if (!node_removal_hook_holder
)
936 node_removal_hook_holder
=
937 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
938 if (!edge_removal_hook_holder
)
939 edge_removal_hook_holder
=
940 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
941 if (!node_duplication_hook_holder
)
942 node_duplication_hook_holder
=
943 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
944 if (!edge_duplication_hook_holder
)
945 edge_duplication_hook_holder
=
946 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
948 if (vec_safe_length (inline_summary_vec
) <= (unsigned) cgraph_max_uid
)
949 vec_safe_grow_cleared (inline_summary_vec
, cgraph_max_uid
+ 1);
950 if (inline_edge_summary_vec
.length () <= (unsigned) cgraph_edge_max_uid
)
951 inline_edge_summary_vec
.safe_grow_cleared (cgraph_edge_max_uid
+ 1);
952 if (!edge_predicate_pool
)
953 edge_predicate_pool
= create_alloc_pool ("edge predicates",
954 sizeof (struct predicate
), 10);
957 /* We are called multiple time for given function; clear
958 data from previous run so they are not cumulated. */
961 reset_inline_edge_summary (struct cgraph_edge
*e
)
963 if (e
->uid
< (int) inline_edge_summary_vec
.length ())
965 struct inline_edge_summary
*es
= inline_edge_summary (e
);
967 es
->call_stmt_size
= es
->call_stmt_time
= 0;
969 pool_free (edge_predicate_pool
, es
->predicate
);
970 es
->predicate
= NULL
;
971 es
->param
.release ();
975 /* We are called multiple time for given function; clear
976 data from previous run so they are not cumulated. */
979 reset_inline_summary (struct cgraph_node
*node
)
981 struct inline_summary
*info
= inline_summary (node
);
982 struct cgraph_edge
*e
;
984 info
->self_size
= info
->self_time
= 0;
985 info
->estimated_stack_size
= 0;
986 info
->estimated_self_stack_size
= 0;
987 info
->stack_frame_offset
= 0;
992 if (info
->loop_iterations
)
994 pool_free (edge_predicate_pool
, info
->loop_iterations
);
995 info
->loop_iterations
= NULL
;
997 if (info
->loop_stride
)
999 pool_free (edge_predicate_pool
, info
->loop_stride
);
1000 info
->loop_stride
= NULL
;
1002 if (info
->array_index
)
1004 pool_free (edge_predicate_pool
, info
->array_index
);
1005 info
->array_index
= NULL
;
1007 vec_free (info
->conds
);
1008 vec_free (info
->entry
);
1009 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1010 reset_inline_edge_summary (e
);
1011 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1012 reset_inline_edge_summary (e
);
1015 /* Hook that is called by cgraph.c when a node is removed. */
1018 inline_node_removal_hook (struct cgraph_node
*node
,
1019 void *data ATTRIBUTE_UNUSED
)
1021 struct inline_summary
*info
;
1022 if (vec_safe_length (inline_summary_vec
) <= (unsigned) node
->uid
)
1024 info
= inline_summary (node
);
1025 reset_inline_summary (node
);
1026 memset (info
, 0, sizeof (inline_summary_t
));
1029 /* Remap predicate P of former function to be predicate of duplicated functoin.
1030 POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
1031 INFO is inline summary of the duplicated node. */
1033 static struct predicate
1034 remap_predicate_after_duplication (struct predicate
*p
,
1035 clause_t possible_truths
,
1036 struct inline_summary
*info
)
1038 struct predicate new_predicate
= true_predicate ();
1040 for (j
= 0; p
->clause
[j
]; j
++)
1041 if (!(possible_truths
& p
->clause
[j
]))
1043 new_predicate
= false_predicate ();
1047 add_clause (info
->conds
, &new_predicate
,
1048 possible_truths
& p
->clause
[j
]);
1049 return new_predicate
;
1052 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
1053 Additionally care about allocating new memory slot for updated predicate
1054 and set it to NULL when it becomes true or false (and thus uninteresting).
1058 remap_hint_predicate_after_duplication (struct predicate
**p
,
1059 clause_t possible_truths
,
1060 struct inline_summary
*info
)
1062 struct predicate new_predicate
;
1067 new_predicate
= remap_predicate_after_duplication (*p
,
1068 possible_truths
, info
);
1069 /* We do not want to free previous predicate; it is used by node origin. */
1071 set_hint_predicate (p
, new_predicate
);
1075 /* Hook that is called by cgraph.c when a node is duplicated. */
1078 inline_node_duplication_hook (struct cgraph_node
*src
,
1079 struct cgraph_node
*dst
,
1080 ATTRIBUTE_UNUSED
void *data
)
1082 struct inline_summary
*info
;
1083 inline_summary_alloc ();
1084 info
= inline_summary (dst
);
1085 memcpy (info
, inline_summary (src
), sizeof (struct inline_summary
));
1086 /* TODO: as an optimization, we may avoid copying conditions
1087 that are known to be false or true. */
1088 info
->conds
= vec_safe_copy (info
->conds
);
1090 /* When there are any replacements in the function body, see if we can figure
1091 out that something was optimized out. */
1092 if (ipa_node_params_vector
.exists () && dst
->clone
.tree_map
)
1094 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
1095 /* Use SRC parm info since it may not be copied yet. */
1096 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1097 vec
<tree
> known_vals
= vNULL
;
1098 int count
= ipa_get_param_count (parms_info
);
1100 clause_t possible_truths
;
1101 struct predicate true_pred
= true_predicate ();
1103 int optimized_out_size
= 0;
1104 bool inlined_to_p
= false;
1105 struct cgraph_edge
*edge
;
1108 known_vals
.safe_grow_cleared (count
);
1109 for (i
= 0; i
< count
; i
++)
1111 struct ipa_replace_map
*r
;
1113 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
1115 if (((!r
->old_tree
&& r
->parm_num
== i
)
1116 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
1117 && r
->replace_p
&& !r
->ref_p
)
1119 known_vals
[i
] = r
->new_tree
;
1124 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1127 known_vals
.release ();
1129 account_size_time (info
, 0, 0, &true_pred
);
1131 /* Remap size_time vectors.
1132 Simplify the predicate by prunning out alternatives that are known
1134 TODO: as on optimization, we can also eliminate conditions known
1136 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
1138 struct predicate new_predicate
;
1139 new_predicate
= remap_predicate_after_duplication (&e
->predicate
,
1142 if (false_predicate_p (&new_predicate
))
1143 optimized_out_size
+= e
->size
;
1145 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1148 /* Remap edge predicates with the same simplification as above.
1149 Also copy constantness arrays. */
1150 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1152 struct predicate new_predicate
;
1153 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1155 if (!edge
->inline_failed
)
1156 inlined_to_p
= true;
1159 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1162 if (false_predicate_p (&new_predicate
)
1163 && !false_predicate_p (es
->predicate
))
1165 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1166 edge
->frequency
= 0;
1168 edge_set_predicate (edge
, &new_predicate
);
1171 /* Remap indirect edge predicates with the same simplificaiton as above.
1172 Also copy constantness arrays. */
1173 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1175 struct predicate new_predicate
;
1176 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1178 gcc_checking_assert (edge
->inline_failed
);
1181 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1184 if (false_predicate_p (&new_predicate
)
1185 && !false_predicate_p (es
->predicate
))
1187 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1188 edge
->frequency
= 0;
1190 edge_set_predicate (edge
, &new_predicate
);
1192 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
1193 possible_truths
, info
);
1194 remap_hint_predicate_after_duplication (&info
->loop_stride
,
1195 possible_truths
, info
);
1196 remap_hint_predicate_after_duplication (&info
->array_index
,
1197 possible_truths
, info
);
1199 /* If inliner or someone after inliner will ever start producing
1200 non-trivial clones, we will get trouble with lack of information
1201 about updating self sizes, because size vectors already contains
1202 sizes of the calees. */
1203 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
1207 info
->entry
= vec_safe_copy (info
->entry
);
1208 if (info
->loop_iterations
)
1210 predicate p
= *info
->loop_iterations
;
1211 info
->loop_iterations
= NULL
;
1212 set_hint_predicate (&info
->loop_iterations
, p
);
1214 if (info
->loop_stride
)
1216 predicate p
= *info
->loop_stride
;
1217 info
->loop_stride
= NULL
;
1218 set_hint_predicate (&info
->loop_stride
, p
);
1220 if (info
->array_index
)
1222 predicate p
= *info
->array_index
;
1223 info
->array_index
= NULL
;
1224 set_hint_predicate (&info
->array_index
, p
);
1227 inline_update_overall_summary (dst
);
1231 /* Hook that is called by cgraph.c when a node is duplicated. */
1234 inline_edge_duplication_hook (struct cgraph_edge
*src
,
1235 struct cgraph_edge
*dst
,
1236 ATTRIBUTE_UNUSED
void *data
)
1238 struct inline_edge_summary
*info
;
1239 struct inline_edge_summary
*srcinfo
;
1240 inline_summary_alloc ();
1241 info
= inline_edge_summary (dst
);
1242 srcinfo
= inline_edge_summary (src
);
1243 memcpy (info
, srcinfo
, sizeof (struct inline_edge_summary
));
1244 info
->predicate
= NULL
;
1245 edge_set_predicate (dst
, srcinfo
->predicate
);
1246 info
->param
= srcinfo
->param
.copy ();
1250 /* Keep edge cache consistent across edge removal. */
1253 inline_edge_removal_hook (struct cgraph_edge
*edge
,
1254 void *data ATTRIBUTE_UNUSED
)
1256 if (edge_growth_cache
.exists ())
1257 reset_edge_growth_cache (edge
);
1258 reset_inline_edge_summary (edge
);
1262 /* Initialize growth caches. */
1265 initialize_growth_caches (void)
1267 if (cgraph_edge_max_uid
)
1268 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
1270 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
1274 /* Free growth caches. */
1277 free_growth_caches (void)
1279 edge_growth_cache
.release ();
1280 node_growth_cache
.release ();
1284 /* Dump edge summaries associated to NODE and recursively to all clones.
1285 Indent by INDENT. */
1288 dump_inline_edge_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
1289 struct inline_summary
*info
)
1291 struct cgraph_edge
*edge
;
1292 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1294 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1295 struct cgraph_node
*callee
=
1296 cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1300 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
1301 " time: %2i callee size:%2i stack:%2i",
1302 indent
, "", cgraph_node_name (callee
), callee
->order
,
1303 !edge
->inline_failed
1304 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
1305 indent
, "", es
->loop_depth
, edge
->frequency
,
1306 es
->call_stmt_size
, es
->call_stmt_time
,
1307 (int) inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1308 (int) inline_summary (callee
)->estimated_stack_size
);
1312 fprintf (f
, " predicate: ");
1313 dump_predicate (f
, info
->conds
, es
->predicate
);
1317 if (es
->param
.exists ())
1318 for (i
= 0; i
< (int) es
->param
.length (); i
++)
1320 int prob
= es
->param
[i
].change_prob
;
1323 fprintf (f
, "%*s op%i is compile time invariant\n",
1325 else if (prob
!= REG_BR_PROB_BASE
)
1326 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1327 prob
* 100.0 / REG_BR_PROB_BASE
);
1329 if (!edge
->inline_failed
)
1331 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1332 " callee size %i\n",
1334 (int) inline_summary (callee
)->stack_frame_offset
,
1335 (int) inline_summary (callee
)->estimated_self_stack_size
,
1336 (int) inline_summary (callee
)->estimated_stack_size
);
1337 dump_inline_edge_summary (f
, indent
+ 2, callee
, info
);
1340 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1342 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1343 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1347 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
1350 fprintf (f
, "predicate: ");
1351 dump_predicate (f
, info
->conds
, es
->predicate
);
1360 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
1362 if (node
->definition
)
1364 struct inline_summary
*s
= inline_summary (node
);
1367 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
1369 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
1370 fprintf (f
, " always_inline");
1372 fprintf (f
, " inlinable");
1373 fprintf (f
, "\n self time: %i\n", s
->self_time
);
1374 fprintf (f
, " global time: %i\n", s
->time
);
1375 fprintf (f
, " self size: %i\n", s
->self_size
);
1376 fprintf (f
, " global size: %i\n", s
->size
);
1377 fprintf (f
, " self stack: %i\n",
1378 (int) s
->estimated_self_stack_size
);
1379 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
1381 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
1383 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
1384 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
1386 fprintf (f
, " size:%f, time:%f, predicate:",
1387 (double) e
->size
/ INLINE_SIZE_SCALE
,
1388 (double) e
->time
/ INLINE_TIME_SCALE
);
1389 dump_predicate (f
, s
->conds
, &e
->predicate
);
1391 if (s
->loop_iterations
)
1393 fprintf (f
, " loop iterations:");
1394 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1398 fprintf (f
, " loop stride:");
1399 dump_predicate (f
, s
->conds
, s
->loop_stride
);
1403 fprintf (f
, " array index:");
1404 dump_predicate (f
, s
->conds
, s
->array_index
);
1406 fprintf (f
, " calls:\n");
1407 dump_inline_edge_summary (f
, 4, node
, s
);
1413 debug_inline_summary (struct cgraph_node
*node
)
1415 dump_inline_summary (stderr
, node
);
1419 dump_inline_summaries (FILE *f
)
1421 struct cgraph_node
*node
;
1423 FOR_EACH_DEFINED_FUNCTION (node
)
1424 if (!node
->global
.inlined_to
)
1425 dump_inline_summary (f
, node
);
1428 /* Give initial reasons why inlining would fail on EDGE. This gets either
1429 nullified or usually overwritten by more precise reasons later. */
1432 initialize_inline_failed (struct cgraph_edge
*e
)
1434 struct cgraph_node
*callee
= e
->callee
;
1436 if (e
->indirect_unknown_callee
)
1437 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1438 else if (!callee
->definition
)
1439 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1440 else if (callee
->local
.redefined_extern_inline
)
1441 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1442 else if (e
->call_stmt_cannot_inline_p
)
1443 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1444 else if (cfun
&& fn_contains_cilk_spawn_p (cfun
))
1445 /* We can't inline if the function is spawing a function. */
1446 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
1448 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1451 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1452 boolean variable pointed to by DATA. */
1455 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1458 bool *b
= (bool *) data
;
1463 /* If OP refers to value of function parameter, return the corresponding
1467 unmodified_parm_1 (gimple stmt
, tree op
)
1469 /* SSA_NAME referring to parm default def? */
1470 if (TREE_CODE (op
) == SSA_NAME
1471 && SSA_NAME_IS_DEFAULT_DEF (op
)
1472 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1473 return SSA_NAME_VAR (op
);
1474 /* Non-SSA parm reference? */
1475 if (TREE_CODE (op
) == PARM_DECL
)
1477 bool modified
= false;
1480 ao_ref_init (&refd
, op
);
1481 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1489 /* If OP refers to value of function parameter, return the corresponding
1490 parameter. Also traverse chains of SSA register assignments. */
1493 unmodified_parm (gimple stmt
, tree op
)
1495 tree res
= unmodified_parm_1 (stmt
, op
);
1499 if (TREE_CODE (op
) == SSA_NAME
1500 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1501 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1502 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1503 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1507 /* If OP refers to a value of a function parameter or value loaded from an
1508 aggregate passed to a parameter (either by value or reference), return TRUE
1509 and store the number of the parameter to *INDEX_P and information whether
1510 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1511 the function parameters, STMT is the statement in which OP is used or
1515 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1516 gimple stmt
, tree op
, int *index_p
,
1517 struct agg_position_info
*aggpos
)
1519 tree res
= unmodified_parm_1 (stmt
, op
);
1521 gcc_checking_assert (aggpos
);
1524 *index_p
= ipa_get_param_decl_index (info
, res
);
1527 aggpos
->agg_contents
= false;
1528 aggpos
->by_ref
= false;
1532 if (TREE_CODE (op
) == SSA_NAME
)
1534 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1535 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1537 stmt
= SSA_NAME_DEF_STMT (op
);
1538 op
= gimple_assign_rhs1 (stmt
);
1539 if (!REFERENCE_CLASS_P (op
))
1540 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1544 aggpos
->agg_contents
= true;
1545 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1549 /* See if statement might disappear after inlining.
1550 0 - means not eliminated
1551 1 - half of statements goes away
1552 2 - for sure it is eliminated.
1553 We are not terribly sophisticated, basically looking for simple abstraction
1554 penalty wrappers. */
1557 eliminated_by_inlining_prob (gimple stmt
)
1559 enum gimple_code code
= gimple_code (stmt
);
1560 enum tree_code rhs_code
;
1570 if (gimple_num_ops (stmt
) != 2)
1573 rhs_code
= gimple_assign_rhs_code (stmt
);
1575 /* Casts of parameters, loads from parameters passed by reference
1576 and stores to return value or parameters are often free after
1577 inlining dua to SRA and further combining.
1578 Assume that half of statements goes away. */
1579 if (rhs_code
== CONVERT_EXPR
1580 || rhs_code
== NOP_EXPR
1581 || rhs_code
== VIEW_CONVERT_EXPR
1582 || rhs_code
== ADDR_EXPR
1583 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1585 tree rhs
= gimple_assign_rhs1 (stmt
);
1586 tree lhs
= gimple_assign_lhs (stmt
);
1587 tree inner_rhs
= get_base_address (rhs
);
1588 tree inner_lhs
= get_base_address (lhs
);
1589 bool rhs_free
= false;
1590 bool lhs_free
= false;
1597 /* Reads of parameter are expected to be free. */
1598 if (unmodified_parm (stmt
, inner_rhs
))
1600 /* Match expressions of form &this->field. Those will most likely
1601 combine with something upstream after inlining. */
1602 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1604 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1605 if (TREE_CODE (op
) == PARM_DECL
)
1607 else if (TREE_CODE (op
) == MEM_REF
1608 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0)))
1612 /* When parameter is not SSA register because its address is taken
1613 and it is just copied into one, the statement will be completely
1614 free after inlining (we will copy propagate backward). */
1615 if (rhs_free
&& is_gimple_reg (lhs
))
1618 /* Reads of parameters passed by reference
1619 expected to be free (i.e. optimized out after inlining). */
1620 if (TREE_CODE (inner_rhs
) == MEM_REF
1621 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1624 /* Copying parameter passed by reference into gimple register is
1625 probably also going to copy propagate, but we can't be quite
1627 if (rhs_free
&& is_gimple_reg (lhs
))
1630 /* Writes to parameters, parameters passed by value and return value
1631 (either dirrectly or passed via invisible reference) are free.
1633 TODO: We ought to handle testcase like
1634 struct a {int a,b;};
1636 retrurnsturct (void)
1642 This translate into:
1657 For that we either need to copy ipa-split logic detecting writes
1659 if (TREE_CODE (inner_lhs
) == PARM_DECL
1660 || TREE_CODE (inner_lhs
) == RESULT_DECL
1661 || (TREE_CODE (inner_lhs
) == MEM_REF
1662 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1663 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1664 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1665 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1667 0))) == RESULT_DECL
))))
1670 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1672 if (lhs_free
&& rhs_free
)
1682 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1683 predicates to the CFG edges. */
1686 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1687 struct inline_summary
*summary
,
1693 struct agg_position_info aggpos
;
1694 enum tree_code code
, inverted_code
;
1700 last
= last_stmt (bb
);
1701 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1703 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1705 op
= gimple_cond_lhs (last
);
1706 /* TODO: handle conditionals like
1709 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1711 code
= gimple_cond_code (last
);
1713 = invert_tree_comparison (code
,
1714 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1716 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1718 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1719 e
->flags
& EDGE_TRUE_VALUE
1720 ? code
: inverted_code
,
1721 gimple_cond_rhs (last
));
1722 e
->aux
= pool_alloc (edge_predicate_pool
);
1723 *(struct predicate
*) e
->aux
= p
;
1727 if (TREE_CODE (op
) != SSA_NAME
)
1730 if (builtin_constant_p (op))
1734 Here we can predicate nonconstant_code. We can't
1735 really handle constant_code since we have no predicate
1736 for this and also the constant code is not known to be
1737 optimized away when inliner doen't see operand is constant.
1738 Other optimizers might think otherwise. */
1739 if (gimple_cond_code (last
) != NE_EXPR
1740 || !integer_zerop (gimple_cond_rhs (last
)))
1742 set_stmt
= SSA_NAME_DEF_STMT (op
);
1743 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1744 || gimple_call_num_args (set_stmt
) != 1)
1746 op2
= gimple_call_arg (set_stmt
, 0);
1747 if (!unmodified_parm_or_parm_agg_item
1748 (info
, set_stmt
, op2
, &index
, &aggpos
))
1750 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1752 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1753 IS_NOT_CONSTANT
, NULL_TREE
);
1754 e
->aux
= pool_alloc (edge_predicate_pool
);
1755 *(struct predicate
*) e
->aux
= p
;
1760 /* If BB ends by a switch we can turn into predicates, attach corresponding
1761 predicates to the CFG edges. */
1764 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1765 struct inline_summary
*summary
,
1771 struct agg_position_info aggpos
;
1777 last
= last_stmt (bb
);
1778 if (!last
|| gimple_code (last
) != GIMPLE_SWITCH
)
1780 op
= gimple_switch_index (last
);
1781 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1784 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1786 e
->aux
= pool_alloc (edge_predicate_pool
);
1787 *(struct predicate
*) e
->aux
= false_predicate ();
1789 n
= gimple_switch_num_labels (last
);
1790 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1792 tree cl
= gimple_switch_label (last
, case_idx
);
1796 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1797 min
= CASE_LOW (cl
);
1798 max
= CASE_HIGH (cl
);
1800 /* For default we might want to construct predicate that none
1801 of cases is met, but it is bit hard to do not having negations
1802 of conditionals handy. */
1804 p
= true_predicate ();
1806 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1809 struct predicate p1
, p2
;
1810 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1811 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1812 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1814 *(struct predicate
*) e
->aux
1815 = or_predicates (summary
->conds
, &p
, (struct predicate
*) e
->aux
);
1820 /* For each BB in NODE attach to its AUX pointer predicate under
1821 which it is executable. */
1824 compute_bb_predicates (struct cgraph_node
*node
,
1825 struct ipa_node_params
*parms_info
,
1826 struct inline_summary
*summary
)
1828 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1832 FOR_EACH_BB_FN (bb
, my_function
)
1834 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1835 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1838 /* Entry block is always executable. */
1839 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1840 = pool_alloc (edge_predicate_pool
);
1841 *(struct predicate
*) ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1842 = true_predicate ();
1844 /* A simple dataflow propagation of predicates forward in the CFG.
1845 TODO: work in reverse postorder. */
1849 FOR_EACH_BB_FN (bb
, my_function
)
1851 struct predicate p
= false_predicate ();
1854 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1858 struct predicate this_bb_predicate
1859 = *(struct predicate
*) e
->src
->aux
;
1862 = and_predicates (summary
->conds
, &this_bb_predicate
,
1863 (struct predicate
*) e
->aux
);
1864 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1865 if (true_predicate_p (&p
))
1869 if (false_predicate_p (&p
))
1870 gcc_assert (!bb
->aux
);
1876 bb
->aux
= pool_alloc (edge_predicate_pool
);
1877 *((struct predicate
*) bb
->aux
) = p
;
1879 else if (!predicates_equal_p (&p
, (struct predicate
*) bb
->aux
))
1882 *((struct predicate
*) bb
->aux
) = p
;
1890 /* We keep info about constantness of SSA names. */
1892 typedef struct predicate predicate_t
;
1893 /* Return predicate specifying when the STMT might have result that is not
1894 a compile time constant. */
1896 static struct predicate
1897 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1898 struct inline_summary
*summary
,
1900 vec
<predicate_t
> nonconstant_names
)
1905 while (UNARY_CLASS_P (expr
))
1906 expr
= TREE_OPERAND (expr
, 0);
1908 parm
= unmodified_parm (NULL
, expr
);
1909 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1910 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1911 if (is_gimple_min_invariant (expr
))
1912 return false_predicate ();
1913 if (TREE_CODE (expr
) == SSA_NAME
)
1914 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1915 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1917 struct predicate p1
= will_be_nonconstant_expr_predicate
1918 (info
, summary
, TREE_OPERAND (expr
, 0),
1920 struct predicate p2
;
1921 if (true_predicate_p (&p1
))
1923 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1924 TREE_OPERAND (expr
, 1),
1926 return or_predicates (summary
->conds
, &p1
, &p2
);
1928 else if (TREE_CODE (expr
) == COND_EXPR
)
1930 struct predicate p1
= will_be_nonconstant_expr_predicate
1931 (info
, summary
, TREE_OPERAND (expr
, 0),
1933 struct predicate p2
;
1934 if (true_predicate_p (&p1
))
1936 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1937 TREE_OPERAND (expr
, 1),
1939 if (true_predicate_p (&p2
))
1941 p1
= or_predicates (summary
->conds
, &p1
, &p2
);
1942 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1943 TREE_OPERAND (expr
, 2),
1945 return or_predicates (summary
->conds
, &p1
, &p2
);
1952 return false_predicate ();
1956 /* Return predicate specifying when the STMT might have result that is not
1957 a compile time constant. */
1959 static struct predicate
1960 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1961 struct inline_summary
*summary
,
1963 vec
<predicate_t
> nonconstant_names
)
1965 struct predicate p
= true_predicate ();
1968 struct predicate op_non_const
;
1971 struct agg_position_info aggpos
;
1973 /* What statments might be optimized away
1974 when their arguments are constant
1975 TODO: also trivial builtins.
1976 builtin_constant_p is already handled later. */
1977 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1978 && gimple_code (stmt
) != GIMPLE_COND
1979 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1982 /* Stores will stay anyway. */
1983 if (gimple_store_p (stmt
))
1986 is_load
= gimple_assign_load_p (stmt
);
1988 /* Loads can be optimized when the value is known. */
1992 gcc_assert (gimple_assign_single_p (stmt
));
1993 op
= gimple_assign_rhs1 (stmt
);
1994 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
2001 /* See if we understand all operands before we start
2002 adding conditionals. */
2003 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2005 tree parm
= unmodified_parm (stmt
, use
);
2006 /* For arguments we can build a condition. */
2007 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
2009 if (TREE_CODE (use
) != SSA_NAME
)
2011 /* If we know when operand is constant,
2012 we still can say something useful. */
2013 if (!true_predicate_p (&nonconstant_names
[SSA_NAME_VERSION (use
)]))
2020 add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
2022 op_non_const
= false_predicate ();
2023 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2025 tree parm
= unmodified_parm (stmt
, use
);
2028 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
2030 if (index
!= base_index
)
2031 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
2036 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2037 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
2039 if (gimple_code (stmt
) == GIMPLE_ASSIGN
2040 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
2041 nonconstant_names
[SSA_NAME_VERSION (gimple_assign_lhs (stmt
))]
2043 return op_non_const
;
2046 struct record_modified_bb_info
2052 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2053 set except for info->stmt. */
2056 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
2058 struct record_modified_bb_info
*info
=
2059 (struct record_modified_bb_info
*) data
;
2060 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2062 bitmap_set_bit (info
->bb_set
,
2063 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2064 ? ENTRY_BLOCK_PTR
->index
2065 : gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
2069 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2070 will change since last invocation of STMT.
2072 Value 0 is reserved for compile time invariants.
2073 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2074 ought to be REG_BR_PROB_BASE / estimated_iters. */
2077 param_change_prob (gimple stmt
, int i
)
2079 tree op
= gimple_call_arg (stmt
, i
);
2080 basic_block bb
= gimple_bb (stmt
);
2083 /* Global invariants neve change. */
2084 if (is_gimple_min_invariant (op
))
2086 /* We would have to do non-trivial analysis to really work out what
2087 is the probability of value to change (i.e. when init statement
2088 is in a sibling loop of the call).
2090 We do an conservative estimate: when call is executed N times more often
2091 than the statement defining value, we take the frequency 1/N. */
2092 if (TREE_CODE (op
) == SSA_NAME
)
2097 return REG_BR_PROB_BASE
;
2099 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2100 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2102 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2106 if (init_freq
< bb
->frequency
)
2107 return MAX (GCOV_COMPUTE_SCALE (init_freq
, bb
->frequency
), 1);
2109 return REG_BR_PROB_BASE
;
2112 base
= get_base_address (op
);
2117 struct record_modified_bb_info info
;
2120 tree init
= ctor_for_folding (base
);
2122 if (init
!= error_mark_node
)
2125 return REG_BR_PROB_BASE
;
2126 ao_ref_init (&refd
, op
);
2128 info
.bb_set
= BITMAP_ALLOC (NULL
);
2129 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2131 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2133 BITMAP_FREE (info
.bb_set
);
2134 return REG_BR_PROB_BASE
;
2137 /* Assume that every memory is initialized at entry.
2138 TODO: Can we easilly determine if value is always defined
2139 and thus we may skip entry block? */
2140 if (ENTRY_BLOCK_PTR
->frequency
)
2141 max
= ENTRY_BLOCK_PTR
->frequency
;
2145 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2146 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2148 BITMAP_FREE (info
.bb_set
);
2149 if (max
< bb
->frequency
)
2150 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->frequency
), 1);
2152 return REG_BR_PROB_BASE
;
2154 return REG_BR_PROB_BASE
;
2157 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2158 sub-graph and if the predicate the condition depends on is known. If so,
2159 return true and store the pointer the predicate in *P. */
2162 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2163 struct inline_summary
*summary
, basic_block bb
,
2164 struct predicate
*p
,
2165 vec
<predicate_t
> nonconstant_names
)
2169 basic_block first_bb
= NULL
;
2172 if (single_pred_p (bb
))
2174 *p
= false_predicate ();
2178 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2180 if (single_succ_p (e
->src
))
2182 if (!single_pred_p (e
->src
))
2185 first_bb
= single_pred (e
->src
);
2186 else if (single_pred (e
->src
) != first_bb
)
2193 else if (e
->src
!= first_bb
)
2201 stmt
= last_stmt (first_bb
);
2203 || gimple_code (stmt
) != GIMPLE_COND
2204 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2207 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2208 gimple_cond_lhs (stmt
),
2210 if (true_predicate_p (p
))
2216 /* Given a PHI statement in a function described by inline properties SUMMARY
2217 and *P being the predicate describing whether the selected PHI argument is
2218 known, store a predicate for the result of the PHI statement into
2219 NONCONSTANT_NAMES, if possible. */
2222 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2223 struct predicate
*p
,
2224 vec
<predicate_t
> nonconstant_names
)
2228 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2230 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2231 if (!is_gimple_min_invariant (arg
))
2233 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2234 *p
= or_predicates (summary
->conds
, p
,
2235 &nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2236 if (true_predicate_p (p
))
2241 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2243 fprintf (dump_file
, "\t\tphi predicate: ");
2244 dump_predicate (dump_file
, summary
->conds
, p
);
2246 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2249 /* Return predicate specifying when array index in access OP becomes non-constant. */
2251 static struct predicate
2252 array_index_predicate (struct inline_summary
*info
,
2253 vec
< predicate_t
> nonconstant_names
, tree op
)
2255 struct predicate p
= false_predicate ();
2256 while (handled_component_p (op
))
2258 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
2260 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2261 p
= or_predicates (info
->conds
, &p
,
2262 &nonconstant_names
[SSA_NAME_VERSION
2263 (TREE_OPERAND (op
, 1))]);
2265 op
= TREE_OPERAND (op
, 0);
2270 /* For a typical usage of __builtin_expect (a<b, 1), we
2271 may introduce an extra relation stmt:
2272 With the builtin, we have
2275 t3 = __builtin_expect (t2, 1);
2278 Without the builtin, we have
2281 This affects the size/time estimation and may have
2282 an impact on the earlier inlining.
2283 Here find this pattern and fix it up later. */
2286 find_foldable_builtin_expect (basic_block bb
)
2288 gimple_stmt_iterator bsi
;
2290 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2292 gimple stmt
= gsi_stmt (bsi
);
2293 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
))
2295 tree var
= gimple_call_lhs (stmt
);
2296 tree arg
= gimple_call_arg (stmt
, 0);
2297 use_operand_p use_p
;
2304 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2306 while (TREE_CODE (arg
) == SSA_NAME
)
2308 gimple stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2309 if (!is_gimple_assign (stmt_tmp
))
2311 switch (gimple_assign_rhs_code (stmt_tmp
))
2330 arg
= gimple_assign_rhs1 (stmt_tmp
);
2333 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2334 && gimple_code (use_stmt
) == GIMPLE_COND
)
2341 /* Compute function body size parameters for NODE.
2342 When EARLY is true, we compute only simple summaries without
2343 non-trivial predicates to drive the early inliner. */
2346 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2349 /* Estimate static overhead for function prologue/epilogue and alignment. */
2351 /* Benefits are scaled by probability of elimination that is in range
2354 gimple_stmt_iterator bsi
;
2355 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2357 struct inline_summary
*info
= inline_summary (node
);
2358 struct predicate bb_predicate
;
2359 struct ipa_node_params
*parms_info
= NULL
;
2360 vec
<predicate_t
> nonconstant_names
= vNULL
;
2363 predicate array_index
= true_predicate ();
2364 gimple fix_builtin_expect_stmt
;
2369 if (optimize
&& !early
)
2371 calculate_dominance_info (CDI_DOMINATORS
);
2372 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2374 if (ipa_node_params_vector
.exists ())
2376 parms_info
= IPA_NODE_REF (node
);
2377 nonconstant_names
.safe_grow_cleared
2378 (SSANAMES (my_function
)->length ());
2383 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2384 cgraph_node_name (node
));
2386 /* When we run into maximal number of entries, we assign everything to the
2387 constant truth case. Be sure to have it in list. */
2388 bb_predicate
= true_predicate ();
2389 account_size_time (info
, 0, 0, &bb_predicate
);
2391 bb_predicate
= not_inlined_predicate ();
2392 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2394 gcc_assert (my_function
&& my_function
->cfg
);
2396 compute_bb_predicates (node
, parms_info
, info
);
2397 gcc_assert (cfun
== my_function
);
2398 order
= XNEWVEC (int, n_basic_blocks
);
2399 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2400 for (n
= 0; n
< nblocks
; n
++)
2402 bb
= BASIC_BLOCK (order
[n
]);
2403 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
2405 /* TODO: Obviously predicates can be propagated down across CFG. */
2409 bb_predicate
= *(struct predicate
*) bb
->aux
;
2411 bb_predicate
= false_predicate ();
2414 bb_predicate
= true_predicate ();
2416 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2418 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2419 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2422 if (parms_info
&& nonconstant_names
.exists ())
2424 struct predicate phi_predicate
;
2425 bool first_phi
= true;
2427 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2430 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2435 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2437 fprintf (dump_file
, " ");
2438 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2440 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2445 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2447 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2449 gimple stmt
= gsi_stmt (bsi
);
2450 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2451 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2453 struct predicate will_be_nonconstant
;
2455 /* This relation stmt should be folded after we remove
2456 buildin_expect call. Adjust the cost here. */
2457 if (stmt
== fix_builtin_expect_stmt
)
2463 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2465 fprintf (dump_file
, " ");
2466 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2467 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2468 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2472 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2474 struct predicate this_array_index
;
2476 array_index_predicate (info
, nonconstant_names
,
2477 gimple_assign_rhs1 (stmt
));
2478 if (!false_predicate_p (&this_array_index
))
2480 and_predicates (info
->conds
, &array_index
,
2483 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2485 struct predicate this_array_index
;
2487 array_index_predicate (info
, nonconstant_names
,
2488 gimple_get_lhs (stmt
));
2489 if (!false_predicate_p (&this_array_index
))
2491 and_predicates (info
->conds
, &array_index
,
2496 if (is_gimple_call (stmt
))
2498 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2499 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2501 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2502 resolved as constant. We however don't want to optimize
2503 out the cgraph edges. */
2504 if (nonconstant_names
.exists ()
2505 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2506 && gimple_call_lhs (stmt
)
2507 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2509 struct predicate false_p
= false_predicate ();
2510 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2513 if (ipa_node_params_vector
.exists ())
2515 int count
= gimple_call_num_args (stmt
);
2519 es
->param
.safe_grow_cleared (count
);
2520 for (i
= 0; i
< count
; i
++)
2522 int prob
= param_change_prob (stmt
, i
);
2523 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2524 es
->param
[i
].change_prob
= prob
;
2528 es
->call_stmt_size
= this_size
;
2529 es
->call_stmt_time
= this_time
;
2530 es
->loop_depth
= bb_loop_depth (bb
);
2531 edge_set_predicate (edge
, &bb_predicate
);
2534 /* TODO: When conditional jump or swithc is known to be constant, but
2535 we did not translate it into the predicates, we really can account
2536 just maximum of the possible paths. */
2539 = will_be_nonconstant_predicate (parms_info
, info
,
2540 stmt
, nonconstant_names
);
2541 if (this_time
|| this_size
)
2547 prob
= eliminated_by_inlining_prob (stmt
);
2548 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2550 "\t\t50%% will be eliminated by inlining\n");
2551 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2552 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2555 p
= and_predicates (info
->conds
, &bb_predicate
,
2556 &will_be_nonconstant
);
2558 p
= true_predicate ();
2560 if (!false_predicate_p (&p
))
2564 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2565 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2568 /* We account everything but the calls. Calls have their own
2569 size/time info attached to cgraph edges. This is necessary
2570 in order to make the cost disappear after inlining. */
2571 if (!is_gimple_call (stmt
))
2575 struct predicate ip
= not_inlined_predicate ();
2576 ip
= and_predicates (info
->conds
, &ip
, &p
);
2577 account_size_time (info
, this_size
* prob
,
2578 this_time
* prob
, &ip
);
2581 account_size_time (info
, this_size
* (2 - prob
),
2582 this_time
* (2 - prob
), &p
);
2585 gcc_assert (time
>= 0);
2586 gcc_assert (size
>= 0);
2590 set_hint_predicate (&inline_summary (node
)->array_index
, array_index
);
2591 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2592 if (time
> MAX_TIME
)
2596 if (!early
&& nonconstant_names
.exists ())
2600 predicate loop_iterations
= true_predicate ();
2601 predicate loop_stride
= true_predicate ();
2603 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2604 flow_loops_dump (dump_file
, NULL
, 0);
2606 FOR_EACH_LOOP (li
, loop
, 0)
2611 struct tree_niter_desc niter_desc
;
2612 basic_block
*body
= get_loop_body (loop
);
2613 bb_predicate
= *(struct predicate
*) loop
->header
->aux
;
2615 exits
= get_loop_exit_edges (loop
);
2616 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2617 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2618 && !is_gimple_min_invariant (niter_desc
.niter
))
2620 predicate will_be_nonconstant
2621 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2624 if (!true_predicate_p (&will_be_nonconstant
))
2625 will_be_nonconstant
= and_predicates (info
->conds
,
2627 &will_be_nonconstant
);
2628 if (!true_predicate_p (&will_be_nonconstant
)
2629 && !false_predicate_p (&will_be_nonconstant
))
2630 /* This is slightly inprecise. We may want to represent each
2631 loop with independent predicate. */
2633 and_predicates (info
->conds
, &loop_iterations
,
2634 &will_be_nonconstant
);
2638 for (i
= 0; i
< loop
->num_nodes
; i
++)
2640 gimple_stmt_iterator gsi
;
2641 bb_predicate
= *(struct predicate
*) body
[i
]->aux
;
2642 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2645 gimple stmt
= gsi_stmt (gsi
);
2650 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2652 predicate will_be_nonconstant
;
2655 (loop
, loop_containing_stmt (stmt
), use
, &iv
, true)
2656 || is_gimple_min_invariant (iv
.step
))
2659 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2662 if (!true_predicate_p (&will_be_nonconstant
))
2664 = and_predicates (info
->conds
,
2666 &will_be_nonconstant
);
2667 if (!true_predicate_p (&will_be_nonconstant
)
2668 && !false_predicate_p (&will_be_nonconstant
))
2669 /* This is slightly inprecise. We may want to represent
2670 each loop with independent predicate. */
2672 and_predicates (info
->conds
, &loop_stride
,
2673 &will_be_nonconstant
);
2679 set_hint_predicate (&inline_summary (node
)->loop_iterations
,
2681 set_hint_predicate (&inline_summary (node
)->loop_stride
, loop_stride
);
2684 FOR_ALL_BB_FN (bb
, my_function
)
2690 pool_free (edge_predicate_pool
, bb
->aux
);
2692 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2695 pool_free (edge_predicate_pool
, e
->aux
);
2699 inline_summary (node
)->self_time
= time
;
2700 inline_summary (node
)->self_size
= size
;
2701 nonconstant_names
.release ();
2702 if (optimize
&& !early
)
2704 loop_optimizer_finalize ();
2705 free_dominance_info (CDI_DOMINATORS
);
2709 fprintf (dump_file
, "\n");
2710 dump_inline_summary (dump_file
, node
);
2715 /* Compute parameters of functions used by inliner.
2716 EARLY is true when we compute parameters for the early inliner */
2719 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2721 HOST_WIDE_INT self_stack_size
;
2722 struct cgraph_edge
*e
;
2723 struct inline_summary
*info
;
2725 gcc_assert (!node
->global
.inlined_to
);
2727 inline_summary_alloc ();
2729 info
= inline_summary (node
);
2730 reset_inline_summary (node
);
2732 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2733 Once this happen, we will need to more curefully predict call
2735 if (node
->thunk
.thunk_p
)
2737 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2738 struct predicate t
= true_predicate ();
2740 info
->inlinable
= 0;
2741 node
->callees
->call_stmt_cannot_inline_p
= true;
2742 node
->local
.can_change_signature
= false;
2743 es
->call_stmt_time
= 1;
2744 es
->call_stmt_size
= 1;
2745 account_size_time (info
, 0, 0, &t
);
2749 /* Even is_gimple_min_invariant rely on current_function_decl. */
2750 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2752 /* Estimate the stack size for the function if we're optimizing. */
2753 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2754 info
->estimated_self_stack_size
= self_stack_size
;
2755 info
->estimated_stack_size
= self_stack_size
;
2756 info
->stack_frame_offset
= 0;
2758 /* Can this function be inlined at all? */
2759 if (!optimize
&& !lookup_attribute ("always_inline",
2760 DECL_ATTRIBUTES (node
->decl
)))
2761 info
->inlinable
= false;
2763 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2765 /* Type attributes can use parameter indices to describe them. */
2766 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
2767 node
->local
.can_change_signature
= false;
2770 /* Otherwise, inlinable functions always can change signature. */
2771 if (info
->inlinable
)
2772 node
->local
.can_change_signature
= true;
2775 /* Functions calling builtin_apply can not change signature. */
2776 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2778 tree
cdecl = e
->callee
->decl
;
2779 if (DECL_BUILT_IN (cdecl)
2780 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2781 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2782 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2785 node
->local
.can_change_signature
= !e
;
2788 estimate_function_body_sizes (node
, early
);
2790 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2791 info
->time
= info
->self_time
;
2792 info
->size
= info
->self_size
;
2793 info
->stack_frame_offset
= 0;
2794 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2795 #ifdef ENABLE_CHECKING
2796 inline_update_overall_summary (node
);
2797 gcc_assert (info
->time
== info
->self_time
&& info
->size
== info
->self_size
);
2804 /* Compute parameters of functions used by inliner using
2805 current_function_decl. */
2808 compute_inline_parameters_for_current (void)
2810 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2816 const pass_data pass_data_inline_parameters
=
2818 GIMPLE_PASS
, /* type */
2819 "inline_param", /* name */
2820 OPTGROUP_INLINE
, /* optinfo_flags */
2821 false, /* has_gate */
2822 true, /* has_execute */
2823 TV_INLINE_PARAMETERS
, /* tv_id */
2824 0, /* properties_required */
2825 0, /* properties_provided */
2826 0, /* properties_destroyed */
2827 0, /* todo_flags_start */
2828 0, /* todo_flags_finish */
2831 class pass_inline_parameters
: public gimple_opt_pass
2834 pass_inline_parameters (gcc::context
*ctxt
)
2835 : gimple_opt_pass (pass_data_inline_parameters
, ctxt
)
2838 /* opt_pass methods: */
2839 opt_pass
* clone () { return new pass_inline_parameters (m_ctxt
); }
2840 unsigned int execute () {
2841 return compute_inline_parameters_for_current ();
2844 }; // class pass_inline_parameters
2849 make_pass_inline_parameters (gcc::context
*ctxt
)
2851 return new pass_inline_parameters (ctxt
);
2855 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2859 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2860 int *size
, int *time
,
2861 vec
<tree
> known_vals
,
2862 vec
<tree
> known_binfos
,
2863 vec
<ipa_agg_jump_function_p
> known_aggs
)
2866 struct cgraph_node
*callee
;
2867 struct inline_summary
*isummary
;
2869 if (!known_vals
.exists () && !known_binfos
.exists ())
2871 if (!flag_indirect_inlining
)
2874 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2879 /* Account for difference in cost between indirect and direct calls. */
2880 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2881 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2882 gcc_checking_assert (*time
>= 0);
2883 gcc_checking_assert (*size
>= 0);
2885 callee
= cgraph_get_node (target
);
2886 if (!callee
|| !callee
->definition
)
2888 isummary
= inline_summary (callee
);
2889 return isummary
->inlinable
;
2892 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2895 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2897 vec
<tree
> known_vals
,
2898 vec
<tree
> known_binfos
,
2899 vec
<ipa_agg_jump_function_p
> known_aggs
,
2900 inline_hints
*hints
)
2902 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2903 int call_size
= es
->call_stmt_size
;
2904 int call_time
= es
->call_stmt_time
;
2906 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2907 known_vals
, known_binfos
, known_aggs
)
2908 && hints
&& cgraph_maybe_hot_edge_p (e
))
2909 *hints
|= INLINE_HINT_indirect_call
;
2910 *size
+= call_size
* INLINE_SIZE_SCALE
;
2911 *time
+= apply_probability ((gcov_type
) call_time
, prob
)
2912 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
);
2913 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2914 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2919 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2920 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2924 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2925 inline_hints
*hints
,
2926 clause_t possible_truths
,
2927 vec
<tree
> known_vals
,
2928 vec
<tree
> known_binfos
,
2929 vec
<ipa_agg_jump_function_p
> known_aggs
)
2931 struct cgraph_edge
*e
;
2932 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2934 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2936 || evaluate_predicate (es
->predicate
, possible_truths
))
2938 if (e
->inline_failed
)
2940 /* Predicates of calls shall not use NOT_CHANGED codes,
2941 sowe do not need to compute probabilities. */
2942 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2943 known_vals
, known_binfos
,
2947 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2949 known_vals
, known_binfos
,
2953 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2955 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2957 || evaluate_predicate (es
->predicate
, possible_truths
))
2958 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2959 known_vals
, known_binfos
, known_aggs
,
2965 /* Estimate size and time needed to execute NODE assuming
2966 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2967 about NODE's arguments. */
2970 estimate_node_size_and_time (struct cgraph_node
*node
,
2971 clause_t possible_truths
,
2972 vec
<tree
> known_vals
,
2973 vec
<tree
> known_binfos
,
2974 vec
<ipa_agg_jump_function_p
> known_aggs
,
2975 int *ret_size
, int *ret_time
,
2976 inline_hints
*ret_hints
,
2977 vec
<inline_param_summary_t
>
2978 inline_param_summary
)
2980 struct inline_summary
*info
= inline_summary (node
);
2984 inline_hints hints
= 0;
2987 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2990 fprintf (dump_file
, " Estimating body: %s/%i\n"
2991 " Known to be false: ", cgraph_node_name (node
),
2994 for (i
= predicate_not_inlined_condition
;
2995 i
< (predicate_first_dynamic_condition
2996 + (int) vec_safe_length (info
->conds
)); i
++)
2997 if (!(possible_truths
& (1 << i
)))
3000 fprintf (dump_file
, ", ");
3002 dump_condition (dump_file
, info
->conds
, i
);
3006 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3007 if (evaluate_predicate (&e
->predicate
, possible_truths
))
3010 gcc_checking_assert (e
->time
>= 0);
3011 gcc_checking_assert (time
>= 0);
3012 if (!inline_param_summary
.exists ())
3016 int prob
= predicate_probability (info
->conds
,
3019 inline_param_summary
);
3020 gcc_checking_assert (prob
>= 0);
3021 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3022 time
+= apply_probability ((gcov_type
) e
->time
, prob
);
3024 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
3025 time
= MAX_TIME
* INLINE_TIME_SCALE
;
3026 gcc_checking_assert (time
>= 0);
3029 gcc_checking_assert (size
>= 0);
3030 gcc_checking_assert (time
>= 0);
3032 if (info
->loop_iterations
3033 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
3034 hints
|= INLINE_HINT_loop_iterations
;
3035 if (info
->loop_stride
3036 && !evaluate_predicate (info
->loop_stride
, possible_truths
))
3037 hints
|= INLINE_HINT_loop_stride
;
3038 if (info
->array_index
3039 && !evaluate_predicate (info
->array_index
, possible_truths
))
3040 hints
|= INLINE_HINT_array_index
;
3042 hints
|= INLINE_HINT_in_scc
;
3043 if (DECL_DECLARED_INLINE_P (node
->decl
))
3044 hints
|= INLINE_HINT_declared_inline
;
3046 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
3047 known_vals
, known_binfos
, known_aggs
);
3048 gcc_checking_assert (size
>= 0);
3049 gcc_checking_assert (time
>= 0);
3050 time
= RDIV (time
, INLINE_TIME_SCALE
);
3051 size
= RDIV (size
, INLINE_SIZE_SCALE
);
3053 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3054 fprintf (dump_file
, "\n size:%i time:%i\n", (int) size
, (int) time
);
3065 /* Estimate size and time needed to execute callee of EDGE assuming that
3066 parameters known to be constant at caller of EDGE are propagated.
3067 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
3068 and types for parameters. */
3071 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
3072 vec
<tree
> known_vals
,
3073 vec
<tree
> known_binfos
,
3074 vec
<ipa_agg_jump_function_p
> known_aggs
,
3075 int *ret_size
, int *ret_time
,
3076 inline_hints
*hints
)
3080 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
,
3082 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
,
3083 known_aggs
, ret_size
, ret_time
, hints
, vNULL
);
3086 /* Translate all conditions from callee representation into caller
3087 representation and symbolically evaluate predicate P into new predicate.
3089 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
3090 is summary of function predicate P is from. OPERAND_MAP is array giving
3091 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
3092 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
3093 predicate under which callee is executed. OFFSET_MAP is an array of of
3094 offsets that need to be added to conditions, negative offset means that
3095 conditions relying on values passed by reference have to be discarded
3096 because they might not be preserved (and should be considered offset zero
3097 for other purposes). */
3099 static struct predicate
3100 remap_predicate (struct inline_summary
*info
,
3101 struct inline_summary
*callee_info
,
3102 struct predicate
*p
,
3103 vec
<int> operand_map
,
3104 vec
<int> offset_map
,
3105 clause_t possible_truths
, struct predicate
*toplev_predicate
)
3108 struct predicate out
= true_predicate ();
3110 /* True predicate is easy. */
3111 if (true_predicate_p (p
))
3112 return *toplev_predicate
;
3113 for (i
= 0; p
->clause
[i
]; i
++)
3115 clause_t clause
= p
->clause
[i
];
3117 struct predicate clause_predicate
= false_predicate ();
3119 gcc_assert (i
< MAX_CLAUSES
);
3121 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
3122 /* Do we have condition we can't disprove? */
3123 if (clause
& possible_truths
& (1 << cond
))
3125 struct predicate cond_predicate
;
3126 /* Work out if the condition can translate to predicate in the
3127 inlined function. */
3128 if (cond
>= predicate_first_dynamic_condition
)
3130 struct condition
*c
;
3132 c
= &(*callee_info
->conds
)[cond
3134 predicate_first_dynamic_condition
];
3135 /* See if we can remap condition operand to caller's operand.
3136 Otherwise give up. */
3137 if (!operand_map
.exists ()
3138 || (int) operand_map
.length () <= c
->operand_num
3139 || operand_map
[c
->operand_num
] == -1
3140 /* TODO: For non-aggregate conditions, adding an offset is
3141 basically an arithmetic jump function processing which
3142 we should support in future. */
3143 || ((!c
->agg_contents
|| !c
->by_ref
)
3144 && offset_map
[c
->operand_num
] > 0)
3145 || (c
->agg_contents
&& c
->by_ref
3146 && offset_map
[c
->operand_num
] < 0))
3147 cond_predicate
= true_predicate ();
3150 struct agg_position_info ap
;
3151 HOST_WIDE_INT offset_delta
= offset_map
[c
->operand_num
];
3152 if (offset_delta
< 0)
3154 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
3157 gcc_assert (!c
->agg_contents
3158 || c
->by_ref
|| offset_delta
== 0);
3159 ap
.offset
= c
->offset
+ offset_delta
;
3160 ap
.agg_contents
= c
->agg_contents
;
3161 ap
.by_ref
= c
->by_ref
;
3162 cond_predicate
= add_condition (info
,
3163 operand_map
[c
->operand_num
],
3164 &ap
, c
->code
, c
->val
);
3167 /* Fixed conditions remains same, construct single
3168 condition predicate. */
3171 cond_predicate
.clause
[0] = 1 << cond
;
3172 cond_predicate
.clause
[1] = 0;
3174 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
3177 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
3179 return and_predicates (info
->conds
, &out
, toplev_predicate
);
3183 /* Update summary information of inline clones after inlining.
3184 Compute peak stack usage. */
3187 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3189 struct cgraph_edge
*e
;
3190 struct inline_summary
*callee_info
= inline_summary (node
);
3191 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
3194 callee_info
->stack_frame_offset
3195 = caller_info
->stack_frame_offset
3196 + caller_info
->estimated_self_stack_size
;
3197 peak
= callee_info
->stack_frame_offset
3198 + callee_info
->estimated_self_stack_size
;
3199 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
3200 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
3201 ipa_propagate_frequency (node
);
3202 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3204 if (!e
->inline_failed
)
3205 inline_update_callee_summaries (e
->callee
, depth
);
3206 inline_edge_summary (e
)->loop_depth
+= depth
;
3208 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3209 inline_edge_summary (e
)->loop_depth
+= depth
;
3212 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3213 When functoin A is inlined in B and A calls C with parameter that
3214 changes with probability PROB1 and C is known to be passthroug
3215 of argument if B that change with probability PROB2, the probability
3216 of change is now PROB1*PROB2. */
3219 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3220 struct cgraph_edge
*edge
)
3222 if (ipa_node_params_vector
.exists ())
3225 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3226 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3227 struct inline_edge_summary
*inlined_es
3228 = inline_edge_summary (inlined_edge
);
3230 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3232 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3233 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3234 && (ipa_get_jf_pass_through_formal_id (jfunc
)
3235 < (int) inlined_es
->param
.length ()))
3237 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
3238 int prob1
= es
->param
[i
].change_prob
;
3239 int prob2
= inlined_es
->param
[jf_formal_id
].change_prob
;
3240 int prob
= combine_probabilities (prob1
, prob2
);
3242 if (prob1
&& prob2
&& !prob
)
3245 es
->param
[i
].change_prob
= prob
;
3251 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3253 Remap predicates of callees of NODE. Rest of arguments match
3256 Also update change probabilities. */
3259 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3260 struct cgraph_node
*node
,
3261 struct inline_summary
*info
,
3262 struct inline_summary
*callee_info
,
3263 vec
<int> operand_map
,
3264 vec
<int> offset_map
,
3265 clause_t possible_truths
,
3266 struct predicate
*toplev_predicate
)
3268 struct cgraph_edge
*e
;
3269 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3271 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3274 if (e
->inline_failed
)
3276 remap_edge_change_prob (inlined_edge
, e
);
3280 p
= remap_predicate (info
, callee_info
,
3281 es
->predicate
, operand_map
, offset_map
,
3282 possible_truths
, toplev_predicate
);
3283 edge_set_predicate (e
, &p
);
3284 /* TODO: We should remove the edge for code that will be
3285 optimized out, but we need to keep verifiers and tree-inline
3286 happy. Make it cold for now. */
3287 if (false_predicate_p (&p
))
3294 edge_set_predicate (e
, toplev_predicate
);
3297 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3298 operand_map
, offset_map
, possible_truths
,
3301 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3303 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3306 remap_edge_change_prob (inlined_edge
, e
);
3309 p
= remap_predicate (info
, callee_info
,
3310 es
->predicate
, operand_map
, offset_map
,
3311 possible_truths
, toplev_predicate
);
3312 edge_set_predicate (e
, &p
);
3313 /* TODO: We should remove the edge for code that will be optimized
3314 out, but we need to keep verifiers and tree-inline happy.
3315 Make it cold for now. */
3316 if (false_predicate_p (&p
))
3323 edge_set_predicate (e
, toplev_predicate
);
3327 /* Same as remap_predicate, but set result into hint *HINT. */
3330 remap_hint_predicate (struct inline_summary
*info
,
3331 struct inline_summary
*callee_info
,
3332 struct predicate
**hint
,
3333 vec
<int> operand_map
,
3334 vec
<int> offset_map
,
3335 clause_t possible_truths
,
3336 struct predicate
*toplev_predicate
)
3342 p
= remap_predicate (info
, callee_info
,
3344 operand_map
, offset_map
,
3345 possible_truths
, toplev_predicate
);
3346 if (!false_predicate_p (&p
) && !true_predicate_p (&p
))
3349 set_hint_predicate (hint
, p
);
3351 **hint
= and_predicates (info
->conds
, *hint
, &p
);
3355 /* We inlined EDGE. Update summary of the function we inlined into. */
3358 inline_merge_summary (struct cgraph_edge
*edge
)
3360 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3361 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3362 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3363 struct inline_summary
*info
= inline_summary (to
);
3364 clause_t clause
= 0; /* not_inline is known to be false. */
3366 vec
<int> operand_map
= vNULL
;
3367 vec
<int> offset_map
= vNULL
;
3369 struct predicate toplev_predicate
;
3370 struct predicate true_p
= true_predicate ();
3371 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3374 toplev_predicate
= *es
->predicate
;
3376 toplev_predicate
= true_predicate ();
3378 if (ipa_node_params_vector
.exists () && callee_info
->conds
)
3380 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3381 int count
= ipa_get_cs_argument_count (args
);
3384 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3387 operand_map
.safe_grow_cleared (count
);
3388 offset_map
.safe_grow_cleared (count
);
3390 for (i
= 0; i
< count
; i
++)
3392 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3395 /* TODO: handle non-NOPs when merging. */
3396 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3398 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3399 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3400 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3403 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3405 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3406 if (offset
>= 0 && offset
< INT_MAX
)
3408 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3409 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3411 offset_map
[i
] = offset
;
3414 operand_map
[i
] = map
;
3415 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3418 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3420 struct predicate p
= remap_predicate (info
, callee_info
,
3421 &e
->predicate
, operand_map
,
3424 if (!false_predicate_p (&p
))
3426 gcov_type add_time
= ((gcov_type
) e
->time
* edge
->frequency
3427 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3428 int prob
= predicate_probability (callee_info
->conds
,
3431 add_time
= apply_probability ((gcov_type
) add_time
, prob
);
3432 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3433 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3434 if (prob
!= REG_BR_PROB_BASE
3435 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3437 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3438 (double) prob
/ REG_BR_PROB_BASE
);
3440 account_size_time (info
, e
->size
, add_time
, &p
);
3443 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3444 offset_map
, clause
, &toplev_predicate
);
3445 remap_hint_predicate (info
, callee_info
,
3446 &callee_info
->loop_iterations
,
3447 operand_map
, offset_map
, clause
, &toplev_predicate
);
3448 remap_hint_predicate (info
, callee_info
,
3449 &callee_info
->loop_stride
,
3450 operand_map
, offset_map
, clause
, &toplev_predicate
);
3451 remap_hint_predicate (info
, callee_info
,
3452 &callee_info
->array_index
,
3453 operand_map
, offset_map
, clause
, &toplev_predicate
);
3455 inline_update_callee_summaries (edge
->callee
,
3456 inline_edge_summary (edge
)->loop_depth
);
3458 /* We do not maintain predicates of inlined edges, free it. */
3459 edge_set_predicate (edge
, &true_p
);
3460 /* Similarly remove param summaries. */
3461 es
->param
.release ();
3462 operand_map
.release ();
3463 offset_map
.release ();
3466 /* For performance reasons inline_merge_summary is not updating overall size
3467 and time. Recompute it. */
3470 inline_update_overall_summary (struct cgraph_node
*node
)
3472 struct inline_summary
*info
= inline_summary (node
);
3478 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3480 info
->size
+= e
->size
, info
->time
+= e
->time
;
3481 if (info
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
3482 info
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
3484 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3485 ~(clause_t
) (1 << predicate_false_condition
),
3486 vNULL
, vNULL
, vNULL
);
3487 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3488 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3491 /* Return hints derrived from EDGE. */
3493 simple_edge_hints (struct cgraph_edge
*edge
)
3496 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3497 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3498 if (inline_summary (to
)->scc_no
3499 && inline_summary (to
)->scc_no
== inline_summary (edge
->callee
)->scc_no
3500 && !cgraph_edge_recursive_p (edge
))
3501 hints
|= INLINE_HINT_same_scc
;
3503 if (to
->lto_file_data
&& edge
->callee
->lto_file_data
3504 && to
->lto_file_data
!= edge
->callee
->lto_file_data
)
3505 hints
|= INLINE_HINT_cross_module
;
3510 /* Estimate the time cost for the caller when inlining EDGE.
3511 Only to be called via estimate_edge_time, that handles the
3514 When caching, also update the cache entry. Compute both time and
3515 size, since we always need both metrics eventually. */
3518 do_estimate_edge_time (struct cgraph_edge
*edge
)
3523 struct cgraph_node
*callee
;
3525 vec
<tree
> known_vals
;
3526 vec
<tree
> known_binfos
;
3527 vec
<ipa_agg_jump_function_p
> known_aggs
;
3528 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3530 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3532 gcc_checking_assert (edge
->inline_failed
);
3533 evaluate_properties_for_edge (edge
, true,
3534 &clause
, &known_vals
, &known_binfos
,
3536 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3537 known_aggs
, &size
, &time
, &hints
, es
->param
);
3538 known_vals
.release ();
3539 known_binfos
.release ();
3540 known_aggs
.release ();
3541 gcc_checking_assert (size
>= 0);
3542 gcc_checking_assert (time
>= 0);
3544 /* When caching, update the cache entry. */
3545 if (edge_growth_cache
.exists ())
3547 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3548 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
3549 edge_growth_cache
[edge
->uid
].time
= time
+ (time
>= 0);
3551 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3552 hints
|= simple_edge_hints (edge
);
3553 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3559 /* Return estimated callee growth after inlining EDGE.
3560 Only to be called via estimate_edge_size. */
3563 do_estimate_edge_size (struct cgraph_edge
*edge
)
3566 struct cgraph_node
*callee
;
3568 vec
<tree
> known_vals
;
3569 vec
<tree
> known_binfos
;
3570 vec
<ipa_agg_jump_function_p
> known_aggs
;
3572 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3574 if (edge_growth_cache
.exists ())
3576 do_estimate_edge_time (edge
);
3577 size
= edge_growth_cache
[edge
->uid
].size
;
3578 gcc_checking_assert (size
);
3579 return size
- (size
> 0);
3582 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3584 /* Early inliner runs without caching, go ahead and do the dirty work. */
3585 gcc_checking_assert (edge
->inline_failed
);
3586 evaluate_properties_for_edge (edge
, true,
3587 &clause
, &known_vals
, &known_binfos
,
3589 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3590 known_aggs
, &size
, NULL
, NULL
, vNULL
);
3591 known_vals
.release ();
3592 known_binfos
.release ();
3593 known_aggs
.release ();
3598 /* Estimate the growth of the caller when inlining EDGE.
3599 Only to be called via estimate_edge_size. */
3602 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3605 struct cgraph_node
*callee
;
3607 vec
<tree
> known_vals
;
3608 vec
<tree
> known_binfos
;
3609 vec
<ipa_agg_jump_function_p
> known_aggs
;
3611 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3613 if (edge_growth_cache
.exists ())
3615 do_estimate_edge_time (edge
);
3616 hints
= edge_growth_cache
[edge
->uid
].hints
;
3617 gcc_checking_assert (hints
);
3621 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3623 /* Early inliner runs without caching, go ahead and do the dirty work. */
3624 gcc_checking_assert (edge
->inline_failed
);
3625 evaluate_properties_for_edge (edge
, true,
3626 &clause
, &known_vals
, &known_binfos
,
3628 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3629 known_aggs
, NULL
, NULL
, &hints
, vNULL
);
3630 known_vals
.release ();
3631 known_binfos
.release ();
3632 known_aggs
.release ();
3633 hints
|= simple_edge_hints (edge
);
3638 /* Estimate self time of the function NODE after inlining EDGE. */
3641 estimate_time_after_inlining (struct cgraph_node
*node
,
3642 struct cgraph_edge
*edge
)
3644 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3645 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3648 inline_summary (node
)->time
+ estimate_edge_time (edge
);
3651 if (time
> MAX_TIME
)
3655 return inline_summary (node
)->time
;
3659 /* Estimate the size of NODE after inlining EDGE which should be an
3660 edge to either NODE or a call inlined into NODE. */
3663 estimate_size_after_inlining (struct cgraph_node
*node
,
3664 struct cgraph_edge
*edge
)
3666 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3667 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3669 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3670 gcc_assert (size
>= 0);
3673 return inline_summary (node
)->size
;
3679 struct cgraph_node
*node
;
3680 bool self_recursive
;
3685 /* Worker for do_estimate_growth. Collect growth for all callers. */
3688 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3690 struct cgraph_edge
*e
;
3691 struct growth_data
*d
= (struct growth_data
*) data
;
3693 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3695 gcc_checking_assert (e
->inline_failed
);
3697 if (e
->caller
== d
->node
3698 || (e
->caller
->global
.inlined_to
3699 && e
->caller
->global
.inlined_to
== d
->node
))
3700 d
->self_recursive
= true;
3701 d
->growth
+= estimate_edge_growth (e
);
3707 /* Estimate the growth caused by inlining NODE into all callees. */
3710 do_estimate_growth (struct cgraph_node
*node
)
3712 struct growth_data d
= { node
, 0, false };
3713 struct inline_summary
*info
= inline_summary (node
);
3715 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3717 /* For self recursive functions the growth estimation really should be
3718 infinity. We don't want to return very large values because the growth
3719 plays various roles in badness computation fractions. Be sure to not
3720 return zero or negative growths. */
3721 if (d
.self_recursive
)
3722 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3723 else if (DECL_EXTERNAL (node
->decl
))
3727 if (cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3728 d
.growth
-= info
->size
;
3729 /* COMDAT functions are very often not shared across multiple units
3730 since they come from various template instantiations.
3731 Take this into account. */
3732 else if (DECL_COMDAT (node
->decl
)
3733 && cgraph_can_remove_if_no_direct_calls_p (node
))
3734 d
.growth
-= (info
->size
3735 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3739 if (node_growth_cache
.exists ())
3741 if ((int) node_growth_cache
.length () <= node
->uid
)
3742 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
3743 node_growth_cache
[node
->uid
] = d
.growth
+ (d
.growth
>= 0);
3749 /* This function performs intraprocedural analysis in NODE that is required to
3750 inline indirect calls. */
3753 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3755 ipa_analyze_node (node
);
3756 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3758 ipa_print_node_params (dump_file
, node
);
3759 ipa_print_node_jump_functions (dump_file
, node
);
3764 /* Note function body size. */
3767 inline_analyze_function (struct cgraph_node
*node
)
3769 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3772 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3773 cgraph_node_name (node
), node
->order
);
3774 if (optimize
&& !node
->thunk
.thunk_p
)
3775 inline_indirect_intraprocedural_analysis (node
);
3776 compute_inline_parameters (node
, false);
3779 struct cgraph_edge
*e
;
3780 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3782 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3783 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3784 e
->call_stmt_cannot_inline_p
= true;
3786 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3788 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3789 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3790 e
->call_stmt_cannot_inline_p
= true;
3798 /* Called when new function is inserted to callgraph late. */
3801 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3803 inline_analyze_function (node
);
3807 /* Note function body size. */
3810 inline_generate_summary (void)
3812 struct cgraph_node
*node
;
3814 /* When not optimizing, do not bother to analyze. Inlining is still done
3815 because edge redirection needs to happen there. */
3816 if (!optimize
&& !flag_lto
&& !flag_wpa
)
3819 function_insertion_hook_holder
=
3820 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3822 ipa_register_cgraph_hooks ();
3823 inline_free_summary ();
3825 FOR_EACH_DEFINED_FUNCTION (node
)
3827 inline_analyze_function (node
);
3831 /* Read predicate from IB. */
3833 static struct predicate
3834 read_predicate (struct lto_input_block
*ib
)
3836 struct predicate out
;
3842 gcc_assert (k
<= MAX_CLAUSES
);
3843 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3847 /* Zero-initialize the remaining clauses in OUT. */
3848 while (k
<= MAX_CLAUSES
)
3849 out
.clause
[k
++] = 0;
3855 /* Write inline summary for edge E to OB. */
3858 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3860 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3864 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3865 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3866 es
->loop_depth
= streamer_read_uhwi (ib
);
3867 p
= read_predicate (ib
);
3868 edge_set_predicate (e
, &p
);
3869 length
= streamer_read_uhwi (ib
);
3872 es
->param
.safe_grow_cleared (length
);
3873 for (i
= 0; i
< length
; i
++)
3874 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3879 /* Stream in inline summaries from the section. */
3882 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3885 const struct lto_function_header
*header
=
3886 (const struct lto_function_header
*) data
;
3887 const int cfg_offset
= sizeof (struct lto_function_header
);
3888 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3889 const int string_offset
= main_offset
+ header
->main_size
;
3890 struct data_in
*data_in
;
3891 struct lto_input_block ib
;
3892 unsigned int i
, count2
, j
;
3893 unsigned int f_count
;
3895 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3899 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3900 header
->string_size
, vNULL
);
3901 f_count
= streamer_read_uhwi (&ib
);
3902 for (i
= 0; i
< f_count
; i
++)
3905 struct cgraph_node
*node
;
3906 struct inline_summary
*info
;
3907 lto_symtab_encoder_t encoder
;
3908 struct bitpack_d bp
;
3909 struct cgraph_edge
*e
;
3912 index
= streamer_read_uhwi (&ib
);
3913 encoder
= file_data
->symtab_node_encoder
;
3914 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3915 info
= inline_summary (node
);
3917 info
->estimated_stack_size
3918 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3919 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3920 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3922 bp
= streamer_read_bitpack (&ib
);
3923 info
->inlinable
= bp_unpack_value (&bp
, 1);
3925 count2
= streamer_read_uhwi (&ib
);
3926 gcc_assert (!info
->conds
);
3927 for (j
= 0; j
< count2
; j
++)
3930 c
.operand_num
= streamer_read_uhwi (&ib
);
3931 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3932 c
.val
= stream_read_tree (&ib
, data_in
);
3933 bp
= streamer_read_bitpack (&ib
);
3934 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3935 c
.by_ref
= bp_unpack_value (&bp
, 1);
3937 c
.offset
= streamer_read_uhwi (&ib
);
3938 vec_safe_push (info
->conds
, c
);
3940 count2
= streamer_read_uhwi (&ib
);
3941 gcc_assert (!info
->entry
);
3942 for (j
= 0; j
< count2
; j
++)
3944 struct size_time_entry e
;
3946 e
.size
= streamer_read_uhwi (&ib
);
3947 e
.time
= streamer_read_uhwi (&ib
);
3948 e
.predicate
= read_predicate (&ib
);
3950 vec_safe_push (info
->entry
, e
);
3953 p
= read_predicate (&ib
);
3954 set_hint_predicate (&info
->loop_iterations
, p
);
3955 p
= read_predicate (&ib
);
3956 set_hint_predicate (&info
->loop_stride
, p
);
3957 p
= read_predicate (&ib
);
3958 set_hint_predicate (&info
->array_index
, p
);
3959 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3960 read_inline_edge_summary (&ib
, e
);
3961 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3962 read_inline_edge_summary (&ib
, e
);
3965 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3967 lto_data_in_delete (data_in
);
3971 /* Read inline summary. Jump functions are shared among ipa-cp
3972 and inliner, so when ipa-cp is active, we don't need to write them
3976 inline_read_summary (void)
3978 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3979 struct lto_file_decl_data
*file_data
;
3982 inline_summary_alloc ();
3984 while ((file_data
= file_data_vec
[j
++]))
3987 const char *data
= lto_get_section_data (file_data
,
3988 LTO_section_inline_summary
,
3991 inline_read_section (file_data
, data
, len
);
3993 /* Fatal error here. We do not want to support compiling ltrans units
3994 with different version of compiler or different flags than the WPA
3995 unit, so this should never happen. */
3996 fatal_error ("ipa inline summary is missing in input file");
4000 ipa_register_cgraph_hooks ();
4002 ipa_prop_read_jump_functions ();
4004 function_insertion_hook_holder
=
4005 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
4009 /* Write predicate P to OB. */
4012 write_predicate (struct output_block
*ob
, struct predicate
*p
)
4016 for (j
= 0; p
->clause
[j
]; j
++)
4018 gcc_assert (j
< MAX_CLAUSES
);
4019 streamer_write_uhwi (ob
, p
->clause
[j
]);
4021 streamer_write_uhwi (ob
, 0);
4025 /* Write inline summary for edge E to OB. */
4028 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4030 struct inline_edge_summary
*es
= inline_edge_summary (e
);
4033 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4034 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4035 streamer_write_uhwi (ob
, es
->loop_depth
);
4036 write_predicate (ob
, es
->predicate
);
4037 streamer_write_uhwi (ob
, es
->param
.length ());
4038 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4039 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4043 /* Write inline summary for node in SET.
4044 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4045 active, we don't need to write them twice. */
4048 inline_write_summary (void)
4050 struct cgraph_node
*node
;
4051 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
4052 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4053 unsigned int count
= 0;
4056 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4058 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
4059 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4060 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4063 streamer_write_uhwi (ob
, count
);
4065 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4067 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
4068 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4069 if (cnode
&& (node
= cnode
)->definition
&& !node
->alias
)
4071 struct inline_summary
*info
= inline_summary (node
);
4072 struct bitpack_d bp
;
4073 struct cgraph_edge
*edge
;
4076 struct condition
*c
;
4078 streamer_write_uhwi (ob
,
4079 lto_symtab_encoder_encode (encoder
,
4082 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
4083 streamer_write_hwi (ob
, info
->self_size
);
4084 streamer_write_hwi (ob
, info
->self_time
);
4085 bp
= bitpack_create (ob
->main_stream
);
4086 bp_pack_value (&bp
, info
->inlinable
, 1);
4087 streamer_write_bitpack (&bp
);
4088 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4089 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4091 streamer_write_uhwi (ob
, c
->operand_num
);
4092 streamer_write_uhwi (ob
, c
->code
);
4093 stream_write_tree (ob
, c
->val
, true);
4094 bp
= bitpack_create (ob
->main_stream
);
4095 bp_pack_value (&bp
, c
->agg_contents
, 1);
4096 bp_pack_value (&bp
, c
->by_ref
, 1);
4097 streamer_write_bitpack (&bp
);
4098 if (c
->agg_contents
)
4099 streamer_write_uhwi (ob
, c
->offset
);
4101 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
4102 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
4104 streamer_write_uhwi (ob
, e
->size
);
4105 streamer_write_uhwi (ob
, e
->time
);
4106 write_predicate (ob
, &e
->predicate
);
4108 write_predicate (ob
, info
->loop_iterations
);
4109 write_predicate (ob
, info
->loop_stride
);
4110 write_predicate (ob
, info
->array_index
);
4111 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
4112 write_inline_edge_summary (ob
, edge
);
4113 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4114 write_inline_edge_summary (ob
, edge
);
4117 streamer_write_char_stream (ob
->main_stream
, 0);
4118 produce_asm (ob
, NULL
);
4119 destroy_output_block (ob
);
4121 if (optimize
&& !flag_ipa_cp
)
4122 ipa_prop_write_jump_functions ();
4126 /* Release inline summary. */
4129 inline_free_summary (void)
4131 struct cgraph_node
*node
;
4132 if (!inline_edge_summary_vec
.exists ())
4134 FOR_EACH_DEFINED_FUNCTION (node
)
4135 reset_inline_summary (node
);
4136 if (function_insertion_hook_holder
)
4137 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
4138 function_insertion_hook_holder
= NULL
;
4139 if (node_removal_hook_holder
)
4140 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
4141 node_removal_hook_holder
= NULL
;
4142 if (edge_removal_hook_holder
)
4143 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
4144 edge_removal_hook_holder
= NULL
;
4145 if (node_duplication_hook_holder
)
4146 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
4147 node_duplication_hook_holder
= NULL
;
4148 if (edge_duplication_hook_holder
)
4149 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
4150 edge_duplication_hook_holder
= NULL
;
4151 vec_free (inline_summary_vec
);
4152 inline_edge_summary_vec
.release ();
4153 if (edge_predicate_pool
)
4154 free_alloc_pool (edge_predicate_pool
);
4155 edge_predicate_pool
= 0;