1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis used by the inliner and other passes limiting code size growth.
23 We estimate for each function
25 - average function execution time
26 - inlining size benefit (that is how much of function body size
27 and its call sequence is expected to disappear by inlining)
28 - inlining time benefit
31 - call statement size and time
33 inlinie_summary datastructures store above information locally (i.e.
34 parameters of the function itself) and globally (i.e. parameters of
35 the function created by applying all the inline decisions already
36 present in the callgraph).
38 We provide accestor to the inline_summary datastructure and
39 basic logic updating the parameters when inlining is performed.
41 The summaries are context sensitive. Context means
42 1) partial assignment of known constant values of operands
43 2) whether function is inlined into the call or not.
44 It is easy to add more variants. To represent function size and time
45 that depends on context (i.e. it is known to be optimized away when
46 context is known either by inlining or from IP-CP and clonning),
47 we use predicates. Predicates are logical formulas in
48 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
49 specifying what conditions must be true. Conditions are simple test
50 of the form described above.
52 In order to make predicate (possibly) true, all of its clauses must
53 be (possibly) true. To make clause (possibly) true, one of conditions
54 it mentions must be (possibly) true. There are fixed bounds on
55 number of clauses and conditions and all the manipulation functions
56 are conservative in positive direction. I.e. we may lose precision
57 by thinking that predicate may be true even when it is not.
59 estimate_edge_size and estimate_edge_growth can be used to query
60 function size/time in the given context. inline_merge_summary merges
61 properties of caller and callee after inlining.
63 Finally pass_inline_parameters is exported. This is used to drive
64 computation of function parameters used by the early inliner. IPA
65 inlined performs analysis via its analyze_function method. */
69 #include "coretypes.h"
72 #include "tree-inline.h"
73 #include "langhooks.h"
75 #include "diagnostic.h"
76 #include "gimple-pretty-print.h"
78 #include "tree-pass.h"
82 #include "gimple-iterator.h"
83 #include "gimple-ssa.h"
85 #include "tree-phinodes.h"
86 #include "ssa-iterators.h"
87 #include "tree-ssanames.h"
88 #include "tree-ssa-loop-niter.h"
89 #include "tree-ssa-loop.h"
91 #include "lto-streamer.h"
92 #include "data-streamer.h"
93 #include "tree-streamer.h"
94 #include "ipa-inline.h"
95 #include "alloc-pool.h"
97 #include "tree-scalar-evolution.h"
98 #include "ipa-utils.h"
100 #include "cfgexpand.h"
102 /* Estimate runtime of function can easilly run into huge numbers with many
103 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
104 integer. For anything larger we use gcov_type. */
105 #define MAX_TIME 500000
107 /* Number of bits in integer, but we really want to be stable across different
109 #define NUM_CONDITIONS 32
111 enum predicate_conditions
113 predicate_false_condition
= 0,
114 predicate_not_inlined_condition
= 1,
115 predicate_first_dynamic_condition
= 2
118 /* Special condition code we use to represent test that operand is compile time
120 #define IS_NOT_CONSTANT ERROR_MARK
121 /* Special condition code we use to represent test that operand is not changed
122 across invocation of the function. When operand IS_NOT_CONSTANT it is always
123 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
124 of executions even when they are not compile time constants. */
125 #define CHANGED IDENTIFIER_NODE
127 /* Holders of ipa cgraph hooks: */
128 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
129 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
130 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
131 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
132 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
133 static void inline_node_removal_hook (struct cgraph_node
*, void *);
134 static void inline_node_duplication_hook (struct cgraph_node
*,
135 struct cgraph_node
*, void *);
136 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
137 static void inline_edge_duplication_hook (struct cgraph_edge
*,
138 struct cgraph_edge
*, void *);
140 /* VECtor holding inline summaries.
141 In GGC memory because conditions might point to constant trees. */
142 vec
<inline_summary_t
, va_gc
> *inline_summary_vec
;
143 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
145 /* Cached node/edge growths. */
146 vec
<int> node_growth_cache
;
147 vec
<edge_growth_cache_entry
> edge_growth_cache
;
149 /* Edge predicates goes here. */
150 static alloc_pool edge_predicate_pool
;
152 /* Return true predicate (tautology).
153 We represent it by empty list of clauses. */
155 static inline struct predicate
156 true_predicate (void)
164 /* Return predicate testing single condition number COND. */
166 static inline struct predicate
167 single_cond_predicate (int cond
)
170 p
.clause
[0] = 1 << cond
;
176 /* Return false predicate. First clause require false condition. */
178 static inline struct predicate
179 false_predicate (void)
181 return single_cond_predicate (predicate_false_condition
);
185 /* Return true if P is (false). */
188 true_predicate_p (struct predicate
*p
)
190 return !p
->clause
[0];
194 /* Return true if P is (false). */
197 false_predicate_p (struct predicate
*p
)
199 if (p
->clause
[0] == (1 << predicate_false_condition
))
201 gcc_checking_assert (!p
->clause
[1]
202 && p
->clause
[0] == 1 << predicate_false_condition
);
209 /* Return predicate that is set true when function is not inlined. */
211 static inline struct predicate
212 not_inlined_predicate (void)
214 return single_cond_predicate (predicate_not_inlined_condition
);
217 /* Simple description of whether a memory load or a condition refers to a load
218 from an aggregate and if so, how and where from in the aggregate.
219 Individual fields have the same meaning like fields with the same name in
222 struct agg_position_info
224 HOST_WIDE_INT offset
;
229 /* Add condition to condition list CONDS. AGGPOS describes whether the used
230 oprand is loaded from an aggregate and where in the aggregate it is. It can
231 be NULL, which means this not a load from an aggregate. */
233 static struct predicate
234 add_condition (struct inline_summary
*summary
, int operand_num
,
235 struct agg_position_info
*aggpos
,
236 enum tree_code code
, tree val
)
240 struct condition new_cond
;
241 HOST_WIDE_INT offset
;
242 bool agg_contents
, by_ref
;
246 offset
= aggpos
->offset
;
247 agg_contents
= aggpos
->agg_contents
;
248 by_ref
= aggpos
->by_ref
;
253 agg_contents
= false;
257 gcc_checking_assert (operand_num
>= 0);
258 for (i
= 0; vec_safe_iterate (summary
->conds
, i
, &c
); i
++)
260 if (c
->operand_num
== operand_num
263 && c
->agg_contents
== agg_contents
264 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
265 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
267 /* Too many conditions. Give up and return constant true. */
268 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
269 return true_predicate ();
271 new_cond
.operand_num
= operand_num
;
272 new_cond
.code
= code
;
274 new_cond
.agg_contents
= agg_contents
;
275 new_cond
.by_ref
= by_ref
;
276 new_cond
.offset
= offset
;
277 vec_safe_push (summary
->conds
, new_cond
);
278 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
282 /* Add clause CLAUSE into the predicate P. */
285 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
289 int insert_here
= -1;
296 /* False clause makes the whole predicate false. Kill the other variants. */
297 if (clause
== (1 << predicate_false_condition
))
299 p
->clause
[0] = (1 << predicate_false_condition
);
303 if (false_predicate_p (p
))
306 /* No one should be sily enough to add false into nontrivial clauses. */
307 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
309 /* Look where to insert the clause. At the same time prune out
310 clauses of P that are implied by the new clause and thus
312 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
314 p
->clause
[i2
] = p
->clause
[i
];
319 /* If p->clause[i] implies clause, there is nothing to add. */
320 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
322 /* We had nothing to add, none of clauses should've become
324 gcc_checking_assert (i
== i2
);
328 if (p
->clause
[i
] < clause
&& insert_here
< 0)
331 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
332 Otherwise the p->clause[i] has to stay. */
333 if ((p
->clause
[i
] & clause
) != clause
)
337 /* Look for clauses that are obviously true. I.e.
338 op0 == 5 || op0 != 5. */
339 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
342 if (!(clause
& (1 << c1
)))
344 cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
345 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
346 and thus there is no point for looking for them. */
347 if (cc1
->code
== CHANGED
|| cc1
->code
== IS_NOT_CONSTANT
)
349 for (c2
= c1
+ 1; c2
< NUM_CONDITIONS
; c2
++)
350 if (clause
& (1 << c2
))
353 &(*conditions
)[c1
- predicate_first_dynamic_condition
];
355 &(*conditions
)[c2
- predicate_first_dynamic_condition
];
356 if (cc1
->operand_num
== cc2
->operand_num
357 && cc1
->val
== cc2
->val
358 && cc2
->code
!= IS_NOT_CONSTANT
359 && cc2
->code
!= CHANGED
360 && cc1
->code
== invert_tree_comparison
362 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
368 /* We run out of variants. Be conservative in positive direction. */
369 if (i2
== MAX_CLAUSES
)
371 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
372 p
->clause
[i2
+ 1] = 0;
373 if (insert_here
>= 0)
374 for (; i2
> insert_here
; i2
--)
375 p
->clause
[i2
] = p
->clause
[i2
- 1];
378 p
->clause
[insert_here
] = clause
;
384 static struct predicate
385 and_predicates (conditions conditions
,
386 struct predicate
*p
, struct predicate
*p2
)
388 struct predicate out
= *p
;
391 /* Avoid busy work. */
392 if (false_predicate_p (p2
) || true_predicate_p (p
))
394 if (false_predicate_p (p
) || true_predicate_p (p2
))
397 /* See how far predicates match. */
398 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
400 gcc_checking_assert (i
< MAX_CLAUSES
);
403 /* Combine the predicates rest. */
404 for (; p2
->clause
[i
]; i
++)
406 gcc_checking_assert (i
< MAX_CLAUSES
);
407 add_clause (conditions
, &out
, p2
->clause
[i
]);
413 /* Return true if predicates are obviously equal. */
416 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
419 for (i
= 0; p
->clause
[i
]; i
++)
421 gcc_checking_assert (i
< MAX_CLAUSES
);
422 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
423 gcc_checking_assert (!p2
->clause
[i
]
424 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
425 if (p
->clause
[i
] != p2
->clause
[i
])
428 return !p2
->clause
[i
];
434 static struct predicate
435 or_predicates (conditions conditions
,
436 struct predicate
*p
, struct predicate
*p2
)
438 struct predicate out
= true_predicate ();
441 /* Avoid busy work. */
442 if (false_predicate_p (p2
) || true_predicate_p (p
))
444 if (false_predicate_p (p
) || true_predicate_p (p2
))
446 if (predicates_equal_p (p
, p2
))
449 /* OK, combine the predicates. */
450 for (i
= 0; p
->clause
[i
]; i
++)
451 for (j
= 0; p2
->clause
[j
]; j
++)
453 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
454 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
460 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
461 if predicate P is known to be false. */
464 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
468 /* True remains true. */
469 if (true_predicate_p (p
))
472 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
474 /* See if we can find clause we can disprove. */
475 for (i
= 0; p
->clause
[i
]; i
++)
477 gcc_checking_assert (i
< MAX_CLAUSES
);
478 if (!(p
->clause
[i
] & possible_truths
))
484 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
485 instruction will be recomputed per invocation of the inlined call. */
488 predicate_probability (conditions conds
,
489 struct predicate
*p
, clause_t possible_truths
,
490 vec
<inline_param_summary_t
> inline_param_summary
)
493 int combined_prob
= REG_BR_PROB_BASE
;
495 /* True remains true. */
496 if (true_predicate_p (p
))
497 return REG_BR_PROB_BASE
;
499 if (false_predicate_p (p
))
502 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
504 /* See if we can find clause we can disprove. */
505 for (i
= 0; p
->clause
[i
]; i
++)
507 gcc_checking_assert (i
< MAX_CLAUSES
);
508 if (!(p
->clause
[i
] & possible_truths
))
514 if (!inline_param_summary
.exists ())
515 return REG_BR_PROB_BASE
;
516 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
517 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
519 if (i2
>= predicate_first_dynamic_condition
)
522 &(*conds
)[i2
- predicate_first_dynamic_condition
];
523 if (c
->code
== CHANGED
525 (int) inline_param_summary
.length ()))
528 inline_param_summary
[c
->operand_num
].change_prob
;
529 this_prob
= MAX (this_prob
, iprob
);
532 this_prob
= REG_BR_PROB_BASE
;
535 this_prob
= REG_BR_PROB_BASE
;
537 combined_prob
= MIN (this_prob
, combined_prob
);
542 return combined_prob
;
546 /* Dump conditional COND. */
549 dump_condition (FILE *f
, conditions conditions
, int cond
)
552 if (cond
== predicate_false_condition
)
553 fprintf (f
, "false");
554 else if (cond
== predicate_not_inlined_condition
)
555 fprintf (f
, "not inlined");
558 c
= &(*conditions
)[cond
- predicate_first_dynamic_condition
];
559 fprintf (f
, "op%i", c
->operand_num
);
561 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
562 c
->by_ref
? "ref " : "", c
->offset
);
563 if (c
->code
== IS_NOT_CONSTANT
)
565 fprintf (f
, " not constant");
568 if (c
->code
== CHANGED
)
570 fprintf (f
, " changed");
573 fprintf (f
, " %s ", op_symbol_code (c
->code
));
574 print_generic_expr (f
, c
->val
, 1);
579 /* Dump clause CLAUSE. */
582 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
589 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
590 if (clause
& (1 << i
))
595 dump_condition (f
, conds
, i
);
601 /* Dump predicate PREDICATE. */
604 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
607 if (true_predicate_p (pred
))
608 dump_clause (f
, conds
, 0);
610 for (i
= 0; pred
->clause
[i
]; i
++)
614 dump_clause (f
, conds
, pred
->clause
[i
]);
620 /* Dump inline hints. */
622 dump_inline_hints (FILE *f
, inline_hints hints
)
626 fprintf (f
, "inline hints:");
627 if (hints
& INLINE_HINT_indirect_call
)
629 hints
&= ~INLINE_HINT_indirect_call
;
630 fprintf (f
, " indirect_call");
632 if (hints
& INLINE_HINT_loop_iterations
)
634 hints
&= ~INLINE_HINT_loop_iterations
;
635 fprintf (f
, " loop_iterations");
637 if (hints
& INLINE_HINT_loop_stride
)
639 hints
&= ~INLINE_HINT_loop_stride
;
640 fprintf (f
, " loop_stride");
642 if (hints
& INLINE_HINT_same_scc
)
644 hints
&= ~INLINE_HINT_same_scc
;
645 fprintf (f
, " same_scc");
647 if (hints
& INLINE_HINT_in_scc
)
649 hints
&= ~INLINE_HINT_in_scc
;
650 fprintf (f
, " in_scc");
652 if (hints
& INLINE_HINT_cross_module
)
654 hints
&= ~INLINE_HINT_cross_module
;
655 fprintf (f
, " cross_module");
657 if (hints
& INLINE_HINT_declared_inline
)
659 hints
&= ~INLINE_HINT_declared_inline
;
660 fprintf (f
, " declared_inline");
662 if (hints
& INLINE_HINT_array_index
)
664 hints
&= ~INLINE_HINT_array_index
;
665 fprintf (f
, " array_index");
671 /* Record SIZE and TIME under condition PRED into the inline summary. */
674 account_size_time (struct inline_summary
*summary
, int size
, int time
,
675 struct predicate
*pred
)
681 if (false_predicate_p (pred
))
684 /* We need to create initial empty unconitional clause, but otherwie
685 we don't need to account empty times and sizes. */
686 if (!size
&& !time
&& summary
->entry
)
689 /* Watch overflow that might result from insane profiles. */
690 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
691 time
= MAX_TIME
* INLINE_TIME_SCALE
;
692 gcc_assert (time
>= 0);
694 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
695 if (predicates_equal_p (&e
->predicate
, pred
))
704 e
= &(*summary
->entry
)[0];
705 gcc_assert (!e
->predicate
.clause
[0]);
706 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
708 "\t\tReached limit on number of entries, "
709 "ignoring the predicate.");
711 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
714 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
715 ((double) size
) / INLINE_SIZE_SCALE
,
716 ((double) time
) / INLINE_TIME_SCALE
, found
? "" : "new ");
717 dump_predicate (dump_file
, summary
->conds
, pred
);
721 struct size_time_entry new_entry
;
722 new_entry
.size
= size
;
723 new_entry
.time
= time
;
724 new_entry
.predicate
= *pred
;
725 vec_safe_push (summary
->entry
, new_entry
);
731 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
732 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
736 /* Set predicate for edge E. */
739 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
741 struct inline_edge_summary
*es
= inline_edge_summary (e
);
742 if (predicate
&& !true_predicate_p (predicate
))
745 es
->predicate
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
746 *es
->predicate
= *predicate
;
751 pool_free (edge_predicate_pool
, es
->predicate
);
752 es
->predicate
= NULL
;
756 /* Set predicate for hint *P. */
759 set_hint_predicate (struct predicate
**p
, struct predicate new_predicate
)
761 if (false_predicate_p (&new_predicate
) || true_predicate_p (&new_predicate
))
764 pool_free (edge_predicate_pool
, *p
);
770 *p
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
776 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
777 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
778 Return clause of possible truths. When INLINE_P is true, assume that we are
781 ERROR_MARK means compile time invariant. */
784 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
786 vec
<tree
> known_vals
,
787 vec
<ipa_agg_jump_function_p
>
790 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
791 struct inline_summary
*info
= inline_summary (node
);
795 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
800 /* We allow call stmt to have fewer arguments than the callee function
801 (especially for K&R style programs). So bound check here (we assume
802 known_aggs vector, if non-NULL, has the same length as
804 gcc_checking_assert (!known_aggs
.exists ()
805 || (known_vals
.length () == known_aggs
.length ()));
806 if (c
->operand_num
>= (int) known_vals
.length ())
808 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
814 struct ipa_agg_jump_function
*agg
;
816 if (c
->code
== CHANGED
818 && (known_vals
[c
->operand_num
] == error_mark_node
))
821 if (known_aggs
.exists ())
823 agg
= known_aggs
[c
->operand_num
];
824 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
831 val
= known_vals
[c
->operand_num
];
832 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
838 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
841 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
843 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
844 if (res
&& integer_zerop (res
))
846 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
852 /* Work out what conditions might be true at invocation of E. */
855 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
856 clause_t
*clause_ptr
,
857 vec
<tree
> *known_vals_ptr
,
858 vec
<tree
> *known_binfos_ptr
,
859 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
861 struct cgraph_node
*callee
=
862 cgraph_function_or_thunk_node (e
->callee
, NULL
);
863 struct inline_summary
*info
= inline_summary (callee
);
864 vec
<tree
> known_vals
= vNULL
;
865 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
868 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
870 known_vals_ptr
->create (0);
871 if (known_binfos_ptr
)
872 known_binfos_ptr
->create (0);
874 if (ipa_node_params_vector
.exists ()
875 && !e
->call_stmt_cannot_inline_p
876 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_binfos_ptr
))
878 struct ipa_node_params
*parms_info
;
879 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
880 struct inline_edge_summary
*es
= inline_edge_summary (e
);
881 int i
, count
= ipa_get_cs_argument_count (args
);
883 if (e
->caller
->global
.inlined_to
)
884 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
886 parms_info
= IPA_NODE_REF (e
->caller
);
888 if (count
&& (info
->conds
|| known_vals_ptr
))
889 known_vals
.safe_grow_cleared (count
);
890 if (count
&& (info
->conds
|| known_aggs_ptr
))
891 known_aggs
.safe_grow_cleared (count
);
892 if (count
&& known_binfos_ptr
)
893 known_binfos_ptr
->safe_grow_cleared (count
);
895 for (i
= 0; i
< count
; i
++)
897 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
898 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
901 if (known_vals
.exists () && TREE_CODE (cst
) != TREE_BINFO
)
903 else if (known_binfos_ptr
!= NULL
904 && TREE_CODE (cst
) == TREE_BINFO
)
905 (*known_binfos_ptr
)[i
] = cst
;
907 else if (inline_p
&& !es
->param
[i
].change_prob
)
908 known_vals
[i
] = error_mark_node
;
909 /* TODO: When IPA-CP starts propagating and merging aggregate jump
910 functions, use its knowledge of the caller too, just like the
911 scalar case above. */
912 known_aggs
[i
] = &jf
->agg
;
917 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
918 known_vals
, known_aggs
);
921 *known_vals_ptr
= known_vals
;
923 known_vals
.release ();
926 *known_aggs_ptr
= known_aggs
;
928 known_aggs
.release ();
932 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
935 inline_summary_alloc (void)
937 if (!node_removal_hook_holder
)
938 node_removal_hook_holder
=
939 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
940 if (!edge_removal_hook_holder
)
941 edge_removal_hook_holder
=
942 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
943 if (!node_duplication_hook_holder
)
944 node_duplication_hook_holder
=
945 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
946 if (!edge_duplication_hook_holder
)
947 edge_duplication_hook_holder
=
948 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
950 if (vec_safe_length (inline_summary_vec
) <= (unsigned) cgraph_max_uid
)
951 vec_safe_grow_cleared (inline_summary_vec
, cgraph_max_uid
+ 1);
952 if (inline_edge_summary_vec
.length () <= (unsigned) cgraph_edge_max_uid
)
953 inline_edge_summary_vec
.safe_grow_cleared (cgraph_edge_max_uid
+ 1);
954 if (!edge_predicate_pool
)
955 edge_predicate_pool
= create_alloc_pool ("edge predicates",
956 sizeof (struct predicate
), 10);
959 /* We are called multiple time for given function; clear
960 data from previous run so they are not cumulated. */
963 reset_inline_edge_summary (struct cgraph_edge
*e
)
965 if (e
->uid
< (int) inline_edge_summary_vec
.length ())
967 struct inline_edge_summary
*es
= inline_edge_summary (e
);
969 es
->call_stmt_size
= es
->call_stmt_time
= 0;
971 pool_free (edge_predicate_pool
, es
->predicate
);
972 es
->predicate
= NULL
;
973 es
->param
.release ();
977 /* We are called multiple time for given function; clear
978 data from previous run so they are not cumulated. */
981 reset_inline_summary (struct cgraph_node
*node
)
983 struct inline_summary
*info
= inline_summary (node
);
984 struct cgraph_edge
*e
;
986 info
->self_size
= info
->self_time
= 0;
987 info
->estimated_stack_size
= 0;
988 info
->estimated_self_stack_size
= 0;
989 info
->stack_frame_offset
= 0;
994 if (info
->loop_iterations
)
996 pool_free (edge_predicate_pool
, info
->loop_iterations
);
997 info
->loop_iterations
= NULL
;
999 if (info
->loop_stride
)
1001 pool_free (edge_predicate_pool
, info
->loop_stride
);
1002 info
->loop_stride
= NULL
;
1004 if (info
->array_index
)
1006 pool_free (edge_predicate_pool
, info
->array_index
);
1007 info
->array_index
= NULL
;
1009 vec_free (info
->conds
);
1010 vec_free (info
->entry
);
1011 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1012 reset_inline_edge_summary (e
);
1013 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1014 reset_inline_edge_summary (e
);
1017 /* Hook that is called by cgraph.c when a node is removed. */
1020 inline_node_removal_hook (struct cgraph_node
*node
,
1021 void *data ATTRIBUTE_UNUSED
)
1023 struct inline_summary
*info
;
1024 if (vec_safe_length (inline_summary_vec
) <= (unsigned) node
->uid
)
1026 info
= inline_summary (node
);
1027 reset_inline_summary (node
);
1028 memset (info
, 0, sizeof (inline_summary_t
));
1031 /* Remap predicate P of former function to be predicate of duplicated functoin.
1032 POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
1033 INFO is inline summary of the duplicated node. */
1035 static struct predicate
1036 remap_predicate_after_duplication (struct predicate
*p
,
1037 clause_t possible_truths
,
1038 struct inline_summary
*info
)
1040 struct predicate new_predicate
= true_predicate ();
1042 for (j
= 0; p
->clause
[j
]; j
++)
1043 if (!(possible_truths
& p
->clause
[j
]))
1045 new_predicate
= false_predicate ();
1049 add_clause (info
->conds
, &new_predicate
,
1050 possible_truths
& p
->clause
[j
]);
1051 return new_predicate
;
1054 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
1055 Additionally care about allocating new memory slot for updated predicate
1056 and set it to NULL when it becomes true or false (and thus uninteresting).
1060 remap_hint_predicate_after_duplication (struct predicate
**p
,
1061 clause_t possible_truths
,
1062 struct inline_summary
*info
)
1064 struct predicate new_predicate
;
1069 new_predicate
= remap_predicate_after_duplication (*p
,
1070 possible_truths
, info
);
1071 /* We do not want to free previous predicate; it is used by node origin. */
1073 set_hint_predicate (p
, new_predicate
);
1077 /* Hook that is called by cgraph.c when a node is duplicated. */
1080 inline_node_duplication_hook (struct cgraph_node
*src
,
1081 struct cgraph_node
*dst
,
1082 ATTRIBUTE_UNUSED
void *data
)
1084 struct inline_summary
*info
;
1085 inline_summary_alloc ();
1086 info
= inline_summary (dst
);
1087 memcpy (info
, inline_summary (src
), sizeof (struct inline_summary
));
1088 /* TODO: as an optimization, we may avoid copying conditions
1089 that are known to be false or true. */
1090 info
->conds
= vec_safe_copy (info
->conds
);
1092 /* When there are any replacements in the function body, see if we can figure
1093 out that something was optimized out. */
1094 if (ipa_node_params_vector
.exists () && dst
->clone
.tree_map
)
1096 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
1097 /* Use SRC parm info since it may not be copied yet. */
1098 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1099 vec
<tree
> known_vals
= vNULL
;
1100 int count
= ipa_get_param_count (parms_info
);
1102 clause_t possible_truths
;
1103 struct predicate true_pred
= true_predicate ();
1105 int optimized_out_size
= 0;
1106 bool inlined_to_p
= false;
1107 struct cgraph_edge
*edge
;
1110 known_vals
.safe_grow_cleared (count
);
1111 for (i
= 0; i
< count
; i
++)
1113 struct ipa_replace_map
*r
;
1115 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
1117 if (((!r
->old_tree
&& r
->parm_num
== i
)
1118 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
1119 && r
->replace_p
&& !r
->ref_p
)
1121 known_vals
[i
] = r
->new_tree
;
1126 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1129 known_vals
.release ();
1131 account_size_time (info
, 0, 0, &true_pred
);
1133 /* Remap size_time vectors.
1134 Simplify the predicate by prunning out alternatives that are known
1136 TODO: as on optimization, we can also eliminate conditions known
1138 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
1140 struct predicate new_predicate
;
1141 new_predicate
= remap_predicate_after_duplication (&e
->predicate
,
1144 if (false_predicate_p (&new_predicate
))
1145 optimized_out_size
+= e
->size
;
1147 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1150 /* Remap edge predicates with the same simplification as above.
1151 Also copy constantness arrays. */
1152 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1154 struct predicate new_predicate
;
1155 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1157 if (!edge
->inline_failed
)
1158 inlined_to_p
= true;
1161 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1164 if (false_predicate_p (&new_predicate
)
1165 && !false_predicate_p (es
->predicate
))
1167 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1168 edge
->frequency
= 0;
1170 edge_set_predicate (edge
, &new_predicate
);
1173 /* Remap indirect edge predicates with the same simplificaiton as above.
1174 Also copy constantness arrays. */
1175 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1177 struct predicate new_predicate
;
1178 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1180 gcc_checking_assert (edge
->inline_failed
);
1183 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1186 if (false_predicate_p (&new_predicate
)
1187 && !false_predicate_p (es
->predicate
))
1189 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1190 edge
->frequency
= 0;
1192 edge_set_predicate (edge
, &new_predicate
);
1194 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
1195 possible_truths
, info
);
1196 remap_hint_predicate_after_duplication (&info
->loop_stride
,
1197 possible_truths
, info
);
1198 remap_hint_predicate_after_duplication (&info
->array_index
,
1199 possible_truths
, info
);
1201 /* If inliner or someone after inliner will ever start producing
1202 non-trivial clones, we will get trouble with lack of information
1203 about updating self sizes, because size vectors already contains
1204 sizes of the calees. */
1205 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
1209 info
->entry
= vec_safe_copy (info
->entry
);
1210 if (info
->loop_iterations
)
1212 predicate p
= *info
->loop_iterations
;
1213 info
->loop_iterations
= NULL
;
1214 set_hint_predicate (&info
->loop_iterations
, p
);
1216 if (info
->loop_stride
)
1218 predicate p
= *info
->loop_stride
;
1219 info
->loop_stride
= NULL
;
1220 set_hint_predicate (&info
->loop_stride
, p
);
1222 if (info
->array_index
)
1224 predicate p
= *info
->array_index
;
1225 info
->array_index
= NULL
;
1226 set_hint_predicate (&info
->array_index
, p
);
1229 inline_update_overall_summary (dst
);
1233 /* Hook that is called by cgraph.c when a node is duplicated. */
1236 inline_edge_duplication_hook (struct cgraph_edge
*src
,
1237 struct cgraph_edge
*dst
,
1238 ATTRIBUTE_UNUSED
void *data
)
1240 struct inline_edge_summary
*info
;
1241 struct inline_edge_summary
*srcinfo
;
1242 inline_summary_alloc ();
1243 info
= inline_edge_summary (dst
);
1244 srcinfo
= inline_edge_summary (src
);
1245 memcpy (info
, srcinfo
, sizeof (struct inline_edge_summary
));
1246 info
->predicate
= NULL
;
1247 edge_set_predicate (dst
, srcinfo
->predicate
);
1248 info
->param
= srcinfo
->param
.copy ();
1252 /* Keep edge cache consistent across edge removal. */
1255 inline_edge_removal_hook (struct cgraph_edge
*edge
,
1256 void *data ATTRIBUTE_UNUSED
)
1258 if (edge_growth_cache
.exists ())
1259 reset_edge_growth_cache (edge
);
1260 reset_inline_edge_summary (edge
);
1264 /* Initialize growth caches. */
1267 initialize_growth_caches (void)
1269 if (cgraph_edge_max_uid
)
1270 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
1272 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
1276 /* Free growth caches. */
1279 free_growth_caches (void)
1281 edge_growth_cache
.release ();
1282 node_growth_cache
.release ();
1286 /* Dump edge summaries associated to NODE and recursively to all clones.
1287 Indent by INDENT. */
1290 dump_inline_edge_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
1291 struct inline_summary
*info
)
1293 struct cgraph_edge
*edge
;
1294 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1296 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1297 struct cgraph_node
*callee
=
1298 cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1302 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
1303 " time: %2i callee size:%2i stack:%2i",
1304 indent
, "", callee
->name (), callee
->order
,
1305 !edge
->inline_failed
1306 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
1307 indent
, "", es
->loop_depth
, edge
->frequency
,
1308 es
->call_stmt_size
, es
->call_stmt_time
,
1309 (int) inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1310 (int) inline_summary (callee
)->estimated_stack_size
);
1314 fprintf (f
, " predicate: ");
1315 dump_predicate (f
, info
->conds
, es
->predicate
);
1319 if (es
->param
.exists ())
1320 for (i
= 0; i
< (int) es
->param
.length (); i
++)
1322 int prob
= es
->param
[i
].change_prob
;
1325 fprintf (f
, "%*s op%i is compile time invariant\n",
1327 else if (prob
!= REG_BR_PROB_BASE
)
1328 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1329 prob
* 100.0 / REG_BR_PROB_BASE
);
1331 if (!edge
->inline_failed
)
1333 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1334 " callee size %i\n",
1336 (int) inline_summary (callee
)->stack_frame_offset
,
1337 (int) inline_summary (callee
)->estimated_self_stack_size
,
1338 (int) inline_summary (callee
)->estimated_stack_size
);
1339 dump_inline_edge_summary (f
, indent
+ 2, callee
, info
);
1342 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1344 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1345 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1349 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
1352 fprintf (f
, "predicate: ");
1353 dump_predicate (f
, info
->conds
, es
->predicate
);
1362 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
1364 if (node
->definition
)
1366 struct inline_summary
*s
= inline_summary (node
);
1369 fprintf (f
, "Inline summary for %s/%i", node
->name (),
1371 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
1372 fprintf (f
, " always_inline");
1374 fprintf (f
, " inlinable");
1375 fprintf (f
, "\n self time: %i\n", s
->self_time
);
1376 fprintf (f
, " global time: %i\n", s
->time
);
1377 fprintf (f
, " self size: %i\n", s
->self_size
);
1378 fprintf (f
, " global size: %i\n", s
->size
);
1379 fprintf (f
, " self stack: %i\n",
1380 (int) s
->estimated_self_stack_size
);
1381 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
1383 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
1385 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
1386 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
1388 fprintf (f
, " size:%f, time:%f, predicate:",
1389 (double) e
->size
/ INLINE_SIZE_SCALE
,
1390 (double) e
->time
/ INLINE_TIME_SCALE
);
1391 dump_predicate (f
, s
->conds
, &e
->predicate
);
1393 if (s
->loop_iterations
)
1395 fprintf (f
, " loop iterations:");
1396 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1400 fprintf (f
, " loop stride:");
1401 dump_predicate (f
, s
->conds
, s
->loop_stride
);
1405 fprintf (f
, " array index:");
1406 dump_predicate (f
, s
->conds
, s
->array_index
);
1408 fprintf (f
, " calls:\n");
1409 dump_inline_edge_summary (f
, 4, node
, s
);
1415 debug_inline_summary (struct cgraph_node
*node
)
1417 dump_inline_summary (stderr
, node
);
1421 dump_inline_summaries (FILE *f
)
1423 struct cgraph_node
*node
;
1425 FOR_EACH_DEFINED_FUNCTION (node
)
1426 if (!node
->global
.inlined_to
)
1427 dump_inline_summary (f
, node
);
1430 /* Give initial reasons why inlining would fail on EDGE. This gets either
1431 nullified or usually overwritten by more precise reasons later. */
1434 initialize_inline_failed (struct cgraph_edge
*e
)
1436 struct cgraph_node
*callee
= e
->callee
;
1438 if (e
->indirect_unknown_callee
)
1439 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1440 else if (!callee
->definition
)
1441 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1442 else if (callee
->local
.redefined_extern_inline
)
1443 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1444 else if (e
->call_stmt_cannot_inline_p
)
1445 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1446 else if (cfun
&& fn_contains_cilk_spawn_p (cfun
))
1447 /* We can't inline if the function is spawing a function. */
1448 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
1450 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1453 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1454 boolean variable pointed to by DATA. */
1457 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1460 bool *b
= (bool *) data
;
1465 /* If OP refers to value of function parameter, return the corresponding
1469 unmodified_parm_1 (gimple stmt
, tree op
)
1471 /* SSA_NAME referring to parm default def? */
1472 if (TREE_CODE (op
) == SSA_NAME
1473 && SSA_NAME_IS_DEFAULT_DEF (op
)
1474 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1475 return SSA_NAME_VAR (op
);
1476 /* Non-SSA parm reference? */
1477 if (TREE_CODE (op
) == PARM_DECL
)
1479 bool modified
= false;
1482 ao_ref_init (&refd
, op
);
1483 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1491 /* If OP refers to value of function parameter, return the corresponding
1492 parameter. Also traverse chains of SSA register assignments. */
1495 unmodified_parm (gimple stmt
, tree op
)
1497 tree res
= unmodified_parm_1 (stmt
, op
);
1501 if (TREE_CODE (op
) == SSA_NAME
1502 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1503 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1504 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1505 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1509 /* If OP refers to a value of a function parameter or value loaded from an
1510 aggregate passed to a parameter (either by value or reference), return TRUE
1511 and store the number of the parameter to *INDEX_P and information whether
1512 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1513 the function parameters, STMT is the statement in which OP is used or
1517 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1518 gimple stmt
, tree op
, int *index_p
,
1519 struct agg_position_info
*aggpos
)
1521 tree res
= unmodified_parm_1 (stmt
, op
);
1523 gcc_checking_assert (aggpos
);
1526 *index_p
= ipa_get_param_decl_index (info
, res
);
1529 aggpos
->agg_contents
= false;
1530 aggpos
->by_ref
= false;
1534 if (TREE_CODE (op
) == SSA_NAME
)
1536 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1537 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1539 stmt
= SSA_NAME_DEF_STMT (op
);
1540 op
= gimple_assign_rhs1 (stmt
);
1541 if (!REFERENCE_CLASS_P (op
))
1542 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1546 aggpos
->agg_contents
= true;
1547 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1551 /* See if statement might disappear after inlining.
1552 0 - means not eliminated
1553 1 - half of statements goes away
1554 2 - for sure it is eliminated.
1555 We are not terribly sophisticated, basically looking for simple abstraction
1556 penalty wrappers. */
1559 eliminated_by_inlining_prob (gimple stmt
)
1561 enum gimple_code code
= gimple_code (stmt
);
1562 enum tree_code rhs_code
;
1572 if (gimple_num_ops (stmt
) != 2)
1575 rhs_code
= gimple_assign_rhs_code (stmt
);
1577 /* Casts of parameters, loads from parameters passed by reference
1578 and stores to return value or parameters are often free after
1579 inlining dua to SRA and further combining.
1580 Assume that half of statements goes away. */
1581 if (rhs_code
== CONVERT_EXPR
1582 || rhs_code
== NOP_EXPR
1583 || rhs_code
== VIEW_CONVERT_EXPR
1584 || rhs_code
== ADDR_EXPR
1585 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1587 tree rhs
= gimple_assign_rhs1 (stmt
);
1588 tree lhs
= gimple_assign_lhs (stmt
);
1589 tree inner_rhs
= get_base_address (rhs
);
1590 tree inner_lhs
= get_base_address (lhs
);
1591 bool rhs_free
= false;
1592 bool lhs_free
= false;
1599 /* Reads of parameter are expected to be free. */
1600 if (unmodified_parm (stmt
, inner_rhs
))
1602 /* Match expressions of form &this->field. Those will most likely
1603 combine with something upstream after inlining. */
1604 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1606 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1607 if (TREE_CODE (op
) == PARM_DECL
)
1609 else if (TREE_CODE (op
) == MEM_REF
1610 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0)))
1614 /* When parameter is not SSA register because its address is taken
1615 and it is just copied into one, the statement will be completely
1616 free after inlining (we will copy propagate backward). */
1617 if (rhs_free
&& is_gimple_reg (lhs
))
1620 /* Reads of parameters passed by reference
1621 expected to be free (i.e. optimized out after inlining). */
1622 if (TREE_CODE (inner_rhs
) == MEM_REF
1623 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1626 /* Copying parameter passed by reference into gimple register is
1627 probably also going to copy propagate, but we can't be quite
1629 if (rhs_free
&& is_gimple_reg (lhs
))
1632 /* Writes to parameters, parameters passed by value and return value
1633 (either dirrectly or passed via invisible reference) are free.
1635 TODO: We ought to handle testcase like
1636 struct a {int a,b;};
1638 retrurnsturct (void)
1644 This translate into:
1659 For that we either need to copy ipa-split logic detecting writes
1661 if (TREE_CODE (inner_lhs
) == PARM_DECL
1662 || TREE_CODE (inner_lhs
) == RESULT_DECL
1663 || (TREE_CODE (inner_lhs
) == MEM_REF
1664 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1665 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1666 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1667 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1669 0))) == RESULT_DECL
))))
1672 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1674 if (lhs_free
&& rhs_free
)
1684 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1685 predicates to the CFG edges. */
1688 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1689 struct inline_summary
*summary
,
1695 struct agg_position_info aggpos
;
1696 enum tree_code code
, inverted_code
;
1702 last
= last_stmt (bb
);
1703 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1705 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1707 op
= gimple_cond_lhs (last
);
1708 /* TODO: handle conditionals like
1711 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1713 code
= gimple_cond_code (last
);
1715 = invert_tree_comparison (code
,
1716 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1718 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1720 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1721 e
->flags
& EDGE_TRUE_VALUE
1722 ? code
: inverted_code
,
1723 gimple_cond_rhs (last
));
1724 e
->aux
= pool_alloc (edge_predicate_pool
);
1725 *(struct predicate
*) e
->aux
= p
;
1729 if (TREE_CODE (op
) != SSA_NAME
)
1732 if (builtin_constant_p (op))
1736 Here we can predicate nonconstant_code. We can't
1737 really handle constant_code since we have no predicate
1738 for this and also the constant code is not known to be
1739 optimized away when inliner doen't see operand is constant.
1740 Other optimizers might think otherwise. */
1741 if (gimple_cond_code (last
) != NE_EXPR
1742 || !integer_zerop (gimple_cond_rhs (last
)))
1744 set_stmt
= SSA_NAME_DEF_STMT (op
);
1745 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1746 || gimple_call_num_args (set_stmt
) != 1)
1748 op2
= gimple_call_arg (set_stmt
, 0);
1749 if (!unmodified_parm_or_parm_agg_item
1750 (info
, set_stmt
, op2
, &index
, &aggpos
))
1752 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1754 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1755 IS_NOT_CONSTANT
, NULL_TREE
);
1756 e
->aux
= pool_alloc (edge_predicate_pool
);
1757 *(struct predicate
*) e
->aux
= p
;
1762 /* If BB ends by a switch we can turn into predicates, attach corresponding
1763 predicates to the CFG edges. */
1766 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1767 struct inline_summary
*summary
,
1773 struct agg_position_info aggpos
;
1779 last
= last_stmt (bb
);
1780 if (!last
|| gimple_code (last
) != GIMPLE_SWITCH
)
1782 op
= gimple_switch_index (last
);
1783 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1786 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1788 e
->aux
= pool_alloc (edge_predicate_pool
);
1789 *(struct predicate
*) e
->aux
= false_predicate ();
1791 n
= gimple_switch_num_labels (last
);
1792 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1794 tree cl
= gimple_switch_label (last
, case_idx
);
1798 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1799 min
= CASE_LOW (cl
);
1800 max
= CASE_HIGH (cl
);
1802 /* For default we might want to construct predicate that none
1803 of cases is met, but it is bit hard to do not having negations
1804 of conditionals handy. */
1806 p
= true_predicate ();
1808 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1811 struct predicate p1
, p2
;
1812 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1813 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1814 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1816 *(struct predicate
*) e
->aux
1817 = or_predicates (summary
->conds
, &p
, (struct predicate
*) e
->aux
);
1822 /* For each BB in NODE attach to its AUX pointer predicate under
1823 which it is executable. */
1826 compute_bb_predicates (struct cgraph_node
*node
,
1827 struct ipa_node_params
*parms_info
,
1828 struct inline_summary
*summary
)
1830 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1834 FOR_EACH_BB_FN (bb
, my_function
)
1836 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1837 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1840 /* Entry block is always executable. */
1841 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1842 = pool_alloc (edge_predicate_pool
);
1843 *(struct predicate
*) ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1844 = true_predicate ();
1846 /* A simple dataflow propagation of predicates forward in the CFG.
1847 TODO: work in reverse postorder. */
1851 FOR_EACH_BB_FN (bb
, my_function
)
1853 struct predicate p
= false_predicate ();
1856 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1860 struct predicate this_bb_predicate
1861 = *(struct predicate
*) e
->src
->aux
;
1864 = and_predicates (summary
->conds
, &this_bb_predicate
,
1865 (struct predicate
*) e
->aux
);
1866 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1867 if (true_predicate_p (&p
))
1871 if (false_predicate_p (&p
))
1872 gcc_assert (!bb
->aux
);
1878 bb
->aux
= pool_alloc (edge_predicate_pool
);
1879 *((struct predicate
*) bb
->aux
) = p
;
1881 else if (!predicates_equal_p (&p
, (struct predicate
*) bb
->aux
))
1884 *((struct predicate
*) bb
->aux
) = p
;
1892 /* We keep info about constantness of SSA names. */
1894 typedef struct predicate predicate_t
;
1895 /* Return predicate specifying when the STMT might have result that is not
1896 a compile time constant. */
1898 static struct predicate
1899 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1900 struct inline_summary
*summary
,
1902 vec
<predicate_t
> nonconstant_names
)
1907 while (UNARY_CLASS_P (expr
))
1908 expr
= TREE_OPERAND (expr
, 0);
1910 parm
= unmodified_parm (NULL
, expr
);
1911 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1912 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1913 if (is_gimple_min_invariant (expr
))
1914 return false_predicate ();
1915 if (TREE_CODE (expr
) == SSA_NAME
)
1916 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1917 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1919 struct predicate p1
= will_be_nonconstant_expr_predicate
1920 (info
, summary
, TREE_OPERAND (expr
, 0),
1922 struct predicate p2
;
1923 if (true_predicate_p (&p1
))
1925 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1926 TREE_OPERAND (expr
, 1),
1928 return or_predicates (summary
->conds
, &p1
, &p2
);
1930 else if (TREE_CODE (expr
) == COND_EXPR
)
1932 struct predicate p1
= will_be_nonconstant_expr_predicate
1933 (info
, summary
, TREE_OPERAND (expr
, 0),
1935 struct predicate p2
;
1936 if (true_predicate_p (&p1
))
1938 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1939 TREE_OPERAND (expr
, 1),
1941 if (true_predicate_p (&p2
))
1943 p1
= or_predicates (summary
->conds
, &p1
, &p2
);
1944 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1945 TREE_OPERAND (expr
, 2),
1947 return or_predicates (summary
->conds
, &p1
, &p2
);
1954 return false_predicate ();
1958 /* Return predicate specifying when the STMT might have result that is not
1959 a compile time constant. */
1961 static struct predicate
1962 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1963 struct inline_summary
*summary
,
1965 vec
<predicate_t
> nonconstant_names
)
1967 struct predicate p
= true_predicate ();
1970 struct predicate op_non_const
;
1973 struct agg_position_info aggpos
;
1975 /* What statments might be optimized away
1976 when their arguments are constant
1977 TODO: also trivial builtins.
1978 builtin_constant_p is already handled later. */
1979 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1980 && gimple_code (stmt
) != GIMPLE_COND
1981 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1984 /* Stores will stay anyway. */
1985 if (gimple_store_p (stmt
))
1988 is_load
= gimple_assign_load_p (stmt
);
1990 /* Loads can be optimized when the value is known. */
1994 gcc_assert (gimple_assign_single_p (stmt
));
1995 op
= gimple_assign_rhs1 (stmt
);
1996 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
2003 /* See if we understand all operands before we start
2004 adding conditionals. */
2005 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2007 tree parm
= unmodified_parm (stmt
, use
);
2008 /* For arguments we can build a condition. */
2009 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
2011 if (TREE_CODE (use
) != SSA_NAME
)
2013 /* If we know when operand is constant,
2014 we still can say something useful. */
2015 if (!true_predicate_p (&nonconstant_names
[SSA_NAME_VERSION (use
)]))
2022 add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
2024 op_non_const
= false_predicate ();
2025 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2027 tree parm
= unmodified_parm (stmt
, use
);
2030 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
2032 if (index
!= base_index
)
2033 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
2038 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2039 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
2041 if (gimple_code (stmt
) == GIMPLE_ASSIGN
2042 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
2043 nonconstant_names
[SSA_NAME_VERSION (gimple_assign_lhs (stmt
))]
2045 return op_non_const
;
2048 struct record_modified_bb_info
2054 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2055 set except for info->stmt. */
2058 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
2060 struct record_modified_bb_info
*info
=
2061 (struct record_modified_bb_info
*) data
;
2062 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2064 bitmap_set_bit (info
->bb_set
,
2065 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2066 ? ENTRY_BLOCK_PTR
->index
2067 : gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
2071 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2072 will change since last invocation of STMT.
2074 Value 0 is reserved for compile time invariants.
2075 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2076 ought to be REG_BR_PROB_BASE / estimated_iters. */
2079 param_change_prob (gimple stmt
, int i
)
2081 tree op
= gimple_call_arg (stmt
, i
);
2082 basic_block bb
= gimple_bb (stmt
);
2085 /* Global invariants neve change. */
2086 if (is_gimple_min_invariant (op
))
2088 /* We would have to do non-trivial analysis to really work out what
2089 is the probability of value to change (i.e. when init statement
2090 is in a sibling loop of the call).
2092 We do an conservative estimate: when call is executed N times more often
2093 than the statement defining value, we take the frequency 1/N. */
2094 if (TREE_CODE (op
) == SSA_NAME
)
2099 return REG_BR_PROB_BASE
;
2101 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2102 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2104 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2108 if (init_freq
< bb
->frequency
)
2109 return MAX (GCOV_COMPUTE_SCALE (init_freq
, bb
->frequency
), 1);
2111 return REG_BR_PROB_BASE
;
2114 base
= get_base_address (op
);
2119 struct record_modified_bb_info info
;
2122 tree init
= ctor_for_folding (base
);
2124 if (init
!= error_mark_node
)
2127 return REG_BR_PROB_BASE
;
2128 ao_ref_init (&refd
, op
);
2130 info
.bb_set
= BITMAP_ALLOC (NULL
);
2131 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2133 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2135 BITMAP_FREE (info
.bb_set
);
2136 return REG_BR_PROB_BASE
;
2139 /* Assume that every memory is initialized at entry.
2140 TODO: Can we easilly determine if value is always defined
2141 and thus we may skip entry block? */
2142 if (ENTRY_BLOCK_PTR
->frequency
)
2143 max
= ENTRY_BLOCK_PTR
->frequency
;
2147 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2148 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2150 BITMAP_FREE (info
.bb_set
);
2151 if (max
< bb
->frequency
)
2152 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->frequency
), 1);
2154 return REG_BR_PROB_BASE
;
2156 return REG_BR_PROB_BASE
;
2159 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2160 sub-graph and if the predicate the condition depends on is known. If so,
2161 return true and store the pointer the predicate in *P. */
2164 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2165 struct inline_summary
*summary
, basic_block bb
,
2166 struct predicate
*p
,
2167 vec
<predicate_t
> nonconstant_names
)
2171 basic_block first_bb
= NULL
;
2174 if (single_pred_p (bb
))
2176 *p
= false_predicate ();
2180 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2182 if (single_succ_p (e
->src
))
2184 if (!single_pred_p (e
->src
))
2187 first_bb
= single_pred (e
->src
);
2188 else if (single_pred (e
->src
) != first_bb
)
2195 else if (e
->src
!= first_bb
)
2203 stmt
= last_stmt (first_bb
);
2205 || gimple_code (stmt
) != GIMPLE_COND
2206 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2209 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2210 gimple_cond_lhs (stmt
),
2212 if (true_predicate_p (p
))
2218 /* Given a PHI statement in a function described by inline properties SUMMARY
2219 and *P being the predicate describing whether the selected PHI argument is
2220 known, store a predicate for the result of the PHI statement into
2221 NONCONSTANT_NAMES, if possible. */
2224 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2225 struct predicate
*p
,
2226 vec
<predicate_t
> nonconstant_names
)
2230 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2232 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2233 if (!is_gimple_min_invariant (arg
))
2235 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2236 *p
= or_predicates (summary
->conds
, p
,
2237 &nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2238 if (true_predicate_p (p
))
2243 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2245 fprintf (dump_file
, "\t\tphi predicate: ");
2246 dump_predicate (dump_file
, summary
->conds
, p
);
2248 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2251 /* Return predicate specifying when array index in access OP becomes non-constant. */
2253 static struct predicate
2254 array_index_predicate (struct inline_summary
*info
,
2255 vec
< predicate_t
> nonconstant_names
, tree op
)
2257 struct predicate p
= false_predicate ();
2258 while (handled_component_p (op
))
2260 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
2262 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2263 p
= or_predicates (info
->conds
, &p
,
2264 &nonconstant_names
[SSA_NAME_VERSION
2265 (TREE_OPERAND (op
, 1))]);
2267 op
= TREE_OPERAND (op
, 0);
2272 /* For a typical usage of __builtin_expect (a<b, 1), we
2273 may introduce an extra relation stmt:
2274 With the builtin, we have
2277 t3 = __builtin_expect (t2, 1);
2280 Without the builtin, we have
2283 This affects the size/time estimation and may have
2284 an impact on the earlier inlining.
2285 Here find this pattern and fix it up later. */
2288 find_foldable_builtin_expect (basic_block bb
)
2290 gimple_stmt_iterator bsi
;
2292 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2294 gimple stmt
= gsi_stmt (bsi
);
2295 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
))
2297 tree var
= gimple_call_lhs (stmt
);
2298 tree arg
= gimple_call_arg (stmt
, 0);
2299 use_operand_p use_p
;
2306 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2308 while (TREE_CODE (arg
) == SSA_NAME
)
2310 gimple stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2311 if (!is_gimple_assign (stmt_tmp
))
2313 switch (gimple_assign_rhs_code (stmt_tmp
))
2332 arg
= gimple_assign_rhs1 (stmt_tmp
);
2335 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2336 && gimple_code (use_stmt
) == GIMPLE_COND
)
2343 /* Compute function body size parameters for NODE.
2344 When EARLY is true, we compute only simple summaries without
2345 non-trivial predicates to drive the early inliner. */
2348 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2351 /* Estimate static overhead for function prologue/epilogue and alignment. */
2353 /* Benefits are scaled by probability of elimination that is in range
2356 gimple_stmt_iterator bsi
;
2357 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2359 struct inline_summary
*info
= inline_summary (node
);
2360 struct predicate bb_predicate
;
2361 struct ipa_node_params
*parms_info
= NULL
;
2362 vec
<predicate_t
> nonconstant_names
= vNULL
;
2365 predicate array_index
= true_predicate ();
2366 gimple fix_builtin_expect_stmt
;
2371 if (optimize
&& !early
)
2373 calculate_dominance_info (CDI_DOMINATORS
);
2374 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2376 if (ipa_node_params_vector
.exists ())
2378 parms_info
= IPA_NODE_REF (node
);
2379 nonconstant_names
.safe_grow_cleared
2380 (SSANAMES (my_function
)->length ());
2385 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2388 /* When we run into maximal number of entries, we assign everything to the
2389 constant truth case. Be sure to have it in list. */
2390 bb_predicate
= true_predicate ();
2391 account_size_time (info
, 0, 0, &bb_predicate
);
2393 bb_predicate
= not_inlined_predicate ();
2394 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2396 gcc_assert (my_function
&& my_function
->cfg
);
2398 compute_bb_predicates (node
, parms_info
, info
);
2399 gcc_assert (cfun
== my_function
);
2400 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2401 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2402 for (n
= 0; n
< nblocks
; n
++)
2404 bb
= BASIC_BLOCK (order
[n
]);
2405 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
2407 /* TODO: Obviously predicates can be propagated down across CFG. */
2411 bb_predicate
= *(struct predicate
*) bb
->aux
;
2413 bb_predicate
= false_predicate ();
2416 bb_predicate
= true_predicate ();
2418 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2420 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2421 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2424 if (parms_info
&& nonconstant_names
.exists ())
2426 struct predicate phi_predicate
;
2427 bool first_phi
= true;
2429 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2432 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2437 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2439 fprintf (dump_file
, " ");
2440 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2442 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2447 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2449 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2451 gimple stmt
= gsi_stmt (bsi
);
2452 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2453 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2455 struct predicate will_be_nonconstant
;
2457 /* This relation stmt should be folded after we remove
2458 buildin_expect call. Adjust the cost here. */
2459 if (stmt
== fix_builtin_expect_stmt
)
2465 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2467 fprintf (dump_file
, " ");
2468 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2469 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2470 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2474 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2476 struct predicate this_array_index
;
2478 array_index_predicate (info
, nonconstant_names
,
2479 gimple_assign_rhs1 (stmt
));
2480 if (!false_predicate_p (&this_array_index
))
2482 and_predicates (info
->conds
, &array_index
,
2485 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2487 struct predicate this_array_index
;
2489 array_index_predicate (info
, nonconstant_names
,
2490 gimple_get_lhs (stmt
));
2491 if (!false_predicate_p (&this_array_index
))
2493 and_predicates (info
->conds
, &array_index
,
2498 if (is_gimple_call (stmt
))
2500 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2501 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2503 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2504 resolved as constant. We however don't want to optimize
2505 out the cgraph edges. */
2506 if (nonconstant_names
.exists ()
2507 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2508 && gimple_call_lhs (stmt
)
2509 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2511 struct predicate false_p
= false_predicate ();
2512 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2515 if (ipa_node_params_vector
.exists ())
2517 int count
= gimple_call_num_args (stmt
);
2521 es
->param
.safe_grow_cleared (count
);
2522 for (i
= 0; i
< count
; i
++)
2524 int prob
= param_change_prob (stmt
, i
);
2525 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2526 es
->param
[i
].change_prob
= prob
;
2530 es
->call_stmt_size
= this_size
;
2531 es
->call_stmt_time
= this_time
;
2532 es
->loop_depth
= bb_loop_depth (bb
);
2533 edge_set_predicate (edge
, &bb_predicate
);
2536 /* TODO: When conditional jump or swithc is known to be constant, but
2537 we did not translate it into the predicates, we really can account
2538 just maximum of the possible paths. */
2541 = will_be_nonconstant_predicate (parms_info
, info
,
2542 stmt
, nonconstant_names
);
2543 if (this_time
|| this_size
)
2549 prob
= eliminated_by_inlining_prob (stmt
);
2550 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2552 "\t\t50%% will be eliminated by inlining\n");
2553 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2554 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2557 p
= and_predicates (info
->conds
, &bb_predicate
,
2558 &will_be_nonconstant
);
2560 p
= true_predicate ();
2562 if (!false_predicate_p (&p
))
2566 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2567 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2570 /* We account everything but the calls. Calls have their own
2571 size/time info attached to cgraph edges. This is necessary
2572 in order to make the cost disappear after inlining. */
2573 if (!is_gimple_call (stmt
))
2577 struct predicate ip
= not_inlined_predicate ();
2578 ip
= and_predicates (info
->conds
, &ip
, &p
);
2579 account_size_time (info
, this_size
* prob
,
2580 this_time
* prob
, &ip
);
2583 account_size_time (info
, this_size
* (2 - prob
),
2584 this_time
* (2 - prob
), &p
);
2587 gcc_assert (time
>= 0);
2588 gcc_assert (size
>= 0);
2592 set_hint_predicate (&inline_summary (node
)->array_index
, array_index
);
2593 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2594 if (time
> MAX_TIME
)
2598 if (!early
&& nonconstant_names
.exists ())
2602 predicate loop_iterations
= true_predicate ();
2603 predicate loop_stride
= true_predicate ();
2605 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2606 flow_loops_dump (dump_file
, NULL
, 0);
2608 FOR_EACH_LOOP (li
, loop
, 0)
2613 struct tree_niter_desc niter_desc
;
2614 basic_block
*body
= get_loop_body (loop
);
2615 bb_predicate
= *(struct predicate
*) loop
->header
->aux
;
2617 exits
= get_loop_exit_edges (loop
);
2618 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2619 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2620 && !is_gimple_min_invariant (niter_desc
.niter
))
2622 predicate will_be_nonconstant
2623 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2626 if (!true_predicate_p (&will_be_nonconstant
))
2627 will_be_nonconstant
= and_predicates (info
->conds
,
2629 &will_be_nonconstant
);
2630 if (!true_predicate_p (&will_be_nonconstant
)
2631 && !false_predicate_p (&will_be_nonconstant
))
2632 /* This is slightly inprecise. We may want to represent each
2633 loop with independent predicate. */
2635 and_predicates (info
->conds
, &loop_iterations
,
2636 &will_be_nonconstant
);
2640 for (i
= 0; i
< loop
->num_nodes
; i
++)
2642 gimple_stmt_iterator gsi
;
2643 bb_predicate
= *(struct predicate
*) body
[i
]->aux
;
2644 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2647 gimple stmt
= gsi_stmt (gsi
);
2652 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2654 predicate will_be_nonconstant
;
2657 (loop
, loop_containing_stmt (stmt
), use
, &iv
, true)
2658 || is_gimple_min_invariant (iv
.step
))
2661 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2664 if (!true_predicate_p (&will_be_nonconstant
))
2666 = and_predicates (info
->conds
,
2668 &will_be_nonconstant
);
2669 if (!true_predicate_p (&will_be_nonconstant
)
2670 && !false_predicate_p (&will_be_nonconstant
))
2671 /* This is slightly inprecise. We may want to represent
2672 each loop with independent predicate. */
2674 and_predicates (info
->conds
, &loop_stride
,
2675 &will_be_nonconstant
);
2681 set_hint_predicate (&inline_summary (node
)->loop_iterations
,
2683 set_hint_predicate (&inline_summary (node
)->loop_stride
, loop_stride
);
2686 FOR_ALL_BB_FN (bb
, my_function
)
2692 pool_free (edge_predicate_pool
, bb
->aux
);
2694 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2697 pool_free (edge_predicate_pool
, e
->aux
);
2701 inline_summary (node
)->self_time
= time
;
2702 inline_summary (node
)->self_size
= size
;
2703 nonconstant_names
.release ();
2704 if (optimize
&& !early
)
2706 loop_optimizer_finalize ();
2707 free_dominance_info (CDI_DOMINATORS
);
2711 fprintf (dump_file
, "\n");
2712 dump_inline_summary (dump_file
, node
);
2717 /* Compute parameters of functions used by inliner.
2718 EARLY is true when we compute parameters for the early inliner */
2721 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2723 HOST_WIDE_INT self_stack_size
;
2724 struct cgraph_edge
*e
;
2725 struct inline_summary
*info
;
2727 gcc_assert (!node
->global
.inlined_to
);
2729 inline_summary_alloc ();
2731 info
= inline_summary (node
);
2732 reset_inline_summary (node
);
2734 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2735 Once this happen, we will need to more curefully predict call
2737 if (node
->thunk
.thunk_p
)
2739 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2740 struct predicate t
= true_predicate ();
2742 info
->inlinable
= 0;
2743 node
->callees
->call_stmt_cannot_inline_p
= true;
2744 node
->local
.can_change_signature
= false;
2745 es
->call_stmt_time
= 1;
2746 es
->call_stmt_size
= 1;
2747 account_size_time (info
, 0, 0, &t
);
2751 /* Even is_gimple_min_invariant rely on current_function_decl. */
2752 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2754 /* Estimate the stack size for the function if we're optimizing. */
2755 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2756 info
->estimated_self_stack_size
= self_stack_size
;
2757 info
->estimated_stack_size
= self_stack_size
;
2758 info
->stack_frame_offset
= 0;
2760 /* Can this function be inlined at all? */
2761 if (!optimize
&& !lookup_attribute ("always_inline",
2762 DECL_ATTRIBUTES (node
->decl
)))
2763 info
->inlinable
= false;
2765 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2767 /* Type attributes can use parameter indices to describe them. */
2768 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
2769 node
->local
.can_change_signature
= false;
2772 /* Otherwise, inlinable functions always can change signature. */
2773 if (info
->inlinable
)
2774 node
->local
.can_change_signature
= true;
2777 /* Functions calling builtin_apply can not change signature. */
2778 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2780 tree
cdecl = e
->callee
->decl
;
2781 if (DECL_BUILT_IN (cdecl)
2782 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2783 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2784 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2787 node
->local
.can_change_signature
= !e
;
2790 estimate_function_body_sizes (node
, early
);
2792 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2793 info
->time
= info
->self_time
;
2794 info
->size
= info
->self_size
;
2795 info
->stack_frame_offset
= 0;
2796 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2797 #ifdef ENABLE_CHECKING
2798 inline_update_overall_summary (node
);
2799 gcc_assert (info
->time
== info
->self_time
&& info
->size
== info
->self_size
);
2806 /* Compute parameters of functions used by inliner using
2807 current_function_decl. */
2810 compute_inline_parameters_for_current (void)
2812 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2818 const pass_data pass_data_inline_parameters
=
2820 GIMPLE_PASS
, /* type */
2821 "inline_param", /* name */
2822 OPTGROUP_INLINE
, /* optinfo_flags */
2823 false, /* has_gate */
2824 true, /* has_execute */
2825 TV_INLINE_PARAMETERS
, /* tv_id */
2826 0, /* properties_required */
2827 0, /* properties_provided */
2828 0, /* properties_destroyed */
2829 0, /* todo_flags_start */
2830 0, /* todo_flags_finish */
2833 class pass_inline_parameters
: public gimple_opt_pass
2836 pass_inline_parameters (gcc::context
*ctxt
)
2837 : gimple_opt_pass (pass_data_inline_parameters
, ctxt
)
2840 /* opt_pass methods: */
2841 opt_pass
* clone () { return new pass_inline_parameters (m_ctxt
); }
2842 unsigned int execute () {
2843 return compute_inline_parameters_for_current ();
2846 }; // class pass_inline_parameters
2851 make_pass_inline_parameters (gcc::context
*ctxt
)
2853 return new pass_inline_parameters (ctxt
);
2857 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2861 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2862 int *size
, int *time
,
2863 vec
<tree
> known_vals
,
2864 vec
<tree
> known_binfos
,
2865 vec
<ipa_agg_jump_function_p
> known_aggs
)
2868 struct cgraph_node
*callee
;
2869 struct inline_summary
*isummary
;
2871 if (!known_vals
.exists () && !known_binfos
.exists ())
2873 if (!flag_indirect_inlining
)
2876 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2881 /* Account for difference in cost between indirect and direct calls. */
2882 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2883 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2884 gcc_checking_assert (*time
>= 0);
2885 gcc_checking_assert (*size
>= 0);
2887 callee
= cgraph_get_node (target
);
2888 if (!callee
|| !callee
->definition
)
2890 isummary
= inline_summary (callee
);
2891 return isummary
->inlinable
;
2894 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2897 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2899 vec
<tree
> known_vals
,
2900 vec
<tree
> known_binfos
,
2901 vec
<ipa_agg_jump_function_p
> known_aggs
,
2902 inline_hints
*hints
)
2904 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2905 int call_size
= es
->call_stmt_size
;
2906 int call_time
= es
->call_stmt_time
;
2908 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2909 known_vals
, known_binfos
, known_aggs
)
2910 && hints
&& cgraph_maybe_hot_edge_p (e
))
2911 *hints
|= INLINE_HINT_indirect_call
;
2912 *size
+= call_size
* INLINE_SIZE_SCALE
;
2913 *time
+= apply_probability ((gcov_type
) call_time
, prob
)
2914 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
);
2915 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2916 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2921 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2922 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2926 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2927 inline_hints
*hints
,
2928 clause_t possible_truths
,
2929 vec
<tree
> known_vals
,
2930 vec
<tree
> known_binfos
,
2931 vec
<ipa_agg_jump_function_p
> known_aggs
)
2933 struct cgraph_edge
*e
;
2934 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2936 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2938 || evaluate_predicate (es
->predicate
, possible_truths
))
2940 if (e
->inline_failed
)
2942 /* Predicates of calls shall not use NOT_CHANGED codes,
2943 sowe do not need to compute probabilities. */
2944 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2945 known_vals
, known_binfos
,
2949 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2951 known_vals
, known_binfos
,
2955 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2957 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2959 || evaluate_predicate (es
->predicate
, possible_truths
))
2960 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2961 known_vals
, known_binfos
, known_aggs
,
2967 /* Estimate size and time needed to execute NODE assuming
2968 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2969 about NODE's arguments. */
2972 estimate_node_size_and_time (struct cgraph_node
*node
,
2973 clause_t possible_truths
,
2974 vec
<tree
> known_vals
,
2975 vec
<tree
> known_binfos
,
2976 vec
<ipa_agg_jump_function_p
> known_aggs
,
2977 int *ret_size
, int *ret_time
,
2978 inline_hints
*ret_hints
,
2979 vec
<inline_param_summary_t
>
2980 inline_param_summary
)
2982 struct inline_summary
*info
= inline_summary (node
);
2986 inline_hints hints
= 0;
2989 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2992 fprintf (dump_file
, " Estimating body: %s/%i\n"
2993 " Known to be false: ", node
->name (),
2996 for (i
= predicate_not_inlined_condition
;
2997 i
< (predicate_first_dynamic_condition
2998 + (int) vec_safe_length (info
->conds
)); i
++)
2999 if (!(possible_truths
& (1 << i
)))
3002 fprintf (dump_file
, ", ");
3004 dump_condition (dump_file
, info
->conds
, i
);
3008 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3009 if (evaluate_predicate (&e
->predicate
, possible_truths
))
3012 gcc_checking_assert (e
->time
>= 0);
3013 gcc_checking_assert (time
>= 0);
3014 if (!inline_param_summary
.exists ())
3018 int prob
= predicate_probability (info
->conds
,
3021 inline_param_summary
);
3022 gcc_checking_assert (prob
>= 0);
3023 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3024 time
+= apply_probability ((gcov_type
) e
->time
, prob
);
3026 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
3027 time
= MAX_TIME
* INLINE_TIME_SCALE
;
3028 gcc_checking_assert (time
>= 0);
3031 gcc_checking_assert (size
>= 0);
3032 gcc_checking_assert (time
>= 0);
3034 if (info
->loop_iterations
3035 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
3036 hints
|= INLINE_HINT_loop_iterations
;
3037 if (info
->loop_stride
3038 && !evaluate_predicate (info
->loop_stride
, possible_truths
))
3039 hints
|= INLINE_HINT_loop_stride
;
3040 if (info
->array_index
3041 && !evaluate_predicate (info
->array_index
, possible_truths
))
3042 hints
|= INLINE_HINT_array_index
;
3044 hints
|= INLINE_HINT_in_scc
;
3045 if (DECL_DECLARED_INLINE_P (node
->decl
))
3046 hints
|= INLINE_HINT_declared_inline
;
3048 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
3049 known_vals
, known_binfos
, known_aggs
);
3050 gcc_checking_assert (size
>= 0);
3051 gcc_checking_assert (time
>= 0);
3052 time
= RDIV (time
, INLINE_TIME_SCALE
);
3053 size
= RDIV (size
, INLINE_SIZE_SCALE
);
3055 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3056 fprintf (dump_file
, "\n size:%i time:%i\n", (int) size
, (int) time
);
3067 /* Estimate size and time needed to execute callee of EDGE assuming that
3068 parameters known to be constant at caller of EDGE are propagated.
3069 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
3070 and types for parameters. */
3073 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
3074 vec
<tree
> known_vals
,
3075 vec
<tree
> known_binfos
,
3076 vec
<ipa_agg_jump_function_p
> known_aggs
,
3077 int *ret_size
, int *ret_time
,
3078 inline_hints
*hints
)
3082 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
,
3084 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
,
3085 known_aggs
, ret_size
, ret_time
, hints
, vNULL
);
3088 /* Translate all conditions from callee representation into caller
3089 representation and symbolically evaluate predicate P into new predicate.
3091 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
3092 is summary of function predicate P is from. OPERAND_MAP is array giving
3093 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
3094 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
3095 predicate under which callee is executed. OFFSET_MAP is an array of of
3096 offsets that need to be added to conditions, negative offset means that
3097 conditions relying on values passed by reference have to be discarded
3098 because they might not be preserved (and should be considered offset zero
3099 for other purposes). */
3101 static struct predicate
3102 remap_predicate (struct inline_summary
*info
,
3103 struct inline_summary
*callee_info
,
3104 struct predicate
*p
,
3105 vec
<int> operand_map
,
3106 vec
<int> offset_map
,
3107 clause_t possible_truths
, struct predicate
*toplev_predicate
)
3110 struct predicate out
= true_predicate ();
3112 /* True predicate is easy. */
3113 if (true_predicate_p (p
))
3114 return *toplev_predicate
;
3115 for (i
= 0; p
->clause
[i
]; i
++)
3117 clause_t clause
= p
->clause
[i
];
3119 struct predicate clause_predicate
= false_predicate ();
3121 gcc_assert (i
< MAX_CLAUSES
);
3123 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
3124 /* Do we have condition we can't disprove? */
3125 if (clause
& possible_truths
& (1 << cond
))
3127 struct predicate cond_predicate
;
3128 /* Work out if the condition can translate to predicate in the
3129 inlined function. */
3130 if (cond
>= predicate_first_dynamic_condition
)
3132 struct condition
*c
;
3134 c
= &(*callee_info
->conds
)[cond
3136 predicate_first_dynamic_condition
];
3137 /* See if we can remap condition operand to caller's operand.
3138 Otherwise give up. */
3139 if (!operand_map
.exists ()
3140 || (int) operand_map
.length () <= c
->operand_num
3141 || operand_map
[c
->operand_num
] == -1
3142 /* TODO: For non-aggregate conditions, adding an offset is
3143 basically an arithmetic jump function processing which
3144 we should support in future. */
3145 || ((!c
->agg_contents
|| !c
->by_ref
)
3146 && offset_map
[c
->operand_num
] > 0)
3147 || (c
->agg_contents
&& c
->by_ref
3148 && offset_map
[c
->operand_num
] < 0))
3149 cond_predicate
= true_predicate ();
3152 struct agg_position_info ap
;
3153 HOST_WIDE_INT offset_delta
= offset_map
[c
->operand_num
];
3154 if (offset_delta
< 0)
3156 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
3159 gcc_assert (!c
->agg_contents
3160 || c
->by_ref
|| offset_delta
== 0);
3161 ap
.offset
= c
->offset
+ offset_delta
;
3162 ap
.agg_contents
= c
->agg_contents
;
3163 ap
.by_ref
= c
->by_ref
;
3164 cond_predicate
= add_condition (info
,
3165 operand_map
[c
->operand_num
],
3166 &ap
, c
->code
, c
->val
);
3169 /* Fixed conditions remains same, construct single
3170 condition predicate. */
3173 cond_predicate
.clause
[0] = 1 << cond
;
3174 cond_predicate
.clause
[1] = 0;
3176 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
3179 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
3181 return and_predicates (info
->conds
, &out
, toplev_predicate
);
3185 /* Update summary information of inline clones after inlining.
3186 Compute peak stack usage. */
3189 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3191 struct cgraph_edge
*e
;
3192 struct inline_summary
*callee_info
= inline_summary (node
);
3193 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
3196 callee_info
->stack_frame_offset
3197 = caller_info
->stack_frame_offset
3198 + caller_info
->estimated_self_stack_size
;
3199 peak
= callee_info
->stack_frame_offset
3200 + callee_info
->estimated_self_stack_size
;
3201 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
3202 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
3203 ipa_propagate_frequency (node
);
3204 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3206 if (!e
->inline_failed
)
3207 inline_update_callee_summaries (e
->callee
, depth
);
3208 inline_edge_summary (e
)->loop_depth
+= depth
;
3210 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3211 inline_edge_summary (e
)->loop_depth
+= depth
;
3214 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3215 When functoin A is inlined in B and A calls C with parameter that
3216 changes with probability PROB1 and C is known to be passthroug
3217 of argument if B that change with probability PROB2, the probability
3218 of change is now PROB1*PROB2. */
3221 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3222 struct cgraph_edge
*edge
)
3224 if (ipa_node_params_vector
.exists ())
3227 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3228 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3229 struct inline_edge_summary
*inlined_es
3230 = inline_edge_summary (inlined_edge
);
3232 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3234 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3235 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3236 && (ipa_get_jf_pass_through_formal_id (jfunc
)
3237 < (int) inlined_es
->param
.length ()))
3239 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
3240 int prob1
= es
->param
[i
].change_prob
;
3241 int prob2
= inlined_es
->param
[jf_formal_id
].change_prob
;
3242 int prob
= combine_probabilities (prob1
, prob2
);
3244 if (prob1
&& prob2
&& !prob
)
3247 es
->param
[i
].change_prob
= prob
;
3253 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3255 Remap predicates of callees of NODE. Rest of arguments match
3258 Also update change probabilities. */
3261 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3262 struct cgraph_node
*node
,
3263 struct inline_summary
*info
,
3264 struct inline_summary
*callee_info
,
3265 vec
<int> operand_map
,
3266 vec
<int> offset_map
,
3267 clause_t possible_truths
,
3268 struct predicate
*toplev_predicate
)
3270 struct cgraph_edge
*e
;
3271 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3273 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3276 if (e
->inline_failed
)
3278 remap_edge_change_prob (inlined_edge
, e
);
3282 p
= remap_predicate (info
, callee_info
,
3283 es
->predicate
, operand_map
, offset_map
,
3284 possible_truths
, toplev_predicate
);
3285 edge_set_predicate (e
, &p
);
3286 /* TODO: We should remove the edge for code that will be
3287 optimized out, but we need to keep verifiers and tree-inline
3288 happy. Make it cold for now. */
3289 if (false_predicate_p (&p
))
3296 edge_set_predicate (e
, toplev_predicate
);
3299 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3300 operand_map
, offset_map
, possible_truths
,
3303 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3305 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3308 remap_edge_change_prob (inlined_edge
, e
);
3311 p
= remap_predicate (info
, callee_info
,
3312 es
->predicate
, operand_map
, offset_map
,
3313 possible_truths
, toplev_predicate
);
3314 edge_set_predicate (e
, &p
);
3315 /* TODO: We should remove the edge for code that will be optimized
3316 out, but we need to keep verifiers and tree-inline happy.
3317 Make it cold for now. */
3318 if (false_predicate_p (&p
))
3325 edge_set_predicate (e
, toplev_predicate
);
3329 /* Same as remap_predicate, but set result into hint *HINT. */
3332 remap_hint_predicate (struct inline_summary
*info
,
3333 struct inline_summary
*callee_info
,
3334 struct predicate
**hint
,
3335 vec
<int> operand_map
,
3336 vec
<int> offset_map
,
3337 clause_t possible_truths
,
3338 struct predicate
*toplev_predicate
)
3344 p
= remap_predicate (info
, callee_info
,
3346 operand_map
, offset_map
,
3347 possible_truths
, toplev_predicate
);
3348 if (!false_predicate_p (&p
) && !true_predicate_p (&p
))
3351 set_hint_predicate (hint
, p
);
3353 **hint
= and_predicates (info
->conds
, *hint
, &p
);
3357 /* We inlined EDGE. Update summary of the function we inlined into. */
3360 inline_merge_summary (struct cgraph_edge
*edge
)
3362 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3363 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3364 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3365 struct inline_summary
*info
= inline_summary (to
);
3366 clause_t clause
= 0; /* not_inline is known to be false. */
3368 vec
<int> operand_map
= vNULL
;
3369 vec
<int> offset_map
= vNULL
;
3371 struct predicate toplev_predicate
;
3372 struct predicate true_p
= true_predicate ();
3373 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3376 toplev_predicate
= *es
->predicate
;
3378 toplev_predicate
= true_predicate ();
3380 if (ipa_node_params_vector
.exists () && callee_info
->conds
)
3382 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3383 int count
= ipa_get_cs_argument_count (args
);
3386 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3389 operand_map
.safe_grow_cleared (count
);
3390 offset_map
.safe_grow_cleared (count
);
3392 for (i
= 0; i
< count
; i
++)
3394 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3397 /* TODO: handle non-NOPs when merging. */
3398 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3400 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3401 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3402 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3405 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3407 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3408 if (offset
>= 0 && offset
< INT_MAX
)
3410 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3411 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3413 offset_map
[i
] = offset
;
3416 operand_map
[i
] = map
;
3417 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3420 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3422 struct predicate p
= remap_predicate (info
, callee_info
,
3423 &e
->predicate
, operand_map
,
3426 if (!false_predicate_p (&p
))
3428 gcov_type add_time
= ((gcov_type
) e
->time
* edge
->frequency
3429 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3430 int prob
= predicate_probability (callee_info
->conds
,
3433 add_time
= apply_probability ((gcov_type
) add_time
, prob
);
3434 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3435 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3436 if (prob
!= REG_BR_PROB_BASE
3437 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3439 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3440 (double) prob
/ REG_BR_PROB_BASE
);
3442 account_size_time (info
, e
->size
, add_time
, &p
);
3445 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3446 offset_map
, clause
, &toplev_predicate
);
3447 remap_hint_predicate (info
, callee_info
,
3448 &callee_info
->loop_iterations
,
3449 operand_map
, offset_map
, clause
, &toplev_predicate
);
3450 remap_hint_predicate (info
, callee_info
,
3451 &callee_info
->loop_stride
,
3452 operand_map
, offset_map
, clause
, &toplev_predicate
);
3453 remap_hint_predicate (info
, callee_info
,
3454 &callee_info
->array_index
,
3455 operand_map
, offset_map
, clause
, &toplev_predicate
);
3457 inline_update_callee_summaries (edge
->callee
,
3458 inline_edge_summary (edge
)->loop_depth
);
3460 /* We do not maintain predicates of inlined edges, free it. */
3461 edge_set_predicate (edge
, &true_p
);
3462 /* Similarly remove param summaries. */
3463 es
->param
.release ();
3464 operand_map
.release ();
3465 offset_map
.release ();
3468 /* For performance reasons inline_merge_summary is not updating overall size
3469 and time. Recompute it. */
3472 inline_update_overall_summary (struct cgraph_node
*node
)
3474 struct inline_summary
*info
= inline_summary (node
);
3480 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3482 info
->size
+= e
->size
, info
->time
+= e
->time
;
3483 if (info
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
3484 info
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
3486 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3487 ~(clause_t
) (1 << predicate_false_condition
),
3488 vNULL
, vNULL
, vNULL
);
3489 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3490 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3493 /* Return hints derrived from EDGE. */
3495 simple_edge_hints (struct cgraph_edge
*edge
)
3498 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3499 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3500 if (inline_summary (to
)->scc_no
3501 && inline_summary (to
)->scc_no
== inline_summary (edge
->callee
)->scc_no
3502 && !cgraph_edge_recursive_p (edge
))
3503 hints
|= INLINE_HINT_same_scc
;
3505 if (to
->lto_file_data
&& edge
->callee
->lto_file_data
3506 && to
->lto_file_data
!= edge
->callee
->lto_file_data
)
3507 hints
|= INLINE_HINT_cross_module
;
3512 /* Estimate the time cost for the caller when inlining EDGE.
3513 Only to be called via estimate_edge_time, that handles the
3516 When caching, also update the cache entry. Compute both time and
3517 size, since we always need both metrics eventually. */
3520 do_estimate_edge_time (struct cgraph_edge
*edge
)
3525 struct cgraph_node
*callee
;
3527 vec
<tree
> known_vals
;
3528 vec
<tree
> known_binfos
;
3529 vec
<ipa_agg_jump_function_p
> known_aggs
;
3530 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3532 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3534 gcc_checking_assert (edge
->inline_failed
);
3535 evaluate_properties_for_edge (edge
, true,
3536 &clause
, &known_vals
, &known_binfos
,
3538 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3539 known_aggs
, &size
, &time
, &hints
, es
->param
);
3540 known_vals
.release ();
3541 known_binfos
.release ();
3542 known_aggs
.release ();
3543 gcc_checking_assert (size
>= 0);
3544 gcc_checking_assert (time
>= 0);
3546 /* When caching, update the cache entry. */
3547 if (edge_growth_cache
.exists ())
3549 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3550 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
3551 edge_growth_cache
[edge
->uid
].time
= time
+ (time
>= 0);
3553 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3554 hints
|= simple_edge_hints (edge
);
3555 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3561 /* Return estimated callee growth after inlining EDGE.
3562 Only to be called via estimate_edge_size. */
3565 do_estimate_edge_size (struct cgraph_edge
*edge
)
3568 struct cgraph_node
*callee
;
3570 vec
<tree
> known_vals
;
3571 vec
<tree
> known_binfos
;
3572 vec
<ipa_agg_jump_function_p
> known_aggs
;
3574 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3576 if (edge_growth_cache
.exists ())
3578 do_estimate_edge_time (edge
);
3579 size
= edge_growth_cache
[edge
->uid
].size
;
3580 gcc_checking_assert (size
);
3581 return size
- (size
> 0);
3584 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3586 /* Early inliner runs without caching, go ahead and do the dirty work. */
3587 gcc_checking_assert (edge
->inline_failed
);
3588 evaluate_properties_for_edge (edge
, true,
3589 &clause
, &known_vals
, &known_binfos
,
3591 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3592 known_aggs
, &size
, NULL
, NULL
, vNULL
);
3593 known_vals
.release ();
3594 known_binfos
.release ();
3595 known_aggs
.release ();
3600 /* Estimate the growth of the caller when inlining EDGE.
3601 Only to be called via estimate_edge_size. */
3604 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3607 struct cgraph_node
*callee
;
3609 vec
<tree
> known_vals
;
3610 vec
<tree
> known_binfos
;
3611 vec
<ipa_agg_jump_function_p
> known_aggs
;
3613 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3615 if (edge_growth_cache
.exists ())
3617 do_estimate_edge_time (edge
);
3618 hints
= edge_growth_cache
[edge
->uid
].hints
;
3619 gcc_checking_assert (hints
);
3623 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3625 /* Early inliner runs without caching, go ahead and do the dirty work. */
3626 gcc_checking_assert (edge
->inline_failed
);
3627 evaluate_properties_for_edge (edge
, true,
3628 &clause
, &known_vals
, &known_binfos
,
3630 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3631 known_aggs
, NULL
, NULL
, &hints
, vNULL
);
3632 known_vals
.release ();
3633 known_binfos
.release ();
3634 known_aggs
.release ();
3635 hints
|= simple_edge_hints (edge
);
3640 /* Estimate self time of the function NODE after inlining EDGE. */
3643 estimate_time_after_inlining (struct cgraph_node
*node
,
3644 struct cgraph_edge
*edge
)
3646 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3647 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3650 inline_summary (node
)->time
+ estimate_edge_time (edge
);
3653 if (time
> MAX_TIME
)
3657 return inline_summary (node
)->time
;
3661 /* Estimate the size of NODE after inlining EDGE which should be an
3662 edge to either NODE or a call inlined into NODE. */
3665 estimate_size_after_inlining (struct cgraph_node
*node
,
3666 struct cgraph_edge
*edge
)
3668 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3669 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3671 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3672 gcc_assert (size
>= 0);
3675 return inline_summary (node
)->size
;
3681 struct cgraph_node
*node
;
3682 bool self_recursive
;
3687 /* Worker for do_estimate_growth. Collect growth for all callers. */
3690 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3692 struct cgraph_edge
*e
;
3693 struct growth_data
*d
= (struct growth_data
*) data
;
3695 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3697 gcc_checking_assert (e
->inline_failed
);
3699 if (e
->caller
== d
->node
3700 || (e
->caller
->global
.inlined_to
3701 && e
->caller
->global
.inlined_to
== d
->node
))
3702 d
->self_recursive
= true;
3703 d
->growth
+= estimate_edge_growth (e
);
3709 /* Estimate the growth caused by inlining NODE into all callees. */
3712 do_estimate_growth (struct cgraph_node
*node
)
3714 struct growth_data d
= { node
, 0, false };
3715 struct inline_summary
*info
= inline_summary (node
);
3717 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3719 /* For self recursive functions the growth estimation really should be
3720 infinity. We don't want to return very large values because the growth
3721 plays various roles in badness computation fractions. Be sure to not
3722 return zero or negative growths. */
3723 if (d
.self_recursive
)
3724 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3725 else if (DECL_EXTERNAL (node
->decl
))
3729 if (cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3730 d
.growth
-= info
->size
;
3731 /* COMDAT functions are very often not shared across multiple units
3732 since they come from various template instantiations.
3733 Take this into account. */
3734 else if (DECL_COMDAT (node
->decl
)
3735 && cgraph_can_remove_if_no_direct_calls_p (node
))
3736 d
.growth
-= (info
->size
3737 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3741 if (node_growth_cache
.exists ())
3743 if ((int) node_growth_cache
.length () <= node
->uid
)
3744 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
3745 node_growth_cache
[node
->uid
] = d
.growth
+ (d
.growth
>= 0);
3751 /* This function performs intraprocedural analysis in NODE that is required to
3752 inline indirect calls. */
3755 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3757 ipa_analyze_node (node
);
3758 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3760 ipa_print_node_params (dump_file
, node
);
3761 ipa_print_node_jump_functions (dump_file
, node
);
3766 /* Note function body size. */
3769 inline_analyze_function (struct cgraph_node
*node
)
3771 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3774 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3775 node
->name (), node
->order
);
3776 if (optimize
&& !node
->thunk
.thunk_p
)
3777 inline_indirect_intraprocedural_analysis (node
);
3778 compute_inline_parameters (node
, false);
3781 struct cgraph_edge
*e
;
3782 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3784 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3785 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3786 e
->call_stmt_cannot_inline_p
= true;
3788 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3790 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3791 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3792 e
->call_stmt_cannot_inline_p
= true;
3800 /* Called when new function is inserted to callgraph late. */
3803 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3805 inline_analyze_function (node
);
3809 /* Note function body size. */
3812 inline_generate_summary (void)
3814 struct cgraph_node
*node
;
3816 /* When not optimizing, do not bother to analyze. Inlining is still done
3817 because edge redirection needs to happen there. */
3818 if (!optimize
&& !flag_lto
&& !flag_wpa
)
3821 function_insertion_hook_holder
=
3822 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3824 ipa_register_cgraph_hooks ();
3825 inline_free_summary ();
3827 FOR_EACH_DEFINED_FUNCTION (node
)
3829 inline_analyze_function (node
);
3833 /* Read predicate from IB. */
3835 static struct predicate
3836 read_predicate (struct lto_input_block
*ib
)
3838 struct predicate out
;
3844 gcc_assert (k
<= MAX_CLAUSES
);
3845 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3849 /* Zero-initialize the remaining clauses in OUT. */
3850 while (k
<= MAX_CLAUSES
)
3851 out
.clause
[k
++] = 0;
3857 /* Write inline summary for edge E to OB. */
3860 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3862 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3866 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3867 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3868 es
->loop_depth
= streamer_read_uhwi (ib
);
3869 p
= read_predicate (ib
);
3870 edge_set_predicate (e
, &p
);
3871 length
= streamer_read_uhwi (ib
);
3874 es
->param
.safe_grow_cleared (length
);
3875 for (i
= 0; i
< length
; i
++)
3876 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3881 /* Stream in inline summaries from the section. */
3884 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3887 const struct lto_function_header
*header
=
3888 (const struct lto_function_header
*) data
;
3889 const int cfg_offset
= sizeof (struct lto_function_header
);
3890 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3891 const int string_offset
= main_offset
+ header
->main_size
;
3892 struct data_in
*data_in
;
3893 struct lto_input_block ib
;
3894 unsigned int i
, count2
, j
;
3895 unsigned int f_count
;
3897 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3901 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3902 header
->string_size
, vNULL
);
3903 f_count
= streamer_read_uhwi (&ib
);
3904 for (i
= 0; i
< f_count
; i
++)
3907 struct cgraph_node
*node
;
3908 struct inline_summary
*info
;
3909 lto_symtab_encoder_t encoder
;
3910 struct bitpack_d bp
;
3911 struct cgraph_edge
*e
;
3914 index
= streamer_read_uhwi (&ib
);
3915 encoder
= file_data
->symtab_node_encoder
;
3916 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3917 info
= inline_summary (node
);
3919 info
->estimated_stack_size
3920 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3921 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3922 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3924 bp
= streamer_read_bitpack (&ib
);
3925 info
->inlinable
= bp_unpack_value (&bp
, 1);
3927 count2
= streamer_read_uhwi (&ib
);
3928 gcc_assert (!info
->conds
);
3929 for (j
= 0; j
< count2
; j
++)
3932 c
.operand_num
= streamer_read_uhwi (&ib
);
3933 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3934 c
.val
= stream_read_tree (&ib
, data_in
);
3935 bp
= streamer_read_bitpack (&ib
);
3936 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3937 c
.by_ref
= bp_unpack_value (&bp
, 1);
3939 c
.offset
= streamer_read_uhwi (&ib
);
3940 vec_safe_push (info
->conds
, c
);
3942 count2
= streamer_read_uhwi (&ib
);
3943 gcc_assert (!info
->entry
);
3944 for (j
= 0; j
< count2
; j
++)
3946 struct size_time_entry e
;
3948 e
.size
= streamer_read_uhwi (&ib
);
3949 e
.time
= streamer_read_uhwi (&ib
);
3950 e
.predicate
= read_predicate (&ib
);
3952 vec_safe_push (info
->entry
, e
);
3955 p
= read_predicate (&ib
);
3956 set_hint_predicate (&info
->loop_iterations
, p
);
3957 p
= read_predicate (&ib
);
3958 set_hint_predicate (&info
->loop_stride
, p
);
3959 p
= read_predicate (&ib
);
3960 set_hint_predicate (&info
->array_index
, p
);
3961 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3962 read_inline_edge_summary (&ib
, e
);
3963 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3964 read_inline_edge_summary (&ib
, e
);
3967 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3969 lto_data_in_delete (data_in
);
3973 /* Read inline summary. Jump functions are shared among ipa-cp
3974 and inliner, so when ipa-cp is active, we don't need to write them
3978 inline_read_summary (void)
3980 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3981 struct lto_file_decl_data
*file_data
;
3984 inline_summary_alloc ();
3986 while ((file_data
= file_data_vec
[j
++]))
3989 const char *data
= lto_get_section_data (file_data
,
3990 LTO_section_inline_summary
,
3993 inline_read_section (file_data
, data
, len
);
3995 /* Fatal error here. We do not want to support compiling ltrans units
3996 with different version of compiler or different flags than the WPA
3997 unit, so this should never happen. */
3998 fatal_error ("ipa inline summary is missing in input file");
4002 ipa_register_cgraph_hooks ();
4004 ipa_prop_read_jump_functions ();
4006 function_insertion_hook_holder
=
4007 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
4011 /* Write predicate P to OB. */
4014 write_predicate (struct output_block
*ob
, struct predicate
*p
)
4018 for (j
= 0; p
->clause
[j
]; j
++)
4020 gcc_assert (j
< MAX_CLAUSES
);
4021 streamer_write_uhwi (ob
, p
->clause
[j
]);
4023 streamer_write_uhwi (ob
, 0);
4027 /* Write inline summary for edge E to OB. */
4030 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4032 struct inline_edge_summary
*es
= inline_edge_summary (e
);
4035 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4036 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4037 streamer_write_uhwi (ob
, es
->loop_depth
);
4038 write_predicate (ob
, es
->predicate
);
4039 streamer_write_uhwi (ob
, es
->param
.length ());
4040 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4041 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4045 /* Write inline summary for node in SET.
4046 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4047 active, we don't need to write them twice. */
4050 inline_write_summary (void)
4052 struct cgraph_node
*node
;
4053 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
4054 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4055 unsigned int count
= 0;
4058 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4060 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4061 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4062 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4065 streamer_write_uhwi (ob
, count
);
4067 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4069 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4070 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4071 if (cnode
&& (node
= cnode
)->definition
&& !node
->alias
)
4073 struct inline_summary
*info
= inline_summary (node
);
4074 struct bitpack_d bp
;
4075 struct cgraph_edge
*edge
;
4078 struct condition
*c
;
4080 streamer_write_uhwi (ob
,
4081 lto_symtab_encoder_encode (encoder
,
4084 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
4085 streamer_write_hwi (ob
, info
->self_size
);
4086 streamer_write_hwi (ob
, info
->self_time
);
4087 bp
= bitpack_create (ob
->main_stream
);
4088 bp_pack_value (&bp
, info
->inlinable
, 1);
4089 streamer_write_bitpack (&bp
);
4090 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4091 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4093 streamer_write_uhwi (ob
, c
->operand_num
);
4094 streamer_write_uhwi (ob
, c
->code
);
4095 stream_write_tree (ob
, c
->val
, true);
4096 bp
= bitpack_create (ob
->main_stream
);
4097 bp_pack_value (&bp
, c
->agg_contents
, 1);
4098 bp_pack_value (&bp
, c
->by_ref
, 1);
4099 streamer_write_bitpack (&bp
);
4100 if (c
->agg_contents
)
4101 streamer_write_uhwi (ob
, c
->offset
);
4103 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
4104 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
4106 streamer_write_uhwi (ob
, e
->size
);
4107 streamer_write_uhwi (ob
, e
->time
);
4108 write_predicate (ob
, &e
->predicate
);
4110 write_predicate (ob
, info
->loop_iterations
);
4111 write_predicate (ob
, info
->loop_stride
);
4112 write_predicate (ob
, info
->array_index
);
4113 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
4114 write_inline_edge_summary (ob
, edge
);
4115 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4116 write_inline_edge_summary (ob
, edge
);
4119 streamer_write_char_stream (ob
->main_stream
, 0);
4120 produce_asm (ob
, NULL
);
4121 destroy_output_block (ob
);
4123 if (optimize
&& !flag_ipa_cp
)
4124 ipa_prop_write_jump_functions ();
4128 /* Release inline summary. */
4131 inline_free_summary (void)
4133 struct cgraph_node
*node
;
4134 if (!inline_edge_summary_vec
.exists ())
4136 FOR_EACH_DEFINED_FUNCTION (node
)
4137 reset_inline_summary (node
);
4138 if (function_insertion_hook_holder
)
4139 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
4140 function_insertion_hook_holder
= NULL
;
4141 if (node_removal_hook_holder
)
4142 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
4143 node_removal_hook_holder
= NULL
;
4144 if (edge_removal_hook_holder
)
4145 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
4146 edge_removal_hook_holder
= NULL
;
4147 if (node_duplication_hook_holder
)
4148 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
4149 node_duplication_hook_holder
= NULL
;
4150 if (edge_duplication_hook_holder
)
4151 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
4152 edge_duplication_hook_holder
= NULL
;
4153 vec_free (inline_summary_vec
);
4154 inline_edge_summary_vec
.release ();
4155 if (edge_predicate_pool
)
4156 free_alloc_pool (edge_predicate_pool
);
4157 edge_predicate_pool
= 0;