1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011
3 Free Software Foundation, Inc.
4 Contributed by Jan Hubicka
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Analysis used by the inliner and other passes limiting code size growth.
24 We estimate for each function
26 - average function execution time
27 - inlining size benefit (that is how much of function body size
28 and its call sequence is expected to disappear by inlining)
29 - inlining time benefit
32 - call statement size and time
34 inlinie_summary datastructures store above information locally (i.e.
35 parameters of the function itself) and globally (i.e. parameters of
36 the function created by applying all the inline decisions already
37 present in the callgraph).
39 We provide accestor to the inline_summary datastructure and
40 basic logic updating the parameters when inlining is performed.
42 The summaries are context sensitive. Context means
43 1) partial assignment of known constant values of operands
44 2) whether function is inlined into the call or not.
45 It is easy to add more variants. To represent function size and time
46 that depends on context (i.e. it is known to be optimized away when
47 context is known either by inlining or from IP-CP and clonning),
48 we use predicates. Predicates are logical formulas in
49 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
50 specifying what conditions must be true. Conditions are simple test
51 of the form described above.
53 In order to make predicate (possibly) true, all of its clauses must
54 be (possibly) true. To make clause (possibly) true, one of conditions
55 it mentions must be (possibly) true. There are fixed bounds on
56 number of clauses and conditions and all the manipulation functions
57 are conservative in positive direction. I.e. we may lose precision
58 by thinking that predicate may be true even when it is not.
60 estimate_edge_size and estimate_edge_growth can be used to query
61 function size/time in the given context. inline_merge_summary merges
62 properties of caller and callee after inlining.
64 Finally pass_inline_parameters is exported. This is used to drive
65 computation of function parameters used by the early inliner. IPA
66 inlined performs analysis via its analyze_function method. */
70 #include "coretypes.h"
73 #include "tree-inline.h"
74 #include "langhooks.h"
77 #include "diagnostic.h"
78 #include "gimple-pretty-print.h"
80 #include "tree-pass.h"
83 #include "tree-flow.h"
85 #include "lto-streamer.h"
86 #include "data-streamer.h"
87 #include "tree-streamer.h"
88 #include "ipa-inline.h"
89 #include "alloc-pool.h"
92 #include "tree-scalar-evolution.h"
94 /* Estimate runtime of function can easilly run into huge numbers with many
95 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
96 integer. For anything larger we use gcov_type. */
97 #define MAX_TIME 500000
99 /* Number of bits in integer, but we really want to be stable across different
101 #define NUM_CONDITIONS 32
103 enum predicate_conditions
105 predicate_false_condition
= 0,
106 predicate_not_inlined_condition
= 1,
107 predicate_first_dynamic_condition
= 2
110 /* Special condition code we use to represent test that operand is compile time
112 #define IS_NOT_CONSTANT ERROR_MARK
113 /* Special condition code we use to represent test that operand is not changed
114 across invocation of the function. When operand IS_NOT_CONSTANT it is always
115 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
116 of executions even when they are not compile time constants. */
117 #define CHANGED IDENTIFIER_NODE
119 /* Holders of ipa cgraph hooks: */
120 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
121 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
122 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
123 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
124 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
125 static void inline_node_removal_hook (struct cgraph_node
*, void *);
126 static void inline_node_duplication_hook (struct cgraph_node
*,
127 struct cgraph_node
*, void *);
128 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
129 static void inline_edge_duplication_hook (struct cgraph_edge
*,
130 struct cgraph_edge
*,
133 /* VECtor holding inline summaries.
134 In GGC memory because conditions might point to constant trees. */
135 vec
<inline_summary_t
, va_gc
> *inline_summary_vec
;
136 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
138 /* Cached node/edge growths. */
139 vec
<int> node_growth_cache
;
140 vec
<edge_growth_cache_entry
> edge_growth_cache
;
142 /* Edge predicates goes here. */
143 static alloc_pool edge_predicate_pool
;
145 /* Return true predicate (tautology).
146 We represent it by empty list of clauses. */
148 static inline struct predicate
149 true_predicate (void)
157 /* Return predicate testing single condition number COND. */
159 static inline struct predicate
160 single_cond_predicate (int cond
)
163 p
.clause
[0] = 1 << cond
;
169 /* Return false predicate. First clause require false condition. */
171 static inline struct predicate
172 false_predicate (void)
174 return single_cond_predicate (predicate_false_condition
);
178 /* Return true if P is (false). */
181 true_predicate_p (struct predicate
*p
)
183 return !p
->clause
[0];
187 /* Return true if P is (false). */
190 false_predicate_p (struct predicate
*p
)
192 if (p
->clause
[0] == (1 << predicate_false_condition
))
194 gcc_checking_assert (!p
->clause
[1]
195 && p
->clause
[0] == 1 << predicate_false_condition
);
202 /* Return predicate that is set true when function is not inlined. */
203 static inline struct predicate
204 not_inlined_predicate (void)
206 return single_cond_predicate (predicate_not_inlined_condition
);
209 /* Simple description of whether a memory load or a condition refers to a load
210 from an aggregate and if so, how and where from in the aggregate.
211 Individual fields have the same meaning like fields with the same name in
214 struct agg_position_info
216 HOST_WIDE_INT offset
;
221 /* Add condition to condition list CONDS. AGGPOS describes whether the used
222 oprand is loaded from an aggregate and where in the aggregate it is. It can
223 be NULL, which means this not a load from an aggregate. */
225 static struct predicate
226 add_condition (struct inline_summary
*summary
, int operand_num
,
227 struct agg_position_info
*aggpos
,
228 enum tree_code code
, tree val
)
232 struct condition new_cond
;
233 HOST_WIDE_INT offset
;
234 bool agg_contents
, by_ref
;
238 offset
= aggpos
->offset
;
239 agg_contents
= aggpos
->agg_contents
;
240 by_ref
= aggpos
->by_ref
;
245 agg_contents
= false;
249 gcc_checking_assert (operand_num
>= 0);
250 for (i
= 0; vec_safe_iterate (summary
->conds
, i
, &c
); i
++)
252 if (c
->operand_num
== operand_num
255 && c
->agg_contents
== agg_contents
256 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
257 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
259 /* Too many conditions. Give up and return constant true. */
260 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
261 return true_predicate ();
263 new_cond
.operand_num
= operand_num
;
264 new_cond
.code
= code
;
266 new_cond
.agg_contents
= agg_contents
;
267 new_cond
.by_ref
= by_ref
;
268 new_cond
.offset
= offset
;
269 vec_safe_push (summary
->conds
, new_cond
);
270 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
274 /* Add clause CLAUSE into the predicate P. */
277 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
281 int insert_here
= -1;
288 /* False clause makes the whole predicate false. Kill the other variants. */
289 if (clause
== (1 << predicate_false_condition
))
291 p
->clause
[0] = (1 << predicate_false_condition
);
295 if (false_predicate_p (p
))
298 /* No one should be sily enough to add false into nontrivial clauses. */
299 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
301 /* Look where to insert the clause. At the same time prune out
302 clauses of P that are implied by the new clause and thus
304 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
306 p
->clause
[i2
] = p
->clause
[i
];
311 /* If p->clause[i] implies clause, there is nothing to add. */
312 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
314 /* We had nothing to add, none of clauses should've become
316 gcc_checking_assert (i
== i2
);
320 if (p
->clause
[i
] < clause
&& insert_here
< 0)
323 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
324 Otherwise the p->clause[i] has to stay. */
325 if ((p
->clause
[i
] & clause
) != clause
)
329 /* Look for clauses that are obviously true. I.e.
330 op0 == 5 || op0 != 5. */
331 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
334 if (!(clause
& (1 << c1
)))
336 cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
337 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
338 and thus there is no point for looking for them. */
339 if (cc1
->code
== CHANGED
340 || cc1
->code
== IS_NOT_CONSTANT
)
342 for (c2
= c1
+ 1; c2
<= NUM_CONDITIONS
; c2
++)
343 if (clause
& (1 << c2
))
345 condition
*cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
346 condition
*cc2
= &(*conditions
)[c2
- predicate_first_dynamic_condition
];
347 if (cc1
->operand_num
== cc2
->operand_num
348 && cc1
->val
== cc2
->val
349 && cc2
->code
!= IS_NOT_CONSTANT
350 && cc2
->code
!= CHANGED
351 && cc1
->code
== invert_tree_comparison
353 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
359 /* We run out of variants. Be conservative in positive direction. */
360 if (i2
== MAX_CLAUSES
)
362 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
363 p
->clause
[i2
+ 1] = 0;
364 if (insert_here
>= 0)
365 for (;i2
> insert_here
; i2
--)
366 p
->clause
[i2
] = p
->clause
[i2
- 1];
369 p
->clause
[insert_here
] = clause
;
375 static struct predicate
376 and_predicates (conditions conditions
,
377 struct predicate
*p
, struct predicate
*p2
)
379 struct predicate out
= *p
;
382 /* Avoid busy work. */
383 if (false_predicate_p (p2
) || true_predicate_p (p
))
385 if (false_predicate_p (p
) || true_predicate_p (p2
))
388 /* See how far predicates match. */
389 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
391 gcc_checking_assert (i
< MAX_CLAUSES
);
394 /* Combine the predicates rest. */
395 for (; p2
->clause
[i
]; i
++)
397 gcc_checking_assert (i
< MAX_CLAUSES
);
398 add_clause (conditions
, &out
, p2
->clause
[i
]);
404 /* Return true if predicates are obviously equal. */
407 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
410 for (i
= 0; p
->clause
[i
]; i
++)
412 gcc_checking_assert (i
< MAX_CLAUSES
);
413 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
414 gcc_checking_assert (!p2
->clause
[i
]
415 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
416 if (p
->clause
[i
] != p2
->clause
[i
])
419 return !p2
->clause
[i
];
425 static struct predicate
426 or_predicates (conditions conditions
, struct predicate
*p
, struct predicate
*p2
)
428 struct predicate out
= true_predicate ();
431 /* Avoid busy work. */
432 if (false_predicate_p (p2
) || true_predicate_p (p
))
434 if (false_predicate_p (p
) || true_predicate_p (p2
))
436 if (predicates_equal_p (p
, p2
))
439 /* OK, combine the predicates. */
440 for (i
= 0; p
->clause
[i
]; i
++)
441 for (j
= 0; p2
->clause
[j
]; j
++)
443 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
444 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
450 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
451 if predicate P is known to be false. */
454 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
458 /* True remains true. */
459 if (true_predicate_p (p
))
462 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
464 /* See if we can find clause we can disprove. */
465 for (i
= 0; p
->clause
[i
]; i
++)
467 gcc_checking_assert (i
< MAX_CLAUSES
);
468 if (!(p
->clause
[i
] & possible_truths
))
474 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
475 instruction will be recomputed per invocation of the inlined call. */
478 predicate_probability (conditions conds
,
479 struct predicate
*p
, clause_t possible_truths
,
480 vec
<inline_param_summary_t
> inline_param_summary
)
483 int combined_prob
= REG_BR_PROB_BASE
;
485 /* True remains true. */
486 if (true_predicate_p (p
))
487 return REG_BR_PROB_BASE
;
489 if (false_predicate_p (p
))
492 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
494 /* See if we can find clause we can disprove. */
495 for (i
= 0; p
->clause
[i
]; i
++)
497 gcc_checking_assert (i
< MAX_CLAUSES
);
498 if (!(p
->clause
[i
] & possible_truths
))
504 if (!inline_param_summary
.exists ())
505 return REG_BR_PROB_BASE
;
506 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
507 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
509 if (i2
>= predicate_first_dynamic_condition
)
511 condition
*c
= &(*conds
)[i2
- predicate_first_dynamic_condition
];
512 if (c
->code
== CHANGED
514 < (int) inline_param_summary
.length ()))
516 int iprob
= inline_param_summary
[c
->operand_num
].change_prob
;
517 this_prob
= MAX (this_prob
, iprob
);
520 this_prob
= REG_BR_PROB_BASE
;
523 this_prob
= REG_BR_PROB_BASE
;
525 combined_prob
= MIN (this_prob
, combined_prob
);
530 return combined_prob
;
534 /* Dump conditional COND. */
537 dump_condition (FILE *f
, conditions conditions
, int cond
)
540 if (cond
== predicate_false_condition
)
541 fprintf (f
, "false");
542 else if (cond
== predicate_not_inlined_condition
)
543 fprintf (f
, "not inlined");
546 c
= &(*conditions
)[cond
- predicate_first_dynamic_condition
];
547 fprintf (f
, "op%i", c
->operand_num
);
549 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
550 c
->by_ref
? "ref " : "", c
->offset
);
551 if (c
->code
== IS_NOT_CONSTANT
)
553 fprintf (f
, " not constant");
556 if (c
->code
== CHANGED
)
558 fprintf (f
, " changed");
561 fprintf (f
, " %s ", op_symbol_code (c
->code
));
562 print_generic_expr (f
, c
->val
, 1);
567 /* Dump clause CLAUSE. */
570 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
577 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
578 if (clause
& (1 << i
))
583 dump_condition (f
, conds
, i
);
589 /* Dump predicate PREDICATE. */
592 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
595 if (true_predicate_p (pred
))
596 dump_clause (f
, conds
, 0);
598 for (i
= 0; pred
->clause
[i
]; i
++)
602 dump_clause (f
, conds
, pred
->clause
[i
]);
608 /* Dump inline hints. */
610 dump_inline_hints (FILE *f
, inline_hints hints
)
614 fprintf (f
, "inline hints:");
615 if (hints
& INLINE_HINT_indirect_call
)
617 hints
&= ~INLINE_HINT_indirect_call
;
618 fprintf (f
, " indirect_call");
620 if (hints
& INLINE_HINT_loop_iterations
)
622 hints
&= ~INLINE_HINT_loop_iterations
;
623 fprintf (f
, " loop_iterations");
625 if (hints
& INLINE_HINT_loop_stride
)
627 hints
&= ~INLINE_HINT_loop_stride
;
628 fprintf (f
, " loop_stride");
630 if (hints
& INLINE_HINT_same_scc
)
632 hints
&= ~INLINE_HINT_same_scc
;
633 fprintf (f
, " same_scc");
635 if (hints
& INLINE_HINT_in_scc
)
637 hints
&= ~INLINE_HINT_in_scc
;
638 fprintf (f
, " in_scc");
640 if (hints
& INLINE_HINT_cross_module
)
642 hints
&= ~INLINE_HINT_cross_module
;
643 fprintf (f
, " cross_module");
645 if (hints
& INLINE_HINT_declared_inline
)
647 hints
&= ~INLINE_HINT_declared_inline
;
648 fprintf (f
, " declared_inline");
650 if (hints
& INLINE_HINT_array_index
)
652 hints
&= ~INLINE_HINT_array_index
;
653 fprintf (f
, " array_index");
659 /* Record SIZE and TIME under condition PRED into the inline summary. */
662 account_size_time (struct inline_summary
*summary
, int size
, int time
,
663 struct predicate
*pred
)
669 if (false_predicate_p (pred
))
672 /* We need to create initial empty unconitional clause, but otherwie
673 we don't need to account empty times and sizes. */
674 if (!size
&& !time
&& summary
->entry
)
677 /* Watch overflow that might result from insane profiles. */
678 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
679 time
= MAX_TIME
* INLINE_TIME_SCALE
;
680 gcc_assert (time
>= 0);
682 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
683 if (predicates_equal_p (&e
->predicate
, pred
))
692 e
= &(*summary
->entry
)[0];
693 gcc_assert (!e
->predicate
.clause
[0]);
694 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
695 fprintf (dump_file
, "\t\tReached limit on number of entries, ignoring the predicate.");
697 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
699 fprintf (dump_file
, "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
700 ((double)size
) / INLINE_SIZE_SCALE
,
701 ((double)time
) / INLINE_TIME_SCALE
,
702 found
? "" : "new ");
703 dump_predicate (dump_file
, summary
->conds
, pred
);
707 struct size_time_entry new_entry
;
708 new_entry
.size
= size
;
709 new_entry
.time
= time
;
710 new_entry
.predicate
= *pred
;
711 vec_safe_push (summary
->entry
, new_entry
);
717 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
718 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
722 /* Set predicate for edge E. */
725 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
727 struct inline_edge_summary
*es
= inline_edge_summary (e
);
728 if (predicate
&& !true_predicate_p (predicate
))
731 es
->predicate
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
732 *es
->predicate
= *predicate
;
737 pool_free (edge_predicate_pool
, es
->predicate
);
738 es
->predicate
= NULL
;
742 /* Set predicate for hint *P. */
745 set_hint_predicate (struct predicate
**p
, struct predicate new_predicate
)
747 if (false_predicate_p (&new_predicate
)
748 || true_predicate_p (&new_predicate
))
751 pool_free (edge_predicate_pool
, *p
);
757 *p
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
763 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
764 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
765 Return clause of possible truths. When INLINE_P is true, assume that we are
768 ERROR_MARK means compile time invariant. */
771 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
773 vec
<tree
> known_vals
,
774 vec
<ipa_agg_jump_function_p
> known_aggs
)
776 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
777 struct inline_summary
*info
= inline_summary (node
);
781 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
786 /* We allow call stmt to have fewer arguments than the callee function
787 (especially for K&R style programs). So bound check here (we assume
788 known_aggs vector, if non-NULL, has the same length as
790 gcc_checking_assert (!known_aggs
.exists ()
791 || (known_vals
.length () == known_aggs
.length ()));
792 if (c
->operand_num
>= (int) known_vals
.length ())
794 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
800 struct ipa_agg_jump_function
*agg
;
802 if (c
->code
== CHANGED
804 && (known_vals
[c
->operand_num
]
808 if (known_aggs
.exists ())
810 agg
= known_aggs
[c
->operand_num
];
811 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
818 val
= known_vals
[c
->operand_num
];
819 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
825 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
828 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
830 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
832 && integer_zerop (res
))
834 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
840 /* Work out what conditions might be true at invocation of E. */
843 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
844 clause_t
*clause_ptr
,
845 vec
<tree
> *known_vals_ptr
,
846 vec
<tree
> *known_binfos_ptr
,
847 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
849 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
850 struct inline_summary
*info
= inline_summary (callee
);
851 vec
<tree
> known_vals
= vNULL
;
852 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
855 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
857 known_vals_ptr
->create (0);
858 if (known_binfos_ptr
)
859 known_binfos_ptr
->create (0);
861 if (ipa_node_params_vector
.exists ()
862 && !e
->call_stmt_cannot_inline_p
863 && ((clause_ptr
&& info
->conds
)
864 || known_vals_ptr
|| known_binfos_ptr
))
866 struct ipa_node_params
*parms_info
;
867 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
868 struct inline_edge_summary
*es
= inline_edge_summary (e
);
869 int i
, count
= ipa_get_cs_argument_count (args
);
871 if (e
->caller
->global
.inlined_to
)
872 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
874 parms_info
= IPA_NODE_REF (e
->caller
);
876 if (count
&& (info
->conds
|| known_vals_ptr
))
877 known_vals
.safe_grow_cleared (count
);
878 if (count
&& (info
->conds
|| known_aggs_ptr
))
879 known_aggs
.safe_grow_cleared (count
);
880 if (count
&& known_binfos_ptr
)
881 known_binfos_ptr
->safe_grow_cleared (count
);
883 for (i
= 0; i
< count
; i
++)
885 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
886 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
889 if (known_vals
.exists () && TREE_CODE (cst
) != TREE_BINFO
)
891 else if (known_binfos_ptr
!= NULL
&& TREE_CODE (cst
) == TREE_BINFO
)
892 (*known_binfos_ptr
)[i
] = cst
;
894 else if (inline_p
&& !es
->param
[i
].change_prob
)
895 known_vals
[i
] = error_mark_node
;
896 /* TODO: When IPA-CP starts propagating and merging aggregate jump
897 functions, use its knowledge of the caller too, just like the
898 scalar case above. */
899 known_aggs
[i
] = &jf
->agg
;
904 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
905 known_vals
, known_aggs
);
908 *known_vals_ptr
= known_vals
;
910 known_vals
.release ();
913 *known_aggs_ptr
= known_aggs
;
915 known_aggs
.release ();
919 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
922 inline_summary_alloc (void)
924 if (!node_removal_hook_holder
)
925 node_removal_hook_holder
=
926 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
927 if (!edge_removal_hook_holder
)
928 edge_removal_hook_holder
=
929 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
930 if (!node_duplication_hook_holder
)
931 node_duplication_hook_holder
=
932 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
933 if (!edge_duplication_hook_holder
)
934 edge_duplication_hook_holder
=
935 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
937 if (vec_safe_length (inline_summary_vec
) <= (unsigned) cgraph_max_uid
)
938 vec_safe_grow_cleared (inline_summary_vec
, cgraph_max_uid
+ 1);
939 if (inline_edge_summary_vec
.length () <= (unsigned) cgraph_edge_max_uid
)
940 inline_edge_summary_vec
.safe_grow_cleared (cgraph_edge_max_uid
+ 1);
941 if (!edge_predicate_pool
)
942 edge_predicate_pool
= create_alloc_pool ("edge predicates",
943 sizeof (struct predicate
),
947 /* We are called multiple time for given function; clear
948 data from previous run so they are not cumulated. */
951 reset_inline_edge_summary (struct cgraph_edge
*e
)
953 if (e
->uid
< (int)inline_edge_summary_vec
.length ())
955 struct inline_edge_summary
*es
= inline_edge_summary (e
);
957 es
->call_stmt_size
= es
->call_stmt_time
= 0;
959 pool_free (edge_predicate_pool
, es
->predicate
);
960 es
->predicate
= NULL
;
961 es
->param
.release ();
965 /* We are called multiple time for given function; clear
966 data from previous run so they are not cumulated. */
969 reset_inline_summary (struct cgraph_node
*node
)
971 struct inline_summary
*info
= inline_summary (node
);
972 struct cgraph_edge
*e
;
974 info
->self_size
= info
->self_time
= 0;
975 info
->estimated_stack_size
= 0;
976 info
->estimated_self_stack_size
= 0;
977 info
->stack_frame_offset
= 0;
982 if (info
->loop_iterations
)
984 pool_free (edge_predicate_pool
, info
->loop_iterations
);
985 info
->loop_iterations
= NULL
;
987 if (info
->loop_stride
)
989 pool_free (edge_predicate_pool
, info
->loop_stride
);
990 info
->loop_stride
= NULL
;
992 if (info
->array_index
)
994 pool_free (edge_predicate_pool
, info
->array_index
);
995 info
->array_index
= NULL
;
997 vec_free (info
->conds
);
998 vec_free (info
->entry
);
999 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1000 reset_inline_edge_summary (e
);
1001 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1002 reset_inline_edge_summary (e
);
1005 /* Hook that is called by cgraph.c when a node is removed. */
1008 inline_node_removal_hook (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
1010 struct inline_summary
*info
;
1011 if (vec_safe_length (inline_summary_vec
) <= (unsigned)node
->uid
)
1013 info
= inline_summary (node
);
1014 reset_inline_summary (node
);
1015 memset (info
, 0, sizeof (inline_summary_t
));
1018 /* Remap predicate P of former function to be predicate of duplicated functoin.
1019 POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
1020 INFO is inline summary of the duplicated node. */
1022 static struct predicate
1023 remap_predicate_after_duplication (struct predicate
*p
,
1024 clause_t possible_truths
,
1025 struct inline_summary
*info
)
1027 struct predicate new_predicate
= true_predicate ();
1029 for (j
= 0; p
->clause
[j
]; j
++)
1030 if (!(possible_truths
& p
->clause
[j
]))
1032 new_predicate
= false_predicate ();
1036 add_clause (info
->conds
, &new_predicate
,
1037 possible_truths
& p
->clause
[j
]);
1038 return new_predicate
;
1041 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
1042 Additionally care about allocating new memory slot for updated predicate
1043 and set it to NULL when it becomes true or false (and thus uninteresting).
1047 remap_hint_predicate_after_duplication (struct predicate
**p
,
1048 clause_t possible_truths
,
1049 struct inline_summary
*info
)
1051 struct predicate new_predicate
;
1056 new_predicate
= remap_predicate_after_duplication (*p
,
1059 /* We do not want to free previous predicate; it is used by node origin. */
1061 set_hint_predicate (p
, new_predicate
);
1065 /* Hook that is called by cgraph.c when a node is duplicated. */
1068 inline_node_duplication_hook (struct cgraph_node
*src
, struct cgraph_node
*dst
,
1069 ATTRIBUTE_UNUSED
void *data
)
1071 struct inline_summary
*info
;
1072 inline_summary_alloc ();
1073 info
= inline_summary (dst
);
1074 memcpy (info
, inline_summary (src
),
1075 sizeof (struct inline_summary
));
1076 /* TODO: as an optimization, we may avoid copying conditions
1077 that are known to be false or true. */
1078 info
->conds
= vec_safe_copy (info
->conds
);
1080 /* When there are any replacements in the function body, see if we can figure
1081 out that something was optimized out. */
1082 if (ipa_node_params_vector
.exists ()
1083 && dst
->clone
.tree_map
)
1085 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
1086 /* Use SRC parm info since it may not be copied yet. */
1087 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1088 vec
<tree
> known_vals
= vNULL
;
1089 int count
= ipa_get_param_count (parms_info
);
1091 clause_t possible_truths
;
1092 struct predicate true_pred
= true_predicate ();
1094 int optimized_out_size
= 0;
1095 bool inlined_to_p
= false;
1096 struct cgraph_edge
*edge
;
1099 known_vals
.safe_grow_cleared (count
);
1100 for (i
= 0; i
< count
; i
++)
1102 tree t
= ipa_get_param (parms_info
, i
);
1103 struct ipa_replace_map
*r
;
1105 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
1107 if (r
->old_tree
== t
1111 known_vals
[i
] = r
->new_tree
;
1116 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1118 known_vals
.release ();
1120 account_size_time (info
, 0, 0, &true_pred
);
1122 /* Remap size_time vectors.
1123 Simplify the predicate by prunning out alternatives that are known
1125 TODO: as on optimization, we can also eliminate conditions known
1127 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
1129 struct predicate new_predicate
;
1130 new_predicate
= remap_predicate_after_duplication (&e
->predicate
,
1133 if (false_predicate_p (&new_predicate
))
1134 optimized_out_size
+= e
->size
;
1136 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1139 /* Remap edge predicates with the same simplification as above.
1140 Also copy constantness arrays. */
1141 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1143 struct predicate new_predicate
;
1144 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1146 if (!edge
->inline_failed
)
1147 inlined_to_p
= true;
1150 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1153 if (false_predicate_p (&new_predicate
)
1154 && !false_predicate_p (es
->predicate
))
1156 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1157 edge
->frequency
= 0;
1159 edge_set_predicate (edge
, &new_predicate
);
1162 /* Remap indirect edge predicates with the same simplificaiton as above.
1163 Also copy constantness arrays. */
1164 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1166 struct predicate new_predicate
;
1167 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1169 gcc_checking_assert (edge
->inline_failed
);
1172 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1175 if (false_predicate_p (&new_predicate
)
1176 && !false_predicate_p (es
->predicate
))
1178 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1179 edge
->frequency
= 0;
1181 edge_set_predicate (edge
, &new_predicate
);
1183 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
1186 remap_hint_predicate_after_duplication (&info
->loop_stride
,
1189 remap_hint_predicate_after_duplication (&info
->array_index
,
1193 /* If inliner or someone after inliner will ever start producing
1194 non-trivial clones, we will get trouble with lack of information
1195 about updating self sizes, because size vectors already contains
1196 sizes of the calees. */
1197 gcc_assert (!inlined_to_p
1198 || !optimized_out_size
);
1202 info
->entry
= vec_safe_copy (info
->entry
);
1203 if (info
->loop_iterations
)
1205 predicate p
= *info
->loop_iterations
;
1206 info
->loop_iterations
= NULL
;
1207 set_hint_predicate (&info
->loop_iterations
, p
);
1209 if (info
->loop_stride
)
1211 predicate p
= *info
->loop_stride
;
1212 info
->loop_stride
= NULL
;
1213 set_hint_predicate (&info
->loop_stride
, p
);
1215 if (info
->array_index
)
1217 predicate p
= *info
->array_index
;
1218 info
->array_index
= NULL
;
1219 set_hint_predicate (&info
->array_index
, p
);
1222 inline_update_overall_summary (dst
);
1226 /* Hook that is called by cgraph.c when a node is duplicated. */
1229 inline_edge_duplication_hook (struct cgraph_edge
*src
, struct cgraph_edge
*dst
,
1230 ATTRIBUTE_UNUSED
void *data
)
1232 struct inline_edge_summary
*info
;
1233 struct inline_edge_summary
*srcinfo
;
1234 inline_summary_alloc ();
1235 info
= inline_edge_summary (dst
);
1236 srcinfo
= inline_edge_summary (src
);
1237 memcpy (info
, srcinfo
,
1238 sizeof (struct inline_edge_summary
));
1239 info
->predicate
= NULL
;
1240 edge_set_predicate (dst
, srcinfo
->predicate
);
1241 info
->param
= srcinfo
->param
.copy ();
1245 /* Keep edge cache consistent across edge removal. */
1248 inline_edge_removal_hook (struct cgraph_edge
*edge
, void *data ATTRIBUTE_UNUSED
)
1250 if (edge_growth_cache
.exists ())
1251 reset_edge_growth_cache (edge
);
1252 reset_inline_edge_summary (edge
);
1256 /* Initialize growth caches. */
1259 initialize_growth_caches (void)
1261 if (cgraph_edge_max_uid
)
1262 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
1264 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
1268 /* Free growth caches. */
1271 free_growth_caches (void)
1273 edge_growth_cache
.release ();
1274 node_growth_cache
.release ();
1278 /* Dump edge summaries associated to NODE and recursively to all clones.
1279 Indent by INDENT. */
1282 dump_inline_edge_summary (FILE * f
, int indent
, struct cgraph_node
*node
,
1283 struct inline_summary
*info
)
1285 struct cgraph_edge
*edge
;
1286 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1288 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1289 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1292 fprintf (f
, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i callee size:%2i stack:%2i",
1293 indent
, "", cgraph_node_name (callee
),
1295 !edge
->inline_failed
? "inlined"
1296 : cgraph_inline_failed_string (edge
->inline_failed
),
1302 (int)inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1303 (int)inline_summary (callee
)->estimated_stack_size
);
1307 fprintf (f
, " predicate: ");
1308 dump_predicate (f
, info
->conds
, es
->predicate
);
1312 if (es
->param
.exists ())
1313 for (i
= 0; i
< (int)es
->param
.length (); i
++)
1315 int prob
= es
->param
[i
].change_prob
;
1318 fprintf (f
, "%*s op%i is compile time invariant\n",
1320 else if (prob
!= REG_BR_PROB_BASE
)
1321 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1322 prob
* 100.0 / REG_BR_PROB_BASE
);
1324 if (!edge
->inline_failed
)
1326 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1327 " callee size %i\n",
1329 (int)inline_summary (callee
)->stack_frame_offset
,
1330 (int)inline_summary (callee
)->estimated_self_stack_size
,
1331 (int)inline_summary (callee
)->estimated_stack_size
);
1332 dump_inline_edge_summary (f
, indent
+2, callee
, info
);
1335 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1337 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1338 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1344 es
->call_stmt_time
);
1347 fprintf (f
, "predicate: ");
1348 dump_predicate (f
, info
->conds
, es
->predicate
);
1357 dump_inline_summary (FILE * f
, struct cgraph_node
*node
)
1361 struct inline_summary
*s
= inline_summary (node
);
1364 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
1366 if (DECL_DISREGARD_INLINE_LIMITS (node
->symbol
.decl
))
1367 fprintf (f
, " always_inline");
1369 fprintf (f
, " inlinable");
1370 fprintf (f
, "\n self time: %i\n",
1372 fprintf (f
, " global time: %i\n", s
->time
);
1373 fprintf (f
, " self size: %i\n",
1375 fprintf (f
, " global size: %i\n", s
->size
);
1376 fprintf (f
, " self stack: %i\n",
1377 (int) s
->estimated_self_stack_size
);
1378 fprintf (f
, " global stack: %i\n",
1379 (int) s
->estimated_stack_size
);
1381 fprintf (f
, " estimated growth:%i\n",
1384 fprintf (f
, " In SCC: %i\n",
1386 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
1388 fprintf (f
, " size:%f, time:%f, predicate:",
1389 (double) e
->size
/ INLINE_SIZE_SCALE
,
1390 (double) e
->time
/ INLINE_TIME_SCALE
);
1391 dump_predicate (f
, s
->conds
, &e
->predicate
);
1393 if (s
->loop_iterations
)
1395 fprintf (f
, " loop iterations:");
1396 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1400 fprintf (f
, " loop stride:");
1401 dump_predicate (f
, s
->conds
, s
->loop_stride
);
1405 fprintf (f
, " array index:");
1406 dump_predicate (f
, s
->conds
, s
->array_index
);
1408 fprintf (f
, " calls:\n");
1409 dump_inline_edge_summary (f
, 4, node
, s
);
1415 debug_inline_summary (struct cgraph_node
*node
)
1417 dump_inline_summary (stderr
, node
);
1421 dump_inline_summaries (FILE *f
)
1423 struct cgraph_node
*node
;
1425 FOR_EACH_DEFINED_FUNCTION (node
)
1426 if (!node
->global
.inlined_to
)
1427 dump_inline_summary (f
, node
);
1430 /* Give initial reasons why inlining would fail on EDGE. This gets either
1431 nullified or usually overwritten by more precise reasons later. */
1434 initialize_inline_failed (struct cgraph_edge
*e
)
1436 struct cgraph_node
*callee
= e
->callee
;
1438 if (e
->indirect_unknown_callee
)
1439 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1440 else if (!callee
->analyzed
)
1441 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1442 else if (callee
->local
.redefined_extern_inline
)
1443 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1444 else if (e
->call_stmt_cannot_inline_p
)
1445 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1447 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1450 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1451 boolean variable pointed to by DATA. */
1454 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1457 bool *b
= (bool *) data
;
1462 /* If OP refers to value of function parameter, return the corresponding
1466 unmodified_parm_1 (gimple stmt
, tree op
)
1468 /* SSA_NAME referring to parm default def? */
1469 if (TREE_CODE (op
) == SSA_NAME
1470 && SSA_NAME_IS_DEFAULT_DEF (op
)
1471 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1472 return SSA_NAME_VAR (op
);
1473 /* Non-SSA parm reference? */
1474 if (TREE_CODE (op
) == PARM_DECL
)
1476 bool modified
= false;
1479 ao_ref_init (&refd
, op
);
1480 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1488 /* If OP refers to value of function parameter, return the corresponding
1489 parameter. Also traverse chains of SSA register assignments. */
1492 unmodified_parm (gimple stmt
, tree op
)
1494 tree res
= unmodified_parm_1 (stmt
, op
);
1498 if (TREE_CODE (op
) == SSA_NAME
1499 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1500 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1501 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1502 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1506 /* If OP refers to a value of a function parameter or value loaded from an
1507 aggregate passed to a parameter (either by value or reference), return TRUE
1508 and store the number of the parameter to *INDEX_P and information whether
1509 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1510 the function parameters, STMT is the statement in which OP is used or
1514 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1515 gimple stmt
, tree op
, int *index_p
,
1516 struct agg_position_info
*aggpos
)
1518 tree res
= unmodified_parm_1 (stmt
, op
);
1520 gcc_checking_assert (aggpos
);
1523 *index_p
= ipa_get_param_decl_index (info
, res
);
1526 aggpos
->agg_contents
= false;
1527 aggpos
->by_ref
= false;
1531 if (TREE_CODE (op
) == SSA_NAME
)
1533 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1534 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1536 stmt
= SSA_NAME_DEF_STMT (op
);
1537 op
= gimple_assign_rhs1 (stmt
);
1538 if (!REFERENCE_CLASS_P (op
))
1539 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1543 aggpos
->agg_contents
= true;
1544 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1548 /* See if statement might disappear after inlining.
1549 0 - means not eliminated
1550 1 - half of statements goes away
1551 2 - for sure it is eliminated.
1552 We are not terribly sophisticated, basically looking for simple abstraction
1553 penalty wrappers. */
1556 eliminated_by_inlining_prob (gimple stmt
)
1558 enum gimple_code code
= gimple_code (stmt
);
1559 enum tree_code rhs_code
;
1569 if (gimple_num_ops (stmt
) != 2)
1572 rhs_code
= gimple_assign_rhs_code (stmt
);
1574 /* Casts of parameters, loads from parameters passed by reference
1575 and stores to return value or parameters are often free after
1576 inlining dua to SRA and further combining.
1577 Assume that half of statements goes away. */
1578 if (rhs_code
== CONVERT_EXPR
1579 || rhs_code
== NOP_EXPR
1580 || rhs_code
== VIEW_CONVERT_EXPR
1581 || rhs_code
== ADDR_EXPR
1582 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1584 tree rhs
= gimple_assign_rhs1 (stmt
);
1585 tree lhs
= gimple_assign_lhs (stmt
);
1586 tree inner_rhs
= get_base_address (rhs
);
1587 tree inner_lhs
= get_base_address (lhs
);
1588 bool rhs_free
= false;
1589 bool lhs_free
= false;
1596 /* Reads of parameter are expected to be free. */
1597 if (unmodified_parm (stmt
, inner_rhs
))
1599 /* Match expressions of form &this->field. Those will most likely
1600 combine with something upstream after inlining. */
1601 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1603 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1604 if (TREE_CODE (op
) == PARM_DECL
)
1606 else if (TREE_CODE (op
) == MEM_REF
1607 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0)))
1611 /* When parameter is not SSA register because its address is taken
1612 and it is just copied into one, the statement will be completely
1613 free after inlining (we will copy propagate backward). */
1614 if (rhs_free
&& is_gimple_reg (lhs
))
1617 /* Reads of parameters passed by reference
1618 expected to be free (i.e. optimized out after inlining). */
1619 if (TREE_CODE(inner_rhs
) == MEM_REF
1620 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1623 /* Copying parameter passed by reference into gimple register is
1624 probably also going to copy propagate, but we can't be quite
1626 if (rhs_free
&& is_gimple_reg (lhs
))
1629 /* Writes to parameters, parameters passed by value and return value
1630 (either dirrectly or passed via invisible reference) are free.
1632 TODO: We ought to handle testcase like
1633 struct a {int a,b;};
1635 retrurnsturct (void)
1641 This translate into:
1656 For that we either need to copy ipa-split logic detecting writes
1658 if (TREE_CODE (inner_lhs
) == PARM_DECL
1659 || TREE_CODE (inner_lhs
) == RESULT_DECL
1660 || (TREE_CODE(inner_lhs
) == MEM_REF
1661 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1662 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1663 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1664 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1665 (inner_lhs
, 0))) == RESULT_DECL
))))
1668 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1670 if (lhs_free
&& rhs_free
)
1680 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1681 predicates to the CFG edges. */
1684 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1685 struct inline_summary
*summary
,
1691 struct agg_position_info aggpos
;
1692 enum tree_code code
, inverted_code
;
1698 last
= last_stmt (bb
);
1700 || gimple_code (last
) != GIMPLE_COND
)
1702 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1704 op
= gimple_cond_lhs (last
);
1705 /* TODO: handle conditionals like
1708 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1710 code
= gimple_cond_code (last
);
1712 = invert_tree_comparison (code
,
1713 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1715 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1717 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1718 e
->flags
& EDGE_TRUE_VALUE
1719 ? code
: inverted_code
,
1720 gimple_cond_rhs (last
));
1721 e
->aux
= pool_alloc (edge_predicate_pool
);
1722 *(struct predicate
*)e
->aux
= p
;
1726 if (TREE_CODE (op
) != SSA_NAME
)
1729 if (builtin_constant_p (op))
1733 Here we can predicate nonconstant_code. We can't
1734 really handle constant_code since we have no predicate
1735 for this and also the constant code is not known to be
1736 optimized away when inliner doen't see operand is constant.
1737 Other optimizers might think otherwise. */
1738 if (gimple_cond_code (last
) != NE_EXPR
1739 || !integer_zerop (gimple_cond_rhs (last
)))
1741 set_stmt
= SSA_NAME_DEF_STMT (op
);
1742 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1743 || gimple_call_num_args (set_stmt
) != 1)
1745 op2
= gimple_call_arg (set_stmt
, 0);
1746 if (!unmodified_parm_or_parm_agg_item (info
, set_stmt
, op2
, &index
, &aggpos
))
1748 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1749 if (e
->flags
& EDGE_FALSE_VALUE
)
1751 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1752 IS_NOT_CONSTANT
, NULL_TREE
);
1753 e
->aux
= pool_alloc (edge_predicate_pool
);
1754 *(struct predicate
*)e
->aux
= p
;
1759 /* If BB ends by a switch we can turn into predicates, attach corresponding
1760 predicates to the CFG edges. */
1763 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1764 struct inline_summary
*summary
,
1770 struct agg_position_info aggpos
;
1776 last
= last_stmt (bb
);
1778 || gimple_code (last
) != GIMPLE_SWITCH
)
1780 op
= gimple_switch_index (last
);
1781 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1784 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1786 e
->aux
= pool_alloc (edge_predicate_pool
);
1787 *(struct predicate
*)e
->aux
= false_predicate ();
1789 n
= gimple_switch_num_labels(last
);
1790 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1792 tree cl
= gimple_switch_label (last
, case_idx
);
1796 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1797 min
= CASE_LOW (cl
);
1798 max
= CASE_HIGH (cl
);
1800 /* For default we might want to construct predicate that none
1801 of cases is met, but it is bit hard to do not having negations
1802 of conditionals handy. */
1804 p
= true_predicate ();
1806 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1809 struct predicate p1
, p2
;
1810 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1811 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1812 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1814 *(struct predicate
*)e
->aux
1815 = or_predicates (summary
->conds
, &p
, (struct predicate
*)e
->aux
);
1820 /* For each BB in NODE attach to its AUX pointer predicate under
1821 which it is executable. */
1824 compute_bb_predicates (struct cgraph_node
*node
,
1825 struct ipa_node_params
*parms_info
,
1826 struct inline_summary
*summary
)
1828 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
1832 FOR_EACH_BB_FN (bb
, my_function
)
1834 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1835 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1838 /* Entry block is always executable. */
1839 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1840 = pool_alloc (edge_predicate_pool
);
1841 *(struct predicate
*)ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1842 = true_predicate ();
1844 /* A simple dataflow propagation of predicates forward in the CFG.
1845 TODO: work in reverse postorder. */
1849 FOR_EACH_BB_FN (bb
, my_function
)
1851 struct predicate p
= false_predicate ();
1854 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1858 struct predicate this_bb_predicate
1859 = *(struct predicate
*)e
->src
->aux
;
1862 = and_predicates (summary
->conds
, &this_bb_predicate
,
1863 (struct predicate
*)e
->aux
);
1864 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1865 if (true_predicate_p (&p
))
1869 if (false_predicate_p (&p
))
1870 gcc_assert (!bb
->aux
);
1876 bb
->aux
= pool_alloc (edge_predicate_pool
);
1877 *((struct predicate
*)bb
->aux
) = p
;
1879 else if (!predicates_equal_p (&p
, (struct predicate
*)bb
->aux
))
1882 *((struct predicate
*)bb
->aux
) = p
;
1890 /* We keep info about constantness of SSA names. */
1892 typedef struct predicate predicate_t
;
1893 /* Return predicate specifying when the STMT might have result that is not
1894 a compile time constant. */
1896 static struct predicate
1897 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1898 struct inline_summary
*summary
,
1900 vec
<predicate_t
> nonconstant_names
)
1905 while (UNARY_CLASS_P (expr
))
1906 expr
= TREE_OPERAND (expr
, 0);
1908 parm
= unmodified_parm (NULL
, expr
);
1910 && (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1911 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1912 if (is_gimple_min_invariant (expr
))
1913 return false_predicate ();
1914 if (TREE_CODE (expr
) == SSA_NAME
)
1915 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1916 if (BINARY_CLASS_P (expr
)
1917 || COMPARISON_CLASS_P (expr
))
1919 struct predicate p1
= will_be_nonconstant_expr_predicate
1920 (info
, summary
, TREE_OPERAND (expr
, 0),
1922 struct predicate p2
;
1923 if (true_predicate_p (&p1
))
1925 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1926 TREE_OPERAND (expr
, 1),
1928 return or_predicates (summary
->conds
, &p1
, &p2
);
1930 else if (TREE_CODE (expr
) == COND_EXPR
)
1932 struct predicate p1
= will_be_nonconstant_expr_predicate
1933 (info
, summary
, TREE_OPERAND (expr
, 0),
1935 struct predicate p2
;
1936 if (true_predicate_p (&p1
))
1938 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1939 TREE_OPERAND (expr
, 1),
1941 if (true_predicate_p (&p2
))
1943 p1
= or_predicates (summary
->conds
, &p1
, &p2
);
1944 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1945 TREE_OPERAND (expr
, 2),
1947 return or_predicates (summary
->conds
, &p1
, &p2
);
1954 return false_predicate ();
1958 /* Return predicate specifying when the STMT might have result that is not
1959 a compile time constant. */
1961 static struct predicate
1962 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1963 struct inline_summary
*summary
,
1965 vec
<predicate_t
> nonconstant_names
)
1967 struct predicate p
= true_predicate ();
1970 struct predicate op_non_const
;
1973 struct agg_position_info aggpos
;
1975 /* What statments might be optimized away
1976 when their arguments are constant
1977 TODO: also trivial builtins.
1978 builtin_constant_p is already handled later. */
1979 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1980 && gimple_code (stmt
) != GIMPLE_COND
1981 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1984 /* Stores will stay anyway. */
1985 if (gimple_store_p (stmt
))
1988 is_load
= gimple_assign_load_p (stmt
);
1990 /* Loads can be optimized when the value is known. */
1994 gcc_assert (gimple_assign_single_p (stmt
));
1995 op
= gimple_assign_rhs1 (stmt
);
1996 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
2003 /* See if we understand all operands before we start
2004 adding conditionals. */
2005 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2007 tree parm
= unmodified_parm (stmt
, use
);
2008 /* For arguments we can build a condition. */
2009 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
2011 if (TREE_CODE (use
) != SSA_NAME
)
2013 /* If we know when operand is constant,
2014 we still can say something useful. */
2015 if (!true_predicate_p (&nonconstant_names
[SSA_NAME_VERSION (use
)]))
2021 op_non_const
= add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
2023 op_non_const
= false_predicate ();
2024 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2026 tree parm
= unmodified_parm (stmt
, use
);
2030 && (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
2032 if (index
!= base_index
)
2033 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
2038 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2039 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
2041 if (gimple_code (stmt
) == GIMPLE_ASSIGN
2042 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
2043 nonconstant_names
[SSA_NAME_VERSION (gimple_assign_lhs (stmt
))]
2045 return op_non_const
;
2048 struct record_modified_bb_info
2054 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2055 set except for info->stmt. */
2058 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
,
2061 struct record_modified_bb_info
*info
= (struct record_modified_bb_info
*) data
;
2062 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2064 bitmap_set_bit (info
->bb_set
,
2065 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2066 ? ENTRY_BLOCK_PTR
->index
: gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
2070 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2071 will change since last invocation of STMT.
2073 Value 0 is reserved for compile time invariants.
2074 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2075 ought to be REG_BR_PROB_BASE / estimated_iters. */
2078 param_change_prob (gimple stmt
, int i
)
2080 tree op
= gimple_call_arg (stmt
, i
);
2081 basic_block bb
= gimple_bb (stmt
);
2084 if (is_gimple_min_invariant (op
))
2086 /* We would have to do non-trivial analysis to really work out what
2087 is the probability of value to change (i.e. when init statement
2088 is in a sibling loop of the call).
2090 We do an conservative estimate: when call is executed N times more often
2091 than the statement defining value, we take the frequency 1/N. */
2092 if (TREE_CODE (op
) == SSA_NAME
)
2097 return REG_BR_PROB_BASE
;
2099 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2100 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2102 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2106 if (init_freq
< bb
->frequency
)
2107 return MAX ((init_freq
* REG_BR_PROB_BASE
+
2108 bb
->frequency
/ 2) / bb
->frequency
, 1);
2110 return REG_BR_PROB_BASE
;
2113 base
= get_base_address (op
);
2118 struct record_modified_bb_info info
;
2122 if (const_value_known_p (base
))
2125 return REG_BR_PROB_BASE
;
2126 ao_ref_init (&refd
, op
);
2128 info
.bb_set
= BITMAP_ALLOC (NULL
);
2129 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2131 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2133 BITMAP_FREE (info
.bb_set
);
2134 return REG_BR_PROB_BASE
;
2137 /* Assume that every memory is initialized at entry.
2138 TODO: Can we easilly determine if value is always defined
2139 and thus we may skip entry block? */
2140 if (ENTRY_BLOCK_PTR
->frequency
)
2141 max
= ENTRY_BLOCK_PTR
->frequency
;
2145 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2146 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2148 BITMAP_FREE (info
.bb_set
);
2149 if (max
< bb
->frequency
)
2150 return MAX ((max
* REG_BR_PROB_BASE
+
2151 bb
->frequency
/ 2) / bb
->frequency
, 1);
2153 return REG_BR_PROB_BASE
;
2155 return REG_BR_PROB_BASE
;
2158 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2159 sub-graph and if the predicate the condition depends on is known. If so,
2160 return true and store the pointer the predicate in *P. */
2163 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2164 struct inline_summary
*summary
, basic_block bb
,
2165 struct predicate
*p
,
2166 vec
<predicate_t
> nonconstant_names
)
2170 basic_block first_bb
= NULL
;
2173 if (single_pred_p (bb
))
2175 *p
= false_predicate ();
2179 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2181 if (single_succ_p (e
->src
))
2183 if (!single_pred_p (e
->src
))
2186 first_bb
= single_pred (e
->src
);
2187 else if (single_pred (e
->src
) != first_bb
)
2194 else if (e
->src
!= first_bb
)
2202 stmt
= last_stmt (first_bb
);
2204 || gimple_code (stmt
) != GIMPLE_COND
2205 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2208 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2209 gimple_cond_lhs (stmt
),
2211 if (true_predicate_p (p
))
2217 /* Given a PHI statement in a function described by inline properties SUMMARY
2218 and *P being the predicate describing whether the selected PHI argument is
2219 known, store a predicate for the result of the PHI statement into
2220 NONCONSTANT_NAMES, if possible. */
2223 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2224 struct predicate
*p
,
2225 vec
<predicate_t
> nonconstant_names
)
2229 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2231 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2232 if (!is_gimple_min_invariant (arg
))
2234 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2235 *p
= or_predicates (summary
->conds
, p
,
2236 &nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2237 if (true_predicate_p (p
))
2242 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2244 fprintf (dump_file
, "\t\tphi predicate: ");
2245 dump_predicate (dump_file
, summary
->conds
, p
);
2247 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2250 /* Return predicate specifying when array index in access OP becomes non-constant. */
2252 static struct predicate
2253 array_index_predicate (struct inline_summary
*info
,
2254 vec
<predicate_t
> nonconstant_names
, tree op
)
2256 struct predicate p
= false_predicate ();
2257 while (handled_component_p (op
))
2259 if (TREE_CODE (op
) == ARRAY_REF
2260 || TREE_CODE (op
) == ARRAY_RANGE_REF
)
2262 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2263 p
= or_predicates (info
->conds
, &p
,
2265 SSA_NAME_VERSION (TREE_OPERAND (op
, 1))]);
2267 op
= TREE_OPERAND (op
, 0);
2272 /* Compute function body size parameters for NODE.
2273 When EARLY is true, we compute only simple summaries without
2274 non-trivial predicates to drive the early inliner. */
2277 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2280 /* Estimate static overhead for function prologue/epilogue and alignment. */
2282 /* Benefits are scaled by probability of elimination that is in range
2285 gimple_stmt_iterator bsi
;
2286 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
2288 struct inline_summary
*info
= inline_summary (node
);
2289 struct predicate bb_predicate
;
2290 struct ipa_node_params
*parms_info
= NULL
;
2291 vec
<predicate_t
> nonconstant_names
= vNULL
;
2294 predicate array_index
= true_predicate ();
2299 if (optimize
&& !early
)
2301 calculate_dominance_info (CDI_DOMINATORS
);
2302 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2304 if (ipa_node_params_vector
.exists ())
2306 parms_info
= IPA_NODE_REF (node
);
2307 nonconstant_names
.safe_grow_cleared(SSANAMES (my_function
)->length());
2312 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2313 cgraph_node_name (node
));
2315 /* When we run into maximal number of entries, we assign everything to the
2316 constant truth case. Be sure to have it in list. */
2317 bb_predicate
= true_predicate ();
2318 account_size_time (info
, 0, 0, &bb_predicate
);
2320 bb_predicate
= not_inlined_predicate ();
2321 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2323 gcc_assert (my_function
&& my_function
->cfg
);
2325 compute_bb_predicates (node
, parms_info
, info
);
2326 gcc_assert (cfun
== my_function
);
2327 order
= XNEWVEC (int, n_basic_blocks
);
2328 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2329 for (n
= 0; n
< nblocks
; n
++)
2331 bb
= BASIC_BLOCK (order
[n
]);
2332 freq
= compute_call_stmt_bb_frequency (node
->symbol
.decl
, bb
);
2334 /* TODO: Obviously predicates can be propagated down across CFG. */
2338 bb_predicate
= *(struct predicate
*)bb
->aux
;
2340 bb_predicate
= false_predicate ();
2343 bb_predicate
= true_predicate ();
2345 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2347 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2348 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2351 if (parms_info
&& nonconstant_names
.exists ())
2353 struct predicate phi_predicate
;
2354 bool first_phi
= true;
2356 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2359 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2364 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2366 fprintf (dump_file
, " ");
2367 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2369 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2374 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2376 gimple stmt
= gsi_stmt (bsi
);
2377 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2378 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2380 struct predicate will_be_nonconstant
;
2382 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2384 fprintf (dump_file
, " ");
2385 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2386 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2387 ((double)freq
)/CGRAPH_FREQ_BASE
, this_size
, this_time
);
2390 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2392 struct predicate this_array_index
;
2393 this_array_index
= array_index_predicate (info
, nonconstant_names
,
2394 gimple_assign_rhs1 (stmt
));
2395 if (!false_predicate_p (&this_array_index
))
2396 array_index
= and_predicates (info
->conds
, &array_index
, &this_array_index
);
2398 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2400 struct predicate this_array_index
;
2401 this_array_index
= array_index_predicate (info
, nonconstant_names
,
2402 gimple_get_lhs (stmt
));
2403 if (!false_predicate_p (&this_array_index
))
2404 array_index
= and_predicates (info
->conds
, &array_index
, &this_array_index
);
2408 if (is_gimple_call (stmt
))
2410 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2411 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2413 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2414 resolved as constant. We however don't want to optimize
2415 out the cgraph edges. */
2416 if (nonconstant_names
.exists ()
2417 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2418 && gimple_call_lhs (stmt
)
2419 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2421 struct predicate false_p
= false_predicate ();
2422 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2425 if (ipa_node_params_vector
.exists ())
2427 int count
= gimple_call_num_args (stmt
);
2431 es
->param
.safe_grow_cleared (count
);
2432 for (i
= 0; i
< count
; i
++)
2434 int prob
= param_change_prob (stmt
, i
);
2435 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2436 es
->param
[i
].change_prob
= prob
;
2440 es
->call_stmt_size
= this_size
;
2441 es
->call_stmt_time
= this_time
;
2442 es
->loop_depth
= bb_loop_depth (bb
);
2443 edge_set_predicate (edge
, &bb_predicate
);
2446 /* TODO: When conditional jump or swithc is known to be constant, but
2447 we did not translate it into the predicates, we really can account
2448 just maximum of the possible paths. */
2451 = will_be_nonconstant_predicate (parms_info
, info
,
2452 stmt
, nonconstant_names
);
2453 if (this_time
|| this_size
)
2459 prob
= eliminated_by_inlining_prob (stmt
);
2460 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2461 fprintf (dump_file
, "\t\t50%% will be eliminated by inlining\n");
2462 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2463 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2466 p
= and_predicates (info
->conds
, &bb_predicate
,
2467 &will_be_nonconstant
);
2469 p
= true_predicate ();
2471 if (!false_predicate_p (&p
))
2475 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2476 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2479 /* We account everything but the calls. Calls have their own
2480 size/time info attached to cgraph edges. This is necessary
2481 in order to make the cost disappear after inlining. */
2482 if (!is_gimple_call (stmt
))
2486 struct predicate ip
= not_inlined_predicate ();
2487 ip
= and_predicates (info
->conds
, &ip
, &p
);
2488 account_size_time (info
, this_size
* prob
,
2489 this_time
* prob
, &ip
);
2492 account_size_time (info
, this_size
* (2 - prob
),
2493 this_time
* (2 - prob
), &p
);
2496 gcc_assert (time
>= 0);
2497 gcc_assert (size
>= 0);
2501 set_hint_predicate (&inline_summary (node
)->array_index
, array_index
);
2502 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2503 if (time
> MAX_TIME
)
2507 if (!early
&& nonconstant_names
.exists ())
2511 predicate loop_iterations
= true_predicate ();
2512 predicate loop_stride
= true_predicate ();
2514 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2515 flow_loops_dump (dump_file
, NULL
, 0);
2517 FOR_EACH_LOOP (li
, loop
, 0)
2522 struct tree_niter_desc niter_desc
;
2523 basic_block
*body
= get_loop_body (loop
);
2524 bb_predicate
= *(struct predicate
*)loop
->header
->aux
;
2526 exits
= get_loop_exit_edges (loop
);
2527 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2528 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2529 && !is_gimple_min_invariant (niter_desc
.niter
))
2531 predicate will_be_nonconstant
2532 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2533 niter_desc
.niter
, nonconstant_names
);
2534 if (!true_predicate_p (&will_be_nonconstant
))
2535 will_be_nonconstant
= and_predicates (info
->conds
,
2537 &will_be_nonconstant
);
2538 if (!true_predicate_p (&will_be_nonconstant
)
2539 && !false_predicate_p (&will_be_nonconstant
))
2540 /* This is slightly inprecise. We may want to represent each loop with
2541 independent predicate. */
2542 loop_iterations
= and_predicates (info
->conds
, &loop_iterations
, &will_be_nonconstant
);
2546 for (i
= 0; i
< loop
->num_nodes
; i
++)
2548 gimple_stmt_iterator gsi
;
2549 bb_predicate
= *(struct predicate
*)body
[i
]->aux
;
2550 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
); gsi_next (&gsi
))
2552 gimple stmt
= gsi_stmt (gsi
);
2557 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2559 predicate will_be_nonconstant
;
2561 if (!simple_iv (loop
, loop_containing_stmt (stmt
), use
, &iv
, true)
2562 || is_gimple_min_invariant (iv
.step
))
2565 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2566 iv
.step
, nonconstant_names
);
2567 if (!true_predicate_p (&will_be_nonconstant
))
2568 will_be_nonconstant
= and_predicates (info
->conds
,
2570 &will_be_nonconstant
);
2571 if (!true_predicate_p (&will_be_nonconstant
)
2572 && !false_predicate_p (&will_be_nonconstant
))
2573 /* This is slightly inprecise. We may want to represent each loop with
2574 independent predicate. */
2575 loop_stride
= and_predicates (info
->conds
, &loop_stride
, &will_be_nonconstant
);
2581 set_hint_predicate (&inline_summary (node
)->loop_iterations
, loop_iterations
);
2582 set_hint_predicate (&inline_summary (node
)->loop_stride
, loop_stride
);
2585 FOR_ALL_BB_FN (bb
, my_function
)
2591 pool_free (edge_predicate_pool
, bb
->aux
);
2593 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2596 pool_free (edge_predicate_pool
, e
->aux
);
2600 inline_summary (node
)->self_time
= time
;
2601 inline_summary (node
)->self_size
= size
;
2602 nonconstant_names
.release ();
2603 if (optimize
&& !early
)
2605 loop_optimizer_finalize ();
2606 free_dominance_info (CDI_DOMINATORS
);
2610 fprintf (dump_file
, "\n");
2611 dump_inline_summary (dump_file
, node
);
2616 /* Compute parameters of functions used by inliner.
2617 EARLY is true when we compute parameters for the early inliner */
2620 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2622 HOST_WIDE_INT self_stack_size
;
2623 struct cgraph_edge
*e
;
2624 struct inline_summary
*info
;
2626 gcc_assert (!node
->global
.inlined_to
);
2628 inline_summary_alloc ();
2630 info
= inline_summary (node
);
2631 reset_inline_summary (node
);
2633 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2634 Once this happen, we will need to more curefully predict call
2636 if (node
->thunk
.thunk_p
)
2638 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2639 struct predicate t
= true_predicate ();
2641 info
->inlinable
= 0;
2642 node
->callees
->call_stmt_cannot_inline_p
= true;
2643 node
->local
.can_change_signature
= false;
2644 es
->call_stmt_time
= 1;
2645 es
->call_stmt_size
= 1;
2646 account_size_time (info
, 0, 0, &t
);
2650 /* Even is_gimple_min_invariant rely on current_function_decl. */
2651 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
2653 /* Estimate the stack size for the function if we're optimizing. */
2654 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2655 info
->estimated_self_stack_size
= self_stack_size
;
2656 info
->estimated_stack_size
= self_stack_size
;
2657 info
->stack_frame_offset
= 0;
2659 /* Can this function be inlined at all? */
2660 info
->inlinable
= tree_inlinable_function_p (node
->symbol
.decl
);
2662 /* Type attributes can use parameter indices to describe them. */
2663 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->symbol
.decl
)))
2664 node
->local
.can_change_signature
= false;
2667 /* Otherwise, inlinable functions always can change signature. */
2668 if (info
->inlinable
)
2669 node
->local
.can_change_signature
= true;
2672 /* Functions calling builtin_apply can not change signature. */
2673 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2675 tree
cdecl = e
->callee
->symbol
.decl
;
2676 if (DECL_BUILT_IN (cdecl)
2677 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2678 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2679 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2682 node
->local
.can_change_signature
= !e
;
2685 estimate_function_body_sizes (node
, early
);
2687 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2688 info
->time
= info
->self_time
;
2689 info
->size
= info
->self_size
;
2690 info
->stack_frame_offset
= 0;
2691 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2692 #ifdef ENABLE_CHECKING
2693 inline_update_overall_summary (node
);
2694 gcc_assert (info
->time
== info
->self_time
2695 && info
->size
== info
->self_size
);
2702 /* Compute parameters of functions used by inliner using
2703 current_function_decl. */
2706 compute_inline_parameters_for_current (void)
2708 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2712 struct gimple_opt_pass pass_inline_parameters
=
2716 "inline_param", /* name */
2717 OPTGROUP_INLINE
, /* optinfo_flags */
2719 compute_inline_parameters_for_current
,/* execute */
2722 0, /* static_pass_number */
2723 TV_INLINE_PARAMETERS
, /* tv_id */
2724 0, /* properties_required */
2725 0, /* properties_provided */
2726 0, /* properties_destroyed */
2727 0, /* todo_flags_start */
2728 0 /* todo_flags_finish */
2733 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2737 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2738 int *size
, int *time
,
2739 vec
<tree
> known_vals
,
2740 vec
<tree
> known_binfos
,
2741 vec
<ipa_agg_jump_function_p
> known_aggs
)
2744 struct cgraph_node
*callee
;
2745 struct inline_summary
*isummary
;
2747 if (!known_vals
.exists () && !known_binfos
.exists ())
2749 if (!flag_indirect_inlining
)
2752 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2757 /* Account for difference in cost between indirect and direct calls. */
2758 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2759 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2760 gcc_checking_assert (*time
>= 0);
2761 gcc_checking_assert (*size
>= 0);
2763 callee
= cgraph_get_node (target
);
2764 if (!callee
|| !callee
->analyzed
)
2766 isummary
= inline_summary (callee
);
2767 return isummary
->inlinable
;
2770 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2773 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2775 vec
<tree
> known_vals
,
2776 vec
<tree
> known_binfos
,
2777 vec
<ipa_agg_jump_function_p
> known_aggs
,
2778 inline_hints
*hints
)
2781 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2782 int call_size
= es
->call_stmt_size
;
2783 int call_time
= es
->call_stmt_time
;
2785 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2786 known_vals
, known_binfos
, known_aggs
)
2788 && cgraph_maybe_hot_edge_p (e
))
2789 *hints
|= INLINE_HINT_indirect_call
;
2790 *size
+= call_size
* INLINE_SIZE_SCALE
;
2791 *time
+= call_time
* prob
/ REG_BR_PROB_BASE
2792 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
);
2793 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2794 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2799 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2800 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2804 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2805 inline_hints
*hints
,
2806 clause_t possible_truths
,
2807 vec
<tree
> known_vals
,
2808 vec
<tree
> known_binfos
,
2809 vec
<ipa_agg_jump_function_p
> known_aggs
)
2811 struct cgraph_edge
*e
;
2812 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2814 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2815 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
2817 if (e
->inline_failed
)
2819 /* Predicates of calls shall not use NOT_CHANGED codes,
2820 sowe do not need to compute probabilities. */
2821 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2822 known_vals
, known_binfos
, known_aggs
,
2826 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2828 known_vals
, known_binfos
, known_aggs
);
2831 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2833 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2834 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
2835 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2836 known_vals
, known_binfos
, known_aggs
,
2842 /* Estimate size and time needed to execute NODE assuming
2843 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2844 about NODE's arguments. */
2847 estimate_node_size_and_time (struct cgraph_node
*node
,
2848 clause_t possible_truths
,
2849 vec
<tree
> known_vals
,
2850 vec
<tree
> known_binfos
,
2851 vec
<ipa_agg_jump_function_p
> known_aggs
,
2852 int *ret_size
, int *ret_time
,
2853 inline_hints
*ret_hints
,
2854 vec
<inline_param_summary_t
>
2855 inline_param_summary
)
2857 struct inline_summary
*info
= inline_summary (node
);
2861 inline_hints hints
= 0;
2865 && (dump_flags
& TDF_DETAILS
))
2868 fprintf (dump_file
, " Estimating body: %s/%i\n"
2869 " Known to be false: ",
2870 cgraph_node_name (node
),
2873 for (i
= predicate_not_inlined_condition
;
2874 i
< (predicate_first_dynamic_condition
2875 + (int)vec_safe_length (info
->conds
)); i
++)
2876 if (!(possible_truths
& (1 << i
)))
2879 fprintf (dump_file
, ", ");
2881 dump_condition (dump_file
, info
->conds
, i
);
2885 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
2886 if (evaluate_predicate (&e
->predicate
, possible_truths
))
2889 gcc_checking_assert (e
->time
>= 0);
2890 gcc_checking_assert (time
>= 0);
2891 if (!inline_param_summary
.exists ())
2895 int prob
= predicate_probability (info
->conds
,
2898 inline_param_summary
);
2899 gcc_checking_assert (prob
>= 0);
2900 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
2901 time
+= ((gcov_type
)e
->time
* prob
) / REG_BR_PROB_BASE
;
2903 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2904 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2905 gcc_checking_assert (time
>= 0);
2908 gcc_checking_assert (size
>= 0);
2909 gcc_checking_assert (time
>= 0);
2911 if (info
->loop_iterations
2912 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
2913 hints
|=INLINE_HINT_loop_iterations
;
2914 if (info
->loop_stride
2915 && !evaluate_predicate (info
->loop_stride
, possible_truths
))
2916 hints
|=INLINE_HINT_loop_stride
;
2917 if (info
->array_index
2918 && !evaluate_predicate (info
->array_index
, possible_truths
))
2919 hints
|=INLINE_HINT_array_index
;
2921 hints
|= INLINE_HINT_in_scc
;
2922 if (DECL_DECLARED_INLINE_P (node
->symbol
.decl
))
2923 hints
|= INLINE_HINT_declared_inline
;
2925 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
2926 known_vals
, known_binfos
, known_aggs
);
2927 gcc_checking_assert (size
>= 0);
2928 gcc_checking_assert (time
>= 0);
2929 time
= RDIV (time
, INLINE_TIME_SCALE
);
2930 size
= RDIV (size
, INLINE_SIZE_SCALE
);
2933 && (dump_flags
& TDF_DETAILS
))
2934 fprintf (dump_file
, "\n size:%i time:%i\n", (int)size
, (int)time
);
2945 /* Estimate size and time needed to execute callee of EDGE assuming that
2946 parameters known to be constant at caller of EDGE are propagated.
2947 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
2948 and types for parameters. */
2951 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
2952 vec
<tree
> known_vals
,
2953 vec
<tree
> known_binfos
,
2954 vec
<ipa_agg_jump_function_p
> known_aggs
,
2955 int *ret_size
, int *ret_time
,
2956 inline_hints
*hints
)
2960 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
,
2962 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
,
2963 known_aggs
, ret_size
, ret_time
, hints
, vNULL
);
2966 /* Translate all conditions from callee representation into caller
2967 representation and symbolically evaluate predicate P into new predicate.
2969 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
2970 is summary of function predicate P is from. OPERAND_MAP is array giving
2971 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
2972 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
2973 predicate under which callee is executed. OFFSET_MAP is an array of of
2974 offsets that need to be added to conditions, negative offset means that
2975 conditions relying on values passed by reference have to be discarded
2976 because they might not be preserved (and should be considered offset zero
2977 for other purposes). */
2979 static struct predicate
2980 remap_predicate (struct inline_summary
*info
,
2981 struct inline_summary
*callee_info
,
2982 struct predicate
*p
,
2983 vec
<int> operand_map
,
2984 vec
<int> offset_map
,
2985 clause_t possible_truths
,
2986 struct predicate
*toplev_predicate
)
2989 struct predicate out
= true_predicate ();
2991 /* True predicate is easy. */
2992 if (true_predicate_p (p
))
2993 return *toplev_predicate
;
2994 for (i
= 0; p
->clause
[i
]; i
++)
2996 clause_t clause
= p
->clause
[i
];
2998 struct predicate clause_predicate
= false_predicate ();
3000 gcc_assert (i
< MAX_CLAUSES
);
3002 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
3003 /* Do we have condition we can't disprove? */
3004 if (clause
& possible_truths
& (1 << cond
))
3006 struct predicate cond_predicate
;
3007 /* Work out if the condition can translate to predicate in the
3008 inlined function. */
3009 if (cond
>= predicate_first_dynamic_condition
)
3011 struct condition
*c
;
3013 c
= &(*callee_info
->conds
)[cond
3014 - predicate_first_dynamic_condition
];
3015 /* See if we can remap condition operand to caller's operand.
3016 Otherwise give up. */
3017 if (!operand_map
.exists ()
3018 || (int)operand_map
.length () <= c
->operand_num
3019 || operand_map
[c
->operand_num
] == -1
3020 /* TODO: For non-aggregate conditions, adding an offset is
3021 basically an arithmetic jump function processing which
3022 we should support in future. */
3023 || ((!c
->agg_contents
|| !c
->by_ref
)
3024 && offset_map
[c
->operand_num
] > 0)
3025 || (c
->agg_contents
&& c
->by_ref
3026 && offset_map
[c
->operand_num
] < 0))
3027 cond_predicate
= true_predicate ();
3030 struct agg_position_info ap
;
3031 HOST_WIDE_INT offset_delta
= offset_map
[c
->operand_num
];
3032 if (offset_delta
< 0)
3034 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
3037 gcc_assert (!c
->agg_contents
3039 || offset_delta
== 0);
3040 ap
.offset
= c
->offset
+ offset_delta
;
3041 ap
.agg_contents
= c
->agg_contents
;
3042 ap
.by_ref
= c
->by_ref
;
3043 cond_predicate
= add_condition (info
,
3044 operand_map
[c
->operand_num
],
3045 &ap
, c
->code
, c
->val
);
3048 /* Fixed conditions remains same, construct single
3049 condition predicate. */
3052 cond_predicate
.clause
[0] = 1 << cond
;
3053 cond_predicate
.clause
[1] = 0;
3055 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
3058 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
3060 return and_predicates (info
->conds
, &out
, toplev_predicate
);
3064 /* Update summary information of inline clones after inlining.
3065 Compute peak stack usage. */
3068 inline_update_callee_summaries (struct cgraph_node
*node
,
3071 struct cgraph_edge
*e
;
3072 struct inline_summary
*callee_info
= inline_summary (node
);
3073 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
3076 callee_info
->stack_frame_offset
3077 = caller_info
->stack_frame_offset
3078 + caller_info
->estimated_self_stack_size
;
3079 peak
= callee_info
->stack_frame_offset
3080 + callee_info
->estimated_self_stack_size
;
3081 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
3083 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
3084 cgraph_propagate_frequency (node
);
3085 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3087 if (!e
->inline_failed
)
3088 inline_update_callee_summaries (e
->callee
, depth
);
3089 inline_edge_summary (e
)->loop_depth
+= depth
;
3091 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3092 inline_edge_summary (e
)->loop_depth
+= depth
;
3095 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3096 When functoin A is inlined in B and A calls C with parameter that
3097 changes with probability PROB1 and C is known to be passthroug
3098 of argument if B that change with probability PROB2, the probability
3099 of change is now PROB1*PROB2. */
3102 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3103 struct cgraph_edge
*edge
)
3105 if (ipa_node_params_vector
.exists ())
3108 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3109 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3110 struct inline_edge_summary
*inlined_es
3111 = inline_edge_summary (inlined_edge
);
3113 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3115 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3116 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3117 && (ipa_get_jf_pass_through_formal_id (jfunc
)
3118 < (int) inlined_es
->param
.length ()))
3120 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
3121 int prob1
= es
->param
[i
].change_prob
;
3122 int prob2
= inlined_es
->param
[jf_formal_id
].change_prob
;
3123 int prob
= ((prob1
* prob2
+ REG_BR_PROB_BASE
/ 2)
3124 / REG_BR_PROB_BASE
);
3126 if (prob1
&& prob2
&& !prob
)
3129 es
->param
[i
].change_prob
= prob
;
3135 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3137 Remap predicates of callees of NODE. Rest of arguments match
3140 Also update change probabilities. */
3143 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3144 struct cgraph_node
*node
,
3145 struct inline_summary
*info
,
3146 struct inline_summary
*callee_info
,
3147 vec
<int> operand_map
,
3148 vec
<int> offset_map
,
3149 clause_t possible_truths
,
3150 struct predicate
*toplev_predicate
)
3152 struct cgraph_edge
*e
;
3153 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3155 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3158 if (e
->inline_failed
)
3160 remap_edge_change_prob (inlined_edge
, e
);
3164 p
= remap_predicate (info
, callee_info
,
3165 es
->predicate
, operand_map
, offset_map
,
3168 edge_set_predicate (e
, &p
);
3169 /* TODO: We should remove the edge for code that will be
3170 optimized out, but we need to keep verifiers and tree-inline
3171 happy. Make it cold for now. */
3172 if (false_predicate_p (&p
))
3179 edge_set_predicate (e
, toplev_predicate
);
3182 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3183 operand_map
, offset_map
, possible_truths
,
3186 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3188 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3191 remap_edge_change_prob (inlined_edge
, e
);
3194 p
= remap_predicate (info
, callee_info
,
3195 es
->predicate
, operand_map
, offset_map
,
3196 possible_truths
, toplev_predicate
);
3197 edge_set_predicate (e
, &p
);
3198 /* TODO: We should remove the edge for code that will be optimized
3199 out, but we need to keep verifiers and tree-inline happy.
3200 Make it cold for now. */
3201 if (false_predicate_p (&p
))
3208 edge_set_predicate (e
, toplev_predicate
);
3212 /* Same as remap_predicate, but set result into hint *HINT. */
3215 remap_hint_predicate (struct inline_summary
*info
,
3216 struct inline_summary
*callee_info
,
3217 struct predicate
**hint
,
3218 vec
<int> operand_map
,
3219 vec
<int> offset_map
,
3220 clause_t possible_truths
,
3221 struct predicate
*toplev_predicate
)
3227 p
= remap_predicate (info
, callee_info
,
3229 operand_map
, offset_map
,
3232 if (!false_predicate_p (&p
)
3233 && !true_predicate_p (&p
))
3236 set_hint_predicate (hint
, p
);
3238 **hint
= and_predicates (info
->conds
,
3244 /* We inlined EDGE. Update summary of the function we inlined into. */
3247 inline_merge_summary (struct cgraph_edge
*edge
)
3249 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3250 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3251 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3252 struct inline_summary
*info
= inline_summary (to
);
3253 clause_t clause
= 0; /* not_inline is known to be false. */
3255 vec
<int> operand_map
= vNULL
;
3256 vec
<int> offset_map
= vNULL
;
3258 struct predicate toplev_predicate
;
3259 struct predicate true_p
= true_predicate ();
3260 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3263 toplev_predicate
= *es
->predicate
;
3265 toplev_predicate
= true_predicate ();
3267 if (ipa_node_params_vector
.exists () && callee_info
->conds
)
3269 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3270 int count
= ipa_get_cs_argument_count (args
);
3273 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3276 operand_map
.safe_grow_cleared (count
);
3277 offset_map
.safe_grow_cleared (count
);
3279 for (i
= 0; i
< count
; i
++)
3281 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3284 /* TODO: handle non-NOPs when merging. */
3285 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3287 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3288 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3289 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3292 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3294 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3295 if (offset
>= 0 && offset
< INT_MAX
)
3297 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3298 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3300 offset_map
[i
] = offset
;
3303 operand_map
[i
] = map
;
3304 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3307 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3309 struct predicate p
= remap_predicate (info
, callee_info
,
3310 &e
->predicate
, operand_map
,
3313 if (!false_predicate_p (&p
))
3315 gcov_type add_time
= ((gcov_type
)e
->time
* edge
->frequency
3316 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3317 int prob
= predicate_probability (callee_info
->conds
,
3320 add_time
= ((gcov_type
)add_time
* prob
) / REG_BR_PROB_BASE
;
3321 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3322 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3323 if (prob
!= REG_BR_PROB_BASE
3324 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3326 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3327 (double)prob
/ REG_BR_PROB_BASE
);
3329 account_size_time (info
, e
->size
, add_time
, &p
);
3332 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3333 offset_map
, clause
, &toplev_predicate
);
3334 remap_hint_predicate (info
, callee_info
,
3335 &callee_info
->loop_iterations
,
3336 operand_map
, offset_map
,
3337 clause
, &toplev_predicate
);
3338 remap_hint_predicate (info
, callee_info
,
3339 &callee_info
->loop_stride
,
3340 operand_map
, offset_map
,
3341 clause
, &toplev_predicate
);
3342 remap_hint_predicate (info
, callee_info
,
3343 &callee_info
->array_index
,
3344 operand_map
, offset_map
,
3345 clause
, &toplev_predicate
);
3347 inline_update_callee_summaries (edge
->callee
,
3348 inline_edge_summary (edge
)->loop_depth
);
3350 /* We do not maintain predicates of inlined edges, free it. */
3351 edge_set_predicate (edge
, &true_p
);
3352 /* Similarly remove param summaries. */
3353 es
->param
.release ();
3354 operand_map
.release ();
3355 offset_map
.release ();
3358 /* For performance reasons inline_merge_summary is not updating overall size
3359 and time. Recompute it. */
3362 inline_update_overall_summary (struct cgraph_node
*node
)
3364 struct inline_summary
*info
= inline_summary (node
);
3370 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3372 info
->size
+= e
->size
, info
->time
+= e
->time
;
3373 if (info
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
3374 info
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
3376 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3377 ~(clause_t
)(1 << predicate_false_condition
),
3378 vNULL
, vNULL
, vNULL
);
3379 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3380 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3383 /* Return hints derrived from EDGE. */
3385 simple_edge_hints (struct cgraph_edge
*edge
)
3388 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3389 ? edge
->caller
->global
.inlined_to
3391 if (inline_summary (to
)->scc_no
3392 && inline_summary (to
)->scc_no
== inline_summary (edge
->callee
)->scc_no
3393 && !cgraph_edge_recursive_p (edge
))
3394 hints
|= INLINE_HINT_same_scc
;
3396 if (to
->symbol
.lto_file_data
&& edge
->callee
->symbol
.lto_file_data
3397 && to
->symbol
.lto_file_data
!= edge
->callee
->symbol
.lto_file_data
)
3398 hints
|= INLINE_HINT_cross_module
;
3403 /* Estimate the time cost for the caller when inlining EDGE.
3404 Only to be called via estimate_edge_time, that handles the
3407 When caching, also update the cache entry. Compute both time and
3408 size, since we always need both metrics eventually. */
3411 do_estimate_edge_time (struct cgraph_edge
*edge
)
3416 struct cgraph_node
*callee
;
3418 vec
<tree
> known_vals
;
3419 vec
<tree
> known_binfos
;
3420 vec
<ipa_agg_jump_function_p
> known_aggs
;
3421 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3423 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3425 gcc_checking_assert (edge
->inline_failed
);
3426 evaluate_properties_for_edge (edge
, true,
3427 &clause
, &known_vals
, &known_binfos
,
3429 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3430 known_aggs
, &size
, &time
, &hints
, es
->param
);
3431 known_vals
.release ();
3432 known_binfos
.release ();
3433 known_aggs
.release ();
3434 gcc_checking_assert (size
>= 0);
3435 gcc_checking_assert (time
>= 0);
3437 /* When caching, update the cache entry. */
3438 if (edge_growth_cache
.exists ())
3440 if ((int)edge_growth_cache
.length () <= edge
->uid
)
3441 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
3442 edge_growth_cache
[edge
->uid
].time
= time
+ (time
>= 0);
3444 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3445 hints
|= simple_edge_hints (edge
);
3446 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3452 /* Return estimated callee growth after inlining EDGE.
3453 Only to be called via estimate_edge_size. */
3456 do_estimate_edge_size (struct cgraph_edge
*edge
)
3459 struct cgraph_node
*callee
;
3461 vec
<tree
> known_vals
;
3462 vec
<tree
> known_binfos
;
3463 vec
<ipa_agg_jump_function_p
> known_aggs
;
3465 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3467 if (edge_growth_cache
.exists ())
3469 do_estimate_edge_time (edge
);
3470 size
= edge_growth_cache
[edge
->uid
].size
;
3471 gcc_checking_assert (size
);
3472 return size
- (size
> 0);
3475 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3477 /* Early inliner runs without caching, go ahead and do the dirty work. */
3478 gcc_checking_assert (edge
->inline_failed
);
3479 evaluate_properties_for_edge (edge
, true,
3480 &clause
, &known_vals
, &known_binfos
,
3482 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3483 known_aggs
, &size
, NULL
, NULL
, vNULL
);
3484 known_vals
.release ();
3485 known_binfos
.release ();
3486 known_aggs
.release ();
3491 /* Estimate the growth of the caller when inlining EDGE.
3492 Only to be called via estimate_edge_size. */
3495 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3498 struct cgraph_node
*callee
;
3500 vec
<tree
> known_vals
;
3501 vec
<tree
> known_binfos
;
3502 vec
<ipa_agg_jump_function_p
> known_aggs
;
3504 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3506 if (edge_growth_cache
.exists ())
3508 do_estimate_edge_time (edge
);
3509 hints
= edge_growth_cache
[edge
->uid
].hints
;
3510 gcc_checking_assert (hints
);
3514 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3516 /* Early inliner runs without caching, go ahead and do the dirty work. */
3517 gcc_checking_assert (edge
->inline_failed
);
3518 evaluate_properties_for_edge (edge
, true,
3519 &clause
, &known_vals
, &known_binfos
,
3521 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3522 known_aggs
, NULL
, NULL
, &hints
, vNULL
);
3523 known_vals
.release ();
3524 known_binfos
.release ();
3525 known_aggs
.release ();
3526 hints
|= simple_edge_hints (edge
);
3531 /* Estimate self time of the function NODE after inlining EDGE. */
3534 estimate_time_after_inlining (struct cgraph_node
*node
,
3535 struct cgraph_edge
*edge
)
3537 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3538 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3540 gcov_type time
= inline_summary (node
)->time
+ estimate_edge_time (edge
);
3543 if (time
> MAX_TIME
)
3547 return inline_summary (node
)->time
;
3551 /* Estimate the size of NODE after inlining EDGE which should be an
3552 edge to either NODE or a call inlined into NODE. */
3555 estimate_size_after_inlining (struct cgraph_node
*node
,
3556 struct cgraph_edge
*edge
)
3558 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3559 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3561 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3562 gcc_assert (size
>= 0);
3565 return inline_summary (node
)->size
;
3571 bool self_recursive
;
3576 /* Worker for do_estimate_growth. Collect growth for all callers. */
3579 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3581 struct cgraph_edge
*e
;
3582 struct growth_data
*d
= (struct growth_data
*) data
;
3584 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3586 gcc_checking_assert (e
->inline_failed
);
3588 if (e
->caller
== node
3589 || (e
->caller
->global
.inlined_to
3590 && e
->caller
->global
.inlined_to
== node
))
3591 d
->self_recursive
= true;
3592 d
->growth
+= estimate_edge_growth (e
);
3598 /* Estimate the growth caused by inlining NODE into all callees. */
3601 do_estimate_growth (struct cgraph_node
*node
)
3603 struct growth_data d
= {0, false};
3604 struct inline_summary
*info
= inline_summary (node
);
3606 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3608 /* For self recursive functions the growth estimation really should be
3609 infinity. We don't want to return very large values because the growth
3610 plays various roles in badness computation fractions. Be sure to not
3611 return zero or negative growths. */
3612 if (d
.self_recursive
)
3613 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3614 else if (DECL_EXTERNAL (node
->symbol
.decl
))
3618 if (cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3619 d
.growth
-= info
->size
;
3620 /* COMDAT functions are very often not shared across multiple units
3621 since they come from various template instantiations.
3622 Take this into account. */
3623 else if (DECL_COMDAT (node
->symbol
.decl
)
3624 && cgraph_can_remove_if_no_direct_calls_p (node
))
3625 d
.growth
-= (info
->size
3626 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3630 if (node_growth_cache
.exists ())
3632 if ((int)node_growth_cache
.length () <= node
->uid
)
3633 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
3634 node_growth_cache
[node
->uid
] = d
.growth
+ (d
.growth
>= 0);
3640 /* This function performs intraprocedural analysis in NODE that is required to
3641 inline indirect calls. */
3644 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3646 ipa_analyze_node (node
);
3647 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3649 ipa_print_node_params (dump_file
, node
);
3650 ipa_print_node_jump_functions (dump_file
, node
);
3655 /* Note function body size. */
3658 inline_analyze_function (struct cgraph_node
*node
)
3660 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
3663 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3664 cgraph_node_name (node
), node
->uid
);
3665 if (optimize
&& !node
->thunk
.thunk_p
)
3666 inline_indirect_intraprocedural_analysis (node
);
3667 compute_inline_parameters (node
, false);
3673 /* Called when new function is inserted to callgraph late. */
3676 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3678 inline_analyze_function (node
);
3682 /* Note function body size. */
3685 inline_generate_summary (void)
3687 struct cgraph_node
*node
;
3689 function_insertion_hook_holder
=
3690 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3692 ipa_register_cgraph_hooks ();
3693 inline_free_summary ();
3695 FOR_EACH_DEFINED_FUNCTION (node
)
3697 inline_analyze_function (node
);
3701 /* Read predicate from IB. */
3703 static struct predicate
3704 read_predicate (struct lto_input_block
*ib
)
3706 struct predicate out
;
3712 gcc_assert (k
<= MAX_CLAUSES
);
3713 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3717 /* Zero-initialize the remaining clauses in OUT. */
3718 while (k
<= MAX_CLAUSES
)
3719 out
.clause
[k
++] = 0;
3725 /* Write inline summary for edge E to OB. */
3728 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3730 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3734 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3735 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3736 es
->loop_depth
= streamer_read_uhwi (ib
);
3737 p
= read_predicate (ib
);
3738 edge_set_predicate (e
, &p
);
3739 length
= streamer_read_uhwi (ib
);
3742 es
->param
.safe_grow_cleared (length
);
3743 for (i
= 0; i
< length
; i
++)
3744 es
->param
[i
].change_prob
3745 = streamer_read_uhwi (ib
);
3750 /* Stream in inline summaries from the section. */
3753 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3756 const struct lto_function_header
*header
=
3757 (const struct lto_function_header
*) data
;
3758 const int cfg_offset
= sizeof (struct lto_function_header
);
3759 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3760 const int string_offset
= main_offset
+ header
->main_size
;
3761 struct data_in
*data_in
;
3762 struct lto_input_block ib
;
3763 unsigned int i
, count2
, j
;
3764 unsigned int f_count
;
3766 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3770 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3771 header
->string_size
, vNULL
);
3772 f_count
= streamer_read_uhwi (&ib
);
3773 for (i
= 0; i
< f_count
; i
++)
3776 struct cgraph_node
*node
;
3777 struct inline_summary
*info
;
3778 lto_symtab_encoder_t encoder
;
3779 struct bitpack_d bp
;
3780 struct cgraph_edge
*e
;
3783 index
= streamer_read_uhwi (&ib
);
3784 encoder
= file_data
->symtab_node_encoder
;
3785 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3786 info
= inline_summary (node
);
3788 info
->estimated_stack_size
3789 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3790 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3791 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3793 bp
= streamer_read_bitpack (&ib
);
3794 info
->inlinable
= bp_unpack_value (&bp
, 1);
3796 count2
= streamer_read_uhwi (&ib
);
3797 gcc_assert (!info
->conds
);
3798 for (j
= 0; j
< count2
; j
++)
3801 c
.operand_num
= streamer_read_uhwi (&ib
);
3802 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3803 c
.val
= stream_read_tree (&ib
, data_in
);
3804 bp
= streamer_read_bitpack (&ib
);
3805 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3806 c
.by_ref
= bp_unpack_value (&bp
, 1);
3808 c
.offset
= streamer_read_uhwi (&ib
);
3809 vec_safe_push (info
->conds
, c
);
3811 count2
= streamer_read_uhwi (&ib
);
3812 gcc_assert (!info
->entry
);
3813 for (j
= 0; j
< count2
; j
++)
3815 struct size_time_entry e
;
3817 e
.size
= streamer_read_uhwi (&ib
);
3818 e
.time
= streamer_read_uhwi (&ib
);
3819 e
.predicate
= read_predicate (&ib
);
3821 vec_safe_push (info
->entry
, e
);
3824 p
= read_predicate (&ib
);
3825 set_hint_predicate (&info
->loop_iterations
, p
);
3826 p
= read_predicate (&ib
);
3827 set_hint_predicate (&info
->loop_stride
, p
);
3828 p
= read_predicate (&ib
);
3829 set_hint_predicate (&info
->array_index
, p
);
3830 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3831 read_inline_edge_summary (&ib
, e
);
3832 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3833 read_inline_edge_summary (&ib
, e
);
3836 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3838 lto_data_in_delete (data_in
);
3842 /* Read inline summary. Jump functions are shared among ipa-cp
3843 and inliner, so when ipa-cp is active, we don't need to write them
3847 inline_read_summary (void)
3849 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3850 struct lto_file_decl_data
*file_data
;
3853 inline_summary_alloc ();
3855 while ((file_data
= file_data_vec
[j
++]))
3858 const char *data
= lto_get_section_data (file_data
,
3859 LTO_section_inline_summary
,
3862 inline_read_section (file_data
, data
, len
);
3864 /* Fatal error here. We do not want to support compiling ltrans units
3865 with different version of compiler or different flags than the WPA
3866 unit, so this should never happen. */
3867 fatal_error ("ipa inline summary is missing in input file");
3871 ipa_register_cgraph_hooks ();
3873 ipa_prop_read_jump_functions ();
3875 function_insertion_hook_holder
=
3876 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3880 /* Write predicate P to OB. */
3883 write_predicate (struct output_block
*ob
, struct predicate
*p
)
3887 for (j
= 0; p
->clause
[j
]; j
++)
3889 gcc_assert (j
< MAX_CLAUSES
);
3890 streamer_write_uhwi (ob
, p
->clause
[j
]);
3892 streamer_write_uhwi (ob
, 0);
3896 /* Write inline summary for edge E to OB. */
3899 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
3901 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3904 streamer_write_uhwi (ob
, es
->call_stmt_size
);
3905 streamer_write_uhwi (ob
, es
->call_stmt_time
);
3906 streamer_write_uhwi (ob
, es
->loop_depth
);
3907 write_predicate (ob
, es
->predicate
);
3908 streamer_write_uhwi (ob
, es
->param
.length ());
3909 for (i
= 0; i
< (int)es
->param
.length (); i
++)
3910 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
3914 /* Write inline summary for node in SET.
3915 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
3916 active, we don't need to write them twice. */
3919 inline_write_summary (void)
3921 struct cgraph_node
*node
;
3922 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
3923 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
3924 unsigned int count
= 0;
3927 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3929 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
3930 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
3931 if (cnode
&& cnode
->analyzed
)
3934 streamer_write_uhwi (ob
, count
);
3936 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3938 symtab_node snode
= lto_symtab_encoder_deref (encoder
, i
);
3939 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
3940 if (cnode
&& (node
= cnode
)->analyzed
)
3942 struct inline_summary
*info
= inline_summary (node
);
3943 struct bitpack_d bp
;
3944 struct cgraph_edge
*edge
;
3947 struct condition
*c
;
3949 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, (symtab_node
)node
));
3950 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
3951 streamer_write_hwi (ob
, info
->self_size
);
3952 streamer_write_hwi (ob
, info
->self_time
);
3953 bp
= bitpack_create (ob
->main_stream
);
3954 bp_pack_value (&bp
, info
->inlinable
, 1);
3955 streamer_write_bitpack (&bp
);
3956 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
3957 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
3959 streamer_write_uhwi (ob
, c
->operand_num
);
3960 streamer_write_uhwi (ob
, c
->code
);
3961 stream_write_tree (ob
, c
->val
, true);
3962 bp
= bitpack_create (ob
->main_stream
);
3963 bp_pack_value (&bp
, c
->agg_contents
, 1);
3964 bp_pack_value (&bp
, c
->by_ref
, 1);
3965 streamer_write_bitpack (&bp
);
3966 if (c
->agg_contents
)
3967 streamer_write_uhwi (ob
, c
->offset
);
3969 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
3970 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3972 streamer_write_uhwi (ob
, e
->size
);
3973 streamer_write_uhwi (ob
, e
->time
);
3974 write_predicate (ob
, &e
->predicate
);
3976 write_predicate (ob
, info
->loop_iterations
);
3977 write_predicate (ob
, info
->loop_stride
);
3978 write_predicate (ob
, info
->array_index
);
3979 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
3980 write_inline_edge_summary (ob
, edge
);
3981 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
3982 write_inline_edge_summary (ob
, edge
);
3985 streamer_write_char_stream (ob
->main_stream
, 0);
3986 produce_asm (ob
, NULL
);
3987 destroy_output_block (ob
);
3989 if (optimize
&& !flag_ipa_cp
)
3990 ipa_prop_write_jump_functions ();
3994 /* Release inline summary. */
3997 inline_free_summary (void)
3999 struct cgraph_node
*node
;
4000 if (!inline_edge_summary_vec
.exists ())
4002 FOR_EACH_DEFINED_FUNCTION (node
)
4003 reset_inline_summary (node
);
4004 if (function_insertion_hook_holder
)
4005 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
4006 function_insertion_hook_holder
= NULL
;
4007 if (node_removal_hook_holder
)
4008 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
4009 node_removal_hook_holder
= NULL
;
4010 if (edge_removal_hook_holder
)
4011 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
4012 edge_removal_hook_holder
= NULL
;
4013 if (node_duplication_hook_holder
)
4014 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
4015 node_duplication_hook_holder
= NULL
;
4016 if (edge_duplication_hook_holder
)
4017 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
4018 edge_duplication_hook_holder
= NULL
;
4019 vec_free (inline_summary_vec
);
4020 inline_edge_summary_vec
.release ();
4021 if (edge_predicate_pool
)
4022 free_alloc_pool (edge_predicate_pool
);
4023 edge_predicate_pool
= 0;