1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis used by the inliner and other passes limiting code size growth.
23 We estimate for each function
25 - average function execution time
26 - inlining size benefit (that is how much of function body size
27 and its call sequence is expected to disappear by inlining)
28 - inlining time benefit
31 - call statement size and time
33 inlinie_summary datastructures store above information locally (i.e.
34 parameters of the function itself) and globally (i.e. parameters of
35 the function created by applying all the inline decisions already
36 present in the callgraph).
38 We provide accestor to the inline_summary datastructure and
39 basic logic updating the parameters when inlining is performed.
41 The summaries are context sensitive. Context means
42 1) partial assignment of known constant values of operands
43 2) whether function is inlined into the call or not.
44 It is easy to add more variants. To represent function size and time
45 that depends on context (i.e. it is known to be optimized away when
46 context is known either by inlining or from IP-CP and clonning),
47 we use predicates. Predicates are logical formulas in
48 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
49 specifying what conditions must be true. Conditions are simple test
50 of the form described above.
52 In order to make predicate (possibly) true, all of its clauses must
53 be (possibly) true. To make clause (possibly) true, one of conditions
54 it mentions must be (possibly) true. There are fixed bounds on
55 number of clauses and conditions and all the manipulation functions
56 are conservative in positive direction. I.e. we may lose precision
57 by thinking that predicate may be true even when it is not.
59 estimate_edge_size and estimate_edge_growth can be used to query
60 function size/time in the given context. inline_merge_summary merges
61 properties of caller and callee after inlining.
63 Finally pass_inline_parameters is exported. This is used to drive
64 computation of function parameters used by the early inliner. IPA
65 inlined performs analysis via its analyze_function method. */
69 #include "coretypes.h"
72 #include "tree-inline.h"
73 #include "langhooks.h"
75 #include "diagnostic.h"
76 #include "gimple-pretty-print.h"
78 #include "tree-pass.h"
82 #include "gimple-iterator.h"
83 #include "gimple-ssa.h"
85 #include "tree-phinodes.h"
86 #include "ssa-iterators.h"
87 #include "tree-ssanames.h"
88 #include "tree-ssa-loop-niter.h"
89 #include "tree-ssa-loop.h"
91 #include "lto-streamer.h"
92 #include "data-streamer.h"
93 #include "tree-streamer.h"
94 #include "ipa-inline.h"
95 #include "alloc-pool.h"
97 #include "tree-scalar-evolution.h"
98 #include "ipa-utils.h"
101 /* Estimate runtime of function can easilly run into huge numbers with many
102 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
103 integer. For anything larger we use gcov_type. */
104 #define MAX_TIME 500000
106 /* Number of bits in integer, but we really want to be stable across different
108 #define NUM_CONDITIONS 32
110 enum predicate_conditions
112 predicate_false_condition
= 0,
113 predicate_not_inlined_condition
= 1,
114 predicate_first_dynamic_condition
= 2
117 /* Special condition code we use to represent test that operand is compile time
119 #define IS_NOT_CONSTANT ERROR_MARK
120 /* Special condition code we use to represent test that operand is not changed
121 across invocation of the function. When operand IS_NOT_CONSTANT it is always
122 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
123 of executions even when they are not compile time constants. */
124 #define CHANGED IDENTIFIER_NODE
126 /* Holders of ipa cgraph hooks: */
127 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
128 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
129 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
130 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
131 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
132 static void inline_node_removal_hook (struct cgraph_node
*, void *);
133 static void inline_node_duplication_hook (struct cgraph_node
*,
134 struct cgraph_node
*, void *);
135 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
136 static void inline_edge_duplication_hook (struct cgraph_edge
*,
137 struct cgraph_edge
*, void *);
139 /* VECtor holding inline summaries.
140 In GGC memory because conditions might point to constant trees. */
141 vec
<inline_summary_t
, va_gc
> *inline_summary_vec
;
142 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
144 /* Cached node/edge growths. */
145 vec
<int> node_growth_cache
;
146 vec
<edge_growth_cache_entry
> edge_growth_cache
;
148 /* Edge predicates goes here. */
149 static alloc_pool edge_predicate_pool
;
151 /* Return true predicate (tautology).
152 We represent it by empty list of clauses. */
154 static inline struct predicate
155 true_predicate (void)
163 /* Return predicate testing single condition number COND. */
165 static inline struct predicate
166 single_cond_predicate (int cond
)
169 p
.clause
[0] = 1 << cond
;
175 /* Return false predicate. First clause require false condition. */
177 static inline struct predicate
178 false_predicate (void)
180 return single_cond_predicate (predicate_false_condition
);
184 /* Return true if P is (false). */
187 true_predicate_p (struct predicate
*p
)
189 return !p
->clause
[0];
193 /* Return true if P is (false). */
196 false_predicate_p (struct predicate
*p
)
198 if (p
->clause
[0] == (1 << predicate_false_condition
))
200 gcc_checking_assert (!p
->clause
[1]
201 && p
->clause
[0] == 1 << predicate_false_condition
);
208 /* Return predicate that is set true when function is not inlined. */
210 static inline struct predicate
211 not_inlined_predicate (void)
213 return single_cond_predicate (predicate_not_inlined_condition
);
216 /* Simple description of whether a memory load or a condition refers to a load
217 from an aggregate and if so, how and where from in the aggregate.
218 Individual fields have the same meaning like fields with the same name in
221 struct agg_position_info
223 HOST_WIDE_INT offset
;
228 /* Add condition to condition list CONDS. AGGPOS describes whether the used
229 oprand is loaded from an aggregate and where in the aggregate it is. It can
230 be NULL, which means this not a load from an aggregate. */
232 static struct predicate
233 add_condition (struct inline_summary
*summary
, int operand_num
,
234 struct agg_position_info
*aggpos
,
235 enum tree_code code
, tree val
)
239 struct condition new_cond
;
240 HOST_WIDE_INT offset
;
241 bool agg_contents
, by_ref
;
245 offset
= aggpos
->offset
;
246 agg_contents
= aggpos
->agg_contents
;
247 by_ref
= aggpos
->by_ref
;
252 agg_contents
= false;
256 gcc_checking_assert (operand_num
>= 0);
257 for (i
= 0; vec_safe_iterate (summary
->conds
, i
, &c
); i
++)
259 if (c
->operand_num
== operand_num
262 && c
->agg_contents
== agg_contents
263 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
264 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
266 /* Too many conditions. Give up and return constant true. */
267 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
268 return true_predicate ();
270 new_cond
.operand_num
= operand_num
;
271 new_cond
.code
= code
;
273 new_cond
.agg_contents
= agg_contents
;
274 new_cond
.by_ref
= by_ref
;
275 new_cond
.offset
= offset
;
276 vec_safe_push (summary
->conds
, new_cond
);
277 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
281 /* Add clause CLAUSE into the predicate P. */
284 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
288 int insert_here
= -1;
295 /* False clause makes the whole predicate false. Kill the other variants. */
296 if (clause
== (1 << predicate_false_condition
))
298 p
->clause
[0] = (1 << predicate_false_condition
);
302 if (false_predicate_p (p
))
305 /* No one should be sily enough to add false into nontrivial clauses. */
306 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
308 /* Look where to insert the clause. At the same time prune out
309 clauses of P that are implied by the new clause and thus
311 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
313 p
->clause
[i2
] = p
->clause
[i
];
318 /* If p->clause[i] implies clause, there is nothing to add. */
319 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
321 /* We had nothing to add, none of clauses should've become
323 gcc_checking_assert (i
== i2
);
327 if (p
->clause
[i
] < clause
&& insert_here
< 0)
330 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
331 Otherwise the p->clause[i] has to stay. */
332 if ((p
->clause
[i
] & clause
) != clause
)
336 /* Look for clauses that are obviously true. I.e.
337 op0 == 5 || op0 != 5. */
338 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
341 if (!(clause
& (1 << c1
)))
343 cc1
= &(*conditions
)[c1
- predicate_first_dynamic_condition
];
344 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
345 and thus there is no point for looking for them. */
346 if (cc1
->code
== CHANGED
|| cc1
->code
== IS_NOT_CONSTANT
)
348 for (c2
= c1
+ 1; c2
< NUM_CONDITIONS
; c2
++)
349 if (clause
& (1 << c2
))
352 &(*conditions
)[c1
- predicate_first_dynamic_condition
];
354 &(*conditions
)[c2
- predicate_first_dynamic_condition
];
355 if (cc1
->operand_num
== cc2
->operand_num
356 && cc1
->val
== cc2
->val
357 && cc2
->code
!= IS_NOT_CONSTANT
358 && cc2
->code
!= CHANGED
359 && cc1
->code
== invert_tree_comparison
361 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
367 /* We run out of variants. Be conservative in positive direction. */
368 if (i2
== MAX_CLAUSES
)
370 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
371 p
->clause
[i2
+ 1] = 0;
372 if (insert_here
>= 0)
373 for (; i2
> insert_here
; i2
--)
374 p
->clause
[i2
] = p
->clause
[i2
- 1];
377 p
->clause
[insert_here
] = clause
;
383 static struct predicate
384 and_predicates (conditions conditions
,
385 struct predicate
*p
, struct predicate
*p2
)
387 struct predicate out
= *p
;
390 /* Avoid busy work. */
391 if (false_predicate_p (p2
) || true_predicate_p (p
))
393 if (false_predicate_p (p
) || true_predicate_p (p2
))
396 /* See how far predicates match. */
397 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
399 gcc_checking_assert (i
< MAX_CLAUSES
);
402 /* Combine the predicates rest. */
403 for (; p2
->clause
[i
]; i
++)
405 gcc_checking_assert (i
< MAX_CLAUSES
);
406 add_clause (conditions
, &out
, p2
->clause
[i
]);
412 /* Return true if predicates are obviously equal. */
415 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
418 for (i
= 0; p
->clause
[i
]; i
++)
420 gcc_checking_assert (i
< MAX_CLAUSES
);
421 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
422 gcc_checking_assert (!p2
->clause
[i
]
423 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
424 if (p
->clause
[i
] != p2
->clause
[i
])
427 return !p2
->clause
[i
];
433 static struct predicate
434 or_predicates (conditions conditions
,
435 struct predicate
*p
, struct predicate
*p2
)
437 struct predicate out
= true_predicate ();
440 /* Avoid busy work. */
441 if (false_predicate_p (p2
) || true_predicate_p (p
))
443 if (false_predicate_p (p
) || true_predicate_p (p2
))
445 if (predicates_equal_p (p
, p2
))
448 /* OK, combine the predicates. */
449 for (i
= 0; p
->clause
[i
]; i
++)
450 for (j
= 0; p2
->clause
[j
]; j
++)
452 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
453 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
459 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
460 if predicate P is known to be false. */
463 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
467 /* True remains true. */
468 if (true_predicate_p (p
))
471 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
473 /* See if we can find clause we can disprove. */
474 for (i
= 0; p
->clause
[i
]; i
++)
476 gcc_checking_assert (i
< MAX_CLAUSES
);
477 if (!(p
->clause
[i
] & possible_truths
))
483 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
484 instruction will be recomputed per invocation of the inlined call. */
487 predicate_probability (conditions conds
,
488 struct predicate
*p
, clause_t possible_truths
,
489 vec
<inline_param_summary_t
> inline_param_summary
)
492 int combined_prob
= REG_BR_PROB_BASE
;
494 /* True remains true. */
495 if (true_predicate_p (p
))
496 return REG_BR_PROB_BASE
;
498 if (false_predicate_p (p
))
501 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
503 /* See if we can find clause we can disprove. */
504 for (i
= 0; p
->clause
[i
]; i
++)
506 gcc_checking_assert (i
< MAX_CLAUSES
);
507 if (!(p
->clause
[i
] & possible_truths
))
513 if (!inline_param_summary
.exists ())
514 return REG_BR_PROB_BASE
;
515 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
516 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
518 if (i2
>= predicate_first_dynamic_condition
)
521 &(*conds
)[i2
- predicate_first_dynamic_condition
];
522 if (c
->code
== CHANGED
524 (int) inline_param_summary
.length ()))
527 inline_param_summary
[c
->operand_num
].change_prob
;
528 this_prob
= MAX (this_prob
, iprob
);
531 this_prob
= REG_BR_PROB_BASE
;
534 this_prob
= REG_BR_PROB_BASE
;
536 combined_prob
= MIN (this_prob
, combined_prob
);
541 return combined_prob
;
545 /* Dump conditional COND. */
548 dump_condition (FILE *f
, conditions conditions
, int cond
)
551 if (cond
== predicate_false_condition
)
552 fprintf (f
, "false");
553 else if (cond
== predicate_not_inlined_condition
)
554 fprintf (f
, "not inlined");
557 c
= &(*conditions
)[cond
- predicate_first_dynamic_condition
];
558 fprintf (f
, "op%i", c
->operand_num
);
560 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
561 c
->by_ref
? "ref " : "", c
->offset
);
562 if (c
->code
== IS_NOT_CONSTANT
)
564 fprintf (f
, " not constant");
567 if (c
->code
== CHANGED
)
569 fprintf (f
, " changed");
572 fprintf (f
, " %s ", op_symbol_code (c
->code
));
573 print_generic_expr (f
, c
->val
, 1);
578 /* Dump clause CLAUSE. */
581 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
588 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
589 if (clause
& (1 << i
))
594 dump_condition (f
, conds
, i
);
600 /* Dump predicate PREDICATE. */
603 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
606 if (true_predicate_p (pred
))
607 dump_clause (f
, conds
, 0);
609 for (i
= 0; pred
->clause
[i
]; i
++)
613 dump_clause (f
, conds
, pred
->clause
[i
]);
619 /* Dump inline hints. */
621 dump_inline_hints (FILE *f
, inline_hints hints
)
625 fprintf (f
, "inline hints:");
626 if (hints
& INLINE_HINT_indirect_call
)
628 hints
&= ~INLINE_HINT_indirect_call
;
629 fprintf (f
, " indirect_call");
631 if (hints
& INLINE_HINT_loop_iterations
)
633 hints
&= ~INLINE_HINT_loop_iterations
;
634 fprintf (f
, " loop_iterations");
636 if (hints
& INLINE_HINT_loop_stride
)
638 hints
&= ~INLINE_HINT_loop_stride
;
639 fprintf (f
, " loop_stride");
641 if (hints
& INLINE_HINT_same_scc
)
643 hints
&= ~INLINE_HINT_same_scc
;
644 fprintf (f
, " same_scc");
646 if (hints
& INLINE_HINT_in_scc
)
648 hints
&= ~INLINE_HINT_in_scc
;
649 fprintf (f
, " in_scc");
651 if (hints
& INLINE_HINT_cross_module
)
653 hints
&= ~INLINE_HINT_cross_module
;
654 fprintf (f
, " cross_module");
656 if (hints
& INLINE_HINT_declared_inline
)
658 hints
&= ~INLINE_HINT_declared_inline
;
659 fprintf (f
, " declared_inline");
661 if (hints
& INLINE_HINT_array_index
)
663 hints
&= ~INLINE_HINT_array_index
;
664 fprintf (f
, " array_index");
670 /* Record SIZE and TIME under condition PRED into the inline summary. */
673 account_size_time (struct inline_summary
*summary
, int size
, int time
,
674 struct predicate
*pred
)
680 if (false_predicate_p (pred
))
683 /* We need to create initial empty unconitional clause, but otherwie
684 we don't need to account empty times and sizes. */
685 if (!size
&& !time
&& summary
->entry
)
688 /* Watch overflow that might result from insane profiles. */
689 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
690 time
= MAX_TIME
* INLINE_TIME_SCALE
;
691 gcc_assert (time
>= 0);
693 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
694 if (predicates_equal_p (&e
->predicate
, pred
))
703 e
= &(*summary
->entry
)[0];
704 gcc_assert (!e
->predicate
.clause
[0]);
705 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
707 "\t\tReached limit on number of entries, "
708 "ignoring the predicate.");
710 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
713 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
714 ((double) size
) / INLINE_SIZE_SCALE
,
715 ((double) time
) / INLINE_TIME_SCALE
, found
? "" : "new ");
716 dump_predicate (dump_file
, summary
->conds
, pred
);
720 struct size_time_entry new_entry
;
721 new_entry
.size
= size
;
722 new_entry
.time
= time
;
723 new_entry
.predicate
= *pred
;
724 vec_safe_push (summary
->entry
, new_entry
);
730 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
731 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
735 /* Set predicate for edge E. */
738 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
740 struct inline_edge_summary
*es
= inline_edge_summary (e
);
741 if (predicate
&& !true_predicate_p (predicate
))
744 es
->predicate
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
745 *es
->predicate
= *predicate
;
750 pool_free (edge_predicate_pool
, es
->predicate
);
751 es
->predicate
= NULL
;
755 /* Set predicate for hint *P. */
758 set_hint_predicate (struct predicate
**p
, struct predicate new_predicate
)
760 if (false_predicate_p (&new_predicate
) || true_predicate_p (&new_predicate
))
763 pool_free (edge_predicate_pool
, *p
);
769 *p
= (struct predicate
*) pool_alloc (edge_predicate_pool
);
775 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
776 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
777 Return clause of possible truths. When INLINE_P is true, assume that we are
780 ERROR_MARK means compile time invariant. */
783 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
785 vec
<tree
> known_vals
,
786 vec
<ipa_agg_jump_function_p
>
789 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
790 struct inline_summary
*info
= inline_summary (node
);
794 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
799 /* We allow call stmt to have fewer arguments than the callee function
800 (especially for K&R style programs). So bound check here (we assume
801 known_aggs vector, if non-NULL, has the same length as
803 gcc_checking_assert (!known_aggs
.exists ()
804 || (known_vals
.length () == known_aggs
.length ()));
805 if (c
->operand_num
>= (int) known_vals
.length ())
807 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
813 struct ipa_agg_jump_function
*agg
;
815 if (c
->code
== CHANGED
817 && (known_vals
[c
->operand_num
] == error_mark_node
))
820 if (known_aggs
.exists ())
822 agg
= known_aggs
[c
->operand_num
];
823 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
830 val
= known_vals
[c
->operand_num
];
831 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
837 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
840 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
842 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
843 if (res
&& integer_zerop (res
))
845 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
851 /* Work out what conditions might be true at invocation of E. */
854 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
855 clause_t
*clause_ptr
,
856 vec
<tree
> *known_vals_ptr
,
857 vec
<tree
> *known_binfos_ptr
,
858 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
860 struct cgraph_node
*callee
=
861 cgraph_function_or_thunk_node (e
->callee
, NULL
);
862 struct inline_summary
*info
= inline_summary (callee
);
863 vec
<tree
> known_vals
= vNULL
;
864 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
867 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
869 known_vals_ptr
->create (0);
870 if (known_binfos_ptr
)
871 known_binfos_ptr
->create (0);
873 if (ipa_node_params_vector
.exists ()
874 && !e
->call_stmt_cannot_inline_p
875 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_binfos_ptr
))
877 struct ipa_node_params
*parms_info
;
878 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
879 struct inline_edge_summary
*es
= inline_edge_summary (e
);
880 int i
, count
= ipa_get_cs_argument_count (args
);
882 if (e
->caller
->global
.inlined_to
)
883 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
885 parms_info
= IPA_NODE_REF (e
->caller
);
887 if (count
&& (info
->conds
|| known_vals_ptr
))
888 known_vals
.safe_grow_cleared (count
);
889 if (count
&& (info
->conds
|| known_aggs_ptr
))
890 known_aggs
.safe_grow_cleared (count
);
891 if (count
&& known_binfos_ptr
)
892 known_binfos_ptr
->safe_grow_cleared (count
);
894 for (i
= 0; i
< count
; i
++)
896 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
897 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
900 if (known_vals
.exists () && TREE_CODE (cst
) != TREE_BINFO
)
902 else if (known_binfos_ptr
!= NULL
903 && TREE_CODE (cst
) == TREE_BINFO
)
904 (*known_binfos_ptr
)[i
] = cst
;
906 else if (inline_p
&& !es
->param
[i
].change_prob
)
907 known_vals
[i
] = error_mark_node
;
908 /* TODO: When IPA-CP starts propagating and merging aggregate jump
909 functions, use its knowledge of the caller too, just like the
910 scalar case above. */
911 known_aggs
[i
] = &jf
->agg
;
916 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
917 known_vals
, known_aggs
);
920 *known_vals_ptr
= known_vals
;
922 known_vals
.release ();
925 *known_aggs_ptr
= known_aggs
;
927 known_aggs
.release ();
931 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
934 inline_summary_alloc (void)
936 if (!node_removal_hook_holder
)
937 node_removal_hook_holder
=
938 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
939 if (!edge_removal_hook_holder
)
940 edge_removal_hook_holder
=
941 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
942 if (!node_duplication_hook_holder
)
943 node_duplication_hook_holder
=
944 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
945 if (!edge_duplication_hook_holder
)
946 edge_duplication_hook_holder
=
947 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
949 if (vec_safe_length (inline_summary_vec
) <= (unsigned) cgraph_max_uid
)
950 vec_safe_grow_cleared (inline_summary_vec
, cgraph_max_uid
+ 1);
951 if (inline_edge_summary_vec
.length () <= (unsigned) cgraph_edge_max_uid
)
952 inline_edge_summary_vec
.safe_grow_cleared (cgraph_edge_max_uid
+ 1);
953 if (!edge_predicate_pool
)
954 edge_predicate_pool
= create_alloc_pool ("edge predicates",
955 sizeof (struct predicate
), 10);
958 /* We are called multiple time for given function; clear
959 data from previous run so they are not cumulated. */
962 reset_inline_edge_summary (struct cgraph_edge
*e
)
964 if (e
->uid
< (int) inline_edge_summary_vec
.length ())
966 struct inline_edge_summary
*es
= inline_edge_summary (e
);
968 es
->call_stmt_size
= es
->call_stmt_time
= 0;
970 pool_free (edge_predicate_pool
, es
->predicate
);
971 es
->predicate
= NULL
;
972 es
->param
.release ();
976 /* We are called multiple time for given function; clear
977 data from previous run so they are not cumulated. */
980 reset_inline_summary (struct cgraph_node
*node
)
982 struct inline_summary
*info
= inline_summary (node
);
983 struct cgraph_edge
*e
;
985 info
->self_size
= info
->self_time
= 0;
986 info
->estimated_stack_size
= 0;
987 info
->estimated_self_stack_size
= 0;
988 info
->stack_frame_offset
= 0;
993 if (info
->loop_iterations
)
995 pool_free (edge_predicate_pool
, info
->loop_iterations
);
996 info
->loop_iterations
= NULL
;
998 if (info
->loop_stride
)
1000 pool_free (edge_predicate_pool
, info
->loop_stride
);
1001 info
->loop_stride
= NULL
;
1003 if (info
->array_index
)
1005 pool_free (edge_predicate_pool
, info
->array_index
);
1006 info
->array_index
= NULL
;
1008 vec_free (info
->conds
);
1009 vec_free (info
->entry
);
1010 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1011 reset_inline_edge_summary (e
);
1012 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1013 reset_inline_edge_summary (e
);
1016 /* Hook that is called by cgraph.c when a node is removed. */
1019 inline_node_removal_hook (struct cgraph_node
*node
,
1020 void *data ATTRIBUTE_UNUSED
)
1022 struct inline_summary
*info
;
1023 if (vec_safe_length (inline_summary_vec
) <= (unsigned) node
->uid
)
1025 info
= inline_summary (node
);
1026 reset_inline_summary (node
);
1027 memset (info
, 0, sizeof (inline_summary_t
));
1030 /* Remap predicate P of former function to be predicate of duplicated functoin.
1031 POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
1032 INFO is inline summary of the duplicated node. */
1034 static struct predicate
1035 remap_predicate_after_duplication (struct predicate
*p
,
1036 clause_t possible_truths
,
1037 struct inline_summary
*info
)
1039 struct predicate new_predicate
= true_predicate ();
1041 for (j
= 0; p
->clause
[j
]; j
++)
1042 if (!(possible_truths
& p
->clause
[j
]))
1044 new_predicate
= false_predicate ();
1048 add_clause (info
->conds
, &new_predicate
,
1049 possible_truths
& p
->clause
[j
]);
1050 return new_predicate
;
1053 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
1054 Additionally care about allocating new memory slot for updated predicate
1055 and set it to NULL when it becomes true or false (and thus uninteresting).
1059 remap_hint_predicate_after_duplication (struct predicate
**p
,
1060 clause_t possible_truths
,
1061 struct inline_summary
*info
)
1063 struct predicate new_predicate
;
1068 new_predicate
= remap_predicate_after_duplication (*p
,
1069 possible_truths
, info
);
1070 /* We do not want to free previous predicate; it is used by node origin. */
1072 set_hint_predicate (p
, new_predicate
);
1076 /* Hook that is called by cgraph.c when a node is duplicated. */
1079 inline_node_duplication_hook (struct cgraph_node
*src
,
1080 struct cgraph_node
*dst
,
1081 ATTRIBUTE_UNUSED
void *data
)
1083 struct inline_summary
*info
;
1084 inline_summary_alloc ();
1085 info
= inline_summary (dst
);
1086 memcpy (info
, inline_summary (src
), sizeof (struct inline_summary
));
1087 /* TODO: as an optimization, we may avoid copying conditions
1088 that are known to be false or true. */
1089 info
->conds
= vec_safe_copy (info
->conds
);
1091 /* When there are any replacements in the function body, see if we can figure
1092 out that something was optimized out. */
1093 if (ipa_node_params_vector
.exists () && dst
->clone
.tree_map
)
1095 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
1096 /* Use SRC parm info since it may not be copied yet. */
1097 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1098 vec
<tree
> known_vals
= vNULL
;
1099 int count
= ipa_get_param_count (parms_info
);
1101 clause_t possible_truths
;
1102 struct predicate true_pred
= true_predicate ();
1104 int optimized_out_size
= 0;
1105 bool inlined_to_p
= false;
1106 struct cgraph_edge
*edge
;
1109 known_vals
.safe_grow_cleared (count
);
1110 for (i
= 0; i
< count
; i
++)
1112 struct ipa_replace_map
*r
;
1114 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
1116 if (((!r
->old_tree
&& r
->parm_num
== i
)
1117 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
1118 && r
->replace_p
&& !r
->ref_p
)
1120 known_vals
[i
] = r
->new_tree
;
1125 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1128 known_vals
.release ();
1130 account_size_time (info
, 0, 0, &true_pred
);
1132 /* Remap size_time vectors.
1133 Simplify the predicate by prunning out alternatives that are known
1135 TODO: as on optimization, we can also eliminate conditions known
1137 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
1139 struct predicate new_predicate
;
1140 new_predicate
= remap_predicate_after_duplication (&e
->predicate
,
1143 if (false_predicate_p (&new_predicate
))
1144 optimized_out_size
+= e
->size
;
1146 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1149 /* Remap edge predicates with the same simplification as above.
1150 Also copy constantness arrays. */
1151 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1153 struct predicate new_predicate
;
1154 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1156 if (!edge
->inline_failed
)
1157 inlined_to_p
= true;
1160 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1163 if (false_predicate_p (&new_predicate
)
1164 && !false_predicate_p (es
->predicate
))
1166 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1167 edge
->frequency
= 0;
1169 edge_set_predicate (edge
, &new_predicate
);
1172 /* Remap indirect edge predicates with the same simplificaiton as above.
1173 Also copy constantness arrays. */
1174 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1176 struct predicate new_predicate
;
1177 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1179 gcc_checking_assert (edge
->inline_failed
);
1182 new_predicate
= remap_predicate_after_duplication (es
->predicate
,
1185 if (false_predicate_p (&new_predicate
)
1186 && !false_predicate_p (es
->predicate
))
1188 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1189 edge
->frequency
= 0;
1191 edge_set_predicate (edge
, &new_predicate
);
1193 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
1194 possible_truths
, info
);
1195 remap_hint_predicate_after_duplication (&info
->loop_stride
,
1196 possible_truths
, info
);
1197 remap_hint_predicate_after_duplication (&info
->array_index
,
1198 possible_truths
, info
);
1200 /* If inliner or someone after inliner will ever start producing
1201 non-trivial clones, we will get trouble with lack of information
1202 about updating self sizes, because size vectors already contains
1203 sizes of the calees. */
1204 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
1208 info
->entry
= vec_safe_copy (info
->entry
);
1209 if (info
->loop_iterations
)
1211 predicate p
= *info
->loop_iterations
;
1212 info
->loop_iterations
= NULL
;
1213 set_hint_predicate (&info
->loop_iterations
, p
);
1215 if (info
->loop_stride
)
1217 predicate p
= *info
->loop_stride
;
1218 info
->loop_stride
= NULL
;
1219 set_hint_predicate (&info
->loop_stride
, p
);
1221 if (info
->array_index
)
1223 predicate p
= *info
->array_index
;
1224 info
->array_index
= NULL
;
1225 set_hint_predicate (&info
->array_index
, p
);
1228 inline_update_overall_summary (dst
);
1232 /* Hook that is called by cgraph.c when a node is duplicated. */
1235 inline_edge_duplication_hook (struct cgraph_edge
*src
,
1236 struct cgraph_edge
*dst
,
1237 ATTRIBUTE_UNUSED
void *data
)
1239 struct inline_edge_summary
*info
;
1240 struct inline_edge_summary
*srcinfo
;
1241 inline_summary_alloc ();
1242 info
= inline_edge_summary (dst
);
1243 srcinfo
= inline_edge_summary (src
);
1244 memcpy (info
, srcinfo
, sizeof (struct inline_edge_summary
));
1245 info
->predicate
= NULL
;
1246 edge_set_predicate (dst
, srcinfo
->predicate
);
1247 info
->param
= srcinfo
->param
.copy ();
1251 /* Keep edge cache consistent across edge removal. */
1254 inline_edge_removal_hook (struct cgraph_edge
*edge
,
1255 void *data ATTRIBUTE_UNUSED
)
1257 if (edge_growth_cache
.exists ())
1258 reset_edge_growth_cache (edge
);
1259 reset_inline_edge_summary (edge
);
1263 /* Initialize growth caches. */
1266 initialize_growth_caches (void)
1268 if (cgraph_edge_max_uid
)
1269 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
1271 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
1275 /* Free growth caches. */
1278 free_growth_caches (void)
1280 edge_growth_cache
.release ();
1281 node_growth_cache
.release ();
1285 /* Dump edge summaries associated to NODE and recursively to all clones.
1286 Indent by INDENT. */
1289 dump_inline_edge_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
1290 struct inline_summary
*info
)
1292 struct cgraph_edge
*edge
;
1293 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1295 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1296 struct cgraph_node
*callee
=
1297 cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1301 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
1302 " time: %2i callee size:%2i stack:%2i",
1303 indent
, "", cgraph_node_name (callee
), callee
->order
,
1304 !edge
->inline_failed
1305 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
1306 indent
, "", es
->loop_depth
, edge
->frequency
,
1307 es
->call_stmt_size
, es
->call_stmt_time
,
1308 (int) inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1309 (int) inline_summary (callee
)->estimated_stack_size
);
1313 fprintf (f
, " predicate: ");
1314 dump_predicate (f
, info
->conds
, es
->predicate
);
1318 if (es
->param
.exists ())
1319 for (i
= 0; i
< (int) es
->param
.length (); i
++)
1321 int prob
= es
->param
[i
].change_prob
;
1324 fprintf (f
, "%*s op%i is compile time invariant\n",
1326 else if (prob
!= REG_BR_PROB_BASE
)
1327 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1328 prob
* 100.0 / REG_BR_PROB_BASE
);
1330 if (!edge
->inline_failed
)
1332 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1333 " callee size %i\n",
1335 (int) inline_summary (callee
)->stack_frame_offset
,
1336 (int) inline_summary (callee
)->estimated_self_stack_size
,
1337 (int) inline_summary (callee
)->estimated_stack_size
);
1338 dump_inline_edge_summary (f
, indent
+ 2, callee
, info
);
1341 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1343 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1344 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1348 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
1351 fprintf (f
, "predicate: ");
1352 dump_predicate (f
, info
->conds
, es
->predicate
);
1361 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
1363 if (node
->definition
)
1365 struct inline_summary
*s
= inline_summary (node
);
1368 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
1370 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
1371 fprintf (f
, " always_inline");
1373 fprintf (f
, " inlinable");
1374 fprintf (f
, "\n self time: %i\n", s
->self_time
);
1375 fprintf (f
, " global time: %i\n", s
->time
);
1376 fprintf (f
, " self size: %i\n", s
->self_size
);
1377 fprintf (f
, " global size: %i\n", s
->size
);
1378 fprintf (f
, " self stack: %i\n",
1379 (int) s
->estimated_self_stack_size
);
1380 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
1382 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
1384 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
1385 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
1387 fprintf (f
, " size:%f, time:%f, predicate:",
1388 (double) e
->size
/ INLINE_SIZE_SCALE
,
1389 (double) e
->time
/ INLINE_TIME_SCALE
);
1390 dump_predicate (f
, s
->conds
, &e
->predicate
);
1392 if (s
->loop_iterations
)
1394 fprintf (f
, " loop iterations:");
1395 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1399 fprintf (f
, " loop stride:");
1400 dump_predicate (f
, s
->conds
, s
->loop_stride
);
1404 fprintf (f
, " array index:");
1405 dump_predicate (f
, s
->conds
, s
->array_index
);
1407 fprintf (f
, " calls:\n");
1408 dump_inline_edge_summary (f
, 4, node
, s
);
1414 debug_inline_summary (struct cgraph_node
*node
)
1416 dump_inline_summary (stderr
, node
);
1420 dump_inline_summaries (FILE *f
)
1422 struct cgraph_node
*node
;
1424 FOR_EACH_DEFINED_FUNCTION (node
)
1425 if (!node
->global
.inlined_to
)
1426 dump_inline_summary (f
, node
);
1429 /* Give initial reasons why inlining would fail on EDGE. This gets either
1430 nullified or usually overwritten by more precise reasons later. */
1433 initialize_inline_failed (struct cgraph_edge
*e
)
1435 struct cgraph_node
*callee
= e
->callee
;
1437 if (e
->indirect_unknown_callee
)
1438 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1439 else if (!callee
->definition
)
1440 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1441 else if (callee
->local
.redefined_extern_inline
)
1442 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1443 else if (e
->call_stmt_cannot_inline_p
)
1444 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1445 else if (cfun
&& fn_contains_cilk_spawn_p (cfun
))
1446 /* We can't inline if the function is spawing a function. */
1447 e
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
1449 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1452 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1453 boolean variable pointed to by DATA. */
1456 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1459 bool *b
= (bool *) data
;
1464 /* If OP refers to value of function parameter, return the corresponding
1468 unmodified_parm_1 (gimple stmt
, tree op
)
1470 /* SSA_NAME referring to parm default def? */
1471 if (TREE_CODE (op
) == SSA_NAME
1472 && SSA_NAME_IS_DEFAULT_DEF (op
)
1473 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1474 return SSA_NAME_VAR (op
);
1475 /* Non-SSA parm reference? */
1476 if (TREE_CODE (op
) == PARM_DECL
)
1478 bool modified
= false;
1481 ao_ref_init (&refd
, op
);
1482 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1490 /* If OP refers to value of function parameter, return the corresponding
1491 parameter. Also traverse chains of SSA register assignments. */
1494 unmodified_parm (gimple stmt
, tree op
)
1496 tree res
= unmodified_parm_1 (stmt
, op
);
1500 if (TREE_CODE (op
) == SSA_NAME
1501 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1502 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1503 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1504 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1508 /* If OP refers to a value of a function parameter or value loaded from an
1509 aggregate passed to a parameter (either by value or reference), return TRUE
1510 and store the number of the parameter to *INDEX_P and information whether
1511 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1512 the function parameters, STMT is the statement in which OP is used or
1516 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1517 gimple stmt
, tree op
, int *index_p
,
1518 struct agg_position_info
*aggpos
)
1520 tree res
= unmodified_parm_1 (stmt
, op
);
1522 gcc_checking_assert (aggpos
);
1525 *index_p
= ipa_get_param_decl_index (info
, res
);
1528 aggpos
->agg_contents
= false;
1529 aggpos
->by_ref
= false;
1533 if (TREE_CODE (op
) == SSA_NAME
)
1535 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1536 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1538 stmt
= SSA_NAME_DEF_STMT (op
);
1539 op
= gimple_assign_rhs1 (stmt
);
1540 if (!REFERENCE_CLASS_P (op
))
1541 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1545 aggpos
->agg_contents
= true;
1546 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1550 /* See if statement might disappear after inlining.
1551 0 - means not eliminated
1552 1 - half of statements goes away
1553 2 - for sure it is eliminated.
1554 We are not terribly sophisticated, basically looking for simple abstraction
1555 penalty wrappers. */
1558 eliminated_by_inlining_prob (gimple stmt
)
1560 enum gimple_code code
= gimple_code (stmt
);
1561 enum tree_code rhs_code
;
1571 if (gimple_num_ops (stmt
) != 2)
1574 rhs_code
= gimple_assign_rhs_code (stmt
);
1576 /* Casts of parameters, loads from parameters passed by reference
1577 and stores to return value or parameters are often free after
1578 inlining dua to SRA and further combining.
1579 Assume that half of statements goes away. */
1580 if (rhs_code
== CONVERT_EXPR
1581 || rhs_code
== NOP_EXPR
1582 || rhs_code
== VIEW_CONVERT_EXPR
1583 || rhs_code
== ADDR_EXPR
1584 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1586 tree rhs
= gimple_assign_rhs1 (stmt
);
1587 tree lhs
= gimple_assign_lhs (stmt
);
1588 tree inner_rhs
= get_base_address (rhs
);
1589 tree inner_lhs
= get_base_address (lhs
);
1590 bool rhs_free
= false;
1591 bool lhs_free
= false;
1598 /* Reads of parameter are expected to be free. */
1599 if (unmodified_parm (stmt
, inner_rhs
))
1601 /* Match expressions of form &this->field. Those will most likely
1602 combine with something upstream after inlining. */
1603 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1605 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1606 if (TREE_CODE (op
) == PARM_DECL
)
1608 else if (TREE_CODE (op
) == MEM_REF
1609 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0)))
1613 /* When parameter is not SSA register because its address is taken
1614 and it is just copied into one, the statement will be completely
1615 free after inlining (we will copy propagate backward). */
1616 if (rhs_free
&& is_gimple_reg (lhs
))
1619 /* Reads of parameters passed by reference
1620 expected to be free (i.e. optimized out after inlining). */
1621 if (TREE_CODE (inner_rhs
) == MEM_REF
1622 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1625 /* Copying parameter passed by reference into gimple register is
1626 probably also going to copy propagate, but we can't be quite
1628 if (rhs_free
&& is_gimple_reg (lhs
))
1631 /* Writes to parameters, parameters passed by value and return value
1632 (either dirrectly or passed via invisible reference) are free.
1634 TODO: We ought to handle testcase like
1635 struct a {int a,b;};
1637 retrurnsturct (void)
1643 This translate into:
1658 For that we either need to copy ipa-split logic detecting writes
1660 if (TREE_CODE (inner_lhs
) == PARM_DECL
1661 || TREE_CODE (inner_lhs
) == RESULT_DECL
1662 || (TREE_CODE (inner_lhs
) == MEM_REF
1663 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1664 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1665 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1666 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1668 0))) == RESULT_DECL
))))
1671 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1673 if (lhs_free
&& rhs_free
)
1683 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1684 predicates to the CFG edges. */
1687 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1688 struct inline_summary
*summary
,
1694 struct agg_position_info aggpos
;
1695 enum tree_code code
, inverted_code
;
1701 last
= last_stmt (bb
);
1702 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1704 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1706 op
= gimple_cond_lhs (last
);
1707 /* TODO: handle conditionals like
1710 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1712 code
= gimple_cond_code (last
);
1714 = invert_tree_comparison (code
,
1715 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1717 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1719 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1720 e
->flags
& EDGE_TRUE_VALUE
1721 ? code
: inverted_code
,
1722 gimple_cond_rhs (last
));
1723 e
->aux
= pool_alloc (edge_predicate_pool
);
1724 *(struct predicate
*) e
->aux
= p
;
1728 if (TREE_CODE (op
) != SSA_NAME
)
1731 if (builtin_constant_p (op))
1735 Here we can predicate nonconstant_code. We can't
1736 really handle constant_code since we have no predicate
1737 for this and also the constant code is not known to be
1738 optimized away when inliner doen't see operand is constant.
1739 Other optimizers might think otherwise. */
1740 if (gimple_cond_code (last
) != NE_EXPR
1741 || !integer_zerop (gimple_cond_rhs (last
)))
1743 set_stmt
= SSA_NAME_DEF_STMT (op
);
1744 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1745 || gimple_call_num_args (set_stmt
) != 1)
1747 op2
= gimple_call_arg (set_stmt
, 0);
1748 if (!unmodified_parm_or_parm_agg_item
1749 (info
, set_stmt
, op2
, &index
, &aggpos
))
1751 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1753 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1754 IS_NOT_CONSTANT
, NULL_TREE
);
1755 e
->aux
= pool_alloc (edge_predicate_pool
);
1756 *(struct predicate
*) e
->aux
= p
;
1761 /* If BB ends by a switch we can turn into predicates, attach corresponding
1762 predicates to the CFG edges. */
1765 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1766 struct inline_summary
*summary
,
1772 struct agg_position_info aggpos
;
1778 last
= last_stmt (bb
);
1779 if (!last
|| gimple_code (last
) != GIMPLE_SWITCH
)
1781 op
= gimple_switch_index (last
);
1782 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1785 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1787 e
->aux
= pool_alloc (edge_predicate_pool
);
1788 *(struct predicate
*) e
->aux
= false_predicate ();
1790 n
= gimple_switch_num_labels (last
);
1791 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1793 tree cl
= gimple_switch_label (last
, case_idx
);
1797 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1798 min
= CASE_LOW (cl
);
1799 max
= CASE_HIGH (cl
);
1801 /* For default we might want to construct predicate that none
1802 of cases is met, but it is bit hard to do not having negations
1803 of conditionals handy. */
1805 p
= true_predicate ();
1807 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1810 struct predicate p1
, p2
;
1811 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1812 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1813 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1815 *(struct predicate
*) e
->aux
1816 = or_predicates (summary
->conds
, &p
, (struct predicate
*) e
->aux
);
1821 /* For each BB in NODE attach to its AUX pointer predicate under
1822 which it is executable. */
1825 compute_bb_predicates (struct cgraph_node
*node
,
1826 struct ipa_node_params
*parms_info
,
1827 struct inline_summary
*summary
)
1829 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1833 FOR_EACH_BB_FN (bb
, my_function
)
1835 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1836 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1839 /* Entry block is always executable. */
1840 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1841 = pool_alloc (edge_predicate_pool
);
1842 *(struct predicate
*) ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1843 = true_predicate ();
1845 /* A simple dataflow propagation of predicates forward in the CFG.
1846 TODO: work in reverse postorder. */
1850 FOR_EACH_BB_FN (bb
, my_function
)
1852 struct predicate p
= false_predicate ();
1855 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1859 struct predicate this_bb_predicate
1860 = *(struct predicate
*) e
->src
->aux
;
1863 = and_predicates (summary
->conds
, &this_bb_predicate
,
1864 (struct predicate
*) e
->aux
);
1865 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1866 if (true_predicate_p (&p
))
1870 if (false_predicate_p (&p
))
1871 gcc_assert (!bb
->aux
);
1877 bb
->aux
= pool_alloc (edge_predicate_pool
);
1878 *((struct predicate
*) bb
->aux
) = p
;
1880 else if (!predicates_equal_p (&p
, (struct predicate
*) bb
->aux
))
1883 *((struct predicate
*) bb
->aux
) = p
;
1891 /* We keep info about constantness of SSA names. */
1893 typedef struct predicate predicate_t
;
1894 /* Return predicate specifying when the STMT might have result that is not
1895 a compile time constant. */
1897 static struct predicate
1898 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1899 struct inline_summary
*summary
,
1901 vec
<predicate_t
> nonconstant_names
)
1906 while (UNARY_CLASS_P (expr
))
1907 expr
= TREE_OPERAND (expr
, 0);
1909 parm
= unmodified_parm (NULL
, expr
);
1910 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1911 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1912 if (is_gimple_min_invariant (expr
))
1913 return false_predicate ();
1914 if (TREE_CODE (expr
) == SSA_NAME
)
1915 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1916 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1918 struct predicate p1
= will_be_nonconstant_expr_predicate
1919 (info
, summary
, TREE_OPERAND (expr
, 0),
1921 struct predicate p2
;
1922 if (true_predicate_p (&p1
))
1924 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1925 TREE_OPERAND (expr
, 1),
1927 return or_predicates (summary
->conds
, &p1
, &p2
);
1929 else if (TREE_CODE (expr
) == COND_EXPR
)
1931 struct predicate p1
= will_be_nonconstant_expr_predicate
1932 (info
, summary
, TREE_OPERAND (expr
, 0),
1934 struct predicate p2
;
1935 if (true_predicate_p (&p1
))
1937 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1938 TREE_OPERAND (expr
, 1),
1940 if (true_predicate_p (&p2
))
1942 p1
= or_predicates (summary
->conds
, &p1
, &p2
);
1943 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1944 TREE_OPERAND (expr
, 2),
1946 return or_predicates (summary
->conds
, &p1
, &p2
);
1953 return false_predicate ();
1957 /* Return predicate specifying when the STMT might have result that is not
1958 a compile time constant. */
1960 static struct predicate
1961 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1962 struct inline_summary
*summary
,
1964 vec
<predicate_t
> nonconstant_names
)
1966 struct predicate p
= true_predicate ();
1969 struct predicate op_non_const
;
1972 struct agg_position_info aggpos
;
1974 /* What statments might be optimized away
1975 when their arguments are constant
1976 TODO: also trivial builtins.
1977 builtin_constant_p is already handled later. */
1978 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1979 && gimple_code (stmt
) != GIMPLE_COND
1980 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1983 /* Stores will stay anyway. */
1984 if (gimple_store_p (stmt
))
1987 is_load
= gimple_assign_load_p (stmt
);
1989 /* Loads can be optimized when the value is known. */
1993 gcc_assert (gimple_assign_single_p (stmt
));
1994 op
= gimple_assign_rhs1 (stmt
);
1995 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
2002 /* See if we understand all operands before we start
2003 adding conditionals. */
2004 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2006 tree parm
= unmodified_parm (stmt
, use
);
2007 /* For arguments we can build a condition. */
2008 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
2010 if (TREE_CODE (use
) != SSA_NAME
)
2012 /* If we know when operand is constant,
2013 we still can say something useful. */
2014 if (!true_predicate_p (&nonconstant_names
[SSA_NAME_VERSION (use
)]))
2021 add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
2023 op_non_const
= false_predicate ();
2024 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2026 tree parm
= unmodified_parm (stmt
, use
);
2029 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
2031 if (index
!= base_index
)
2032 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
2037 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2038 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
2040 if (gimple_code (stmt
) == GIMPLE_ASSIGN
2041 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
2042 nonconstant_names
[SSA_NAME_VERSION (gimple_assign_lhs (stmt
))]
2044 return op_non_const
;
2047 struct record_modified_bb_info
2053 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2054 set except for info->stmt. */
2057 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
2059 struct record_modified_bb_info
*info
=
2060 (struct record_modified_bb_info
*) data
;
2061 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2063 bitmap_set_bit (info
->bb_set
,
2064 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2065 ? ENTRY_BLOCK_PTR
->index
2066 : gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
2070 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2071 will change since last invocation of STMT.
2073 Value 0 is reserved for compile time invariants.
2074 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2075 ought to be REG_BR_PROB_BASE / estimated_iters. */
2078 param_change_prob (gimple stmt
, int i
)
2080 tree op
= gimple_call_arg (stmt
, i
);
2081 basic_block bb
= gimple_bb (stmt
);
2084 /* Global invariants neve change. */
2085 if (is_gimple_min_invariant (op
))
2087 /* We would have to do non-trivial analysis to really work out what
2088 is the probability of value to change (i.e. when init statement
2089 is in a sibling loop of the call).
2091 We do an conservative estimate: when call is executed N times more often
2092 than the statement defining value, we take the frequency 1/N. */
2093 if (TREE_CODE (op
) == SSA_NAME
)
2098 return REG_BR_PROB_BASE
;
2100 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2101 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2103 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2107 if (init_freq
< bb
->frequency
)
2108 return MAX (GCOV_COMPUTE_SCALE (init_freq
, bb
->frequency
), 1);
2110 return REG_BR_PROB_BASE
;
2113 base
= get_base_address (op
);
2118 struct record_modified_bb_info info
;
2121 tree init
= ctor_for_folding (base
);
2123 if (init
!= error_mark_node
)
2126 return REG_BR_PROB_BASE
;
2127 ao_ref_init (&refd
, op
);
2129 info
.bb_set
= BITMAP_ALLOC (NULL
);
2130 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2132 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2134 BITMAP_FREE (info
.bb_set
);
2135 return REG_BR_PROB_BASE
;
2138 /* Assume that every memory is initialized at entry.
2139 TODO: Can we easilly determine if value is always defined
2140 and thus we may skip entry block? */
2141 if (ENTRY_BLOCK_PTR
->frequency
)
2142 max
= ENTRY_BLOCK_PTR
->frequency
;
2146 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2147 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2149 BITMAP_FREE (info
.bb_set
);
2150 if (max
< bb
->frequency
)
2151 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->frequency
), 1);
2153 return REG_BR_PROB_BASE
;
2155 return REG_BR_PROB_BASE
;
2158 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2159 sub-graph and if the predicate the condition depends on is known. If so,
2160 return true and store the pointer the predicate in *P. */
2163 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2164 struct inline_summary
*summary
, basic_block bb
,
2165 struct predicate
*p
,
2166 vec
<predicate_t
> nonconstant_names
)
2170 basic_block first_bb
= NULL
;
2173 if (single_pred_p (bb
))
2175 *p
= false_predicate ();
2179 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2181 if (single_succ_p (e
->src
))
2183 if (!single_pred_p (e
->src
))
2186 first_bb
= single_pred (e
->src
);
2187 else if (single_pred (e
->src
) != first_bb
)
2194 else if (e
->src
!= first_bb
)
2202 stmt
= last_stmt (first_bb
);
2204 || gimple_code (stmt
) != GIMPLE_COND
2205 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2208 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2209 gimple_cond_lhs (stmt
),
2211 if (true_predicate_p (p
))
2217 /* Given a PHI statement in a function described by inline properties SUMMARY
2218 and *P being the predicate describing whether the selected PHI argument is
2219 known, store a predicate for the result of the PHI statement into
2220 NONCONSTANT_NAMES, if possible. */
2223 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2224 struct predicate
*p
,
2225 vec
<predicate_t
> nonconstant_names
)
2229 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2231 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2232 if (!is_gimple_min_invariant (arg
))
2234 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2235 *p
= or_predicates (summary
->conds
, p
,
2236 &nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2237 if (true_predicate_p (p
))
2242 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2244 fprintf (dump_file
, "\t\tphi predicate: ");
2245 dump_predicate (dump_file
, summary
->conds
, p
);
2247 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2250 /* Return predicate specifying when array index in access OP becomes non-constant. */
2252 static struct predicate
2253 array_index_predicate (struct inline_summary
*info
,
2254 vec
< predicate_t
> nonconstant_names
, tree op
)
2256 struct predicate p
= false_predicate ();
2257 while (handled_component_p (op
))
2259 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
2261 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2262 p
= or_predicates (info
->conds
, &p
,
2263 &nonconstant_names
[SSA_NAME_VERSION
2264 (TREE_OPERAND (op
, 1))]);
2266 op
= TREE_OPERAND (op
, 0);
2271 /* For a typical usage of __builtin_expect (a<b, 1), we
2272 may introduce an extra relation stmt:
2273 With the builtin, we have
2276 t3 = __builtin_expect (t2, 1);
2279 Without the builtin, we have
2282 This affects the size/time estimation and may have
2283 an impact on the earlier inlining.
2284 Here find this pattern and fix it up later. */
2287 find_foldable_builtin_expect (basic_block bb
)
2289 gimple_stmt_iterator bsi
;
2291 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2293 gimple stmt
= gsi_stmt (bsi
);
2294 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
))
2296 tree var
= gimple_call_lhs (stmt
);
2297 tree arg
= gimple_call_arg (stmt
, 0);
2298 use_operand_p use_p
;
2305 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2307 while (TREE_CODE (arg
) == SSA_NAME
)
2309 gimple stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2310 if (!is_gimple_assign (stmt_tmp
))
2312 switch (gimple_assign_rhs_code (stmt_tmp
))
2331 arg
= gimple_assign_rhs1 (stmt_tmp
);
2334 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2335 && gimple_code (use_stmt
) == GIMPLE_COND
)
2342 /* Compute function body size parameters for NODE.
2343 When EARLY is true, we compute only simple summaries without
2344 non-trivial predicates to drive the early inliner. */
2347 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2350 /* Estimate static overhead for function prologue/epilogue and alignment. */
2352 /* Benefits are scaled by probability of elimination that is in range
2355 gimple_stmt_iterator bsi
;
2356 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2358 struct inline_summary
*info
= inline_summary (node
);
2359 struct predicate bb_predicate
;
2360 struct ipa_node_params
*parms_info
= NULL
;
2361 vec
<predicate_t
> nonconstant_names
= vNULL
;
2364 predicate array_index
= true_predicate ();
2365 gimple fix_builtin_expect_stmt
;
2370 if (optimize
&& !early
)
2372 calculate_dominance_info (CDI_DOMINATORS
);
2373 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2375 if (ipa_node_params_vector
.exists ())
2377 parms_info
= IPA_NODE_REF (node
);
2378 nonconstant_names
.safe_grow_cleared
2379 (SSANAMES (my_function
)->length ());
2384 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2385 cgraph_node_name (node
));
2387 /* When we run into maximal number of entries, we assign everything to the
2388 constant truth case. Be sure to have it in list. */
2389 bb_predicate
= true_predicate ();
2390 account_size_time (info
, 0, 0, &bb_predicate
);
2392 bb_predicate
= not_inlined_predicate ();
2393 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2395 gcc_assert (my_function
&& my_function
->cfg
);
2397 compute_bb_predicates (node
, parms_info
, info
);
2398 gcc_assert (cfun
== my_function
);
2399 order
= XNEWVEC (int, n_basic_blocks
);
2400 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2401 for (n
= 0; n
< nblocks
; n
++)
2403 bb
= BASIC_BLOCK (order
[n
]);
2404 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
2406 /* TODO: Obviously predicates can be propagated down across CFG. */
2410 bb_predicate
= *(struct predicate
*) bb
->aux
;
2412 bb_predicate
= false_predicate ();
2415 bb_predicate
= true_predicate ();
2417 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2419 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2420 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2423 if (parms_info
&& nonconstant_names
.exists ())
2425 struct predicate phi_predicate
;
2426 bool first_phi
= true;
2428 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2431 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2436 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2438 fprintf (dump_file
, " ");
2439 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2441 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2446 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2448 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2450 gimple stmt
= gsi_stmt (bsi
);
2451 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2452 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2454 struct predicate will_be_nonconstant
;
2456 /* This relation stmt should be folded after we remove
2457 buildin_expect call. Adjust the cost here. */
2458 if (stmt
== fix_builtin_expect_stmt
)
2464 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2466 fprintf (dump_file
, " ");
2467 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2468 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2469 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2473 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2475 struct predicate this_array_index
;
2477 array_index_predicate (info
, nonconstant_names
,
2478 gimple_assign_rhs1 (stmt
));
2479 if (!false_predicate_p (&this_array_index
))
2481 and_predicates (info
->conds
, &array_index
,
2484 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2486 struct predicate this_array_index
;
2488 array_index_predicate (info
, nonconstant_names
,
2489 gimple_get_lhs (stmt
));
2490 if (!false_predicate_p (&this_array_index
))
2492 and_predicates (info
->conds
, &array_index
,
2497 if (is_gimple_call (stmt
))
2499 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2500 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2502 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2503 resolved as constant. We however don't want to optimize
2504 out the cgraph edges. */
2505 if (nonconstant_names
.exists ()
2506 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2507 && gimple_call_lhs (stmt
)
2508 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2510 struct predicate false_p
= false_predicate ();
2511 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2514 if (ipa_node_params_vector
.exists ())
2516 int count
= gimple_call_num_args (stmt
);
2520 es
->param
.safe_grow_cleared (count
);
2521 for (i
= 0; i
< count
; i
++)
2523 int prob
= param_change_prob (stmt
, i
);
2524 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2525 es
->param
[i
].change_prob
= prob
;
2529 es
->call_stmt_size
= this_size
;
2530 es
->call_stmt_time
= this_time
;
2531 es
->loop_depth
= bb_loop_depth (bb
);
2532 edge_set_predicate (edge
, &bb_predicate
);
2535 /* TODO: When conditional jump or swithc is known to be constant, but
2536 we did not translate it into the predicates, we really can account
2537 just maximum of the possible paths. */
2540 = will_be_nonconstant_predicate (parms_info
, info
,
2541 stmt
, nonconstant_names
);
2542 if (this_time
|| this_size
)
2548 prob
= eliminated_by_inlining_prob (stmt
);
2549 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2551 "\t\t50%% will be eliminated by inlining\n");
2552 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2553 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2556 p
= and_predicates (info
->conds
, &bb_predicate
,
2557 &will_be_nonconstant
);
2559 p
= true_predicate ();
2561 if (!false_predicate_p (&p
))
2565 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2566 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2569 /* We account everything but the calls. Calls have their own
2570 size/time info attached to cgraph edges. This is necessary
2571 in order to make the cost disappear after inlining. */
2572 if (!is_gimple_call (stmt
))
2576 struct predicate ip
= not_inlined_predicate ();
2577 ip
= and_predicates (info
->conds
, &ip
, &p
);
2578 account_size_time (info
, this_size
* prob
,
2579 this_time
* prob
, &ip
);
2582 account_size_time (info
, this_size
* (2 - prob
),
2583 this_time
* (2 - prob
), &p
);
2586 gcc_assert (time
>= 0);
2587 gcc_assert (size
>= 0);
2591 set_hint_predicate (&inline_summary (node
)->array_index
, array_index
);
2592 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2593 if (time
> MAX_TIME
)
2597 if (!early
&& nonconstant_names
.exists ())
2601 predicate loop_iterations
= true_predicate ();
2602 predicate loop_stride
= true_predicate ();
2604 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2605 flow_loops_dump (dump_file
, NULL
, 0);
2607 FOR_EACH_LOOP (li
, loop
, 0)
2612 struct tree_niter_desc niter_desc
;
2613 basic_block
*body
= get_loop_body (loop
);
2614 bb_predicate
= *(struct predicate
*) loop
->header
->aux
;
2616 exits
= get_loop_exit_edges (loop
);
2617 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2618 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2619 && !is_gimple_min_invariant (niter_desc
.niter
))
2621 predicate will_be_nonconstant
2622 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2625 if (!true_predicate_p (&will_be_nonconstant
))
2626 will_be_nonconstant
= and_predicates (info
->conds
,
2628 &will_be_nonconstant
);
2629 if (!true_predicate_p (&will_be_nonconstant
)
2630 && !false_predicate_p (&will_be_nonconstant
))
2631 /* This is slightly inprecise. We may want to represent each
2632 loop with independent predicate. */
2634 and_predicates (info
->conds
, &loop_iterations
,
2635 &will_be_nonconstant
);
2639 for (i
= 0; i
< loop
->num_nodes
; i
++)
2641 gimple_stmt_iterator gsi
;
2642 bb_predicate
= *(struct predicate
*) body
[i
]->aux
;
2643 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2646 gimple stmt
= gsi_stmt (gsi
);
2651 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2653 predicate will_be_nonconstant
;
2656 (loop
, loop_containing_stmt (stmt
), use
, &iv
, true)
2657 || is_gimple_min_invariant (iv
.step
))
2660 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2663 if (!true_predicate_p (&will_be_nonconstant
))
2665 = and_predicates (info
->conds
,
2667 &will_be_nonconstant
);
2668 if (!true_predicate_p (&will_be_nonconstant
)
2669 && !false_predicate_p (&will_be_nonconstant
))
2670 /* This is slightly inprecise. We may want to represent
2671 each loop with independent predicate. */
2673 and_predicates (info
->conds
, &loop_stride
,
2674 &will_be_nonconstant
);
2680 set_hint_predicate (&inline_summary (node
)->loop_iterations
,
2682 set_hint_predicate (&inline_summary (node
)->loop_stride
, loop_stride
);
2685 FOR_ALL_BB_FN (bb
, my_function
)
2691 pool_free (edge_predicate_pool
, bb
->aux
);
2693 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2696 pool_free (edge_predicate_pool
, e
->aux
);
2700 inline_summary (node
)->self_time
= time
;
2701 inline_summary (node
)->self_size
= size
;
2702 nonconstant_names
.release ();
2703 if (optimize
&& !early
)
2705 loop_optimizer_finalize ();
2706 free_dominance_info (CDI_DOMINATORS
);
2710 fprintf (dump_file
, "\n");
2711 dump_inline_summary (dump_file
, node
);
2716 /* Compute parameters of functions used by inliner.
2717 EARLY is true when we compute parameters for the early inliner */
2720 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2722 HOST_WIDE_INT self_stack_size
;
2723 struct cgraph_edge
*e
;
2724 struct inline_summary
*info
;
2726 gcc_assert (!node
->global
.inlined_to
);
2728 inline_summary_alloc ();
2730 info
= inline_summary (node
);
2731 reset_inline_summary (node
);
2733 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2734 Once this happen, we will need to more curefully predict call
2736 if (node
->thunk
.thunk_p
)
2738 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2739 struct predicate t
= true_predicate ();
2741 info
->inlinable
= 0;
2742 node
->callees
->call_stmt_cannot_inline_p
= true;
2743 node
->local
.can_change_signature
= false;
2744 es
->call_stmt_time
= 1;
2745 es
->call_stmt_size
= 1;
2746 account_size_time (info
, 0, 0, &t
);
2750 /* Even is_gimple_min_invariant rely on current_function_decl. */
2751 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2753 /* Estimate the stack size for the function if we're optimizing. */
2754 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2755 info
->estimated_self_stack_size
= self_stack_size
;
2756 info
->estimated_stack_size
= self_stack_size
;
2757 info
->stack_frame_offset
= 0;
2759 /* Can this function be inlined at all? */
2760 if (!optimize
&& !lookup_attribute ("always_inline",
2761 DECL_ATTRIBUTES (node
->decl
)))
2762 info
->inlinable
= false;
2764 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2766 /* Type attributes can use parameter indices to describe them. */
2767 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
2768 node
->local
.can_change_signature
= false;
2771 /* Otherwise, inlinable functions always can change signature. */
2772 if (info
->inlinable
)
2773 node
->local
.can_change_signature
= true;
2776 /* Functions calling builtin_apply can not change signature. */
2777 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2779 tree
cdecl = e
->callee
->decl
;
2780 if (DECL_BUILT_IN (cdecl)
2781 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2782 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2783 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2786 node
->local
.can_change_signature
= !e
;
2789 estimate_function_body_sizes (node
, early
);
2791 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2792 info
->time
= info
->self_time
;
2793 info
->size
= info
->self_size
;
2794 info
->stack_frame_offset
= 0;
2795 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2796 #ifdef ENABLE_CHECKING
2797 inline_update_overall_summary (node
);
2798 gcc_assert (info
->time
== info
->self_time
&& info
->size
== info
->self_size
);
2805 /* Compute parameters of functions used by inliner using
2806 current_function_decl. */
2809 compute_inline_parameters_for_current (void)
2811 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2817 const pass_data pass_data_inline_parameters
=
2819 GIMPLE_PASS
, /* type */
2820 "inline_param", /* name */
2821 OPTGROUP_INLINE
, /* optinfo_flags */
2822 false, /* has_gate */
2823 true, /* has_execute */
2824 TV_INLINE_PARAMETERS
, /* tv_id */
2825 0, /* properties_required */
2826 0, /* properties_provided */
2827 0, /* properties_destroyed */
2828 0, /* todo_flags_start */
2829 0, /* todo_flags_finish */
2832 class pass_inline_parameters
: public gimple_opt_pass
2835 pass_inline_parameters (gcc::context
*ctxt
)
2836 : gimple_opt_pass (pass_data_inline_parameters
, ctxt
)
2839 /* opt_pass methods: */
2840 opt_pass
* clone () { return new pass_inline_parameters (m_ctxt
); }
2841 unsigned int execute () {
2842 return compute_inline_parameters_for_current ();
2845 }; // class pass_inline_parameters
2850 make_pass_inline_parameters (gcc::context
*ctxt
)
2852 return new pass_inline_parameters (ctxt
);
2856 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2860 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2861 int *size
, int *time
,
2862 vec
<tree
> known_vals
,
2863 vec
<tree
> known_binfos
,
2864 vec
<ipa_agg_jump_function_p
> known_aggs
)
2867 struct cgraph_node
*callee
;
2868 struct inline_summary
*isummary
;
2870 if (!known_vals
.exists () && !known_binfos
.exists ())
2872 if (!flag_indirect_inlining
)
2875 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2880 /* Account for difference in cost between indirect and direct calls. */
2881 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2882 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2883 gcc_checking_assert (*time
>= 0);
2884 gcc_checking_assert (*size
>= 0);
2886 callee
= cgraph_get_node (target
);
2887 if (!callee
|| !callee
->definition
)
2889 isummary
= inline_summary (callee
);
2890 return isummary
->inlinable
;
2893 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2896 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2898 vec
<tree
> known_vals
,
2899 vec
<tree
> known_binfos
,
2900 vec
<ipa_agg_jump_function_p
> known_aggs
,
2901 inline_hints
*hints
)
2903 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2904 int call_size
= es
->call_stmt_size
;
2905 int call_time
= es
->call_stmt_time
;
2907 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2908 known_vals
, known_binfos
, known_aggs
)
2909 && hints
&& cgraph_maybe_hot_edge_p (e
))
2910 *hints
|= INLINE_HINT_indirect_call
;
2911 *size
+= call_size
* INLINE_SIZE_SCALE
;
2912 *time
+= apply_probability ((gcov_type
) call_time
, prob
)
2913 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
);
2914 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2915 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2920 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2921 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2925 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2926 inline_hints
*hints
,
2927 clause_t possible_truths
,
2928 vec
<tree
> known_vals
,
2929 vec
<tree
> known_binfos
,
2930 vec
<ipa_agg_jump_function_p
> known_aggs
)
2932 struct cgraph_edge
*e
;
2933 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2935 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2937 || evaluate_predicate (es
->predicate
, possible_truths
))
2939 if (e
->inline_failed
)
2941 /* Predicates of calls shall not use NOT_CHANGED codes,
2942 sowe do not need to compute probabilities. */
2943 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2944 known_vals
, known_binfos
,
2948 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2950 known_vals
, known_binfos
,
2954 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2956 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2958 || evaluate_predicate (es
->predicate
, possible_truths
))
2959 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
,
2960 known_vals
, known_binfos
, known_aggs
,
2966 /* Estimate size and time needed to execute NODE assuming
2967 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2968 about NODE's arguments. */
2971 estimate_node_size_and_time (struct cgraph_node
*node
,
2972 clause_t possible_truths
,
2973 vec
<tree
> known_vals
,
2974 vec
<tree
> known_binfos
,
2975 vec
<ipa_agg_jump_function_p
> known_aggs
,
2976 int *ret_size
, int *ret_time
,
2977 inline_hints
*ret_hints
,
2978 vec
<inline_param_summary_t
>
2979 inline_param_summary
)
2981 struct inline_summary
*info
= inline_summary (node
);
2985 inline_hints hints
= 0;
2988 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2991 fprintf (dump_file
, " Estimating body: %s/%i\n"
2992 " Known to be false: ", cgraph_node_name (node
),
2995 for (i
= predicate_not_inlined_condition
;
2996 i
< (predicate_first_dynamic_condition
2997 + (int) vec_safe_length (info
->conds
)); i
++)
2998 if (!(possible_truths
& (1 << i
)))
3001 fprintf (dump_file
, ", ");
3003 dump_condition (dump_file
, info
->conds
, i
);
3007 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3008 if (evaluate_predicate (&e
->predicate
, possible_truths
))
3011 gcc_checking_assert (e
->time
>= 0);
3012 gcc_checking_assert (time
>= 0);
3013 if (!inline_param_summary
.exists ())
3017 int prob
= predicate_probability (info
->conds
,
3020 inline_param_summary
);
3021 gcc_checking_assert (prob
>= 0);
3022 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3023 time
+= apply_probability ((gcov_type
) e
->time
, prob
);
3025 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
3026 time
= MAX_TIME
* INLINE_TIME_SCALE
;
3027 gcc_checking_assert (time
>= 0);
3030 gcc_checking_assert (size
>= 0);
3031 gcc_checking_assert (time
>= 0);
3033 if (info
->loop_iterations
3034 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
3035 hints
|= INLINE_HINT_loop_iterations
;
3036 if (info
->loop_stride
3037 && !evaluate_predicate (info
->loop_stride
, possible_truths
))
3038 hints
|= INLINE_HINT_loop_stride
;
3039 if (info
->array_index
3040 && !evaluate_predicate (info
->array_index
, possible_truths
))
3041 hints
|= INLINE_HINT_array_index
;
3043 hints
|= INLINE_HINT_in_scc
;
3044 if (DECL_DECLARED_INLINE_P (node
->decl
))
3045 hints
|= INLINE_HINT_declared_inline
;
3047 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
3048 known_vals
, known_binfos
, known_aggs
);
3049 gcc_checking_assert (size
>= 0);
3050 gcc_checking_assert (time
>= 0);
3051 time
= RDIV (time
, INLINE_TIME_SCALE
);
3052 size
= RDIV (size
, INLINE_SIZE_SCALE
);
3054 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3055 fprintf (dump_file
, "\n size:%i time:%i\n", (int) size
, (int) time
);
3066 /* Estimate size and time needed to execute callee of EDGE assuming that
3067 parameters known to be constant at caller of EDGE are propagated.
3068 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
3069 and types for parameters. */
3072 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
3073 vec
<tree
> known_vals
,
3074 vec
<tree
> known_binfos
,
3075 vec
<ipa_agg_jump_function_p
> known_aggs
,
3076 int *ret_size
, int *ret_time
,
3077 inline_hints
*hints
)
3081 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
,
3083 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
,
3084 known_aggs
, ret_size
, ret_time
, hints
, vNULL
);
3087 /* Translate all conditions from callee representation into caller
3088 representation and symbolically evaluate predicate P into new predicate.
3090 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
3091 is summary of function predicate P is from. OPERAND_MAP is array giving
3092 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
3093 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
3094 predicate under which callee is executed. OFFSET_MAP is an array of of
3095 offsets that need to be added to conditions, negative offset means that
3096 conditions relying on values passed by reference have to be discarded
3097 because they might not be preserved (and should be considered offset zero
3098 for other purposes). */
3100 static struct predicate
3101 remap_predicate (struct inline_summary
*info
,
3102 struct inline_summary
*callee_info
,
3103 struct predicate
*p
,
3104 vec
<int> operand_map
,
3105 vec
<int> offset_map
,
3106 clause_t possible_truths
, struct predicate
*toplev_predicate
)
3109 struct predicate out
= true_predicate ();
3111 /* True predicate is easy. */
3112 if (true_predicate_p (p
))
3113 return *toplev_predicate
;
3114 for (i
= 0; p
->clause
[i
]; i
++)
3116 clause_t clause
= p
->clause
[i
];
3118 struct predicate clause_predicate
= false_predicate ();
3120 gcc_assert (i
< MAX_CLAUSES
);
3122 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
3123 /* Do we have condition we can't disprove? */
3124 if (clause
& possible_truths
& (1 << cond
))
3126 struct predicate cond_predicate
;
3127 /* Work out if the condition can translate to predicate in the
3128 inlined function. */
3129 if (cond
>= predicate_first_dynamic_condition
)
3131 struct condition
*c
;
3133 c
= &(*callee_info
->conds
)[cond
3135 predicate_first_dynamic_condition
];
3136 /* See if we can remap condition operand to caller's operand.
3137 Otherwise give up. */
3138 if (!operand_map
.exists ()
3139 || (int) operand_map
.length () <= c
->operand_num
3140 || operand_map
[c
->operand_num
] == -1
3141 /* TODO: For non-aggregate conditions, adding an offset is
3142 basically an arithmetic jump function processing which
3143 we should support in future. */
3144 || ((!c
->agg_contents
|| !c
->by_ref
)
3145 && offset_map
[c
->operand_num
] > 0)
3146 || (c
->agg_contents
&& c
->by_ref
3147 && offset_map
[c
->operand_num
] < 0))
3148 cond_predicate
= true_predicate ();
3151 struct agg_position_info ap
;
3152 HOST_WIDE_INT offset_delta
= offset_map
[c
->operand_num
];
3153 if (offset_delta
< 0)
3155 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
3158 gcc_assert (!c
->agg_contents
3159 || c
->by_ref
|| offset_delta
== 0);
3160 ap
.offset
= c
->offset
+ offset_delta
;
3161 ap
.agg_contents
= c
->agg_contents
;
3162 ap
.by_ref
= c
->by_ref
;
3163 cond_predicate
= add_condition (info
,
3164 operand_map
[c
->operand_num
],
3165 &ap
, c
->code
, c
->val
);
3168 /* Fixed conditions remains same, construct single
3169 condition predicate. */
3172 cond_predicate
.clause
[0] = 1 << cond
;
3173 cond_predicate
.clause
[1] = 0;
3175 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
3178 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
3180 return and_predicates (info
->conds
, &out
, toplev_predicate
);
3184 /* Update summary information of inline clones after inlining.
3185 Compute peak stack usage. */
3188 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3190 struct cgraph_edge
*e
;
3191 struct inline_summary
*callee_info
= inline_summary (node
);
3192 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
3195 callee_info
->stack_frame_offset
3196 = caller_info
->stack_frame_offset
3197 + caller_info
->estimated_self_stack_size
;
3198 peak
= callee_info
->stack_frame_offset
3199 + callee_info
->estimated_self_stack_size
;
3200 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
3201 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
3202 ipa_propagate_frequency (node
);
3203 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3205 if (!e
->inline_failed
)
3206 inline_update_callee_summaries (e
->callee
, depth
);
3207 inline_edge_summary (e
)->loop_depth
+= depth
;
3209 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3210 inline_edge_summary (e
)->loop_depth
+= depth
;
3213 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3214 When functoin A is inlined in B and A calls C with parameter that
3215 changes with probability PROB1 and C is known to be passthroug
3216 of argument if B that change with probability PROB2, the probability
3217 of change is now PROB1*PROB2. */
3220 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3221 struct cgraph_edge
*edge
)
3223 if (ipa_node_params_vector
.exists ())
3226 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3227 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3228 struct inline_edge_summary
*inlined_es
3229 = inline_edge_summary (inlined_edge
);
3231 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3233 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3234 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3235 && (ipa_get_jf_pass_through_formal_id (jfunc
)
3236 < (int) inlined_es
->param
.length ()))
3238 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
3239 int prob1
= es
->param
[i
].change_prob
;
3240 int prob2
= inlined_es
->param
[jf_formal_id
].change_prob
;
3241 int prob
= combine_probabilities (prob1
, prob2
);
3243 if (prob1
&& prob2
&& !prob
)
3246 es
->param
[i
].change_prob
= prob
;
3252 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3254 Remap predicates of callees of NODE. Rest of arguments match
3257 Also update change probabilities. */
3260 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3261 struct cgraph_node
*node
,
3262 struct inline_summary
*info
,
3263 struct inline_summary
*callee_info
,
3264 vec
<int> operand_map
,
3265 vec
<int> offset_map
,
3266 clause_t possible_truths
,
3267 struct predicate
*toplev_predicate
)
3269 struct cgraph_edge
*e
;
3270 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3272 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3275 if (e
->inline_failed
)
3277 remap_edge_change_prob (inlined_edge
, e
);
3281 p
= remap_predicate (info
, callee_info
,
3282 es
->predicate
, operand_map
, offset_map
,
3283 possible_truths
, toplev_predicate
);
3284 edge_set_predicate (e
, &p
);
3285 /* TODO: We should remove the edge for code that will be
3286 optimized out, but we need to keep verifiers and tree-inline
3287 happy. Make it cold for now. */
3288 if (false_predicate_p (&p
))
3295 edge_set_predicate (e
, toplev_predicate
);
3298 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3299 operand_map
, offset_map
, possible_truths
,
3302 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3304 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3307 remap_edge_change_prob (inlined_edge
, e
);
3310 p
= remap_predicate (info
, callee_info
,
3311 es
->predicate
, operand_map
, offset_map
,
3312 possible_truths
, toplev_predicate
);
3313 edge_set_predicate (e
, &p
);
3314 /* TODO: We should remove the edge for code that will be optimized
3315 out, but we need to keep verifiers and tree-inline happy.
3316 Make it cold for now. */
3317 if (false_predicate_p (&p
))
3324 edge_set_predicate (e
, toplev_predicate
);
3328 /* Same as remap_predicate, but set result into hint *HINT. */
3331 remap_hint_predicate (struct inline_summary
*info
,
3332 struct inline_summary
*callee_info
,
3333 struct predicate
**hint
,
3334 vec
<int> operand_map
,
3335 vec
<int> offset_map
,
3336 clause_t possible_truths
,
3337 struct predicate
*toplev_predicate
)
3343 p
= remap_predicate (info
, callee_info
,
3345 operand_map
, offset_map
,
3346 possible_truths
, toplev_predicate
);
3347 if (!false_predicate_p (&p
) && !true_predicate_p (&p
))
3350 set_hint_predicate (hint
, p
);
3352 **hint
= and_predicates (info
->conds
, *hint
, &p
);
3356 /* We inlined EDGE. Update summary of the function we inlined into. */
3359 inline_merge_summary (struct cgraph_edge
*edge
)
3361 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3362 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3363 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3364 struct inline_summary
*info
= inline_summary (to
);
3365 clause_t clause
= 0; /* not_inline is known to be false. */
3367 vec
<int> operand_map
= vNULL
;
3368 vec
<int> offset_map
= vNULL
;
3370 struct predicate toplev_predicate
;
3371 struct predicate true_p
= true_predicate ();
3372 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3375 toplev_predicate
= *es
->predicate
;
3377 toplev_predicate
= true_predicate ();
3379 if (ipa_node_params_vector
.exists () && callee_info
->conds
)
3381 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3382 int count
= ipa_get_cs_argument_count (args
);
3385 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3388 operand_map
.safe_grow_cleared (count
);
3389 offset_map
.safe_grow_cleared (count
);
3391 for (i
= 0; i
< count
; i
++)
3393 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3396 /* TODO: handle non-NOPs when merging. */
3397 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3399 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3400 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3401 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3404 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3406 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3407 if (offset
>= 0 && offset
< INT_MAX
)
3409 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3410 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3412 offset_map
[i
] = offset
;
3415 operand_map
[i
] = map
;
3416 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3419 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3421 struct predicate p
= remap_predicate (info
, callee_info
,
3422 &e
->predicate
, operand_map
,
3425 if (!false_predicate_p (&p
))
3427 gcov_type add_time
= ((gcov_type
) e
->time
* edge
->frequency
3428 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3429 int prob
= predicate_probability (callee_info
->conds
,
3432 add_time
= apply_probability ((gcov_type
) add_time
, prob
);
3433 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3434 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3435 if (prob
!= REG_BR_PROB_BASE
3436 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3438 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3439 (double) prob
/ REG_BR_PROB_BASE
);
3441 account_size_time (info
, e
->size
, add_time
, &p
);
3444 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3445 offset_map
, clause
, &toplev_predicate
);
3446 remap_hint_predicate (info
, callee_info
,
3447 &callee_info
->loop_iterations
,
3448 operand_map
, offset_map
, clause
, &toplev_predicate
);
3449 remap_hint_predicate (info
, callee_info
,
3450 &callee_info
->loop_stride
,
3451 operand_map
, offset_map
, clause
, &toplev_predicate
);
3452 remap_hint_predicate (info
, callee_info
,
3453 &callee_info
->array_index
,
3454 operand_map
, offset_map
, clause
, &toplev_predicate
);
3456 inline_update_callee_summaries (edge
->callee
,
3457 inline_edge_summary (edge
)->loop_depth
);
3459 /* We do not maintain predicates of inlined edges, free it. */
3460 edge_set_predicate (edge
, &true_p
);
3461 /* Similarly remove param summaries. */
3462 es
->param
.release ();
3463 operand_map
.release ();
3464 offset_map
.release ();
3467 /* For performance reasons inline_merge_summary is not updating overall size
3468 and time. Recompute it. */
3471 inline_update_overall_summary (struct cgraph_node
*node
)
3473 struct inline_summary
*info
= inline_summary (node
);
3479 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3481 info
->size
+= e
->size
, info
->time
+= e
->time
;
3482 if (info
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
3483 info
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
3485 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3486 ~(clause_t
) (1 << predicate_false_condition
),
3487 vNULL
, vNULL
, vNULL
);
3488 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3489 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3492 /* Return hints derrived from EDGE. */
3494 simple_edge_hints (struct cgraph_edge
*edge
)
3497 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3498 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3499 if (inline_summary (to
)->scc_no
3500 && inline_summary (to
)->scc_no
== inline_summary (edge
->callee
)->scc_no
3501 && !cgraph_edge_recursive_p (edge
))
3502 hints
|= INLINE_HINT_same_scc
;
3504 if (to
->lto_file_data
&& edge
->callee
->lto_file_data
3505 && to
->lto_file_data
!= edge
->callee
->lto_file_data
)
3506 hints
|= INLINE_HINT_cross_module
;
3511 /* Estimate the time cost for the caller when inlining EDGE.
3512 Only to be called via estimate_edge_time, that handles the
3515 When caching, also update the cache entry. Compute both time and
3516 size, since we always need both metrics eventually. */
3519 do_estimate_edge_time (struct cgraph_edge
*edge
)
3524 struct cgraph_node
*callee
;
3526 vec
<tree
> known_vals
;
3527 vec
<tree
> known_binfos
;
3528 vec
<ipa_agg_jump_function_p
> known_aggs
;
3529 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3531 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3533 gcc_checking_assert (edge
->inline_failed
);
3534 evaluate_properties_for_edge (edge
, true,
3535 &clause
, &known_vals
, &known_binfos
,
3537 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3538 known_aggs
, &size
, &time
, &hints
, es
->param
);
3539 known_vals
.release ();
3540 known_binfos
.release ();
3541 known_aggs
.release ();
3542 gcc_checking_assert (size
>= 0);
3543 gcc_checking_assert (time
>= 0);
3545 /* When caching, update the cache entry. */
3546 if (edge_growth_cache
.exists ())
3548 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3549 edge_growth_cache
.safe_grow_cleared (cgraph_edge_max_uid
);
3550 edge_growth_cache
[edge
->uid
].time
= time
+ (time
>= 0);
3552 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3553 hints
|= simple_edge_hints (edge
);
3554 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3560 /* Return estimated callee growth after inlining EDGE.
3561 Only to be called via estimate_edge_size. */
3564 do_estimate_edge_size (struct cgraph_edge
*edge
)
3567 struct cgraph_node
*callee
;
3569 vec
<tree
> known_vals
;
3570 vec
<tree
> known_binfos
;
3571 vec
<ipa_agg_jump_function_p
> known_aggs
;
3573 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3575 if (edge_growth_cache
.exists ())
3577 do_estimate_edge_time (edge
);
3578 size
= edge_growth_cache
[edge
->uid
].size
;
3579 gcc_checking_assert (size
);
3580 return size
- (size
> 0);
3583 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3585 /* Early inliner runs without caching, go ahead and do the dirty work. */
3586 gcc_checking_assert (edge
->inline_failed
);
3587 evaluate_properties_for_edge (edge
, true,
3588 &clause
, &known_vals
, &known_binfos
,
3590 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3591 known_aggs
, &size
, NULL
, NULL
, vNULL
);
3592 known_vals
.release ();
3593 known_binfos
.release ();
3594 known_aggs
.release ();
3599 /* Estimate the growth of the caller when inlining EDGE.
3600 Only to be called via estimate_edge_size. */
3603 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3606 struct cgraph_node
*callee
;
3608 vec
<tree
> known_vals
;
3609 vec
<tree
> known_binfos
;
3610 vec
<ipa_agg_jump_function_p
> known_aggs
;
3612 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3614 if (edge_growth_cache
.exists ())
3616 do_estimate_edge_time (edge
);
3617 hints
= edge_growth_cache
[edge
->uid
].hints
;
3618 gcc_checking_assert (hints
);
3622 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3624 /* Early inliner runs without caching, go ahead and do the dirty work. */
3625 gcc_checking_assert (edge
->inline_failed
);
3626 evaluate_properties_for_edge (edge
, true,
3627 &clause
, &known_vals
, &known_binfos
,
3629 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3630 known_aggs
, NULL
, NULL
, &hints
, vNULL
);
3631 known_vals
.release ();
3632 known_binfos
.release ();
3633 known_aggs
.release ();
3634 hints
|= simple_edge_hints (edge
);
3639 /* Estimate self time of the function NODE after inlining EDGE. */
3642 estimate_time_after_inlining (struct cgraph_node
*node
,
3643 struct cgraph_edge
*edge
)
3645 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3646 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3649 inline_summary (node
)->time
+ estimate_edge_time (edge
);
3652 if (time
> MAX_TIME
)
3656 return inline_summary (node
)->time
;
3660 /* Estimate the size of NODE after inlining EDGE which should be an
3661 edge to either NODE or a call inlined into NODE. */
3664 estimate_size_after_inlining (struct cgraph_node
*node
,
3665 struct cgraph_edge
*edge
)
3667 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3668 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3670 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3671 gcc_assert (size
>= 0);
3674 return inline_summary (node
)->size
;
3680 struct cgraph_node
*node
;
3681 bool self_recursive
;
3686 /* Worker for do_estimate_growth. Collect growth for all callers. */
3689 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3691 struct cgraph_edge
*e
;
3692 struct growth_data
*d
= (struct growth_data
*) data
;
3694 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3696 gcc_checking_assert (e
->inline_failed
);
3698 if (e
->caller
== d
->node
3699 || (e
->caller
->global
.inlined_to
3700 && e
->caller
->global
.inlined_to
== d
->node
))
3701 d
->self_recursive
= true;
3702 d
->growth
+= estimate_edge_growth (e
);
3708 /* Estimate the growth caused by inlining NODE into all callees. */
3711 do_estimate_growth (struct cgraph_node
*node
)
3713 struct growth_data d
= { node
, 0, false };
3714 struct inline_summary
*info
= inline_summary (node
);
3716 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3718 /* For self recursive functions the growth estimation really should be
3719 infinity. We don't want to return very large values because the growth
3720 plays various roles in badness computation fractions. Be sure to not
3721 return zero or negative growths. */
3722 if (d
.self_recursive
)
3723 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3724 else if (DECL_EXTERNAL (node
->decl
))
3728 if (cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3729 d
.growth
-= info
->size
;
3730 /* COMDAT functions are very often not shared across multiple units
3731 since they come from various template instantiations.
3732 Take this into account. */
3733 else if (DECL_COMDAT (node
->decl
)
3734 && cgraph_can_remove_if_no_direct_calls_p (node
))
3735 d
.growth
-= (info
->size
3736 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3740 if (node_growth_cache
.exists ())
3742 if ((int) node_growth_cache
.length () <= node
->uid
)
3743 node_growth_cache
.safe_grow_cleared (cgraph_max_uid
);
3744 node_growth_cache
[node
->uid
] = d
.growth
+ (d
.growth
>= 0);
3750 /* This function performs intraprocedural analysis in NODE that is required to
3751 inline indirect calls. */
3754 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3756 ipa_analyze_node (node
);
3757 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3759 ipa_print_node_params (dump_file
, node
);
3760 ipa_print_node_jump_functions (dump_file
, node
);
3765 /* Note function body size. */
3768 inline_analyze_function (struct cgraph_node
*node
)
3770 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3773 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3774 cgraph_node_name (node
), node
->order
);
3775 if (optimize
&& !node
->thunk
.thunk_p
)
3776 inline_indirect_intraprocedural_analysis (node
);
3777 compute_inline_parameters (node
, false);
3780 struct cgraph_edge
*e
;
3781 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3783 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3784 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3785 e
->call_stmt_cannot_inline_p
= true;
3787 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3789 if (e
->inline_failed
== CIF_FUNCTION_NOT_CONSIDERED
)
3790 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3791 e
->call_stmt_cannot_inline_p
= true;
3799 /* Called when new function is inserted to callgraph late. */
3802 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3804 inline_analyze_function (node
);
3808 /* Note function body size. */
3811 inline_generate_summary (void)
3813 struct cgraph_node
*node
;
3815 /* When not optimizing, do not bother to analyze. Inlining is still done
3816 because edge redirection needs to happen there. */
3817 if (!optimize
&& !flag_lto
&& !flag_wpa
)
3820 function_insertion_hook_holder
=
3821 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3823 ipa_register_cgraph_hooks ();
3824 inline_free_summary ();
3826 FOR_EACH_DEFINED_FUNCTION (node
)
3828 inline_analyze_function (node
);
3832 /* Read predicate from IB. */
3834 static struct predicate
3835 read_predicate (struct lto_input_block
*ib
)
3837 struct predicate out
;
3843 gcc_assert (k
<= MAX_CLAUSES
);
3844 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3848 /* Zero-initialize the remaining clauses in OUT. */
3849 while (k
<= MAX_CLAUSES
)
3850 out
.clause
[k
++] = 0;
3856 /* Write inline summary for edge E to OB. */
3859 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3861 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3865 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3866 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3867 es
->loop_depth
= streamer_read_uhwi (ib
);
3868 p
= read_predicate (ib
);
3869 edge_set_predicate (e
, &p
);
3870 length
= streamer_read_uhwi (ib
);
3873 es
->param
.safe_grow_cleared (length
);
3874 for (i
= 0; i
< length
; i
++)
3875 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3880 /* Stream in inline summaries from the section. */
3883 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3886 const struct lto_function_header
*header
=
3887 (const struct lto_function_header
*) data
;
3888 const int cfg_offset
= sizeof (struct lto_function_header
);
3889 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3890 const int string_offset
= main_offset
+ header
->main_size
;
3891 struct data_in
*data_in
;
3892 struct lto_input_block ib
;
3893 unsigned int i
, count2
, j
;
3894 unsigned int f_count
;
3896 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3900 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3901 header
->string_size
, vNULL
);
3902 f_count
= streamer_read_uhwi (&ib
);
3903 for (i
= 0; i
< f_count
; i
++)
3906 struct cgraph_node
*node
;
3907 struct inline_summary
*info
;
3908 lto_symtab_encoder_t encoder
;
3909 struct bitpack_d bp
;
3910 struct cgraph_edge
*e
;
3913 index
= streamer_read_uhwi (&ib
);
3914 encoder
= file_data
->symtab_node_encoder
;
3915 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3916 info
= inline_summary (node
);
3918 info
->estimated_stack_size
3919 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3920 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3921 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3923 bp
= streamer_read_bitpack (&ib
);
3924 info
->inlinable
= bp_unpack_value (&bp
, 1);
3926 count2
= streamer_read_uhwi (&ib
);
3927 gcc_assert (!info
->conds
);
3928 for (j
= 0; j
< count2
; j
++)
3931 c
.operand_num
= streamer_read_uhwi (&ib
);
3932 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3933 c
.val
= stream_read_tree (&ib
, data_in
);
3934 bp
= streamer_read_bitpack (&ib
);
3935 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3936 c
.by_ref
= bp_unpack_value (&bp
, 1);
3938 c
.offset
= streamer_read_uhwi (&ib
);
3939 vec_safe_push (info
->conds
, c
);
3941 count2
= streamer_read_uhwi (&ib
);
3942 gcc_assert (!info
->entry
);
3943 for (j
= 0; j
< count2
; j
++)
3945 struct size_time_entry e
;
3947 e
.size
= streamer_read_uhwi (&ib
);
3948 e
.time
= streamer_read_uhwi (&ib
);
3949 e
.predicate
= read_predicate (&ib
);
3951 vec_safe_push (info
->entry
, e
);
3954 p
= read_predicate (&ib
);
3955 set_hint_predicate (&info
->loop_iterations
, p
);
3956 p
= read_predicate (&ib
);
3957 set_hint_predicate (&info
->loop_stride
, p
);
3958 p
= read_predicate (&ib
);
3959 set_hint_predicate (&info
->array_index
, p
);
3960 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3961 read_inline_edge_summary (&ib
, e
);
3962 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3963 read_inline_edge_summary (&ib
, e
);
3966 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3968 lto_data_in_delete (data_in
);
3972 /* Read inline summary. Jump functions are shared among ipa-cp
3973 and inliner, so when ipa-cp is active, we don't need to write them
3977 inline_read_summary (void)
3979 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3980 struct lto_file_decl_data
*file_data
;
3983 inline_summary_alloc ();
3985 while ((file_data
= file_data_vec
[j
++]))
3988 const char *data
= lto_get_section_data (file_data
,
3989 LTO_section_inline_summary
,
3992 inline_read_section (file_data
, data
, len
);
3994 /* Fatal error here. We do not want to support compiling ltrans units
3995 with different version of compiler or different flags than the WPA
3996 unit, so this should never happen. */
3997 fatal_error ("ipa inline summary is missing in input file");
4001 ipa_register_cgraph_hooks ();
4003 ipa_prop_read_jump_functions ();
4005 function_insertion_hook_holder
=
4006 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
4010 /* Write predicate P to OB. */
4013 write_predicate (struct output_block
*ob
, struct predicate
*p
)
4017 for (j
= 0; p
->clause
[j
]; j
++)
4019 gcc_assert (j
< MAX_CLAUSES
);
4020 streamer_write_uhwi (ob
, p
->clause
[j
]);
4022 streamer_write_uhwi (ob
, 0);
4026 /* Write inline summary for edge E to OB. */
4029 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4031 struct inline_edge_summary
*es
= inline_edge_summary (e
);
4034 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4035 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4036 streamer_write_uhwi (ob
, es
->loop_depth
);
4037 write_predicate (ob
, es
->predicate
);
4038 streamer_write_uhwi (ob
, es
->param
.length ());
4039 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4040 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4044 /* Write inline summary for node in SET.
4045 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4046 active, we don't need to write them twice. */
4049 inline_write_summary (void)
4051 struct cgraph_node
*node
;
4052 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
4053 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4054 unsigned int count
= 0;
4057 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4059 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4060 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4061 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4064 streamer_write_uhwi (ob
, count
);
4066 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4068 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4069 cgraph_node
*cnode
= dyn_cast
<cgraph_node
> (snode
);
4070 if (cnode
&& (node
= cnode
)->definition
&& !node
->alias
)
4072 struct inline_summary
*info
= inline_summary (node
);
4073 struct bitpack_d bp
;
4074 struct cgraph_edge
*edge
;
4077 struct condition
*c
;
4079 streamer_write_uhwi (ob
,
4080 lto_symtab_encoder_encode (encoder
,
4083 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
4084 streamer_write_hwi (ob
, info
->self_size
);
4085 streamer_write_hwi (ob
, info
->self_time
);
4086 bp
= bitpack_create (ob
->main_stream
);
4087 bp_pack_value (&bp
, info
->inlinable
, 1);
4088 streamer_write_bitpack (&bp
);
4089 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4090 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4092 streamer_write_uhwi (ob
, c
->operand_num
);
4093 streamer_write_uhwi (ob
, c
->code
);
4094 stream_write_tree (ob
, c
->val
, true);
4095 bp
= bitpack_create (ob
->main_stream
);
4096 bp_pack_value (&bp
, c
->agg_contents
, 1);
4097 bp_pack_value (&bp
, c
->by_ref
, 1);
4098 streamer_write_bitpack (&bp
);
4099 if (c
->agg_contents
)
4100 streamer_write_uhwi (ob
, c
->offset
);
4102 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
4103 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
4105 streamer_write_uhwi (ob
, e
->size
);
4106 streamer_write_uhwi (ob
, e
->time
);
4107 write_predicate (ob
, &e
->predicate
);
4109 write_predicate (ob
, info
->loop_iterations
);
4110 write_predicate (ob
, info
->loop_stride
);
4111 write_predicate (ob
, info
->array_index
);
4112 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
4113 write_inline_edge_summary (ob
, edge
);
4114 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4115 write_inline_edge_summary (ob
, edge
);
4118 streamer_write_char_stream (ob
->main_stream
, 0);
4119 produce_asm (ob
, NULL
);
4120 destroy_output_block (ob
);
4122 if (optimize
&& !flag_ipa_cp
)
4123 ipa_prop_write_jump_functions ();
4127 /* Release inline summary. */
4130 inline_free_summary (void)
4132 struct cgraph_node
*node
;
4133 if (!inline_edge_summary_vec
.exists ())
4135 FOR_EACH_DEFINED_FUNCTION (node
)
4136 reset_inline_summary (node
);
4137 if (function_insertion_hook_holder
)
4138 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
4139 function_insertion_hook_holder
= NULL
;
4140 if (node_removal_hook_holder
)
4141 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
4142 node_removal_hook_holder
= NULL
;
4143 if (edge_removal_hook_holder
)
4144 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
4145 edge_removal_hook_holder
= NULL
;
4146 if (node_duplication_hook_holder
)
4147 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
4148 node_duplication_hook_holder
= NULL
;
4149 if (edge_duplication_hook_holder
)
4150 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
4151 edge_duplication_hook_holder
= NULL
;
4152 vec_free (inline_summary_vec
);
4153 inline_edge_summary_vec
.release ();
4154 if (edge_predicate_pool
)
4155 free_alloc_pool (edge_predicate_pool
);
4156 edge_predicate_pool
= 0;