1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011
3 Free Software Foundation, Inc.
4 Contributed by Jan Hubicka
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Analysis used by the inliner and other passes limiting code size growth.
24 We estimate for each function
26 - average function execution time
27 - inlining size benefit (that is how much of function body size
28 and its call sequence is expected to disappear by inlining)
29 - inlining time benefit
32 - call statement size and time
34 inlinie_summary datastructures store above information locally (i.e.
35 parameters of the function itself) and globally (i.e. parameters of
36 the function created by applying all the inline decisions already
37 present in the callgraph).
39 We provide accestor to the inline_summary datastructure and
40 basic logic updating the parameters when inlining is performed.
42 The summaries are context sensitive. Context means
43 1) partial assignment of known constant values of operands
44 2) whether function is inlined into the call or not.
45 It is easy to add more variants. To represent function size and time
46 that depends on context (i.e. it is known to be optimized away when
47 context is known either by inlining or from IP-CP and clonning),
48 we use predicates. Predicates are logical formulas in
49 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
50 specifying what conditions must be true. Conditions are simple test
51 of the form described above.
53 In order to make predicate (possibly) true, all of its clauses must
54 be (possibly) true. To make clause (possibly) true, one of conditions
55 it mentions must be (possibly) true. There are fixed bounds on
56 number of clauses and conditions and all the manipulation functions
57 are conservative in positive direction. I.e. we may lose precision
58 by thinking that predicate may be true even when it is not.
60 estimate_edge_size and estimate_edge_growth can be used to query
61 function size/time in the given context. inline_merge_summary merges
62 properties of caller and callee after inlining.
64 Finally pass_inline_parameters is exported. This is used to drive
65 computation of function parameters used by the early inliner. IPA
66 inlined performs analysis via its analyze_function method. */
70 #include "coretypes.h"
73 #include "tree-inline.h"
74 #include "langhooks.h"
77 #include "diagnostic.h"
78 #include "gimple-pretty-print.h"
81 #include "tree-pass.h"
84 #include "tree-flow.h"
86 #include "lto-streamer.h"
87 #include "ipa-inline.h"
88 #include "alloc-pool.h"
90 /* Estimate runtime of function can easilly run into huge numbers with many
91 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE in integer.
92 For anything larger we use gcov_type. */
93 #define MAX_TIME 1000000
95 /* Number of bits in integer, but we really want to be stable across different
97 #define NUM_CONDITIONS 32
99 enum predicate_conditions
101 predicate_false_condition
= 0,
102 predicate_not_inlined_condition
= 1,
103 predicate_first_dynamic_condition
= 2
106 /* Special condition code we use to represent test that operand is compile time
108 #define IS_NOT_CONSTANT ERROR_MARK
110 /* Holders of ipa cgraph hooks: */
111 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
112 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
113 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
114 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
115 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
116 static void inline_node_removal_hook (struct cgraph_node
*, void *);
117 static void inline_node_duplication_hook (struct cgraph_node
*,
118 struct cgraph_node
*, void *);
119 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
120 static void inline_edge_duplication_hook (struct cgraph_edge
*,
121 struct cgraph_edge
*,
124 /* VECtor holding inline summaries.
125 In GGC memory because conditions might point to constant trees. */
126 VEC(inline_summary_t
,gc
) *inline_summary_vec
;
127 VEC(inline_edge_summary_t
,heap
) *inline_edge_summary_vec
;
129 /* Cached node/edge growths. */
130 VEC(int,heap
) *node_growth_cache
;
131 VEC(edge_growth_cache_entry
,heap
) *edge_growth_cache
;
133 /* Edge predicates goes here. */
134 static alloc_pool edge_predicate_pool
;
136 /* Return true predicate (tautology).
137 We represent it by empty list of clauses. */
139 static inline struct predicate
140 true_predicate (void)
148 /* Return predicate testing single condition number COND. */
150 static inline struct predicate
151 single_cond_predicate (int cond
)
154 p
.clause
[0]=1 << cond
;
160 /* Return false predicate. First clause require false condition. */
162 static inline struct predicate
163 false_predicate (void)
165 return single_cond_predicate (predicate_false_condition
);
169 /* Return true if P is (false). */
172 true_predicate_p (struct predicate
*p
)
174 return !p
->clause
[0];
178 /* Return true if P is (false). */
181 false_predicate_p (struct predicate
*p
)
183 if (p
->clause
[0] == (1 << predicate_false_condition
))
185 gcc_checking_assert (!p
->clause
[1]
186 && p
->clause
[0] == 1 << predicate_false_condition
);
193 /* Return predicate that is set true when function is not inlined. */
194 static inline struct predicate
195 not_inlined_predicate (void)
197 return single_cond_predicate (predicate_not_inlined_condition
);
201 /* Add condition to condition list CONDS. */
203 static struct predicate
204 add_condition (struct inline_summary
*summary
, int operand_num
,
205 enum tree_code code
, tree val
)
209 struct condition new_cond
;
211 for (i
= 0; VEC_iterate (condition
, summary
->conds
, i
, c
); i
++)
213 if (c
->operand_num
== operand_num
216 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
218 /* Too many conditions. Give up and return constant true. */
219 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
220 return true_predicate ();
222 new_cond
.operand_num
= operand_num
;
223 new_cond
.code
= code
;
225 VEC_safe_push (condition
, gc
, summary
->conds
, &new_cond
);
226 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
230 /* Add clause CLAUSE into the predicate P. */
233 add_clause (struct predicate
*p
, clause_t clause
)
237 int insert_here
= -1;
243 /* False clause makes the whole predicate false. Kill the other variants. */
244 if (clause
== (1 << predicate_false_condition
))
246 p
->clause
[0] = (1 << predicate_false_condition
);
250 if (false_predicate_p (p
))
253 /* No one should be sily enough to add false into nontrivial clauses. */
254 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
256 /* Look where to insert the clause. At the same time prune out
257 clauses of P that are implied by the new clause and thus
259 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
261 p
->clause
[i2
] = p
->clause
[i
];
266 /* If p->clause[i] implies clause, there is nothing to add. */
267 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
269 /* We had nothing to add, none of clauses should've become redundant. */
270 gcc_checking_assert (i
== i2
);
274 if (p
->clause
[i
] < clause
&& insert_here
< 0)
277 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
278 Otherwise the p->clause[i] has to stay. */
279 if ((p
->clause
[i
] & clause
) != clause
)
282 /* We run out of variants. Be conservative in positive direction. */
283 if (i2
== MAX_CLAUSES
)
285 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
286 p
->clause
[i2
+ 1] = 0;
287 if (insert_here
>= 0)
288 for (;i2
> insert_here
; i2
--)
289 p
->clause
[i2
] = p
->clause
[i2
- 1];
292 p
->clause
[insert_here
] = clause
;
298 static struct predicate
299 and_predicates (struct predicate
*p
, struct predicate
*p2
)
301 struct predicate out
= *p
;
304 /* Avoid busy work. */
305 if (false_predicate_p (p2
) || true_predicate_p (p
))
307 if (false_predicate_p (p
) || true_predicate_p (p2
))
310 /* See how far predicates match. */
311 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
313 gcc_checking_assert (i
< MAX_CLAUSES
);
316 /* Combine the predicates rest. */
317 for (; p2
->clause
[i
]; i
++)
319 gcc_checking_assert (i
< MAX_CLAUSES
);
320 add_clause (&out
, p2
->clause
[i
]);
326 /* Return true if predicates are obviously equal. */
329 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
332 for (i
= 0; p
->clause
[i
]; i
++)
334 gcc_checking_assert (i
< MAX_CLAUSES
);
335 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
336 gcc_checking_assert (!p2
->clause
[i
] || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
337 if (p
->clause
[i
] != p2
->clause
[i
])
340 return !p2
->clause
[i
];
346 static struct predicate
347 or_predicates (struct predicate
*p
, struct predicate
*p2
)
349 struct predicate out
= true_predicate ();
352 /* Avoid busy work. */
353 if (false_predicate_p (p2
) || true_predicate_p (p
))
355 if (false_predicate_p (p
) || true_predicate_p (p2
))
357 if (predicates_equal_p (p
, p2
))
360 /* OK, combine the predicates. */
361 for (i
= 0; p
->clause
[i
]; i
++)
362 for (j
= 0; p2
->clause
[j
]; j
++)
364 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
365 add_clause (&out
, p
->clause
[i
] | p2
->clause
[j
]);
371 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false if predicate P
375 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
379 /* True remains true. */
380 if (true_predicate_p (p
))
383 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
385 /* See if we can find clause we can disprove. */
386 for (i
= 0; p
->clause
[i
]; i
++)
388 gcc_checking_assert (i
< MAX_CLAUSES
);
389 if (!(p
->clause
[i
] & possible_truths
))
396 /* Dump conditional COND. */
399 dump_condition (FILE *f
, conditions conditions
, int cond
)
402 if (cond
== predicate_false_condition
)
403 fprintf (f
, "false");
404 else if (cond
== predicate_not_inlined_condition
)
405 fprintf (f
, "not inlined");
408 c
= VEC_index (condition
, conditions
, cond
- predicate_first_dynamic_condition
);
409 fprintf (f
, "op%i", c
->operand_num
);
410 if (c
->code
== IS_NOT_CONSTANT
)
412 fprintf (f
, " not constant");
415 fprintf (f
, " %s ", op_symbol_code (c
->code
));
416 print_generic_expr (f
, c
->val
, 1);
421 /* Dump clause CLAUSE. */
424 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
431 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
432 if (clause
& (1 << i
))
437 dump_condition (f
, conds
, i
);
443 /* Dump predicate PREDICATE. */
446 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
449 if (true_predicate_p (pred
))
450 dump_clause (f
, conds
, 0);
452 for (i
= 0; pred
->clause
[i
]; i
++)
456 dump_clause (f
, conds
, pred
->clause
[i
]);
462 /* Record SIZE and TIME under condition PRED into the inline summary. */
465 account_size_time (struct inline_summary
*summary
, int size
, int time
, struct predicate
*pred
)
471 if (false_predicate_p (pred
))
474 /* We need to create initial empty unconitional clause, but otherwie
475 we don't need to account empty times and sizes. */
476 if (!size
&& !time
&& summary
->entry
)
479 /* Watch overflow that might result from insane profiles. */
480 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
481 time
= MAX_TIME
* INLINE_TIME_SCALE
;
482 gcc_assert (time
>= 0);
484 for (i
= 0; VEC_iterate (size_time_entry
, summary
->entry
, i
, e
); i
++)
485 if (predicates_equal_p (&e
->predicate
, pred
))
494 e
= VEC_index (size_time_entry
, summary
->entry
, 0);
495 gcc_assert (!e
->predicate
.clause
[0]);
497 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
499 fprintf (dump_file
, "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
500 ((double)size
) / INLINE_SIZE_SCALE
, ((double)time
) / INLINE_TIME_SCALE
,
501 found
? "" : "new ");
502 dump_predicate (dump_file
, summary
->conds
, pred
);
506 struct size_time_entry new_entry
;
507 new_entry
.size
= size
;
508 new_entry
.time
= time
;
509 new_entry
.predicate
= *pred
;
510 VEC_safe_push (size_time_entry
, gc
, summary
->entry
, &new_entry
);
516 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
517 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
521 /* Set predicate for edge E. */
524 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
526 struct inline_edge_summary
*es
= inline_edge_summary (e
);
527 if (predicate
&& !true_predicate_p (predicate
))
530 es
->predicate
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
531 *es
->predicate
= *predicate
;
536 pool_free (edge_predicate_pool
, es
->predicate
);
537 es
->predicate
= NULL
;
542 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
543 Return clause of possible truths. When INLINE_P is true, assume that
547 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
549 VEC (tree
, heap
) *known_vals
)
551 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
552 struct inline_summary
*info
= inline_summary (node
);
556 for (i
= 0; VEC_iterate (condition
, info
->conds
, i
, c
); i
++)
561 /* We allow call stmt to have fewer arguments than the callee
562 function (especially for K&R style programs). So bound
564 if (c
->operand_num
< (int)VEC_length (tree
, known_vals
))
565 val
= VEC_index (tree
, known_vals
, c
->operand_num
);
571 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
574 if (c
->code
== IS_NOT_CONSTANT
)
576 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
578 && integer_zerop (res
))
580 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
586 /* Work out what conditions might be true at invocation of E. */
589 evaluate_conditions_for_edge (struct cgraph_edge
*e
, bool inline_p
)
591 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
592 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
593 struct inline_summary
*info
= inline_summary (callee
);
596 if (ipa_node_params_vector
&& info
->conds
597 /* FIXME: it seems that we forget to get argument count in some cases,
598 probaby for previously indirect edges or so. */
599 && ipa_get_cs_argument_count (IPA_EDGE_REF (e
)))
601 struct ipa_node_params
*parms_info
;
602 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
603 int i
, count
= ipa_get_cs_argument_count (args
);
604 VEC (tree
, heap
) *known_vals
= NULL
;
606 if (e
->caller
->global
.inlined_to
)
607 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
609 parms_info
= IPA_NODE_REF (e
->caller
);
611 VEC_safe_grow_cleared (tree
, heap
, known_vals
, count
);
612 for (i
= 0; i
< count
; i
++)
614 tree cst
= ipa_cst_from_jfunc (parms_info
,
615 ipa_get_ith_jump_func (args
, i
));
617 VEC_replace (tree
, known_vals
, i
, cst
);
619 clause
= evaluate_conditions_for_known_args (callee
,
620 inline_p
, known_vals
);
621 VEC_free (tree
, heap
, known_vals
);
624 for (i
= 0; i
< (int)VEC_length (condition
, info
->conds
); i
++)
625 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
631 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
634 inline_summary_alloc (void)
636 if (!node_removal_hook_holder
)
637 node_removal_hook_holder
=
638 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
639 if (!edge_removal_hook_holder
)
640 edge_removal_hook_holder
=
641 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
642 if (!node_duplication_hook_holder
)
643 node_duplication_hook_holder
=
644 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
645 if (!edge_duplication_hook_holder
)
646 edge_duplication_hook_holder
=
647 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
649 if (VEC_length (inline_summary_t
, inline_summary_vec
)
650 <= (unsigned) cgraph_max_uid
)
651 VEC_safe_grow_cleared (inline_summary_t
, gc
,
652 inline_summary_vec
, cgraph_max_uid
+ 1);
653 if (VEC_length (inline_edge_summary_t
, inline_edge_summary_vec
)
654 <= (unsigned) cgraph_edge_max_uid
)
655 VEC_safe_grow_cleared (inline_edge_summary_t
, heap
,
656 inline_edge_summary_vec
, cgraph_edge_max_uid
+ 1);
657 if (!edge_predicate_pool
)
658 edge_predicate_pool
= create_alloc_pool ("edge predicates", sizeof (struct predicate
),
662 /* Hook that is called by cgraph.c when a node is removed. */
665 inline_node_removal_hook (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
667 struct inline_summary
*info
;
668 if (VEC_length (inline_summary_t
, inline_summary_vec
)
669 <= (unsigned)node
->uid
)
671 info
= inline_summary (node
);
672 reset_node_growth_cache (node
);
673 VEC_free (condition
, gc
, info
->conds
);
674 VEC_free (size_time_entry
, gc
, info
->entry
);
677 memset (info
, 0, sizeof (inline_summary_t
));
681 /* Hook that is called by cgraph.c when a node is duplicated. */
684 inline_node_duplication_hook (struct cgraph_node
*src
, struct cgraph_node
*dst
,
685 ATTRIBUTE_UNUSED
void *data
)
687 struct inline_summary
*info
;
688 inline_summary_alloc ();
689 info
= inline_summary (dst
);
690 memcpy (info
, inline_summary (src
),
691 sizeof (struct inline_summary
));
692 /* TODO: as an optimization, we may avoid copying conditions
693 that are known to be false or true. */
694 info
->conds
= VEC_copy (condition
, gc
, info
->conds
);
696 /* When there are any replacements in the function body, see if we can figure
697 out that something was optimized out. */
698 if (ipa_node_params_vector
&& dst
->clone
.tree_map
)
700 VEC(size_time_entry
,gc
) *entry
= info
->entry
;
701 /* Use SRC parm info since it may not be copied yet. */
702 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
703 VEC (tree
, heap
) *known_vals
= NULL
;
704 int count
= ipa_get_param_count (parms_info
);
706 clause_t possible_truths
;
707 struct predicate true_pred
= true_predicate ();
709 int optimized_out_size
= 0;
710 gcov_type optimized_out_time
= 0;
711 bool inlined_to_p
= false;
712 struct cgraph_edge
*edge
;
715 VEC_safe_grow_cleared (tree
, heap
, known_vals
, count
);
716 for (i
= 0; i
< count
; i
++)
718 tree t
= ipa_get_param (parms_info
, i
);
719 struct ipa_replace_map
*r
;
722 VEC_iterate (ipa_replace_map_p
, dst
->clone
.tree_map
, j
, r
);
729 VEC_replace (tree
, known_vals
, i
, r
->new_tree
);
734 possible_truths
= evaluate_conditions_for_known_args (dst
,
736 VEC_free (tree
, heap
, known_vals
);
738 account_size_time (info
, 0, 0, &true_pred
);
740 /* Remap size_time vectors.
741 Simplify the predicate by prunning out alternatives that are known
743 TODO: as on optimization, we can also eliminate conditions known to be true. */
744 for (i
= 0; VEC_iterate (size_time_entry
, entry
, i
, e
); i
++)
746 struct predicate new_predicate
= true_predicate ();
747 for (j
= 0; e
->predicate
.clause
[j
]; j
++)
748 if (!(possible_truths
& e
->predicate
.clause
[j
]))
750 new_predicate
= false_predicate ();
754 add_clause (&new_predicate
,
755 possible_truths
& e
->predicate
.clause
[j
]);
756 if (false_predicate_p (&new_predicate
))
758 optimized_out_size
+= e
->size
;
759 optimized_out_time
+= e
->time
;
762 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
765 /* Remap edge predicates with the same simplificaiton as above. */
766 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
768 struct predicate new_predicate
= true_predicate ();
769 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
771 if (!edge
->inline_failed
)
775 for (j
= 0; es
->predicate
->clause
[j
]; j
++)
776 if (!(possible_truths
& es
->predicate
->clause
[j
]))
778 new_predicate
= false_predicate ();
782 add_clause (&new_predicate
,
783 possible_truths
& es
->predicate
->clause
[j
]);
784 if (false_predicate_p (&new_predicate
)
785 && !false_predicate_p (es
->predicate
))
787 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
788 optimized_out_time
+= (es
->call_stmt_time
789 * (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
)
793 *es
->predicate
= new_predicate
;
796 /* Remap indirect edge predicates with the same simplificaiton as above. */
797 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
799 struct predicate new_predicate
= true_predicate ();
800 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
802 if (!edge
->inline_failed
)
806 for (j
= 0; es
->predicate
->clause
[j
]; j
++)
807 if (!(possible_truths
& es
->predicate
->clause
[j
]))
809 new_predicate
= false_predicate ();
813 add_clause (&new_predicate
,
814 possible_truths
& es
->predicate
->clause
[j
]);
815 if (false_predicate_p (&new_predicate
)
816 && !false_predicate_p (es
->predicate
))
818 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
819 optimized_out_time
+= (es
->call_stmt_time
820 * (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
)
824 *es
->predicate
= new_predicate
;
827 /* If inliner or someone after inliner will ever start producing
828 non-trivial clones, we will get trouble with lack of information
829 about updating self sizes, because size vectors already contains
830 sizes of the calees. */
831 gcc_assert (!inlined_to_p
832 || (!optimized_out_size
&& !optimized_out_time
));
834 info
->size
-= optimized_out_size
/ INLINE_SIZE_SCALE
;
835 info
->self_size
-= optimized_out_size
/ INLINE_SIZE_SCALE
;
836 gcc_assert (info
->size
> 0);
837 gcc_assert (info
->self_size
> 0);
839 optimized_out_time
/= INLINE_TIME_SCALE
;
840 if (optimized_out_time
> MAX_TIME
)
841 optimized_out_time
= MAX_TIME
;
842 info
->time
-= optimized_out_time
;
843 info
->self_time
-= optimized_out_time
;
846 if (info
->self_time
< 0)
850 info
->entry
= VEC_copy (size_time_entry
, gc
, info
->entry
);
854 /* Hook that is called by cgraph.c when a node is duplicated. */
857 inline_edge_duplication_hook (struct cgraph_edge
*src
, struct cgraph_edge
*dst
,
858 ATTRIBUTE_UNUSED
void *data
)
860 struct inline_edge_summary
*info
;
861 struct inline_edge_summary
*srcinfo
;
862 inline_summary_alloc ();
863 info
= inline_edge_summary (dst
);
864 srcinfo
= inline_edge_summary (src
);
865 memcpy (info
, srcinfo
,
866 sizeof (struct inline_edge_summary
));
867 info
->predicate
= NULL
;
868 edge_set_predicate (dst
, srcinfo
->predicate
);
872 /* Keep edge cache consistent across edge removal. */
875 inline_edge_removal_hook (struct cgraph_edge
*edge
, void *data ATTRIBUTE_UNUSED
)
877 if (edge_growth_cache
)
878 reset_edge_growth_cache (edge
);
879 if (edge
->uid
< (int)VEC_length (inline_edge_summary_t
, inline_edge_summary_vec
))
881 edge_set_predicate (edge
, NULL
);
882 memset (inline_edge_summary (edge
), 0, sizeof (struct inline_edge_summary
));
887 /* Initialize growth caches. */
890 initialize_growth_caches (void)
892 if (cgraph_edge_max_uid
)
893 VEC_safe_grow_cleared (edge_growth_cache_entry
, heap
, edge_growth_cache
,
894 cgraph_edge_max_uid
);
896 VEC_safe_grow_cleared (int, heap
, node_growth_cache
, cgraph_max_uid
);
900 /* Free growth caches. */
903 free_growth_caches (void)
905 VEC_free (edge_growth_cache_entry
, heap
, edge_growth_cache
);
906 edge_growth_cache
= 0;
907 VEC_free (int, heap
, node_growth_cache
);
908 node_growth_cache
= 0;
912 /* Dump edge summaries associated to NODE and recursively to all clones.
916 dump_inline_edge_summary (FILE * f
, int indent
, struct cgraph_node
*node
,
917 struct inline_summary
*info
)
919 struct cgraph_edge
*edge
;
920 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
922 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
923 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
924 fprintf (f
, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i callee size:%2i stack:%2i",
925 indent
, "", cgraph_node_name (callee
),
927 !edge
->inline_failed
? "inlined"
928 : cgraph_inline_failed_string (edge
->inline_failed
),
934 (int)inline_summary (callee
)->size
,
935 (int)inline_summary (callee
)->estimated_stack_size
);
938 fprintf (f
, " predicate: ");
939 dump_predicate (f
, info
->conds
, es
->predicate
);
943 if (!edge
->inline_failed
)
945 fprintf (f
, "%*sStack frame offset %i, callee self size %i, callee size %i\n",
947 (int)inline_summary (callee
)->stack_frame_offset
,
948 (int)inline_summary (callee
)->estimated_self_stack_size
,
949 (int)inline_summary (callee
)->estimated_stack_size
);
950 dump_inline_edge_summary (f
, indent
+2, callee
, info
);
953 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
955 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
956 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i time: %2i\n",
964 fprintf (f
, "predicate: ");
965 dump_predicate (f
, info
->conds
, es
->predicate
);
974 dump_inline_summary (FILE * f
, struct cgraph_node
*node
)
978 struct inline_summary
*s
= inline_summary (node
);
981 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
983 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
984 fprintf (f
, " always_inline");
986 fprintf (f
, " inlinable");
988 fprintf (f
, " versionable");
989 fprintf (f
, "\n self time: %i\n",
991 fprintf (f
, " global time: %i\n", s
->time
);
992 fprintf (f
, " self size: %i\n",
994 fprintf (f
, " global size: %i\n", s
->size
);
995 fprintf (f
, " self stack: %i\n",
996 (int) s
->estimated_self_stack_size
);
997 fprintf (f
, " global stack: %i\n",
998 (int) s
->estimated_stack_size
);
1000 VEC_iterate (size_time_entry
, s
->entry
, i
, e
);
1003 fprintf (f
, " size:%f, time:%f, predicate:",
1004 (double) e
->size
/ INLINE_SIZE_SCALE
,
1005 (double) e
->time
/ INLINE_TIME_SCALE
);
1006 dump_predicate (f
, s
->conds
, &e
->predicate
);
1008 fprintf (f
, " calls:\n");
1009 dump_inline_edge_summary (f
, 4, node
, s
);
1015 debug_inline_summary (struct cgraph_node
*node
)
1017 dump_inline_summary (stderr
, node
);
1021 dump_inline_summaries (FILE *f
)
1023 struct cgraph_node
*node
;
1025 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1026 if (node
->analyzed
&& !node
->global
.inlined_to
)
1027 dump_inline_summary (f
, node
);
1030 /* Give initial reasons why inlining would fail on EDGE. This gets either
1031 nullified or usually overwritten by more precise reasons later. */
1034 initialize_inline_failed (struct cgraph_edge
*e
)
1036 struct cgraph_node
*callee
= e
->callee
;
1038 if (e
->indirect_unknown_callee
)
1039 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1040 else if (!callee
->analyzed
)
1041 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1042 else if (callee
->local
.redefined_extern_inline
)
1043 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1044 else if (e
->call_stmt
&& gimple_call_cannot_inline_p (e
->call_stmt
))
1045 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1047 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1050 /* See if statement might disappear after inlining.
1051 0 - means not eliminated
1052 1 - half of statements goes away
1053 2 - for sure it is eliminated.
1054 We are not terribly sophisticated, basically looking for simple abstraction
1055 penalty wrappers. */
1058 eliminated_by_inlining_prob (gimple stmt
)
1060 enum gimple_code code
= gimple_code (stmt
);
1066 if (gimple_num_ops (stmt
) != 2)
1069 /* Casts of parameters, loads from parameters passed by reference
1070 and stores to return value or parameters are often free after
1071 inlining dua to SRA and further combining.
1072 Assume that half of statements goes away. */
1073 if (gimple_assign_rhs_code (stmt
) == CONVERT_EXPR
1074 || gimple_assign_rhs_code (stmt
) == NOP_EXPR
1075 || gimple_assign_rhs_code (stmt
) == VIEW_CONVERT_EXPR
1076 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1078 tree rhs
= gimple_assign_rhs1 (stmt
);
1079 tree lhs
= gimple_assign_lhs (stmt
);
1080 tree inner_rhs
= rhs
;
1081 tree inner_lhs
= lhs
;
1082 bool rhs_free
= false;
1083 bool lhs_free
= false;
1085 while (handled_component_p (inner_lhs
)
1086 || TREE_CODE (inner_lhs
) == MEM_REF
)
1087 inner_lhs
= TREE_OPERAND (inner_lhs
, 0);
1088 while (handled_component_p (inner_rhs
)
1089 || TREE_CODE (inner_rhs
) == ADDR_EXPR
1090 || TREE_CODE (inner_rhs
) == MEM_REF
)
1091 inner_rhs
= TREE_OPERAND (inner_rhs
, 0);
1094 if (TREE_CODE (inner_rhs
) == PARM_DECL
1095 || (TREE_CODE (inner_rhs
) == SSA_NAME
1096 && SSA_NAME_IS_DEFAULT_DEF (inner_rhs
)
1097 && TREE_CODE (SSA_NAME_VAR (inner_rhs
)) == PARM_DECL
))
1099 if (rhs_free
&& is_gimple_reg (lhs
))
1101 if (((TREE_CODE (inner_lhs
) == PARM_DECL
1102 || (TREE_CODE (inner_lhs
) == SSA_NAME
1103 && SSA_NAME_IS_DEFAULT_DEF (inner_lhs
)
1104 && TREE_CODE (SSA_NAME_VAR (inner_lhs
)) == PARM_DECL
))
1105 && inner_lhs
!= lhs
)
1106 || TREE_CODE (inner_lhs
) == RESULT_DECL
1107 || (TREE_CODE (inner_lhs
) == SSA_NAME
1108 && TREE_CODE (SSA_NAME_VAR (inner_lhs
)) == RESULT_DECL
))
1111 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1113 if (lhs_free
&& rhs_free
)
1123 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1124 predicates to the CFG edges. */
1127 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1128 struct inline_summary
*summary
,
1134 enum tree_code code
, inverted_code
;
1140 last
= last_stmt (bb
);
1142 || gimple_code (last
) != GIMPLE_COND
)
1144 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1146 op
= gimple_cond_lhs (last
);
1147 /* TODO: handle conditionals like
1150 if (TREE_CODE (op
) != SSA_NAME
)
1152 if (SSA_NAME_IS_DEFAULT_DEF (op
))
1154 index
= ipa_get_param_decl_index (info
, SSA_NAME_VAR (op
));
1157 code
= gimple_cond_code (last
);
1158 inverted_code
= invert_tree_comparison (code
,
1159 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1161 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1163 struct predicate p
= add_condition (summary
,
1165 e
->flags
& EDGE_TRUE_VALUE
1166 ? code
: inverted_code
,
1167 gimple_cond_rhs (last
));
1168 e
->aux
= pool_alloc (edge_predicate_pool
);
1169 *(struct predicate
*)e
->aux
= p
;
1174 if (builtin_constant_p (op))
1178 Here we can predicate nonconstant_code. We can't
1179 really handle constant_code since we have no predicate
1180 for this and also the constant code is not known to be
1181 optimized away when inliner doen't see operand is constant.
1182 Other optimizers might think otherwise. */
1183 set_stmt
= SSA_NAME_DEF_STMT (op
);
1184 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1185 || gimple_call_num_args (set_stmt
) != 1)
1187 op2
= gimple_call_arg (set_stmt
, 0);
1188 if (!SSA_NAME_IS_DEFAULT_DEF (op2
))
1190 index
= ipa_get_param_decl_index (info
, SSA_NAME_VAR (op2
));
1193 if (gimple_cond_code (last
) != NE_EXPR
1194 || !integer_zerop (gimple_cond_rhs (last
)))
1196 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1197 if (e
->flags
& EDGE_FALSE_VALUE
)
1199 struct predicate p
= add_condition (summary
,
1203 e
->aux
= pool_alloc (edge_predicate_pool
);
1204 *(struct predicate
*)e
->aux
= p
;
1209 /* If BB ends by a switch we can turn into predicates, attach corresponding
1210 predicates to the CFG edges. */
1213 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1214 struct inline_summary
*summary
,
1225 last
= last_stmt (bb
);
1227 || gimple_code (last
) != GIMPLE_SWITCH
)
1229 op
= gimple_switch_index (last
);
1230 if (TREE_CODE (op
) != SSA_NAME
1231 || !SSA_NAME_IS_DEFAULT_DEF (op
))
1233 index
= ipa_get_param_decl_index (info
, SSA_NAME_VAR (op
));
1237 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1239 e
->aux
= pool_alloc (edge_predicate_pool
);
1240 *(struct predicate
*)e
->aux
= false_predicate ();
1242 n
= gimple_switch_num_labels(last
);
1243 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1245 tree cl
= gimple_switch_label (last
, case_idx
);
1249 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1250 min
= CASE_LOW (cl
);
1251 max
= CASE_HIGH (cl
);
1253 /* For default we might want to construct predicate that none
1254 of cases is met, but it is bit hard to do not having negations
1255 of conditionals handy. */
1257 p
= true_predicate ();
1259 p
= add_condition (summary
, index
,
1264 struct predicate p1
, p2
;
1265 p1
= add_condition (summary
, index
,
1268 p2
= add_condition (summary
, index
,
1271 p
= and_predicates (&p1
, &p2
);
1273 *(struct predicate
*)e
->aux
1274 = or_predicates (&p
, (struct predicate
*)e
->aux
);
1279 /* For each BB in NODE attach to its AUX pointer predicate under
1280 which it is executable. */
1283 compute_bb_predicates (struct cgraph_node
*node
,
1284 struct ipa_node_params
*parms_info
,
1285 struct inline_summary
*summary
)
1287 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1291 FOR_EACH_BB_FN (bb
, my_function
)
1293 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1294 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1297 /* Entry block is always executable. */
1298 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
= pool_alloc (edge_predicate_pool
);
1299 *(struct predicate
*)ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1300 = true_predicate ();
1302 /* A simple dataflow propagation of predicates forward in the CFG.
1303 TODO: work in reverse postorder. */
1307 FOR_EACH_BB_FN (bb
, my_function
)
1309 struct predicate p
= false_predicate ();
1312 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1316 struct predicate this_bb_predicate
= *(struct predicate
*)e
->src
->aux
;
1318 this_bb_predicate
= and_predicates (&this_bb_predicate
,
1319 (struct predicate
*)e
->aux
);
1320 p
= or_predicates (&p
, &this_bb_predicate
);
1321 if (true_predicate_p (&p
))
1325 if (false_predicate_p (&p
))
1326 gcc_assert (!bb
->aux
);
1332 bb
->aux
= pool_alloc (edge_predicate_pool
);
1333 *((struct predicate
*)bb
->aux
) = p
;
1335 else if (!predicates_equal_p (&p
, (struct predicate
*)bb
->aux
))
1338 *((struct predicate
*)bb
->aux
) = p
;
1346 /* We keep info about constantness of SSA names. */
1348 typedef struct predicate predicate_t
;
1349 DEF_VEC_O (predicate_t
);
1350 DEF_VEC_ALLOC_O (predicate_t
, heap
);
1353 /* Return predicate specifying when the STMT might have result that is not a compile
1356 static struct predicate
1357 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1358 struct inline_summary
*summary
,
1360 VEC (predicate_t
, heap
) *nonconstant_names
)
1363 struct predicate p
= true_predicate ();
1366 struct predicate op_non_const
;
1368 /* What statments might be optimized away
1369 when their arguments are constant
1370 TODO: also trivial builtins.
1371 builtin_constant_p is already handled later. */
1372 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1373 && gimple_code (stmt
) != GIMPLE_COND
1374 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1377 /* Stores and loads will stay anyway.
1378 TODO: Constant memory accesses could be handled here, too. */
1379 if (gimple_vuse (stmt
))
1382 /* See if we understand all operands before we start
1383 adding conditionals. */
1384 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1386 if (TREE_CODE (use
) != SSA_NAME
)
1388 /* For arguments we can build a condition. */
1389 if (SSA_NAME_IS_DEFAULT_DEF (use
)
1390 && ipa_get_param_decl_index (info
, SSA_NAME_VAR (use
)) >= 0)
1392 /* If we know when operand is constant,
1393 we still can say something useful. */
1394 if (!true_predicate_p (VEC_index (predicate_t
, nonconstant_names
,
1395 SSA_NAME_VERSION (use
))))
1399 op_non_const
= false_predicate ();
1400 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1402 if (SSA_NAME_IS_DEFAULT_DEF (use
)
1403 && ipa_get_param_decl_index (info
, SSA_NAME_VAR (use
)) >= 0)
1404 p
= add_condition (summary
,
1405 ipa_get_param_decl_index (info
, SSA_NAME_VAR (use
)),
1406 IS_NOT_CONSTANT
, NULL
);
1408 p
= *VEC_index (predicate_t
, nonconstant_names
,
1409 SSA_NAME_VERSION (use
));
1410 op_non_const
= or_predicates (&p
, &op_non_const
);
1412 if (gimple_code (stmt
) == GIMPLE_ASSIGN
1413 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
1414 VEC_replace (predicate_t
, nonconstant_names
,
1415 SSA_NAME_VERSION (gimple_assign_lhs (stmt
)), &op_non_const
);
1416 return op_non_const
;
1420 /* Compute function body size parameters for NODE.
1421 When EARLY is true, we compute only simple summaries without
1422 non-trivial predicates to drive the early inliner. */
1425 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
1428 /* Estimate static overhead for function prologue/epilogue and alignment. */
1430 /* Benefits are scaled by probability of elimination that is in range
1433 gimple_stmt_iterator bsi
;
1434 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1436 struct inline_summary
*info
= inline_summary (node
);
1437 struct predicate bb_predicate
;
1438 struct ipa_node_params
*parms_info
= NULL
;
1439 VEC (predicate_t
, heap
) *nonconstant_names
= NULL
;
1441 if (ipa_node_params_vector
&& !early
&& optimize
)
1443 parms_info
= IPA_NODE_REF (node
);
1444 VEC_safe_grow_cleared (predicate_t
, heap
, nonconstant_names
,
1445 VEC_length (tree
, SSANAMES (my_function
)));
1453 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
1454 cgraph_node_name (node
));
1456 /* When we run into maximal number of entries, we assign everything to the
1457 constant truth case. Be sure to have it in list. */
1458 bb_predicate
= true_predicate ();
1459 account_size_time (info
, 0, 0, &bb_predicate
);
1461 bb_predicate
= not_inlined_predicate ();
1462 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
1464 gcc_assert (my_function
&& my_function
->cfg
);
1466 compute_bb_predicates (node
, parms_info
, info
);
1467 FOR_EACH_BB_FN (bb
, my_function
)
1469 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
1471 /* TODO: Obviously predicates can be propagated down across CFG. */
1475 bb_predicate
= *(struct predicate
*)bb
->aux
;
1477 bb_predicate
= false_predicate ();
1480 bb_predicate
= true_predicate ();
1482 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1484 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
1485 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
1488 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1490 gimple stmt
= gsi_stmt (bsi
);
1491 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
1492 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
1494 struct predicate will_be_nonconstant
;
1496 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1498 fprintf (dump_file
, " ");
1499 print_gimple_stmt (dump_file
, stmt
, 0, 0);
1500 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
1501 ((double)freq
)/CGRAPH_FREQ_BASE
, this_size
, this_time
);
1504 if (is_gimple_call (stmt
))
1506 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
1507 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1509 /* Special case: results of BUILT_IN_CONSTANT_P will be always
1510 resolved as constant. We however don't want to optimize
1511 out the cgraph edges. */
1512 if (nonconstant_names
1513 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
1514 && gimple_call_lhs (stmt
)
1515 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
1517 struct predicate false_p
= false_predicate ();
1518 VEC_replace (predicate_t
, nonconstant_names
,
1519 SSA_NAME_VERSION (gimple_call_lhs (stmt
)), &false_p
);
1522 es
->call_stmt_size
= this_size
;
1523 es
->call_stmt_time
= this_time
;
1524 es
->loop_depth
= bb
->loop_depth
;
1525 edge_set_predicate (edge
, &bb_predicate
);
1527 /* Do not inline calls where we cannot triviall work around
1528 mismatches in argument or return types. */
1530 && cgraph_function_or_thunk_node (edge
->callee
, NULL
)
1531 && !gimple_check_call_matching_types (stmt
,
1532 cgraph_function_or_thunk_node (edge
->callee
,
1535 edge
->call_stmt_cannot_inline_p
= true;
1536 gimple_call_set_cannot_inline (stmt
, true);
1539 gcc_assert (!gimple_call_cannot_inline_p (stmt
));
1542 /* TODO: When conditional jump or swithc is known to be constant, but
1543 we did not translate it into the predicates, we really can account
1544 just maximum of the possible paths. */
1547 = will_be_nonconstant_predicate (parms_info
, info
,
1548 stmt
, nonconstant_names
);
1549 if (this_time
|| this_size
)
1557 prob
= eliminated_by_inlining_prob (stmt
);
1558 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
1559 fprintf (dump_file
, "\t\t50%% will be eliminated by inlining\n");
1560 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
1561 fprintf (dump_file
, "\t\twill eliminated by inlining\n");
1564 p
= and_predicates (&bb_predicate
, &will_be_nonconstant
);
1566 p
= true_predicate ();
1568 /* We account everything but the calls. Calls have their own
1569 size/time info attached to cgraph edges. This is neccesary
1570 in order to make the cost disappear after inlining. */
1571 if (!is_gimple_call (stmt
))
1575 struct predicate ip
= not_inlined_predicate ();
1576 ip
= and_predicates (&ip
, &p
);
1577 account_size_time (info
, this_size
* prob
,
1578 this_time
* prob
, &ip
);
1581 account_size_time (info
, this_size
* (2 - prob
),
1582 this_time
* (2 - prob
), &p
);
1585 gcc_assert (time
>= 0);
1586 gcc_assert (size
>= 0);
1590 FOR_ALL_BB_FN (bb
, my_function
)
1596 pool_free (edge_predicate_pool
, bb
->aux
);
1598 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1601 pool_free (edge_predicate_pool
, e
->aux
);
1605 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
1606 if (time
> MAX_TIME
)
1608 inline_summary (node
)->self_time
= time
;
1609 inline_summary (node
)->self_size
= size
;
1610 VEC_free (predicate_t
, heap
, nonconstant_names
);
1613 fprintf (dump_file
, "\n");
1614 dump_inline_summary (dump_file
, node
);
1619 /* Compute parameters of functions used by inliner.
1620 EARLY is true when we compute parameters for the early inliner */
1623 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
1625 HOST_WIDE_INT self_stack_size
;
1626 struct cgraph_edge
*e
;
1627 struct inline_summary
*info
;
1629 gcc_assert (!node
->global
.inlined_to
);
1631 inline_summary_alloc ();
1633 info
= inline_summary (node
);
1635 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
1636 Once this happen, we will need to more curefully predict call
1638 if (node
->thunk
.thunk_p
)
1640 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
1641 struct predicate t
= true_predicate ();
1643 info
->inlinable
= info
->versionable
= 0;
1644 node
->callees
->call_stmt_cannot_inline_p
= true;
1645 node
->local
.can_change_signature
= false;
1646 es
->call_stmt_time
= 1;
1647 es
->call_stmt_size
= 1;
1648 account_size_time (info
, 0, 0, &t
);
1652 /* Estimate the stack size for the function if we're optimizing. */
1653 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
1654 info
->estimated_self_stack_size
= self_stack_size
;
1655 info
->estimated_stack_size
= self_stack_size
;
1656 info
->stack_frame_offset
= 0;
1658 /* Can this function be inlined at all? */
1659 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
1661 /* Inlinable functions always can change signature. */
1662 if (info
->inlinable
)
1663 node
->local
.can_change_signature
= true;
1666 /* Functions calling builtin_apply can not change signature. */
1667 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1668 if (DECL_BUILT_IN (e
->callee
->decl
)
1669 && DECL_BUILT_IN_CLASS (e
->callee
->decl
) == BUILT_IN_NORMAL
1670 && DECL_FUNCTION_CODE (e
->callee
->decl
) == BUILT_IN_APPLY_ARGS
)
1672 node
->local
.can_change_signature
= !e
;
1674 estimate_function_body_sizes (node
, early
);
1676 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
1677 info
->time
= info
->self_time
;
1678 info
->size
= info
->self_size
;
1679 info
->stack_frame_offset
= 0;
1680 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
1684 /* Compute parameters of functions used by inliner using
1685 current_function_decl. */
1688 compute_inline_parameters_for_current (void)
1690 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
1694 struct gimple_opt_pass pass_inline_parameters
=
1698 "inline_param", /* name */
1700 compute_inline_parameters_for_current
,/* execute */
1703 0, /* static_pass_number */
1704 TV_INLINE_HEURISTICS
, /* tv_id */
1705 0, /* properties_required */
1706 0, /* properties_provided */
1707 0, /* properties_destroyed */
1708 0, /* todo_flags_start */
1709 0 /* todo_flags_finish */
1714 /* Increase SIZE and TIME for size and time needed to handle edge E. */
1717 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
)
1719 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1720 *size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1721 *time
+= (es
->call_stmt_time
1722 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
));
1723 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
1724 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
1728 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE. */
1731 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
1732 clause_t possible_truths
)
1734 struct cgraph_edge
*e
;
1735 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1737 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1738 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
1740 if (e
->inline_failed
)
1741 estimate_edge_size_and_time (e
, size
, time
);
1743 estimate_calls_size_and_time (e
->callee
, size
, time
,
1747 /* TODO: look for devirtualizing oppurtunities. */
1748 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1750 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1751 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
1752 estimate_edge_size_and_time (e
, size
, time
);
1757 /* Estimate size and time needed to execute NODE assuming
1758 POSSIBLE_TRUTHS clause. */
1761 estimate_node_size_and_time (struct cgraph_node
*node
,
1762 clause_t possible_truths
,
1763 int *ret_size
, int *ret_time
)
1765 struct inline_summary
*info
= inline_summary (node
);
1767 int size
= 0, time
= 0;
1771 && (dump_flags
& TDF_DETAILS
))
1774 fprintf (dump_file
, " Estimating body: %s/%i\n"
1775 " Known to be false: ",
1776 cgraph_node_name (node
),
1779 for (i
= predicate_not_inlined_condition
;
1780 i
< (predicate_first_dynamic_condition
1781 + (int)VEC_length (condition
, info
->conds
)); i
++)
1782 if (!(possible_truths
& (1 << i
)))
1785 fprintf (dump_file
, ", ");
1787 dump_condition (dump_file
, info
->conds
, i
);
1791 for (i
= 0; VEC_iterate (size_time_entry
, info
->entry
, i
, e
); i
++)
1792 if (evaluate_predicate (&e
->predicate
, possible_truths
))
1793 time
+= e
->time
, size
+= e
->size
;
1795 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
1796 time
= MAX_TIME
* INLINE_TIME_SCALE
;
1798 estimate_calls_size_and_time (node
, &size
, &time
, possible_truths
);
1799 time
= (time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
1800 size
= (size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
1804 && (dump_flags
& TDF_DETAILS
))
1805 fprintf (dump_file
, "\n size:%i time:%i\n", size
, time
);
1814 /* Estimate size and time needed to execute callee of EDGE assuming that
1815 parameters known to be constant at caller of EDGE are propagated.
1816 KNOWN_VALs is a vector of assumed known constant values for parameters. */
1819 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
1820 VEC (tree
, heap
) *known_vals
,
1821 int *ret_size
, int *ret_time
)
1825 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
);
1826 estimate_node_size_and_time (node
, clause
, ret_size
, ret_time
);
1830 /* Translate all conditions from callee representation into caller representation and
1831 symbolically evaluate predicate P into new predicate.
1833 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO is summary
1834 of function predicate P is from. OPERAND_MAP is array giving callee formal IDs the
1835 caller formal IDs. POSSSIBLE_TRUTHS is clausule of all callee conditions that
1836 may be true in caller context. TOPLEV_PREDICATE is predicate under which callee
1839 static struct predicate
1840 remap_predicate (struct inline_summary
*info
, struct inline_summary
*callee_info
,
1841 struct predicate
*p
,
1842 VEC (int, heap
) *operand_map
,
1843 clause_t possible_truths
,
1844 struct predicate
*toplev_predicate
)
1847 struct predicate out
= true_predicate ();
1849 /* True predicate is easy. */
1850 if (true_predicate_p (p
))
1851 return *toplev_predicate
;
1852 for (i
= 0; p
->clause
[i
]; i
++)
1854 clause_t clause
= p
->clause
[i
];
1856 struct predicate clause_predicate
= false_predicate ();
1858 gcc_assert (i
< MAX_CLAUSES
);
1860 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
1861 /* Do we have condition we can't disprove? */
1862 if (clause
& possible_truths
& (1 << cond
))
1864 struct predicate cond_predicate
;
1865 /* Work out if the condition can translate to predicate in the
1866 inlined function. */
1867 if (cond
>= predicate_first_dynamic_condition
)
1869 struct condition
*c
;
1871 c
= VEC_index (condition
, callee_info
->conds
,
1872 cond
- predicate_first_dynamic_condition
);
1873 /* See if we can remap condition operand to caller's operand.
1874 Otherwise give up. */
1876 || VEC_length (int, operand_map
) <= c
->operand_num
1877 || VEC_index (int, operand_map
, c
->operand_num
) == -1)
1878 cond_predicate
= true_predicate ();
1880 cond_predicate
= add_condition (info
,
1881 VEC_index (int, operand_map
,
1885 /* Fixed conditions remains same, construct single
1886 condition predicate. */
1889 cond_predicate
.clause
[0] = 1 << cond
;
1890 cond_predicate
.clause
[1] = 0;
1892 clause_predicate
= or_predicates (&clause_predicate
, &cond_predicate
);
1894 out
= and_predicates (&out
, &clause_predicate
);
1896 return and_predicates (&out
, toplev_predicate
);
1900 /* Update summary information of inline clones after inlining.
1901 Compute peak stack usage. */
1904 inline_update_callee_summaries (struct cgraph_node
*node
,
1907 struct cgraph_edge
*e
;
1908 struct inline_summary
*callee_info
= inline_summary (node
);
1909 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
1912 callee_info
->stack_frame_offset
1913 = caller_info
->stack_frame_offset
1914 + caller_info
->estimated_self_stack_size
;
1915 peak
= callee_info
->stack_frame_offset
1916 + callee_info
->estimated_self_stack_size
;
1917 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
1919 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
1920 cgraph_propagate_frequency (node
);
1921 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1923 if (!e
->inline_failed
)
1924 inline_update_callee_summaries (e
->callee
, depth
);
1925 inline_edge_summary (e
)->loop_depth
+= depth
;
1927 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1928 inline_edge_summary (e
)->loop_depth
+= depth
;
1932 /* Remap predicates of callees of NODE. Rest of arguments match
1936 remap_edge_predicates (struct cgraph_node
*node
,
1937 struct inline_summary
*info
,
1938 struct inline_summary
*callee_info
,
1939 VEC (int, heap
) *operand_map
,
1940 clause_t possible_truths
,
1941 struct predicate
*toplev_predicate
)
1943 struct cgraph_edge
*e
;
1944 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1946 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1950 p
= remap_predicate (info
, callee_info
,
1951 es
->predicate
, operand_map
, possible_truths
,
1953 edge_set_predicate (e
, &p
);
1954 /* TODO: We should remove the edge for code that will be optimized out,
1955 but we need to keep verifiers and tree-inline happy.
1956 Make it cold for now. */
1957 if (false_predicate_p (&p
))
1963 if (!e
->inline_failed
)
1964 remap_edge_predicates (e
->callee
, info
, callee_info
, operand_map
,
1965 possible_truths
, toplev_predicate
);
1967 edge_set_predicate (e
, toplev_predicate
);
1969 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1971 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1975 p
= remap_predicate (info
, callee_info
,
1976 es
->predicate
, operand_map
, possible_truths
,
1978 edge_set_predicate (e
, &p
);
1979 /* TODO: We should remove the edge for code that will be optimized out,
1980 but we need to keep verifiers and tree-inline happy.
1981 Make it cold for now. */
1982 if (false_predicate_p (&p
))
1989 edge_set_predicate (e
, toplev_predicate
);
1994 /* We inlined EDGE. Update summary of the function we inlined into. */
1997 inline_merge_summary (struct cgraph_edge
*edge
)
1999 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
2000 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
2001 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
2002 struct inline_summary
*info
= inline_summary (to
);
2003 clause_t clause
= 0; /* not_inline is known to be false. */
2005 VEC (int, heap
) *operand_map
= NULL
;
2007 struct predicate toplev_predicate
;
2008 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2011 toplev_predicate
= *es
->predicate
;
2013 toplev_predicate
= true_predicate ();
2015 if (ipa_node_params_vector
&& callee_info
->conds
2016 /* FIXME: it seems that we forget to get argument count in some cases,
2017 probaby for previously indirect edges or so.
2018 Removing the test leads to ICE on tramp3d. */
2019 && ipa_get_cs_argument_count (IPA_EDGE_REF (edge
)))
2021 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
2022 int count
= ipa_get_cs_argument_count (args
);
2025 clause
= evaluate_conditions_for_edge (edge
, true);
2026 VEC_safe_grow_cleared (int, heap
, operand_map
, count
);
2027 for (i
= 0; i
< count
; i
++)
2029 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
2031 /* TODO: handle non-NOPs when merging. */
2032 if (jfunc
->type
== IPA_JF_PASS_THROUGH
2033 && jfunc
->value
.pass_through
.operation
== NOP_EXPR
)
2034 map
= jfunc
->value
.pass_through
.formal_id
;
2035 VEC_replace (int, operand_map
, i
, map
);
2036 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
2039 for (i
= 0; VEC_iterate (size_time_entry
, callee_info
->entry
, i
, e
); i
++)
2041 struct predicate p
= remap_predicate (info
, callee_info
,
2042 &e
->predicate
, operand_map
, clause
,
2044 gcov_type add_time
= ((gcov_type
)e
->time
* edge
->frequency
2045 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2046 if (add_time
> MAX_TIME
)
2047 add_time
= MAX_TIME
;
2048 account_size_time (info
, e
->size
, add_time
, &p
);
2050 remap_edge_predicates (edge
->callee
, info
, callee_info
, operand_map
,
2051 clause
, &toplev_predicate
);
2054 for (i
= 0; VEC_iterate (size_time_entry
, info
->entry
, i
, e
); i
++)
2055 info
->size
+= e
->size
, info
->time
+= e
->time
;
2056 estimate_calls_size_and_time (to
, &info
->size
, &info
->time
,
2057 ~(clause_t
)(1 << predicate_false_condition
));
2059 inline_update_callee_summaries (edge
->callee
,
2060 inline_edge_summary (edge
)->loop_depth
);
2062 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
2063 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
2067 /* Estimate the time cost for the caller when inlining EDGE.
2068 Only to be called via estimate_edge_time, that handles the
2071 When caching, also update the cache entry. Compute both time and
2072 size, since we always need both metrics eventually. */
2075 do_estimate_edge_time (struct cgraph_edge
*edge
)
2080 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2082 gcc_checking_assert (edge
->inline_failed
);
2083 estimate_node_size_and_time (cgraph_function_or_thunk_node (edge
->callee
, NULL
),
2084 evaluate_conditions_for_edge (edge
, true),
2087 ret
= (((gcov_type
)time
- es
->call_stmt_time
) * edge
->frequency
2088 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2092 /* When caching, update the cache entry. */
2093 if (edge_growth_cache
)
2096 if ((int)VEC_length (edge_growth_cache_entry
, edge_growth_cache
)
2098 VEC_safe_grow_cleared (edge_growth_cache_entry
, heap
, edge_growth_cache
,
2099 cgraph_edge_max_uid
);
2100 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
)->time
2103 ret_size
= size
- es
->call_stmt_size
;
2104 gcc_checking_assert (es
->call_stmt_size
);
2105 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
)->size
2106 = ret_size
+ (ret_size
>= 0);
2112 /* Estimate the growth of the caller when inlining EDGE.
2113 Only to be called via estimate_edge_size. */
2116 do_estimate_edge_growth (struct cgraph_edge
*edge
)
2119 struct cgraph_node
*callee
;
2121 /* When we do caching, use do_estimate_edge_time to populate the entry. */
2123 if (edge_growth_cache
)
2125 do_estimate_edge_time (edge
);
2126 size
= VEC_index (edge_growth_cache_entry
,
2129 gcc_checking_assert (size
);
2130 return size
- (size
> 0);
2132 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
2134 /* Early inliner runs without caching, go ahead and do the dirty work. */
2135 gcc_checking_assert (edge
->inline_failed
);
2136 estimate_node_size_and_time (callee
,
2137 evaluate_conditions_for_edge (edge
, true),
2139 gcc_checking_assert (inline_edge_summary (edge
)->call_stmt_size
);
2140 return size
- inline_edge_summary (edge
)->call_stmt_size
;
2144 /* Estimate self time of the function NODE after inlining EDGE. */
2147 estimate_time_after_inlining (struct cgraph_node
*node
,
2148 struct cgraph_edge
*edge
)
2150 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2151 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
2153 gcov_type time
= inline_summary (node
)->time
+ estimate_edge_time (edge
);
2156 if (time
> MAX_TIME
)
2160 return inline_summary (node
)->time
;
2164 /* Estimate the size of NODE after inlining EDGE which should be an
2165 edge to either NODE or a call inlined into NODE. */
2168 estimate_size_after_inlining (struct cgraph_node
*node
,
2169 struct cgraph_edge
*edge
)
2171 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2172 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
2174 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
2175 gcc_assert (size
>= 0);
2178 return inline_summary (node
)->size
;
2184 bool self_recursive
;
2189 /* Worker for do_estimate_growth. Collect growth for all callers. */
2192 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
2194 struct cgraph_edge
*e
;
2195 struct growth_data
*d
= (struct growth_data
*) data
;
2197 for (e
= node
->callers
; e
; e
= e
->next_caller
)
2199 gcc_checking_assert (e
->inline_failed
);
2201 if (e
->caller
== node
2202 || (e
->caller
->global
.inlined_to
2203 && e
->caller
->global
.inlined_to
== node
))
2204 d
->self_recursive
= true;
2205 d
->growth
+= estimate_edge_growth (e
);
2211 /* Estimate the growth caused by inlining NODE into all callees. */
2214 do_estimate_growth (struct cgraph_node
*node
)
2216 struct growth_data d
= {0, false};
2217 struct inline_summary
*info
= inline_summary (node
);
2219 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
2221 /* For self recursive functions the growth estimation really should be
2222 infinity. We don't want to return very large values because the growth
2223 plays various roles in badness computation fractions. Be sure to not
2224 return zero or negative growths. */
2225 if (d
.self_recursive
)
2226 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
2229 if (!DECL_EXTERNAL (node
->decl
)
2230 && cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
2231 d
.growth
-= info
->size
;
2232 /* COMDAT functions are very often not shared across multiple units since they
2233 come from various template instantiations. Take this into account. */
2234 else if (DECL_COMDAT (node
->decl
)
2235 && cgraph_can_remove_if_no_direct_calls_p (node
))
2236 d
.growth
-= (info
->size
2237 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
)) + 50) / 100;
2240 if (node_growth_cache
)
2242 if ((int)VEC_length (int, node_growth_cache
) <= node
->uid
)
2243 VEC_safe_grow_cleared (int, heap
, node_growth_cache
, cgraph_max_uid
);
2244 VEC_replace (int, node_growth_cache
, node
->uid
, d
.growth
+ (d
.growth
>= 0));
2250 /* This function performs intraprocedural analysis in NODE that is required to
2251 inline indirect calls. */
2254 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
2256 ipa_analyze_node (node
);
2257 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2259 ipa_print_node_params (dump_file
, node
);
2260 ipa_print_node_jump_functions (dump_file
, node
);
2265 /* Note function body size. */
2268 inline_analyze_function (struct cgraph_node
*node
)
2270 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2271 current_function_decl
= node
->decl
;
2274 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
2275 cgraph_node_name (node
), node
->uid
);
2276 /* FIXME: We should remove the optimize check after we ensure we never run
2277 IPA passes when not optimizing. */
2278 if (flag_indirect_inlining
&& optimize
&& !node
->thunk
.thunk_p
)
2279 inline_indirect_intraprocedural_analysis (node
);
2280 compute_inline_parameters (node
, false);
2282 current_function_decl
= NULL
;
2287 /* Called when new function is inserted to callgraph late. */
2290 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
2292 inline_analyze_function (node
);
2296 /* Note function body size. */
2299 inline_generate_summary (void)
2301 struct cgraph_node
*node
;
2303 function_insertion_hook_holder
=
2304 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
2306 if (flag_indirect_inlining
)
2307 ipa_register_cgraph_hooks ();
2309 FOR_EACH_DEFINED_FUNCTION (node
)
2311 inline_analyze_function (node
);
2315 /* Read predicate from IB. */
2317 static struct predicate
2318 read_predicate (struct lto_input_block
*ib
)
2320 struct predicate out
;
2326 gcc_assert (k
<= MAX_CLAUSES
);
2327 clause
= out
.clause
[k
++] = lto_input_uleb128 (ib
);
2331 /* Zero-initialize the remaining clauses in OUT. */
2332 while (k
<= MAX_CLAUSES
)
2333 out
.clause
[k
++] = 0;
2339 /* Write inline summary for edge E to OB. */
2342 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
2344 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2347 es
->call_stmt_size
= lto_input_uleb128 (ib
);
2348 es
->call_stmt_time
= lto_input_uleb128 (ib
);
2349 es
->loop_depth
= lto_input_uleb128 (ib
);
2350 p
= read_predicate (ib
);
2351 edge_set_predicate (e
, &p
);
2355 /* Stream in inline summaries from the section. */
2358 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
2361 const struct lto_function_header
*header
=
2362 (const struct lto_function_header
*) data
;
2363 const int32_t cfg_offset
= sizeof (struct lto_function_header
);
2364 const int32_t main_offset
= cfg_offset
+ header
->cfg_size
;
2365 const int32_t string_offset
= main_offset
+ header
->main_size
;
2366 struct data_in
*data_in
;
2367 struct lto_input_block ib
;
2368 unsigned int i
, count2
, j
;
2369 unsigned int f_count
;
2371 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
2375 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
2376 header
->string_size
, NULL
);
2377 f_count
= lto_input_uleb128 (&ib
);
2378 for (i
= 0; i
< f_count
; i
++)
2381 struct cgraph_node
*node
;
2382 struct inline_summary
*info
;
2383 lto_cgraph_encoder_t encoder
;
2384 struct bitpack_d bp
;
2385 struct cgraph_edge
*e
;
2387 index
= lto_input_uleb128 (&ib
);
2388 encoder
= file_data
->cgraph_node_encoder
;
2389 node
= lto_cgraph_encoder_deref (encoder
, index
);
2390 info
= inline_summary (node
);
2392 info
->estimated_stack_size
2393 = info
->estimated_self_stack_size
= lto_input_uleb128 (&ib
);
2394 info
->size
= info
->self_size
= lto_input_uleb128 (&ib
);
2395 info
->time
= info
->self_time
= lto_input_uleb128 (&ib
);
2397 bp
= lto_input_bitpack (&ib
);
2398 info
->inlinable
= bp_unpack_value (&bp
, 1);
2399 info
->versionable
= bp_unpack_value (&bp
, 1);
2401 count2
= lto_input_uleb128 (&ib
);
2402 gcc_assert (!info
->conds
);
2403 for (j
= 0; j
< count2
; j
++)
2406 c
.operand_num
= lto_input_uleb128 (&ib
);
2407 c
.code
= (enum tree_code
) lto_input_uleb128 (&ib
);
2408 c
.val
= lto_input_tree (&ib
, data_in
);
2409 VEC_safe_push (condition
, gc
, info
->conds
, &c
);
2411 count2
= lto_input_uleb128 (&ib
);
2412 gcc_assert (!info
->entry
);
2413 for (j
= 0; j
< count2
; j
++)
2415 struct size_time_entry e
;
2417 e
.size
= lto_input_uleb128 (&ib
);
2418 e
.time
= lto_input_uleb128 (&ib
);
2419 e
.predicate
= read_predicate (&ib
);
2421 VEC_safe_push (size_time_entry
, gc
, info
->entry
, &e
);
2423 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2424 read_inline_edge_summary (&ib
, e
);
2425 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2426 read_inline_edge_summary (&ib
, e
);
2429 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
2431 lto_data_in_delete (data_in
);
2435 /* Read inline summary. Jump functions are shared among ipa-cp
2436 and inliner, so when ipa-cp is active, we don't need to write them
2440 inline_read_summary (void)
2442 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
2443 struct lto_file_decl_data
*file_data
;
2446 inline_summary_alloc ();
2448 while ((file_data
= file_data_vec
[j
++]))
2451 const char *data
= lto_get_section_data (file_data
, LTO_section_inline_summary
, NULL
, &len
);
2453 inline_read_section (file_data
, data
, len
);
2455 /* Fatal error here. We do not want to support compiling ltrans units with
2456 different version of compiler or different flags than the WPA unit, so
2457 this should never happen. */
2458 fatal_error ("ipa inline summary is missing in input file");
2460 if (flag_indirect_inlining
)
2462 ipa_register_cgraph_hooks ();
2464 ipa_prop_read_jump_functions ();
2466 function_insertion_hook_holder
=
2467 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
2471 /* Write predicate P to OB. */
2474 write_predicate (struct output_block
*ob
, struct predicate
*p
)
2478 for (j
= 0; p
->clause
[j
]; j
++)
2480 gcc_assert (j
< MAX_CLAUSES
);
2481 lto_output_uleb128_stream (ob
->main_stream
,
2484 lto_output_uleb128_stream (ob
->main_stream
, 0);
2488 /* Write inline summary for edge E to OB. */
2491 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
2493 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2494 lto_output_uleb128_stream (ob
->main_stream
, es
->call_stmt_size
);
2495 lto_output_uleb128_stream (ob
->main_stream
, es
->call_stmt_time
);
2496 lto_output_uleb128_stream (ob
->main_stream
, es
->loop_depth
);
2497 write_predicate (ob
, es
->predicate
);
2501 /* Write inline summary for node in SET.
2502 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
2503 active, we don't need to write them twice. */
2506 inline_write_summary (cgraph_node_set set
,
2507 varpool_node_set vset ATTRIBUTE_UNUSED
)
2509 struct cgraph_node
*node
;
2510 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
2511 lto_cgraph_encoder_t encoder
= ob
->decl_state
->cgraph_node_encoder
;
2512 unsigned int count
= 0;
2515 for (i
= 0; i
< lto_cgraph_encoder_size (encoder
); i
++)
2516 if (lto_cgraph_encoder_deref (encoder
, i
)->analyzed
)
2518 lto_output_uleb128_stream (ob
->main_stream
, count
);
2520 for (i
= 0; i
< lto_cgraph_encoder_size (encoder
); i
++)
2522 node
= lto_cgraph_encoder_deref (encoder
, i
);
2525 struct inline_summary
*info
= inline_summary (node
);
2526 struct bitpack_d bp
;
2527 struct cgraph_edge
*edge
;
2530 struct condition
*c
;
2533 lto_output_uleb128_stream (ob
->main_stream
,
2534 lto_cgraph_encoder_encode (encoder
, node
));
2535 lto_output_sleb128_stream (ob
->main_stream
,
2536 info
->estimated_self_stack_size
);
2537 lto_output_sleb128_stream (ob
->main_stream
,
2539 lto_output_sleb128_stream (ob
->main_stream
,
2541 bp
= bitpack_create (ob
->main_stream
);
2542 bp_pack_value (&bp
, info
->inlinable
, 1);
2543 bp_pack_value (&bp
, info
->versionable
, 1);
2544 lto_output_bitpack (&bp
);
2545 lto_output_uleb128_stream (ob
->main_stream
,
2546 VEC_length (condition
, info
->conds
));
2547 for (i
= 0; VEC_iterate (condition
, info
->conds
, i
, c
); i
++)
2549 lto_output_uleb128_stream (ob
->main_stream
,
2551 lto_output_uleb128_stream (ob
->main_stream
,
2553 lto_output_tree (ob
, c
->val
, true);
2555 lto_output_uleb128_stream (ob
->main_stream
,
2556 VEC_length (size_time_entry
, info
->entry
));
2558 VEC_iterate (size_time_entry
, info
->entry
, i
, e
);
2561 lto_output_uleb128_stream (ob
->main_stream
,
2563 lto_output_uleb128_stream (ob
->main_stream
,
2565 write_predicate (ob
, &e
->predicate
);
2567 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2568 write_inline_edge_summary (ob
, edge
);
2569 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
2570 write_inline_edge_summary (ob
, edge
);
2573 lto_output_1_stream (ob
->main_stream
, 0);
2574 produce_asm (ob
, NULL
);
2575 destroy_output_block (ob
);
2577 if (flag_indirect_inlining
&& !flag_ipa_cp
)
2578 ipa_prop_write_jump_functions (set
);
2582 /* Release inline summary. */
2585 inline_free_summary (void)
2587 if (function_insertion_hook_holder
)
2588 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
2589 function_insertion_hook_holder
= NULL
;
2590 if (node_removal_hook_holder
)
2591 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
2592 if (edge_removal_hook_holder
)
2593 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
2594 node_removal_hook_holder
= NULL
;
2595 if (node_duplication_hook_holder
)
2596 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
2597 if (edge_duplication_hook_holder
)
2598 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
2599 node_duplication_hook_holder
= NULL
;
2600 VEC_free (inline_summary_t
, gc
, inline_summary_vec
);
2601 inline_summary_vec
= NULL
;
2602 VEC_free (inline_edge_summary_t
, heap
, inline_edge_summary_vec
);
2603 inline_edge_summary_vec
= NULL
;
2604 if (edge_predicate_pool
)
2605 free_alloc_pool (edge_predicate_pool
);
2606 edge_predicate_pool
= 0;