1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011
3 Free Software Foundation, Inc.
4 Contributed by Jan Hubicka
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Analysis used by the inliner and other passes limiting code size growth.
24 We estimate for each function
26 - average function execution time
27 - inlining size benefit (that is how much of function body size
28 and its call sequence is expected to disappear by inlining)
29 - inlining time benefit
32 - call statement size and time
34 inlinie_summary datastructures store above information locally (i.e.
35 parameters of the function itself) and globally (i.e. parameters of
36 the function created by applying all the inline decisions already
37 present in the callgraph).
39 We provide accestor to the inline_summary datastructure and
40 basic logic updating the parameters when inlining is performed.
42 The summaries are context sensitive. Context means
43 1) partial assignment of known constant values of operands
44 2) whether function is inlined into the call or not.
45 It is easy to add more variants. To represent function size and time
46 that depends on context (i.e. it is known to be optimized away when
47 context is known either by inlining or from IP-CP and clonning),
48 we use predicates. Predicates are logical formulas in
49 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
50 specifying what conditions must be true. Conditions are simple test
51 of the form described above.
53 In order to make predicate (possibly) true, all of its clauses must
54 be (possibly) true. To make clause (possibly) true, one of conditions
55 it mentions must be (possibly) true. There are fixed bounds on
56 number of clauses and conditions and all the manipulation functions
57 are conservative in positive direction. I.e. we may lose precision
58 by thinking that predicate may be true even when it is not.
60 estimate_edge_size and estimate_edge_growth can be used to query
61 function size/time in the given context. inline_merge_summary merges
62 properties of caller and callee after inlining.
64 Finally pass_inline_parameters is exported. This is used to drive
65 computation of function parameters used by the early inliner. IPA
66 inlined performs analysis via its analyze_function method. */
70 #include "coretypes.h"
73 #include "tree-inline.h"
74 #include "langhooks.h"
77 #include "diagnostic.h"
78 #include "gimple-pretty-print.h"
81 #include "tree-pass.h"
84 #include "tree-flow.h"
86 #include "lto-streamer.h"
87 #include "data-streamer.h"
88 #include "tree-streamer.h"
89 #include "ipa-inline.h"
90 #include "alloc-pool.h"
92 /* Estimate runtime of function can easilly run into huge numbers with many
93 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE in integer.
94 For anything larger we use gcov_type. */
95 #define MAX_TIME 1000000
97 /* Number of bits in integer, but we really want to be stable across different
99 #define NUM_CONDITIONS 32
101 enum predicate_conditions
103 predicate_false_condition
= 0,
104 predicate_not_inlined_condition
= 1,
105 predicate_first_dynamic_condition
= 2
108 /* Special condition code we use to represent test that operand is compile time
110 #define IS_NOT_CONSTANT ERROR_MARK
112 /* Holders of ipa cgraph hooks: */
113 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
114 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
115 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
116 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
117 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
118 static void inline_node_removal_hook (struct cgraph_node
*, void *);
119 static void inline_node_duplication_hook (struct cgraph_node
*,
120 struct cgraph_node
*, void *);
121 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
122 static void inline_edge_duplication_hook (struct cgraph_edge
*,
123 struct cgraph_edge
*,
126 /* VECtor holding inline summaries.
127 In GGC memory because conditions might point to constant trees. */
128 VEC(inline_summary_t
,gc
) *inline_summary_vec
;
129 VEC(inline_edge_summary_t
,heap
) *inline_edge_summary_vec
;
131 /* Cached node/edge growths. */
132 VEC(int,heap
) *node_growth_cache
;
133 VEC(edge_growth_cache_entry
,heap
) *edge_growth_cache
;
135 /* Edge predicates goes here. */
136 static alloc_pool edge_predicate_pool
;
138 /* Return true predicate (tautology).
139 We represent it by empty list of clauses. */
141 static inline struct predicate
142 true_predicate (void)
150 /* Return predicate testing single condition number COND. */
152 static inline struct predicate
153 single_cond_predicate (int cond
)
156 p
.clause
[0]=1 << cond
;
162 /* Return false predicate. First clause require false condition. */
164 static inline struct predicate
165 false_predicate (void)
167 return single_cond_predicate (predicate_false_condition
);
171 /* Return true if P is (false). */
174 true_predicate_p (struct predicate
*p
)
176 return !p
->clause
[0];
180 /* Return true if P is (false). */
183 false_predicate_p (struct predicate
*p
)
185 if (p
->clause
[0] == (1 << predicate_false_condition
))
187 gcc_checking_assert (!p
->clause
[1]
188 && p
->clause
[0] == 1 << predicate_false_condition
);
195 /* Return predicate that is set true when function is not inlined. */
196 static inline struct predicate
197 not_inlined_predicate (void)
199 return single_cond_predicate (predicate_not_inlined_condition
);
203 /* Add condition to condition list CONDS. */
205 static struct predicate
206 add_condition (struct inline_summary
*summary
, int operand_num
,
207 enum tree_code code
, tree val
)
211 struct condition new_cond
;
213 for (i
= 0; VEC_iterate (condition
, summary
->conds
, i
, c
); i
++)
215 if (c
->operand_num
== operand_num
218 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
220 /* Too many conditions. Give up and return constant true. */
221 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
222 return true_predicate ();
224 new_cond
.operand_num
= operand_num
;
225 new_cond
.code
= code
;
227 VEC_safe_push (condition
, gc
, summary
->conds
, &new_cond
);
228 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
232 /* Add clause CLAUSE into the predicate P. */
235 add_clause (struct predicate
*p
, clause_t clause
)
239 int insert_here
= -1;
245 /* False clause makes the whole predicate false. Kill the other variants. */
246 if (clause
== (1 << predicate_false_condition
))
248 p
->clause
[0] = (1 << predicate_false_condition
);
252 if (false_predicate_p (p
))
255 /* No one should be sily enough to add false into nontrivial clauses. */
256 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
258 /* Look where to insert the clause. At the same time prune out
259 clauses of P that are implied by the new clause and thus
261 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
263 p
->clause
[i2
] = p
->clause
[i
];
268 /* If p->clause[i] implies clause, there is nothing to add. */
269 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
271 /* We had nothing to add, none of clauses should've become redundant. */
272 gcc_checking_assert (i
== i2
);
276 if (p
->clause
[i
] < clause
&& insert_here
< 0)
279 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
280 Otherwise the p->clause[i] has to stay. */
281 if ((p
->clause
[i
] & clause
) != clause
)
284 /* We run out of variants. Be conservative in positive direction. */
285 if (i2
== MAX_CLAUSES
)
287 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
288 p
->clause
[i2
+ 1] = 0;
289 if (insert_here
>= 0)
290 for (;i2
> insert_here
; i2
--)
291 p
->clause
[i2
] = p
->clause
[i2
- 1];
294 p
->clause
[insert_here
] = clause
;
300 static struct predicate
301 and_predicates (struct predicate
*p
, struct predicate
*p2
)
303 struct predicate out
= *p
;
306 /* Avoid busy work. */
307 if (false_predicate_p (p2
) || true_predicate_p (p
))
309 if (false_predicate_p (p
) || true_predicate_p (p2
))
312 /* See how far predicates match. */
313 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
315 gcc_checking_assert (i
< MAX_CLAUSES
);
318 /* Combine the predicates rest. */
319 for (; p2
->clause
[i
]; i
++)
321 gcc_checking_assert (i
< MAX_CLAUSES
);
322 add_clause (&out
, p2
->clause
[i
]);
328 /* Return true if predicates are obviously equal. */
331 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
334 for (i
= 0; p
->clause
[i
]; i
++)
336 gcc_checking_assert (i
< MAX_CLAUSES
);
337 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
338 gcc_checking_assert (!p2
->clause
[i
] || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
339 if (p
->clause
[i
] != p2
->clause
[i
])
342 return !p2
->clause
[i
];
348 static struct predicate
349 or_predicates (struct predicate
*p
, struct predicate
*p2
)
351 struct predicate out
= true_predicate ();
354 /* Avoid busy work. */
355 if (false_predicate_p (p2
) || true_predicate_p (p
))
357 if (false_predicate_p (p
) || true_predicate_p (p2
))
359 if (predicates_equal_p (p
, p2
))
362 /* OK, combine the predicates. */
363 for (i
= 0; p
->clause
[i
]; i
++)
364 for (j
= 0; p2
->clause
[j
]; j
++)
366 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
367 add_clause (&out
, p
->clause
[i
] | p2
->clause
[j
]);
373 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false if predicate P
377 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
381 /* True remains true. */
382 if (true_predicate_p (p
))
385 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
387 /* See if we can find clause we can disprove. */
388 for (i
= 0; p
->clause
[i
]; i
++)
390 gcc_checking_assert (i
< MAX_CLAUSES
);
391 if (!(p
->clause
[i
] & possible_truths
))
398 /* Dump conditional COND. */
401 dump_condition (FILE *f
, conditions conditions
, int cond
)
404 if (cond
== predicate_false_condition
)
405 fprintf (f
, "false");
406 else if (cond
== predicate_not_inlined_condition
)
407 fprintf (f
, "not inlined");
410 c
= VEC_index (condition
, conditions
, cond
- predicate_first_dynamic_condition
);
411 fprintf (f
, "op%i", c
->operand_num
);
412 if (c
->code
== IS_NOT_CONSTANT
)
414 fprintf (f
, " not constant");
417 fprintf (f
, " %s ", op_symbol_code (c
->code
));
418 print_generic_expr (f
, c
->val
, 1);
423 /* Dump clause CLAUSE. */
426 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
433 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
434 if (clause
& (1 << i
))
439 dump_condition (f
, conds
, i
);
445 /* Dump predicate PREDICATE. */
448 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
451 if (true_predicate_p (pred
))
452 dump_clause (f
, conds
, 0);
454 for (i
= 0; pred
->clause
[i
]; i
++)
458 dump_clause (f
, conds
, pred
->clause
[i
]);
464 /* Record SIZE and TIME under condition PRED into the inline summary. */
467 account_size_time (struct inline_summary
*summary
, int size
, int time
, struct predicate
*pred
)
473 if (false_predicate_p (pred
))
476 /* We need to create initial empty unconitional clause, but otherwie
477 we don't need to account empty times and sizes. */
478 if (!size
&& !time
&& summary
->entry
)
481 /* Watch overflow that might result from insane profiles. */
482 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
483 time
= MAX_TIME
* INLINE_TIME_SCALE
;
484 gcc_assert (time
>= 0);
486 for (i
= 0; VEC_iterate (size_time_entry
, summary
->entry
, i
, e
); i
++)
487 if (predicates_equal_p (&e
->predicate
, pred
))
496 e
= VEC_index (size_time_entry
, summary
->entry
, 0);
497 gcc_assert (!e
->predicate
.clause
[0]);
499 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
501 fprintf (dump_file
, "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
502 ((double)size
) / INLINE_SIZE_SCALE
, ((double)time
) / INLINE_TIME_SCALE
,
503 found
? "" : "new ");
504 dump_predicate (dump_file
, summary
->conds
, pred
);
508 struct size_time_entry new_entry
;
509 new_entry
.size
= size
;
510 new_entry
.time
= time
;
511 new_entry
.predicate
= *pred
;
512 VEC_safe_push (size_time_entry
, gc
, summary
->entry
, &new_entry
);
518 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
519 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
523 /* Set predicate for edge E. */
526 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
528 struct inline_edge_summary
*es
= inline_edge_summary (e
);
529 if (predicate
&& !true_predicate_p (predicate
))
532 es
->predicate
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
533 *es
->predicate
= *predicate
;
538 pool_free (edge_predicate_pool
, es
->predicate
);
539 es
->predicate
= NULL
;
544 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
545 Return clause of possible truths. When INLINE_P is true, assume that
549 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
551 VEC (tree
, heap
) *known_vals
)
553 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
554 struct inline_summary
*info
= inline_summary (node
);
558 for (i
= 0; VEC_iterate (condition
, info
->conds
, i
, c
); i
++)
563 /* We allow call stmt to have fewer arguments than the callee
564 function (especially for K&R style programs). So bound
566 if (c
->operand_num
< (int)VEC_length (tree
, known_vals
))
567 val
= VEC_index (tree
, known_vals
, c
->operand_num
);
573 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
576 if (c
->code
== IS_NOT_CONSTANT
)
578 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
580 && integer_zerop (res
))
582 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
588 /* Work out what conditions might be true at invocation of E. */
591 evaluate_conditions_for_edge (struct cgraph_edge
*e
, bool inline_p
)
593 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
594 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
595 struct inline_summary
*info
= inline_summary (callee
);
598 if (ipa_node_params_vector
&& info
->conds
599 /* FIXME: it seems that we forget to get argument count in some cases,
600 probaby for previously indirect edges or so. */
601 && ipa_get_cs_argument_count (IPA_EDGE_REF (e
)))
603 struct ipa_node_params
*parms_info
;
604 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
605 int i
, count
= ipa_get_cs_argument_count (args
);
606 VEC (tree
, heap
) *known_vals
= NULL
;
608 if (e
->caller
->global
.inlined_to
)
609 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
611 parms_info
= IPA_NODE_REF (e
->caller
);
613 VEC_safe_grow_cleared (tree
, heap
, known_vals
, count
);
614 for (i
= 0; i
< count
; i
++)
616 tree cst
= ipa_cst_from_jfunc (parms_info
,
617 ipa_get_ith_jump_func (args
, i
));
619 VEC_replace (tree
, known_vals
, i
, cst
);
621 clause
= evaluate_conditions_for_known_args (callee
,
622 inline_p
, known_vals
);
623 VEC_free (tree
, heap
, known_vals
);
626 for (i
= 0; i
< (int)VEC_length (condition
, info
->conds
); i
++)
627 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
633 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
636 inline_summary_alloc (void)
638 if (!node_removal_hook_holder
)
639 node_removal_hook_holder
=
640 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
641 if (!edge_removal_hook_holder
)
642 edge_removal_hook_holder
=
643 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
644 if (!node_duplication_hook_holder
)
645 node_duplication_hook_holder
=
646 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
647 if (!edge_duplication_hook_holder
)
648 edge_duplication_hook_holder
=
649 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
651 if (VEC_length (inline_summary_t
, inline_summary_vec
)
652 <= (unsigned) cgraph_max_uid
)
653 VEC_safe_grow_cleared (inline_summary_t
, gc
,
654 inline_summary_vec
, cgraph_max_uid
+ 1);
655 if (VEC_length (inline_edge_summary_t
, inline_edge_summary_vec
)
656 <= (unsigned) cgraph_edge_max_uid
)
657 VEC_safe_grow_cleared (inline_edge_summary_t
, heap
,
658 inline_edge_summary_vec
, cgraph_edge_max_uid
+ 1);
659 if (!edge_predicate_pool
)
660 edge_predicate_pool
= create_alloc_pool ("edge predicates", sizeof (struct predicate
),
664 /* Hook that is called by cgraph.c when a node is removed. */
667 inline_node_removal_hook (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
669 struct inline_summary
*info
;
670 if (VEC_length (inline_summary_t
, inline_summary_vec
)
671 <= (unsigned)node
->uid
)
673 info
= inline_summary (node
);
674 reset_node_growth_cache (node
);
675 VEC_free (condition
, gc
, info
->conds
);
676 VEC_free (size_time_entry
, gc
, info
->entry
);
679 memset (info
, 0, sizeof (inline_summary_t
));
683 /* Hook that is called by cgraph.c when a node is duplicated. */
686 inline_node_duplication_hook (struct cgraph_node
*src
, struct cgraph_node
*dst
,
687 ATTRIBUTE_UNUSED
void *data
)
689 struct inline_summary
*info
;
690 inline_summary_alloc ();
691 info
= inline_summary (dst
);
692 memcpy (info
, inline_summary (src
),
693 sizeof (struct inline_summary
));
694 /* TODO: as an optimization, we may avoid copying conditions
695 that are known to be false or true. */
696 info
->conds
= VEC_copy (condition
, gc
, info
->conds
);
698 /* When there are any replacements in the function body, see if we can figure
699 out that something was optimized out. */
700 if (ipa_node_params_vector
&& dst
->clone
.tree_map
)
702 VEC(size_time_entry
,gc
) *entry
= info
->entry
;
703 /* Use SRC parm info since it may not be copied yet. */
704 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
705 VEC (tree
, heap
) *known_vals
= NULL
;
706 int count
= ipa_get_param_count (parms_info
);
708 clause_t possible_truths
;
709 struct predicate true_pred
= true_predicate ();
711 int optimized_out_size
= 0;
712 gcov_type optimized_out_time
= 0;
713 bool inlined_to_p
= false;
714 struct cgraph_edge
*edge
;
717 VEC_safe_grow_cleared (tree
, heap
, known_vals
, count
);
718 for (i
= 0; i
< count
; i
++)
720 tree t
= ipa_get_param (parms_info
, i
);
721 struct ipa_replace_map
*r
;
724 VEC_iterate (ipa_replace_map_p
, dst
->clone
.tree_map
, j
, r
);
731 VEC_replace (tree
, known_vals
, i
, r
->new_tree
);
736 possible_truths
= evaluate_conditions_for_known_args (dst
,
738 VEC_free (tree
, heap
, known_vals
);
740 account_size_time (info
, 0, 0, &true_pred
);
742 /* Remap size_time vectors.
743 Simplify the predicate by prunning out alternatives that are known
745 TODO: as on optimization, we can also eliminate conditions known to be true. */
746 for (i
= 0; VEC_iterate (size_time_entry
, entry
, i
, e
); i
++)
748 struct predicate new_predicate
= true_predicate ();
749 for (j
= 0; e
->predicate
.clause
[j
]; j
++)
750 if (!(possible_truths
& e
->predicate
.clause
[j
]))
752 new_predicate
= false_predicate ();
756 add_clause (&new_predicate
,
757 possible_truths
& e
->predicate
.clause
[j
]);
758 if (false_predicate_p (&new_predicate
))
760 optimized_out_size
+= e
->size
;
761 optimized_out_time
+= e
->time
;
764 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
767 /* Remap edge predicates with the same simplificaiton as above. */
768 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
770 struct predicate new_predicate
= true_predicate ();
771 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
773 if (!edge
->inline_failed
)
777 for (j
= 0; es
->predicate
->clause
[j
]; j
++)
778 if (!(possible_truths
& es
->predicate
->clause
[j
]))
780 new_predicate
= false_predicate ();
784 add_clause (&new_predicate
,
785 possible_truths
& es
->predicate
->clause
[j
]);
786 if (false_predicate_p (&new_predicate
)
787 && !false_predicate_p (es
->predicate
))
789 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
790 optimized_out_time
+= (es
->call_stmt_time
791 * (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
)
795 *es
->predicate
= new_predicate
;
798 /* Remap indirect edge predicates with the same simplificaiton as above. */
799 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
801 struct predicate new_predicate
= true_predicate ();
802 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
804 if (!edge
->inline_failed
)
808 for (j
= 0; es
->predicate
->clause
[j
]; j
++)
809 if (!(possible_truths
& es
->predicate
->clause
[j
]))
811 new_predicate
= false_predicate ();
815 add_clause (&new_predicate
,
816 possible_truths
& es
->predicate
->clause
[j
]);
817 if (false_predicate_p (&new_predicate
)
818 && !false_predicate_p (es
->predicate
))
820 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
821 optimized_out_time
+= (es
->call_stmt_time
822 * (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
)
826 *es
->predicate
= new_predicate
;
829 /* If inliner or someone after inliner will ever start producing
830 non-trivial clones, we will get trouble with lack of information
831 about updating self sizes, because size vectors already contains
832 sizes of the calees. */
833 gcc_assert (!inlined_to_p
834 || (!optimized_out_size
&& !optimized_out_time
));
836 info
->size
-= optimized_out_size
/ INLINE_SIZE_SCALE
;
837 info
->self_size
-= optimized_out_size
/ INLINE_SIZE_SCALE
;
838 gcc_assert (info
->size
> 0);
839 gcc_assert (info
->self_size
> 0);
841 optimized_out_time
/= INLINE_TIME_SCALE
;
842 if (optimized_out_time
> MAX_TIME
)
843 optimized_out_time
= MAX_TIME
;
844 info
->time
-= optimized_out_time
;
845 info
->self_time
-= optimized_out_time
;
848 if (info
->self_time
< 0)
852 info
->entry
= VEC_copy (size_time_entry
, gc
, info
->entry
);
856 /* Hook that is called by cgraph.c when a node is duplicated. */
859 inline_edge_duplication_hook (struct cgraph_edge
*src
, struct cgraph_edge
*dst
,
860 ATTRIBUTE_UNUSED
void *data
)
862 struct inline_edge_summary
*info
;
863 struct inline_edge_summary
*srcinfo
;
864 inline_summary_alloc ();
865 info
= inline_edge_summary (dst
);
866 srcinfo
= inline_edge_summary (src
);
867 memcpy (info
, srcinfo
,
868 sizeof (struct inline_edge_summary
));
869 info
->predicate
= NULL
;
870 edge_set_predicate (dst
, srcinfo
->predicate
);
874 /* Keep edge cache consistent across edge removal. */
877 inline_edge_removal_hook (struct cgraph_edge
*edge
, void *data ATTRIBUTE_UNUSED
)
879 if (edge_growth_cache
)
880 reset_edge_growth_cache (edge
);
881 if (edge
->uid
< (int)VEC_length (inline_edge_summary_t
, inline_edge_summary_vec
))
883 edge_set_predicate (edge
, NULL
);
884 memset (inline_edge_summary (edge
), 0, sizeof (struct inline_edge_summary
));
889 /* Initialize growth caches. */
892 initialize_growth_caches (void)
894 if (cgraph_edge_max_uid
)
895 VEC_safe_grow_cleared (edge_growth_cache_entry
, heap
, edge_growth_cache
,
896 cgraph_edge_max_uid
);
898 VEC_safe_grow_cleared (int, heap
, node_growth_cache
, cgraph_max_uid
);
902 /* Free growth caches. */
905 free_growth_caches (void)
907 VEC_free (edge_growth_cache_entry
, heap
, edge_growth_cache
);
908 edge_growth_cache
= 0;
909 VEC_free (int, heap
, node_growth_cache
);
910 node_growth_cache
= 0;
914 /* Dump edge summaries associated to NODE and recursively to all clones.
918 dump_inline_edge_summary (FILE * f
, int indent
, struct cgraph_node
*node
,
919 struct inline_summary
*info
)
921 struct cgraph_edge
*edge
;
922 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
924 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
925 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
926 fprintf (f
, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i callee size:%2i stack:%2i",
927 indent
, "", cgraph_node_name (callee
),
929 !edge
->inline_failed
? "inlined"
930 : cgraph_inline_failed_string (edge
->inline_failed
),
936 (int)inline_summary (callee
)->size
,
937 (int)inline_summary (callee
)->estimated_stack_size
);
940 fprintf (f
, " predicate: ");
941 dump_predicate (f
, info
->conds
, es
->predicate
);
945 if (!edge
->inline_failed
)
947 fprintf (f
, "%*sStack frame offset %i, callee self size %i, callee size %i\n",
949 (int)inline_summary (callee
)->stack_frame_offset
,
950 (int)inline_summary (callee
)->estimated_self_stack_size
,
951 (int)inline_summary (callee
)->estimated_stack_size
);
952 dump_inline_edge_summary (f
, indent
+2, callee
, info
);
955 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
957 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
958 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i time: %2i\n",
966 fprintf (f
, "predicate: ");
967 dump_predicate (f
, info
->conds
, es
->predicate
);
976 dump_inline_summary (FILE * f
, struct cgraph_node
*node
)
980 struct inline_summary
*s
= inline_summary (node
);
983 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
985 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
986 fprintf (f
, " always_inline");
988 fprintf (f
, " inlinable");
990 fprintf (f
, " versionable");
991 fprintf (f
, "\n self time: %i\n",
993 fprintf (f
, " global time: %i\n", s
->time
);
994 fprintf (f
, " self size: %i\n",
996 fprintf (f
, " global size: %i\n", s
->size
);
997 fprintf (f
, " self stack: %i\n",
998 (int) s
->estimated_self_stack_size
);
999 fprintf (f
, " global stack: %i\n",
1000 (int) s
->estimated_stack_size
);
1002 VEC_iterate (size_time_entry
, s
->entry
, i
, e
);
1005 fprintf (f
, " size:%f, time:%f, predicate:",
1006 (double) e
->size
/ INLINE_SIZE_SCALE
,
1007 (double) e
->time
/ INLINE_TIME_SCALE
);
1008 dump_predicate (f
, s
->conds
, &e
->predicate
);
1010 fprintf (f
, " calls:\n");
1011 dump_inline_edge_summary (f
, 4, node
, s
);
1017 debug_inline_summary (struct cgraph_node
*node
)
1019 dump_inline_summary (stderr
, node
);
1023 dump_inline_summaries (FILE *f
)
1025 struct cgraph_node
*node
;
1027 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1028 if (node
->analyzed
&& !node
->global
.inlined_to
)
1029 dump_inline_summary (f
, node
);
1032 /* Give initial reasons why inlining would fail on EDGE. This gets either
1033 nullified or usually overwritten by more precise reasons later. */
1036 initialize_inline_failed (struct cgraph_edge
*e
)
1038 struct cgraph_node
*callee
= e
->callee
;
1040 if (e
->indirect_unknown_callee
)
1041 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1042 else if (!callee
->analyzed
)
1043 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1044 else if (callee
->local
.redefined_extern_inline
)
1045 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1046 else if (e
->call_stmt
&& gimple_call_cannot_inline_p (e
->call_stmt
))
1047 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1049 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1052 /* See if statement might disappear after inlining.
1053 0 - means not eliminated
1054 1 - half of statements goes away
1055 2 - for sure it is eliminated.
1056 We are not terribly sophisticated, basically looking for simple abstraction
1057 penalty wrappers. */
1060 eliminated_by_inlining_prob (gimple stmt
)
1062 enum gimple_code code
= gimple_code (stmt
);
1068 if (gimple_num_ops (stmt
) != 2)
1071 /* Casts of parameters, loads from parameters passed by reference
1072 and stores to return value or parameters are often free after
1073 inlining dua to SRA and further combining.
1074 Assume that half of statements goes away. */
1075 if (gimple_assign_rhs_code (stmt
) == CONVERT_EXPR
1076 || gimple_assign_rhs_code (stmt
) == NOP_EXPR
1077 || gimple_assign_rhs_code (stmt
) == VIEW_CONVERT_EXPR
1078 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1080 tree rhs
= gimple_assign_rhs1 (stmt
);
1081 tree lhs
= gimple_assign_lhs (stmt
);
1082 tree inner_rhs
= rhs
;
1083 tree inner_lhs
= lhs
;
1084 bool rhs_free
= false;
1085 bool lhs_free
= false;
1087 while (handled_component_p (inner_lhs
)
1088 || TREE_CODE (inner_lhs
) == MEM_REF
)
1089 inner_lhs
= TREE_OPERAND (inner_lhs
, 0);
1090 while (handled_component_p (inner_rhs
)
1091 || TREE_CODE (inner_rhs
) == ADDR_EXPR
1092 || TREE_CODE (inner_rhs
) == MEM_REF
)
1093 inner_rhs
= TREE_OPERAND (inner_rhs
, 0);
1096 if (TREE_CODE (inner_rhs
) == PARM_DECL
1097 || (TREE_CODE (inner_rhs
) == SSA_NAME
1098 && SSA_NAME_IS_DEFAULT_DEF (inner_rhs
)
1099 && TREE_CODE (SSA_NAME_VAR (inner_rhs
)) == PARM_DECL
))
1101 if (rhs_free
&& is_gimple_reg (lhs
))
1103 if (((TREE_CODE (inner_lhs
) == PARM_DECL
1104 || (TREE_CODE (inner_lhs
) == SSA_NAME
1105 && SSA_NAME_IS_DEFAULT_DEF (inner_lhs
)
1106 && TREE_CODE (SSA_NAME_VAR (inner_lhs
)) == PARM_DECL
))
1107 && inner_lhs
!= lhs
)
1108 || TREE_CODE (inner_lhs
) == RESULT_DECL
1109 || (TREE_CODE (inner_lhs
) == SSA_NAME
1110 && TREE_CODE (SSA_NAME_VAR (inner_lhs
)) == RESULT_DECL
))
1113 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1115 if (lhs_free
&& rhs_free
)
1125 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1126 predicates to the CFG edges. */
1129 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1130 struct inline_summary
*summary
,
1136 enum tree_code code
, inverted_code
;
1142 last
= last_stmt (bb
);
1144 || gimple_code (last
) != GIMPLE_COND
)
1146 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1148 op
= gimple_cond_lhs (last
);
1149 /* TODO: handle conditionals like
1152 if (TREE_CODE (op
) != SSA_NAME
)
1154 if (SSA_NAME_IS_DEFAULT_DEF (op
))
1156 index
= ipa_get_param_decl_index (info
, SSA_NAME_VAR (op
));
1159 code
= gimple_cond_code (last
);
1160 inverted_code
= invert_tree_comparison (code
,
1161 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1163 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1165 struct predicate p
= add_condition (summary
,
1167 e
->flags
& EDGE_TRUE_VALUE
1168 ? code
: inverted_code
,
1169 gimple_cond_rhs (last
));
1170 e
->aux
= pool_alloc (edge_predicate_pool
);
1171 *(struct predicate
*)e
->aux
= p
;
1176 if (builtin_constant_p (op))
1180 Here we can predicate nonconstant_code. We can't
1181 really handle constant_code since we have no predicate
1182 for this and also the constant code is not known to be
1183 optimized away when inliner doen't see operand is constant.
1184 Other optimizers might think otherwise. */
1185 set_stmt
= SSA_NAME_DEF_STMT (op
);
1186 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1187 || gimple_call_num_args (set_stmt
) != 1)
1189 op2
= gimple_call_arg (set_stmt
, 0);
1190 if (!SSA_NAME_IS_DEFAULT_DEF (op2
))
1192 index
= ipa_get_param_decl_index (info
, SSA_NAME_VAR (op2
));
1195 if (gimple_cond_code (last
) != NE_EXPR
1196 || !integer_zerop (gimple_cond_rhs (last
)))
1198 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1199 if (e
->flags
& EDGE_FALSE_VALUE
)
1201 struct predicate p
= add_condition (summary
,
1205 e
->aux
= pool_alloc (edge_predicate_pool
);
1206 *(struct predicate
*)e
->aux
= p
;
1211 /* If BB ends by a switch we can turn into predicates, attach corresponding
1212 predicates to the CFG edges. */
1215 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1216 struct inline_summary
*summary
,
1227 last
= last_stmt (bb
);
1229 || gimple_code (last
) != GIMPLE_SWITCH
)
1231 op
= gimple_switch_index (last
);
1232 if (TREE_CODE (op
) != SSA_NAME
1233 || !SSA_NAME_IS_DEFAULT_DEF (op
))
1235 index
= ipa_get_param_decl_index (info
, SSA_NAME_VAR (op
));
1239 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1241 e
->aux
= pool_alloc (edge_predicate_pool
);
1242 *(struct predicate
*)e
->aux
= false_predicate ();
1244 n
= gimple_switch_num_labels(last
);
1245 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1247 tree cl
= gimple_switch_label (last
, case_idx
);
1251 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1252 min
= CASE_LOW (cl
);
1253 max
= CASE_HIGH (cl
);
1255 /* For default we might want to construct predicate that none
1256 of cases is met, but it is bit hard to do not having negations
1257 of conditionals handy. */
1259 p
= true_predicate ();
1261 p
= add_condition (summary
, index
,
1266 struct predicate p1
, p2
;
1267 p1
= add_condition (summary
, index
,
1270 p2
= add_condition (summary
, index
,
1273 p
= and_predicates (&p1
, &p2
);
1275 *(struct predicate
*)e
->aux
1276 = or_predicates (&p
, (struct predicate
*)e
->aux
);
1281 /* For each BB in NODE attach to its AUX pointer predicate under
1282 which it is executable. */
1285 compute_bb_predicates (struct cgraph_node
*node
,
1286 struct ipa_node_params
*parms_info
,
1287 struct inline_summary
*summary
)
1289 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1293 FOR_EACH_BB_FN (bb
, my_function
)
1295 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1296 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1299 /* Entry block is always executable. */
1300 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
= pool_alloc (edge_predicate_pool
);
1301 *(struct predicate
*)ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1302 = true_predicate ();
1304 /* A simple dataflow propagation of predicates forward in the CFG.
1305 TODO: work in reverse postorder. */
1309 FOR_EACH_BB_FN (bb
, my_function
)
1311 struct predicate p
= false_predicate ();
1314 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1318 struct predicate this_bb_predicate
= *(struct predicate
*)e
->src
->aux
;
1320 this_bb_predicate
= and_predicates (&this_bb_predicate
,
1321 (struct predicate
*)e
->aux
);
1322 p
= or_predicates (&p
, &this_bb_predicate
);
1323 if (true_predicate_p (&p
))
1327 if (false_predicate_p (&p
))
1328 gcc_assert (!bb
->aux
);
1334 bb
->aux
= pool_alloc (edge_predicate_pool
);
1335 *((struct predicate
*)bb
->aux
) = p
;
1337 else if (!predicates_equal_p (&p
, (struct predicate
*)bb
->aux
))
1340 *((struct predicate
*)bb
->aux
) = p
;
1348 /* We keep info about constantness of SSA names. */
1350 typedef struct predicate predicate_t
;
1351 DEF_VEC_O (predicate_t
);
1352 DEF_VEC_ALLOC_O (predicate_t
, heap
);
1355 /* Return predicate specifying when the STMT might have result that is not a compile
1358 static struct predicate
1359 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1360 struct inline_summary
*summary
,
1362 VEC (predicate_t
, heap
) *nonconstant_names
)
1365 struct predicate p
= true_predicate ();
1368 struct predicate op_non_const
;
1370 /* What statments might be optimized away
1371 when their arguments are constant
1372 TODO: also trivial builtins.
1373 builtin_constant_p is already handled later. */
1374 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1375 && gimple_code (stmt
) != GIMPLE_COND
1376 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1379 /* Stores and loads will stay anyway.
1380 TODO: Constant memory accesses could be handled here, too. */
1381 if (gimple_vuse (stmt
))
1384 /* See if we understand all operands before we start
1385 adding conditionals. */
1386 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1388 if (TREE_CODE (use
) != SSA_NAME
)
1390 /* For arguments we can build a condition. */
1391 if (SSA_NAME_IS_DEFAULT_DEF (use
)
1392 && ipa_get_param_decl_index (info
, SSA_NAME_VAR (use
)) >= 0)
1394 /* If we know when operand is constant,
1395 we still can say something useful. */
1396 if (!true_predicate_p (VEC_index (predicate_t
, nonconstant_names
,
1397 SSA_NAME_VERSION (use
))))
1401 op_non_const
= false_predicate ();
1402 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1404 if (SSA_NAME_IS_DEFAULT_DEF (use
)
1405 && ipa_get_param_decl_index (info
, SSA_NAME_VAR (use
)) >= 0)
1406 p
= add_condition (summary
,
1407 ipa_get_param_decl_index (info
, SSA_NAME_VAR (use
)),
1408 IS_NOT_CONSTANT
, NULL
);
1410 p
= *VEC_index (predicate_t
, nonconstant_names
,
1411 SSA_NAME_VERSION (use
));
1412 op_non_const
= or_predicates (&p
, &op_non_const
);
1414 if (gimple_code (stmt
) == GIMPLE_ASSIGN
1415 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
1416 VEC_replace (predicate_t
, nonconstant_names
,
1417 SSA_NAME_VERSION (gimple_assign_lhs (stmt
)), &op_non_const
);
1418 return op_non_const
;
1422 /* Compute function body size parameters for NODE.
1423 When EARLY is true, we compute only simple summaries without
1424 non-trivial predicates to drive the early inliner. */
1427 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
1430 /* Estimate static overhead for function prologue/epilogue and alignment. */
1432 /* Benefits are scaled by probability of elimination that is in range
1435 gimple_stmt_iterator bsi
;
1436 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1438 struct inline_summary
*info
= inline_summary (node
);
1439 struct predicate bb_predicate
;
1440 struct ipa_node_params
*parms_info
= NULL
;
1441 VEC (predicate_t
, heap
) *nonconstant_names
= NULL
;
1443 if (ipa_node_params_vector
&& !early
&& optimize
)
1445 parms_info
= IPA_NODE_REF (node
);
1446 VEC_safe_grow_cleared (predicate_t
, heap
, nonconstant_names
,
1447 VEC_length (tree
, SSANAMES (my_function
)));
1455 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
1456 cgraph_node_name (node
));
1458 /* When we run into maximal number of entries, we assign everything to the
1459 constant truth case. Be sure to have it in list. */
1460 bb_predicate
= true_predicate ();
1461 account_size_time (info
, 0, 0, &bb_predicate
);
1463 bb_predicate
= not_inlined_predicate ();
1464 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
1466 gcc_assert (my_function
&& my_function
->cfg
);
1468 compute_bb_predicates (node
, parms_info
, info
);
1469 FOR_EACH_BB_FN (bb
, my_function
)
1471 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
1473 /* TODO: Obviously predicates can be propagated down across CFG. */
1477 bb_predicate
= *(struct predicate
*)bb
->aux
;
1479 bb_predicate
= false_predicate ();
1482 bb_predicate
= true_predicate ();
1484 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1486 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
1487 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
1490 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1492 gimple stmt
= gsi_stmt (bsi
);
1493 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
1494 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
1496 struct predicate will_be_nonconstant
;
1498 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1500 fprintf (dump_file
, " ");
1501 print_gimple_stmt (dump_file
, stmt
, 0, 0);
1502 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
1503 ((double)freq
)/CGRAPH_FREQ_BASE
, this_size
, this_time
);
1506 if (is_gimple_call (stmt
))
1508 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
1509 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1511 /* Special case: results of BUILT_IN_CONSTANT_P will be always
1512 resolved as constant. We however don't want to optimize
1513 out the cgraph edges. */
1514 if (nonconstant_names
1515 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
1516 && gimple_call_lhs (stmt
)
1517 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
1519 struct predicate false_p
= false_predicate ();
1520 VEC_replace (predicate_t
, nonconstant_names
,
1521 SSA_NAME_VERSION (gimple_call_lhs (stmt
)), &false_p
);
1524 es
->call_stmt_size
= this_size
;
1525 es
->call_stmt_time
= this_time
;
1526 es
->loop_depth
= bb
->loop_depth
;
1527 edge_set_predicate (edge
, &bb_predicate
);
1529 /* Do not inline calls where we cannot triviall work around
1530 mismatches in argument or return types. */
1532 && cgraph_function_or_thunk_node (edge
->callee
, NULL
)
1533 && !gimple_check_call_matching_types (stmt
,
1534 cgraph_function_or_thunk_node (edge
->callee
,
1537 edge
->call_stmt_cannot_inline_p
= true;
1538 gimple_call_set_cannot_inline (stmt
, true);
1541 gcc_assert (!gimple_call_cannot_inline_p (stmt
));
1544 /* TODO: When conditional jump or swithc is known to be constant, but
1545 we did not translate it into the predicates, we really can account
1546 just maximum of the possible paths. */
1549 = will_be_nonconstant_predicate (parms_info
, info
,
1550 stmt
, nonconstant_names
);
1551 if (this_time
|| this_size
)
1559 prob
= eliminated_by_inlining_prob (stmt
);
1560 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
1561 fprintf (dump_file
, "\t\t50%% will be eliminated by inlining\n");
1562 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
1563 fprintf (dump_file
, "\t\twill eliminated by inlining\n");
1566 p
= and_predicates (&bb_predicate
, &will_be_nonconstant
);
1568 p
= true_predicate ();
1570 /* We account everything but the calls. Calls have their own
1571 size/time info attached to cgraph edges. This is neccesary
1572 in order to make the cost disappear after inlining. */
1573 if (!is_gimple_call (stmt
))
1577 struct predicate ip
= not_inlined_predicate ();
1578 ip
= and_predicates (&ip
, &p
);
1579 account_size_time (info
, this_size
* prob
,
1580 this_time
* prob
, &ip
);
1583 account_size_time (info
, this_size
* (2 - prob
),
1584 this_time
* (2 - prob
), &p
);
1587 gcc_assert (time
>= 0);
1588 gcc_assert (size
>= 0);
1592 FOR_ALL_BB_FN (bb
, my_function
)
1598 pool_free (edge_predicate_pool
, bb
->aux
);
1600 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1603 pool_free (edge_predicate_pool
, e
->aux
);
1607 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
1608 if (time
> MAX_TIME
)
1610 inline_summary (node
)->self_time
= time
;
1611 inline_summary (node
)->self_size
= size
;
1612 VEC_free (predicate_t
, heap
, nonconstant_names
);
1615 fprintf (dump_file
, "\n");
1616 dump_inline_summary (dump_file
, node
);
1621 /* Compute parameters of functions used by inliner.
1622 EARLY is true when we compute parameters for the early inliner */
1625 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
1627 HOST_WIDE_INT self_stack_size
;
1628 struct cgraph_edge
*e
;
1629 struct inline_summary
*info
;
1631 gcc_assert (!node
->global
.inlined_to
);
1633 inline_summary_alloc ();
1635 info
= inline_summary (node
);
1637 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
1638 Once this happen, we will need to more curefully predict call
1640 if (node
->thunk
.thunk_p
)
1642 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
1643 struct predicate t
= true_predicate ();
1645 info
->inlinable
= info
->versionable
= 0;
1646 node
->callees
->call_stmt_cannot_inline_p
= true;
1647 node
->local
.can_change_signature
= false;
1648 es
->call_stmt_time
= 1;
1649 es
->call_stmt_size
= 1;
1650 account_size_time (info
, 0, 0, &t
);
1654 /* Estimate the stack size for the function if we're optimizing. */
1655 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
1656 info
->estimated_self_stack_size
= self_stack_size
;
1657 info
->estimated_stack_size
= self_stack_size
;
1658 info
->stack_frame_offset
= 0;
1660 /* Can this function be inlined at all? */
1661 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
1663 /* Inlinable functions always can change signature. */
1664 if (info
->inlinable
)
1665 node
->local
.can_change_signature
= true;
1668 /* Functions calling builtin_apply can not change signature. */
1669 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1670 if (DECL_BUILT_IN (e
->callee
->decl
)
1671 && DECL_BUILT_IN_CLASS (e
->callee
->decl
) == BUILT_IN_NORMAL
1672 && DECL_FUNCTION_CODE (e
->callee
->decl
) == BUILT_IN_APPLY_ARGS
)
1674 node
->local
.can_change_signature
= !e
;
1676 estimate_function_body_sizes (node
, early
);
1678 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
1679 info
->time
= info
->self_time
;
1680 info
->size
= info
->self_size
;
1681 info
->stack_frame_offset
= 0;
1682 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
1686 /* Compute parameters of functions used by inliner using
1687 current_function_decl. */
1690 compute_inline_parameters_for_current (void)
1692 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
1696 struct gimple_opt_pass pass_inline_parameters
=
1700 "inline_param", /* name */
1702 compute_inline_parameters_for_current
,/* execute */
1705 0, /* static_pass_number */
1706 TV_INLINE_HEURISTICS
, /* tv_id */
1707 0, /* properties_required */
1708 0, /* properties_provided */
1709 0, /* properties_destroyed */
1710 0, /* todo_flags_start */
1711 0 /* todo_flags_finish */
1716 /* Increase SIZE and TIME for size and time needed to handle edge E. */
1719 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
)
1721 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1722 *size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1723 *time
+= (es
->call_stmt_time
1724 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
));
1725 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
1726 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
1730 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE. */
1733 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
1734 clause_t possible_truths
)
1736 struct cgraph_edge
*e
;
1737 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1739 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1740 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
1742 if (e
->inline_failed
)
1743 estimate_edge_size_and_time (e
, size
, time
);
1745 estimate_calls_size_and_time (e
->callee
, size
, time
,
1749 /* TODO: look for devirtualizing oppurtunities. */
1750 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1752 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1753 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
1754 estimate_edge_size_and_time (e
, size
, time
);
1759 /* Estimate size and time needed to execute NODE assuming
1760 POSSIBLE_TRUTHS clause. */
1763 estimate_node_size_and_time (struct cgraph_node
*node
,
1764 clause_t possible_truths
,
1765 int *ret_size
, int *ret_time
)
1767 struct inline_summary
*info
= inline_summary (node
);
1769 int size
= 0, time
= 0;
1773 && (dump_flags
& TDF_DETAILS
))
1776 fprintf (dump_file
, " Estimating body: %s/%i\n"
1777 " Known to be false: ",
1778 cgraph_node_name (node
),
1781 for (i
= predicate_not_inlined_condition
;
1782 i
< (predicate_first_dynamic_condition
1783 + (int)VEC_length (condition
, info
->conds
)); i
++)
1784 if (!(possible_truths
& (1 << i
)))
1787 fprintf (dump_file
, ", ");
1789 dump_condition (dump_file
, info
->conds
, i
);
1793 for (i
= 0; VEC_iterate (size_time_entry
, info
->entry
, i
, e
); i
++)
1794 if (evaluate_predicate (&e
->predicate
, possible_truths
))
1795 time
+= e
->time
, size
+= e
->size
;
1797 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
1798 time
= MAX_TIME
* INLINE_TIME_SCALE
;
1800 estimate_calls_size_and_time (node
, &size
, &time
, possible_truths
);
1801 time
= (time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
1802 size
= (size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
1806 && (dump_flags
& TDF_DETAILS
))
1807 fprintf (dump_file
, "\n size:%i time:%i\n", size
, time
);
1816 /* Estimate size and time needed to execute callee of EDGE assuming that
1817 parameters known to be constant at caller of EDGE are propagated.
1818 KNOWN_VALs is a vector of assumed known constant values for parameters. */
1821 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
1822 VEC (tree
, heap
) *known_vals
,
1823 int *ret_size
, int *ret_time
)
1827 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
);
1828 estimate_node_size_and_time (node
, clause
, ret_size
, ret_time
);
1832 /* Translate all conditions from callee representation into caller representation and
1833 symbolically evaluate predicate P into new predicate.
1835 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO is summary
1836 of function predicate P is from. OPERAND_MAP is array giving callee formal IDs the
1837 caller formal IDs. POSSSIBLE_TRUTHS is clausule of all callee conditions that
1838 may be true in caller context. TOPLEV_PREDICATE is predicate under which callee
1841 static struct predicate
1842 remap_predicate (struct inline_summary
*info
, struct inline_summary
*callee_info
,
1843 struct predicate
*p
,
1844 VEC (int, heap
) *operand_map
,
1845 clause_t possible_truths
,
1846 struct predicate
*toplev_predicate
)
1849 struct predicate out
= true_predicate ();
1851 /* True predicate is easy. */
1852 if (true_predicate_p (p
))
1853 return *toplev_predicate
;
1854 for (i
= 0; p
->clause
[i
]; i
++)
1856 clause_t clause
= p
->clause
[i
];
1858 struct predicate clause_predicate
= false_predicate ();
1860 gcc_assert (i
< MAX_CLAUSES
);
1862 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
1863 /* Do we have condition we can't disprove? */
1864 if (clause
& possible_truths
& (1 << cond
))
1866 struct predicate cond_predicate
;
1867 /* Work out if the condition can translate to predicate in the
1868 inlined function. */
1869 if (cond
>= predicate_first_dynamic_condition
)
1871 struct condition
*c
;
1873 c
= VEC_index (condition
, callee_info
->conds
,
1874 cond
- predicate_first_dynamic_condition
);
1875 /* See if we can remap condition operand to caller's operand.
1876 Otherwise give up. */
1878 || (int)VEC_length (int, operand_map
) <= c
->operand_num
1879 || VEC_index (int, operand_map
, c
->operand_num
) == -1)
1880 cond_predicate
= true_predicate ();
1882 cond_predicate
= add_condition (info
,
1883 VEC_index (int, operand_map
,
1887 /* Fixed conditions remains same, construct single
1888 condition predicate. */
1891 cond_predicate
.clause
[0] = 1 << cond
;
1892 cond_predicate
.clause
[1] = 0;
1894 clause_predicate
= or_predicates (&clause_predicate
, &cond_predicate
);
1896 out
= and_predicates (&out
, &clause_predicate
);
1898 return and_predicates (&out
, toplev_predicate
);
1902 /* Update summary information of inline clones after inlining.
1903 Compute peak stack usage. */
1906 inline_update_callee_summaries (struct cgraph_node
*node
,
1909 struct cgraph_edge
*e
;
1910 struct inline_summary
*callee_info
= inline_summary (node
);
1911 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
1914 callee_info
->stack_frame_offset
1915 = caller_info
->stack_frame_offset
1916 + caller_info
->estimated_self_stack_size
;
1917 peak
= callee_info
->stack_frame_offset
1918 + callee_info
->estimated_self_stack_size
;
1919 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
1921 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
1922 cgraph_propagate_frequency (node
);
1923 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1925 if (!e
->inline_failed
)
1926 inline_update_callee_summaries (e
->callee
, depth
);
1927 inline_edge_summary (e
)->loop_depth
+= depth
;
1929 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1930 inline_edge_summary (e
)->loop_depth
+= depth
;
1934 /* Remap predicates of callees of NODE. Rest of arguments match
1938 remap_edge_predicates (struct cgraph_node
*node
,
1939 struct inline_summary
*info
,
1940 struct inline_summary
*callee_info
,
1941 VEC (int, heap
) *operand_map
,
1942 clause_t possible_truths
,
1943 struct predicate
*toplev_predicate
)
1945 struct cgraph_edge
*e
;
1946 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1948 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1952 p
= remap_predicate (info
, callee_info
,
1953 es
->predicate
, operand_map
, possible_truths
,
1955 edge_set_predicate (e
, &p
);
1956 /* TODO: We should remove the edge for code that will be optimized out,
1957 but we need to keep verifiers and tree-inline happy.
1958 Make it cold for now. */
1959 if (false_predicate_p (&p
))
1965 if (!e
->inline_failed
)
1966 remap_edge_predicates (e
->callee
, info
, callee_info
, operand_map
,
1967 possible_truths
, toplev_predicate
);
1969 edge_set_predicate (e
, toplev_predicate
);
1971 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
1973 struct inline_edge_summary
*es
= inline_edge_summary (e
);
1977 p
= remap_predicate (info
, callee_info
,
1978 es
->predicate
, operand_map
, possible_truths
,
1980 edge_set_predicate (e
, &p
);
1981 /* TODO: We should remove the edge for code that will be optimized out,
1982 but we need to keep verifiers and tree-inline happy.
1983 Make it cold for now. */
1984 if (false_predicate_p (&p
))
1991 edge_set_predicate (e
, toplev_predicate
);
1996 /* We inlined EDGE. Update summary of the function we inlined into. */
1999 inline_merge_summary (struct cgraph_edge
*edge
)
2001 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
2002 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
2003 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
2004 struct inline_summary
*info
= inline_summary (to
);
2005 clause_t clause
= 0; /* not_inline is known to be false. */
2007 VEC (int, heap
) *operand_map
= NULL
;
2009 struct predicate toplev_predicate
;
2010 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2013 toplev_predicate
= *es
->predicate
;
2015 toplev_predicate
= true_predicate ();
2017 if (ipa_node_params_vector
&& callee_info
->conds
2018 /* FIXME: it seems that we forget to get argument count in some cases,
2019 probaby for previously indirect edges or so.
2020 Removing the test leads to ICE on tramp3d. */
2021 && ipa_get_cs_argument_count (IPA_EDGE_REF (edge
)))
2023 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
2024 int count
= ipa_get_cs_argument_count (args
);
2027 clause
= evaluate_conditions_for_edge (edge
, true);
2028 VEC_safe_grow_cleared (int, heap
, operand_map
, count
);
2029 for (i
= 0; i
< count
; i
++)
2031 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
2033 /* TODO: handle non-NOPs when merging. */
2034 if (jfunc
->type
== IPA_JF_PASS_THROUGH
2035 && jfunc
->value
.pass_through
.operation
== NOP_EXPR
)
2036 map
= jfunc
->value
.pass_through
.formal_id
;
2037 VEC_replace (int, operand_map
, i
, map
);
2038 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
2041 for (i
= 0; VEC_iterate (size_time_entry
, callee_info
->entry
, i
, e
); i
++)
2043 struct predicate p
= remap_predicate (info
, callee_info
,
2044 &e
->predicate
, operand_map
, clause
,
2046 gcov_type add_time
= ((gcov_type
)e
->time
* edge
->frequency
2047 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2048 if (add_time
> MAX_TIME
)
2049 add_time
= MAX_TIME
;
2050 account_size_time (info
, e
->size
, add_time
, &p
);
2052 remap_edge_predicates (edge
->callee
, info
, callee_info
, operand_map
,
2053 clause
, &toplev_predicate
);
2056 for (i
= 0; VEC_iterate (size_time_entry
, info
->entry
, i
, e
); i
++)
2057 info
->size
+= e
->size
, info
->time
+= e
->time
;
2058 estimate_calls_size_and_time (to
, &info
->size
, &info
->time
,
2059 ~(clause_t
)(1 << predicate_false_condition
));
2061 inline_update_callee_summaries (edge
->callee
,
2062 inline_edge_summary (edge
)->loop_depth
);
2064 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
2065 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
2069 /* Estimate the time cost for the caller when inlining EDGE.
2070 Only to be called via estimate_edge_time, that handles the
2073 When caching, also update the cache entry. Compute both time and
2074 size, since we always need both metrics eventually. */
2077 do_estimate_edge_time (struct cgraph_edge
*edge
)
2082 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2084 gcc_checking_assert (edge
->inline_failed
);
2085 estimate_node_size_and_time (cgraph_function_or_thunk_node (edge
->callee
, NULL
),
2086 evaluate_conditions_for_edge (edge
, true),
2089 ret
= (((gcov_type
)time
- es
->call_stmt_time
) * edge
->frequency
2090 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2094 /* When caching, update the cache entry. */
2095 if (edge_growth_cache
)
2098 if ((int)VEC_length (edge_growth_cache_entry
, edge_growth_cache
)
2100 VEC_safe_grow_cleared (edge_growth_cache_entry
, heap
, edge_growth_cache
,
2101 cgraph_edge_max_uid
);
2102 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
)->time
2105 ret_size
= size
- es
->call_stmt_size
;
2106 gcc_checking_assert (es
->call_stmt_size
);
2107 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
)->size
2108 = ret_size
+ (ret_size
>= 0);
2114 /* Estimate the growth of the caller when inlining EDGE.
2115 Only to be called via estimate_edge_size. */
2118 do_estimate_edge_growth (struct cgraph_edge
*edge
)
2121 struct cgraph_node
*callee
;
2123 /* When we do caching, use do_estimate_edge_time to populate the entry. */
2125 if (edge_growth_cache
)
2127 do_estimate_edge_time (edge
);
2128 size
= VEC_index (edge_growth_cache_entry
,
2131 gcc_checking_assert (size
);
2132 return size
- (size
> 0);
2134 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
2136 /* Early inliner runs without caching, go ahead and do the dirty work. */
2137 gcc_checking_assert (edge
->inline_failed
);
2138 estimate_node_size_and_time (callee
,
2139 evaluate_conditions_for_edge (edge
, true),
2141 gcc_checking_assert (inline_edge_summary (edge
)->call_stmt_size
);
2142 return size
- inline_edge_summary (edge
)->call_stmt_size
;
2146 /* Estimate self time of the function NODE after inlining EDGE. */
2149 estimate_time_after_inlining (struct cgraph_node
*node
,
2150 struct cgraph_edge
*edge
)
2152 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2153 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
2155 gcov_type time
= inline_summary (node
)->time
+ estimate_edge_time (edge
);
2158 if (time
> MAX_TIME
)
2162 return inline_summary (node
)->time
;
2166 /* Estimate the size of NODE after inlining EDGE which should be an
2167 edge to either NODE or a call inlined into NODE. */
2170 estimate_size_after_inlining (struct cgraph_node
*node
,
2171 struct cgraph_edge
*edge
)
2173 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2174 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
2176 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
2177 gcc_assert (size
>= 0);
2180 return inline_summary (node
)->size
;
2186 bool self_recursive
;
2191 /* Worker for do_estimate_growth. Collect growth for all callers. */
2194 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
2196 struct cgraph_edge
*e
;
2197 struct growth_data
*d
= (struct growth_data
*) data
;
2199 for (e
= node
->callers
; e
; e
= e
->next_caller
)
2201 gcc_checking_assert (e
->inline_failed
);
2203 if (e
->caller
== node
2204 || (e
->caller
->global
.inlined_to
2205 && e
->caller
->global
.inlined_to
== node
))
2206 d
->self_recursive
= true;
2207 d
->growth
+= estimate_edge_growth (e
);
2213 /* Estimate the growth caused by inlining NODE into all callees. */
2216 do_estimate_growth (struct cgraph_node
*node
)
2218 struct growth_data d
= {0, false};
2219 struct inline_summary
*info
= inline_summary (node
);
2221 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
2223 /* For self recursive functions the growth estimation really should be
2224 infinity. We don't want to return very large values because the growth
2225 plays various roles in badness computation fractions. Be sure to not
2226 return zero or negative growths. */
2227 if (d
.self_recursive
)
2228 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
2231 if (!DECL_EXTERNAL (node
->decl
)
2232 && cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
2233 d
.growth
-= info
->size
;
2234 /* COMDAT functions are very often not shared across multiple units since they
2235 come from various template instantiations. Take this into account. */
2236 else if (DECL_COMDAT (node
->decl
)
2237 && cgraph_can_remove_if_no_direct_calls_p (node
))
2238 d
.growth
-= (info
->size
2239 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
)) + 50) / 100;
2242 if (node_growth_cache
)
2244 if ((int)VEC_length (int, node_growth_cache
) <= node
->uid
)
2245 VEC_safe_grow_cleared (int, heap
, node_growth_cache
, cgraph_max_uid
);
2246 VEC_replace (int, node_growth_cache
, node
->uid
, d
.growth
+ (d
.growth
>= 0));
2252 /* This function performs intraprocedural analysis in NODE that is required to
2253 inline indirect calls. */
2256 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
2258 ipa_analyze_node (node
);
2259 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2261 ipa_print_node_params (dump_file
, node
);
2262 ipa_print_node_jump_functions (dump_file
, node
);
2267 /* Note function body size. */
2270 inline_analyze_function (struct cgraph_node
*node
)
2272 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2273 current_function_decl
= node
->decl
;
2276 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
2277 cgraph_node_name (node
), node
->uid
);
2278 /* FIXME: We should remove the optimize check after we ensure we never run
2279 IPA passes when not optimizing. */
2280 if (flag_indirect_inlining
&& optimize
&& !node
->thunk
.thunk_p
)
2281 inline_indirect_intraprocedural_analysis (node
);
2282 compute_inline_parameters (node
, false);
2284 current_function_decl
= NULL
;
2289 /* Called when new function is inserted to callgraph late. */
2292 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
2294 inline_analyze_function (node
);
2298 /* Note function body size. */
2301 inline_generate_summary (void)
2303 struct cgraph_node
*node
;
2305 function_insertion_hook_holder
=
2306 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
2308 if (flag_indirect_inlining
)
2309 ipa_register_cgraph_hooks ();
2311 FOR_EACH_DEFINED_FUNCTION (node
)
2313 inline_analyze_function (node
);
2317 /* Read predicate from IB. */
2319 static struct predicate
2320 read_predicate (struct lto_input_block
*ib
)
2322 struct predicate out
;
2328 gcc_assert (k
<= MAX_CLAUSES
);
2329 clause
= out
.clause
[k
++] = lto_input_uleb128 (ib
);
2333 /* Zero-initialize the remaining clauses in OUT. */
2334 while (k
<= MAX_CLAUSES
)
2335 out
.clause
[k
++] = 0;
2341 /* Write inline summary for edge E to OB. */
2344 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
2346 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2349 es
->call_stmt_size
= lto_input_uleb128 (ib
);
2350 es
->call_stmt_time
= lto_input_uleb128 (ib
);
2351 es
->loop_depth
= lto_input_uleb128 (ib
);
2352 p
= read_predicate (ib
);
2353 edge_set_predicate (e
, &p
);
2357 /* Stream in inline summaries from the section. */
2360 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
2363 const struct lto_function_header
*header
=
2364 (const struct lto_function_header
*) data
;
2365 const int32_t cfg_offset
= sizeof (struct lto_function_header
);
2366 const int32_t main_offset
= cfg_offset
+ header
->cfg_size
;
2367 const int32_t string_offset
= main_offset
+ header
->main_size
;
2368 struct data_in
*data_in
;
2369 struct lto_input_block ib
;
2370 unsigned int i
, count2
, j
;
2371 unsigned int f_count
;
2373 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
2377 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
2378 header
->string_size
, NULL
);
2379 f_count
= lto_input_uleb128 (&ib
);
2380 for (i
= 0; i
< f_count
; i
++)
2383 struct cgraph_node
*node
;
2384 struct inline_summary
*info
;
2385 lto_cgraph_encoder_t encoder
;
2386 struct bitpack_d bp
;
2387 struct cgraph_edge
*e
;
2389 index
= lto_input_uleb128 (&ib
);
2390 encoder
= file_data
->cgraph_node_encoder
;
2391 node
= lto_cgraph_encoder_deref (encoder
, index
);
2392 info
= inline_summary (node
);
2394 info
->estimated_stack_size
2395 = info
->estimated_self_stack_size
= lto_input_uleb128 (&ib
);
2396 info
->size
= info
->self_size
= lto_input_uleb128 (&ib
);
2397 info
->time
= info
->self_time
= lto_input_uleb128 (&ib
);
2399 bp
= lto_input_bitpack (&ib
);
2400 info
->inlinable
= bp_unpack_value (&bp
, 1);
2401 info
->versionable
= bp_unpack_value (&bp
, 1);
2403 count2
= lto_input_uleb128 (&ib
);
2404 gcc_assert (!info
->conds
);
2405 for (j
= 0; j
< count2
; j
++)
2408 c
.operand_num
= lto_input_uleb128 (&ib
);
2409 c
.code
= (enum tree_code
) lto_input_uleb128 (&ib
);
2410 c
.val
= lto_input_tree (&ib
, data_in
);
2411 VEC_safe_push (condition
, gc
, info
->conds
, &c
);
2413 count2
= lto_input_uleb128 (&ib
);
2414 gcc_assert (!info
->entry
);
2415 for (j
= 0; j
< count2
; j
++)
2417 struct size_time_entry e
;
2419 e
.size
= lto_input_uleb128 (&ib
);
2420 e
.time
= lto_input_uleb128 (&ib
);
2421 e
.predicate
= read_predicate (&ib
);
2423 VEC_safe_push (size_time_entry
, gc
, info
->entry
, &e
);
2425 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2426 read_inline_edge_summary (&ib
, e
);
2427 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2428 read_inline_edge_summary (&ib
, e
);
2431 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
2433 lto_data_in_delete (data_in
);
2437 /* Read inline summary. Jump functions are shared among ipa-cp
2438 and inliner, so when ipa-cp is active, we don't need to write them
2442 inline_read_summary (void)
2444 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
2445 struct lto_file_decl_data
*file_data
;
2448 inline_summary_alloc ();
2450 while ((file_data
= file_data_vec
[j
++]))
2453 const char *data
= lto_get_section_data (file_data
, LTO_section_inline_summary
, NULL
, &len
);
2455 inline_read_section (file_data
, data
, len
);
2457 /* Fatal error here. We do not want to support compiling ltrans units with
2458 different version of compiler or different flags than the WPA unit, so
2459 this should never happen. */
2460 fatal_error ("ipa inline summary is missing in input file");
2462 if (flag_indirect_inlining
)
2464 ipa_register_cgraph_hooks ();
2466 ipa_prop_read_jump_functions ();
2468 function_insertion_hook_holder
=
2469 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
2473 /* Write predicate P to OB. */
2476 write_predicate (struct output_block
*ob
, struct predicate
*p
)
2480 for (j
= 0; p
->clause
[j
]; j
++)
2482 gcc_assert (j
< MAX_CLAUSES
);
2483 lto_output_uleb128_stream (ob
->main_stream
,
2486 lto_output_uleb128_stream (ob
->main_stream
, 0);
2490 /* Write inline summary for edge E to OB. */
2493 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
2495 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2496 lto_output_uleb128_stream (ob
->main_stream
, es
->call_stmt_size
);
2497 lto_output_uleb128_stream (ob
->main_stream
, es
->call_stmt_time
);
2498 lto_output_uleb128_stream (ob
->main_stream
, es
->loop_depth
);
2499 write_predicate (ob
, es
->predicate
);
2503 /* Write inline summary for node in SET.
2504 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
2505 active, we don't need to write them twice. */
2508 inline_write_summary (cgraph_node_set set
,
2509 varpool_node_set vset ATTRIBUTE_UNUSED
)
2511 struct cgraph_node
*node
;
2512 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
2513 lto_cgraph_encoder_t encoder
= ob
->decl_state
->cgraph_node_encoder
;
2514 unsigned int count
= 0;
2517 for (i
= 0; i
< lto_cgraph_encoder_size (encoder
); i
++)
2518 if (lto_cgraph_encoder_deref (encoder
, i
)->analyzed
)
2520 lto_output_uleb128_stream (ob
->main_stream
, count
);
2522 for (i
= 0; i
< lto_cgraph_encoder_size (encoder
); i
++)
2524 node
= lto_cgraph_encoder_deref (encoder
, i
);
2527 struct inline_summary
*info
= inline_summary (node
);
2528 struct bitpack_d bp
;
2529 struct cgraph_edge
*edge
;
2532 struct condition
*c
;
2535 lto_output_uleb128_stream (ob
->main_stream
,
2536 lto_cgraph_encoder_encode (encoder
, node
));
2537 lto_output_sleb128_stream (ob
->main_stream
,
2538 info
->estimated_self_stack_size
);
2539 lto_output_sleb128_stream (ob
->main_stream
,
2541 lto_output_sleb128_stream (ob
->main_stream
,
2543 bp
= bitpack_create (ob
->main_stream
);
2544 bp_pack_value (&bp
, info
->inlinable
, 1);
2545 bp_pack_value (&bp
, info
->versionable
, 1);
2546 lto_output_bitpack (&bp
);
2547 lto_output_uleb128_stream (ob
->main_stream
,
2548 VEC_length (condition
, info
->conds
));
2549 for (i
= 0; VEC_iterate (condition
, info
->conds
, i
, c
); i
++)
2551 lto_output_uleb128_stream (ob
->main_stream
,
2553 lto_output_uleb128_stream (ob
->main_stream
,
2555 lto_output_tree (ob
, c
->val
, true);
2557 lto_output_uleb128_stream (ob
->main_stream
,
2558 VEC_length (size_time_entry
, info
->entry
));
2560 VEC_iterate (size_time_entry
, info
->entry
, i
, e
);
2563 lto_output_uleb128_stream (ob
->main_stream
,
2565 lto_output_uleb128_stream (ob
->main_stream
,
2567 write_predicate (ob
, &e
->predicate
);
2569 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
2570 write_inline_edge_summary (ob
, edge
);
2571 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
2572 write_inline_edge_summary (ob
, edge
);
2575 lto_output_1_stream (ob
->main_stream
, 0);
2576 produce_asm (ob
, NULL
);
2577 destroy_output_block (ob
);
2579 if (flag_indirect_inlining
&& !flag_ipa_cp
)
2580 ipa_prop_write_jump_functions (set
);
2584 /* Release inline summary. */
2587 inline_free_summary (void)
2589 if (function_insertion_hook_holder
)
2590 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
2591 function_insertion_hook_holder
= NULL
;
2592 if (node_removal_hook_holder
)
2593 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
2594 if (edge_removal_hook_holder
)
2595 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
2596 node_removal_hook_holder
= NULL
;
2597 if (node_duplication_hook_holder
)
2598 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
2599 if (edge_duplication_hook_holder
)
2600 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
2601 node_duplication_hook_holder
= NULL
;
2602 VEC_free (inline_summary_t
, gc
, inline_summary_vec
);
2603 inline_summary_vec
= NULL
;
2604 VEC_free (inline_edge_summary_t
, heap
, inline_edge_summary_vec
);
2605 inline_edge_summary_vec
= NULL
;
2606 if (edge_predicate_pool
)
2607 free_alloc_pool (edge_predicate_pool
);
2608 edge_predicate_pool
= 0;