1 /* Function summary pass.
2 Copyright (C) 2003-2017 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis of function bodies used by inter-procedural passes
23 We estimate for each function
24 - function body size and size after specializing into given context
25 - average function execution time in a given context
28 - call statement size, time and how often the parameters change
30 ipa_fn_summary data structures store above information locally (i.e.
31 parameters of the function itself) and globally (i.e. parameters of
32 the function created by applying all the inline decisions already
33 present in the callgraph).
35 We provide access to the ipa_fn_summary data structure and
36 basic logic updating the parameters when inlining is performed.
38 The summaries are context sensitive. Context means
39 1) partial assignment of known constant values of operands
40 2) whether function is inlined into the call or not.
41 It is easy to add more variants. To represent function size and time
42 that depends on context (i.e. it is known to be optimized away when
43 context is known either by inlining or from IP-CP and cloning),
46 estimate_edge_size_and_time can be used to query
47 function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 properties of caller and callee after inlining.
50 Finally pass_inline_parameters is exported. This is used to drive
51 computation of function parameters used by the early inliner. IPA
52 inlined performs analysis via its analyze_function method. */
56 #include "coretypes.h"
60 #include "alloc-pool.h"
61 #include "tree-pass.h"
63 #include "tree-streamer.h"
65 #include "diagnostic.h"
66 #include "fold-const.h"
67 #include "print-tree.h"
68 #include "tree-inline.h"
69 #include "gimple-pretty-print.h"
72 #include "gimple-iterator.h"
74 #include "tree-ssa-loop-niter.h"
75 #include "tree-ssa-loop.h"
76 #include "symbol-summary.h"
78 #include "ipa-fnsummary.h"
80 #include "tree-scalar-evolution.h"
81 #include "ipa-utils.h"
83 #include "cfgexpand.h"
85 #include "stringpool.h"
89 function_summary
<ipa_fn_summary
*> *ipa_fn_summaries
;
90 call_summary
<ipa_call_summary
*> *ipa_call_summaries
;
92 /* Edge predicates goes here. */
93 static object_allocator
<predicate
> edge_predicate_pool ("edge predicates");
98 ipa_dump_hints (FILE *f
, ipa_hints hints
)
102 fprintf (f
, "IPA hints:");
103 if (hints
& INLINE_HINT_indirect_call
)
105 hints
&= ~INLINE_HINT_indirect_call
;
106 fprintf (f
, " indirect_call");
108 if (hints
& INLINE_HINT_loop_iterations
)
110 hints
&= ~INLINE_HINT_loop_iterations
;
111 fprintf (f
, " loop_iterations");
113 if (hints
& INLINE_HINT_loop_stride
)
115 hints
&= ~INLINE_HINT_loop_stride
;
116 fprintf (f
, " loop_stride");
118 if (hints
& INLINE_HINT_same_scc
)
120 hints
&= ~INLINE_HINT_same_scc
;
121 fprintf (f
, " same_scc");
123 if (hints
& INLINE_HINT_in_scc
)
125 hints
&= ~INLINE_HINT_in_scc
;
126 fprintf (f
, " in_scc");
128 if (hints
& INLINE_HINT_cross_module
)
130 hints
&= ~INLINE_HINT_cross_module
;
131 fprintf (f
, " cross_module");
133 if (hints
& INLINE_HINT_declared_inline
)
135 hints
&= ~INLINE_HINT_declared_inline
;
136 fprintf (f
, " declared_inline");
138 if (hints
& INLINE_HINT_array_index
)
140 hints
&= ~INLINE_HINT_array_index
;
141 fprintf (f
, " array_index");
143 if (hints
& INLINE_HINT_known_hot
)
145 hints
&= ~INLINE_HINT_known_hot
;
146 fprintf (f
, " known_hot");
152 /* Record SIZE and TIME to SUMMARY.
153 The accounted code will be executed when EXEC_PRED is true.
154 When NONCONST_PRED is false the code will evaulate to constant and
155 will get optimized out in specialized clones of the function. */
158 ipa_fn_summary::account_size_time (int size
, sreal time
,
159 const predicate
&exec_pred
,
160 const predicate
&nonconst_pred_in
)
165 predicate nonconst_pred
;
167 if (exec_pred
== false)
170 nonconst_pred
= nonconst_pred_in
& exec_pred
;
172 if (nonconst_pred
== false)
175 /* We need to create initial empty unconitional clause, but otherwie
176 we don't need to account empty times and sizes. */
177 if (!size
&& time
== 0 && size_time_table
)
180 gcc_assert (time
>= 0);
182 for (i
= 0; vec_safe_iterate (size_time_table
, i
, &e
); i
++)
183 if (e
->exec_predicate
== exec_pred
184 && e
->nonconst_predicate
== nonconst_pred
)
193 e
= &(*size_time_table
)[0];
194 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
196 "\t\tReached limit on number of entries, "
197 "ignoring the predicate.");
199 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
!= 0 || size
))
202 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
203 ((double) size
) / ipa_fn_summary::size_scale
,
204 (time
.to_double ()), found
? "" : "new ");
205 exec_pred
.dump (dump_file
, conds
, 0);
206 if (exec_pred
!= nonconst_pred
)
208 fprintf (dump_file
, " nonconst:");
209 nonconst_pred
.dump (dump_file
, conds
);
212 fprintf (dump_file
, "\n");
216 struct size_time_entry new_entry
;
217 new_entry
.size
= size
;
218 new_entry
.time
= time
;
219 new_entry
.exec_predicate
= exec_pred
;
220 new_entry
.nonconst_predicate
= nonconst_pred
;
221 vec_safe_push (size_time_table
, new_entry
);
230 /* We proved E to be unreachable, redirect it to __bultin_unreachable. */
232 static struct cgraph_edge
*
233 redirect_to_unreachable (struct cgraph_edge
*e
)
235 struct cgraph_node
*callee
= !e
->inline_failed
? e
->callee
: NULL
;
236 struct cgraph_node
*target
= cgraph_node::get_create
237 (builtin_decl_implicit (BUILT_IN_UNREACHABLE
));
240 e
= e
->resolve_speculation (target
->decl
);
242 e
->make_direct (target
);
244 e
->redirect_callee (target
);
245 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
246 e
->inline_failed
= CIF_UNREACHABLE
;
247 e
->count
= profile_count::zero ();
248 es
->call_stmt_size
= 0;
249 es
->call_stmt_time
= 0;
251 callee
->remove_symbol_and_inline_clones ();
255 /* Set predicate for edge E. */
258 edge_set_predicate (struct cgraph_edge
*e
, predicate
*predicate
)
260 /* If the edge is determined to be never executed, redirect it
261 to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
263 if (predicate
&& *predicate
== false
264 /* When handling speculative edges, we need to do the redirection
265 just once. Do it always on the direct edge, so we do not
266 attempt to resolve speculation while duplicating the edge. */
267 && (!e
->speculative
|| e
->callee
))
268 e
= redirect_to_unreachable (e
);
270 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
271 if (predicate
&& *predicate
!= true)
274 es
->predicate
= edge_predicate_pool
.allocate ();
275 *es
->predicate
= *predicate
;
280 edge_predicate_pool
.remove (es
->predicate
);
281 es
->predicate
= NULL
;
285 /* Set predicate for hint *P. */
288 set_hint_predicate (predicate
**p
, predicate new_predicate
)
290 if (new_predicate
== false || new_predicate
== true)
293 edge_predicate_pool
.remove (*p
);
299 *p
= edge_predicate_pool
.allocate ();
305 /* Compute what conditions may or may not hold given invormation about
306 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
307 whie RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
308 copy when called in a given context. It is a bitmask of conditions. Bit
309 0 means that condition is known to be false, while bit 1 means that condition
310 may or may not be true. These differs - for example NOT_INLINED condition
311 is always false in the second and also builtin_constant_p tests can not use
312 the fact that parameter is indeed a constant.
314 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
315 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
316 Return clause of possible truths. When INLINE_P is true, assume that we are
319 ERROR_MARK means compile time invariant. */
322 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
324 vec
<tree
> known_vals
,
325 vec
<ipa_agg_jump_function_p
>
327 clause_t
*ret_clause
,
328 clause_t
*ret_nonspec_clause
)
330 clause_t clause
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
331 clause_t nonspec_clause
= 1 << predicate::not_inlined_condition
;
332 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
336 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
341 /* We allow call stmt to have fewer arguments than the callee function
342 (especially for K&R style programs). So bound check here (we assume
343 known_aggs vector, if non-NULL, has the same length as
345 gcc_checking_assert (!known_aggs
.exists ()
346 || (known_vals
.length () == known_aggs
.length ()));
347 if (c
->operand_num
>= (int) known_vals
.length ())
349 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
350 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
356 struct ipa_agg_jump_function
*agg
;
358 if (c
->code
== predicate::changed
360 && (known_vals
[c
->operand_num
] == error_mark_node
))
363 if (known_aggs
.exists ())
365 agg
= known_aggs
[c
->operand_num
];
366 val
= ipa_find_agg_cst_for_param (agg
, known_vals
[c
->operand_num
],
367 c
->offset
, c
->by_ref
);
374 val
= known_vals
[c
->operand_num
];
375 if (val
== error_mark_node
&& c
->code
!= predicate::changed
)
381 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
382 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
385 if (c
->code
== predicate::changed
)
387 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
391 if (tree_to_shwi (TYPE_SIZE (TREE_TYPE (val
))) != c
->size
)
393 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
394 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
397 if (c
->code
== predicate::is_not_constant
)
399 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
403 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (c
->val
), val
);
405 ? fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
)
408 if (res
&& integer_zerop (res
))
411 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
412 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
414 *ret_clause
= clause
;
415 if (ret_nonspec_clause
)
416 *ret_nonspec_clause
= nonspec_clause
;
420 /* Work out what conditions might be true at invocation of E. */
423 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
424 clause_t
*clause_ptr
,
425 clause_t
*nonspec_clause_ptr
,
426 vec
<tree
> *known_vals_ptr
,
427 vec
<ipa_polymorphic_call_context
>
429 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
431 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
432 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (callee
);
433 vec
<tree
> known_vals
= vNULL
;
434 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
437 *clause_ptr
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
439 known_vals_ptr
->create (0);
440 if (known_contexts_ptr
)
441 known_contexts_ptr
->create (0);
443 if (ipa_node_params_sum
444 && !e
->call_stmt_cannot_inline_p
445 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_contexts_ptr
))
447 struct ipa_node_params
*parms_info
;
448 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
449 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
450 int i
, count
= ipa_get_cs_argument_count (args
);
452 if (e
->caller
->global
.inlined_to
)
453 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
455 parms_info
= IPA_NODE_REF (e
->caller
);
457 if (count
&& (info
->conds
|| known_vals_ptr
))
458 known_vals
.safe_grow_cleared (count
);
459 if (count
&& (info
->conds
|| known_aggs_ptr
))
460 known_aggs
.safe_grow_cleared (count
);
461 if (count
&& known_contexts_ptr
)
462 known_contexts_ptr
->safe_grow_cleared (count
);
464 for (i
= 0; i
< count
; i
++)
466 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
467 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
469 if (!cst
&& e
->call_stmt
470 && i
< (int)gimple_call_num_args (e
->call_stmt
))
472 cst
= gimple_call_arg (e
->call_stmt
, i
);
473 if (!is_gimple_min_invariant (cst
))
478 gcc_checking_assert (TREE_CODE (cst
) != TREE_BINFO
);
479 if (known_vals
.exists ())
482 else if (inline_p
&& !es
->param
[i
].change_prob
)
483 known_vals
[i
] = error_mark_node
;
485 if (known_contexts_ptr
)
486 (*known_contexts_ptr
)[i
] = ipa_context_from_jfunc (parms_info
, e
,
488 /* TODO: When IPA-CP starts propagating and merging aggregate jump
489 functions, use its knowledge of the caller too, just like the
490 scalar case above. */
491 known_aggs
[i
] = &jf
->agg
;
494 else if (e
->call_stmt
&& !e
->call_stmt_cannot_inline_p
495 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
))
497 int i
, count
= (int)gimple_call_num_args (e
->call_stmt
);
499 if (count
&& (info
->conds
|| known_vals_ptr
))
500 known_vals
.safe_grow_cleared (count
);
501 for (i
= 0; i
< count
; i
++)
503 tree cst
= gimple_call_arg (e
->call_stmt
, i
);
504 if (!is_gimple_min_invariant (cst
))
511 evaluate_conditions_for_known_args (callee
, inline_p
,
512 known_vals
, known_aggs
, clause_ptr
,
516 *known_vals_ptr
= known_vals
;
518 known_vals
.release ();
521 *known_aggs_ptr
= known_aggs
;
523 known_aggs
.release ();
527 /* Allocate the function summary. */
530 ipa_fn_summary_alloc (void)
532 gcc_checking_assert (!ipa_fn_summaries
);
533 ipa_fn_summaries
= ipa_fn_summary_t::create_ggc (symtab
);
534 ipa_call_summaries
= new ipa_call_summary_t (symtab
, false);
537 /* We are called multiple time for given function; clear
538 data from previous run so they are not cumulated. */
541 ipa_call_summary::reset ()
543 call_stmt_size
= call_stmt_time
= 0;
544 is_return_callee_uncaptured
= false;
546 edge_predicate_pool
.remove (predicate
);
551 /* We are called multiple time for given function; clear
552 data from previous run so they are not cumulated. */
555 ipa_fn_summary::reset (struct cgraph_node
*node
)
557 struct cgraph_edge
*e
;
560 estimated_stack_size
= 0;
561 estimated_self_stack_size
= 0;
562 stack_frame_offset
= 0;
569 edge_predicate_pool
.remove (loop_iterations
);
570 loop_iterations
= NULL
;
574 edge_predicate_pool
.remove (loop_stride
);
579 edge_predicate_pool
.remove (array_index
);
583 vec_free (size_time_table
);
584 for (e
= node
->callees
; e
; e
= e
->next_callee
)
585 ipa_call_summaries
->get (e
)->reset ();
586 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
587 ipa_call_summaries
->get (e
)->reset ();
588 fp_expressions
= false;
591 /* Hook that is called by cgraph.c when a node is removed. */
594 ipa_fn_summary_t::remove (cgraph_node
*node
, ipa_fn_summary
*info
)
599 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
600 Additionally care about allocating new memory slot for updated predicate
601 and set it to NULL when it becomes true or false (and thus uninteresting).
605 remap_hint_predicate_after_duplication (predicate
**p
,
606 clause_t possible_truths
)
608 predicate new_predicate
;
613 new_predicate
= (*p
)->remap_after_duplication (possible_truths
);
614 /* We do not want to free previous predicate; it is used by node origin. */
616 set_hint_predicate (p
, new_predicate
);
620 /* Hook that is called by cgraph.c when a node is duplicated. */
622 ipa_fn_summary_t::duplicate (cgraph_node
*src
,
625 ipa_fn_summary
*info
)
627 memcpy (info
, ipa_fn_summaries
->get (src
), sizeof (ipa_fn_summary
));
628 /* TODO: as an optimization, we may avoid copying conditions
629 that are known to be false or true. */
630 info
->conds
= vec_safe_copy (info
->conds
);
632 /* When there are any replacements in the function body, see if we can figure
633 out that something was optimized out. */
634 if (ipa_node_params_sum
&& dst
->clone
.tree_map
)
636 vec
<size_time_entry
, va_gc
> *entry
= info
->size_time_table
;
637 /* Use SRC parm info since it may not be copied yet. */
638 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
639 vec
<tree
> known_vals
= vNULL
;
640 int count
= ipa_get_param_count (parms_info
);
642 clause_t possible_truths
;
643 predicate true_pred
= true;
645 int optimized_out_size
= 0;
646 bool inlined_to_p
= false;
647 struct cgraph_edge
*edge
, *next
;
649 info
->size_time_table
= 0;
650 known_vals
.safe_grow_cleared (count
);
651 for (i
= 0; i
< count
; i
++)
653 struct ipa_replace_map
*r
;
655 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
657 if (((!r
->old_tree
&& r
->parm_num
== i
)
658 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
659 && r
->replace_p
&& !r
->ref_p
)
661 known_vals
[i
] = r
->new_tree
;
666 evaluate_conditions_for_known_args (dst
, false,
670 /* We are going to specialize,
671 so ignore nonspec truths. */
673 known_vals
.release ();
675 info
->account_size_time (0, 0, true_pred
, true_pred
);
677 /* Remap size_time vectors.
678 Simplify the predicate by prunning out alternatives that are known
680 TODO: as on optimization, we can also eliminate conditions known
682 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
684 predicate new_exec_pred
;
685 predicate new_nonconst_pred
;
686 new_exec_pred
= e
->exec_predicate
.remap_after_duplication
688 new_nonconst_pred
= e
->nonconst_predicate
.remap_after_duplication
690 if (new_exec_pred
== false || new_nonconst_pred
== false)
691 optimized_out_size
+= e
->size
;
693 info
->account_size_time (e
->size
, e
->time
, new_exec_pred
,
697 /* Remap edge predicates with the same simplification as above.
698 Also copy constantness arrays. */
699 for (edge
= dst
->callees
; edge
; edge
= next
)
701 predicate new_predicate
;
702 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
703 next
= edge
->next_callee
;
705 if (!edge
->inline_failed
)
709 new_predicate
= es
->predicate
->remap_after_duplication
711 if (new_predicate
== false && *es
->predicate
!= false)
712 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
713 edge_set_predicate (edge
, &new_predicate
);
716 /* Remap indirect edge predicates with the same simplificaiton as above.
717 Also copy constantness arrays. */
718 for (edge
= dst
->indirect_calls
; edge
; edge
= next
)
720 predicate new_predicate
;
721 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
722 next
= edge
->next_callee
;
724 gcc_checking_assert (edge
->inline_failed
);
727 new_predicate
= es
->predicate
->remap_after_duplication
729 if (new_predicate
== false && *es
->predicate
!= false)
730 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
731 edge_set_predicate (edge
, &new_predicate
);
733 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
735 remap_hint_predicate_after_duplication (&info
->loop_stride
,
737 remap_hint_predicate_after_duplication (&info
->array_index
,
740 /* If inliner or someone after inliner will ever start producing
741 non-trivial clones, we will get trouble with lack of information
742 about updating self sizes, because size vectors already contains
743 sizes of the calees. */
744 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
748 info
->size_time_table
= vec_safe_copy (info
->size_time_table
);
749 if (info
->loop_iterations
)
751 predicate p
= *info
->loop_iterations
;
752 info
->loop_iterations
= NULL
;
753 set_hint_predicate (&info
->loop_iterations
, p
);
755 if (info
->loop_stride
)
757 predicate p
= *info
->loop_stride
;
758 info
->loop_stride
= NULL
;
759 set_hint_predicate (&info
->loop_stride
, p
);
761 if (info
->array_index
)
763 predicate p
= *info
->array_index
;
764 info
->array_index
= NULL
;
765 set_hint_predicate (&info
->array_index
, p
);
768 if (!dst
->global
.inlined_to
)
769 ipa_update_overall_fn_summary (dst
);
773 /* Hook that is called by cgraph.c when a node is duplicated. */
776 ipa_call_summary_t::duplicate (struct cgraph_edge
*src
,
777 struct cgraph_edge
*dst
,
778 struct ipa_call_summary
*srcinfo
,
779 struct ipa_call_summary
*info
)
782 info
->predicate
= NULL
;
783 edge_set_predicate (dst
, srcinfo
->predicate
);
784 info
->param
= srcinfo
->param
.copy ();
785 if (!dst
->indirect_unknown_callee
&& src
->indirect_unknown_callee
)
787 info
->call_stmt_size
-= (eni_size_weights
.indirect_call_cost
788 - eni_size_weights
.call_cost
);
789 info
->call_stmt_time
-= (eni_time_weights
.indirect_call_cost
790 - eni_time_weights
.call_cost
);
795 /* Keep edge cache consistent across edge removal. */
798 ipa_call_summary_t::remove (struct cgraph_edge
*,
799 struct ipa_call_summary
*sum
)
805 /* Dump edge summaries associated to NODE and recursively to all clones.
809 dump_ipa_call_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
810 struct ipa_fn_summary
*info
)
812 struct cgraph_edge
*edge
;
813 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
815 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
816 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
820 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4.2f size:%2i"
821 " time: %2i callee size:%2i stack:%2i",
822 indent
, "", callee
->name (), callee
->order
,
824 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
825 indent
, "", es
->loop_depth
, edge
->sreal_frequency ().to_double (),
826 es
->call_stmt_size
, es
->call_stmt_time
,
827 (int) ipa_fn_summaries
->get (callee
)->size
/ ipa_fn_summary::size_scale
,
828 (int) ipa_fn_summaries
->get (callee
)->estimated_stack_size
);
832 fprintf (f
, " predicate: ");
833 es
->predicate
->dump (f
, info
->conds
);
837 if (es
->param
.exists ())
838 for (i
= 0; i
< (int) es
->param
.length (); i
++)
840 int prob
= es
->param
[i
].change_prob
;
843 fprintf (f
, "%*s op%i is compile time invariant\n",
845 else if (prob
!= REG_BR_PROB_BASE
)
846 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
847 prob
* 100.0 / REG_BR_PROB_BASE
);
849 if (!edge
->inline_failed
)
851 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
854 (int) ipa_fn_summaries
->get (callee
)->stack_frame_offset
,
855 (int) ipa_fn_summaries
->get (callee
)->estimated_self_stack_size
,
856 (int) ipa_fn_summaries
->get (callee
)->estimated_stack_size
);
857 dump_ipa_call_summary (f
, indent
+ 2, callee
, info
);
860 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
862 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
863 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
867 edge
->sreal_frequency ().to_double (), es
->call_stmt_size
,
871 fprintf (f
, "predicate: ");
872 es
->predicate
->dump (f
, info
->conds
);
881 ipa_dump_fn_summary (FILE *f
, struct cgraph_node
*node
)
883 if (node
->definition
)
885 struct ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
888 fprintf (f
, "IPA function summary for %s/%i", node
->name (),
890 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
891 fprintf (f
, " always_inline");
893 fprintf (f
, " inlinable");
894 if (s
->contains_cilk_spawn
)
895 fprintf (f
, " contains_cilk_spawn");
896 if (s
->fp_expressions
)
897 fprintf (f
, " fp_expression");
898 fprintf (f
, "\n global time: %f\n", s
->time
.to_double ());
899 fprintf (f
, " self size: %i\n", s
->self_size
);
900 fprintf (f
, " global size: %i\n", s
->size
);
901 fprintf (f
, " min size: %i\n", s
->min_size
);
902 fprintf (f
, " self stack: %i\n",
903 (int) s
->estimated_self_stack_size
);
904 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
906 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
908 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
909 for (i
= 0; vec_safe_iterate (s
->size_time_table
, i
, &e
); i
++)
911 fprintf (f
, " size:%f, time:%f",
912 (double) e
->size
/ ipa_fn_summary::size_scale
,
913 e
->time
.to_double ());
914 if (e
->exec_predicate
!= true)
916 fprintf (f
, ", executed if:");
917 e
->exec_predicate
.dump (f
, s
->conds
, 0);
919 if (e
->exec_predicate
!= e
->nonconst_predicate
)
921 fprintf (f
, ", nonconst if:");
922 e
->nonconst_predicate
.dump (f
, s
->conds
, 0);
926 if (s
->loop_iterations
)
928 fprintf (f
, " loop iterations:");
929 s
->loop_iterations
->dump (f
, s
->conds
);
933 fprintf (f
, " loop stride:");
934 s
->loop_stride
->dump (f
, s
->conds
);
938 fprintf (f
, " array index:");
939 s
->array_index
->dump (f
, s
->conds
);
941 fprintf (f
, " calls:\n");
942 dump_ipa_call_summary (f
, 4, node
, s
);
948 ipa_debug_fn_summary (struct cgraph_node
*node
)
950 ipa_dump_fn_summary (stderr
, node
);
954 ipa_dump_fn_summaries (FILE *f
)
956 struct cgraph_node
*node
;
958 FOR_EACH_DEFINED_FUNCTION (node
)
959 if (!node
->global
.inlined_to
)
960 ipa_dump_fn_summary (f
, node
);
963 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
964 boolean variable pointed to by DATA. */
967 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
970 bool *b
= (bool *) data
;
975 /* If OP refers to value of function parameter, return the corresponding
976 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
977 PARM_DECL) will be stored to *SIZE_P in that case too. */
980 unmodified_parm_1 (gimple
*stmt
, tree op
, HOST_WIDE_INT
*size_p
)
982 /* SSA_NAME referring to parm default def? */
983 if (TREE_CODE (op
) == SSA_NAME
984 && SSA_NAME_IS_DEFAULT_DEF (op
)
985 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
988 *size_p
= tree_to_shwi (TYPE_SIZE (TREE_TYPE (op
)));
989 return SSA_NAME_VAR (op
);
991 /* Non-SSA parm reference? */
992 if (TREE_CODE (op
) == PARM_DECL
)
994 bool modified
= false;
997 ao_ref_init (&refd
, op
);
998 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1003 *size_p
= tree_to_shwi (TYPE_SIZE (TREE_TYPE (op
)));
1010 /* If OP refers to value of function parameter, return the corresponding
1011 parameter. Also traverse chains of SSA register assignments. If non-NULL,
1012 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1013 stored to *SIZE_P in that case too. */
1016 unmodified_parm (gimple
*stmt
, tree op
, HOST_WIDE_INT
*size_p
)
1018 tree res
= unmodified_parm_1 (stmt
, op
, size_p
);
1022 if (TREE_CODE (op
) == SSA_NAME
1023 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1024 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1025 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1026 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)),
1031 /* If OP refers to a value of a function parameter or value loaded from an
1032 aggregate passed to a parameter (either by value or reference), return TRUE
1033 and store the number of the parameter to *INDEX_P, the access size into
1034 *SIZE_P, and information whether and how it has been loaded from an
1035 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1036 statement in which OP is used or loaded. */
1039 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info
*fbi
,
1040 gimple
*stmt
, tree op
, int *index_p
,
1041 HOST_WIDE_INT
*size_p
,
1042 struct agg_position_info
*aggpos
)
1044 tree res
= unmodified_parm_1 (stmt
, op
, size_p
);
1046 gcc_checking_assert (aggpos
);
1049 *index_p
= ipa_get_param_decl_index (fbi
->info
, res
);
1052 aggpos
->agg_contents
= false;
1053 aggpos
->by_ref
= false;
1057 if (TREE_CODE (op
) == SSA_NAME
)
1059 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1060 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1062 stmt
= SSA_NAME_DEF_STMT (op
);
1063 op
= gimple_assign_rhs1 (stmt
);
1064 if (!REFERENCE_CLASS_P (op
))
1065 return unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, index_p
, size_p
,
1069 aggpos
->agg_contents
= true;
1070 return ipa_load_from_parm_agg (fbi
, fbi
->info
->descriptors
,
1071 stmt
, op
, index_p
, &aggpos
->offset
,
1072 size_p
, &aggpos
->by_ref
);
1075 /* See if statement might disappear after inlining.
1076 0 - means not eliminated
1077 1 - half of statements goes away
1078 2 - for sure it is eliminated.
1079 We are not terribly sophisticated, basically looking for simple abstraction
1080 penalty wrappers. */
1083 eliminated_by_inlining_prob (gimple
*stmt
)
1085 enum gimple_code code
= gimple_code (stmt
);
1086 enum tree_code rhs_code
;
1096 if (gimple_num_ops (stmt
) != 2)
1099 rhs_code
= gimple_assign_rhs_code (stmt
);
1101 /* Casts of parameters, loads from parameters passed by reference
1102 and stores to return value or parameters are often free after
1103 inlining dua to SRA and further combining.
1104 Assume that half of statements goes away. */
1105 if (CONVERT_EXPR_CODE_P (rhs_code
)
1106 || rhs_code
== VIEW_CONVERT_EXPR
1107 || rhs_code
== ADDR_EXPR
1108 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1110 tree rhs
= gimple_assign_rhs1 (stmt
);
1111 tree lhs
= gimple_assign_lhs (stmt
);
1112 tree inner_rhs
= get_base_address (rhs
);
1113 tree inner_lhs
= get_base_address (lhs
);
1114 bool rhs_free
= false;
1115 bool lhs_free
= false;
1122 /* Reads of parameter are expected to be free. */
1123 if (unmodified_parm (stmt
, inner_rhs
, NULL
))
1125 /* Match expressions of form &this->field. Those will most likely
1126 combine with something upstream after inlining. */
1127 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1129 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1130 if (TREE_CODE (op
) == PARM_DECL
)
1132 else if (TREE_CODE (op
) == MEM_REF
1133 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0), NULL
))
1137 /* When parameter is not SSA register because its address is taken
1138 and it is just copied into one, the statement will be completely
1139 free after inlining (we will copy propagate backward). */
1140 if (rhs_free
&& is_gimple_reg (lhs
))
1143 /* Reads of parameters passed by reference
1144 expected to be free (i.e. optimized out after inlining). */
1145 if (TREE_CODE (inner_rhs
) == MEM_REF
1146 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0), NULL
))
1149 /* Copying parameter passed by reference into gimple register is
1150 probably also going to copy propagate, but we can't be quite
1152 if (rhs_free
&& is_gimple_reg (lhs
))
1155 /* Writes to parameters, parameters passed by value and return value
1156 (either dirrectly or passed via invisible reference) are free.
1158 TODO: We ought to handle testcase like
1159 struct a {int a,b;};
1161 retrurnsturct (void)
1167 This translate into:
1182 For that we either need to copy ipa-split logic detecting writes
1184 if (TREE_CODE (inner_lhs
) == PARM_DECL
1185 || TREE_CODE (inner_lhs
) == RESULT_DECL
1186 || (TREE_CODE (inner_lhs
) == MEM_REF
1187 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0), NULL
)
1188 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1189 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1190 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1192 0))) == RESULT_DECL
))))
1195 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1197 if (lhs_free
&& rhs_free
)
1207 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1208 predicates to the CFG edges. */
1211 set_cond_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1212 struct ipa_fn_summary
*summary
,
1219 struct agg_position_info aggpos
;
1220 enum tree_code code
, inverted_code
;
1226 last
= last_stmt (bb
);
1227 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1229 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1231 op
= gimple_cond_lhs (last
);
1232 /* TODO: handle conditionals like
1235 if (unmodified_parm_or_parm_agg_item (fbi
, last
, op
, &index
, &size
, &aggpos
))
1237 code
= gimple_cond_code (last
);
1238 inverted_code
= invert_tree_comparison (code
, HONOR_NANS (op
));
1240 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1242 enum tree_code this_code
= (e
->flags
& EDGE_TRUE_VALUE
1243 ? code
: inverted_code
);
1244 /* invert_tree_comparison will return ERROR_MARK on FP
1245 comparsions that are not EQ/NE instead of returning proper
1246 unordered one. Be sure it is not confused with NON_CONSTANT. */
1247 if (this_code
!= ERROR_MARK
)
1250 = add_condition (summary
, index
, size
, &aggpos
, this_code
,
1251 unshare_expr_without_location
1252 (gimple_cond_rhs (last
)));
1253 e
->aux
= edge_predicate_pool
.allocate ();
1254 *(predicate
*) e
->aux
= p
;
1259 if (TREE_CODE (op
) != SSA_NAME
)
1262 if (builtin_constant_p (op))
1266 Here we can predicate nonconstant_code. We can't
1267 really handle constant_code since we have no predicate
1268 for this and also the constant code is not known to be
1269 optimized away when inliner doen't see operand is constant.
1270 Other optimizers might think otherwise. */
1271 if (gimple_cond_code (last
) != NE_EXPR
1272 || !integer_zerop (gimple_cond_rhs (last
)))
1274 set_stmt
= SSA_NAME_DEF_STMT (op
);
1275 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1276 || gimple_call_num_args (set_stmt
) != 1)
1278 op2
= gimple_call_arg (set_stmt
, 0);
1279 if (!unmodified_parm_or_parm_agg_item (fbi
, set_stmt
, op2
, &index
, &size
,
1282 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1284 predicate p
= add_condition (summary
, index
, size
, &aggpos
,
1285 predicate::is_not_constant
, NULL_TREE
);
1286 e
->aux
= edge_predicate_pool
.allocate ();
1287 *(predicate
*) e
->aux
= p
;
1292 /* If BB ends by a switch we can turn into predicates, attach corresponding
1293 predicates to the CFG edges. */
1296 set_switch_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1297 struct ipa_fn_summary
*summary
,
1304 struct agg_position_info aggpos
;
1310 lastg
= last_stmt (bb
);
1311 if (!lastg
|| gimple_code (lastg
) != GIMPLE_SWITCH
)
1313 gswitch
*last
= as_a
<gswitch
*> (lastg
);
1314 op
= gimple_switch_index (last
);
1315 if (!unmodified_parm_or_parm_agg_item (fbi
, last
, op
, &index
, &size
, &aggpos
))
1318 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1320 e
->aux
= edge_predicate_pool
.allocate ();
1321 *(predicate
*) e
->aux
= false;
1323 n
= gimple_switch_num_labels (last
);
1324 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1326 tree cl
= gimple_switch_label (last
, case_idx
);
1330 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1331 min
= CASE_LOW (cl
);
1332 max
= CASE_HIGH (cl
);
1334 /* For default we might want to construct predicate that none
1335 of cases is met, but it is bit hard to do not having negations
1336 of conditionals handy. */
1340 p
= add_condition (summary
, index
, size
, &aggpos
, EQ_EXPR
,
1341 unshare_expr_without_location (min
));
1345 p1
= add_condition (summary
, index
, size
, &aggpos
, GE_EXPR
,
1346 unshare_expr_without_location (min
));
1347 p2
= add_condition (summary
, index
, size
, &aggpos
, LE_EXPR
,
1348 unshare_expr_without_location (max
));
1351 *(struct predicate
*) e
->aux
1352 = p
.or_with (summary
->conds
, *(struct predicate
*) e
->aux
);
1357 /* For each BB in NODE attach to its AUX pointer predicate under
1358 which it is executable. */
1361 compute_bb_predicates (struct ipa_func_body_info
*fbi
,
1362 struct cgraph_node
*node
,
1363 struct ipa_fn_summary
*summary
)
1365 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1369 FOR_EACH_BB_FN (bb
, my_function
)
1371 set_cond_stmt_execution_predicate (fbi
, summary
, bb
);
1372 set_switch_stmt_execution_predicate (fbi
, summary
, bb
);
1375 /* Entry block is always executable. */
1376 ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
1377 = edge_predicate_pool
.allocate ();
1378 *(predicate
*) ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
= true;
1380 /* A simple dataflow propagation of predicates forward in the CFG.
1381 TODO: work in reverse postorder. */
1385 FOR_EACH_BB_FN (bb
, my_function
)
1387 predicate p
= false;
1390 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1394 predicate this_bb_predicate
1395 = *(predicate
*) e
->src
->aux
;
1397 this_bb_predicate
&= (*(struct predicate
*) e
->aux
);
1398 p
= p
.or_with (summary
->conds
, this_bb_predicate
);
1404 gcc_checking_assert (!bb
->aux
);
1410 bb
->aux
= edge_predicate_pool
.allocate ();
1411 *((predicate
*) bb
->aux
) = p
;
1413 else if (p
!= *(predicate
*) bb
->aux
)
1415 /* This OR operation is needed to ensure monotonous data flow
1416 in the case we hit the limit on number of clauses and the
1417 and/or operations above give approximate answers. */
1418 p
= p
.or_with (summary
->conds
, *(predicate
*)bb
->aux
);
1419 if (p
!= *(predicate
*) bb
->aux
)
1422 *((predicate
*) bb
->aux
) = p
;
1431 /* Return predicate specifying when the STMT might have result that is not
1432 a compile time constant. */
1435 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1436 struct ipa_fn_summary
*summary
,
1438 vec
<predicate
> nonconstant_names
)
1444 while (UNARY_CLASS_P (expr
))
1445 expr
= TREE_OPERAND (expr
, 0);
1447 parm
= unmodified_parm (NULL
, expr
, &size
);
1448 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1449 return add_condition (summary
, index
, size
, NULL
, predicate::changed
,
1451 if (is_gimple_min_invariant (expr
))
1453 if (TREE_CODE (expr
) == SSA_NAME
)
1454 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1455 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1457 predicate p1
= will_be_nonconstant_expr_predicate
1458 (info
, summary
, TREE_OPERAND (expr
, 0),
1464 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1465 TREE_OPERAND (expr
, 1),
1467 return p1
.or_with (summary
->conds
, p2
);
1469 else if (TREE_CODE (expr
) == COND_EXPR
)
1471 predicate p1
= will_be_nonconstant_expr_predicate
1472 (info
, summary
, TREE_OPERAND (expr
, 0),
1478 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1479 TREE_OPERAND (expr
, 1),
1483 p1
= p1
.or_with (summary
->conds
, p2
);
1484 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1485 TREE_OPERAND (expr
, 2),
1487 return p2
.or_with (summary
->conds
, p1
);
1498 /* Return predicate specifying when the STMT might have result that is not
1499 a compile time constant. */
1502 will_be_nonconstant_predicate (struct ipa_func_body_info
*fbi
,
1503 struct ipa_fn_summary
*summary
,
1505 vec
<predicate
> nonconstant_names
)
1510 predicate op_non_const
;
1514 struct agg_position_info aggpos
;
1516 /* What statments might be optimized away
1517 when their arguments are constant. */
1518 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1519 && gimple_code (stmt
) != GIMPLE_COND
1520 && gimple_code (stmt
) != GIMPLE_SWITCH
1521 && (gimple_code (stmt
) != GIMPLE_CALL
1522 || !(gimple_call_flags (stmt
) & ECF_CONST
)))
1525 /* Stores will stay anyway. */
1526 if (gimple_store_p (stmt
))
1529 is_load
= gimple_assign_load_p (stmt
);
1531 /* Loads can be optimized when the value is known. */
1535 gcc_assert (gimple_assign_single_p (stmt
));
1536 op
= gimple_assign_rhs1 (stmt
);
1537 if (!unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, &base_index
, &size
,
1544 /* See if we understand all operands before we start
1545 adding conditionals. */
1546 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1548 tree parm
= unmodified_parm (stmt
, use
, NULL
);
1549 /* For arguments we can build a condition. */
1550 if (parm
&& ipa_get_param_decl_index (fbi
->info
, parm
) >= 0)
1552 if (TREE_CODE (use
) != SSA_NAME
)
1554 /* If we know when operand is constant,
1555 we still can say something useful. */
1556 if (nonconstant_names
[SSA_NAME_VERSION (use
)] != true)
1563 add_condition (summary
, base_index
, size
, &aggpos
, predicate::changed
,
1566 op_non_const
= false;
1567 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1570 tree parm
= unmodified_parm (stmt
, use
, &size
);
1573 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1575 if (index
!= base_index
)
1576 p
= add_condition (summary
, index
, size
, NULL
, predicate::changed
,
1582 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
1583 op_non_const
= p
.or_with (summary
->conds
, op_non_const
);
1585 if ((gimple_code (stmt
) == GIMPLE_ASSIGN
|| gimple_code (stmt
) == GIMPLE_CALL
)
1586 && gimple_op (stmt
, 0)
1587 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
1588 nonconstant_names
[SSA_NAME_VERSION (gimple_op (stmt
, 0))]
1590 return op_non_const
;
1593 struct record_modified_bb_info
1599 /* Value is initialized in INIT_BB and used in USE_BB. We want to copute
1600 probability how often it changes between USE_BB.
1601 INIT_BB->frequency/USE_BB->frequency is an estimate, but if INIT_BB
1602 is in different loop nest, we can do better.
1603 This is all just estimate. In theory we look for minimal cut separating
1604 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
1608 get_minimal_bb (basic_block init_bb
, basic_block use_bb
)
1610 struct loop
*l
= find_common_loop (init_bb
->loop_father
, use_bb
->loop_father
);
1611 if (l
&& l
->header
->count
< init_bb
->count
)
1616 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
1617 set except for info->stmt. */
1620 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
1622 struct record_modified_bb_info
*info
=
1623 (struct record_modified_bb_info
*) data
;
1624 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
1626 bitmap_set_bit (info
->bb_set
,
1627 SSA_NAME_IS_DEFAULT_DEF (vdef
)
1628 ? ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
1630 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1631 gimple_bb (info
->stmt
))->index
);
1635 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
1636 will change since last invocation of STMT.
1638 Value 0 is reserved for compile time invariants.
1639 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
1640 ought to be REG_BR_PROB_BASE / estimated_iters. */
1643 param_change_prob (gimple
*stmt
, int i
)
1645 tree op
= gimple_call_arg (stmt
, i
);
1646 basic_block bb
= gimple_bb (stmt
);
1648 if (TREE_CODE (op
) == WITH_SIZE_EXPR
)
1649 op
= TREE_OPERAND (op
, 0);
1651 tree base
= get_base_address (op
);
1653 /* Global invariants never change. */
1654 if (is_gimple_min_invariant (base
))
1657 /* We would have to do non-trivial analysis to really work out what
1658 is the probability of value to change (i.e. when init statement
1659 is in a sibling loop of the call).
1661 We do an conservative estimate: when call is executed N times more often
1662 than the statement defining value, we take the frequency 1/N. */
1663 if (TREE_CODE (base
) == SSA_NAME
)
1667 if (!bb
->count
.to_frequency (cfun
))
1668 return REG_BR_PROB_BASE
;
1670 if (SSA_NAME_IS_DEFAULT_DEF (base
))
1671 init_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.to_frequency (cfun
);
1673 init_freq
= get_minimal_bb
1674 (gimple_bb (SSA_NAME_DEF_STMT (base
)),
1675 gimple_bb (stmt
))->count
.to_frequency (cfun
);
1679 if (init_freq
< bb
->count
.to_frequency (cfun
))
1680 return MAX (GCOV_COMPUTE_SCALE (init_freq
,
1681 bb
->count
.to_frequency (cfun
)), 1);
1683 return REG_BR_PROB_BASE
;
1689 struct record_modified_bb_info info
;
1692 tree init
= ctor_for_folding (base
);
1694 if (init
!= error_mark_node
)
1696 if (!bb
->count
.to_frequency (cfun
))
1697 return REG_BR_PROB_BASE
;
1698 ao_ref_init (&refd
, op
);
1700 info
.bb_set
= BITMAP_ALLOC (NULL
);
1701 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
1703 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
1705 BITMAP_FREE (info
.bb_set
);
1706 return REG_BR_PROB_BASE
;
1709 /* Assume that every memory is initialized at entry.
1710 TODO: Can we easilly determine if value is always defined
1711 and thus we may skip entry block? */
1712 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.to_frequency (cfun
))
1713 max
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.to_frequency (cfun
);
1717 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
1718 max
= MIN (max
, BASIC_BLOCK_FOR_FN (cfun
, index
)->count
.to_frequency (cfun
));
1720 BITMAP_FREE (info
.bb_set
);
1721 if (max
< bb
->count
.to_frequency (cfun
))
1722 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->count
.to_frequency (cfun
)), 1);
1724 return REG_BR_PROB_BASE
;
1728 /* Find whether a basic block BB is the final block of a (half) diamond CFG
1729 sub-graph and if the predicate the condition depends on is known. If so,
1730 return true and store the pointer the predicate in *P. */
1733 phi_result_unknown_predicate (struct ipa_node_params
*info
,
1734 ipa_fn_summary
*summary
, basic_block bb
,
1736 vec
<predicate
> nonconstant_names
)
1740 basic_block first_bb
= NULL
;
1743 if (single_pred_p (bb
))
1749 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1751 if (single_succ_p (e
->src
))
1753 if (!single_pred_p (e
->src
))
1756 first_bb
= single_pred (e
->src
);
1757 else if (single_pred (e
->src
) != first_bb
)
1764 else if (e
->src
!= first_bb
)
1772 stmt
= last_stmt (first_bb
);
1774 || gimple_code (stmt
) != GIMPLE_COND
1775 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
1778 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
1779 gimple_cond_lhs (stmt
),
1787 /* Given a PHI statement in a function described by inline properties SUMMARY
1788 and *P being the predicate describing whether the selected PHI argument is
1789 known, store a predicate for the result of the PHI statement into
1790 NONCONSTANT_NAMES, if possible. */
1793 predicate_for_phi_result (struct ipa_fn_summary
*summary
, gphi
*phi
,
1795 vec
<predicate
> nonconstant_names
)
1799 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
1801 tree arg
= gimple_phi_arg (phi
, i
)->def
;
1802 if (!is_gimple_min_invariant (arg
))
1804 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
1805 *p
= p
->or_with (summary
->conds
,
1806 nonconstant_names
[SSA_NAME_VERSION (arg
)]);
1812 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1814 fprintf (dump_file
, "\t\tphi predicate: ");
1815 p
->dump (dump_file
, summary
->conds
);
1817 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
1820 /* Return predicate specifying when array index in access OP becomes non-constant. */
1823 array_index_predicate (ipa_fn_summary
*info
,
1824 vec
< predicate
> nonconstant_names
, tree op
)
1826 predicate p
= false;
1827 while (handled_component_p (op
))
1829 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
1831 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
1832 p
= p
.or_with (info
->conds
,
1833 nonconstant_names
[SSA_NAME_VERSION
1834 (TREE_OPERAND (op
, 1))]);
1836 op
= TREE_OPERAND (op
, 0);
1841 /* For a typical usage of __builtin_expect (a<b, 1), we
1842 may introduce an extra relation stmt:
1843 With the builtin, we have
1846 t3 = __builtin_expect (t2, 1);
1849 Without the builtin, we have
1852 This affects the size/time estimation and may have
1853 an impact on the earlier inlining.
1854 Here find this pattern and fix it up later. */
1857 find_foldable_builtin_expect (basic_block bb
)
1859 gimple_stmt_iterator bsi
;
1861 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1863 gimple
*stmt
= gsi_stmt (bsi
);
1864 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
)
1865 || gimple_call_internal_p (stmt
, IFN_BUILTIN_EXPECT
))
1867 tree var
= gimple_call_lhs (stmt
);
1868 tree arg
= gimple_call_arg (stmt
, 0);
1869 use_operand_p use_p
;
1876 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
1878 while (TREE_CODE (arg
) == SSA_NAME
)
1880 gimple
*stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
1881 if (!is_gimple_assign (stmt_tmp
))
1883 switch (gimple_assign_rhs_code (stmt_tmp
))
1902 arg
= gimple_assign_rhs1 (stmt_tmp
);
1905 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
1906 && gimple_code (use_stmt
) == GIMPLE_COND
)
1913 /* Return true when the basic blocks contains only clobbers followed by RESX.
1914 Such BBs are kept around to make removal of dead stores possible with
1915 presence of EH and will be optimized out by optimize_clobbers later in the
1918 NEED_EH is used to recurse in case the clobber has non-EH predecestors
1919 that can be clobber only, too.. When it is false, the RESX is not necessary
1920 on the end of basic block. */
1923 clobber_only_eh_bb_p (basic_block bb
, bool need_eh
= true)
1925 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1931 if (gsi_end_p (gsi
))
1933 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_RESX
)
1937 else if (!single_succ_p (bb
))
1940 for (; !gsi_end_p (gsi
); gsi_prev (&gsi
))
1942 gimple
*stmt
= gsi_stmt (gsi
);
1943 if (is_gimple_debug (stmt
))
1945 if (gimple_clobber_p (stmt
))
1947 if (gimple_code (stmt
) == GIMPLE_LABEL
)
1952 /* See if all predecestors are either throws or clobber only BBs. */
1953 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1954 if (!(e
->flags
& EDGE_EH
)
1955 && !clobber_only_eh_bb_p (e
->src
, false))
1961 /* Return true if STMT compute a floating point expression that may be affected
1962 by -ffast-math and similar flags. */
1965 fp_expression_p (gimple
*stmt
)
1970 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_DEF
|SSA_OP_USE
)
1971 if (FLOAT_TYPE_P (TREE_TYPE (op
)))
1976 /* Analyze function body for NODE.
1977 EARLY indicates run from early optimization pipeline. */
1980 analyze_function_body (struct cgraph_node
*node
, bool early
)
1983 /* Estimate static overhead for function prologue/epilogue and alignment. */
1985 /* Benefits are scaled by probability of elimination that is in range
1988 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1990 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
1991 predicate bb_predicate
;
1992 struct ipa_func_body_info fbi
;
1993 vec
<predicate
> nonconstant_names
= vNULL
;
1996 predicate array_index
= true;
1997 gimple
*fix_builtin_expect_stmt
;
1999 gcc_assert (my_function
&& my_function
->cfg
);
2000 gcc_assert (cfun
== my_function
);
2002 memset(&fbi
, 0, sizeof(fbi
));
2004 info
->size_time_table
= NULL
;
2006 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2007 so we can produce proper inline hints.
2009 When optimizing and analyzing for early inliner, initialize node params
2010 so we can produce correct BB predicates. */
2012 if (opt_for_fn (node
->decl
, optimize
))
2014 calculate_dominance_info (CDI_DOMINATORS
);
2016 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2019 ipa_check_create_node_params ();
2020 ipa_initialize_node_params (node
);
2023 if (ipa_node_params_sum
)
2026 fbi
.info
= IPA_NODE_REF (node
);
2027 fbi
.bb_infos
= vNULL
;
2028 fbi
.bb_infos
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
2029 fbi
.param_count
= count_formal_params(node
->decl
);
2030 nonconstant_names
.safe_grow_cleared
2031 (SSANAMES (my_function
)->length ());
2036 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2039 /* When we run into maximal number of entries, we assign everything to the
2040 constant truth case. Be sure to have it in list. */
2041 bb_predicate
= true;
2042 info
->account_size_time (0, 0, bb_predicate
, bb_predicate
);
2044 bb_predicate
= predicate::not_inlined ();
2045 info
->account_size_time (2 * ipa_fn_summary::size_scale
, 0, bb_predicate
,
2049 compute_bb_predicates (&fbi
, node
, info
);
2050 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2051 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2052 for (n
= 0; n
< nblocks
; n
++)
2054 bb
= BASIC_BLOCK_FOR_FN (cfun
, order
[n
]);
2055 freq
= bb
->count
.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
);
2056 if (clobber_only_eh_bb_p (bb
))
2058 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2059 fprintf (dump_file
, "\n Ignoring BB %i;"
2060 " it will be optimized away by cleanup_clobbers\n",
2065 /* TODO: Obviously predicates can be propagated down across CFG. */
2069 bb_predicate
= *(predicate
*) bb
->aux
;
2071 bb_predicate
= false;
2074 bb_predicate
= true;
2076 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2078 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2079 bb_predicate
.dump (dump_file
, info
->conds
);
2082 if (fbi
.info
&& nonconstant_names
.exists ())
2084 predicate phi_predicate
;
2085 bool first_phi
= true;
2087 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
2091 && !phi_result_unknown_predicate (fbi
.info
, info
, bb
,
2096 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2098 fprintf (dump_file
, " ");
2099 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0);
2101 predicate_for_phi_result (info
, bsi
.phi (), &phi_predicate
,
2106 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2108 for (gimple_stmt_iterator bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
);
2111 gimple
*stmt
= gsi_stmt (bsi
);
2112 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2113 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2115 predicate will_be_nonconstant
;
2117 /* This relation stmt should be folded after we remove
2118 buildin_expect call. Adjust the cost here. */
2119 if (stmt
== fix_builtin_expect_stmt
)
2125 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2127 fprintf (dump_file
, " ");
2128 print_gimple_stmt (dump_file
, stmt
, 0);
2129 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2130 freq
.to_double (), this_size
,
2134 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2136 predicate this_array_index
;
2138 array_index_predicate (info
, nonconstant_names
,
2139 gimple_assign_rhs1 (stmt
));
2140 if (this_array_index
!= false)
2141 array_index
&= this_array_index
;
2143 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2145 predicate this_array_index
;
2147 array_index_predicate (info
, nonconstant_names
,
2148 gimple_get_lhs (stmt
));
2149 if (this_array_index
!= false)
2150 array_index
&= this_array_index
;
2154 if (is_gimple_call (stmt
)
2155 && !gimple_call_internal_p (stmt
))
2157 struct cgraph_edge
*edge
= node
->get_edge (stmt
);
2158 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
2160 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2161 resolved as constant. We however don't want to optimize
2162 out the cgraph edges. */
2163 if (nonconstant_names
.exists ()
2164 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2165 && gimple_call_lhs (stmt
)
2166 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2168 predicate false_p
= false;
2169 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2172 if (ipa_node_params_sum
)
2174 int count
= gimple_call_num_args (stmt
);
2178 es
->param
.safe_grow_cleared (count
);
2179 for (i
= 0; i
< count
; i
++)
2181 int prob
= param_change_prob (stmt
, i
);
2182 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2183 es
->param
[i
].change_prob
= prob
;
2187 es
->call_stmt_size
= this_size
;
2188 es
->call_stmt_time
= this_time
;
2189 es
->loop_depth
= bb_loop_depth (bb
);
2190 edge_set_predicate (edge
, &bb_predicate
);
2193 /* TODO: When conditional jump or swithc is known to be constant, but
2194 we did not translate it into the predicates, we really can account
2195 just maximum of the possible paths. */
2198 = will_be_nonconstant_predicate (&fbi
, info
,
2199 stmt
, nonconstant_names
);
2201 will_be_nonconstant
= true;
2202 if (this_time
|| this_size
)
2204 sreal final_time
= (sreal
)this_time
* freq
;
2206 prob
= eliminated_by_inlining_prob (stmt
);
2207 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2209 "\t\t50%% will be eliminated by inlining\n");
2210 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2211 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2213 struct predicate p
= bb_predicate
& will_be_nonconstant
;
2215 /* We can ignore statement when we proved it is never going
2216 to happen, but we can not do that for call statements
2217 because edges are accounted specially. */
2219 if (*(is_gimple_call (stmt
) ? &bb_predicate
: &p
) != false)
2225 /* We account everything but the calls. Calls have their own
2226 size/time info attached to cgraph edges. This is necessary
2227 in order to make the cost disappear after inlining. */
2228 if (!is_gimple_call (stmt
))
2232 predicate ip
= bb_predicate
& predicate::not_inlined ();
2233 info
->account_size_time (this_size
* prob
,
2234 (this_time
* prob
) / 2, ip
,
2238 info
->account_size_time (this_size
* (2 - prob
),
2239 (this_time
* (2 - prob
) / 2),
2244 if (!info
->fp_expressions
&& fp_expression_p (stmt
))
2246 info
->fp_expressions
= true;
2248 fprintf (dump_file
, " fp_expression set\n");
2251 gcc_assert (time
>= 0);
2252 gcc_assert (size
>= 0);
2256 set_hint_predicate (&ipa_fn_summaries
->get (node
)->array_index
, array_index
);
2259 if (nonconstant_names
.exists () && !early
)
2262 predicate loop_iterations
= true;
2263 predicate loop_stride
= true;
2265 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2266 flow_loops_dump (dump_file
, NULL
, 0);
2268 FOR_EACH_LOOP (loop
, 0)
2273 struct tree_niter_desc niter_desc
;
2274 bb_predicate
= *(predicate
*) loop
->header
->aux
;
2276 exits
= get_loop_exit_edges (loop
);
2277 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2278 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2279 && !is_gimple_min_invariant (niter_desc
.niter
))
2281 predicate will_be_nonconstant
2282 = will_be_nonconstant_expr_predicate (fbi
.info
, info
,
2285 if (will_be_nonconstant
!= true)
2286 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2287 if (will_be_nonconstant
!= true
2288 && will_be_nonconstant
!= false)
2289 /* This is slightly inprecise. We may want to represent each
2290 loop with independent predicate. */
2291 loop_iterations
&= will_be_nonconstant
;
2296 /* To avoid quadratic behavior we analyze stride predicates only
2297 with respect to the containing loop. Thus we simply iterate
2298 over all defs in the outermost loop body. */
2299 for (loop
= loops_for_fn (cfun
)->tree_root
->inner
;
2300 loop
!= NULL
; loop
= loop
->next
)
2302 basic_block
*body
= get_loop_body (loop
);
2303 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
2305 gimple_stmt_iterator gsi
;
2306 bb_predicate
= *(predicate
*) body
[i
]->aux
;
2307 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2310 gimple
*stmt
= gsi_stmt (gsi
);
2312 if (!is_gimple_assign (stmt
))
2315 tree def
= gimple_assign_lhs (stmt
);
2316 if (TREE_CODE (def
) != SSA_NAME
)
2320 if (!simple_iv (loop_containing_stmt (stmt
),
2321 loop_containing_stmt (stmt
),
2323 || is_gimple_min_invariant (iv
.step
))
2326 predicate will_be_nonconstant
2327 = will_be_nonconstant_expr_predicate (fbi
.info
, info
,
2330 if (will_be_nonconstant
!= true)
2331 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2332 if (will_be_nonconstant
!= true
2333 && will_be_nonconstant
!= false)
2334 /* This is slightly inprecise. We may want to represent
2335 each loop with independent predicate. */
2336 loop_stride
= loop_stride
& will_be_nonconstant
;
2341 set_hint_predicate (&ipa_fn_summaries
->get (node
)->loop_iterations
,
2343 set_hint_predicate (&ipa_fn_summaries
->get (node
)->loop_stride
,
2347 FOR_ALL_BB_FN (bb
, my_function
)
2353 edge_predicate_pool
.remove ((predicate
*)bb
->aux
);
2355 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2358 edge_predicate_pool
.remove ((predicate
*) e
->aux
);
2362 ipa_fn_summaries
->get (node
)->time
= time
;
2363 ipa_fn_summaries
->get (node
)->self_size
= size
;
2364 nonconstant_names
.release ();
2365 ipa_release_body_info (&fbi
);
2366 if (opt_for_fn (node
->decl
, optimize
))
2369 loop_optimizer_finalize ();
2370 else if (!ipa_edge_args_sum
)
2371 ipa_free_all_node_params ();
2372 free_dominance_info (CDI_DOMINATORS
);
2376 fprintf (dump_file
, "\n");
2377 ipa_dump_fn_summary (dump_file
, node
);
2382 /* Compute function summary.
2383 EARLY is true when we compute parameters during early opts. */
2386 compute_fn_summary (struct cgraph_node
*node
, bool early
)
2388 HOST_WIDE_INT self_stack_size
;
2389 struct cgraph_edge
*e
;
2390 struct ipa_fn_summary
*info
;
2392 gcc_assert (!node
->global
.inlined_to
);
2394 if (!ipa_fn_summaries
)
2395 ipa_fn_summary_alloc ();
2397 info
= ipa_fn_summaries
->get (node
);
2400 /* Estimate the stack size for the function if we're optimizing. */
2401 self_stack_size
= optimize
&& !node
->thunk
.thunk_p
2402 ? estimated_stack_frame_size (node
) : 0;
2403 info
->estimated_self_stack_size
= self_stack_size
;
2404 info
->estimated_stack_size
= self_stack_size
;
2405 info
->stack_frame_offset
= 0;
2407 if (node
->thunk
.thunk_p
)
2409 struct ipa_call_summary
*es
= ipa_call_summaries
->get (node
->callees
);
2412 node
->local
.can_change_signature
= false;
2413 es
->call_stmt_size
= eni_size_weights
.call_cost
;
2414 es
->call_stmt_time
= eni_time_weights
.call_cost
;
2415 info
->account_size_time (ipa_fn_summary::size_scale
* 2, 2, t
, t
);
2416 t
= predicate::not_inlined ();
2417 info
->account_size_time (2 * ipa_fn_summary::size_scale
, 0, t
, t
);
2418 ipa_update_overall_fn_summary (node
);
2419 info
->self_size
= info
->size
;
2420 /* We can not inline instrumentation clones. */
2421 if (node
->thunk
.add_pointer_bounds_args
)
2423 info
->inlinable
= false;
2424 node
->callees
->inline_failed
= CIF_CHKP
;
2427 info
->inlinable
= true;
2431 /* Even is_gimple_min_invariant rely on current_function_decl. */
2432 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2434 /* Can this function be inlined at all? */
2435 if (!opt_for_fn (node
->decl
, optimize
)
2436 && !lookup_attribute ("always_inline",
2437 DECL_ATTRIBUTES (node
->decl
)))
2438 info
->inlinable
= false;
2440 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2442 info
->contains_cilk_spawn
= fn_contains_cilk_spawn_p (cfun
);
2444 /* Type attributes can use parameter indices to describe them. */
2445 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
2446 node
->local
.can_change_signature
= false;
2449 /* Otherwise, inlinable functions always can change signature. */
2450 if (info
->inlinable
)
2451 node
->local
.can_change_signature
= true;
2454 /* Functions calling builtin_apply can not change signature. */
2455 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2457 tree
cdecl = e
->callee
->decl
;
2458 if (DECL_BUILT_IN (cdecl)
2459 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2460 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2461 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2464 node
->local
.can_change_signature
= !e
;
2467 /* Functions called by instrumentation thunk can't change signature
2468 because instrumentation thunk modification is not supported. */
2469 if (node
->local
.can_change_signature
)
2470 for (e
= node
->callers
; e
; e
= e
->next_caller
)
2471 if (e
->caller
->thunk
.thunk_p
2472 && e
->caller
->thunk
.add_pointer_bounds_args
)
2474 node
->local
.can_change_signature
= false;
2477 analyze_function_body (node
, early
);
2480 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2481 if (e
->callee
->comdat_local_p ())
2483 node
->calls_comdat_local
= (e
!= NULL
);
2485 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2486 info
->size
= info
->self_size
;
2487 info
->stack_frame_offset
= 0;
2488 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2490 /* Code above should compute exactly the same result as
2491 ipa_update_overall_fn_summary but because computation happens in
2492 different order the roundoff errors result in slight changes. */
2493 ipa_update_overall_fn_summary (node
);
2494 gcc_assert (info
->size
== info
->self_size
);
2498 /* Compute parameters of functions used by inliner using
2499 current_function_decl. */
2502 compute_fn_summary_for_current (void)
2504 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
2508 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
2509 KNOWN_CONTEXTS and KNOWN_AGGS. */
2512 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2513 int *size
, int *time
,
2514 vec
<tree
> known_vals
,
2515 vec
<ipa_polymorphic_call_context
> known_contexts
,
2516 vec
<ipa_agg_jump_function_p
> known_aggs
)
2519 struct cgraph_node
*callee
;
2520 struct ipa_fn_summary
*isummary
;
2521 enum availability avail
;
2524 if (!known_vals
.exists () && !known_contexts
.exists ())
2526 if (!opt_for_fn (ie
->caller
->decl
, flag_indirect_inlining
))
2529 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_contexts
,
2530 known_aggs
, &speculative
);
2531 if (!target
|| speculative
)
2534 /* Account for difference in cost between indirect and direct calls. */
2535 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2536 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2537 gcc_checking_assert (*time
>= 0);
2538 gcc_checking_assert (*size
>= 0);
2540 callee
= cgraph_node::get (target
);
2541 if (!callee
|| !callee
->definition
)
2543 callee
= callee
->function_symbol (&avail
);
2544 if (avail
< AVAIL_AVAILABLE
)
2546 isummary
= ipa_fn_summaries
->get (callee
);
2547 return isummary
->inlinable
;
2550 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
2551 handle edge E with probability PROB.
2552 Set HINTS if edge may be devirtualized.
2553 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
2557 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *min_size
,
2560 vec
<tree
> known_vals
,
2561 vec
<ipa_polymorphic_call_context
> known_contexts
,
2562 vec
<ipa_agg_jump_function_p
> known_aggs
,
2565 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2566 int call_size
= es
->call_stmt_size
;
2567 int call_time
= es
->call_stmt_time
;
2570 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2571 known_vals
, known_contexts
, known_aggs
)
2572 && hints
&& e
->maybe_hot_p ())
2573 *hints
|= INLINE_HINT_indirect_call
;
2574 cur_size
= call_size
* ipa_fn_summary::size_scale
;
2577 *min_size
+= cur_size
;
2578 if (prob
== REG_BR_PROB_BASE
)
2579 *time
+= ((sreal
)call_time
) * e
->sreal_frequency ();
2581 *time
+= ((sreal
)call_time
* prob
) * e
->sreal_frequency ();
2586 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
2587 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2588 describe context of the call site. */
2591 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
,
2592 int *min_size
, sreal
*time
,
2594 clause_t possible_truths
,
2595 vec
<tree
> known_vals
,
2596 vec
<ipa_polymorphic_call_context
> known_contexts
,
2597 vec
<ipa_agg_jump_function_p
> known_aggs
)
2599 struct cgraph_edge
*e
;
2600 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2602 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2604 /* Do not care about zero sized builtins. */
2605 if (e
->inline_failed
&& !es
->call_stmt_size
)
2607 gcc_checking_assert (!es
->call_stmt_time
);
2611 || es
->predicate
->evaluate (possible_truths
))
2613 if (e
->inline_failed
)
2615 /* Predicates of calls shall not use NOT_CHANGED codes,
2616 sowe do not need to compute probabilities. */
2617 estimate_edge_size_and_time (e
, size
,
2618 es
->predicate
? NULL
: min_size
,
2619 time
, REG_BR_PROB_BASE
,
2620 known_vals
, known_contexts
,
2624 estimate_calls_size_and_time (e
->callee
, size
, min_size
, time
,
2627 known_vals
, known_contexts
,
2631 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2633 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2635 || es
->predicate
->evaluate (possible_truths
))
2636 estimate_edge_size_and_time (e
, size
,
2637 es
->predicate
? NULL
: min_size
,
2638 time
, REG_BR_PROB_BASE
,
2639 known_vals
, known_contexts
, known_aggs
,
2645 /* Estimate size and time needed to execute NODE assuming
2646 POSSIBLE_TRUTHS clause, and KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2647 information about NODE's arguments. If non-NULL use also probability
2648 information present in INLINE_PARAM_SUMMARY vector.
2649 Additionally detemine hints determined by the context. Finally compute
2650 minimal size needed for the call that is independent on the call context and
2651 can be used for fast estimates. Return the values in RET_SIZE,
2652 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
2655 estimate_node_size_and_time (struct cgraph_node
*node
,
2656 clause_t possible_truths
,
2657 clause_t nonspec_possible_truths
,
2658 vec
<tree
> known_vals
,
2659 vec
<ipa_polymorphic_call_context
> known_contexts
,
2660 vec
<ipa_agg_jump_function_p
> known_aggs
,
2661 int *ret_size
, int *ret_min_size
,
2663 sreal
*ret_nonspecialized_time
,
2664 ipa_hints
*ret_hints
,
2665 vec
<inline_param_summary
>
2666 inline_param_summary
)
2668 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
2673 ipa_hints hints
= 0;
2676 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2679 fprintf (dump_file
, " Estimating body: %s/%i\n"
2680 " Known to be false: ", node
->name (),
2683 for (i
= predicate::not_inlined_condition
;
2684 i
< (predicate::first_dynamic_condition
2685 + (int) vec_safe_length (info
->conds
)); i
++)
2686 if (!(possible_truths
& (1 << i
)))
2689 fprintf (dump_file
, ", ");
2691 dump_condition (dump_file
, info
->conds
, i
);
2695 estimate_calls_size_and_time (node
, &size
, &min_size
, &time
, &hints
, possible_truths
,
2696 known_vals
, known_contexts
, known_aggs
);
2697 sreal nonspecialized_time
= time
;
2699 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
2701 bool exec
= e
->exec_predicate
.evaluate (nonspec_possible_truths
);
2703 /* Because predicates are conservative, it can happen that nonconst is 1
2707 bool nonconst
= e
->nonconst_predicate
.evaluate (possible_truths
);
2709 gcc_checking_assert (e
->time
>= 0);
2710 gcc_checking_assert (time
>= 0);
2712 /* We compute specialized size only because size of nonspecialized
2713 copy is context independent.
2715 The difference between nonspecialized execution and specialized is
2716 that nonspecialized is not going to have optimized out computations
2717 known to be constant in a specialized setting. */
2720 nonspecialized_time
+= e
->time
;
2723 else if (!inline_param_summary
.exists ())
2730 int prob
= e
->nonconst_predicate
.probability
2731 (info
->conds
, possible_truths
,
2732 inline_param_summary
);
2733 gcc_checking_assert (prob
>= 0);
2734 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
2735 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
2737 gcc_checking_assert (time
>= 0);
2740 gcc_checking_assert ((*info
->size_time_table
)[0].exec_predicate
== true);
2741 gcc_checking_assert ((*info
->size_time_table
)[0].nonconst_predicate
== true);
2742 min_size
= (*info
->size_time_table
)[0].size
;
2743 gcc_checking_assert (size
>= 0);
2744 gcc_checking_assert (time
>= 0);
2745 /* nonspecialized_time should be always bigger than specialized time.
2746 Roundoff issues however may get into the way. */
2747 gcc_checking_assert ((nonspecialized_time
- time
) >= -1);
2749 /* Roundoff issues may make specialized time bigger than nonspecialized
2750 time. We do not really want that to happen because some heurstics
2751 may get confused by seeing negative speedups. */
2752 if (time
> nonspecialized_time
)
2753 time
= nonspecialized_time
;
2755 if (info
->loop_iterations
2756 && !info
->loop_iterations
->evaluate (possible_truths
))
2757 hints
|= INLINE_HINT_loop_iterations
;
2758 if (info
->loop_stride
2759 && !info
->loop_stride
->evaluate (possible_truths
))
2760 hints
|= INLINE_HINT_loop_stride
;
2761 if (info
->array_index
2762 && !info
->array_index
->evaluate (possible_truths
))
2763 hints
|= INLINE_HINT_array_index
;
2765 hints
|= INLINE_HINT_in_scc
;
2766 if (DECL_DECLARED_INLINE_P (node
->decl
))
2767 hints
|= INLINE_HINT_declared_inline
;
2769 size
= RDIV (size
, ipa_fn_summary::size_scale
);
2770 min_size
= RDIV (min_size
, ipa_fn_summary::size_scale
);
2772 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2773 fprintf (dump_file
, "\n size:%i time:%f nonspec time:%f\n", (int) size
,
2774 time
.to_double (), nonspecialized_time
.to_double ());
2777 if (ret_nonspecialized_time
)
2778 *ret_nonspecialized_time
= nonspecialized_time
;
2782 *ret_min_size
= min_size
;
2789 /* Estimate size and time needed to execute callee of EDGE assuming that
2790 parameters known to be constant at caller of EDGE are propagated.
2791 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
2792 and types for parameters. */
2795 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
2796 vec
<tree
> known_vals
,
2797 vec
<ipa_polymorphic_call_context
>
2799 vec
<ipa_agg_jump_function_p
> known_aggs
,
2800 int *ret_size
, sreal
*ret_time
,
2801 sreal
*ret_nonspec_time
,
2804 clause_t clause
, nonspec_clause
;
2806 evaluate_conditions_for_known_args (node
, false, known_vals
, known_aggs
,
2807 &clause
, &nonspec_clause
);
2808 estimate_node_size_and_time (node
, clause
, nonspec_clause
,
2809 known_vals
, known_contexts
,
2810 known_aggs
, ret_size
, NULL
, ret_time
,
2811 ret_nonspec_time
, hints
, vNULL
);
2815 /* Update summary information of inline clones after inlining.
2816 Compute peak stack usage. */
2819 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
2821 struct cgraph_edge
*e
;
2822 struct ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (node
);
2823 struct ipa_fn_summary
*caller_info
= ipa_fn_summaries
->get (node
->callers
->caller
);
2826 callee_info
->stack_frame_offset
2827 = caller_info
->stack_frame_offset
2828 + caller_info
->estimated_self_stack_size
;
2829 peak
= callee_info
->stack_frame_offset
2830 + callee_info
->estimated_self_stack_size
;
2831 if (ipa_fn_summaries
->get (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
2832 ipa_fn_summaries
->get (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
2833 ipa_propagate_frequency (node
);
2834 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2836 if (!e
->inline_failed
)
2837 inline_update_callee_summaries (e
->callee
, depth
);
2838 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
2840 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2841 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
2844 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
2845 When functoin A is inlined in B and A calls C with parameter that
2846 changes with probability PROB1 and C is known to be passthroug
2847 of argument if B that change with probability PROB2, the probability
2848 of change is now PROB1*PROB2. */
2851 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
2852 struct cgraph_edge
*edge
)
2854 if (ipa_node_params_sum
)
2857 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
2858 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
2859 struct ipa_call_summary
*inlined_es
2860 = ipa_call_summaries
->get (inlined_edge
);
2862 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
2864 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
2865 if (jfunc
->type
== IPA_JF_PASS_THROUGH
2866 || jfunc
->type
== IPA_JF_ANCESTOR
)
2868 int id
= jfunc
->type
== IPA_JF_PASS_THROUGH
2869 ? ipa_get_jf_pass_through_formal_id (jfunc
)
2870 : ipa_get_jf_ancestor_formal_id (jfunc
);
2871 if (id
< (int) inlined_es
->param
.length ())
2873 int prob1
= es
->param
[i
].change_prob
;
2874 int prob2
= inlined_es
->param
[id
].change_prob
;
2875 int prob
= combine_probabilities (prob1
, prob2
);
2877 if (prob1
&& prob2
&& !prob
)
2880 es
->param
[i
].change_prob
= prob
;
2887 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
2889 Remap predicates of callees of NODE. Rest of arguments match
2892 Also update change probabilities. */
2895 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
2896 struct cgraph_node
*node
,
2897 struct ipa_fn_summary
*info
,
2898 struct ipa_fn_summary
*callee_info
,
2899 vec
<int> operand_map
,
2900 vec
<int> offset_map
,
2901 clause_t possible_truths
,
2902 predicate
*toplev_predicate
)
2904 struct cgraph_edge
*e
, *next
;
2905 for (e
= node
->callees
; e
; e
= next
)
2907 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2909 next
= e
->next_callee
;
2911 if (e
->inline_failed
)
2913 remap_edge_change_prob (inlined_edge
, e
);
2917 p
= es
->predicate
->remap_after_inlining
2918 (info
, callee_info
, operand_map
,
2919 offset_map
, possible_truths
,
2921 edge_set_predicate (e
, &p
);
2924 edge_set_predicate (e
, toplev_predicate
);
2927 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
2928 operand_map
, offset_map
, possible_truths
,
2931 for (e
= node
->indirect_calls
; e
; e
= next
)
2933 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2935 next
= e
->next_callee
;
2937 remap_edge_change_prob (inlined_edge
, e
);
2940 p
= es
->predicate
->remap_after_inlining
2941 (info
, callee_info
, operand_map
, offset_map
,
2942 possible_truths
, *toplev_predicate
);
2943 edge_set_predicate (e
, &p
);
2946 edge_set_predicate (e
, toplev_predicate
);
2950 /* Same as remap_predicate, but set result into hint *HINT. */
2953 remap_hint_predicate (struct ipa_fn_summary
*info
,
2954 struct ipa_fn_summary
*callee_info
,
2956 vec
<int> operand_map
,
2957 vec
<int> offset_map
,
2958 clause_t possible_truths
,
2959 predicate
*toplev_predicate
)
2965 p
= (*hint
)->remap_after_inlining
2967 operand_map
, offset_map
,
2968 possible_truths
, *toplev_predicate
);
2969 if (p
!= false && p
!= true)
2972 set_hint_predicate (hint
, p
);
2978 /* We inlined EDGE. Update summary of the function we inlined into. */
2981 ipa_merge_fn_summary_after_inlining (struct cgraph_edge
*edge
)
2983 struct ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (edge
->callee
);
2984 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
2985 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
2986 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (to
);
2987 clause_t clause
= 0; /* not_inline is known to be false. */
2989 vec
<int> operand_map
= vNULL
;
2990 vec
<int> offset_map
= vNULL
;
2992 predicate toplev_predicate
;
2993 predicate true_p
= true;
2994 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
2997 toplev_predicate
= *es
->predicate
;
2999 toplev_predicate
= true;
3001 info
->fp_expressions
|= callee_info
->fp_expressions
;
3003 if (callee_info
->conds
)
3004 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
, NULL
);
3005 if (ipa_node_params_sum
&& callee_info
->conds
)
3007 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3008 int count
= ipa_get_cs_argument_count (args
);
3013 operand_map
.safe_grow_cleared (count
);
3014 offset_map
.safe_grow_cleared (count
);
3016 for (i
= 0; i
< count
; i
++)
3018 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3021 /* TODO: handle non-NOPs when merging. */
3022 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3024 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3025 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3026 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3029 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3031 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3032 if (offset
>= 0 && offset
< INT_MAX
)
3034 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3035 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3037 offset_map
[i
] = offset
;
3040 operand_map
[i
] = map
;
3041 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3044 for (i
= 0; vec_safe_iterate (callee_info
->size_time_table
, i
, &e
); i
++)
3047 p
= e
->exec_predicate
.remap_after_inlining
3048 (info
, callee_info
, operand_map
,
3051 predicate nonconstp
;
3052 nonconstp
= e
->nonconst_predicate
.remap_after_inlining
3053 (info
, callee_info
, operand_map
,
3056 if (p
!= false && nonconstp
!= false)
3058 sreal add_time
= ((sreal
)e
->time
* edge
->sreal_frequency ());
3059 int prob
= e
->nonconst_predicate
.probability (callee_info
->conds
,
3061 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
3062 if (prob
!= REG_BR_PROB_BASE
3063 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3065 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3066 (double) prob
/ REG_BR_PROB_BASE
);
3068 info
->account_size_time (e
->size
, add_time
, p
, nonconstp
);
3071 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3072 offset_map
, clause
, &toplev_predicate
);
3073 remap_hint_predicate (info
, callee_info
,
3074 &callee_info
->loop_iterations
,
3075 operand_map
, offset_map
, clause
, &toplev_predicate
);
3076 remap_hint_predicate (info
, callee_info
,
3077 &callee_info
->loop_stride
,
3078 operand_map
, offset_map
, clause
, &toplev_predicate
);
3079 remap_hint_predicate (info
, callee_info
,
3080 &callee_info
->array_index
,
3081 operand_map
, offset_map
, clause
, &toplev_predicate
);
3083 inline_update_callee_summaries (edge
->callee
,
3084 ipa_call_summaries
->get (edge
)->loop_depth
);
3086 /* We do not maintain predicates of inlined edges, free it. */
3087 edge_set_predicate (edge
, &true_p
);
3088 /* Similarly remove param summaries. */
3089 es
->param
.release ();
3090 operand_map
.release ();
3091 offset_map
.release ();
3094 /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating overall size
3095 and time. Recompute it. */
3098 ipa_update_overall_fn_summary (struct cgraph_node
*node
)
3100 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
3106 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
3108 info
->size
+= e
->size
;
3109 info
->time
+= e
->time
;
3111 estimate_calls_size_and_time (node
, &info
->size
, &info
->min_size
,
3113 ~(clause_t
) (1 << predicate::false_condition
),
3114 vNULL
, vNULL
, vNULL
);
3115 info
->size
= (info
->size
+ ipa_fn_summary::size_scale
/ 2) / ipa_fn_summary::size_scale
;
3119 /* This function performs intraprocedural analysis in NODE that is required to
3120 inline indirect calls. */
3123 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3125 ipa_analyze_node (node
);
3126 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3128 ipa_print_node_params (dump_file
, node
);
3129 ipa_print_node_jump_functions (dump_file
, node
);
3134 /* Note function body size. */
3137 inline_analyze_function (struct cgraph_node
*node
)
3139 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3142 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3143 node
->name (), node
->order
);
3144 if (opt_for_fn (node
->decl
, optimize
) && !node
->thunk
.thunk_p
)
3145 inline_indirect_intraprocedural_analysis (node
);
3146 compute_fn_summary (node
, false);
3149 struct cgraph_edge
*e
;
3150 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3151 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3152 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3153 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3160 /* Called when new function is inserted to callgraph late. */
3163 ipa_fn_summary_t::insert (struct cgraph_node
*node
, ipa_fn_summary
*)
3165 inline_analyze_function (node
);
3168 /* Note function body size. */
3171 ipa_fn_summary_generate (void)
3173 struct cgraph_node
*node
;
3175 FOR_EACH_DEFINED_FUNCTION (node
)
3176 if (DECL_STRUCT_FUNCTION (node
->decl
))
3177 node
->local
.versionable
= tree_versionable_function_p (node
->decl
);
3179 ipa_fn_summary_alloc ();
3181 ipa_fn_summaries
->enable_insertion_hook ();
3183 ipa_register_cgraph_hooks ();
3185 FOR_EACH_DEFINED_FUNCTION (node
)
3187 && (flag_generate_lto
|| flag_generate_offload
|| flag_wpa
3188 || opt_for_fn (node
->decl
, optimize
)))
3189 inline_analyze_function (node
);
3193 /* Write inline summary for edge E to OB. */
3196 read_ipa_call_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3198 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3202 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3203 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3204 es
->loop_depth
= streamer_read_uhwi (ib
);
3206 bitpack_d bp
= streamer_read_bitpack (ib
);
3207 es
->is_return_callee_uncaptured
= bp_unpack_value (&bp
, 1);
3210 edge_set_predicate (e
, &p
);
3211 length
= streamer_read_uhwi (ib
);
3214 es
->param
.safe_grow_cleared (length
);
3215 for (i
= 0; i
< length
; i
++)
3216 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3221 /* Stream in inline summaries from the section. */
3224 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3227 const struct lto_function_header
*header
=
3228 (const struct lto_function_header
*) data
;
3229 const int cfg_offset
= sizeof (struct lto_function_header
);
3230 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3231 const int string_offset
= main_offset
+ header
->main_size
;
3232 struct data_in
*data_in
;
3233 unsigned int i
, count2
, j
;
3234 unsigned int f_count
;
3236 lto_input_block
ib ((const char *) data
+ main_offset
, header
->main_size
,
3237 file_data
->mode_table
);
3240 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3241 header
->string_size
, vNULL
);
3242 f_count
= streamer_read_uhwi (&ib
);
3243 for (i
= 0; i
< f_count
; i
++)
3246 struct cgraph_node
*node
;
3247 struct ipa_fn_summary
*info
;
3248 lto_symtab_encoder_t encoder
;
3249 struct bitpack_d bp
;
3250 struct cgraph_edge
*e
;
3253 index
= streamer_read_uhwi (&ib
);
3254 encoder
= file_data
->symtab_node_encoder
;
3255 node
= dyn_cast
<cgraph_node
*> (lto_symtab_encoder_deref (encoder
,
3257 info
= ipa_fn_summaries
->get (node
);
3259 info
->estimated_stack_size
3260 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3261 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3262 info
->time
= sreal::stream_in (&ib
);
3264 bp
= streamer_read_bitpack (&ib
);
3265 info
->inlinable
= bp_unpack_value (&bp
, 1);
3266 info
->contains_cilk_spawn
= bp_unpack_value (&bp
, 1);
3267 info
->fp_expressions
= bp_unpack_value (&bp
, 1);
3269 count2
= streamer_read_uhwi (&ib
);
3270 gcc_assert (!info
->conds
);
3271 for (j
= 0; j
< count2
; j
++)
3274 c
.operand_num
= streamer_read_uhwi (&ib
);
3275 c
.size
= streamer_read_uhwi (&ib
);
3276 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3277 c
.val
= stream_read_tree (&ib
, data_in
);
3278 bp
= streamer_read_bitpack (&ib
);
3279 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3280 c
.by_ref
= bp_unpack_value (&bp
, 1);
3282 c
.offset
= streamer_read_uhwi (&ib
);
3283 vec_safe_push (info
->conds
, c
);
3285 count2
= streamer_read_uhwi (&ib
);
3286 gcc_assert (!info
->size_time_table
);
3287 for (j
= 0; j
< count2
; j
++)
3289 struct size_time_entry e
;
3291 e
.size
= streamer_read_uhwi (&ib
);
3292 e
.time
= sreal::stream_in (&ib
);
3293 e
.exec_predicate
.stream_in (&ib
);
3294 e
.nonconst_predicate
.stream_in (&ib
);
3296 vec_safe_push (info
->size_time_table
, e
);
3300 set_hint_predicate (&info
->loop_iterations
, p
);
3302 set_hint_predicate (&info
->loop_stride
, p
);
3304 set_hint_predicate (&info
->array_index
, p
);
3305 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3306 read_ipa_call_summary (&ib
, e
);
3307 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3308 read_ipa_call_summary (&ib
, e
);
3311 lto_free_section_data (file_data
, LTO_section_ipa_fn_summary
, NULL
, data
,
3313 lto_data_in_delete (data_in
);
3317 /* Read inline summary. Jump functions are shared among ipa-cp
3318 and inliner, so when ipa-cp is active, we don't need to write them
3322 ipa_fn_summary_read (void)
3324 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3325 struct lto_file_decl_data
*file_data
;
3328 ipa_fn_summary_alloc ();
3330 while ((file_data
= file_data_vec
[j
++]))
3333 const char *data
= lto_get_section_data (file_data
,
3334 LTO_section_ipa_fn_summary
,
3337 inline_read_section (file_data
, data
, len
);
3339 /* Fatal error here. We do not want to support compiling ltrans units
3340 with different version of compiler or different flags than the WPA
3341 unit, so this should never happen. */
3342 fatal_error (input_location
,
3343 "ipa inline summary is missing in input file");
3345 ipa_register_cgraph_hooks ();
3347 ipa_prop_read_jump_functions ();
3349 gcc_assert (ipa_fn_summaries
);
3350 ipa_fn_summaries
->enable_insertion_hook ();
3354 /* Write inline summary for edge E to OB. */
3357 write_ipa_call_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
3359 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3362 streamer_write_uhwi (ob
, es
->call_stmt_size
);
3363 streamer_write_uhwi (ob
, es
->call_stmt_time
);
3364 streamer_write_uhwi (ob
, es
->loop_depth
);
3366 bitpack_d bp
= bitpack_create (ob
->main_stream
);
3367 bp_pack_value (&bp
, es
->is_return_callee_uncaptured
, 1);
3368 streamer_write_bitpack (&bp
);
3371 es
->predicate
->stream_out (ob
);
3373 streamer_write_uhwi (ob
, 0);
3374 streamer_write_uhwi (ob
, es
->param
.length ());
3375 for (i
= 0; i
< (int) es
->param
.length (); i
++)
3376 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
3380 /* Write inline summary for node in SET.
3381 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
3382 active, we don't need to write them twice. */
3385 ipa_fn_summary_write (void)
3387 struct output_block
*ob
= create_output_block (LTO_section_ipa_fn_summary
);
3388 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
3389 unsigned int count
= 0;
3392 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3394 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
3395 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
3396 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
3399 streamer_write_uhwi (ob
, count
);
3401 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3403 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
3404 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
3405 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
3407 struct ipa_fn_summary
*info
= ipa_fn_summaries
->get (cnode
);
3408 struct bitpack_d bp
;
3409 struct cgraph_edge
*edge
;
3412 struct condition
*c
;
3414 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, cnode
));
3415 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
3416 streamer_write_hwi (ob
, info
->self_size
);
3417 info
->time
.stream_out (ob
);
3418 bp
= bitpack_create (ob
->main_stream
);
3419 bp_pack_value (&bp
, info
->inlinable
, 1);
3420 bp_pack_value (&bp
, info
->contains_cilk_spawn
, 1);
3421 bp_pack_value (&bp
, info
->fp_expressions
, 1);
3422 streamer_write_bitpack (&bp
);
3423 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
3424 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
3426 streamer_write_uhwi (ob
, c
->operand_num
);
3427 streamer_write_uhwi (ob
, c
->size
);
3428 streamer_write_uhwi (ob
, c
->code
);
3429 stream_write_tree (ob
, c
->val
, true);
3430 bp
= bitpack_create (ob
->main_stream
);
3431 bp_pack_value (&bp
, c
->agg_contents
, 1);
3432 bp_pack_value (&bp
, c
->by_ref
, 1);
3433 streamer_write_bitpack (&bp
);
3434 if (c
->agg_contents
)
3435 streamer_write_uhwi (ob
, c
->offset
);
3437 streamer_write_uhwi (ob
, vec_safe_length (info
->size_time_table
));
3438 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
3440 streamer_write_uhwi (ob
, e
->size
);
3441 e
->time
.stream_out (ob
);
3442 e
->exec_predicate
.stream_out (ob
);
3443 e
->nonconst_predicate
.stream_out (ob
);
3445 if (info
->loop_iterations
)
3446 info
->loop_iterations
->stream_out (ob
);
3448 streamer_write_uhwi (ob
, 0);
3449 if (info
->loop_stride
)
3450 info
->loop_stride
->stream_out (ob
);
3452 streamer_write_uhwi (ob
, 0);
3453 if (info
->array_index
)
3454 info
->array_index
->stream_out (ob
);
3456 streamer_write_uhwi (ob
, 0);
3457 for (edge
= cnode
->callees
; edge
; edge
= edge
->next_callee
)
3458 write_ipa_call_summary (ob
, edge
);
3459 for (edge
= cnode
->indirect_calls
; edge
; edge
= edge
->next_callee
)
3460 write_ipa_call_summary (ob
, edge
);
3463 streamer_write_char_stream (ob
->main_stream
, 0);
3464 produce_asm (ob
, NULL
);
3465 destroy_output_block (ob
);
3468 ipa_prop_write_jump_functions ();
3472 /* Release inline summary. */
3475 ipa_free_fn_summary (void)
3477 struct cgraph_node
*node
;
3478 if (!ipa_call_summaries
)
3480 FOR_EACH_DEFINED_FUNCTION (node
)
3482 ipa_fn_summaries
->get (node
)->reset (node
);
3483 ipa_fn_summaries
->release ();
3484 ipa_fn_summaries
= NULL
;
3485 ipa_call_summaries
->release ();
3486 delete ipa_call_summaries
;
3487 ipa_call_summaries
= NULL
;
3488 edge_predicate_pool
.release ();
3493 const pass_data pass_data_local_fn_summary
=
3495 GIMPLE_PASS
, /* type */
3496 "local-fnsummary", /* name */
3497 OPTGROUP_INLINE
, /* optinfo_flags */
3498 TV_INLINE_PARAMETERS
, /* tv_id */
3499 0, /* properties_required */
3500 0, /* properties_provided */
3501 0, /* properties_destroyed */
3502 0, /* todo_flags_start */
3503 0, /* todo_flags_finish */
3506 class pass_local_fn_summary
: public gimple_opt_pass
3509 pass_local_fn_summary (gcc::context
*ctxt
)
3510 : gimple_opt_pass (pass_data_local_fn_summary
, ctxt
)
3513 /* opt_pass methods: */
3514 opt_pass
* clone () { return new pass_local_fn_summary (m_ctxt
); }
3515 virtual unsigned int execute (function
*)
3517 return compute_fn_summary_for_current ();
3520 }; // class pass_local_fn_summary
3525 make_pass_local_fn_summary (gcc::context
*ctxt
)
3527 return new pass_local_fn_summary (ctxt
);
3531 /* Free inline summary. */
3535 const pass_data pass_data_ipa_free_fn_summary
=
3537 SIMPLE_IPA_PASS
, /* type */
3538 "free-fnsummary", /* name */
3539 OPTGROUP_NONE
, /* optinfo_flags */
3540 TV_IPA_FREE_INLINE_SUMMARY
, /* tv_id */
3541 0, /* properties_required */
3542 0, /* properties_provided */
3543 0, /* properties_destroyed */
3544 0, /* todo_flags_start */
3545 /* Early optimizations may make function unreachable. We can not
3546 remove unreachable functions as part of the ealry opts pass because
3547 TODOs are run before subpasses. Do it here. */
3548 ( TODO_remove_functions
| TODO_dump_symtab
), /* todo_flags_finish */
3551 class pass_ipa_free_fn_summary
: public simple_ipa_opt_pass
3554 pass_ipa_free_fn_summary (gcc::context
*ctxt
)
3555 : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary
, ctxt
)
3558 /* opt_pass methods: */
3559 virtual unsigned int execute (function
*)
3561 ipa_free_fn_summary ();
3565 }; // class pass_ipa_free_fn_summary
3569 simple_ipa_opt_pass
*
3570 make_pass_ipa_free_fn_summary (gcc::context
*ctxt
)
3572 return new pass_ipa_free_fn_summary (ctxt
);
3577 const pass_data pass_data_ipa_fn_summary
=
3579 IPA_PASS
, /* type */
3580 "fnsummary", /* name */
3581 OPTGROUP_INLINE
, /* optinfo_flags */
3582 TV_IPA_FNSUMMARY
, /* tv_id */
3583 0, /* properties_required */
3584 0, /* properties_provided */
3585 0, /* properties_destroyed */
3586 0, /* todo_flags_start */
3587 ( TODO_dump_symtab
), /* todo_flags_finish */
3590 class pass_ipa_fn_summary
: public ipa_opt_pass_d
3593 pass_ipa_fn_summary (gcc::context
*ctxt
)
3594 : ipa_opt_pass_d (pass_data_ipa_fn_summary
, ctxt
,
3595 ipa_fn_summary_generate
, /* generate_summary */
3596 ipa_fn_summary_write
, /* write_summary */
3597 ipa_fn_summary_read
, /* read_summary */
3598 NULL
, /* write_optimization_summary */
3599 NULL
, /* read_optimization_summary */
3600 NULL
, /* stmt_fixup */
3601 0, /* function_transform_todo_flags_start */
3602 NULL
, /* function_transform */
3603 NULL
) /* variable_transform */
3606 /* opt_pass methods: */
3607 virtual unsigned int execute (function
*) { return 0; }
3609 }; // class pass_ipa_fn_summary
3614 make_pass_ipa_fn_summary (gcc::context
*ctxt
)
3616 return new pass_ipa_fn_summary (ctxt
);
3619 /* Reset all state within ipa-fnsummary.c so that we can rerun the compiler
3620 within the same process. For use by toplev::finalize. */
3623 ipa_fnsummary_c_finalize (void)
3625 ipa_free_fn_summary ();