1 /* Function summary pass.
2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis of function bodies used by inter-procedural passes
23 We estimate for each function
24 - function body size and size after specializing into given context
25 - average function execution time in a given context
28 - call statement size, time and how often the parameters change
30 ipa_fn_summary data structures store above information locally (i.e.
31 parameters of the function itself) and globally (i.e. parameters of
32 the function created by applying all the inline decisions already
33 present in the callgraph).
35 We provide access to the ipa_fn_summary data structure and
36 basic logic updating the parameters when inlining is performed.
38 The summaries are context sensitive. Context means
39 1) partial assignment of known constant values of operands
40 2) whether function is inlined into the call or not.
41 It is easy to add more variants. To represent function size and time
42 that depends on context (i.e. it is known to be optimized away when
43 context is known either by inlining or from IP-CP and cloning),
46 estimate_edge_size_and_time can be used to query
47 function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 properties of caller and callee after inlining.
50 Finally pass_inline_parameters is exported. This is used to drive
51 computation of function parameters used by the early inliner. IPA
52 inlined performs analysis via its analyze_function method. */
55 #define INCLUDE_VECTOR
57 #include "coretypes.h"
62 #include "alloc-pool.h"
63 #include "tree-pass.h"
65 #include "tree-streamer.h"
67 #include "diagnostic.h"
68 #include "fold-const.h"
69 #include "print-tree.h"
70 #include "tree-inline.h"
71 #include "gimple-pretty-print.h"
73 #include "gimple-iterator.h"
75 #include "tree-ssa-loop-niter.h"
76 #include "tree-ssa-loop.h"
77 #include "symbol-summary.h"
81 #include "ipa-fnsummary.h"
83 #include "tree-scalar-evolution.h"
84 #include "ipa-utils.h"
85 #include "cfgexpand.h"
87 #include "stringpool.h"
89 #include "tree-into-ssa.h"
90 #include "symtab-clones.h"
91 #include "gimple-range.h"
95 fast_function_summary
<ipa_fn_summary
*, va_gc
> *ipa_fn_summaries
;
96 fast_function_summary
<ipa_size_summary
*, va_heap
> *ipa_size_summaries
;
97 fast_call_summary
<ipa_call_summary
*, va_heap
> *ipa_call_summaries
;
99 /* Edge predicates goes here. */
100 static object_allocator
<ipa_predicate
> edge_predicate_pool ("edge predicates");
103 /* Dump IPA hints. */
105 ipa_dump_hints (FILE *f
, ipa_hints hints
)
109 fprintf (f
, "IPA hints:");
110 if (hints
& INLINE_HINT_indirect_call
)
112 hints
&= ~INLINE_HINT_indirect_call
;
113 fprintf (f
, " indirect_call");
115 if (hints
& INLINE_HINT_loop_iterations
)
117 hints
&= ~INLINE_HINT_loop_iterations
;
118 fprintf (f
, " loop_iterations");
120 if (hints
& INLINE_HINT_loop_stride
)
122 hints
&= ~INLINE_HINT_loop_stride
;
123 fprintf (f
, " loop_stride");
125 if (hints
& INLINE_HINT_same_scc
)
127 hints
&= ~INLINE_HINT_same_scc
;
128 fprintf (f
, " same_scc");
130 if (hints
& INLINE_HINT_in_scc
)
132 hints
&= ~INLINE_HINT_in_scc
;
133 fprintf (f
, " in_scc");
135 if (hints
& INLINE_HINT_cross_module
)
137 hints
&= ~INLINE_HINT_cross_module
;
138 fprintf (f
, " cross_module");
140 if (hints
& INLINE_HINT_declared_inline
)
142 hints
&= ~INLINE_HINT_declared_inline
;
143 fprintf (f
, " declared_inline");
145 if (hints
& INLINE_HINT_known_hot
)
147 hints
&= ~INLINE_HINT_known_hot
;
148 fprintf (f
, " known_hot");
150 if (hints
& INLINE_HINT_builtin_constant_p
)
152 hints
&= ~INLINE_HINT_builtin_constant_p
;
153 fprintf (f
, " builtin_constant_p");
159 /* Record SIZE and TIME to SUMMARY.
160 The accounted code will be executed when EXEC_PRED is true.
161 When NONCONST_PRED is false the code will evaluate to constant and
162 will get optimized out in specialized clones of the function.
163 If CALL is true account to call_size_time_table rather than
167 ipa_fn_summary::account_size_time (int size
, sreal time
,
168 const ipa_predicate
&exec_pred
,
169 const ipa_predicate
&nonconst_pred_in
,
175 ipa_predicate nonconst_pred
;
176 vec
<size_time_entry
> *table
= call
? &call_size_time_table
: &size_time_table
;
178 if (exec_pred
== false)
181 nonconst_pred
= nonconst_pred_in
& exec_pred
;
183 if (nonconst_pred
== false)
186 /* We need to create initial empty unconditional clause, but otherwise
187 we don't need to account empty times and sizes. */
188 if (!size
&& time
== 0 && table
->length ())
191 /* Only for calls we are unaccounting what we previously recorded. */
192 gcc_checking_assert (time
>= 0 || call
);
194 for (i
= 0; table
->iterate (i
, &e
); i
++)
195 if (e
->exec_predicate
== exec_pred
196 && e
->nonconst_predicate
== nonconst_pred
)
201 if (i
== max_size_time_table_size
)
206 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
208 "\t\tReached limit on number of entries, "
209 "ignoring the predicate.");
211 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
!= 0 || size
))
214 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
215 ((double) size
) / ipa_fn_summary::size_scale
,
216 (time
.to_double ()), found
? "" : "new ");
217 exec_pred
.dump (dump_file
, conds
, 0);
218 if (exec_pred
!= nonconst_pred
)
220 fprintf (dump_file
, " nonconst:");
221 nonconst_pred
.dump (dump_file
, conds
);
224 fprintf (dump_file
, "\n");
228 class size_time_entry new_entry
;
229 new_entry
.size
= size
;
230 new_entry
.time
= time
;
231 new_entry
.exec_predicate
= exec_pred
;
232 new_entry
.nonconst_predicate
= nonconst_pred
;
234 call_size_time_table
.safe_push (new_entry
);
236 size_time_table
.safe_push (new_entry
);
242 /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
243 /* Tolerate small roundoff issues. */
249 /* We proved E to be unreachable, redirect it to __builtin_unreachable. */
251 static struct cgraph_edge
*
252 redirect_to_unreachable (struct cgraph_edge
*e
)
254 struct cgraph_node
*callee
= !e
->inline_failed
? e
->callee
: NULL
;
255 struct cgraph_node
*target
256 = cgraph_node::get_create (builtin_decl_unreachable ());
259 e
= cgraph_edge::resolve_speculation (e
, target
->decl
);
261 e
= cgraph_edge::make_direct (e
, target
);
263 e
->redirect_callee (target
);
264 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
265 e
->inline_failed
= CIF_UNREACHABLE
;
266 e
->count
= profile_count::zero ();
267 es
->call_stmt_size
= 0;
268 es
->call_stmt_time
= 0;
270 callee
->remove_symbol_and_inline_clones ();
274 /* Set predicate for edge E. */
277 edge_set_predicate (struct cgraph_edge
*e
, ipa_predicate
*predicate
)
279 /* If the edge is determined to be never executed, redirect it
280 to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
282 if (predicate
&& *predicate
== false
283 /* When handling speculative edges, we need to do the redirection
284 just once. Do it always on the direct edge, so we do not
285 attempt to resolve speculation while duplicating the edge. */
286 && (!e
->speculative
|| e
->callee
))
287 e
= redirect_to_unreachable (e
);
289 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
290 if (predicate
&& *predicate
!= true)
293 es
->predicate
= edge_predicate_pool
.allocate ();
294 *es
->predicate
= *predicate
;
299 edge_predicate_pool
.remove (es
->predicate
);
300 es
->predicate
= NULL
;
304 /* Set predicate for hint *P. */
307 set_hint_predicate (ipa_predicate
**p
, ipa_predicate new_predicate
)
309 if (new_predicate
== false || new_predicate
== true)
312 edge_predicate_pool
.remove (*p
);
318 *p
= edge_predicate_pool
.allocate ();
323 /* Find if NEW_PREDICATE is already in V and if so, increment its freq.
324 Otherwise add a new item to the vector with this predicate and frerq equal
325 to add_freq, unless the number of predicates would exceed MAX_NUM_PREDICATES
326 in which case the function does nothing. */
329 add_freqcounting_predicate (vec
<ipa_freqcounting_predicate
, va_gc
> **v
,
330 const ipa_predicate
&new_predicate
, sreal add_freq
,
331 unsigned max_num_predicates
)
333 if (new_predicate
== false || new_predicate
== true)
335 ipa_freqcounting_predicate
*f
;
336 for (int i
= 0; vec_safe_iterate (*v
, i
, &f
); i
++)
337 if (new_predicate
== f
->predicate
)
342 if (vec_safe_length (*v
) >= max_num_predicates
)
343 /* Too many different predicates to account for. */
346 ipa_freqcounting_predicate fcp
;
347 fcp
.predicate
= NULL
;
348 set_hint_predicate (&fcp
.predicate
, new_predicate
);
350 vec_safe_push (*v
, fcp
);
354 /* Compute what conditions may or may not hold given information about
355 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
356 while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
357 copy when called in a given context. It is a bitmask of conditions. Bit
358 0 means that condition is known to be false, while bit 1 means that condition
359 may or may not be true. These differs - for example NOT_INLINED condition
360 is always false in the second and also builtin_constant_p tests cannot use
361 the fact that parameter is indeed a constant.
363 When INLINE_P is true, assume that we are inlining. AVAL contains known
364 information about argument values. The function does not modify its content
365 and so AVALs could also be of type ipa_call_arg_values but so far all
366 callers work with the auto version and so we avoid the conversion for
369 ERROR_MARK value of an argument means compile time invariant. */
372 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
374 ipa_auto_call_arg_values
*avals
,
375 clause_t
*ret_clause
,
376 clause_t
*ret_nonspec_clause
,
377 ipa_call_summary
*es
)
379 clause_t clause
= inline_p
? 0 : 1 << ipa_predicate::not_inlined_condition
;
380 clause_t nonspec_clause
= 1 << ipa_predicate::not_inlined_condition
;
381 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
385 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
390 struct expr_eval_op
*op
;
392 if (c
->code
== ipa_predicate::not_sra_candidate
)
396 || (int)es
->param
.length () <= c
->operand_num
397 || !es
->param
[c
->operand_num
].points_to_possible_sra_candidate
)
398 clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
399 nonspec_clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
405 if (c
->code
== ipa_predicate::changed
407 && (avals
->safe_sval_at(c
->operand_num
) == error_mark_node
))
410 if (tree sval
= avals
->safe_sval_at (c
->operand_num
))
411 val
= ipa_find_agg_cst_from_init (sval
, c
->offset
, c
->by_ref
);
414 ipa_argagg_value_list
avs (avals
);
415 val
= avs
.get_value (c
->operand_num
, c
->offset
/ BITS_PER_UNIT
,
421 val
= avals
->safe_sval_at (c
->operand_num
);
422 if (val
&& val
== error_mark_node
423 && c
->code
!= ipa_predicate::changed
)
428 && (c
->code
== ipa_predicate::changed
429 || c
->code
== ipa_predicate::is_not_constant
))
431 clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
432 nonspec_clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
435 if (c
->code
== ipa_predicate::changed
)
437 nonspec_clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
441 if (c
->code
== ipa_predicate::is_not_constant
)
443 nonspec_clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
447 if (val
&& TYPE_SIZE (c
->type
) == TYPE_SIZE (TREE_TYPE (val
)))
449 if (c
->type
!= TREE_TYPE (val
))
450 val
= fold_unary (VIEW_CONVERT_EXPR
, c
->type
, val
);
451 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
456 val
= fold_unary (op
->code
, op
->type
, val
);
457 else if (!op
->val
[1])
458 val
= fold_binary (op
->code
, op
->type
,
459 op
->index
? op
->val
[0] : val
,
460 op
->index
? val
: op
->val
[0]);
461 else if (op
->index
== 0)
462 val
= fold_ternary (op
->code
, op
->type
,
463 val
, op
->val
[0], op
->val
[1]);
464 else if (op
->index
== 1)
465 val
= fold_ternary (op
->code
, op
->type
,
466 op
->val
[0], val
, op
->val
[1]);
467 else if (op
->index
== 2)
468 val
= fold_ternary (op
->code
, op
->type
,
469 op
->val
[0], op
->val
[1], val
);
475 ? fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
)
478 if (res
&& integer_zerop (res
))
480 if (res
&& integer_onep (res
))
482 clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
484 |= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
488 if (c
->operand_num
< (int) avals
->m_known_value_ranges
.length ()
490 && (!val
|| TREE_CODE (val
) != INTEGER_CST
))
492 Value_Range
vr (avals
->m_known_value_ranges
[c
->operand_num
]);
493 if (!vr
.undefined_p ()
495 && (TYPE_SIZE (c
->type
) == TYPE_SIZE (vr
.type ())))
497 if (!useless_type_conversion_p (c
->type
, vr
.type ()))
498 range_cast (vr
, c
->type
);
500 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
502 if (vr
.varying_p () || vr
.undefined_p ())
505 Value_Range
res (op
->type
);
508 Value_Range
varying (op
->type
);
509 varying
.set_varying (op
->type
);
510 range_op_handler
handler (op
->code
);
512 || !res
.supports_type_p (op
->type
)
513 || !handler
.fold_range (res
, op
->type
, vr
, varying
))
514 res
.set_varying (op
->type
);
516 else if (!op
->val
[1])
518 Value_Range
op0 (op
->type
);
519 range_op_handler
handler (op
->code
);
521 ipa_range_set_and_normalize (op0
, op
->val
[0]);
524 || !res
.supports_type_p (op
->type
)
525 || !handler
.fold_range (res
, op
->type
,
526 op
->index
? op0
: vr
,
527 op
->index
? vr
: op0
))
528 res
.set_varying (op
->type
);
531 res
.set_varying (op
->type
);
534 if (!vr
.varying_p () && !vr
.undefined_p ())
537 Value_Range
val_vr (TREE_TYPE (c
->val
));
538 range_op_handler
handler (c
->code
);
540 ipa_range_set_and_normalize (val_vr
, c
->val
);
543 || !val_vr
.supports_type_p (TREE_TYPE (c
->val
))
544 || !handler
.fold_range (res
, boolean_type_node
, vr
, val_vr
))
545 res
.set_varying (boolean_type_node
);
553 clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
554 nonspec_clause
|= 1 << (i
+ ipa_predicate::first_dynamic_condition
);
556 *ret_clause
= clause
;
557 if (ret_nonspec_clause
)
558 *ret_nonspec_clause
= nonspec_clause
;
561 /* Return true if VRP will be exectued on the function.
562 We do not want to anticipate optimizations that will not happen.
564 FIXME: This can be confused with -fdisable and debug counters and thus
565 it should not be used for correctness (only to make heuristics work).
566 This means that inliner should do its own optimizations of expressions
567 that it predicts to be constant so wrong code can not be triggered by
568 builtin_constant_p. */
571 vrp_will_run_p (struct cgraph_node
*node
)
573 return (opt_for_fn (node
->decl
, optimize
)
574 && !opt_for_fn (node
->decl
, optimize_debug
)
575 && opt_for_fn (node
->decl
, flag_tree_vrp
));
578 /* Similarly about FRE. */
581 fre_will_run_p (struct cgraph_node
*node
)
583 return (opt_for_fn (node
->decl
, optimize
)
584 && !opt_for_fn (node
->decl
, optimize_debug
)
585 && opt_for_fn (node
->decl
, flag_tree_fre
));
588 /* Work out what conditions might be true at invocation of E.
589 Compute costs for inlined edge if INLINE_P is true.
591 Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
592 (if non-NULL) conditions evaluated for nonspecialized clone called
595 Vectors in AVALS will be populated with useful known information about
596 argument values - information not known to have any uses will be omitted -
597 except for m_known_contexts which will only be calculated if
598 COMPUTE_CONTEXTS is true. */
601 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
602 clause_t
*clause_ptr
,
603 clause_t
*nonspec_clause_ptr
,
604 ipa_auto_call_arg_values
*avals
,
605 bool compute_contexts
)
607 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
608 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (callee
);
609 class ipa_edge_args
*args
;
610 class ipa_call_summary
*es
= NULL
;
613 *clause_ptr
= inline_p
? 0 : 1 << ipa_predicate::not_inlined_condition
;
615 if (ipa_node_params_sum
616 && !e
->call_stmt_cannot_inline_p
617 && (info
->conds
|| compute_contexts
)
618 && (args
= ipa_edge_args_sum
->get (e
)) != NULL
)
620 struct cgraph_node
*caller
;
621 class ipa_node_params
*caller_parms_info
, *callee_pi
= NULL
;
622 int i
, count
= ipa_get_cs_argument_count (args
);
623 es
= ipa_call_summaries
->get (e
);
627 if (e
->caller
->inlined_to
)
628 caller
= e
->caller
->inlined_to
;
631 caller_parms_info
= ipa_node_params_sum
->get (caller
);
632 callee_pi
= ipa_node_params_sum
->get (callee
);
634 /* Watch for thunks. */
636 /* Watch for variadic functions. */
637 count
= MIN (count
, ipa_get_param_count (callee_pi
));
641 for (i
= 0; i
< count
; i
++)
643 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
645 if (ipa_is_param_used_by_indirect_call (callee_pi
, i
)
646 || ipa_is_param_used_by_ipa_predicates (callee_pi
, i
))
648 /* Determine if we know constant value of the parameter. */
649 tree type
= ipa_get_type (callee_pi
, i
);
650 tree cst
= ipa_value_from_jfunc (caller_parms_info
, jf
, type
);
652 if (!cst
&& e
->call_stmt
653 && i
< (int)gimple_call_num_args (e
->call_stmt
))
655 cst
= gimple_call_arg (e
->call_stmt
, i
);
656 if (!is_gimple_min_invariant (cst
))
661 gcc_checking_assert (TREE_CODE (cst
) != TREE_BINFO
);
662 if (!avals
->m_known_vals
.length ())
663 avals
->m_known_vals
.safe_grow_cleared (count
, true);
664 avals
->m_known_vals
[i
] = cst
;
666 else if (inline_p
&& !es
->param
[i
].change_prob
)
668 if (!avals
->m_known_vals
.length ())
669 avals
->m_known_vals
.safe_grow_cleared (count
, true);
670 avals
->m_known_vals
[i
] = error_mark_node
;
673 /* If we failed to get simple constant, try value range. */
674 if ((!cst
|| TREE_CODE (cst
) != INTEGER_CST
)
675 && vrp_will_run_p (caller
)
676 && ipa_is_param_used_by_ipa_predicates (callee_pi
, i
))
678 Value_Range
vr (type
);
680 ipa_value_range_from_jfunc (vr
, caller_parms_info
, e
, jf
, type
);
681 if (!vr
.undefined_p () && !vr
.varying_p ())
683 if (!avals
->m_known_value_ranges
.length ())
684 avals
->m_known_value_ranges
.safe_grow_cleared (count
,
686 avals
->m_known_value_ranges
[i
] = vr
;
690 /* Determine known aggregate values. */
691 if (fre_will_run_p (caller
))
692 ipa_push_agg_values_from_jfunc (caller_parms_info
,
694 &avals
->m_known_aggs
);
697 /* For calls used in polymorphic calls we further determine
698 polymorphic call context. */
700 && ipa_is_param_used_by_polymorphic_call (callee_pi
, i
))
702 ipa_polymorphic_call_context
703 ctx
= ipa_context_from_jfunc (caller_parms_info
, e
, i
, jf
);
704 if (!ctx
.useless_p ())
706 if (!avals
->m_known_contexts
.length ())
707 avals
->m_known_contexts
.safe_grow_cleared (count
, true);
708 avals
->m_known_contexts
[i
]
709 = ipa_context_from_jfunc (caller_parms_info
, e
, i
, jf
);
714 gcc_assert (!count
|| callee
->thunk
);
716 else if (e
->call_stmt
&& !e
->call_stmt_cannot_inline_p
&& info
->conds
)
718 int i
, count
= (int)gimple_call_num_args (e
->call_stmt
);
720 for (i
= 0; i
< count
; i
++)
722 tree cst
= gimple_call_arg (e
->call_stmt
, i
);
723 if (!is_gimple_min_invariant (cst
))
727 if (!avals
->m_known_vals
.length ())
728 avals
->m_known_vals
.safe_grow_cleared (count
, true);
729 avals
->m_known_vals
[i
] = cst
;
734 evaluate_conditions_for_known_args (callee
, inline_p
, avals
, clause_ptr
,
735 nonspec_clause_ptr
, es
);
739 /* Allocate the function summary. */
742 ipa_fn_summary_alloc (void)
744 gcc_checking_assert (!ipa_fn_summaries
);
745 ipa_size_summaries
= new ipa_size_summary_t (symtab
);
746 ipa_fn_summaries
= ipa_fn_summary_t::create_ggc (symtab
);
747 ipa_call_summaries
= new ipa_call_summary_t (symtab
);
750 ipa_call_summary::~ipa_call_summary ()
753 edge_predicate_pool
.remove (predicate
);
758 ipa_fn_summary::~ipa_fn_summary ()
760 unsigned len
= vec_safe_length (loop_iterations
);
761 for (unsigned i
= 0; i
< len
; i
++)
762 edge_predicate_pool
.remove ((*loop_iterations
)[i
].predicate
);
763 len
= vec_safe_length (loop_strides
);
764 for (unsigned i
= 0; i
< len
; i
++)
765 edge_predicate_pool
.remove ((*loop_strides
)[i
].predicate
);
767 call_size_time_table
.release ();
768 vec_free (loop_iterations
);
769 vec_free (loop_strides
);
770 builtin_constant_p_parms
.release ();
774 ipa_fn_summary_t::remove_callees (cgraph_node
*node
)
777 for (e
= node
->callees
; e
; e
= e
->next_callee
)
778 ipa_call_summaries
->remove (e
);
779 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
780 ipa_call_summaries
->remove (e
);
783 /* Duplicate predicates in loop hint vector, allocating memory for them and
784 remove and deallocate any uninteresting (true or false) ones. Return the
787 static vec
<ipa_freqcounting_predicate
, va_gc
> *
788 remap_freqcounting_preds_after_dup (vec
<ipa_freqcounting_predicate
, va_gc
> *v
,
789 clause_t possible_truths
)
791 if (vec_safe_length (v
) == 0)
794 vec
<ipa_freqcounting_predicate
, va_gc
> *res
= v
->copy ();
795 int len
= res
->length();
796 for (int i
= len
- 1; i
>= 0; i
--)
798 ipa_predicate new_predicate
799 = (*res
)[i
].predicate
->remap_after_duplication (possible_truths
);
800 /* We do not want to free previous predicate; it is used by node
802 (*res
)[i
].predicate
= NULL
;
803 set_hint_predicate (&(*res
)[i
].predicate
, new_predicate
);
805 if (!(*res
)[i
].predicate
)
806 res
->unordered_remove (i
);
813 /* Hook that is called by cgraph.cc when a node is duplicated. */
815 ipa_fn_summary_t::duplicate (cgraph_node
*src
,
817 ipa_fn_summary
*src_info
,
818 ipa_fn_summary
*info
)
820 new (info
) ipa_fn_summary (*src_info
);
821 /* TODO: as an optimization, we may avoid copying conditions
822 that are known to be false or true. */
823 info
->conds
= vec_safe_copy (info
->conds
);
825 clone_info
*cinfo
= clone_info::get (dst
);
826 /* When there are any replacements in the function body, see if we can figure
827 out that something was optimized out. */
828 if (ipa_node_params_sum
&& cinfo
&& cinfo
->tree_map
)
830 /* Use SRC parm info since it may not be copied yet. */
831 ipa_node_params
*parms_info
= ipa_node_params_sum
->get (src
);
832 ipa_auto_call_arg_values avals
;
833 int count
= ipa_get_param_count (parms_info
);
835 clause_t possible_truths
;
836 ipa_predicate true_pred
= true;
838 int optimized_out_size
= 0;
839 bool inlined_to_p
= false;
840 struct cgraph_edge
*edge
, *next
;
842 info
->size_time_table
.release ();
843 avals
.m_known_vals
.safe_grow_cleared (count
, true);
844 for (i
= 0; i
< count
; i
++)
846 struct ipa_replace_map
*r
;
848 for (j
= 0; vec_safe_iterate (cinfo
->tree_map
, j
, &r
); j
++)
850 if (r
->parm_num
== i
)
852 avals
.m_known_vals
[i
] = r
->new_tree
;
857 evaluate_conditions_for_known_args (dst
, false,
860 /* We are going to specialize,
861 so ignore nonspec truths. */
865 info
->account_size_time (0, 0, true_pred
, true_pred
);
867 /* Remap size_time vectors.
868 Simplify the predicate by pruning out alternatives that are known
870 TODO: as on optimization, we can also eliminate conditions known
872 for (i
= 0; src_info
->size_time_table
.iterate (i
, &e
); i
++)
874 ipa_predicate new_exec_pred
;
875 ipa_predicate new_nonconst_pred
;
876 new_exec_pred
= e
->exec_predicate
.remap_after_duplication
878 new_nonconst_pred
= e
->nonconst_predicate
.remap_after_duplication
880 if (new_exec_pred
== false || new_nonconst_pred
== false)
881 optimized_out_size
+= e
->size
;
883 info
->account_size_time (e
->size
, e
->time
, new_exec_pred
,
887 /* Remap edge predicates with the same simplification as above.
888 Also copy constantness arrays. */
889 for (edge
= dst
->callees
; edge
; edge
= next
)
891 ipa_predicate new_predicate
;
892 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
893 next
= edge
->next_callee
;
895 if (!edge
->inline_failed
)
899 new_predicate
= es
->predicate
->remap_after_duplication
901 if (new_predicate
== false && *es
->predicate
!= false)
902 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
903 edge_set_predicate (edge
, &new_predicate
);
906 /* Remap indirect edge predicates with the same simplification as above.
907 Also copy constantness arrays. */
908 for (edge
= dst
->indirect_calls
; edge
; edge
= next
)
910 ipa_predicate new_predicate
;
911 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
912 next
= edge
->next_callee
;
914 gcc_checking_assert (edge
->inline_failed
);
917 new_predicate
= es
->predicate
->remap_after_duplication
919 if (new_predicate
== false && *es
->predicate
!= false)
921 += es
->call_stmt_size
* ipa_fn_summary::size_scale
;
922 edge_set_predicate (edge
, &new_predicate
);
924 info
->loop_iterations
925 = remap_freqcounting_preds_after_dup (info
->loop_iterations
,
928 = remap_freqcounting_preds_after_dup (info
->loop_strides
,
930 if (info
->builtin_constant_p_parms
.length())
932 vec
<int, va_heap
, vl_ptr
> parms
= info
->builtin_constant_p_parms
;
934 info
->builtin_constant_p_parms
= vNULL
;
935 for (i
= 0; parms
.iterate (i
, &ip
); i
++)
936 if (!avals
.m_known_vals
[ip
])
937 info
->builtin_constant_p_parms
.safe_push (ip
);
940 /* If inliner or someone after inliner will ever start producing
941 non-trivial clones, we will get trouble with lack of information
942 about updating self sizes, because size vectors already contains
943 sizes of the callees. */
944 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
948 info
->size_time_table
= src_info
->size_time_table
.copy ();
949 info
->loop_iterations
= vec_safe_copy (src_info
->loop_iterations
);
950 info
->loop_strides
= vec_safe_copy (info
->loop_strides
);
952 info
->builtin_constant_p_parms
953 = info
->builtin_constant_p_parms
.copy ();
955 ipa_freqcounting_predicate
*f
;
956 for (int i
= 0; vec_safe_iterate (info
->loop_iterations
, i
, &f
); i
++)
958 ipa_predicate p
= *f
->predicate
;
960 set_hint_predicate (&f
->predicate
, p
);
962 for (int i
= 0; vec_safe_iterate (info
->loop_strides
, i
, &f
); i
++)
964 ipa_predicate p
= *f
->predicate
;
966 set_hint_predicate (&f
->predicate
, p
);
969 if (!dst
->inlined_to
)
970 ipa_update_overall_fn_summary (dst
);
974 /* Hook that is called by cgraph.cc when a node is duplicated. */
977 ipa_call_summary_t::duplicate (struct cgraph_edge
*src
,
978 struct cgraph_edge
*dst
,
979 class ipa_call_summary
*srcinfo
,
980 class ipa_call_summary
*info
)
982 new (info
) ipa_call_summary (*srcinfo
);
983 info
->predicate
= NULL
;
984 edge_set_predicate (dst
, srcinfo
->predicate
);
985 info
->param
= srcinfo
->param
.copy ();
986 if (!dst
->indirect_unknown_callee
&& src
->indirect_unknown_callee
)
988 info
->call_stmt_size
-= (eni_size_weights
.indirect_call_cost
989 - eni_size_weights
.call_cost
);
990 info
->call_stmt_time
-= (eni_time_weights
.indirect_call_cost
991 - eni_time_weights
.call_cost
);
995 /* Dump edge summaries associated to NODE and recursively to all clones.
999 dump_ipa_call_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
1000 class ipa_fn_summary
*info
)
1002 struct cgraph_edge
*edge
;
1003 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1005 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
1006 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
1010 "%*s%s %s\n%*s freq:%4.2f",
1011 indent
, "", callee
->dump_name (),
1012 !edge
->inline_failed
1013 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
1014 indent
, "", edge
->sreal_frequency ().to_double ());
1016 if (cross_module_call_p (edge
))
1017 fprintf (f
, " cross module");
1020 fprintf (f
, " loop depth:%2i size:%2i time: %2i",
1021 es
->loop_depth
, es
->call_stmt_size
, es
->call_stmt_time
);
1023 ipa_fn_summary
*s
= ipa_fn_summaries
->get (callee
);
1024 ipa_size_summary
*ss
= ipa_size_summaries
->get (callee
);
1026 fprintf (f
, " callee size:%2i stack:%2i",
1027 (int) (ss
->size
/ ipa_fn_summary::size_scale
),
1028 (int) s
->estimated_stack_size
);
1030 if (es
&& es
->predicate
)
1032 fprintf (f
, " predicate: ");
1033 es
->predicate
->dump (f
, info
->conds
);
1037 if (es
&& es
->param
.exists ())
1038 for (i
= 0; i
< (int) es
->param
.length (); i
++)
1040 int prob
= es
->param
[i
].change_prob
;
1043 fprintf (f
, "%*s op%i is compile time invariant\n",
1045 else if (prob
!= REG_BR_PROB_BASE
)
1046 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1047 prob
* 100.0 / REG_BR_PROB_BASE
);
1048 if (es
->param
[i
].points_to_local_or_readonly_memory
)
1049 fprintf (f
, "%*s op%i points to local or readonly memory\n",
1051 if (es
->param
[i
].points_to_possible_sra_candidate
)
1052 fprintf (f
, "%*s op%i points to possible sra candidate\n",
1055 if (!edge
->inline_failed
)
1057 ipa_size_summary
*ss
= ipa_size_summaries
->get (callee
);
1058 fprintf (f
, "%*sStack frame offset %i, callee self size %i\n",
1060 (int) ipa_get_stack_frame_offset (callee
),
1061 (int) ss
->estimated_self_stack_size
);
1062 dump_ipa_call_summary (f
, indent
+ 2, callee
, info
);
1065 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1067 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
1068 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
1072 edge
->sreal_frequency ().to_double (), es
->call_stmt_size
,
1073 es
->call_stmt_time
);
1076 fprintf (f
, "predicate: ");
1077 es
->predicate
->dump (f
, info
->conds
);
1086 ipa_dump_fn_summary (FILE *f
, struct cgraph_node
*node
)
1088 if (node
->definition
)
1090 class ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
1091 class ipa_size_summary
*ss
= ipa_size_summaries
->get (node
);
1096 fprintf (f
, "IPA function summary for %s", node
->dump_name ());
1097 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
1098 fprintf (f
, " always_inline");
1100 fprintf (f
, " inlinable");
1101 if (s
->fp_expressions
)
1102 fprintf (f
, " fp_expression");
1103 if (s
->builtin_constant_p_parms
.length ())
1105 fprintf (f
, " builtin_constant_p_parms");
1106 for (unsigned int i
= 0;
1107 i
< s
->builtin_constant_p_parms
.length (); i
++)
1108 fprintf (f
, " %i", s
->builtin_constant_p_parms
[i
]);
1110 fprintf (f
, "\n global time: %f\n", s
->time
.to_double ());
1111 fprintf (f
, " self size: %i\n", ss
->self_size
);
1112 fprintf (f
, " global size: %i\n", ss
->size
);
1113 fprintf (f
, " min size: %i\n", s
->min_size
);
1114 fprintf (f
, " self stack: %i\n",
1115 (int) ss
->estimated_self_stack_size
);
1116 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
1118 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
1120 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
1121 for (i
= 0; s
->size_time_table
.iterate (i
, &e
); i
++)
1123 fprintf (f
, " size:%f, time:%f",
1124 (double) e
->size
/ ipa_fn_summary::size_scale
,
1125 e
->time
.to_double ());
1126 if (e
->exec_predicate
!= true)
1128 fprintf (f
, ", executed if:");
1129 e
->exec_predicate
.dump (f
, s
->conds
, 0);
1131 if (e
->exec_predicate
!= e
->nonconst_predicate
)
1133 fprintf (f
, ", nonconst if:");
1134 e
->nonconst_predicate
.dump (f
, s
->conds
, 0);
1138 ipa_freqcounting_predicate
*fcp
;
1139 bool first_fcp
= true;
1140 for (int i
= 0; vec_safe_iterate (s
->loop_iterations
, i
, &fcp
); i
++)
1144 fprintf (f
, " loop iterations:");
1147 fprintf (f
, " %3.2f for ", fcp
->freq
.to_double ());
1148 fcp
->predicate
->dump (f
, s
->conds
);
1151 for (int i
= 0; vec_safe_iterate (s
->loop_strides
, i
, &fcp
); i
++)
1155 fprintf (f
, " loop strides:");
1158 fprintf (f
, " %3.2f for :", fcp
->freq
.to_double ());
1159 fcp
->predicate
->dump (f
, s
->conds
);
1161 fprintf (f
, " calls:\n");
1162 dump_ipa_call_summary (f
, 4, node
, s
);
1165 fprintf (f
, " target_info: %x\n", s
->target_info
);
1168 fprintf (f
, "IPA summary for %s is missing.\n", node
->dump_name ());
1173 ipa_debug_fn_summary (struct cgraph_node
*node
)
1175 ipa_dump_fn_summary (stderr
, node
);
1179 ipa_dump_fn_summaries (FILE *f
)
1181 struct cgraph_node
*node
;
1183 FOR_EACH_DEFINED_FUNCTION (node
)
1184 if (!node
->inlined_to
)
1185 ipa_dump_fn_summary (f
, node
);
1188 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1189 boolean variable pointed to by DATA. */
1192 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1195 bool *b
= (bool *) data
;
1200 /* If OP refers to value of function parameter, return the corresponding
1201 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1202 PARM_DECL) will be stored to *SIZE_P in that case too. */
1205 unmodified_parm_1 (ipa_func_body_info
*fbi
, gimple
*stmt
, tree op
,
1208 /* SSA_NAME referring to parm default def? */
1209 if (TREE_CODE (op
) == SSA_NAME
1210 && SSA_NAME_IS_DEFAULT_DEF (op
)
1211 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1214 *size_p
= tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op
)));
1215 return SSA_NAME_VAR (op
);
1217 /* Non-SSA parm reference? */
1218 if (TREE_CODE (op
) == PARM_DECL
1219 && fbi
->aa_walk_budget
> 0)
1221 bool modified
= false;
1224 ao_ref_init (&refd
, op
);
1225 int walked
= walk_aliased_vdefs (&refd
, gimple_vuse (stmt
),
1226 mark_modified
, &modified
, NULL
, NULL
,
1227 fbi
->aa_walk_budget
);
1230 fbi
->aa_walk_budget
= 0;
1233 fbi
->aa_walk_budget
-= walked
;
1237 *size_p
= tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op
)));
1244 /* If OP refers to value of function parameter, return the corresponding
1245 parameter. Also traverse chains of SSA register assignments. If non-NULL,
1246 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1247 stored to *SIZE_P in that case too. */
1250 unmodified_parm (ipa_func_body_info
*fbi
, gimple
*stmt
, tree op
,
1253 tree res
= unmodified_parm_1 (fbi
, stmt
, op
, size_p
);
1257 if (TREE_CODE (op
) == SSA_NAME
1258 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1259 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1260 return unmodified_parm (fbi
, SSA_NAME_DEF_STMT (op
),
1261 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)),
1266 /* If OP refers to a value of a function parameter or value loaded from an
1267 aggregate passed to a parameter (either by value or reference), return TRUE
1268 and store the number of the parameter to *INDEX_P, the access size into
1269 *SIZE_P, and information whether and how it has been loaded from an
1270 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1271 statement in which OP is used or loaded. */
1274 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info
*fbi
,
1275 gimple
*stmt
, tree op
, int *index_p
,
1277 struct agg_position_info
*aggpos
)
1279 tree res
= unmodified_parm_1 (fbi
, stmt
, op
, size_p
);
1281 gcc_checking_assert (aggpos
);
1284 *index_p
= ipa_get_param_decl_index (fbi
->info
, res
);
1287 aggpos
->agg_contents
= false;
1288 aggpos
->by_ref
= false;
1292 if (TREE_CODE (op
) == SSA_NAME
)
1294 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1295 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1297 stmt
= SSA_NAME_DEF_STMT (op
);
1298 op
= gimple_assign_rhs1 (stmt
);
1299 if (!REFERENCE_CLASS_P (op
))
1300 return unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, index_p
, size_p
,
1304 aggpos
->agg_contents
= true;
1305 return ipa_load_from_parm_agg (fbi
, fbi
->info
->descriptors
,
1306 stmt
, op
, index_p
, &aggpos
->offset
,
1307 size_p
, &aggpos
->by_ref
);
1310 /* If stmt is simple load or store of value pointed to by a function parmaeter,
1311 return its index. */
1314 load_or_store_of_ptr_parameter (ipa_func_body_info
*fbi
, gimple
*stmt
)
1318 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
1322 if (gimple_assign_load_p (stmt
))
1323 param
= gimple_assign_rhs1 (stmt
);
1324 else if (gimple_store_p (stmt
))
1325 param
= gimple_assign_lhs (stmt
);
1328 tree base
= get_base_address (param
);
1329 if (TREE_CODE (base
) != MEM_REF
1330 || TREE_CODE (TREE_OPERAND (base
, 0)) != SSA_NAME
1331 || !SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base
, 0)))
1333 tree p
= SSA_NAME_VAR (TREE_OPERAND (base
, 0));
1334 if (TREE_CODE (p
) != PARM_DECL
)
1336 return ipa_get_param_decl_index (fbi
->info
, p
);
1339 /* See if statement might disappear after inlining.
1340 0 - means not eliminated
1341 1 - half of statements goes away
1342 2 - for sure it is eliminated.
1343 We are not terribly sophisticated, basically looking for simple abstraction
1344 penalty wrappers. */
1347 eliminated_by_inlining_prob (ipa_func_body_info
*fbi
, gimple
*stmt
)
1349 enum gimple_code code
= gimple_code (stmt
);
1350 enum tree_code rhs_code
;
1360 if (gimple_num_ops (stmt
) != 2)
1363 rhs_code
= gimple_assign_rhs_code (stmt
);
1365 /* Casts of parameters, loads from parameters passed by reference
1366 and stores to return value or parameters are often free after
1367 inlining due to SRA and further combining.
1368 Assume that half of statements goes away. */
1369 if (CONVERT_EXPR_CODE_P (rhs_code
)
1370 || rhs_code
== VIEW_CONVERT_EXPR
1371 || rhs_code
== ADDR_EXPR
1372 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1374 tree rhs
= gimple_assign_rhs1 (stmt
);
1375 tree lhs
= gimple_assign_lhs (stmt
);
1376 tree inner_rhs
= get_base_address (rhs
);
1377 tree inner_lhs
= get_base_address (lhs
);
1378 bool rhs_free
= false;
1379 bool lhs_free
= false;
1386 /* Reads of parameter are expected to be free. */
1387 if (unmodified_parm (fbi
, stmt
, inner_rhs
, NULL
))
1389 /* Match expressions of form &this->field. Those will most likely
1390 combine with something upstream after inlining. */
1391 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1393 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1394 if (TREE_CODE (op
) == PARM_DECL
)
1396 else if (TREE_CODE (op
) == MEM_REF
1397 && unmodified_parm (fbi
, stmt
, TREE_OPERAND (op
, 0),
1402 /* When parameter is not SSA register because its address is taken
1403 and it is just copied into one, the statement will be completely
1404 free after inlining (we will copy propagate backward). */
1405 if (rhs_free
&& is_gimple_reg (lhs
))
1408 /* Reads of parameters passed by reference
1409 expected to be free (i.e. optimized out after inlining). */
1410 if (TREE_CODE (inner_rhs
) == MEM_REF
1411 && unmodified_parm (fbi
, stmt
, TREE_OPERAND (inner_rhs
, 0), NULL
))
1414 /* Copying parameter passed by reference into gimple register is
1415 probably also going to copy propagate, but we can't be quite
1417 if (rhs_free
&& is_gimple_reg (lhs
))
1420 /* Writes to parameters, parameters passed by value and return value
1421 (either directly or passed via invisible reference) are free.
1423 TODO: We ought to handle testcase like
1424 struct a {int a,b;};
1432 This translate into:
1447 For that we either need to copy ipa-split logic detecting writes
1449 if (TREE_CODE (inner_lhs
) == PARM_DECL
1450 || TREE_CODE (inner_lhs
) == RESULT_DECL
1451 || (TREE_CODE (inner_lhs
) == MEM_REF
1452 && (unmodified_parm (fbi
, stmt
, TREE_OPERAND (inner_lhs
, 0),
1454 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1455 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1456 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1458 0))) == RESULT_DECL
))))
1461 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1463 if (lhs_free
&& rhs_free
)
1472 /* Analyze EXPR if it represents a series of simple operations performed on
1473 a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1474 AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1475 Type of the parameter or load from an aggregate via the parameter is
1476 stored in *TYPE_P. Operations on the parameter are recorded to
1477 PARAM_OPS_P if it is not NULL. */
1480 decompose_param_expr (struct ipa_func_body_info
*fbi
,
1481 gimple
*stmt
, tree expr
,
1482 int *index_p
, tree
*type_p
,
1483 struct agg_position_info
*aggpos
,
1484 expr_eval_ops
*param_ops_p
= NULL
)
1486 int op_limit
= opt_for_fn (fbi
->node
->decl
, param_ipa_max_param_expr_ops
);
1490 *param_ops_p
= NULL
;
1494 expr_eval_op eval_op
;
1496 unsigned cst_count
= 0;
1498 if (unmodified_parm_or_parm_agg_item (fbi
, stmt
, expr
, index_p
, NULL
,
1501 tree type
= TREE_TYPE (expr
);
1503 if (aggpos
->agg_contents
)
1505 /* Stop if containing bit-field. */
1506 if (TREE_CODE (expr
) == BIT_FIELD_REF
1507 || contains_bitfld_component_ref_p (expr
))
1515 if (TREE_CODE (expr
) != SSA_NAME
|| SSA_NAME_IS_DEFAULT_DEF (expr
))
1517 stmt
= SSA_NAME_DEF_STMT (expr
);
1519 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
1521 int flags
= gimple_call_return_flags (call
);
1522 if (!(flags
& ERF_RETURNS_ARG
))
1524 int arg
= flags
& ERF_RETURN_ARG_MASK
;
1525 if (arg
>= (int)gimple_call_num_args (call
))
1527 expr
= gimple_call_arg (stmt
, arg
);
1531 if (!is_gimple_assign (stmt
= SSA_NAME_DEF_STMT (expr
)))
1534 switch (gimple_assign_rhs_class (stmt
))
1536 case GIMPLE_SINGLE_RHS
:
1537 expr
= gimple_assign_rhs1 (stmt
);
1540 case GIMPLE_UNARY_RHS
:
1544 case GIMPLE_BINARY_RHS
:
1548 case GIMPLE_TERNARY_RHS
:
1556 /* Stop if expression is too complex. */
1557 if (op_count
++ == op_limit
)
1562 eval_op
.code
= gimple_assign_rhs_code (stmt
);
1563 eval_op
.type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1564 eval_op
.val
[0] = NULL_TREE
;
1565 eval_op
.val
[1] = NULL_TREE
;
1569 for (unsigned i
= 0; i
< rhs_count
; i
++)
1571 tree op
= gimple_op (stmt
, i
+ 1);
1573 gcc_assert (op
&& !TYPE_P (op
));
1574 if (is_gimple_ip_invariant (op
))
1576 if (++cst_count
== rhs_count
)
1579 eval_op
.val
[cst_count
- 1] = op
;
1583 /* Found a non-constant operand, and record its index in rhs
1590 /* Found more than one non-constant operands. */
1596 vec_safe_insert (*param_ops_p
, 0, eval_op
);
1599 /* Failed to decompose, free resource and return. */
1602 vec_free (*param_ops_p
);
1607 /* Record to SUMMARY that PARM is used by builtin_constant_p. */
1610 add_builtin_constant_p_parm (class ipa_fn_summary
*summary
, int parm
)
1614 /* Avoid duplicates. */
1615 for (unsigned int i
= 0;
1616 summary
->builtin_constant_p_parms
.iterate (i
, &ip
); i
++)
1619 summary
->builtin_constant_p_parms
.safe_push (parm
);
1622 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1623 predicates to the CFG edges. */
1626 set_cond_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1627 class ipa_fn_summary
*summary
,
1628 class ipa_node_params
*params_summary
,
1633 struct agg_position_info aggpos
;
1634 enum tree_code code
, inverted_code
;
1639 expr_eval_ops param_ops
;
1641 gcond
*last
= safe_dyn_cast
<gcond
*> (*gsi_last_bb (bb
));
1644 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1646 op
= gimple_cond_lhs (last
);
1648 if (decompose_param_expr (fbi
, last
, op
, &index
, ¶m_type
, &aggpos
,
1651 code
= gimple_cond_code (last
);
1652 inverted_code
= invert_tree_comparison (code
, HONOR_NANS (op
));
1654 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1656 enum tree_code this_code
= (e
->flags
& EDGE_TRUE_VALUE
1657 ? code
: inverted_code
);
1658 /* invert_tree_comparison will return ERROR_MARK on FP
1659 comparisons that are not EQ/NE instead of returning proper
1660 unordered one. Be sure it is not confused with NON_CONSTANT.
1662 And if the edge's target is the final block of diamond CFG graph
1663 of this conditional statement, we do not need to compute
1664 predicate for the edge because the final block's predicate must
1665 be at least as that of the first block of the statement. */
1666 if (this_code
!= ERROR_MARK
1667 && !dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1670 = add_condition (summary
, params_summary
, index
,
1671 param_type
, &aggpos
,
1672 this_code
, gimple_cond_rhs (last
), param_ops
);
1673 e
->aux
= edge_predicate_pool
.allocate ();
1674 *(ipa_predicate
*) e
->aux
= p
;
1677 vec_free (param_ops
);
1681 if (TREE_CODE (op
) != SSA_NAME
)
1684 if (builtin_constant_p (op))
1688 Here we can predicate nonconstant_code. We can't
1689 really handle constant_code since we have no predicate
1690 for this and also the constant code is not known to be
1691 optimized away when inliner doesn't see operand is constant.
1692 Other optimizers might think otherwise. */
1693 if (gimple_cond_code (last
) != NE_EXPR
1694 || !integer_zerop (gimple_cond_rhs (last
)))
1696 set_stmt
= SSA_NAME_DEF_STMT (op
);
1697 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1698 || gimple_call_num_args (set_stmt
) != 1)
1700 op2
= gimple_call_arg (set_stmt
, 0);
1701 if (!decompose_param_expr (fbi
, set_stmt
, op2
, &index
, ¶m_type
, &aggpos
))
1704 add_builtin_constant_p_parm (summary
, index
);
1705 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1707 ipa_predicate p
= add_condition (summary
, params_summary
, index
,
1708 param_type
, &aggpos
,
1709 ipa_predicate::is_not_constant
, NULL_TREE
);
1710 e
->aux
= edge_predicate_pool
.allocate ();
1711 *(ipa_predicate
*) e
->aux
= p
;
1716 /* If BB ends by a switch we can turn into predicates, attach corresponding
1717 predicates to the CFG edges. */
1720 set_switch_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1721 class ipa_fn_summary
*summary
,
1722 class ipa_node_params
*params_summary
,
1727 struct agg_position_info aggpos
;
1733 expr_eval_ops param_ops
;
1735 gswitch
*last
= safe_dyn_cast
<gswitch
*> (*gsi_last_bb (bb
));
1738 op
= gimple_switch_index (last
);
1739 if (!decompose_param_expr (fbi
, last
, op
, &index
, ¶m_type
, &aggpos
,
1743 auto_vec
<std::pair
<tree
, tree
> > ranges
;
1744 tree type
= TREE_TYPE (op
);
1745 int bound_limit
= opt_for_fn (fbi
->node
->decl
,
1746 param_ipa_max_switch_predicate_bounds
);
1747 int bound_count
= 0;
1748 // This can safely be an integer range, as switches can only hold
1752 get_range_query (cfun
)->range_of_expr (vr
, op
);
1753 if (vr
.undefined_p ())
1754 vr
.set_varying (TREE_TYPE (op
));
1755 tree vr_min
, vr_max
;
1756 // TODO: This entire function could use a rewrite to use the irange
1757 // API, instead of trying to recreate its intersection/union logic.
1758 // Any use of get_legacy_range() is a serious code smell.
1759 value_range_kind vr_type
= get_legacy_range (vr
, vr_min
, vr_max
);
1760 wide_int vr_wmin
= wi::to_wide (vr_min
);
1761 wide_int vr_wmax
= wi::to_wide (vr_max
);
1763 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1765 e
->aux
= edge_predicate_pool
.allocate ();
1766 *(ipa_predicate
*) e
->aux
= false;
1769 e
= gimple_switch_edge (cfun
, last
, 0);
1770 /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1771 default case if its target basic block is in convergence point of all
1772 switch cases, which can be determined by checking whether it
1773 post-dominates the switch statement. */
1774 if (dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1775 bound_count
= INT_MAX
;
1777 n
= gimple_switch_num_labels (last
);
1778 for (case_idx
= 1; case_idx
< n
; ++case_idx
)
1780 tree cl
= gimple_switch_label (last
, case_idx
);
1781 tree min
= CASE_LOW (cl
);
1782 tree max
= CASE_HIGH (cl
);
1785 e
= gimple_switch_edge (cfun
, last
, case_idx
);
1787 /* The case value might not have same type as switch expression,
1788 extend the value based on the expression type. */
1789 if (TREE_TYPE (min
) != type
)
1790 min
= wide_int_to_tree (type
, wi::to_wide (min
));
1794 else if (TREE_TYPE (max
) != type
)
1795 max
= wide_int_to_tree (type
, wi::to_wide (max
));
1797 /* The case's target basic block is in convergence point of all switch
1798 cases, its predicate should be at least as that of the switch
1800 if (dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1802 else if (min
== max
)
1803 p
= add_condition (summary
, params_summary
, index
, param_type
,
1804 &aggpos
, EQ_EXPR
, min
, param_ops
);
1807 ipa_predicate p1
, p2
;
1808 p1
= add_condition (summary
, params_summary
, index
, param_type
,
1809 &aggpos
, GE_EXPR
, min
, param_ops
);
1810 p2
= add_condition (summary
, params_summary
,index
, param_type
,
1811 &aggpos
, LE_EXPR
, max
, param_ops
);
1814 *(ipa_predicate
*) e
->aux
1815 = p
.or_with (summary
->conds
, *(ipa_predicate
*) e
->aux
);
1817 /* If there are too many disjoint case ranges, predicate for default
1818 case might become too complicated. So add a limit here. */
1819 if (bound_count
> bound_limit
)
1822 bool new_range
= true;
1824 if (!ranges
.is_empty ())
1826 wide_int curr_wmin
= wi::to_wide (min
);
1827 wide_int last_wmax
= wi::to_wide (ranges
.last ().second
);
1829 /* Merge case ranges if they are continuous. */
1830 if (curr_wmin
== last_wmax
+ 1)
1832 else if (vr_type
== VR_ANTI_RANGE
)
1834 /* If two disjoint case ranges can be connected by anti-range
1835 of switch index, combine them to one range. */
1836 if (wi::lt_p (vr_wmax
, curr_wmin
- 1, TYPE_SIGN (type
)))
1837 vr_type
= VR_UNDEFINED
;
1838 else if (wi::le_p (vr_wmin
, last_wmax
+ 1, TYPE_SIGN (type
)))
1843 /* Create/extend a case range. And we count endpoints of range set,
1844 this number nearly equals to number of conditions that we will create
1845 for predicate of default case. */
1848 bound_count
+= (min
== max
) ? 1 : 2;
1849 ranges
.safe_push (std::make_pair (min
, max
));
1853 bound_count
+= (ranges
.last ().first
== ranges
.last ().second
);
1854 ranges
.last ().second
= max
;
1858 e
= gimple_switch_edge (cfun
, last
, 0);
1859 if (bound_count
> bound_limit
)
1861 *(ipa_predicate
*) e
->aux
= true;
1862 vec_free (param_ops
);
1866 ipa_predicate p_seg
= true;
1867 ipa_predicate p_all
= false;
1869 if (vr_type
!= VR_RANGE
)
1871 vr_wmin
= wi::to_wide (TYPE_MIN_VALUE (type
));
1872 vr_wmax
= wi::to_wide (TYPE_MAX_VALUE (type
));
1875 /* Construct predicate to represent default range set that is negation of
1876 all case ranges. Case range is classified as containing single/non-single
1877 values. Suppose a piece of case ranges in the following.
1879 [D1...D2] [S1] ... [Sn] [D3...D4]
1881 To represent default case's range sets between two non-single value
1882 case ranges (From D2 to D3), we construct predicate as:
1884 D2 < x < D3 && x != S1 && ... && x != Sn
1886 for (size_t i
= 0; i
< ranges
.length (); i
++)
1888 tree min
= ranges
[i
].first
;
1889 tree max
= ranges
[i
].second
;
1892 p_seg
&= add_condition (summary
, params_summary
, index
,
1893 param_type
, &aggpos
, NE_EXPR
,
1897 /* Do not create sub-predicate for range that is beyond low bound
1899 if (wi::lt_p (vr_wmin
, wi::to_wide (min
), TYPE_SIGN (type
)))
1901 p_seg
&= add_condition (summary
, params_summary
, index
,
1902 param_type
, &aggpos
,
1903 LT_EXPR
, min
, param_ops
);
1904 p_all
= p_all
.or_with (summary
->conds
, p_seg
);
1907 /* Do not create sub-predicate for range that is beyond up bound
1909 if (wi::le_p (vr_wmax
, wi::to_wide (max
), TYPE_SIGN (type
)))
1915 p_seg
= add_condition (summary
, params_summary
, index
,
1916 param_type
, &aggpos
, GT_EXPR
,
1921 p_all
= p_all
.or_with (summary
->conds
, p_seg
);
1922 *(ipa_predicate
*) e
->aux
1923 = p_all
.or_with (summary
->conds
, *(ipa_predicate
*) e
->aux
);
1925 vec_free (param_ops
);
1929 /* For each BB in NODE attach to its AUX pointer predicate under
1930 which it is executable. */
1933 compute_bb_predicates (struct ipa_func_body_info
*fbi
,
1934 struct cgraph_node
*node
,
1935 class ipa_fn_summary
*summary
,
1936 class ipa_node_params
*params_summary
)
1938 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1942 FOR_EACH_BB_FN (bb
, my_function
)
1944 set_cond_stmt_execution_predicate (fbi
, summary
, params_summary
, bb
);
1945 set_switch_stmt_execution_predicate (fbi
, summary
, params_summary
, bb
);
1948 /* Entry block is always executable. */
1949 ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
1950 = edge_predicate_pool
.allocate ();
1951 *(ipa_predicate
*) ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
= true;
1953 /* A simple dataflow propagation of predicates forward in the CFG.
1954 TODO: work in reverse postorder. */
1958 FOR_EACH_BB_FN (bb
, my_function
)
1960 ipa_predicate p
= false;
1963 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1967 ipa_predicate this_bb_predicate
1968 = *(ipa_predicate
*) e
->src
->aux
;
1970 this_bb_predicate
&= (*(ipa_predicate
*) e
->aux
);
1971 p
= p
.or_with (summary
->conds
, this_bb_predicate
);
1978 basic_block pdom_bb
;
1983 bb
->aux
= edge_predicate_pool
.allocate ();
1984 *((ipa_predicate
*) bb
->aux
) = p
;
1986 else if (p
!= *(ipa_predicate
*) bb
->aux
)
1988 /* This OR operation is needed to ensure monotonous data flow
1989 in the case we hit the limit on number of clauses and the
1990 and/or operations above give approximate answers. */
1991 p
= p
.or_with (summary
->conds
, *(ipa_predicate
*)bb
->aux
);
1992 if (p
!= *(ipa_predicate
*)bb
->aux
)
1995 *((ipa_predicate
*)bb
->aux
) = p
;
1999 /* For switch/if statement, we can OR-combine predicates of all
2000 its cases/branches to get predicate for basic block in their
2001 convergence point, but sometimes this will generate very
2002 complicated predicate. Actually, we can get simplified
2003 predicate in another way by using the fact that predicate
2004 for a basic block must also hold true for its post dominators.
2005 To be specific, basic block in convergence point of
2006 conditional statement should include predicate of the
2008 pdom_bb
= get_immediate_dominator (CDI_POST_DOMINATORS
, bb
);
2009 if (pdom_bb
== EXIT_BLOCK_PTR_FOR_FN (my_function
) || !pdom_bb
)
2011 else if (!pdom_bb
->aux
)
2014 pdom_bb
->aux
= edge_predicate_pool
.allocate ();
2015 *((ipa_predicate
*)pdom_bb
->aux
) = p
;
2017 else if (p
!= *(ipa_predicate
*)pdom_bb
->aux
)
2019 p
= p
.or_with (summary
->conds
,
2020 *(ipa_predicate
*)pdom_bb
->aux
);
2021 if (p
!= *(ipa_predicate
*)pdom_bb
->aux
)
2024 *((ipa_predicate
*)pdom_bb
->aux
) = p
;
2033 /* Return predicate specifying when the STMT might have result that is not
2034 a compile time constant. */
2036 static ipa_predicate
2037 will_be_nonconstant_expr_predicate (ipa_func_body_info
*fbi
,
2038 class ipa_fn_summary
*summary
,
2039 class ipa_node_params
*params_summary
,
2041 vec
<ipa_predicate
> nonconstant_names
)
2046 while (UNARY_CLASS_P (expr
))
2047 expr
= TREE_OPERAND (expr
, 0);
2049 parm
= unmodified_parm (fbi
, NULL
, expr
, NULL
);
2050 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
2051 return add_condition (summary
, params_summary
, index
, TREE_TYPE (parm
), NULL
,
2052 ipa_predicate::changed
, NULL_TREE
);
2053 if (is_gimple_min_invariant (expr
))
2055 if (TREE_CODE (expr
) == SSA_NAME
)
2056 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
2057 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
2060 = will_be_nonconstant_expr_predicate (fbi
, summary
,
2062 TREE_OPERAND (expr
, 0),
2068 = will_be_nonconstant_expr_predicate (fbi
, summary
,
2070 TREE_OPERAND (expr
, 1),
2072 return p1
.or_with (summary
->conds
, p2
);
2074 else if (TREE_CODE (expr
) == COND_EXPR
)
2077 = will_be_nonconstant_expr_predicate (fbi
, summary
,
2079 TREE_OPERAND (expr
, 0),
2085 = will_be_nonconstant_expr_predicate (fbi
, summary
,
2087 TREE_OPERAND (expr
, 1),
2091 p1
= p1
.or_with (summary
->conds
, p2
);
2092 p2
= will_be_nonconstant_expr_predicate (fbi
, summary
,
2094 TREE_OPERAND (expr
, 2),
2096 return p2
.or_with (summary
->conds
, p1
);
2098 else if (TREE_CODE (expr
) == CALL_EXPR
)
2108 /* Return predicate specifying when the STMT might have result that is not
2109 a compile time constant. */
2111 static ipa_predicate
2112 will_be_nonconstant_predicate (struct ipa_func_body_info
*fbi
,
2113 class ipa_fn_summary
*summary
,
2114 class ipa_node_params
*params_summary
,
2116 vec
<ipa_predicate
> nonconstant_names
)
2118 ipa_predicate p
= true;
2121 tree param_type
= NULL_TREE
;
2122 ipa_predicate op_non_const
;
2125 struct agg_position_info aggpos
;
2127 /* What statements might be optimized away
2128 when their arguments are constant. */
2129 if (gimple_code (stmt
) != GIMPLE_ASSIGN
2130 && gimple_code (stmt
) != GIMPLE_COND
2131 && gimple_code (stmt
) != GIMPLE_SWITCH
2132 && (gimple_code (stmt
) != GIMPLE_CALL
2133 || !(gimple_call_flags (stmt
) & ECF_CONST
)))
2136 /* Stores will stay anyway. */
2137 if (gimple_store_p (stmt
))
2140 is_load
= gimple_assign_load_p (stmt
);
2142 /* Loads can be optimized when the value is known. */
2145 tree op
= gimple_assign_rhs1 (stmt
);
2146 if (!decompose_param_expr (fbi
, stmt
, op
, &base_index
, ¶m_type
,
2153 /* See if we understand all operands before we start
2154 adding conditionals. */
2155 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2157 tree parm
= unmodified_parm (fbi
, stmt
, use
, NULL
);
2158 /* For arguments we can build a condition. */
2159 if (parm
&& ipa_get_param_decl_index (fbi
->info
, parm
) >= 0)
2161 if (TREE_CODE (use
) != SSA_NAME
)
2163 /* If we know when operand is constant,
2164 we still can say something useful. */
2165 if (nonconstant_names
[SSA_NAME_VERSION (use
)] != true)
2172 add_condition (summary
, params_summary
,
2173 base_index
, param_type
, &aggpos
,
2174 ipa_predicate::changed
, NULL_TREE
);
2176 op_non_const
= false;
2177 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
2179 tree parm
= unmodified_parm (fbi
, stmt
, use
, NULL
);
2182 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
2184 if (index
!= base_index
)
2185 p
= add_condition (summary
, params_summary
, index
,
2186 TREE_TYPE (parm
), NULL
,
2187 ipa_predicate::changed
, NULL_TREE
);
2192 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
2193 op_non_const
= p
.or_with (summary
->conds
, op_non_const
);
2195 if ((gimple_code (stmt
) == GIMPLE_ASSIGN
|| gimple_code (stmt
) == GIMPLE_CALL
)
2196 && gimple_op (stmt
, 0)
2197 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
2198 nonconstant_names
[SSA_NAME_VERSION (gimple_op (stmt
, 0))]
2200 return op_non_const
;
2203 struct record_modified_bb_info
2210 /* Value is initialized in INIT_BB and used in USE_BB. We want to compute
2211 probability how often it changes between USE_BB.
2212 INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
2213 is in different loop nest, we can do better.
2214 This is all just estimate. In theory we look for minimal cut separating
2215 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2219 get_minimal_bb (basic_block init_bb
, basic_block use_bb
)
2221 class loop
*l
= find_common_loop (init_bb
->loop_father
, use_bb
->loop_father
);
2222 if (l
&& l
->header
->count
< init_bb
->count
)
2227 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2228 set except for info->stmt. */
2231 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
2233 struct record_modified_bb_info
*info
=
2234 (struct record_modified_bb_info
*) data
;
2235 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
2237 if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef
)))
2239 bitmap_set_bit (info
->bb_set
,
2240 SSA_NAME_IS_DEFAULT_DEF (vdef
)
2241 ? ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
2243 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
2244 gimple_bb (info
->stmt
))->index
);
2247 fprintf (dump_file
, " Param ");
2248 print_generic_expr (dump_file
, info
->op
, TDF_SLIM
);
2249 fprintf (dump_file
, " changed at bb %i, minimal: %i stmt: ",
2250 gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
,
2252 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
2253 gimple_bb (info
->stmt
))->index
);
2254 print_gimple_stmt (dump_file
, SSA_NAME_DEF_STMT (vdef
), 0);
2259 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2260 will change since last invocation of STMT.
2262 Value 0 is reserved for compile time invariants.
2263 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2264 ought to be REG_BR_PROB_BASE / estimated_iters. */
2267 param_change_prob (ipa_func_body_info
*fbi
, gimple
*stmt
, int i
)
2269 tree op
= gimple_call_arg (stmt
, i
);
2270 basic_block bb
= gimple_bb (stmt
);
2272 if (TREE_CODE (op
) == WITH_SIZE_EXPR
)
2273 op
= TREE_OPERAND (op
, 0);
2275 tree base
= get_base_address (op
);
2277 /* Global invariants never change. */
2278 if (is_gimple_min_invariant (base
))
2281 /* We would have to do non-trivial analysis to really work out what
2282 is the probability of value to change (i.e. when init statement
2283 is in a sibling loop of the call).
2285 We do an conservative estimate: when call is executed N times more often
2286 than the statement defining value, we take the frequency 1/N. */
2287 if (TREE_CODE (base
) == SSA_NAME
)
2289 profile_count init_count
;
2291 if (!bb
->count
.nonzero_p ())
2292 return REG_BR_PROB_BASE
;
2294 if (SSA_NAME_IS_DEFAULT_DEF (base
))
2295 init_count
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
2297 init_count
= get_minimal_bb
2298 (gimple_bb (SSA_NAME_DEF_STMT (base
)),
2299 gimple_bb (stmt
))->count
;
2301 if (init_count
< bb
->count
)
2302 return MAX ((init_count
.to_sreal_scale (bb
->count
)
2303 * REG_BR_PROB_BASE
).to_int (), 1);
2304 return REG_BR_PROB_BASE
;
2309 profile_count max
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
2310 struct record_modified_bb_info info
;
2311 tree init
= ctor_for_folding (base
);
2313 if (init
!= error_mark_node
)
2315 if (!bb
->count
.nonzero_p () || fbi
->aa_walk_budget
== 0)
2316 return REG_BR_PROB_BASE
;
2319 fprintf (dump_file
, " Analyzing param change probability of ");
2320 print_generic_expr (dump_file
, op
, TDF_SLIM
);
2321 fprintf (dump_file
, "\n");
2323 ao_ref_init (&refd
, op
);
2326 info
.bb_set
= BITMAP_ALLOC (NULL
);
2328 = walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2329 NULL
, NULL
, fbi
->aa_walk_budget
);
2331 fbi
->aa_walk_budget
-= walked
;
2332 if (walked
< 0 || bitmap_bit_p (info
.bb_set
, bb
->index
))
2335 fbi
->aa_walk_budget
= 0;
2339 fprintf (dump_file
, " Ran out of AA walking budget.\n");
2341 fprintf (dump_file
, " Set in same BB as used.\n");
2343 BITMAP_FREE (info
.bb_set
);
2344 return REG_BR_PROB_BASE
;
2349 /* Lookup the most frequent update of the value and believe that
2350 it dominates all the other; precise analysis here is difficult. */
2351 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2352 max
= max
.max (BASIC_BLOCK_FOR_FN (cfun
, index
)->count
);
2355 fprintf (dump_file
, " Set with count ");
2356 max
.dump (dump_file
);
2357 fprintf (dump_file
, " and used with count ");
2358 bb
->count
.dump (dump_file
);
2359 fprintf (dump_file
, " freq %f\n",
2360 max
.to_sreal_scale (bb
->count
).to_double ());
2363 BITMAP_FREE (info
.bb_set
);
2364 if (max
< bb
->count
)
2365 return MAX ((max
.to_sreal_scale (bb
->count
)
2366 * REG_BR_PROB_BASE
).to_int (), 1);
2367 return REG_BR_PROB_BASE
;
2371 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2372 sub-graph and if the predicate the condition depends on is known. If so,
2373 return true and store the pointer the predicate in *P. */
2376 phi_result_unknown_predicate (ipa_func_body_info
*fbi
,
2377 ipa_fn_summary
*summary
,
2378 class ipa_node_params
*params_summary
,
2381 vec
<ipa_predicate
> nonconstant_names
)
2385 basic_block first_bb
= NULL
;
2387 if (single_pred_p (bb
))
2393 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2395 if (single_succ_p (e
->src
))
2397 if (!single_pred_p (e
->src
))
2400 first_bb
= single_pred (e
->src
);
2401 else if (single_pred (e
->src
) != first_bb
)
2408 else if (e
->src
!= first_bb
)
2416 gcond
*stmt
= safe_dyn_cast
<gcond
*> (*gsi_last_bb (first_bb
));
2418 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2421 *p
= will_be_nonconstant_expr_predicate (fbi
, summary
, params_summary
,
2422 gimple_cond_lhs (stmt
),
2430 /* Given a PHI statement in a function described by inline properties SUMMARY
2431 and *P being the predicate describing whether the selected PHI argument is
2432 known, store a predicate for the result of the PHI statement into
2433 NONCONSTANT_NAMES, if possible. */
2436 predicate_for_phi_result (class ipa_fn_summary
*summary
, gphi
*phi
,
2438 vec
<ipa_predicate
> nonconstant_names
)
2442 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2444 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2445 if (!is_gimple_min_invariant (arg
))
2447 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2448 *p
= p
->or_with (summary
->conds
,
2449 nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2455 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2457 fprintf (dump_file
, "\t\tphi predicate: ");
2458 p
->dump (dump_file
, summary
->conds
);
2460 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2463 /* For a typical usage of __builtin_expect (a<b, 1), we
2464 may introduce an extra relation stmt:
2465 With the builtin, we have
2468 t3 = __builtin_expect (t2, 1);
2471 Without the builtin, we have
2474 This affects the size/time estimation and may have
2475 an impact on the earlier inlining.
2476 Here find this pattern and fix it up later. */
2479 find_foldable_builtin_expect (basic_block bb
)
2481 gimple_stmt_iterator bsi
;
2483 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2485 gimple
*stmt
= gsi_stmt (bsi
);
2486 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
)
2487 || gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT_WITH_PROBABILITY
)
2488 || gimple_call_internal_p (stmt
, IFN_BUILTIN_EXPECT
))
2490 tree var
= gimple_call_lhs (stmt
);
2491 tree arg
= gimple_call_arg (stmt
, 0);
2492 use_operand_p use_p
;
2499 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2501 while (TREE_CODE (arg
) == SSA_NAME
)
2503 gimple
*stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2504 if (!is_gimple_assign (stmt_tmp
))
2506 switch (gimple_assign_rhs_code (stmt_tmp
))
2525 arg
= gimple_assign_rhs1 (stmt_tmp
);
2528 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2529 && gimple_code (use_stmt
) == GIMPLE_COND
)
2536 /* Return true when the basic blocks contains only clobbers followed by RESX.
2537 Such BBs are kept around to make removal of dead stores possible with
2538 presence of EH and will be optimized out by optimize_clobbers later in the
2541 NEED_EH is used to recurse in case the clobber has non-EH predecessors
2542 that can be clobber only, too.. When it is false, the RESX is not necessary
2543 on the end of basic block. */
2546 clobber_only_eh_bb_p (basic_block bb
, bool need_eh
= true)
2548 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
2554 if (gsi_end_p (gsi
))
2556 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_RESX
)
2560 else if (!single_succ_p (bb
))
2563 for (; !gsi_end_p (gsi
); gsi_prev (&gsi
))
2565 gimple
*stmt
= gsi_stmt (gsi
);
2566 if (is_gimple_debug (stmt
))
2568 if (gimple_clobber_p (stmt
))
2570 if (gimple_code (stmt
) == GIMPLE_LABEL
)
2575 /* See if all predecessors are either throws or clobber only BBs. */
2576 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2577 if (!(e
->flags
& EDGE_EH
)
2578 && !clobber_only_eh_bb_p (e
->src
, false))
2584 /* Return true if STMT compute a floating point expression that may be affected
2585 by -ffast-math and similar flags. */
2588 fp_expression_p (gimple
*stmt
)
2593 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_DEF
|SSA_OP_USE
)
2594 if (FLOAT_TYPE_P (TREE_TYPE (op
)))
2599 /* Return true if T references memory location that is local
2600 for the function (that means, dead after return) or read-only. */
2603 refs_local_or_readonly_memory_p (tree t
)
2605 /* Non-escaping memory is fine. */
2606 t
= get_base_address (t
);
2607 if ((TREE_CODE (t
) == MEM_REF
2608 || TREE_CODE (t
) == TARGET_MEM_REF
))
2609 return points_to_local_or_readonly_memory_p (TREE_OPERAND (t
, 0));
2611 /* Automatic variables are fine. */
2613 && auto_var_in_fn_p (t
, current_function_decl
))
2616 /* Read-only variables are fine. */
2617 if (DECL_P (t
) && TREE_READONLY (t
))
2623 /* Return true if T is a pointer pointing to memory location that is local
2624 for the function (that means, dead after return) or read-only. */
2627 points_to_local_or_readonly_memory_p (tree t
)
2629 /* See if memory location is clearly invalid. */
2630 if (integer_zerop (t
))
2631 return flag_delete_null_pointer_checks
;
2632 if (TREE_CODE (t
) == SSA_NAME
)
2634 /* For IPA passes we can consinder accesses to return slot local
2635 even if it is not local in the sense that memory is dead by
2636 the end of founction.
2637 The outer function will see a store in the call assignment
2638 and thus this will do right thing for all uses of this
2639 function in the current IPA passes (modref, pure/const discovery
2640 and inlining heuristics). */
2641 if (DECL_RESULT (current_function_decl
)
2642 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
))
2643 && t
== ssa_default_def (cfun
, DECL_RESULT (current_function_decl
)))
2645 return !ptr_deref_may_alias_global_p (t
, false);
2647 if (TREE_CODE (t
) == ADDR_EXPR
)
2648 return refs_local_or_readonly_memory_p (TREE_OPERAND (t
, 0));
2652 /* Return true if T is a pointer pointing to memory location that is possible
2653 sra candidate if all functions it is passed to are inlined. */
2656 points_to_possible_sra_candidate_p (tree t
)
2658 if (TREE_CODE (t
) != ADDR_EXPR
)
2661 t
= get_base_address (TREE_OPERAND (t
, 0));
2663 /* Automatic variables are fine. */
2665 && auto_var_in_fn_p (t
, current_function_decl
))
2670 /* Analyze function body for NODE.
2671 EARLY indicates run from early optimization pipeline. */
2674 analyze_function_body (struct cgraph_node
*node
, bool early
)
2676 sreal time
= opt_for_fn (node
->decl
, param_uninlined_function_time
);
2677 /* Estimate static overhead for function prologue/epilogue and alignment. */
2678 int size
= opt_for_fn (node
->decl
, param_uninlined_function_insns
);
2679 /* Benefits are scaled by probability of elimination that is in range
2682 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2684 class ipa_fn_summary
*info
= ipa_fn_summaries
->get_create (node
);
2685 ipa_node_params
*params_summary
2686 = early
? NULL
: ipa_node_params_sum
->get (node
);
2687 ipa_predicate bb_predicate
;
2688 struct ipa_func_body_info fbi
;
2689 vec
<ipa_predicate
> nonconstant_names
= vNULL
;
2692 gimple
*fix_builtin_expect_stmt
;
2694 gcc_assert (my_function
&& my_function
->cfg
);
2695 gcc_assert (cfun
== my_function
);
2697 memset(&fbi
, 0, sizeof(fbi
));
2698 vec_free (info
->conds
);
2700 info
->size_time_table
.release ();
2701 info
->call_size_time_table
.release ();
2703 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2704 so we can produce proper inline hints.
2706 When optimizing and analyzing for early inliner, initialize node params
2707 so we can produce correct BB predicates. */
2709 if (opt_for_fn (node
->decl
, optimize
))
2711 calculate_dominance_info (CDI_DOMINATORS
);
2712 calculate_dominance_info (CDI_POST_DOMINATORS
);
2714 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2717 ipa_check_create_node_params ();
2718 ipa_initialize_node_params (node
);
2721 if (ipa_node_params_sum
)
2724 fbi
.info
= ipa_node_params_sum
->get (node
);
2725 fbi
.bb_infos
= vNULL
;
2726 fbi
.bb_infos
.safe_grow_cleared (last_basic_block_for_fn (cfun
), true);
2727 fbi
.param_count
= count_formal_params (node
->decl
);
2728 fbi
.aa_walk_budget
= opt_for_fn (node
->decl
, param_ipa_max_aa_steps
);
2730 nonconstant_names
.safe_grow_cleared
2731 (SSANAMES (my_function
)->length (), true);
2736 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2737 node
->dump_name ());
2739 /* When we run into maximal number of entries, we assign everything to the
2740 constant truth case. Be sure to have it in list. */
2741 bb_predicate
= true;
2742 info
->account_size_time (0, 0, bb_predicate
, bb_predicate
);
2744 bb_predicate
= ipa_predicate::not_inlined ();
2745 info
->account_size_time (opt_for_fn (node
->decl
,
2746 param_uninlined_function_insns
)
2747 * ipa_fn_summary::size_scale
,
2748 opt_for_fn (node
->decl
,
2749 param_uninlined_function_time
),
2753 /* Only look for target information for inlinable functions. */
2754 bool scan_for_target_info
=
2756 && targetm
.target_option
.need_ipa_fn_target_info (node
->decl
,
2760 compute_bb_predicates (&fbi
, node
, info
, params_summary
);
2761 const profile_count entry_count
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
2762 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2763 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2764 for (n
= 0; n
< nblocks
; n
++)
2766 bb
= BASIC_BLOCK_FOR_FN (cfun
, order
[n
]);
2767 freq
= bb
->count
.to_sreal_scale (entry_count
);
2768 if (clobber_only_eh_bb_p (bb
))
2770 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2771 fprintf (dump_file
, "\n Ignoring BB %i;"
2772 " it will be optimized away by cleanup_clobbers\n",
2777 /* TODO: Obviously predicates can be propagated down across CFG. */
2781 bb_predicate
= *(ipa_predicate
*)bb
->aux
;
2783 bb_predicate
= false;
2786 bb_predicate
= true;
2788 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2790 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2791 bb_predicate
.dump (dump_file
, info
->conds
);
2794 if (fbi
.info
&& nonconstant_names
.exists ())
2796 ipa_predicate phi_predicate
;
2797 bool first_phi
= true;
2799 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
2803 && !phi_result_unknown_predicate (&fbi
, info
,
2810 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2812 fprintf (dump_file
, " ");
2813 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0);
2815 predicate_for_phi_result (info
, bsi
.phi (), &phi_predicate
,
2820 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2822 for (gimple_stmt_iterator bsi
= gsi_start_nondebug_bb (bb
);
2823 !gsi_end_p (bsi
); gsi_next_nondebug (&bsi
))
2825 gimple
*stmt
= gsi_stmt (bsi
);
2826 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2827 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2829 ipa_predicate will_be_nonconstant
;
2831 /* This relation stmt should be folded after we remove
2832 __builtin_expect call. Adjust the cost here. */
2833 if (stmt
== fix_builtin_expect_stmt
)
2839 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2841 fprintf (dump_file
, " ");
2842 print_gimple_stmt (dump_file
, stmt
, 0);
2843 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2844 freq
.to_double (), this_size
,
2848 if (is_gimple_call (stmt
)
2849 && !gimple_call_internal_p (stmt
))
2851 struct cgraph_edge
*edge
= node
->get_edge (stmt
);
2852 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2854 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2855 resolved as constant. We however don't want to optimize
2856 out the cgraph edges. */
2857 if (nonconstant_names
.exists ()
2858 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2859 && gimple_call_lhs (stmt
)
2860 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2862 ipa_predicate false_p
= false;
2863 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2866 if (ipa_node_params_sum
)
2868 int count
= gimple_call_num_args (stmt
);
2872 es
->param
.safe_grow_cleared (count
, true);
2873 for (i
= 0; i
< count
; i
++)
2875 int prob
= param_change_prob (&fbi
, stmt
, i
);
2876 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2877 es
->param
[i
].change_prob
= prob
;
2878 es
->param
[i
].points_to_local_or_readonly_memory
2879 = points_to_local_or_readonly_memory_p
2880 (gimple_call_arg (stmt
, i
));
2881 es
->param
[i
].points_to_possible_sra_candidate
2882 = points_to_possible_sra_candidate_p
2883 (gimple_call_arg (stmt
, i
));
2886 /* We cannot setup VLA parameters during inlining. */
2887 for (unsigned int i
= 0; i
< gimple_call_num_args (stmt
); ++i
)
2888 if (TREE_CODE (gimple_call_arg (stmt
, i
)) == WITH_SIZE_EXPR
)
2890 edge
->inline_failed
= CIF_FUNCTION_NOT_INLINABLE
;
2893 es
->call_stmt_size
= this_size
;
2894 es
->call_stmt_time
= this_time
;
2895 es
->loop_depth
= bb_loop_depth (bb
);
2896 edge_set_predicate (edge
, &bb_predicate
);
2897 if (edge
->speculative
)
2899 cgraph_edge
*indirect
2900 = edge
->speculative_call_indirect_edge ();
2901 ipa_call_summary
*es2
2902 = ipa_call_summaries
->get_create (indirect
);
2903 ipa_call_summaries
->duplicate (edge
, indirect
,
2906 /* Edge is the first direct call.
2907 create and duplicate call summaries for multiple
2908 speculative call targets. */
2909 for (cgraph_edge
*direct
2910 = edge
->next_speculative_call_target ();
2912 direct
= direct
->next_speculative_call_target ())
2914 ipa_call_summary
*es3
2915 = ipa_call_summaries
->get_create (direct
);
2916 ipa_call_summaries
->duplicate (edge
, direct
,
2922 /* TODO: When conditional jump or switch is known to be constant, but
2923 we did not translate it into the predicates, we really can account
2924 just maximum of the possible paths. */
2927 = will_be_nonconstant_predicate (&fbi
, info
, params_summary
,
2928 stmt
, nonconstant_names
);
2930 will_be_nonconstant
= true;
2931 if (this_time
|| this_size
)
2933 sreal final_time
= (sreal
)this_time
* freq
;
2934 prob
= eliminated_by_inlining_prob (&fbi
, stmt
);
2935 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2937 "\t\t50%% will be eliminated by inlining\n");
2938 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2939 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2941 ipa_predicate p
= bb_predicate
& will_be_nonconstant
;
2942 int parm
= load_or_store_of_ptr_parameter (&fbi
, stmt
);
2943 ipa_predicate sra_predicate
= true;
2945 sra_predicate
&= add_condition (info
, params_summary
, parm
,
2946 ptr_type_node
, NULL
,
2947 ipa_predicate::not_sra_candidate
, NULL
, 0);
2949 /* We can ignore statement when we proved it is never going
2950 to happen, but we cannot do that for call statements
2951 because edges are accounted specially. */
2953 if (*(is_gimple_call (stmt
) ? &bb_predicate
: &p
) != false)
2959 /* We account everything but the calls. Calls have their own
2960 size/time info attached to cgraph edges. This is necessary
2961 in order to make the cost disappear after inlining. */
2962 if (!is_gimple_call (stmt
))
2967 = bb_predicate
& ipa_predicate::not_inlined () & sra_predicate
;
2968 info
->account_size_time (this_size
* prob
,
2969 (final_time
* prob
) / 2, ip
,
2973 info
->account_size_time (this_size
* (2 - prob
),
2974 (final_time
* (2 - prob
) / 2),
2975 bb_predicate
& sra_predicate
,
2979 if (!info
->fp_expressions
&& fp_expression_p (stmt
))
2981 info
->fp_expressions
= true;
2983 fprintf (dump_file
, " fp_expression set\n");
2987 /* For target specific information, we want to scan all statements
2988 rather than those statements with non-zero weights, to avoid
2989 missing to scan something interesting for target information,
2990 such as: internal function calls. */
2991 if (scan_for_target_info
)
2992 scan_for_target_info
=
2993 targetm
.target_option
.update_ipa_fn_target_info
2994 (info
->target_info
, stmt
);
2996 /* Account cost of address calculations in the statements. */
2997 for (unsigned int i
= 0; i
< gimple_num_ops (stmt
); i
++)
2999 for (tree op
= gimple_op (stmt
, i
);
3000 op
&& handled_component_p (op
);
3001 op
= TREE_OPERAND (op
, 0))
3002 if ((TREE_CODE (op
) == ARRAY_REF
3003 || TREE_CODE (op
) == ARRAY_RANGE_REF
)
3004 && TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
3006 ipa_predicate p
= bb_predicate
;
3008 p
= p
& will_be_nonconstant_expr_predicate
3009 (&fbi
, info
, params_summary
,
3010 TREE_OPERAND (op
, 1),
3018 "\t\tAccounting address calculation.\n");
3019 info
->account_size_time (ipa_fn_summary::size_scale
,
3031 if (nonconstant_names
.exists () && !early
)
3033 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
3034 unsigned max_loop_predicates
= opt_for_fn (node
->decl
,
3035 param_ipa_max_loop_predicates
);
3037 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3038 flow_loops_dump (dump_file
, NULL
, 0);
3040 for (auto loop
: loops_list (cfun
, 0))
3042 ipa_predicate loop_iterations
= true;
3046 class tree_niter_desc niter_desc
;
3047 if (!loop
->header
->aux
)
3050 profile_count phdr_count
= loop_preheader_edge (loop
)->count ();
3051 sreal phdr_freq
= phdr_count
.to_sreal_scale (entry_count
);
3053 bb_predicate
= *(ipa_predicate
*)loop
->header
->aux
;
3054 auto_vec
<edge
> exits
= get_loop_exit_edges (loop
);
3055 FOR_EACH_VEC_ELT (exits
, j
, ex
)
3056 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
3057 && !is_gimple_min_invariant (niter_desc
.niter
))
3059 ipa_predicate will_be_nonconstant
3060 = will_be_nonconstant_expr_predicate (&fbi
, info
,
3064 if (will_be_nonconstant
!= true)
3065 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
3066 if (will_be_nonconstant
!= true
3067 && will_be_nonconstant
!= false)
3068 loop_iterations
&= will_be_nonconstant
;
3070 add_freqcounting_predicate (&s
->loop_iterations
, loop_iterations
,
3071 phdr_freq
, max_loop_predicates
);
3074 /* To avoid quadratic behavior we analyze stride predicates only
3075 with respect to the containing loop. Thus we simply iterate
3076 over all defs in the outermost loop body. */
3077 for (class loop
*loop
= loops_for_fn (cfun
)->tree_root
->inner
;
3078 loop
!= NULL
; loop
= loop
->next
)
3080 ipa_predicate loop_stride
= true;
3081 basic_block
*body
= get_loop_body (loop
);
3082 profile_count phdr_count
= loop_preheader_edge (loop
)->count ();
3083 sreal phdr_freq
= phdr_count
.to_sreal_scale (entry_count
);
3084 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
3086 gimple_stmt_iterator gsi
;
3090 bb_predicate
= *(ipa_predicate
*)body
[i
]->aux
;
3091 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
3094 gimple
*stmt
= gsi_stmt (gsi
);
3096 if (!is_gimple_assign (stmt
))
3099 tree def
= gimple_assign_lhs (stmt
);
3100 if (TREE_CODE (def
) != SSA_NAME
)
3104 if (!simple_iv (loop_containing_stmt (stmt
),
3105 loop_containing_stmt (stmt
),
3107 || is_gimple_min_invariant (iv
.step
))
3110 ipa_predicate will_be_nonconstant
3111 = will_be_nonconstant_expr_predicate (&fbi
, info
,
3115 if (will_be_nonconstant
!= true)
3116 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
3117 if (will_be_nonconstant
!= true
3118 && will_be_nonconstant
!= false)
3119 loop_stride
= loop_stride
& will_be_nonconstant
;
3122 add_freqcounting_predicate (&s
->loop_strides
, loop_stride
,
3123 phdr_freq
, max_loop_predicates
);
3128 FOR_ALL_BB_FN (bb
, my_function
)
3134 edge_predicate_pool
.remove ((ipa_predicate
*)bb
->aux
);
3136 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3139 edge_predicate_pool
.remove ((ipa_predicate
*)e
->aux
);
3143 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
3144 ipa_size_summary
*ss
= ipa_size_summaries
->get (node
);
3146 ss
->self_size
= size
;
3147 nonconstant_names
.release ();
3148 ipa_release_body_info (&fbi
);
3149 if (opt_for_fn (node
->decl
, optimize
))
3152 loop_optimizer_finalize ();
3153 else if (!ipa_edge_args_sum
)
3154 ipa_free_all_node_params ();
3155 free_dominance_info (CDI_DOMINATORS
);
3156 free_dominance_info (CDI_POST_DOMINATORS
);
3160 fprintf (dump_file
, "\n");
3161 ipa_dump_fn_summary (dump_file
, node
);
3166 /* Compute function summary.
3167 EARLY is true when we compute parameters during early opts. */
3170 compute_fn_summary (struct cgraph_node
*node
, bool early
)
3172 HOST_WIDE_INT self_stack_size
;
3173 struct cgraph_edge
*e
;
3175 gcc_assert (!node
->inlined_to
);
3177 if (!ipa_fn_summaries
)
3178 ipa_fn_summary_alloc ();
3180 /* Create a new ipa_fn_summary. */
3181 ((ipa_fn_summary_t
*)ipa_fn_summaries
)->remove_callees (node
);
3182 ipa_fn_summaries
->remove (node
);
3183 class ipa_fn_summary
*info
= ipa_fn_summaries
->get_create (node
);
3184 class ipa_size_summary
*size_info
= ipa_size_summaries
->get_create (node
);
3186 /* Estimate the stack size for the function if we're optimizing. */
3187 self_stack_size
= optimize
&& !node
->thunk
3188 ? estimated_stack_frame_size (node
) : 0;
3189 size_info
->estimated_self_stack_size
= self_stack_size
;
3190 info
->estimated_stack_size
= self_stack_size
;
3194 ipa_call_summary
*es
= ipa_call_summaries
->get_create (node
->callees
);
3195 ipa_predicate t
= true;
3197 node
->can_change_signature
= false;
3198 es
->call_stmt_size
= eni_size_weights
.call_cost
;
3199 es
->call_stmt_time
= eni_time_weights
.call_cost
;
3200 info
->account_size_time (ipa_fn_summary::size_scale
3201 * opt_for_fn (node
->decl
,
3202 param_uninlined_function_thunk_insns
),
3203 opt_for_fn (node
->decl
,
3204 param_uninlined_function_thunk_time
), t
, t
);
3205 t
= ipa_predicate::not_inlined ();
3206 info
->account_size_time (2 * ipa_fn_summary::size_scale
, 0, t
, t
);
3207 ipa_update_overall_fn_summary (node
);
3208 size_info
->self_size
= size_info
->size
;
3209 if (stdarg_p (TREE_TYPE (node
->decl
)))
3211 info
->inlinable
= false;
3212 node
->callees
->inline_failed
= CIF_VARIADIC_THUNK
;
3215 info
->inlinable
= true;
3219 /* Even is_gimple_min_invariant rely on current_function_decl. */
3220 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3222 /* During IPA profile merging we may be called w/o virtual SSA form
3224 update_ssa (TODO_update_ssa_only_virtuals
);
3226 /* Can this function be inlined at all? */
3227 if (!opt_for_fn (node
->decl
, optimize
)
3228 && !lookup_attribute ("always_inline",
3229 DECL_ATTRIBUTES (node
->decl
)))
3230 info
->inlinable
= false;
3232 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
3234 bool no_signature
= false;
3235 /* Type attributes can use parameter indices to describe them.
3236 Special case fn spec since we can safely preserve them in
3237 modref summaries. */
3238 for (tree list
= TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
));
3239 list
&& !no_signature
; list
= TREE_CHAIN (list
))
3240 if (!ipa_param_adjustments::type_attribute_allowed_p
3241 (get_attribute_name (list
)))
3245 fprintf (dump_file
, "No signature change:"
3246 " function type has unhandled attribute %s.\n",
3247 IDENTIFIER_POINTER (get_attribute_name (list
)));
3249 no_signature
= true;
3251 for (tree parm
= DECL_ARGUMENTS (node
->decl
);
3252 parm
&& !no_signature
; parm
= DECL_CHAIN (parm
))
3253 if (variably_modified_type_p (TREE_TYPE (parm
), node
->decl
))
3257 fprintf (dump_file
, "No signature change:"
3258 " has parameter with variably modified type.\n");
3260 no_signature
= true;
3263 /* Likewise for #pragma omp declare simd functions or functions
3264 with simd attribute. */
3266 || lookup_attribute ("omp declare simd",
3267 DECL_ATTRIBUTES (node
->decl
)))
3268 node
->can_change_signature
= false;
3271 /* Otherwise, inlinable functions always can change signature. */
3272 if (info
->inlinable
)
3273 node
->can_change_signature
= true;
3276 /* Functions calling builtin_apply cannot change signature. */
3277 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3279 tree
cdecl = e
->callee
->decl
;
3280 if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS
,
3284 node
->can_change_signature
= !e
;
3287 analyze_function_body (node
, early
);
3291 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
3292 size_info
->size
= size_info
->self_size
;
3293 info
->estimated_stack_size
= size_info
->estimated_self_stack_size
;
3295 /* Code above should compute exactly the same result as
3296 ipa_update_overall_fn_summary except for case when speculative
3297 edges are present since these are accounted to size but not
3298 self_size. Do not compare time since different order the roundoff
3299 errors result in slight changes. */
3300 ipa_update_overall_fn_summary (node
);
3303 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3306 gcc_assert (e
|| size_info
->size
== size_info
->self_size
);
3311 /* Compute parameters of functions used by inliner using
3312 current_function_decl. */
3315 compute_fn_summary_for_current (void)
3317 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
3321 /* Estimate benefit devirtualizing indirect edge IE and return true if it can
3322 be devirtualized and inlined, provided m_known_vals, m_known_contexts and
3323 m_known_aggs in AVALS. Return false straight away if AVALS is NULL. */
3326 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
3327 int *size
, int *time
,
3328 ipa_call_arg_values
*avals
)
3331 struct cgraph_node
*callee
;
3332 class ipa_fn_summary
*isummary
;
3333 enum availability avail
;
3337 || (!avals
->m_known_vals
.length() && !avals
->m_known_contexts
.length ()))
3339 if (!opt_for_fn (ie
->caller
->decl
, flag_indirect_inlining
))
3342 target
= ipa_get_indirect_edge_target (ie
, avals
, &speculative
);
3343 if (!target
|| speculative
)
3346 /* Account for difference in cost between indirect and direct calls. */
3347 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
3348 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
3349 gcc_checking_assert (*time
>= 0);
3350 gcc_checking_assert (*size
>= 0);
3352 callee
= cgraph_node::get (target
);
3353 if (!callee
|| !callee
->definition
)
3355 callee
= callee
->function_symbol (&avail
);
3356 if (avail
< AVAIL_AVAILABLE
)
3358 isummary
= ipa_fn_summaries
->get (callee
);
3359 if (isummary
== NULL
)
3362 return isummary
->inlinable
;
3365 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3366 handle edge E with probability PROB. Set HINTS accordingly if edge may be
3367 devirtualized. AVALS, if non-NULL, describes the context of the call site
3368 as far as values of parameters are concerened. */
3371 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *min_size
,
3372 sreal
*time
, ipa_call_arg_values
*avals
,
3375 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3376 int call_size
= es
->call_stmt_size
;
3377 int call_time
= es
->call_stmt_time
;
3380 if (!e
->callee
&& hints
&& e
->maybe_hot_p ()
3381 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
, avals
))
3382 *hints
|= INLINE_HINT_indirect_call
;
3383 cur_size
= call_size
* ipa_fn_summary::size_scale
;
3386 *min_size
+= cur_size
;
3388 *time
+= ((sreal
)call_time
) * e
->sreal_frequency ();
3392 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3393 calls in NODE. POSSIBLE_TRUTHS and AVALS describe the context of the call
3396 Helper for estimate_calls_size_and_time which does the same but
3397 (in most cases) faster. */
3400 estimate_calls_size_and_time_1 (struct cgraph_node
*node
, int *size
,
3401 int *min_size
, sreal
*time
,
3403 clause_t possible_truths
,
3404 ipa_call_arg_values
*avals
)
3406 struct cgraph_edge
*e
;
3407 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3409 if (!e
->inline_failed
)
3411 gcc_checking_assert (!ipa_call_summaries
->get (e
));
3412 estimate_calls_size_and_time_1 (e
->callee
, size
, min_size
, time
,
3413 hints
, possible_truths
, avals
);
3417 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3419 /* Do not care about zero sized builtins. */
3420 if (!es
->call_stmt_size
)
3422 gcc_checking_assert (!es
->call_stmt_time
);
3426 || es
->predicate
->evaluate (possible_truths
))
3428 /* Predicates of calls shall not use NOT_CHANGED codes,
3429 so we do not need to compute probabilities. */
3430 estimate_edge_size_and_time (e
, size
,
3431 es
->predicate
? NULL
: min_size
,
3432 time
, avals
, hints
);
3435 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3437 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3439 || es
->predicate
->evaluate (possible_truths
))
3440 estimate_edge_size_and_time (e
, size
,
3441 es
->predicate
? NULL
: min_size
,
3442 time
, avals
, hints
);
3446 /* Populate sum->call_size_time_table for edges from NODE. */
3449 summarize_calls_size_and_time (struct cgraph_node
*node
,
3450 ipa_fn_summary
*sum
)
3452 struct cgraph_edge
*e
;
3453 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3455 if (!e
->inline_failed
)
3457 gcc_checking_assert (!ipa_call_summaries
->get (e
));
3458 summarize_calls_size_and_time (e
->callee
, sum
);
3464 estimate_edge_size_and_time (e
, &size
, NULL
, &time
, NULL
, NULL
);
3466 ipa_predicate pred
= true;
3467 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3470 pred
= *es
->predicate
;
3471 sum
->account_size_time (size
, time
, pred
, pred
, true);
3473 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3478 estimate_edge_size_and_time (e
, &size
, NULL
, &time
, NULL
, NULL
);
3479 ipa_predicate pred
= true;
3480 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3483 pred
= *es
->predicate
;
3484 sum
->account_size_time (size
, time
, pred
, pred
, true);
3488 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3489 calls in NODE. POSSIBLE_TRUTHS and AVALS (the latter if non-NULL) describe
3490 context of the call site. */
3493 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
,
3494 int *min_size
, sreal
*time
,
3496 clause_t possible_truths
,
3497 ipa_call_arg_values
*avals
)
3499 class ipa_fn_summary
*sum
= ipa_fn_summaries
->get (node
);
3500 bool use_table
= true;
3502 gcc_assert (node
->callees
|| node
->indirect_calls
);
3504 /* During early inlining we do not calculate info for very
3505 large functions and thus there is no need for producing
3507 if (!ipa_node_params_sum
)
3509 /* Do not calculate summaries for simple wrappers; it is waste
3511 else if (node
->callees
&& node
->indirect_calls
3512 && node
->callees
->inline_failed
&& !node
->callees
->next_callee
)
3514 /* If there is an indirect edge that may be optimized, we need
3515 to go the slow way. */
3516 else if (avals
&& hints
3517 && (avals
->m_known_vals
.length ()
3518 || avals
->m_known_contexts
.length ()
3519 || avals
->m_known_aggs
.length ()))
3521 ipa_node_params
*params_summary
= ipa_node_params_sum
->get (node
);
3522 unsigned int nargs
= params_summary
3523 ? ipa_get_param_count (params_summary
) : 0;
3525 for (unsigned int i
= 0; i
< nargs
&& use_table
; i
++)
3527 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3528 && (avals
->safe_sval_at (i
)
3529 || (ipa_argagg_value_list (avals
).value_for_index_p (i
))))
3531 else if (ipa_is_param_used_by_polymorphic_call (params_summary
, i
)
3532 && (avals
->m_known_contexts
.length () > i
3533 && !avals
->m_known_contexts
[i
].useless_p ()))
3538 /* Fast path is via the call size time table. */
3541 /* Build summary if it is absent. */
3542 if (!sum
->call_size_time_table
.length ())
3544 ipa_predicate true_pred
= true;
3545 sum
->account_size_time (0, 0, true_pred
, true_pred
, true);
3546 summarize_calls_size_and_time (node
, sum
);
3549 int old_size
= *size
;
3550 sreal old_time
= time
? *time
: 0;
3553 *min_size
+= sum
->call_size_time_table
[0].size
;
3558 /* Walk the table and account sizes and times. */
3559 for (i
= 0; sum
->call_size_time_table
.iterate (i
, &e
);
3561 if (e
->exec_predicate
.evaluate (possible_truths
))
3568 /* Be careful and see if both methods agree. */
3569 if ((flag_checking
|| dump_file
)
3570 /* Do not try to sanity check when we know we lost some
3572 && sum
->call_size_time_table
.length ()
3573 < ipa_fn_summary::max_size_time_table_size
)
3575 estimate_calls_size_and_time_1 (node
, &old_size
, NULL
, &old_time
, NULL
,
3576 possible_truths
, avals
);
3577 gcc_assert (*size
== old_size
);
3578 if (time
&& (*time
- old_time
> 1 || *time
- old_time
< -1)
3580 fprintf (dump_file
, "Time mismatch in call summary %f!=%f\n",
3581 old_time
.to_double (),
3582 time
->to_double ());
3585 /* Slow path by walking all edges. */
3587 estimate_calls_size_and_time_1 (node
, size
, min_size
, time
, hints
,
3588 possible_truths
, avals
);
3591 /* Main constructor for ipa call context. Memory allocation of ARG_VALUES
3592 is owned by the caller. INLINE_PARAM_SUMMARY is also owned by the
3595 ipa_call_context::ipa_call_context (cgraph_node
*node
, clause_t possible_truths
,
3596 clause_t nonspec_possible_truths
,
3597 vec
<inline_param_summary
>
3598 inline_param_summary
,
3599 ipa_auto_call_arg_values
*arg_values
)
3600 : m_node (node
), m_possible_truths (possible_truths
),
3601 m_nonspec_possible_truths (nonspec_possible_truths
),
3602 m_inline_param_summary (inline_param_summary
),
3603 m_avals (arg_values
)
3607 /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3610 ipa_cached_call_context::duplicate_from (const ipa_call_context
&ctx
)
3612 m_node
= ctx
.m_node
;
3613 m_possible_truths
= ctx
.m_possible_truths
;
3614 m_nonspec_possible_truths
= ctx
.m_nonspec_possible_truths
;
3615 ipa_node_params
*params_summary
= ipa_node_params_sum
->get (m_node
);
3616 unsigned int nargs
= params_summary
3617 ? ipa_get_param_count (params_summary
) : 0;
3619 m_inline_param_summary
= vNULL
;
3620 /* Copy the info only if there is at least one useful entry. */
3621 if (ctx
.m_inline_param_summary
.exists ())
3623 unsigned int n
= MIN (ctx
.m_inline_param_summary
.length (), nargs
);
3625 for (unsigned int i
= 0; i
< n
; i
++)
3626 if (ipa_is_param_used_by_ipa_predicates (params_summary
, i
)
3627 && !ctx
.m_inline_param_summary
[i
].useless_p ())
3629 m_inline_param_summary
3630 = ctx
.m_inline_param_summary
.copy ();
3634 m_avals
.m_known_vals
= vNULL
;
3635 if (ctx
.m_avals
.m_known_vals
.exists ())
3637 unsigned int n
= MIN (ctx
.m_avals
.m_known_vals
.length (), nargs
);
3639 for (unsigned int i
= 0; i
< n
; i
++)
3640 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3641 && ctx
.m_avals
.m_known_vals
[i
])
3643 m_avals
.m_known_vals
= ctx
.m_avals
.m_known_vals
.copy ();
3648 m_avals
.m_known_contexts
= vNULL
;
3649 if (ctx
.m_avals
.m_known_contexts
.exists ())
3651 unsigned int n
= MIN (ctx
.m_avals
.m_known_contexts
.length (), nargs
);
3653 for (unsigned int i
= 0; i
< n
; i
++)
3654 if (ipa_is_param_used_by_polymorphic_call (params_summary
, i
)
3655 && !ctx
.m_avals
.m_known_contexts
[i
].useless_p ())
3657 m_avals
.m_known_contexts
= ctx
.m_avals
.m_known_contexts
.copy ();
3662 m_avals
.m_known_aggs
= vNULL
;
3663 if (ctx
.m_avals
.m_known_aggs
.exists ())
3665 const ipa_argagg_value_list
avl (&ctx
.m_avals
);
3666 for (unsigned int i
= 0; i
< nargs
; i
++)
3667 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3668 && avl
.value_for_index_p (i
))
3670 m_avals
.m_known_aggs
= ctx
.m_avals
.m_known_aggs
.copy ();
3675 m_avals
.m_known_value_ranges
= vNULL
;
3678 /* Release memory used by known_vals/contexts/aggs vectors. and
3679 inline_param_summary. */
3682 ipa_cached_call_context::release ()
3684 /* See if context is initialized at first place. */
3687 m_avals
.m_known_aggs
.release ();
3688 m_avals
.m_known_vals
.release ();
3689 m_avals
.m_known_contexts
.release ();
3690 m_inline_param_summary
.release ();
3693 /* Return true if CTX describes the same call context as THIS. */
3696 ipa_call_context::equal_to (const ipa_call_context
&ctx
)
3698 if (m_node
!= ctx
.m_node
3699 || m_possible_truths
!= ctx
.m_possible_truths
3700 || m_nonspec_possible_truths
!= ctx
.m_nonspec_possible_truths
)
3703 ipa_node_params
*params_summary
= ipa_node_params_sum
->get (m_node
);
3704 unsigned int nargs
= params_summary
3705 ? ipa_get_param_count (params_summary
) : 0;
3707 if (m_inline_param_summary
.exists () || ctx
.m_inline_param_summary
.exists ())
3709 for (unsigned int i
= 0; i
< nargs
; i
++)
3711 if (!ipa_is_param_used_by_ipa_predicates (params_summary
, i
))
3713 if (i
>= m_inline_param_summary
.length ()
3714 || m_inline_param_summary
[i
].useless_p ())
3716 if (i
< ctx
.m_inline_param_summary
.length ()
3717 && !ctx
.m_inline_param_summary
[i
].useless_p ())
3721 if (i
>= ctx
.m_inline_param_summary
.length ()
3722 || ctx
.m_inline_param_summary
[i
].useless_p ())
3724 if (i
< m_inline_param_summary
.length ()
3725 && !m_inline_param_summary
[i
].useless_p ())
3729 if (!m_inline_param_summary
[i
].equal_to
3730 (ctx
.m_inline_param_summary
[i
]))
3734 if (m_avals
.m_known_vals
.exists () || ctx
.m_avals
.m_known_vals
.exists ())
3736 for (unsigned int i
= 0; i
< nargs
; i
++)
3738 if (!ipa_is_param_used_by_indirect_call (params_summary
, i
))
3740 if (i
>= m_avals
.m_known_vals
.length () || !m_avals
.m_known_vals
[i
])
3742 if (i
< ctx
.m_avals
.m_known_vals
.length ()
3743 && ctx
.m_avals
.m_known_vals
[i
])
3747 if (i
>= ctx
.m_avals
.m_known_vals
.length ()
3748 || !ctx
.m_avals
.m_known_vals
[i
])
3750 if (i
< m_avals
.m_known_vals
.length () && m_avals
.m_known_vals
[i
])
3754 if (m_avals
.m_known_vals
[i
] != ctx
.m_avals
.m_known_vals
[i
])
3758 if (m_avals
.m_known_contexts
.exists ()
3759 || ctx
.m_avals
.m_known_contexts
.exists ())
3761 for (unsigned int i
= 0; i
< nargs
; i
++)
3763 if (!ipa_is_param_used_by_polymorphic_call (params_summary
, i
))
3765 if (i
>= m_avals
.m_known_contexts
.length ()
3766 || m_avals
.m_known_contexts
[i
].useless_p ())
3768 if (i
< ctx
.m_avals
.m_known_contexts
.length ()
3769 && !ctx
.m_avals
.m_known_contexts
[i
].useless_p ())
3773 if (i
>= ctx
.m_avals
.m_known_contexts
.length ()
3774 || ctx
.m_avals
.m_known_contexts
[i
].useless_p ())
3776 if (i
< m_avals
.m_known_contexts
.length ()
3777 && !m_avals
.m_known_contexts
[i
].useless_p ())
3781 if (!m_avals
.m_known_contexts
[i
].equal_to
3782 (ctx
.m_avals
.m_known_contexts
[i
]))
3786 if (m_avals
.m_known_aggs
.exists () || ctx
.m_avals
.m_known_aggs
.exists ())
3788 unsigned i
= 0, j
= 0;
3789 while (i
< m_avals
.m_known_aggs
.length ()
3790 || j
< ctx
.m_avals
.m_known_aggs
.length ())
3792 if (i
>= m_avals
.m_known_aggs
.length ())
3794 int idx2
= ctx
.m_avals
.m_known_aggs
[j
].index
;
3795 if (ipa_is_param_used_by_indirect_call (params_summary
, idx2
))
3800 if (j
>= ctx
.m_avals
.m_known_aggs
.length ())
3802 int idx1
= m_avals
.m_known_aggs
[i
].index
;
3803 if (ipa_is_param_used_by_indirect_call (params_summary
, idx1
))
3809 int idx1
= m_avals
.m_known_aggs
[i
].index
;
3810 int idx2
= ctx
.m_avals
.m_known_aggs
[j
].index
;
3813 if (ipa_is_param_used_by_indirect_call (params_summary
, idx1
))
3820 if (ipa_is_param_used_by_indirect_call (params_summary
, idx2
))
3825 if (!ipa_is_param_used_by_indirect_call (params_summary
, idx1
))
3832 if ((m_avals
.m_known_aggs
[i
].unit_offset
3833 != ctx
.m_avals
.m_known_aggs
[j
].unit_offset
)
3834 || (m_avals
.m_known_aggs
[i
].by_ref
3835 != ctx
.m_avals
.m_known_aggs
[j
].by_ref
)
3836 || !operand_equal_p (m_avals
.m_known_aggs
[i
].value
,
3837 ctx
.m_avals
.m_known_aggs
[j
].value
))
3846 /* Fill in the selected fields in ESTIMATES with value estimated for call in
3847 this context. Always compute size and min_size. Only compute time and
3848 nonspecialized_time if EST_TIMES is true. Only compute hints if EST_HINTS
3852 ipa_call_context::estimate_size_and_time (ipa_call_estimates
*estimates
,
3853 bool est_times
, bool est_hints
)
3855 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (m_node
);
3860 ipa_hints hints
= 0;
3861 sreal loops_with_known_iterations
= 0;
3862 sreal loops_with_known_strides
= 0;
3865 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3868 fprintf (dump_file
, " Estimating body: %s\n"
3869 " Known to be false: ", m_node
->dump_name ());
3871 for (i
= ipa_predicate::not_inlined_condition
;
3872 i
< (ipa_predicate::first_dynamic_condition
3873 + (int) vec_safe_length (info
->conds
)); i
++)
3874 if (!(m_possible_truths
& (1 << i
)))
3877 fprintf (dump_file
, ", ");
3879 dump_condition (dump_file
, info
->conds
, i
);
3883 if (m_node
->callees
|| m_node
->indirect_calls
)
3884 estimate_calls_size_and_time (m_node
, &size
, &min_size
,
3885 est_times
? &time
: NULL
,
3886 est_hints
? &hints
: NULL
, m_possible_truths
,
3889 sreal nonspecialized_time
= time
;
3891 min_size
+= info
->size_time_table
[0].size
;
3892 for (i
= 0; info
->size_time_table
.iterate (i
, &e
); i
++)
3894 bool exec
= e
->exec_predicate
.evaluate (m_nonspec_possible_truths
);
3896 /* Because predicates are conservative, it can happen that nonconst is 1
3900 bool nonconst
= e
->nonconst_predicate
.evaluate (m_possible_truths
);
3902 gcc_checking_assert (e
->time
>= 0);
3903 gcc_checking_assert (time
>= 0);
3905 /* We compute specialized size only because size of nonspecialized
3906 copy is context independent.
3908 The difference between nonspecialized execution and specialized is
3909 that nonspecialized is not going to have optimized out computations
3910 known to be constant in a specialized setting. */
3915 nonspecialized_time
+= e
->time
;
3918 else if (!m_inline_param_summary
.exists ())
3925 int prob
= e
->nonconst_predicate
.probability
3926 (info
->conds
, m_possible_truths
,
3927 m_inline_param_summary
);
3928 gcc_checking_assert (prob
>= 0);
3929 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3930 if (prob
== REG_BR_PROB_BASE
)
3933 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
3935 gcc_checking_assert (time
>= 0);
3938 gcc_checking_assert (info
->size_time_table
[0].exec_predicate
== true);
3939 gcc_checking_assert (info
->size_time_table
[0].nonconst_predicate
== true);
3940 gcc_checking_assert (min_size
>= 0);
3941 gcc_checking_assert (size
>= 0);
3942 gcc_checking_assert (time
>= 0);
3943 /* nonspecialized_time should be always bigger than specialized time.
3944 Roundoff issues however may get into the way. */
3945 gcc_checking_assert ((nonspecialized_time
- time
* 99 / 100) >= -1);
3947 /* Roundoff issues may make specialized time bigger than nonspecialized
3948 time. We do not really want that to happen because some heuristics
3949 may get confused by seeing negative speedups. */
3950 if (time
> nonspecialized_time
)
3951 time
= nonspecialized_time
;
3956 hints
|= INLINE_HINT_in_scc
;
3957 if (DECL_DECLARED_INLINE_P (m_node
->decl
))
3958 hints
|= INLINE_HINT_declared_inline
;
3959 if (info
->builtin_constant_p_parms
.length ()
3960 && DECL_DECLARED_INLINE_P (m_node
->decl
))
3961 hints
|= INLINE_HINT_builtin_constant_p
;
3963 ipa_freqcounting_predicate
*fcp
;
3964 for (i
= 0; vec_safe_iterate (info
->loop_iterations
, i
, &fcp
); i
++)
3965 if (!fcp
->predicate
->evaluate (m_possible_truths
))
3967 hints
|= INLINE_HINT_loop_iterations
;
3968 loops_with_known_iterations
+= fcp
->freq
;
3970 estimates
->loops_with_known_iterations
= loops_with_known_iterations
;
3972 for (i
= 0; vec_safe_iterate (info
->loop_strides
, i
, &fcp
); i
++)
3973 if (!fcp
->predicate
->evaluate (m_possible_truths
))
3975 hints
|= INLINE_HINT_loop_stride
;
3976 loops_with_known_strides
+= fcp
->freq
;
3978 estimates
->loops_with_known_strides
= loops_with_known_strides
;
3981 size
= RDIV (size
, ipa_fn_summary::size_scale
);
3982 min_size
= RDIV (min_size
, ipa_fn_summary::size_scale
);
3984 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3986 fprintf (dump_file
, "\n size:%i", (int) size
);
3988 fprintf (dump_file
, " time:%f nonspec time:%f",
3989 time
.to_double (), nonspecialized_time
.to_double ());
3991 fprintf (dump_file
, " loops with known iterations:%f "
3992 "known strides:%f", loops_with_known_iterations
.to_double (),
3993 loops_with_known_strides
.to_double ());
3994 fprintf (dump_file
, "\n");
3998 estimates
->time
= time
;
3999 estimates
->nonspecialized_time
= nonspecialized_time
;
4001 estimates
->size
= size
;
4002 estimates
->min_size
= min_size
;
4004 estimates
->hints
= hints
;
4009 /* Estimate size and time needed to execute callee of EDGE assuming that
4010 parameters known to be constant at caller of EDGE are propagated.
4011 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
4012 and types for parameters. */
4015 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
4016 ipa_auto_call_arg_values
*avals
,
4017 ipa_call_estimates
*estimates
)
4019 clause_t clause
, nonspec_clause
;
4021 evaluate_conditions_for_known_args (node
, false, avals
, &clause
,
4022 &nonspec_clause
, NULL
);
4023 ipa_call_context
ctx (node
, clause
, nonspec_clause
, vNULL
, avals
);
4024 ctx
.estimate_size_and_time (estimates
);
4027 /* Return stack frame offset where frame of NODE is supposed to start inside
4028 of the function it is inlined to.
4029 Return 0 for functions that are not inlined. */
4032 ipa_get_stack_frame_offset (struct cgraph_node
*node
)
4034 HOST_WIDE_INT offset
= 0;
4035 if (!node
->inlined_to
)
4037 node
= node
->callers
->caller
;
4040 offset
+= ipa_size_summaries
->get (node
)->estimated_self_stack_size
;
4041 if (!node
->inlined_to
)
4043 node
= node
->callers
->caller
;
4048 /* Update summary information of inline clones after inlining.
4049 Compute peak stack usage. */
4052 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
4054 struct cgraph_edge
*e
;
4056 ipa_propagate_frequency (node
);
4057 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4059 if (!e
->inline_failed
)
4060 inline_update_callee_summaries (e
->callee
, depth
);
4062 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
4064 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
4065 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
4068 /* Update change_prob and points_to_local_or_readonly_memory of EDGE after
4069 INLINED_EDGE has been inlined.
4071 When function A is inlined in B and A calls C with parameter that
4072 changes with probability PROB1 and C is known to be passthrough
4073 of argument if B that change with probability PROB2, the probability
4074 of change is now PROB1*PROB2. */
4077 remap_edge_params (struct cgraph_edge
*inlined_edge
,
4078 struct cgraph_edge
*edge
)
4080 if (ipa_node_params_sum
)
4083 ipa_edge_args
*args
= ipa_edge_args_sum
->get (edge
);
4086 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
4087 class ipa_call_summary
*inlined_es
4088 = ipa_call_summaries
->get (inlined_edge
);
4090 if (es
->param
.length () == 0)
4093 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
4095 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
4096 if (jfunc
->type
== IPA_JF_PASS_THROUGH
4097 || jfunc
->type
== IPA_JF_ANCESTOR
)
4099 int id
= jfunc
->type
== IPA_JF_PASS_THROUGH
4100 ? ipa_get_jf_pass_through_formal_id (jfunc
)
4101 : ipa_get_jf_ancestor_formal_id (jfunc
);
4102 if (id
< (int) inlined_es
->param
.length ())
4104 int prob1
= es
->param
[i
].change_prob
;
4105 int prob2
= inlined_es
->param
[id
].change_prob
;
4106 int prob
= combine_probabilities (prob1
, prob2
);
4108 if (prob1
&& prob2
&& !prob
)
4111 es
->param
[i
].change_prob
= prob
;
4114 ->param
[id
].points_to_local_or_readonly_memory
)
4115 es
->param
[i
].points_to_local_or_readonly_memory
= true;
4117 ->param
[id
].points_to_possible_sra_candidate
)
4118 es
->param
[i
].points_to_possible_sra_candidate
= true;
4120 if (!es
->param
[i
].points_to_local_or_readonly_memory
4121 && jfunc
->type
== IPA_JF_CONST
4122 && points_to_local_or_readonly_memory_p
4123 (ipa_get_jf_constant (jfunc
)))
4124 es
->param
[i
].points_to_local_or_readonly_memory
= true;
4130 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
4132 Remap predicates of callees of NODE. Rest of arguments match
4135 Also update change probabilities. */
4138 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
4139 struct cgraph_node
*node
,
4140 class ipa_fn_summary
*info
,
4141 class ipa_node_params
*params_summary
,
4142 class ipa_fn_summary
*callee_info
,
4143 const vec
<int> &operand_map
,
4144 const vec
<HOST_WIDE_INT
> &offset_map
,
4145 clause_t possible_truths
,
4146 ipa_predicate
*toplev_predicate
)
4148 struct cgraph_edge
*e
, *next
;
4149 for (e
= node
->callees
; e
; e
= next
)
4152 next
= e
->next_callee
;
4154 if (e
->inline_failed
)
4156 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
4157 remap_edge_params (inlined_edge
, e
);
4161 p
= es
->predicate
->remap_after_inlining
4162 (info
, params_summary
,
4163 callee_info
, operand_map
,
4164 offset_map
, possible_truths
,
4166 edge_set_predicate (e
, &p
);
4169 edge_set_predicate (e
, toplev_predicate
);
4172 remap_edge_summaries (inlined_edge
, e
->callee
, info
,
4173 params_summary
, callee_info
,
4174 operand_map
, offset_map
, possible_truths
,
4177 for (e
= node
->indirect_calls
; e
; e
= next
)
4179 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
4181 next
= e
->next_callee
;
4183 remap_edge_params (inlined_edge
, e
);
4186 p
= es
->predicate
->remap_after_inlining
4187 (info
, params_summary
,
4188 callee_info
, operand_map
, offset_map
,
4189 possible_truths
, *toplev_predicate
);
4190 edge_set_predicate (e
, &p
);
4193 edge_set_predicate (e
, toplev_predicate
);
4197 /* Run remap_after_inlining on each predicate in V. */
4200 remap_freqcounting_predicate (class ipa_fn_summary
*info
,
4201 class ipa_node_params
*params_summary
,
4202 class ipa_fn_summary
*callee_info
,
4203 vec
<ipa_freqcounting_predicate
, va_gc
> *v
,
4204 const vec
<int> &operand_map
,
4205 const vec
<HOST_WIDE_INT
> &offset_map
,
4206 clause_t possible_truths
,
4207 ipa_predicate
*toplev_predicate
)
4210 ipa_freqcounting_predicate
*fcp
;
4211 for (int i
= 0; vec_safe_iterate (v
, i
, &fcp
); i
++)
4214 = fcp
->predicate
->remap_after_inlining (info
, params_summary
,
4215 callee_info
, operand_map
,
4216 offset_map
, possible_truths
,
4218 if (p
!= false && p
!= true)
4219 *fcp
->predicate
&= p
;
4223 /* We inlined EDGE. Update summary of the function we inlined into. */
4226 ipa_merge_fn_summary_after_inlining (struct cgraph_edge
*edge
)
4228 ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (edge
->callee
);
4229 struct cgraph_node
*to
= (edge
->caller
->inlined_to
4230 ? edge
->caller
->inlined_to
: edge
->caller
);
4231 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (to
);
4232 clause_t clause
= 0; /* not_inline is known to be false. */
4234 auto_vec
<int, 8> operand_map
;
4235 auto_vec
<HOST_WIDE_INT
, 8> offset_map
;
4237 ipa_predicate toplev_predicate
;
4238 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
4239 ipa_node_params
*params_summary
= (ipa_node_params_sum
4240 ? ipa_node_params_sum
->get (to
) : NULL
);
4243 toplev_predicate
= *es
->predicate
;
4245 toplev_predicate
= true;
4247 info
->fp_expressions
|= callee_info
->fp_expressions
;
4248 info
->target_info
|= callee_info
->target_info
;
4250 if (callee_info
->conds
)
4252 ipa_auto_call_arg_values avals
;
4253 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, &avals
, false);
4255 if (ipa_node_params_sum
&& callee_info
->conds
)
4257 ipa_edge_args
*args
= ipa_edge_args_sum
->get (edge
);
4258 int count
= args
? ipa_get_cs_argument_count (args
) : 0;
4263 operand_map
.safe_grow_cleared (count
, true);
4264 offset_map
.safe_grow_cleared (count
, true);
4266 for (i
= 0; i
< count
; i
++)
4268 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
4271 /* TODO: handle non-NOPs when merging. */
4272 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
4274 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
4275 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
4276 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
4279 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
4281 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
4282 if (offset
>= 0 && offset
< INT_MAX
)
4284 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
4285 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
4287 offset_map
[i
] = offset
;
4290 operand_map
[i
] = map
;
4291 gcc_assert (map
< ipa_get_param_count (params_summary
));
4295 for (i
= 0; callee_info
->builtin_constant_p_parms
.iterate (i
, &ip
); i
++)
4296 if (ip
< count
&& operand_map
[ip
] >= 0)
4297 add_builtin_constant_p_parm (info
, operand_map
[ip
]);
4299 sreal freq
= edge
->sreal_frequency ();
4300 for (i
= 0; callee_info
->size_time_table
.iterate (i
, &e
); i
++)
4303 p
= e
->exec_predicate
.remap_after_inlining
4304 (info
, params_summary
,
4305 callee_info
, operand_map
,
4308 ipa_predicate nonconstp
;
4309 nonconstp
= e
->nonconst_predicate
.remap_after_inlining
4310 (info
, params_summary
,
4311 callee_info
, operand_map
,
4314 if (p
!= false && nonconstp
!= false)
4316 sreal add_time
= ((sreal
)e
->time
* freq
);
4317 int prob
= e
->nonconst_predicate
.probability (callee_info
->conds
,
4319 if (prob
!= REG_BR_PROB_BASE
)
4320 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
4321 if (prob
!= REG_BR_PROB_BASE
4322 && dump_file
&& (dump_flags
& TDF_DETAILS
))
4324 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
4325 (double) prob
/ REG_BR_PROB_BASE
);
4327 info
->account_size_time (e
->size
, add_time
, p
, nonconstp
);
4330 remap_edge_summaries (edge
, edge
->callee
, info
, params_summary
,
4331 callee_info
, operand_map
,
4332 offset_map
, clause
, &toplev_predicate
);
4333 remap_freqcounting_predicate (info
, params_summary
, callee_info
,
4334 info
->loop_iterations
, operand_map
,
4335 offset_map
, clause
, &toplev_predicate
);
4336 remap_freqcounting_predicate (info
, params_summary
, callee_info
,
4337 info
->loop_strides
, operand_map
,
4338 offset_map
, clause
, &toplev_predicate
);
4340 HOST_WIDE_INT stack_frame_offset
= ipa_get_stack_frame_offset (edge
->callee
);
4341 HOST_WIDE_INT peak
= stack_frame_offset
+ callee_info
->estimated_stack_size
;
4343 if (info
->estimated_stack_size
< peak
)
4344 info
->estimated_stack_size
= peak
;
4346 inline_update_callee_summaries (edge
->callee
, es
->loop_depth
);
4347 if (info
->call_size_time_table
.length ())
4350 sreal edge_time
= 0;
4352 estimate_edge_size_and_time (edge
, &edge_size
, NULL
, &edge_time
, NULL
, 0);
4353 /* Unaccount size and time of the optimized out call. */
4354 info
->account_size_time (-edge_size
, -edge_time
,
4355 es
->predicate
? *es
->predicate
: true,
4356 es
->predicate
? *es
->predicate
: true,
4358 /* Account new calls. */
4359 summarize_calls_size_and_time (edge
->callee
, info
);
4362 /* Free summaries that are not maintained for inline clones/edges. */
4363 ipa_call_summaries
->remove (edge
);
4364 ipa_fn_summaries
->remove (edge
->callee
);
4365 ipa_remove_from_growth_caches (edge
);
4368 /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
4369 overall size and time. Recompute it.
4370 If RESET is true also recompute call_time_size_table. */
4373 ipa_update_overall_fn_summary (struct cgraph_node
*node
, bool reset
)
4375 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
4376 class ipa_size_summary
*size_info
= ipa_size_summaries
->get (node
);
4380 size_info
->size
= 0;
4382 for (i
= 0; info
->size_time_table
.iterate (i
, &e
); i
++)
4384 size_info
->size
+= e
->size
;
4385 info
->time
+= e
->time
;
4387 info
->min_size
= info
->size_time_table
[0].size
;
4389 info
->call_size_time_table
.release ();
4390 if (node
->callees
|| node
->indirect_calls
)
4391 estimate_calls_size_and_time (node
, &size_info
->size
, &info
->min_size
,
4393 ~(clause_t
) (1 << ipa_predicate::false_condition
),
4395 size_info
->size
= RDIV (size_info
->size
, ipa_fn_summary::size_scale
);
4396 info
->min_size
= RDIV (info
->min_size
, ipa_fn_summary::size_scale
);
4400 /* This function performs intraprocedural analysis in NODE that is required to
4401 inline indirect calls. */
4404 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
4406 ipa_analyze_node (node
);
4407 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
4409 ipa_print_node_params (dump_file
, node
);
4410 ipa_print_node_jump_functions (dump_file
, node
);
4415 /* Note function body size. */
4418 inline_analyze_function (struct cgraph_node
*node
)
4420 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4423 fprintf (dump_file
, "\nAnalyzing function: %s\n", node
->dump_name ());
4424 if (opt_for_fn (node
->decl
, optimize
) && !node
->thunk
)
4425 inline_indirect_intraprocedural_analysis (node
);
4426 compute_fn_summary (node
, false);
4429 struct cgraph_edge
*e
;
4430 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4431 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
4432 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
4433 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
4440 /* Called when new function is inserted to callgraph late. */
4443 ipa_fn_summary_t::insert (struct cgraph_node
*node
, ipa_fn_summary
*)
4445 inline_analyze_function (node
);
4448 /* Note function body size. */
4451 ipa_fn_summary_generate (void)
4453 struct cgraph_node
*node
;
4455 FOR_EACH_DEFINED_FUNCTION (node
)
4456 if (DECL_STRUCT_FUNCTION (node
->decl
))
4457 node
->versionable
= tree_versionable_function_p (node
->decl
);
4459 ipa_fn_summary_alloc ();
4461 ipa_fn_summaries
->enable_insertion_hook ();
4463 ipa_register_cgraph_hooks ();
4465 FOR_EACH_DEFINED_FUNCTION (node
)
4467 && (flag_generate_lto
|| flag_generate_offload
|| flag_wpa
4468 || opt_for_fn (node
->decl
, optimize
)))
4469 inline_analyze_function (node
);
4473 /* Write inline summary for edge E to OB. */
4476 read_ipa_call_summary (class lto_input_block
*ib
, struct cgraph_edge
*e
,
4479 class ipa_call_summary
*es
= prevails
4480 ? ipa_call_summaries
->get_create (e
) : NULL
;
4484 int size
= streamer_read_uhwi (ib
);
4485 int time
= streamer_read_uhwi (ib
);
4486 int depth
= streamer_read_uhwi (ib
);
4490 es
->call_stmt_size
= size
;
4491 es
->call_stmt_time
= time
;
4492 es
->loop_depth
= depth
;
4495 bitpack_d bp
= streamer_read_bitpack (ib
);
4497 es
->is_return_callee_uncaptured
= bp_unpack_value (&bp
, 1);
4499 bp_unpack_value (&bp
, 1);
4503 edge_set_predicate (e
, &p
);
4504 length
= streamer_read_uhwi (ib
);
4506 && (e
->possibly_call_in_translation_unit_p ()
4507 /* Also stream in jump functions to builtins in hope that they
4508 will get fnspecs. */
4509 || fndecl_built_in_p (e
->callee
->decl
, BUILT_IN_NORMAL
)))
4511 es
->param
.safe_grow_cleared (length
, true);
4512 for (i
= 0; i
< length
; i
++)
4514 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
4515 bitpack_d bp
= streamer_read_bitpack (ib
);
4516 es
->param
[i
].points_to_local_or_readonly_memory
4517 = bp_unpack_value (&bp
, 1);
4518 es
->param
[i
].points_to_possible_sra_candidate
4519 = bp_unpack_value (&bp
, 1);
4524 for (i
= 0; i
< length
; i
++)
4526 streamer_read_uhwi (ib
);
4527 streamer_read_uhwi (ib
);
4533 /* Stream in inline summaries from the section. */
4536 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
4539 const struct lto_function_header
*header
=
4540 (const struct lto_function_header
*) data
;
4541 const int cfg_offset
= sizeof (struct lto_function_header
);
4542 const int main_offset
= cfg_offset
+ header
->cfg_size
;
4543 const int string_offset
= main_offset
+ header
->main_size
;
4544 class data_in
*data_in
;
4545 unsigned int i
, count2
, j
;
4546 unsigned int f_count
;
4548 lto_input_block
ib ((const char *) data
+ main_offset
, header
->main_size
,
4552 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
4553 header
->string_size
, vNULL
);
4554 f_count
= streamer_read_uhwi (&ib
);
4555 for (i
= 0; i
< f_count
; i
++)
4558 struct cgraph_node
*node
;
4559 class ipa_fn_summary
*info
;
4560 class ipa_node_params
*params_summary
;
4561 class ipa_size_summary
*size_info
;
4562 lto_symtab_encoder_t encoder
;
4563 struct bitpack_d bp
;
4564 struct cgraph_edge
*e
;
4567 index
= streamer_read_uhwi (&ib
);
4568 encoder
= file_data
->symtab_node_encoder
;
4569 node
= dyn_cast
<cgraph_node
*> (lto_symtab_encoder_deref (encoder
,
4571 info
= node
->prevailing_p () ? ipa_fn_summaries
->get_create (node
) : NULL
;
4572 params_summary
= node
->prevailing_p ()
4573 ? ipa_node_params_sum
->get (node
) : NULL
;
4574 size_info
= node
->prevailing_p ()
4575 ? ipa_size_summaries
->get_create (node
) : NULL
;
4577 int stack_size
= streamer_read_uhwi (&ib
);
4578 int size
= streamer_read_uhwi (&ib
);
4579 sreal time
= sreal::stream_in (&ib
);
4583 info
->estimated_stack_size
4584 = size_info
->estimated_self_stack_size
= stack_size
;
4585 size_info
->size
= size_info
->self_size
= size
;
4589 bp
= streamer_read_bitpack (&ib
);
4592 info
->inlinable
= bp_unpack_value (&bp
, 1);
4593 info
->fp_expressions
= bp_unpack_value (&bp
, 1);
4594 if (!lto_stream_offload_p
)
4595 info
->target_info
= streamer_read_uhwi (&ib
);
4599 bp_unpack_value (&bp
, 1);
4600 bp_unpack_value (&bp
, 1);
4601 if (!lto_stream_offload_p
)
4602 streamer_read_uhwi (&ib
);
4605 count2
= streamer_read_uhwi (&ib
);
4606 gcc_assert (!info
|| !info
->conds
);
4608 vec_safe_reserve_exact (info
->conds
, count2
);
4609 for (j
= 0; j
< count2
; j
++)
4612 unsigned int k
, count3
;
4613 c
.operand_num
= streamer_read_uhwi (&ib
);
4614 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
4615 c
.type
= stream_read_tree (&ib
, data_in
);
4616 c
.val
= stream_read_tree (&ib
, data_in
);
4617 bp
= streamer_read_bitpack (&ib
);
4618 c
.agg_contents
= bp_unpack_value (&bp
, 1);
4619 c
.by_ref
= bp_unpack_value (&bp
, 1);
4621 c
.offset
= streamer_read_uhwi (&ib
);
4622 count3
= streamer_read_uhwi (&ib
);
4625 vec_safe_reserve_exact (c
.param_ops
, count3
);
4627 ipa_set_param_used_by_ipa_predicates
4628 (params_summary
, c
.operand_num
, true);
4629 for (k
= 0; k
< count3
; k
++)
4631 struct expr_eval_op op
;
4632 enum gimple_rhs_class rhs_class
;
4633 op
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
4634 op
.type
= stream_read_tree (&ib
, data_in
);
4635 switch (rhs_class
= get_gimple_rhs_class (op
.code
))
4637 case GIMPLE_UNARY_RHS
:
4639 op
.val
[0] = NULL_TREE
;
4640 op
.val
[1] = NULL_TREE
;
4643 case GIMPLE_BINARY_RHS
:
4644 case GIMPLE_TERNARY_RHS
:
4645 bp
= streamer_read_bitpack (&ib
);
4646 op
.index
= bp_unpack_value (&bp
, 2);
4647 op
.val
[0] = stream_read_tree (&ib
, data_in
);
4648 if (rhs_class
== GIMPLE_BINARY_RHS
)
4649 op
.val
[1] = NULL_TREE
;
4651 op
.val
[1] = stream_read_tree (&ib
, data_in
);
4655 fatal_error (UNKNOWN_LOCATION
,
4656 "invalid fnsummary in LTO stream");
4659 c
.param_ops
->quick_push (op
);
4662 info
->conds
->quick_push (c
);
4664 count2
= streamer_read_uhwi (&ib
);
4665 gcc_assert (!info
|| !info
->size_time_table
.length ());
4667 info
->size_time_table
.reserve_exact (count2
);
4668 for (j
= 0; j
< count2
; j
++)
4670 class size_time_entry e
;
4672 e
.size
= streamer_read_uhwi (&ib
);
4673 e
.time
= sreal::stream_in (&ib
);
4674 e
.exec_predicate
.stream_in (&ib
);
4675 e
.nonconst_predicate
.stream_in (&ib
);
4678 info
->size_time_table
.quick_push (e
);
4681 count2
= streamer_read_uhwi (&ib
);
4682 for (j
= 0; j
< count2
; j
++)
4685 sreal fcp_freq
= sreal::stream_in (&ib
);
4688 ipa_freqcounting_predicate fcp
;
4689 fcp
.predicate
= NULL
;
4690 set_hint_predicate (&fcp
.predicate
, p
);
4691 fcp
.freq
= fcp_freq
;
4692 vec_safe_push (info
->loop_iterations
, fcp
);
4695 count2
= streamer_read_uhwi (&ib
);
4696 for (j
= 0; j
< count2
; j
++)
4699 sreal fcp_freq
= sreal::stream_in (&ib
);
4702 ipa_freqcounting_predicate fcp
;
4703 fcp
.predicate
= NULL
;
4704 set_hint_predicate (&fcp
.predicate
, p
);
4705 fcp
.freq
= fcp_freq
;
4706 vec_safe_push (info
->loop_strides
, fcp
);
4709 count2
= streamer_read_uhwi (&ib
);
4711 info
->builtin_constant_p_parms
.reserve_exact (count2
);
4712 for (j
= 0; j
< count2
; j
++)
4714 int parm
= streamer_read_uhwi (&ib
);
4716 info
->builtin_constant_p_parms
.quick_push (parm
);
4718 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4719 read_ipa_call_summary (&ib
, e
, info
!= NULL
);
4720 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
4721 read_ipa_call_summary (&ib
, e
, info
!= NULL
);
4724 lto_free_section_data (file_data
, LTO_section_ipa_fn_summary
, NULL
, data
,
4726 lto_data_in_delete (data_in
);
4730 /* Read inline summary. Jump functions are shared among ipa-cp
4731 and inliner, so when ipa-cp is active, we don't need to write them
4735 ipa_fn_summary_read (void)
4737 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
4738 struct lto_file_decl_data
*file_data
;
4741 ipa_prop_read_jump_functions ();
4742 ipa_fn_summary_alloc ();
4744 while ((file_data
= file_data_vec
[j
++]))
4748 = lto_get_summary_section_data (file_data
, LTO_section_ipa_fn_summary
,
4751 inline_read_section (file_data
, data
, len
);
4753 /* Fatal error here. We do not want to support compiling ltrans units
4754 with different version of compiler or different flags than the WPA
4755 unit, so this should never happen. */
4756 fatal_error (input_location
,
4757 "ipa inline summary is missing in input file");
4759 ipa_register_cgraph_hooks ();
4761 gcc_assert (ipa_fn_summaries
);
4762 ipa_fn_summaries
->enable_insertion_hook ();
4766 /* Write inline summary for edge E to OB. */
4769 write_ipa_call_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4771 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
4774 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4775 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4776 streamer_write_uhwi (ob
, es
->loop_depth
);
4778 bitpack_d bp
= bitpack_create (ob
->main_stream
);
4779 bp_pack_value (&bp
, es
->is_return_callee_uncaptured
, 1);
4780 streamer_write_bitpack (&bp
);
4783 es
->predicate
->stream_out (ob
);
4785 streamer_write_uhwi (ob
, 0);
4786 streamer_write_uhwi (ob
, es
->param
.length ());
4787 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4789 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4790 bp
= bitpack_create (ob
->main_stream
);
4791 bp_pack_value (&bp
, es
->param
[i
].points_to_local_or_readonly_memory
, 1);
4792 bp_pack_value (&bp
, es
->param
[i
].points_to_possible_sra_candidate
, 1);
4793 streamer_write_bitpack (&bp
);
4798 /* Write inline summary for node in SET.
4799 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4800 active, we don't need to write them twice. */
4803 ipa_fn_summary_write (void)
4805 struct output_block
*ob
= create_output_block (LTO_section_ipa_fn_summary
);
4806 lto_symtab_encoder_iterator lsei
;
4807 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4808 unsigned int count
= 0;
4810 for (lsei
= lsei_start_function_in_partition (encoder
); !lsei_end_p (lsei
);
4811 lsei_next_function_in_partition (&lsei
))
4813 cgraph_node
*cnode
= lsei_cgraph_node (lsei
);
4814 if (cnode
->definition
&& !cnode
->alias
)
4817 streamer_write_uhwi (ob
, count
);
4819 for (lsei
= lsei_start_function_in_partition (encoder
); !lsei_end_p (lsei
);
4820 lsei_next_function_in_partition (&lsei
))
4822 cgraph_node
*cnode
= lsei_cgraph_node (lsei
);
4823 if (cnode
->definition
&& !cnode
->alias
)
4825 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (cnode
);
4826 class ipa_size_summary
*size_info
= ipa_size_summaries
->get (cnode
);
4827 struct bitpack_d bp
;
4828 struct cgraph_edge
*edge
;
4831 struct condition
*c
;
4833 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, cnode
));
4834 streamer_write_hwi (ob
, size_info
->estimated_self_stack_size
);
4835 streamer_write_hwi (ob
, size_info
->self_size
);
4836 info
->time
.stream_out (ob
);
4837 bp
= bitpack_create (ob
->main_stream
);
4838 bp_pack_value (&bp
, info
->inlinable
, 1);
4839 bp_pack_value (&bp
, info
->fp_expressions
, 1);
4840 streamer_write_bitpack (&bp
);
4841 if (!lto_stream_offload_p
)
4842 streamer_write_uhwi (ob
, info
->target_info
);
4843 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4844 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4847 struct expr_eval_op
*op
;
4849 streamer_write_uhwi (ob
, c
->operand_num
);
4850 streamer_write_uhwi (ob
, c
->code
);
4851 stream_write_tree (ob
, c
->type
, true);
4852 stream_write_tree (ob
, c
->val
, true);
4853 bp
= bitpack_create (ob
->main_stream
);
4854 bp_pack_value (&bp
, c
->agg_contents
, 1);
4855 bp_pack_value (&bp
, c
->by_ref
, 1);
4856 streamer_write_bitpack (&bp
);
4857 if (c
->agg_contents
)
4858 streamer_write_uhwi (ob
, c
->offset
);
4859 streamer_write_uhwi (ob
, vec_safe_length (c
->param_ops
));
4860 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
4862 streamer_write_uhwi (ob
, op
->code
);
4863 stream_write_tree (ob
, op
->type
, true);
4866 bp
= bitpack_create (ob
->main_stream
);
4867 bp_pack_value (&bp
, op
->index
, 2);
4868 streamer_write_bitpack (&bp
);
4869 stream_write_tree (ob
, op
->val
[0], true);
4871 stream_write_tree (ob
, op
->val
[1], true);
4875 streamer_write_uhwi (ob
, info
->size_time_table
.length ());
4876 for (i
= 0; info
->size_time_table
.iterate (i
, &e
); i
++)
4878 streamer_write_uhwi (ob
, e
->size
);
4879 e
->time
.stream_out (ob
);
4880 e
->exec_predicate
.stream_out (ob
);
4881 e
->nonconst_predicate
.stream_out (ob
);
4883 ipa_freqcounting_predicate
*fcp
;
4884 streamer_write_uhwi (ob
, vec_safe_length (info
->loop_iterations
));
4885 for (i
= 0; vec_safe_iterate (info
->loop_iterations
, i
, &fcp
); i
++)
4887 fcp
->predicate
->stream_out (ob
);
4888 fcp
->freq
.stream_out (ob
);
4890 streamer_write_uhwi (ob
, vec_safe_length (info
->loop_strides
));
4891 for (i
= 0; vec_safe_iterate (info
->loop_strides
, i
, &fcp
); i
++)
4893 fcp
->predicate
->stream_out (ob
);
4894 fcp
->freq
.stream_out (ob
);
4896 streamer_write_uhwi (ob
, info
->builtin_constant_p_parms
.length ());
4898 for (i
= 0; info
->builtin_constant_p_parms
.iterate (i
, &ip
);
4900 streamer_write_uhwi (ob
, ip
);
4901 for (edge
= cnode
->callees
; edge
; edge
= edge
->next_callee
)
4902 write_ipa_call_summary (ob
, edge
);
4903 for (edge
= cnode
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4904 write_ipa_call_summary (ob
, edge
);
4907 streamer_write_char_stream (ob
->main_stream
, 0);
4908 produce_asm (ob
, NULL
);
4909 destroy_output_block (ob
);
4911 ipa_prop_write_jump_functions ();
4915 /* Release function summary. */
4918 ipa_free_fn_summary (void)
4920 if (!ipa_call_summaries
)
4922 ggc_delete (ipa_fn_summaries
);
4923 ipa_fn_summaries
= NULL
;
4924 delete ipa_call_summaries
;
4925 ipa_call_summaries
= NULL
;
4926 edge_predicate_pool
.release ();
4927 /* During IPA this is one of largest datastructures to release. */
4932 /* Release function summary. */
4935 ipa_free_size_summary (void)
4937 if (!ipa_size_summaries
)
4939 delete ipa_size_summaries
;
4940 ipa_size_summaries
= NULL
;
4945 const pass_data pass_data_local_fn_summary
=
4947 GIMPLE_PASS
, /* type */
4948 "local-fnsummary", /* name */
4949 OPTGROUP_INLINE
, /* optinfo_flags */
4950 TV_INLINE_PARAMETERS
, /* tv_id */
4951 0, /* properties_required */
4952 0, /* properties_provided */
4953 0, /* properties_destroyed */
4954 0, /* todo_flags_start */
4955 0, /* todo_flags_finish */
4958 class pass_local_fn_summary
: public gimple_opt_pass
4961 pass_local_fn_summary (gcc::context
*ctxt
)
4962 : gimple_opt_pass (pass_data_local_fn_summary
, ctxt
)
4965 /* opt_pass methods: */
4966 opt_pass
* clone () final override
4968 return new pass_local_fn_summary (m_ctxt
);
4970 unsigned int execute (function
*) final override
4972 return compute_fn_summary_for_current ();
4975 }; // class pass_local_fn_summary
4980 make_pass_local_fn_summary (gcc::context
*ctxt
)
4982 return new pass_local_fn_summary (ctxt
);
4986 /* Free inline summary. */
4990 const pass_data pass_data_ipa_free_fn_summary
=
4992 SIMPLE_IPA_PASS
, /* type */
4993 "free-fnsummary", /* name */
4994 OPTGROUP_NONE
, /* optinfo_flags */
4995 TV_IPA_FREE_INLINE_SUMMARY
, /* tv_id */
4996 0, /* properties_required */
4997 0, /* properties_provided */
4998 0, /* properties_destroyed */
4999 0, /* todo_flags_start */
5000 0, /* todo_flags_finish */
5003 class pass_ipa_free_fn_summary
: public simple_ipa_opt_pass
5006 pass_ipa_free_fn_summary (gcc::context
*ctxt
)
5007 : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary
, ctxt
),
5011 /* opt_pass methods: */
5012 opt_pass
*clone () final override
5014 return new pass_ipa_free_fn_summary (m_ctxt
);
5016 void set_pass_param (unsigned int n
, bool param
) final override
5018 gcc_assert (n
== 0);
5021 bool gate (function
*) final override
{ return true; }
5022 unsigned int execute (function
*) final override
5024 ipa_free_fn_summary ();
5025 /* Free ipa-prop structures if they are no longer needed. */
5026 ipa_free_all_structures_after_iinln ();
5028 ipa_free_size_summary ();
5034 }; // class pass_ipa_free_fn_summary
5038 simple_ipa_opt_pass
*
5039 make_pass_ipa_free_fn_summary (gcc::context
*ctxt
)
5041 return new pass_ipa_free_fn_summary (ctxt
);
5046 const pass_data pass_data_ipa_fn_summary
=
5048 IPA_PASS
, /* type */
5049 "fnsummary", /* name */
5050 OPTGROUP_INLINE
, /* optinfo_flags */
5051 TV_IPA_FNSUMMARY
, /* tv_id */
5052 0, /* properties_required */
5053 0, /* properties_provided */
5054 0, /* properties_destroyed */
5055 0, /* todo_flags_start */
5056 ( TODO_dump_symtab
), /* todo_flags_finish */
5059 class pass_ipa_fn_summary
: public ipa_opt_pass_d
5062 pass_ipa_fn_summary (gcc::context
*ctxt
)
5063 : ipa_opt_pass_d (pass_data_ipa_fn_summary
, ctxt
,
5064 ipa_fn_summary_generate
, /* generate_summary */
5065 ipa_fn_summary_write
, /* write_summary */
5066 ipa_fn_summary_read
, /* read_summary */
5067 NULL
, /* write_optimization_summary */
5068 NULL
, /* read_optimization_summary */
5069 NULL
, /* stmt_fixup */
5070 0, /* function_transform_todo_flags_start */
5071 NULL
, /* function_transform */
5072 NULL
) /* variable_transform */
5075 /* opt_pass methods: */
5076 unsigned int execute (function
*) final override
{ return 0; }
5078 }; // class pass_ipa_fn_summary
5083 make_pass_ipa_fn_summary (gcc::context
*ctxt
)
5085 return new pass_ipa_fn_summary (ctxt
);
5088 /* Reset all state within ipa-fnsummary.cc so that we can rerun the compiler
5089 within the same process. For use by toplev::finalize. */
5092 ipa_fnsummary_cc_finalize (void)
5094 ipa_free_fn_summary ();
5095 ipa_free_size_summary ();