testsuite: remove SPE tests.
[official-gcc.git] / gcc / ipa-fnsummary.c
blob045a0ecf76689372d19e2f3b18257f43c446f0a3
1 /* Function summary pass.
2 Copyright (C) 2003-2020 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis of function bodies used by inter-procedural passes
23 We estimate for each function
24 - function body size and size after specializing into given context
25 - average function execution time in a given context
26 - function frame size
27 For each call
28 - call statement size, time and how often the parameters change
30 ipa_fn_summary data structures store above information locally (i.e.
31 parameters of the function itself) and globally (i.e. parameters of
32 the function created by applying all the inline decisions already
33 present in the callgraph).
35 We provide access to the ipa_fn_summary data structure and
36 basic logic updating the parameters when inlining is performed.
38 The summaries are context sensitive. Context means
39 1) partial assignment of known constant values of operands
40 2) whether function is inlined into the call or not.
41 It is easy to add more variants. To represent function size and time
42 that depends on context (i.e. it is known to be optimized away when
43 context is known either by inlining or from IP-CP and cloning),
44 we use predicates.
46 estimate_edge_size_and_time can be used to query
47 function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 properties of caller and callee after inlining.
50 Finally pass_inline_parameters is exported. This is used to drive
51 computation of function parameters used by the early inliner. IPA
52 inlined performs analysis via its analyze_function method. */
54 #include "config.h"
55 #include "system.h"
56 #include "coretypes.h"
57 #include "backend.h"
58 #include "tree.h"
59 #include "gimple.h"
60 #include "alloc-pool.h"
61 #include "tree-pass.h"
62 #include "ssa.h"
63 #include "tree-streamer.h"
64 #include "cgraph.h"
65 #include "diagnostic.h"
66 #include "fold-const.h"
67 #include "print-tree.h"
68 #include "tree-inline.h"
69 #include "gimple-pretty-print.h"
70 #include "cfganal.h"
71 #include "gimple-iterator.h"
72 #include "tree-cfg.h"
73 #include "tree-ssa-loop-niter.h"
74 #include "tree-ssa-loop.h"
75 #include "symbol-summary.h"
76 #include "ipa-prop.h"
77 #include "ipa-fnsummary.h"
78 #include "cfgloop.h"
79 #include "tree-scalar-evolution.h"
80 #include "ipa-utils.h"
81 #include "cfgexpand.h"
82 #include "gimplify.h"
83 #include "stringpool.h"
84 #include "attribs.h"
85 #include "tree-into-ssa.h"
87 /* Summaries. */
88 fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
89 fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
90 fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
92 /* Edge predicates goes here. */
93 static object_allocator<predicate> edge_predicate_pool ("edge predicates");
96 /* Dump IPA hints. */
97 void
98 ipa_dump_hints (FILE *f, ipa_hints hints)
100 if (!hints)
101 return;
102 fprintf (f, "IPA hints:");
103 if (hints & INLINE_HINT_indirect_call)
105 hints &= ~INLINE_HINT_indirect_call;
106 fprintf (f, " indirect_call");
108 if (hints & INLINE_HINT_loop_iterations)
110 hints &= ~INLINE_HINT_loop_iterations;
111 fprintf (f, " loop_iterations");
113 if (hints & INLINE_HINT_loop_stride)
115 hints &= ~INLINE_HINT_loop_stride;
116 fprintf (f, " loop_stride");
118 if (hints & INLINE_HINT_same_scc)
120 hints &= ~INLINE_HINT_same_scc;
121 fprintf (f, " same_scc");
123 if (hints & INLINE_HINT_in_scc)
125 hints &= ~INLINE_HINT_in_scc;
126 fprintf (f, " in_scc");
128 if (hints & INLINE_HINT_cross_module)
130 hints &= ~INLINE_HINT_cross_module;
131 fprintf (f, " cross_module");
133 if (hints & INLINE_HINT_declared_inline)
135 hints &= ~INLINE_HINT_declared_inline;
136 fprintf (f, " declared_inline");
138 if (hints & INLINE_HINT_known_hot)
140 hints &= ~INLINE_HINT_known_hot;
141 fprintf (f, " known_hot");
143 gcc_assert (!hints);
147 /* Record SIZE and TIME to SUMMARY.
148 The accounted code will be executed when EXEC_PRED is true.
149 When NONCONST_PRED is false the code will evaluate to constant and
150 will get optimized out in specialized clones of the function.
151 If CALL is true account to call_size_time_table rather than
152 size_time_table. */
154 void
155 ipa_fn_summary::account_size_time (int size, sreal time,
156 const predicate &exec_pred,
157 const predicate &nonconst_pred_in,
158 bool call)
160 size_time_entry *e;
161 bool found = false;
162 int i;
163 predicate nonconst_pred;
164 vec<size_time_entry, va_gc> *table = call
165 ? call_size_time_table : size_time_table;
167 if (exec_pred == false)
168 return;
170 nonconst_pred = nonconst_pred_in & exec_pred;
172 if (nonconst_pred == false)
173 return;
175 /* We need to create initial empty unconditional clause, but otherwise
176 we don't need to account empty times and sizes. */
177 if (!size && time == 0 && table)
178 return;
180 /* Only for calls we are unaccounting what we previously recorded. */
181 gcc_checking_assert (time >= 0 || call);
183 for (i = 0; vec_safe_iterate (table, i, &e); i++)
184 if (e->exec_predicate == exec_pred
185 && e->nonconst_predicate == nonconst_pred)
187 found = true;
188 break;
190 if (i == max_size_time_table_size)
192 i = 0;
193 found = true;
194 e = &(*table)[0];
195 if (dump_file && (dump_flags & TDF_DETAILS))
196 fprintf (dump_file,
197 "\t\tReached limit on number of entries, "
198 "ignoring the predicate.");
200 if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
202 fprintf (dump_file,
203 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
204 ((double) size) / ipa_fn_summary::size_scale,
205 (time.to_double ()), found ? "" : "new ");
206 exec_pred.dump (dump_file, conds, 0);
207 if (exec_pred != nonconst_pred)
209 fprintf (dump_file, " nonconst:");
210 nonconst_pred.dump (dump_file, conds);
212 else
213 fprintf (dump_file, "\n");
215 if (!found)
217 class size_time_entry new_entry;
218 new_entry.size = size;
219 new_entry.time = time;
220 new_entry.exec_predicate = exec_pred;
221 new_entry.nonconst_predicate = nonconst_pred;
222 if (call)
223 vec_safe_push (call_size_time_table, new_entry);
224 else
225 vec_safe_push (size_time_table, new_entry);
227 else
229 e->size += size;
230 e->time += time;
231 /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
232 /* Tolerate small roundoff issues. */
233 if (e->time < 0)
234 e->time = 0;
238 /* We proved E to be unreachable, redirect it to __builtin_unreachable. */
240 static struct cgraph_edge *
241 redirect_to_unreachable (struct cgraph_edge *e)
243 struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
244 struct cgraph_node *target = cgraph_node::get_create
245 (builtin_decl_implicit (BUILT_IN_UNREACHABLE));
247 if (e->speculative)
248 e = cgraph_edge::resolve_speculation (e, target->decl);
249 else if (!e->callee)
250 e = cgraph_edge::make_direct (e, target);
251 else
252 e->redirect_callee (target);
253 class ipa_call_summary *es = ipa_call_summaries->get (e);
254 e->inline_failed = CIF_UNREACHABLE;
255 e->count = profile_count::zero ();
256 es->call_stmt_size = 0;
257 es->call_stmt_time = 0;
258 if (callee)
259 callee->remove_symbol_and_inline_clones ();
260 return e;
263 /* Set predicate for edge E. */
265 static void
266 edge_set_predicate (struct cgraph_edge *e, predicate *predicate)
268 /* If the edge is determined to be never executed, redirect it
269 to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
270 be optimized out. */
271 if (predicate && *predicate == false
272 /* When handling speculative edges, we need to do the redirection
273 just once. Do it always on the direct edge, so we do not
274 attempt to resolve speculation while duplicating the edge. */
275 && (!e->speculative || e->callee))
276 e = redirect_to_unreachable (e);
278 class ipa_call_summary *es = ipa_call_summaries->get (e);
279 if (predicate && *predicate != true)
281 if (!es->predicate)
282 es->predicate = edge_predicate_pool.allocate ();
283 *es->predicate = *predicate;
285 else
287 if (es->predicate)
288 edge_predicate_pool.remove (es->predicate);
289 es->predicate = NULL;
293 /* Set predicate for hint *P. */
295 static void
296 set_hint_predicate (predicate **p, predicate new_predicate)
298 if (new_predicate == false || new_predicate == true)
300 if (*p)
301 edge_predicate_pool.remove (*p);
302 *p = NULL;
304 else
306 if (!*p)
307 *p = edge_predicate_pool.allocate ();
308 **p = new_predicate;
313 /* Compute what conditions may or may not hold given information about
314 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
315 while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
316 copy when called in a given context. It is a bitmask of conditions. Bit
317 0 means that condition is known to be false, while bit 1 means that condition
318 may or may not be true. These differs - for example NOT_INLINED condition
319 is always false in the second and also builtin_constant_p tests cannot use
320 the fact that parameter is indeed a constant.
322 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
323 KNOWN_AGGS is a vector of aggregate known offset/value set for each
324 parameter. Return clause of possible truths. When INLINE_P is true, assume
325 that we are inlining.
327 ERROR_MARK means compile time invariant. */
329 static void
330 evaluate_conditions_for_known_args (struct cgraph_node *node,
331 bool inline_p,
332 vec<tree> known_vals,
333 vec<value_range> known_value_ranges,
334 vec<ipa_agg_value_set> known_aggs,
335 clause_t *ret_clause,
336 clause_t *ret_nonspec_clause)
338 clause_t clause = inline_p ? 0 : 1 << predicate::not_inlined_condition;
339 clause_t nonspec_clause = 1 << predicate::not_inlined_condition;
340 class ipa_fn_summary *info = ipa_fn_summaries->get (node);
341 int i;
342 struct condition *c;
344 for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
346 tree val = NULL;
347 tree res;
348 int j;
349 struct expr_eval_op *op;
351 /* We allow call stmt to have fewer arguments than the callee function
352 (especially for K&R style programs). So bound check here (we assume
353 known_aggs vector, if non-NULL, has the same length as
354 known_vals). */
355 gcc_checking_assert (!known_aggs.length () || !known_vals.length ()
356 || (known_vals.length () == known_aggs.length ()));
358 if (c->agg_contents)
360 struct ipa_agg_value_set *agg;
362 if (c->code == predicate::changed
363 && !c->by_ref
364 && c->operand_num < (int)known_vals.length ()
365 && (known_vals[c->operand_num] == error_mark_node))
366 continue;
368 if (c->operand_num < (int)known_aggs.length ())
370 agg = &known_aggs[c->operand_num];
371 val = ipa_find_agg_cst_for_param (agg,
372 c->operand_num
373 < (int) known_vals.length ()
374 ? known_vals[c->operand_num]
375 : NULL,
376 c->offset, c->by_ref);
378 else
379 val = NULL_TREE;
381 else if (c->operand_num < (int) known_vals.length ())
383 val = known_vals[c->operand_num];
384 if (val == error_mark_node && c->code != predicate::changed)
385 val = NULL_TREE;
388 if (!val
389 && (c->code == predicate::changed
390 || c->code == predicate::is_not_constant))
392 clause |= 1 << (i + predicate::first_dynamic_condition);
393 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
394 continue;
396 if (c->code == predicate::changed)
398 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
399 continue;
402 if (c->code == predicate::is_not_constant)
404 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
405 continue;
408 if (val && TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val)))
410 if (c->type != TREE_TYPE (val))
411 val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
412 for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
414 if (!val)
415 break;
416 if (!op->val[0])
417 val = fold_unary (op->code, op->type, val);
418 else if (!op->val[1])
419 val = fold_binary (op->code, op->type,
420 op->index ? op->val[0] : val,
421 op->index ? val : op->val[0]);
422 else if (op->index == 0)
423 val = fold_ternary (op->code, op->type,
424 val, op->val[0], op->val[1]);
425 else if (op->index == 1)
426 val = fold_ternary (op->code, op->type,
427 op->val[0], val, op->val[1]);
428 else if (op->index == 2)
429 val = fold_ternary (op->code, op->type,
430 op->val[0], op->val[1], val);
431 else
432 val = NULL_TREE;
435 res = val
436 ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
437 : NULL;
439 if (res && integer_zerop (res))
440 continue;
441 if (res && integer_onep (res))
443 clause |= 1 << (i + predicate::first_dynamic_condition);
444 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
445 continue;
448 if (c->operand_num < (int) known_value_ranges.length ()
449 && !c->agg_contents
450 && !known_value_ranges[c->operand_num].undefined_p ()
451 && !known_value_ranges[c->operand_num].varying_p ()
452 && TYPE_SIZE (c->type)
453 == TYPE_SIZE (known_value_ranges[c->operand_num].type ())
454 && (!val || TREE_CODE (val) != INTEGER_CST))
456 value_range vr = known_value_ranges[c->operand_num];
457 if (!useless_type_conversion_p (c->type, vr.type ()))
459 value_range res;
460 range_fold_unary_expr (&res, NOP_EXPR,
461 c->type, &vr, vr.type ());
462 vr = res;
464 tree type = c->type;
466 for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
468 if (vr.varying_p () || vr.undefined_p ())
469 break;
471 value_range res;
472 if (!op->val[0])
473 range_fold_unary_expr (&res, op->code, op->type, &vr, type);
474 else if (!op->val[1])
476 value_range op0 (op->val[0], op->val[0]);
477 range_fold_binary_expr (&res, op->code, op->type,
478 op->index ? &op0 : &vr,
479 op->index ? &vr : &op0);
481 else
482 gcc_unreachable ();
483 type = op->type;
484 vr = res;
486 if (!vr.varying_p () && !vr.undefined_p ())
488 value_range res;
489 value_range val_vr (c->val, c->val);
490 range_fold_binary_expr (&res, c->code, boolean_type_node,
491 &vr,
492 &val_vr);
493 if (res.zero_p ())
494 continue;
498 clause |= 1 << (i + predicate::first_dynamic_condition);
499 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
501 *ret_clause = clause;
502 if (ret_nonspec_clause)
503 *ret_nonspec_clause = nonspec_clause;
506 /* Return true if VRP will be exectued on the function.
507 We do not want to anticipate optimizations that will not happen.
509 FIXME: This can be confused with -fdisable and debug counters and thus
510 it should not be used for correctness (only to make heuristics work).
511 This means that inliner should do its own optimizations of expressions
512 that it predicts to be constant so wrong code can not be triggered by
513 builtin_constant_p. */
515 static bool
516 vrp_will_run_p (struct cgraph_node *node)
518 return (opt_for_fn (node->decl, optimize)
519 && !opt_for_fn (node->decl, optimize_debug)
520 && opt_for_fn (node->decl, flag_tree_vrp));
523 /* Similarly about FRE. */
525 static bool
526 fre_will_run_p (struct cgraph_node *node)
528 return (opt_for_fn (node->decl, optimize)
529 && !opt_for_fn (node->decl, optimize_debug)
530 && opt_for_fn (node->decl, flag_tree_fre));
533 /* Work out what conditions might be true at invocation of E.
534 Compute costs for inlined edge if INLINE_P is true.
536 Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
537 (if non-NULL) conditions evaluated for nonspecialized clone called
538 in a given context.
540 KNOWN_VALS_PTR and KNOWN_AGGS_PTR must be non-NULL and will be filled by
541 known constant and aggregate values of parameters.
543 KNOWN_CONTEXT_PTR, if non-NULL, will be filled by polymorphic call contexts
544 of parameter used by a polymorphic call. */
546 void
547 evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
548 clause_t *clause_ptr,
549 clause_t *nonspec_clause_ptr,
550 vec<tree> *known_vals_ptr,
551 vec<ipa_polymorphic_call_context>
552 *known_contexts_ptr,
553 vec<ipa_agg_value_set> *known_aggs_ptr)
555 struct cgraph_node *callee = e->callee->ultimate_alias_target ();
556 class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
557 auto_vec<value_range, 32> known_value_ranges;
558 class ipa_edge_args *args;
560 if (clause_ptr)
561 *clause_ptr = inline_p ? 0 : 1 << predicate::not_inlined_condition;
563 if (ipa_node_params_sum
564 && !e->call_stmt_cannot_inline_p
565 && (info->conds || known_contexts_ptr)
566 && (args = IPA_EDGE_REF (e)) != NULL)
568 struct cgraph_node *caller;
569 class ipa_node_params *caller_parms_info, *callee_pi = NULL;
570 class ipa_call_summary *es = ipa_call_summaries->get (e);
571 int i, count = ipa_get_cs_argument_count (args);
573 if (count)
575 if (e->caller->inlined_to)
576 caller = e->caller->inlined_to;
577 else
578 caller = e->caller;
579 caller_parms_info = IPA_NODE_REF (caller);
580 callee_pi = IPA_NODE_REF (callee);
582 /* Watch for thunks. */
583 if (callee_pi)
584 /* Watch for variadic functions. */
585 count = MIN (count, ipa_get_param_count (callee_pi));
588 if (callee_pi)
589 for (i = 0; i < count; i++)
591 struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
593 if (ipa_is_param_used_by_indirect_call (callee_pi, i)
594 || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
596 /* Determine if we know constant value of the parameter. */
597 tree cst = ipa_value_from_jfunc (caller_parms_info, jf,
598 ipa_get_type (callee_pi, i));
600 if (!cst && e->call_stmt
601 && i < (int)gimple_call_num_args (e->call_stmt))
603 cst = gimple_call_arg (e->call_stmt, i);
604 if (!is_gimple_min_invariant (cst))
605 cst = NULL;
607 if (cst)
609 gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
610 if (!known_vals_ptr->length ())
611 vec_safe_grow_cleared (known_vals_ptr, count);
612 (*known_vals_ptr)[i] = cst;
614 else if (inline_p && !es->param[i].change_prob)
616 if (!known_vals_ptr->length ())
617 vec_safe_grow_cleared (known_vals_ptr, count);
618 (*known_vals_ptr)[i] = error_mark_node;
621 /* If we failed to get simple constant, try value range. */
622 if ((!cst || TREE_CODE (cst) != INTEGER_CST)
623 && vrp_will_run_p (caller)
624 && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
626 value_range vr
627 = ipa_value_range_from_jfunc (caller_parms_info, e, jf,
628 ipa_get_type (callee_pi,
629 i));
630 if (!vr.undefined_p () && !vr.varying_p ())
632 if (!known_value_ranges.length ())
633 known_value_ranges.safe_grow_cleared (count);
634 known_value_ranges[i] = vr;
638 /* Determine known aggregate values. */
639 if (fre_will_run_p (caller))
641 ipa_agg_value_set agg
642 = ipa_agg_value_set_from_jfunc (caller_parms_info,
643 caller, &jf->agg);
644 if (agg.items.length ())
646 if (!known_aggs_ptr->length ())
647 vec_safe_grow_cleared (known_aggs_ptr, count);
648 (*known_aggs_ptr)[i] = agg;
653 /* For calls used in polymorphic calls we further determine
654 polymorphic call context. */
655 if (known_contexts_ptr
656 && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
658 ipa_polymorphic_call_context
659 ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
660 if (!ctx.useless_p ())
662 if (!known_contexts_ptr->length ())
663 known_contexts_ptr->safe_grow_cleared (count);
664 (*known_contexts_ptr)[i]
665 = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
669 else
670 gcc_assert (!count || callee->thunk.thunk_p);
672 else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
674 int i, count = (int)gimple_call_num_args (e->call_stmt);
676 for (i = 0; i < count; i++)
678 tree cst = gimple_call_arg (e->call_stmt, i);
679 if (!is_gimple_min_invariant (cst))
680 cst = NULL;
681 if (cst)
683 if (!known_vals_ptr->length ())
684 vec_safe_grow_cleared (known_vals_ptr, count);
685 (*known_vals_ptr)[i] = cst;
690 evaluate_conditions_for_known_args (callee, inline_p,
691 *known_vals_ptr,
692 known_value_ranges,
693 *known_aggs_ptr,
694 clause_ptr,
695 nonspec_clause_ptr);
699 /* Allocate the function summary. */
701 static void
702 ipa_fn_summary_alloc (void)
704 gcc_checking_assert (!ipa_fn_summaries);
705 ipa_size_summaries = new ipa_size_summary_t (symtab);
706 ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
707 ipa_call_summaries = new ipa_call_summary_t (symtab);
710 ipa_call_summary::~ipa_call_summary ()
712 if (predicate)
713 edge_predicate_pool.remove (predicate);
715 param.release ();
718 ipa_fn_summary::~ipa_fn_summary ()
720 if (loop_iterations)
721 edge_predicate_pool.remove (loop_iterations);
722 if (loop_stride)
723 edge_predicate_pool.remove (loop_stride);
724 vec_free (conds);
725 vec_free (size_time_table);
726 vec_free (call_size_time_table);
729 void
730 ipa_fn_summary_t::remove_callees (cgraph_node *node)
732 cgraph_edge *e;
733 for (e = node->callees; e; e = e->next_callee)
734 ipa_call_summaries->remove (e);
735 for (e = node->indirect_calls; e; e = e->next_callee)
736 ipa_call_summaries->remove (e);
739 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
740 Additionally care about allocating new memory slot for updated predicate
741 and set it to NULL when it becomes true or false (and thus uninteresting).
744 static void
745 remap_hint_predicate_after_duplication (predicate **p,
746 clause_t possible_truths)
748 predicate new_predicate;
750 if (!*p)
751 return;
753 new_predicate = (*p)->remap_after_duplication (possible_truths);
754 /* We do not want to free previous predicate; it is used by node origin. */
755 *p = NULL;
756 set_hint_predicate (p, new_predicate);
760 /* Hook that is called by cgraph.c when a node is duplicated. */
761 void
762 ipa_fn_summary_t::duplicate (cgraph_node *src,
763 cgraph_node *dst,
764 ipa_fn_summary *,
765 ipa_fn_summary *info)
767 new (info) ipa_fn_summary (*ipa_fn_summaries->get (src));
768 /* TODO: as an optimization, we may avoid copying conditions
769 that are known to be false or true. */
770 info->conds = vec_safe_copy (info->conds);
772 /* When there are any replacements in the function body, see if we can figure
773 out that something was optimized out. */
774 if (ipa_node_params_sum && dst->clone.tree_map)
776 vec<size_time_entry, va_gc> *entry = info->size_time_table;
777 /* Use SRC parm info since it may not be copied yet. */
778 class ipa_node_params *parms_info = IPA_NODE_REF (src);
779 vec<tree> known_vals = vNULL;
780 int count = ipa_get_param_count (parms_info);
781 int i, j;
782 clause_t possible_truths;
783 predicate true_pred = true;
784 size_time_entry *e;
785 int optimized_out_size = 0;
786 bool inlined_to_p = false;
787 struct cgraph_edge *edge, *next;
789 info->size_time_table = 0;
790 known_vals.safe_grow_cleared (count);
791 for (i = 0; i < count; i++)
793 struct ipa_replace_map *r;
795 for (j = 0; vec_safe_iterate (dst->clone.tree_map, j, &r); j++)
797 if (r->parm_num == i)
799 known_vals[i] = r->new_tree;
800 break;
804 evaluate_conditions_for_known_args (dst, false,
805 known_vals,
806 vNULL,
807 vNULL,
808 &possible_truths,
809 /* We are going to specialize,
810 so ignore nonspec truths. */
811 NULL);
812 known_vals.release ();
814 info->account_size_time (0, 0, true_pred, true_pred);
816 /* Remap size_time vectors.
817 Simplify the predicate by pruning out alternatives that are known
818 to be false.
819 TODO: as on optimization, we can also eliminate conditions known
820 to be true. */
821 for (i = 0; vec_safe_iterate (entry, i, &e); i++)
823 predicate new_exec_pred;
824 predicate new_nonconst_pred;
825 new_exec_pred = e->exec_predicate.remap_after_duplication
826 (possible_truths);
827 new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
828 (possible_truths);
829 if (new_exec_pred == false || new_nonconst_pred == false)
830 optimized_out_size += e->size;
831 else
832 info->account_size_time (e->size, e->time, new_exec_pred,
833 new_nonconst_pred);
836 /* Remap edge predicates with the same simplification as above.
837 Also copy constantness arrays. */
838 for (edge = dst->callees; edge; edge = next)
840 predicate new_predicate;
841 class ipa_call_summary *es = ipa_call_summaries->get (edge);
842 next = edge->next_callee;
844 if (!edge->inline_failed)
845 inlined_to_p = true;
846 if (!es->predicate)
847 continue;
848 new_predicate = es->predicate->remap_after_duplication
849 (possible_truths);
850 if (new_predicate == false && *es->predicate != false)
851 optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
852 edge_set_predicate (edge, &new_predicate);
855 /* Remap indirect edge predicates with the same simplification as above.
856 Also copy constantness arrays. */
857 for (edge = dst->indirect_calls; edge; edge = next)
859 predicate new_predicate;
860 class ipa_call_summary *es = ipa_call_summaries->get (edge);
861 next = edge->next_callee;
863 gcc_checking_assert (edge->inline_failed);
864 if (!es->predicate)
865 continue;
866 new_predicate = es->predicate->remap_after_duplication
867 (possible_truths);
868 if (new_predicate == false && *es->predicate != false)
869 optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
870 edge_set_predicate (edge, &new_predicate);
872 remap_hint_predicate_after_duplication (&info->loop_iterations,
873 possible_truths);
874 remap_hint_predicate_after_duplication (&info->loop_stride,
875 possible_truths);
877 /* If inliner or someone after inliner will ever start producing
878 non-trivial clones, we will get trouble with lack of information
879 about updating self sizes, because size vectors already contains
880 sizes of the callees. */
881 gcc_assert (!inlined_to_p || !optimized_out_size);
883 else
885 info->size_time_table = vec_safe_copy (info->size_time_table);
886 if (info->loop_iterations)
888 predicate p = *info->loop_iterations;
889 info->loop_iterations = NULL;
890 set_hint_predicate (&info->loop_iterations, p);
892 if (info->loop_stride)
894 predicate p = *info->loop_stride;
895 info->loop_stride = NULL;
896 set_hint_predicate (&info->loop_stride, p);
899 if (!dst->inlined_to)
900 ipa_update_overall_fn_summary (dst);
904 /* Hook that is called by cgraph.c when a node is duplicated. */
906 void
907 ipa_call_summary_t::duplicate (struct cgraph_edge *src,
908 struct cgraph_edge *dst,
909 class ipa_call_summary *srcinfo,
910 class ipa_call_summary *info)
912 new (info) ipa_call_summary (*srcinfo);
913 info->predicate = NULL;
914 edge_set_predicate (dst, srcinfo->predicate);
915 info->param = srcinfo->param.copy ();
916 if (!dst->indirect_unknown_callee && src->indirect_unknown_callee)
918 info->call_stmt_size -= (eni_size_weights.indirect_call_cost
919 - eni_size_weights.call_cost);
920 info->call_stmt_time -= (eni_time_weights.indirect_call_cost
921 - eni_time_weights.call_cost);
925 /* Dump edge summaries associated to NODE and recursively to all clones.
926 Indent by INDENT. */
928 static void
929 dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
930 class ipa_fn_summary *info)
932 struct cgraph_edge *edge;
933 for (edge = node->callees; edge; edge = edge->next_callee)
935 class ipa_call_summary *es = ipa_call_summaries->get (edge);
936 struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
937 int i;
939 fprintf (f,
940 "%*s%s %s\n%*s freq:%4.2f",
941 indent, "", callee->dump_name (),
942 !edge->inline_failed
943 ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
944 indent, "", edge->sreal_frequency ().to_double ());
946 if (cross_module_call_p (edge))
947 fprintf (f, " cross module");
949 if (es)
950 fprintf (f, " loop depth:%2i size:%2i time: %2i",
951 es->loop_depth, es->call_stmt_size, es->call_stmt_time);
953 ipa_fn_summary *s = ipa_fn_summaries->get (callee);
954 ipa_size_summary *ss = ipa_size_summaries->get (callee);
955 if (s != NULL)
956 fprintf (f, " callee size:%2i stack:%2i",
957 (int) (ss->size / ipa_fn_summary::size_scale),
958 (int) s->estimated_stack_size);
960 if (es && es->predicate)
962 fprintf (f, " predicate: ");
963 es->predicate->dump (f, info->conds);
965 else
966 fprintf (f, "\n");
967 if (es && es->param.exists ())
968 for (i = 0; i < (int) es->param.length (); i++)
970 int prob = es->param[i].change_prob;
972 if (!prob)
973 fprintf (f, "%*s op%i is compile time invariant\n",
974 indent + 2, "", i);
975 else if (prob != REG_BR_PROB_BASE)
976 fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
977 prob * 100.0 / REG_BR_PROB_BASE);
979 if (!edge->inline_failed)
981 ipa_size_summary *ss = ipa_size_summaries->get (callee);
982 fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
983 indent + 2, "",
984 (int) ipa_get_stack_frame_offset (callee),
985 (int) ss->estimated_self_stack_size);
986 dump_ipa_call_summary (f, indent + 2, callee, info);
989 for (edge = node->indirect_calls; edge; edge = edge->next_callee)
991 class ipa_call_summary *es = ipa_call_summaries->get (edge);
992 fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
993 " time: %2i",
994 indent, "",
995 es->loop_depth,
996 edge->sreal_frequency ().to_double (), es->call_stmt_size,
997 es->call_stmt_time);
998 if (es->predicate)
1000 fprintf (f, "predicate: ");
1001 es->predicate->dump (f, info->conds);
1003 else
1004 fprintf (f, "\n");
1009 void
1010 ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
1012 if (node->definition)
1014 class ipa_fn_summary *s = ipa_fn_summaries->get (node);
1015 class ipa_size_summary *ss = ipa_size_summaries->get (node);
1016 if (s != NULL)
1018 size_time_entry *e;
1019 int i;
1020 fprintf (f, "IPA function summary for %s", node->dump_name ());
1021 if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1022 fprintf (f, " always_inline");
1023 if (s->inlinable)
1024 fprintf (f, " inlinable");
1025 if (s->fp_expressions)
1026 fprintf (f, " fp_expression");
1027 fprintf (f, "\n global time: %f\n", s->time.to_double ());
1028 fprintf (f, " self size: %i\n", ss->self_size);
1029 fprintf (f, " global size: %i\n", ss->size);
1030 fprintf (f, " min size: %i\n", s->min_size);
1031 fprintf (f, " self stack: %i\n",
1032 (int) ss->estimated_self_stack_size);
1033 fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
1034 if (s->growth)
1035 fprintf (f, " estimated growth:%i\n", (int) s->growth);
1036 if (s->scc_no)
1037 fprintf (f, " In SCC: %i\n", (int) s->scc_no);
1038 for (i = 0; vec_safe_iterate (s->size_time_table, i, &e); i++)
1040 fprintf (f, " size:%f, time:%f",
1041 (double) e->size / ipa_fn_summary::size_scale,
1042 e->time.to_double ());
1043 if (e->exec_predicate != true)
1045 fprintf (f, ", executed if:");
1046 e->exec_predicate.dump (f, s->conds, 0);
1048 if (e->exec_predicate != e->nonconst_predicate)
1050 fprintf (f, ", nonconst if:");
1051 e->nonconst_predicate.dump (f, s->conds, 0);
1053 fprintf (f, "\n");
1055 if (s->loop_iterations)
1057 fprintf (f, " loop iterations:");
1058 s->loop_iterations->dump (f, s->conds);
1060 if (s->loop_stride)
1062 fprintf (f, " loop stride:");
1063 s->loop_stride->dump (f, s->conds);
1065 fprintf (f, " calls:\n");
1066 dump_ipa_call_summary (f, 4, node, s);
1067 fprintf (f, "\n");
1069 else
1070 fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
1074 DEBUG_FUNCTION void
1075 ipa_debug_fn_summary (struct cgraph_node *node)
1077 ipa_dump_fn_summary (stderr, node);
1080 void
1081 ipa_dump_fn_summaries (FILE *f)
1083 struct cgraph_node *node;
1085 FOR_EACH_DEFINED_FUNCTION (node)
1086 if (!node->inlined_to)
1087 ipa_dump_fn_summary (f, node);
1090 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1091 boolean variable pointed to by DATA. */
1093 static bool
1094 mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
1095 void *data)
1097 bool *b = (bool *) data;
1098 *b = true;
1099 return true;
1102 /* If OP refers to value of function parameter, return the corresponding
1103 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1104 PARM_DECL) will be stored to *SIZE_P in that case too. */
1106 static tree
1107 unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
1108 poly_int64 *size_p)
1110 /* SSA_NAME referring to parm default def? */
1111 if (TREE_CODE (op) == SSA_NAME
1112 && SSA_NAME_IS_DEFAULT_DEF (op)
1113 && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
1115 if (size_p)
1116 *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1117 return SSA_NAME_VAR (op);
1119 /* Non-SSA parm reference? */
1120 if (TREE_CODE (op) == PARM_DECL)
1122 bool modified = false;
1124 ao_ref refd;
1125 ao_ref_init (&refd, op);
1126 int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
1127 mark_modified, &modified, NULL, NULL,
1128 fbi->aa_walk_budget + 1);
1129 if (walked < 0)
1131 fbi->aa_walk_budget = 0;
1132 return NULL_TREE;
1134 if (!modified)
1136 if (size_p)
1137 *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
1138 return op;
1141 return NULL_TREE;
1144 /* If OP refers to value of function parameter, return the corresponding
1145 parameter. Also traverse chains of SSA register assignments. If non-NULL,
1146 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1147 stored to *SIZE_P in that case too. */
1149 static tree
1150 unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
1151 poly_int64 *size_p)
1153 tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1154 if (res)
1155 return res;
1157 if (TREE_CODE (op) == SSA_NAME
1158 && !SSA_NAME_IS_DEFAULT_DEF (op)
1159 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1160 return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
1161 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
1162 size_p);
1163 return NULL_TREE;
1166 /* If OP refers to a value of a function parameter or value loaded from an
1167 aggregate passed to a parameter (either by value or reference), return TRUE
1168 and store the number of the parameter to *INDEX_P, the access size into
1169 *SIZE_P, and information whether and how it has been loaded from an
1170 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1171 statement in which OP is used or loaded. */
1173 static bool
1174 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
1175 gimple *stmt, tree op, int *index_p,
1176 poly_int64 *size_p,
1177 struct agg_position_info *aggpos)
1179 tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
1181 gcc_checking_assert (aggpos);
1182 if (res)
1184 *index_p = ipa_get_param_decl_index (fbi->info, res);
1185 if (*index_p < 0)
1186 return false;
1187 aggpos->agg_contents = false;
1188 aggpos->by_ref = false;
1189 return true;
1192 if (TREE_CODE (op) == SSA_NAME)
1194 if (SSA_NAME_IS_DEFAULT_DEF (op)
1195 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1196 return false;
1197 stmt = SSA_NAME_DEF_STMT (op);
1198 op = gimple_assign_rhs1 (stmt);
1199 if (!REFERENCE_CLASS_P (op))
1200 return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
1201 aggpos);
1204 aggpos->agg_contents = true;
1205 return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
1206 stmt, op, index_p, &aggpos->offset,
1207 size_p, &aggpos->by_ref);
1210 /* See if statement might disappear after inlining.
1211 0 - means not eliminated
1212 1 - half of statements goes away
1213 2 - for sure it is eliminated.
1214 We are not terribly sophisticated, basically looking for simple abstraction
1215 penalty wrappers. */
1217 static int
1218 eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
1220 enum gimple_code code = gimple_code (stmt);
1221 enum tree_code rhs_code;
1223 if (!optimize)
1224 return 0;
1226 switch (code)
1228 case GIMPLE_RETURN:
1229 return 2;
1230 case GIMPLE_ASSIGN:
1231 if (gimple_num_ops (stmt) != 2)
1232 return 0;
1234 rhs_code = gimple_assign_rhs_code (stmt);
1236 /* Casts of parameters, loads from parameters passed by reference
1237 and stores to return value or parameters are often free after
1238 inlining due to SRA and further combining.
1239 Assume that half of statements goes away. */
1240 if (CONVERT_EXPR_CODE_P (rhs_code)
1241 || rhs_code == VIEW_CONVERT_EXPR
1242 || rhs_code == ADDR_EXPR
1243 || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
1245 tree rhs = gimple_assign_rhs1 (stmt);
1246 tree lhs = gimple_assign_lhs (stmt);
1247 tree inner_rhs = get_base_address (rhs);
1248 tree inner_lhs = get_base_address (lhs);
1249 bool rhs_free = false;
1250 bool lhs_free = false;
1252 if (!inner_rhs)
1253 inner_rhs = rhs;
1254 if (!inner_lhs)
1255 inner_lhs = lhs;
1257 /* Reads of parameter are expected to be free. */
1258 if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
1259 rhs_free = true;
1260 /* Match expressions of form &this->field. Those will most likely
1261 combine with something upstream after inlining. */
1262 else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
1264 tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
1265 if (TREE_CODE (op) == PARM_DECL)
1266 rhs_free = true;
1267 else if (TREE_CODE (op) == MEM_REF
1268 && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
1269 NULL))
1270 rhs_free = true;
1273 /* When parameter is not SSA register because its address is taken
1274 and it is just copied into one, the statement will be completely
1275 free after inlining (we will copy propagate backward). */
1276 if (rhs_free && is_gimple_reg (lhs))
1277 return 2;
1279 /* Reads of parameters passed by reference
1280 expected to be free (i.e. optimized out after inlining). */
1281 if (TREE_CODE (inner_rhs) == MEM_REF
1282 && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
1283 rhs_free = true;
1285 /* Copying parameter passed by reference into gimple register is
1286 probably also going to copy propagate, but we can't be quite
1287 sure. */
1288 if (rhs_free && is_gimple_reg (lhs))
1289 lhs_free = true;
1291 /* Writes to parameters, parameters passed by value and return value
1292 (either directly or passed via invisible reference) are free.
1294 TODO: We ought to handle testcase like
1295 struct a {int a,b;};
1296 struct a
1297 returnstruct (void)
1299 struct a a ={1,2};
1300 return a;
1303 This translate into:
1305 returnstruct ()
1307 int a$b;
1308 int a$a;
1309 struct a a;
1310 struct a D.2739;
1312 <bb 2>:
1313 D.2739.a = 1;
1314 D.2739.b = 2;
1315 return D.2739;
1318 For that we either need to copy ipa-split logic detecting writes
1319 to return value. */
1320 if (TREE_CODE (inner_lhs) == PARM_DECL
1321 || TREE_CODE (inner_lhs) == RESULT_DECL
1322 || (TREE_CODE (inner_lhs) == MEM_REF
1323 && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
1324 NULL)
1325 || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
1326 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
1327 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1328 (inner_lhs,
1329 0))) == RESULT_DECL))))
1330 lhs_free = true;
1331 if (lhs_free
1332 && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
1333 rhs_free = true;
1334 if (lhs_free && rhs_free)
1335 return 1;
1337 return 0;
1338 default:
1339 return 0;
1343 /* Analyze EXPR if it represents a series of simple operations performed on
1344 a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1345 AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1346 Type of the parameter or load from an aggregate via the parameter is
1347 stored in *TYPE_P. Operations on the parameter are recorded to
1348 PARAM_OPS_P if it is not NULL. */
1350 static bool
1351 decompose_param_expr (struct ipa_func_body_info *fbi,
1352 gimple *stmt, tree expr,
1353 int *index_p, tree *type_p,
1354 struct agg_position_info *aggpos,
1355 expr_eval_ops *param_ops_p = NULL)
1357 int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
1358 int op_count = 0;
1360 if (param_ops_p)
1361 *param_ops_p = NULL;
1363 while (true)
1365 expr_eval_op eval_op;
1366 unsigned rhs_count;
1367 unsigned cst_count = 0;
1369 if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
1370 aggpos))
1372 tree type = TREE_TYPE (expr);
1374 if (aggpos->agg_contents)
1376 /* Stop if containing bit-field. */
1377 if (TREE_CODE (expr) == BIT_FIELD_REF
1378 || contains_bitfld_component_ref_p (expr))
1379 break;
1382 *type_p = type;
1383 return true;
1386 if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
1387 break;
1389 if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
1390 break;
1392 switch (gimple_assign_rhs_class (stmt))
1394 case GIMPLE_SINGLE_RHS:
1395 expr = gimple_assign_rhs1 (stmt);
1396 continue;
1398 case GIMPLE_UNARY_RHS:
1399 rhs_count = 1;
1400 break;
1402 case GIMPLE_BINARY_RHS:
1403 rhs_count = 2;
1404 break;
1406 case GIMPLE_TERNARY_RHS:
1407 rhs_count = 3;
1408 break;
1410 default:
1411 goto fail;
1414 /* Stop if expression is too complex. */
1415 if (op_count++ == op_limit)
1416 break;
1418 if (param_ops_p)
1420 eval_op.code = gimple_assign_rhs_code (stmt);
1421 eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
1422 eval_op.val[0] = NULL_TREE;
1423 eval_op.val[1] = NULL_TREE;
1426 expr = NULL_TREE;
1427 for (unsigned i = 0; i < rhs_count; i++)
1429 tree op = gimple_op (stmt, i + 1);
1431 gcc_assert (op && !TYPE_P (op));
1432 if (is_gimple_ip_invariant (op))
1434 if (++cst_count == rhs_count)
1435 goto fail;
1437 eval_op.val[cst_count - 1] = op;
1439 else if (!expr)
1441 /* Found a non-constant operand, and record its index in rhs
1442 operands. */
1443 eval_op.index = i;
1444 expr = op;
1446 else
1448 /* Found more than one non-constant operands. */
1449 goto fail;
1453 if (param_ops_p)
1454 vec_safe_insert (*param_ops_p, 0, eval_op);
1457 /* Failed to decompose, free resource and return. */
1458 fail:
1459 if (param_ops_p)
1460 vec_free (*param_ops_p);
1462 return false;
1465 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1466 predicates to the CFG edges. */
1468 static void
1469 set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1470 class ipa_fn_summary *summary,
1471 class ipa_node_params *params_summary,
1472 basic_block bb)
1474 gimple *last;
1475 tree op, op2;
1476 int index;
1477 struct agg_position_info aggpos;
1478 enum tree_code code, inverted_code;
1479 edge e;
1480 edge_iterator ei;
1481 gimple *set_stmt;
1482 tree param_type;
1483 expr_eval_ops param_ops;
1485 last = last_stmt (bb);
1486 if (!last || gimple_code (last) != GIMPLE_COND)
1487 return;
1488 if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
1489 return;
1490 op = gimple_cond_lhs (last);
1492 if (decompose_param_expr (fbi, last, op, &index, &param_type, &aggpos,
1493 &param_ops))
1495 code = gimple_cond_code (last);
1496 inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
1498 FOR_EACH_EDGE (e, ei, bb->succs)
1500 enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
1501 ? code : inverted_code);
1502 /* invert_tree_comparison will return ERROR_MARK on FP
1503 comparisons that are not EQ/NE instead of returning proper
1504 unordered one. Be sure it is not confused with NON_CONSTANT.
1506 And if the edge's target is the final block of diamond CFG graph
1507 of this conditional statement, we do not need to compute
1508 predicate for the edge because the final block's predicate must
1509 be at least as that of the first block of the statement. */
1510 if (this_code != ERROR_MARK
1511 && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1513 predicate p
1514 = add_condition (summary, params_summary, index,
1515 param_type, &aggpos,
1516 this_code, gimple_cond_rhs (last), param_ops);
1517 e->aux = edge_predicate_pool.allocate ();
1518 *(predicate *) e->aux = p;
1521 vec_free (param_ops);
1524 if (TREE_CODE (op) != SSA_NAME)
1525 return;
1526 /* Special case
1527 if (builtin_constant_p (op))
1528 constant_code
1529 else
1530 nonconstant_code.
1531 Here we can predicate nonconstant_code. We can't
1532 really handle constant_code since we have no predicate
1533 for this and also the constant code is not known to be
1534 optimized away when inliner doesn't see operand is constant.
1535 Other optimizers might think otherwise. */
1536 if (gimple_cond_code (last) != NE_EXPR
1537 || !integer_zerop (gimple_cond_rhs (last)))
1538 return;
1539 set_stmt = SSA_NAME_DEF_STMT (op);
1540 if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
1541 || gimple_call_num_args (set_stmt) != 1)
1542 return;
1543 op2 = gimple_call_arg (set_stmt, 0);
1544 if (!decompose_param_expr (fbi, set_stmt, op2, &index, &param_type, &aggpos))
1545 return;
1546 FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
1548 predicate p = add_condition (summary, params_summary, index,
1549 param_type, &aggpos,
1550 predicate::is_not_constant, NULL_TREE);
1551 e->aux = edge_predicate_pool.allocate ();
1552 *(predicate *) e->aux = p;
1557 /* If BB ends by a switch we can turn into predicates, attach corresponding
1558 predicates to the CFG edges. */
1560 static void
1561 set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
1562 class ipa_fn_summary *summary,
1563 class ipa_node_params *params_summary,
1564 basic_block bb)
1566 gimple *lastg;
1567 tree op;
1568 int index;
1569 struct agg_position_info aggpos;
1570 edge e;
1571 edge_iterator ei;
1572 size_t n;
1573 size_t case_idx;
1574 tree param_type;
1575 expr_eval_ops param_ops;
1577 lastg = last_stmt (bb);
1578 if (!lastg || gimple_code (lastg) != GIMPLE_SWITCH)
1579 return;
1580 gswitch *last = as_a <gswitch *> (lastg);
1581 op = gimple_switch_index (last);
1582 if (!decompose_param_expr (fbi, last, op, &index, &param_type, &aggpos,
1583 &param_ops))
1584 return;
1586 auto_vec<std::pair<tree, tree> > ranges;
1587 tree type = TREE_TYPE (op);
1588 int bound_limit = opt_for_fn (fbi->node->decl,
1589 param_ipa_max_switch_predicate_bounds);
1590 int bound_count = 0;
1591 wide_int vr_wmin, vr_wmax;
1592 value_range_kind vr_type = get_range_info (op, &vr_wmin, &vr_wmax);
1594 FOR_EACH_EDGE (e, ei, bb->succs)
1596 e->aux = edge_predicate_pool.allocate ();
1597 *(predicate *) e->aux = false;
1600 e = gimple_switch_edge (cfun, last, 0);
1601 /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1602 default case if its target basic block is in convergence point of all
1603 switch cases, which can be determined by checking whether it
1604 post-dominates the switch statement. */
1605 if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1606 bound_count = INT_MAX;
1608 n = gimple_switch_num_labels (last);
1609 for (case_idx = 1; case_idx < n; ++case_idx)
1611 tree cl = gimple_switch_label (last, case_idx);
1612 tree min = CASE_LOW (cl);
1613 tree max = CASE_HIGH (cl);
1614 predicate p;
1616 e = gimple_switch_edge (cfun, last, case_idx);
1618 /* The case value might not have same type as switch expression,
1619 extend the value based on the expression type. */
1620 if (TREE_TYPE (min) != type)
1621 min = wide_int_to_tree (type, wi::to_wide (min));
1623 if (!max)
1624 max = min;
1625 else if (TREE_TYPE (max) != type)
1626 max = wide_int_to_tree (type, wi::to_wide (max));
1628 /* The case's target basic block is in convergence point of all switch
1629 cases, its predicate should be at least as that of the switch
1630 statement. */
1631 if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1632 p = true;
1633 else if (min == max)
1634 p = add_condition (summary, params_summary, index, param_type,
1635 &aggpos, EQ_EXPR, min, param_ops);
1636 else
1638 predicate p1, p2;
1639 p1 = add_condition (summary, params_summary, index, param_type,
1640 &aggpos, GE_EXPR, min, param_ops);
1641 p2 = add_condition (summary, params_summary,index, param_type,
1642 &aggpos, LE_EXPR, max, param_ops);
1643 p = p1 & p2;
1645 *(class predicate *) e->aux
1646 = p.or_with (summary->conds, *(class predicate *) e->aux);
1648 /* If there are too many disjoint case ranges, predicate for default
1649 case might become too complicated. So add a limit here. */
1650 if (bound_count > bound_limit)
1651 continue;
1653 bool new_range = true;
1655 if (!ranges.is_empty ())
1657 wide_int curr_wmin = wi::to_wide (min);
1658 wide_int last_wmax = wi::to_wide (ranges.last ().second);
1660 /* Merge case ranges if they are continuous. */
1661 if (curr_wmin == last_wmax + 1)
1662 new_range = false;
1663 else if (vr_type == VR_ANTI_RANGE)
1665 /* If two disjoint case ranges can be connected by anti-range
1666 of switch index, combine them to one range. */
1667 if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
1668 vr_type = VR_UNDEFINED;
1669 else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
1670 new_range = false;
1674 /* Create/extend a case range. And we count endpoints of range set,
1675 this number nearly equals to number of conditions that we will create
1676 for predicate of default case. */
1677 if (new_range)
1679 bound_count += (min == max) ? 1 : 2;
1680 ranges.safe_push (std::make_pair (min, max));
1682 else
1684 bound_count += (ranges.last ().first == ranges.last ().second);
1685 ranges.last ().second = max;
1689 e = gimple_switch_edge (cfun, last, 0);
1690 if (bound_count > bound_limit)
1692 *(class predicate *) e->aux = true;
1693 vec_free (param_ops);
1694 return;
1697 predicate p_seg = true;
1698 predicate p_all = false;
1700 if (vr_type != VR_RANGE)
1702 vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
1703 vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
1706 /* Construct predicate to represent default range set that is negation of
1707 all case ranges. Case range is classified as containing single/non-single
1708 values. Suppose a piece of case ranges in the following.
1710 [D1...D2] [S1] ... [Sn] [D3...D4]
1712 To represent default case's range sets between two non-single value
1713 case ranges (From D2 to D3), we construct predicate as:
1715 D2 < x < D3 && x != S1 && ... && x != Sn
1717 for (size_t i = 0; i < ranges.length (); i++)
1719 tree min = ranges[i].first;
1720 tree max = ranges[i].second;
1722 if (min == max)
1723 p_seg &= add_condition (summary, params_summary, index,
1724 param_type, &aggpos, NE_EXPR,
1725 min, param_ops);
1726 else
1728 /* Do not create sub-predicate for range that is beyond low bound
1729 of switch index. */
1730 if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
1732 p_seg &= add_condition (summary, params_summary, index,
1733 param_type, &aggpos,
1734 LT_EXPR, min, param_ops);
1735 p_all = p_all.or_with (summary->conds, p_seg);
1738 /* Do not create sub-predicate for range that is beyond up bound
1739 of switch index. */
1740 if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
1742 p_seg = false;
1743 break;
1746 p_seg = add_condition (summary, params_summary, index,
1747 param_type, &aggpos, GT_EXPR,
1748 max, param_ops);
1752 p_all = p_all.or_with (summary->conds, p_seg);
1753 *(class predicate *) e->aux
1754 = p_all.or_with (summary->conds, *(class predicate *) e->aux);
1756 vec_free (param_ops);
1760 /* For each BB in NODE attach to its AUX pointer predicate under
1761 which it is executable. */
1763 static void
1764 compute_bb_predicates (struct ipa_func_body_info *fbi,
1765 struct cgraph_node *node,
1766 class ipa_fn_summary *summary,
1767 class ipa_node_params *params_summary)
1769 struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
1770 bool done = false;
1771 basic_block bb;
1773 FOR_EACH_BB_FN (bb, my_function)
1775 set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
1776 set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
1779 /* Entry block is always executable. */
1780 ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
1781 = edge_predicate_pool.allocate ();
1782 *(predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
1784 /* A simple dataflow propagation of predicates forward in the CFG.
1785 TODO: work in reverse postorder. */
1786 while (!done)
1788 done = true;
1789 FOR_EACH_BB_FN (bb, my_function)
1791 predicate p = false;
1792 edge e;
1793 edge_iterator ei;
1794 FOR_EACH_EDGE (e, ei, bb->preds)
1796 if (e->src->aux)
1798 predicate this_bb_predicate
1799 = *(predicate *) e->src->aux;
1800 if (e->aux)
1801 this_bb_predicate &= (*(class predicate *) e->aux);
1802 p = p.or_with (summary->conds, this_bb_predicate);
1803 if (p == true)
1804 break;
1807 if (p != false)
1809 basic_block pdom_bb;
1811 if (!bb->aux)
1813 done = false;
1814 bb->aux = edge_predicate_pool.allocate ();
1815 *((predicate *) bb->aux) = p;
1817 else if (p != *(predicate *) bb->aux)
1819 /* This OR operation is needed to ensure monotonous data flow
1820 in the case we hit the limit on number of clauses and the
1821 and/or operations above give approximate answers. */
1822 p = p.or_with (summary->conds, *(predicate *)bb->aux);
1823 if (p != *(predicate *) bb->aux)
1825 done = false;
1826 *((predicate *) bb->aux) = p;
1830 /* For switch/if statement, we can OR-combine predicates of all
1831 its cases/branches to get predicate for basic block in their
1832 convergence point, but sometimes this will generate very
1833 complicated predicate. Actually, we can get simplified
1834 predicate in another way by using the fact that predicate
1835 for a basic block must also hold true for its post dominators.
1836 To be specific, basic block in convergence point of
1837 conditional statement should include predicate of the
1838 statement. */
1839 pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
1840 if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
1842 else if (!pdom_bb->aux)
1844 done = false;
1845 pdom_bb->aux = edge_predicate_pool.allocate ();
1846 *((predicate *) pdom_bb->aux) = p;
1848 else if (p != *(predicate *) pdom_bb->aux)
1850 p = p.or_with (summary->conds, *(predicate *)pdom_bb->aux);
1851 if (p != *(predicate *) pdom_bb->aux)
1853 done = false;
1854 *((predicate *) pdom_bb->aux) = p;
1863 /* Return predicate specifying when the STMT might have result that is not
1864 a compile time constant. */
1866 static predicate
1867 will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
1868 class ipa_fn_summary *summary,
1869 class ipa_node_params *params_summary,
1870 tree expr,
1871 vec<predicate> nonconstant_names)
1873 tree parm;
1874 int index;
1876 while (UNARY_CLASS_P (expr))
1877 expr = TREE_OPERAND (expr, 0);
1879 parm = unmodified_parm (fbi, NULL, expr, NULL);
1880 if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
1881 return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
1882 predicate::changed, NULL_TREE);
1883 if (is_gimple_min_invariant (expr))
1884 return false;
1885 if (TREE_CODE (expr) == SSA_NAME)
1886 return nonconstant_names[SSA_NAME_VERSION (expr)];
1887 if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
1889 predicate p1
1890 = will_be_nonconstant_expr_predicate (fbi, summary,
1891 params_summary,
1892 TREE_OPERAND (expr, 0),
1893 nonconstant_names);
1894 if (p1 == true)
1895 return p1;
1897 predicate p2
1898 = will_be_nonconstant_expr_predicate (fbi, summary,
1899 params_summary,
1900 TREE_OPERAND (expr, 1),
1901 nonconstant_names);
1902 return p1.or_with (summary->conds, p2);
1904 else if (TREE_CODE (expr) == COND_EXPR)
1906 predicate p1
1907 = will_be_nonconstant_expr_predicate (fbi, summary,
1908 params_summary,
1909 TREE_OPERAND (expr, 0),
1910 nonconstant_names);
1911 if (p1 == true)
1912 return p1;
1914 predicate p2
1915 = will_be_nonconstant_expr_predicate (fbi, summary,
1916 params_summary,
1917 TREE_OPERAND (expr, 1),
1918 nonconstant_names);
1919 if (p2 == true)
1920 return p2;
1921 p1 = p1.or_with (summary->conds, p2);
1922 p2 = will_be_nonconstant_expr_predicate (fbi, summary,
1923 params_summary,
1924 TREE_OPERAND (expr, 2),
1925 nonconstant_names);
1926 return p2.or_with (summary->conds, p1);
1928 else if (TREE_CODE (expr) == CALL_EXPR)
1929 return true;
1930 else
1932 debug_tree (expr);
1933 gcc_unreachable ();
1935 return false;
1939 /* Return predicate specifying when the STMT might have result that is not
1940 a compile time constant. */
1942 static predicate
1943 will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
1944 class ipa_fn_summary *summary,
1945 class ipa_node_params *params_summary,
1946 gimple *stmt,
1947 vec<predicate> nonconstant_names)
1949 predicate p = true;
1950 ssa_op_iter iter;
1951 tree use;
1952 tree param_type = NULL_TREE;
1953 predicate op_non_const;
1954 bool is_load;
1955 int base_index;
1956 struct agg_position_info aggpos;
1958 /* What statements might be optimized away
1959 when their arguments are constant. */
1960 if (gimple_code (stmt) != GIMPLE_ASSIGN
1961 && gimple_code (stmt) != GIMPLE_COND
1962 && gimple_code (stmt) != GIMPLE_SWITCH
1963 && (gimple_code (stmt) != GIMPLE_CALL
1964 || !(gimple_call_flags (stmt) & ECF_CONST)))
1965 return p;
1967 /* Stores will stay anyway. */
1968 if (gimple_store_p (stmt))
1969 return p;
1971 is_load = gimple_assign_load_p (stmt);
1973 /* Loads can be optimized when the value is known. */
1974 if (is_load)
1976 tree op = gimple_assign_rhs1 (stmt);
1977 if (!decompose_param_expr (fbi, stmt, op, &base_index, &param_type,
1978 &aggpos))
1979 return p;
1981 else
1982 base_index = -1;
1984 /* See if we understand all operands before we start
1985 adding conditionals. */
1986 FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
1988 tree parm = unmodified_parm (fbi, stmt, use, NULL);
1989 /* For arguments we can build a condition. */
1990 if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
1991 continue;
1992 if (TREE_CODE (use) != SSA_NAME)
1993 return p;
1994 /* If we know when operand is constant,
1995 we still can say something useful. */
1996 if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
1997 continue;
1998 return p;
2001 if (is_load)
2002 op_non_const =
2003 add_condition (summary, params_summary,
2004 base_index, param_type, &aggpos,
2005 predicate::changed, NULL_TREE);
2006 else
2007 op_non_const = false;
2008 FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2010 tree parm = unmodified_parm (fbi, stmt, use, NULL);
2011 int index;
2013 if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2015 if (index != base_index)
2016 p = add_condition (summary, params_summary, index,
2017 TREE_TYPE (parm), NULL,
2018 predicate::changed, NULL_TREE);
2019 else
2020 continue;
2022 else
2023 p = nonconstant_names[SSA_NAME_VERSION (use)];
2024 op_non_const = p.or_with (summary->conds, op_non_const);
2026 if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
2027 && gimple_op (stmt, 0)
2028 && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
2029 nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
2030 = op_non_const;
2031 return op_non_const;
2034 struct record_modified_bb_info
2036 tree op;
2037 bitmap bb_set;
2038 gimple *stmt;
2041 /* Value is initialized in INIT_BB and used in USE_BB. We want to compute
2042 probability how often it changes between USE_BB.
2043 INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
2044 is in different loop nest, we can do better.
2045 This is all just estimate. In theory we look for minimal cut separating
2046 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2047 anyway. */
2049 static basic_block
2050 get_minimal_bb (basic_block init_bb, basic_block use_bb)
2052 class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
2053 if (l && l->header->count < init_bb->count)
2054 return l->header;
2055 return init_bb;
2058 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2059 set except for info->stmt. */
2061 static bool
2062 record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
2064 struct record_modified_bb_info *info =
2065 (struct record_modified_bb_info *) data;
2066 if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
2067 return false;
2068 if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
2069 return false;
2070 bitmap_set_bit (info->bb_set,
2071 SSA_NAME_IS_DEFAULT_DEF (vdef)
2072 ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
2073 : get_minimal_bb
2074 (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2075 gimple_bb (info->stmt))->index);
2076 if (dump_file)
2078 fprintf (dump_file, " Param ");
2079 print_generic_expr (dump_file, info->op, TDF_SLIM);
2080 fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
2081 gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
2082 get_minimal_bb
2083 (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2084 gimple_bb (info->stmt))->index);
2085 print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
2087 return false;
2090 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2091 will change since last invocation of STMT.
2093 Value 0 is reserved for compile time invariants.
2094 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2095 ought to be REG_BR_PROB_BASE / estimated_iters. */
2097 static int
2098 param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
2100 tree op = gimple_call_arg (stmt, i);
2101 basic_block bb = gimple_bb (stmt);
2103 if (TREE_CODE (op) == WITH_SIZE_EXPR)
2104 op = TREE_OPERAND (op, 0);
2106 tree base = get_base_address (op);
2108 /* Global invariants never change. */
2109 if (is_gimple_min_invariant (base))
2110 return 0;
2112 /* We would have to do non-trivial analysis to really work out what
2113 is the probability of value to change (i.e. when init statement
2114 is in a sibling loop of the call).
2116 We do an conservative estimate: when call is executed N times more often
2117 than the statement defining value, we take the frequency 1/N. */
2118 if (TREE_CODE (base) == SSA_NAME)
2120 profile_count init_count;
2122 if (!bb->count.nonzero_p ())
2123 return REG_BR_PROB_BASE;
2125 if (SSA_NAME_IS_DEFAULT_DEF (base))
2126 init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2127 else
2128 init_count = get_minimal_bb
2129 (gimple_bb (SSA_NAME_DEF_STMT (base)),
2130 gimple_bb (stmt))->count;
2132 if (init_count < bb->count)
2133 return MAX ((init_count.to_sreal_scale (bb->count)
2134 * REG_BR_PROB_BASE).to_int (), 1);
2135 return REG_BR_PROB_BASE;
2137 else
2139 ao_ref refd;
2140 profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
2141 struct record_modified_bb_info info;
2142 tree init = ctor_for_folding (base);
2144 if (init != error_mark_node)
2145 return 0;
2146 if (!bb->count.nonzero_p ())
2147 return REG_BR_PROB_BASE;
2148 if (dump_file)
2150 fprintf (dump_file, " Analyzing param change probability of ");
2151 print_generic_expr (dump_file, op, TDF_SLIM);
2152 fprintf (dump_file, "\n");
2154 ao_ref_init (&refd, op);
2155 info.op = op;
2156 info.stmt = stmt;
2157 info.bb_set = BITMAP_ALLOC (NULL);
2158 int walked
2159 = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
2160 NULL, NULL, fbi->aa_walk_budget);
2161 if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
2163 if (dump_file)
2165 if (walked < 0)
2166 fprintf (dump_file, " Ran out of AA walking budget.\n");
2167 else
2168 fprintf (dump_file, " Set in same BB as used.\n");
2170 BITMAP_FREE (info.bb_set);
2171 return REG_BR_PROB_BASE;
2174 bitmap_iterator bi;
2175 unsigned index;
2176 /* Lookup the most frequent update of the value and believe that
2177 it dominates all the other; precise analysis here is difficult. */
2178 EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
2179 max = max.max (BASIC_BLOCK_FOR_FN (cfun, index)->count);
2180 if (dump_file)
2182 fprintf (dump_file, " Set with count ");
2183 max.dump (dump_file);
2184 fprintf (dump_file, " and used with count ");
2185 bb->count.dump (dump_file);
2186 fprintf (dump_file, " freq %f\n",
2187 max.to_sreal_scale (bb->count).to_double ());
2190 BITMAP_FREE (info.bb_set);
2191 if (max < bb->count)
2192 return MAX ((max.to_sreal_scale (bb->count)
2193 * REG_BR_PROB_BASE).to_int (), 1);
2194 return REG_BR_PROB_BASE;
2198 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2199 sub-graph and if the predicate the condition depends on is known. If so,
2200 return true and store the pointer the predicate in *P. */
2202 static bool
2203 phi_result_unknown_predicate (ipa_func_body_info *fbi,
2204 ipa_fn_summary *summary,
2205 class ipa_node_params *params_summary,
2206 basic_block bb,
2207 predicate *p,
2208 vec<predicate> nonconstant_names)
2210 edge e;
2211 edge_iterator ei;
2212 basic_block first_bb = NULL;
2213 gimple *stmt;
2215 if (single_pred_p (bb))
2217 *p = false;
2218 return true;
2221 FOR_EACH_EDGE (e, ei, bb->preds)
2223 if (single_succ_p (e->src))
2225 if (!single_pred_p (e->src))
2226 return false;
2227 if (!first_bb)
2228 first_bb = single_pred (e->src);
2229 else if (single_pred (e->src) != first_bb)
2230 return false;
2232 else
2234 if (!first_bb)
2235 first_bb = e->src;
2236 else if (e->src != first_bb)
2237 return false;
2241 if (!first_bb)
2242 return false;
2244 stmt = last_stmt (first_bb);
2245 if (!stmt
2246 || gimple_code (stmt) != GIMPLE_COND
2247 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
2248 return false;
2250 *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
2251 gimple_cond_lhs (stmt),
2252 nonconstant_names);
2253 if (*p == true)
2254 return false;
2255 else
2256 return true;
2259 /* Given a PHI statement in a function described by inline properties SUMMARY
2260 and *P being the predicate describing whether the selected PHI argument is
2261 known, store a predicate for the result of the PHI statement into
2262 NONCONSTANT_NAMES, if possible. */
2264 static void
2265 predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
2266 predicate *p,
2267 vec<predicate> nonconstant_names)
2269 unsigned i;
2271 for (i = 0; i < gimple_phi_num_args (phi); i++)
2273 tree arg = gimple_phi_arg (phi, i)->def;
2274 if (!is_gimple_min_invariant (arg))
2276 gcc_assert (TREE_CODE (arg) == SSA_NAME);
2277 *p = p->or_with (summary->conds,
2278 nonconstant_names[SSA_NAME_VERSION (arg)]);
2279 if (*p == true)
2280 return;
2284 if (dump_file && (dump_flags & TDF_DETAILS))
2286 fprintf (dump_file, "\t\tphi predicate: ");
2287 p->dump (dump_file, summary->conds);
2289 nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
2292 /* For a typical usage of __builtin_expect (a<b, 1), we
2293 may introduce an extra relation stmt:
2294 With the builtin, we have
2295 t1 = a <= b;
2296 t2 = (long int) t1;
2297 t3 = __builtin_expect (t2, 1);
2298 if (t3 != 0)
2299 goto ...
2300 Without the builtin, we have
2301 if (a<=b)
2302 goto...
2303 This affects the size/time estimation and may have
2304 an impact on the earlier inlining.
2305 Here find this pattern and fix it up later. */
2307 static gimple *
2308 find_foldable_builtin_expect (basic_block bb)
2310 gimple_stmt_iterator bsi;
2312 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2314 gimple *stmt = gsi_stmt (bsi);
2315 if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
2316 || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
2317 || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
2319 tree var = gimple_call_lhs (stmt);
2320 tree arg = gimple_call_arg (stmt, 0);
2321 use_operand_p use_p;
2322 gimple *use_stmt;
2323 bool match = false;
2324 bool done = false;
2326 if (!var || !arg)
2327 continue;
2328 gcc_assert (TREE_CODE (var) == SSA_NAME);
2330 while (TREE_CODE (arg) == SSA_NAME)
2332 gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
2333 if (!is_gimple_assign (stmt_tmp))
2334 break;
2335 switch (gimple_assign_rhs_code (stmt_tmp))
2337 case LT_EXPR:
2338 case LE_EXPR:
2339 case GT_EXPR:
2340 case GE_EXPR:
2341 case EQ_EXPR:
2342 case NE_EXPR:
2343 match = true;
2344 done = true;
2345 break;
2346 CASE_CONVERT:
2347 break;
2348 default:
2349 done = true;
2350 break;
2352 if (done)
2353 break;
2354 arg = gimple_assign_rhs1 (stmt_tmp);
2357 if (match && single_imm_use (var, &use_p, &use_stmt)
2358 && gimple_code (use_stmt) == GIMPLE_COND)
2359 return use_stmt;
2362 return NULL;
2365 /* Return true when the basic blocks contains only clobbers followed by RESX.
2366 Such BBs are kept around to make removal of dead stores possible with
2367 presence of EH and will be optimized out by optimize_clobbers later in the
2368 game.
2370 NEED_EH is used to recurse in case the clobber has non-EH predecessors
2371 that can be clobber only, too.. When it is false, the RESX is not necessary
2372 on the end of basic block. */
2374 static bool
2375 clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
2377 gimple_stmt_iterator gsi = gsi_last_bb (bb);
2378 edge_iterator ei;
2379 edge e;
2381 if (need_eh)
2383 if (gsi_end_p (gsi))
2384 return false;
2385 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
2386 return false;
2387 gsi_prev (&gsi);
2389 else if (!single_succ_p (bb))
2390 return false;
2392 for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2394 gimple *stmt = gsi_stmt (gsi);
2395 if (is_gimple_debug (stmt))
2396 continue;
2397 if (gimple_clobber_p (stmt))
2398 continue;
2399 if (gimple_code (stmt) == GIMPLE_LABEL)
2400 break;
2401 return false;
2404 /* See if all predecessors are either throws or clobber only BBs. */
2405 FOR_EACH_EDGE (e, ei, bb->preds)
2406 if (!(e->flags & EDGE_EH)
2407 && !clobber_only_eh_bb_p (e->src, false))
2408 return false;
2410 return true;
2413 /* Return true if STMT compute a floating point expression that may be affected
2414 by -ffast-math and similar flags. */
2416 static bool
2417 fp_expression_p (gimple *stmt)
2419 ssa_op_iter i;
2420 tree op;
2422 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
2423 if (FLOAT_TYPE_P (TREE_TYPE (op)))
2424 return true;
2425 return false;
2428 /* Analyze function body for NODE.
2429 EARLY indicates run from early optimization pipeline. */
2431 static void
2432 analyze_function_body (struct cgraph_node *node, bool early)
2434 sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
2435 /* Estimate static overhead for function prologue/epilogue and alignment. */
2436 int size = opt_for_fn (node->decl, param_uninlined_function_insns);
2437 /* Benefits are scaled by probability of elimination that is in range
2438 <0,2>. */
2439 basic_block bb;
2440 struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
2441 sreal freq;
2442 class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2443 class ipa_node_params *params_summary = early ? NULL : IPA_NODE_REF (node);
2444 predicate bb_predicate;
2445 struct ipa_func_body_info fbi;
2446 vec<predicate> nonconstant_names = vNULL;
2447 int nblocks, n;
2448 int *order;
2449 gimple *fix_builtin_expect_stmt;
2451 gcc_assert (my_function && my_function->cfg);
2452 gcc_assert (cfun == my_function);
2454 memset(&fbi, 0, sizeof(fbi));
2455 vec_free (info->conds);
2456 info->conds = NULL;
2457 vec_free (info->size_time_table);
2458 info->size_time_table = NULL;
2460 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2461 so we can produce proper inline hints.
2463 When optimizing and analyzing for early inliner, initialize node params
2464 so we can produce correct BB predicates. */
2466 if (opt_for_fn (node->decl, optimize))
2468 calculate_dominance_info (CDI_DOMINATORS);
2469 calculate_dominance_info (CDI_POST_DOMINATORS);
2470 if (!early)
2471 loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
2472 else
2474 ipa_check_create_node_params ();
2475 ipa_initialize_node_params (node);
2478 if (ipa_node_params_sum)
2480 fbi.node = node;
2481 fbi.info = IPA_NODE_REF (node);
2482 fbi.bb_infos = vNULL;
2483 fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun));
2484 fbi.param_count = count_formal_params (node->decl);
2485 fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
2487 nonconstant_names.safe_grow_cleared
2488 (SSANAMES (my_function)->length ());
2492 if (dump_file)
2493 fprintf (dump_file, "\nAnalyzing function body size: %s\n",
2494 node->dump_name ());
2496 /* When we run into maximal number of entries, we assign everything to the
2497 constant truth case. Be sure to have it in list. */
2498 bb_predicate = true;
2499 info->account_size_time (0, 0, bb_predicate, bb_predicate);
2501 bb_predicate = predicate::not_inlined ();
2502 info->account_size_time (opt_for_fn (node->decl,
2503 param_uninlined_function_insns)
2504 * ipa_fn_summary::size_scale,
2505 opt_for_fn (node->decl,
2506 param_uninlined_function_time),
2507 bb_predicate,
2508 bb_predicate);
2510 if (fbi.info)
2511 compute_bb_predicates (&fbi, node, info, params_summary);
2512 order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
2513 nblocks = pre_and_rev_post_order_compute (NULL, order, false);
2514 for (n = 0; n < nblocks; n++)
2516 bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
2517 freq = bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
2518 if (clobber_only_eh_bb_p (bb))
2520 if (dump_file && (dump_flags & TDF_DETAILS))
2521 fprintf (dump_file, "\n Ignoring BB %i;"
2522 " it will be optimized away by cleanup_clobbers\n",
2523 bb->index);
2524 continue;
2527 /* TODO: Obviously predicates can be propagated down across CFG. */
2528 if (fbi.info)
2530 if (bb->aux)
2531 bb_predicate = *(predicate *) bb->aux;
2532 else
2533 bb_predicate = false;
2535 else
2536 bb_predicate = true;
2538 if (dump_file && (dump_flags & TDF_DETAILS))
2540 fprintf (dump_file, "\n BB %i predicate:", bb->index);
2541 bb_predicate.dump (dump_file, info->conds);
2544 if (fbi.info && nonconstant_names.exists ())
2546 predicate phi_predicate;
2547 bool first_phi = true;
2549 for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
2550 gsi_next (&bsi))
2552 if (first_phi
2553 && !phi_result_unknown_predicate (&fbi, info,
2554 params_summary,
2556 &phi_predicate,
2557 nonconstant_names))
2558 break;
2559 first_phi = false;
2560 if (dump_file && (dump_flags & TDF_DETAILS))
2562 fprintf (dump_file, " ");
2563 print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
2565 predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
2566 nonconstant_names);
2570 fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
2572 for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
2573 !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
2575 gimple *stmt = gsi_stmt (bsi);
2576 int this_size = estimate_num_insns (stmt, &eni_size_weights);
2577 int this_time = estimate_num_insns (stmt, &eni_time_weights);
2578 int prob;
2579 predicate will_be_nonconstant;
2581 /* This relation stmt should be folded after we remove
2582 __builtin_expect call. Adjust the cost here. */
2583 if (stmt == fix_builtin_expect_stmt)
2585 this_size--;
2586 this_time--;
2589 if (dump_file && (dump_flags & TDF_DETAILS))
2591 fprintf (dump_file, " ");
2592 print_gimple_stmt (dump_file, stmt, 0);
2593 fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2594 freq.to_double (), this_size,
2595 this_time);
2598 if (is_gimple_call (stmt)
2599 && !gimple_call_internal_p (stmt))
2601 struct cgraph_edge *edge = node->get_edge (stmt);
2602 ipa_call_summary *es = ipa_call_summaries->get_create (edge);
2604 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2605 resolved as constant. We however don't want to optimize
2606 out the cgraph edges. */
2607 if (nonconstant_names.exists ()
2608 && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
2609 && gimple_call_lhs (stmt)
2610 && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
2612 predicate false_p = false;
2613 nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
2614 = false_p;
2616 if (ipa_node_params_sum)
2618 int count = gimple_call_num_args (stmt);
2619 int i;
2621 if (count)
2622 es->param.safe_grow_cleared (count);
2623 for (i = 0; i < count; i++)
2625 int prob = param_change_prob (&fbi, stmt, i);
2626 gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
2627 es->param[i].change_prob = prob;
2631 es->call_stmt_size = this_size;
2632 es->call_stmt_time = this_time;
2633 es->loop_depth = bb_loop_depth (bb);
2634 edge_set_predicate (edge, &bb_predicate);
2635 if (edge->speculative)
2637 cgraph_edge *indirect
2638 = edge->speculative_call_indirect_edge ();
2639 ipa_call_summary *es2
2640 = ipa_call_summaries->get_create (indirect);
2641 ipa_call_summaries->duplicate (edge, indirect,
2642 es, es2);
2644 /* Edge is the first direct call.
2645 create and duplicate call summaries for multiple
2646 speculative call targets. */
2647 for (cgraph_edge *direct
2648 = edge->next_speculative_call_target ();
2649 direct;
2650 direct = direct->next_speculative_call_target ())
2652 ipa_call_summary *es3
2653 = ipa_call_summaries->get_create (direct);
2654 ipa_call_summaries->duplicate (edge, direct,
2655 es, es3);
2660 /* TODO: When conditional jump or switch is known to be constant, but
2661 we did not translate it into the predicates, we really can account
2662 just maximum of the possible paths. */
2663 if (fbi.info)
2664 will_be_nonconstant
2665 = will_be_nonconstant_predicate (&fbi, info, params_summary,
2666 stmt, nonconstant_names);
2667 else
2668 will_be_nonconstant = true;
2669 if (this_time || this_size)
2671 sreal final_time = (sreal)this_time * freq;
2673 prob = eliminated_by_inlining_prob (&fbi, stmt);
2674 if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
2675 fprintf (dump_file,
2676 "\t\t50%% will be eliminated by inlining\n");
2677 if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
2678 fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
2680 class predicate p = bb_predicate & will_be_nonconstant;
2682 /* We can ignore statement when we proved it is never going
2683 to happen, but we cannot do that for call statements
2684 because edges are accounted specially. */
2686 if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
2688 time += final_time;
2689 size += this_size;
2692 /* We account everything but the calls. Calls have their own
2693 size/time info attached to cgraph edges. This is necessary
2694 in order to make the cost disappear after inlining. */
2695 if (!is_gimple_call (stmt))
2697 if (prob)
2699 predicate ip = bb_predicate & predicate::not_inlined ();
2700 info->account_size_time (this_size * prob,
2701 (final_time * prob) / 2, ip,
2704 if (prob != 2)
2705 info->account_size_time (this_size * (2 - prob),
2706 (final_time * (2 - prob) / 2),
2707 bb_predicate,
2711 if (!info->fp_expressions && fp_expression_p (stmt))
2713 info->fp_expressions = true;
2714 if (dump_file)
2715 fprintf (dump_file, " fp_expression set\n");
2719 /* Account cost of address calculations in the statements. */
2720 for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
2722 for (tree op = gimple_op (stmt, i);
2723 op && handled_component_p (op);
2724 op = TREE_OPERAND (op, 0))
2725 if ((TREE_CODE (op) == ARRAY_REF
2726 || TREE_CODE (op) == ARRAY_RANGE_REF)
2727 && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
2729 predicate p = bb_predicate;
2730 if (fbi.info)
2731 p = p & will_be_nonconstant_expr_predicate
2732 (&fbi, info, params_summary,
2733 TREE_OPERAND (op, 1),
2734 nonconstant_names);
2735 if (p != false)
2737 time += freq;
2738 size += 1;
2739 if (dump_file)
2740 fprintf (dump_file,
2741 "\t\tAccounting address calculation.\n");
2742 info->account_size_time (ipa_fn_summary::size_scale,
2743 freq,
2744 bb_predicate,
2752 free (order);
2754 if (nonconstant_names.exists () && !early)
2756 class loop *loop;
2757 predicate loop_iterations = true;
2758 predicate loop_stride = true;
2760 if (dump_file && (dump_flags & TDF_DETAILS))
2761 flow_loops_dump (dump_file, NULL, 0);
2762 scev_initialize ();
2763 FOR_EACH_LOOP (loop, 0)
2765 vec<edge> exits;
2766 edge ex;
2767 unsigned int j;
2768 class tree_niter_desc niter_desc;
2769 bb_predicate = *(predicate *) loop->header->aux;
2771 exits = get_loop_exit_edges (loop);
2772 FOR_EACH_VEC_ELT (exits, j, ex)
2773 if (number_of_iterations_exit (loop, ex, &niter_desc, false)
2774 && !is_gimple_min_invariant (niter_desc.niter))
2776 predicate will_be_nonconstant
2777 = will_be_nonconstant_expr_predicate (&fbi, info,
2778 params_summary,
2779 niter_desc.niter,
2780 nonconstant_names);
2781 if (will_be_nonconstant != true)
2782 will_be_nonconstant = bb_predicate & will_be_nonconstant;
2783 if (will_be_nonconstant != true
2784 && will_be_nonconstant != false)
2785 /* This is slightly inprecise. We may want to represent each
2786 loop with independent predicate. */
2787 loop_iterations &= will_be_nonconstant;
2789 exits.release ();
2792 /* To avoid quadratic behavior we analyze stride predicates only
2793 with respect to the containing loop. Thus we simply iterate
2794 over all defs in the outermost loop body. */
2795 for (loop = loops_for_fn (cfun)->tree_root->inner;
2796 loop != NULL; loop = loop->next)
2798 basic_block *body = get_loop_body (loop);
2799 for (unsigned i = 0; i < loop->num_nodes; i++)
2801 gimple_stmt_iterator gsi;
2802 bb_predicate = *(predicate *) body[i]->aux;
2803 for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
2804 gsi_next (&gsi))
2806 gimple *stmt = gsi_stmt (gsi);
2808 if (!is_gimple_assign (stmt))
2809 continue;
2811 tree def = gimple_assign_lhs (stmt);
2812 if (TREE_CODE (def) != SSA_NAME)
2813 continue;
2815 affine_iv iv;
2816 if (!simple_iv (loop_containing_stmt (stmt),
2817 loop_containing_stmt (stmt),
2818 def, &iv, true)
2819 || is_gimple_min_invariant (iv.step))
2820 continue;
2822 predicate will_be_nonconstant
2823 = will_be_nonconstant_expr_predicate (&fbi, info,
2824 params_summary,
2825 iv.step,
2826 nonconstant_names);
2827 if (will_be_nonconstant != true)
2828 will_be_nonconstant = bb_predicate & will_be_nonconstant;
2829 if (will_be_nonconstant != true
2830 && will_be_nonconstant != false)
2831 /* This is slightly inprecise. We may want to represent
2832 each loop with independent predicate. */
2833 loop_stride = loop_stride & will_be_nonconstant;
2836 free (body);
2838 ipa_fn_summary *s = ipa_fn_summaries->get (node);
2839 set_hint_predicate (&s->loop_iterations, loop_iterations);
2840 set_hint_predicate (&s->loop_stride, loop_stride);
2841 scev_finalize ();
2843 FOR_ALL_BB_FN (bb, my_function)
2845 edge e;
2846 edge_iterator ei;
2848 if (bb->aux)
2849 edge_predicate_pool.remove ((predicate *)bb->aux);
2850 bb->aux = NULL;
2851 FOR_EACH_EDGE (e, ei, bb->succs)
2853 if (e->aux)
2854 edge_predicate_pool.remove ((predicate *) e->aux);
2855 e->aux = NULL;
2858 ipa_fn_summary *s = ipa_fn_summaries->get (node);
2859 ipa_size_summary *ss = ipa_size_summaries->get (node);
2860 s->time = time;
2861 ss->self_size = size;
2862 nonconstant_names.release ();
2863 ipa_release_body_info (&fbi);
2864 if (opt_for_fn (node->decl, optimize))
2866 if (!early)
2867 loop_optimizer_finalize ();
2868 else if (!ipa_edge_args_sum)
2869 ipa_free_all_node_params ();
2870 free_dominance_info (CDI_DOMINATORS);
2871 free_dominance_info (CDI_POST_DOMINATORS);
2873 if (dump_file)
2875 fprintf (dump_file, "\n");
2876 ipa_dump_fn_summary (dump_file, node);
2881 /* Compute function summary.
2882 EARLY is true when we compute parameters during early opts. */
2884 void
2885 compute_fn_summary (struct cgraph_node *node, bool early)
2887 HOST_WIDE_INT self_stack_size;
2888 struct cgraph_edge *e;
2890 gcc_assert (!node->inlined_to);
2892 if (!ipa_fn_summaries)
2893 ipa_fn_summary_alloc ();
2895 /* Create a new ipa_fn_summary. */
2896 ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
2897 ipa_fn_summaries->remove (node);
2898 class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2899 class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
2901 /* Estimate the stack size for the function if we're optimizing. */
2902 self_stack_size = optimize && !node->thunk.thunk_p
2903 ? estimated_stack_frame_size (node) : 0;
2904 size_info->estimated_self_stack_size = self_stack_size;
2905 info->estimated_stack_size = self_stack_size;
2907 if (node->thunk.thunk_p)
2909 ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
2910 predicate t = true;
2912 node->can_change_signature = false;
2913 es->call_stmt_size = eni_size_weights.call_cost;
2914 es->call_stmt_time = eni_time_weights.call_cost;
2915 info->account_size_time (ipa_fn_summary::size_scale
2916 * opt_for_fn (node->decl,
2917 param_uninlined_function_thunk_insns),
2918 opt_for_fn (node->decl,
2919 param_uninlined_function_thunk_time), t, t);
2920 t = predicate::not_inlined ();
2921 info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
2922 ipa_update_overall_fn_summary (node);
2923 size_info->self_size = size_info->size;
2924 if (stdarg_p (TREE_TYPE (node->decl)))
2926 info->inlinable = false;
2927 node->callees->inline_failed = CIF_VARIADIC_THUNK;
2929 else
2930 info->inlinable = true;
2932 else
2934 /* Even is_gimple_min_invariant rely on current_function_decl. */
2935 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2937 /* During IPA profile merging we may be called w/o virtual SSA form
2938 built. */
2939 update_ssa (TODO_update_ssa_only_virtuals);
2941 /* Can this function be inlined at all? */
2942 if (!opt_for_fn (node->decl, optimize)
2943 && !lookup_attribute ("always_inline",
2944 DECL_ATTRIBUTES (node->decl)))
2945 info->inlinable = false;
2946 else
2947 info->inlinable = tree_inlinable_function_p (node->decl);
2949 /* Type attributes can use parameter indices to describe them. */
2950 if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl))
2951 /* Likewise for #pragma omp declare simd functions or functions
2952 with simd attribute. */
2953 || lookup_attribute ("omp declare simd",
2954 DECL_ATTRIBUTES (node->decl)))
2955 node->can_change_signature = false;
2956 else
2958 /* Otherwise, inlinable functions always can change signature. */
2959 if (info->inlinable)
2960 node->can_change_signature = true;
2961 else
2963 /* Functions calling builtin_apply cannot change signature. */
2964 for (e = node->callees; e; e = e->next_callee)
2966 tree cdecl = e->callee->decl;
2967 if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS)
2968 || fndecl_built_in_p (cdecl, BUILT_IN_VA_START))
2969 break;
2971 node->can_change_signature = !e;
2974 analyze_function_body (node, early);
2975 pop_cfun ();
2978 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2979 size_info->size = size_info->self_size;
2980 info->estimated_stack_size = size_info->estimated_self_stack_size;
2982 /* Code above should compute exactly the same result as
2983 ipa_update_overall_fn_summary but because computation happens in
2984 different order the roundoff errors result in slight changes. */
2985 ipa_update_overall_fn_summary (node);
2986 /* In LTO mode we may have speculative edges set. */
2987 gcc_assert (in_lto_p || size_info->size == size_info->self_size);
2991 /* Compute parameters of functions used by inliner using
2992 current_function_decl. */
2994 static unsigned int
2995 compute_fn_summary_for_current (void)
2997 compute_fn_summary (cgraph_node::get (current_function_decl), true);
2998 return 0;
3001 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
3002 KNOWN_CONTEXTS and KNOWN_AGGS. */
3004 static bool
3005 estimate_edge_devirt_benefit (struct cgraph_edge *ie,
3006 int *size, int *time,
3007 vec<tree> known_vals,
3008 vec<ipa_polymorphic_call_context> known_contexts,
3009 vec<ipa_agg_value_set> known_aggs)
3011 tree target;
3012 struct cgraph_node *callee;
3013 class ipa_fn_summary *isummary;
3014 enum availability avail;
3015 bool speculative;
3017 if (!known_vals.length () && !known_contexts.length ())
3018 return false;
3019 if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
3020 return false;
3022 target = ipa_get_indirect_edge_target (ie, known_vals, known_contexts,
3023 known_aggs, &speculative);
3024 if (!target || speculative)
3025 return false;
3027 /* Account for difference in cost between indirect and direct calls. */
3028 *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
3029 *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
3030 gcc_checking_assert (*time >= 0);
3031 gcc_checking_assert (*size >= 0);
3033 callee = cgraph_node::get (target);
3034 if (!callee || !callee->definition)
3035 return false;
3036 callee = callee->function_symbol (&avail);
3037 if (avail < AVAIL_AVAILABLE)
3038 return false;
3039 isummary = ipa_fn_summaries->get (callee);
3040 if (isummary == NULL)
3041 return false;
3043 return isummary->inlinable;
3046 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3047 handle edge E with probability PROB.
3048 Set HINTS if edge may be devirtualized.
3049 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
3050 site. */
3052 static inline void
3053 estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
3054 sreal *time,
3055 vec<tree> known_vals,
3056 vec<ipa_polymorphic_call_context> known_contexts,
3057 vec<ipa_agg_value_set> known_aggs,
3058 ipa_hints *hints)
3060 class ipa_call_summary *es = ipa_call_summaries->get (e);
3061 int call_size = es->call_stmt_size;
3062 int call_time = es->call_stmt_time;
3063 int cur_size;
3065 if (!e->callee && hints && e->maybe_hot_p ()
3066 && estimate_edge_devirt_benefit (e, &call_size, &call_time,
3067 known_vals, known_contexts, known_aggs))
3068 *hints |= INLINE_HINT_indirect_call;
3069 cur_size = call_size * ipa_fn_summary::size_scale;
3070 *size += cur_size;
3071 if (min_size)
3072 *min_size += cur_size;
3073 if (time)
3074 *time += ((sreal)call_time) * e->sreal_frequency ();
3078 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3079 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
3080 describe context of the call site.
3082 Helper for estimate_calls_size_and_time which does the same but
3083 (in most cases) faster. */
3085 static void
3086 estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
3087 int *min_size, sreal *time,
3088 ipa_hints *hints,
3089 clause_t possible_truths,
3090 vec<tree> known_vals,
3091 vec<ipa_polymorphic_call_context> known_contexts,
3092 vec<ipa_agg_value_set> known_aggs)
3094 struct cgraph_edge *e;
3095 for (e = node->callees; e; e = e->next_callee)
3097 if (!e->inline_failed)
3099 gcc_checking_assert (!ipa_call_summaries->get (e));
3100 estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
3101 hints,
3102 possible_truths,
3103 known_vals, known_contexts,
3104 known_aggs);
3105 continue;
3107 class ipa_call_summary *es = ipa_call_summaries->get (e);
3109 /* Do not care about zero sized builtins. */
3110 if (!es->call_stmt_size)
3112 gcc_checking_assert (!es->call_stmt_time);
3113 continue;
3115 if (!es->predicate
3116 || es->predicate->evaluate (possible_truths))
3118 /* Predicates of calls shall not use NOT_CHANGED codes,
3119 so we do not need to compute probabilities. */
3120 estimate_edge_size_and_time (e, size,
3121 es->predicate ? NULL : min_size,
3122 time,
3123 known_vals, known_contexts,
3124 known_aggs, hints);
3127 for (e = node->indirect_calls; e; e = e->next_callee)
3129 class ipa_call_summary *es = ipa_call_summaries->get (e);
3130 if (!es->predicate
3131 || es->predicate->evaluate (possible_truths))
3132 estimate_edge_size_and_time (e, size,
3133 es->predicate ? NULL : min_size,
3134 time,
3135 known_vals, known_contexts, known_aggs,
3136 hints);
3140 /* Populate sum->call_size_time_table for edges from NODE. */
3142 static void
3143 summarize_calls_size_and_time (struct cgraph_node *node,
3144 ipa_fn_summary *sum)
3146 struct cgraph_edge *e;
3147 for (e = node->callees; e; e = e->next_callee)
3149 if (!e->inline_failed)
3151 gcc_checking_assert (!ipa_call_summaries->get (e));
3152 summarize_calls_size_and_time (e->callee, sum);
3153 continue;
3155 int size = 0;
3156 sreal time = 0;
3158 estimate_edge_size_and_time (e, &size, NULL, &time,
3159 vNULL, vNULL, vNULL, NULL);
3161 struct predicate pred = true;
3162 class ipa_call_summary *es = ipa_call_summaries->get (e);
3164 if (es->predicate)
3165 pred = *es->predicate;
3166 sum->account_size_time (size, time, pred, pred, true);
3168 for (e = node->indirect_calls; e; e = e->next_callee)
3170 int size = 0;
3171 sreal time = 0;
3173 estimate_edge_size_and_time (e, &size, NULL, &time,
3174 vNULL, vNULL, vNULL, NULL);
3175 struct predicate pred = true;
3176 class ipa_call_summary *es = ipa_call_summaries->get (e);
3178 if (es->predicate)
3179 pred = *es->predicate;
3180 sum->account_size_time (size, time, pred, pred, true);
3184 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3185 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
3186 describe context of the call site. */
3188 static void
3189 estimate_calls_size_and_time (struct cgraph_node *node, int *size,
3190 int *min_size, sreal *time,
3191 ipa_hints *hints,
3192 clause_t possible_truths,
3193 vec<tree> known_vals,
3194 vec<ipa_polymorphic_call_context> known_contexts,
3195 vec<ipa_agg_value_set> known_aggs)
3197 class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
3198 bool use_table = true;
3200 gcc_assert (node->callees || node->indirect_calls);
3202 /* During early inlining we do not calculate info for very
3203 large functions and thus there is no need for producing
3204 summaries. */
3205 if (!ipa_node_params_sum)
3206 use_table = false;
3207 /* Do not calculate summaries for simple wrappers; it is waste
3208 of memory. */
3209 else if (node->callees && node->indirect_calls
3210 && node->callees->inline_failed && !node->callees->next_callee)
3211 use_table = false;
3212 /* If there is an indirect edge that may be optimized, we need
3213 to go the slow way. */
3214 else if ((known_vals.length ()
3215 || known_contexts.length ()
3216 || known_aggs.length ()) && hints)
3218 class ipa_node_params *params_summary = IPA_NODE_REF (node);
3219 unsigned int nargs = params_summary
3220 ? ipa_get_param_count (params_summary) : 0;
3222 for (unsigned int i = 0; i < nargs && use_table; i++)
3224 if (ipa_is_param_used_by_indirect_call (params_summary, i)
3225 && ((known_vals.length () > i && known_vals[i])
3226 || (known_aggs.length () > i
3227 && known_aggs[i].items.length ())))
3228 use_table = false;
3229 else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3230 && (known_contexts.length () > i
3231 && !known_contexts[i].useless_p ()))
3232 use_table = false;
3236 /* Fast path is via the call size time table. */
3237 if (use_table)
3239 /* Build summary if it is absent. */
3240 if (!sum->call_size_time_table)
3242 predicate true_pred = true;
3243 sum->account_size_time (0, 0, true_pred, true_pred, true);
3244 summarize_calls_size_and_time (node, sum);
3247 int old_size = *size;
3248 sreal old_time = time ? *time : 0;
3250 if (min_size)
3251 *min_size += (*sum->call_size_time_table)[0].size;
3253 unsigned int i;
3254 size_time_entry *e;
3256 /* Walk the table and account sizes and times. */
3257 for (i = 0; vec_safe_iterate (sum->call_size_time_table, i, &e);
3258 i++)
3259 if (e->exec_predicate.evaluate (possible_truths))
3261 *size += e->size;
3262 if (time)
3263 *time += e->time;
3266 /* Be careful and see if both methods agree. */
3267 if ((flag_checking || dump_file)
3268 /* Do not try to sanity check when we know we lost some
3269 precision. */
3270 && sum->call_size_time_table->length ()
3271 < ipa_fn_summary::max_size_time_table_size)
3273 estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
3274 possible_truths, known_vals,
3275 known_contexts, known_aggs);
3276 gcc_assert (*size == old_size);
3277 if (time && (*time - old_time > 1 || *time - old_time < -1)
3278 && dump_file)
3279 fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
3280 old_time.to_double (),
3281 time->to_double ());
3284 /* Slow path by walking all edges. */
3285 else
3286 estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
3287 possible_truths, known_vals, known_contexts,
3288 known_aggs);
3291 /* Default constructor for ipa call context.
3292 Memory allocation of known_vals, known_contexts
3293 and known_aggs vectors is owned by the caller, but can
3294 be release by ipa_call_context::release.
3296 inline_param_summary is owned by the caller. */
3297 ipa_call_context::ipa_call_context (cgraph_node *node,
3298 clause_t possible_truths,
3299 clause_t nonspec_possible_truths,
3300 vec<tree> known_vals,
3301 vec<ipa_polymorphic_call_context>
3302 known_contexts,
3303 vec<ipa_agg_value_set> known_aggs,
3304 vec<inline_param_summary>
3305 inline_param_summary)
3306 : m_node (node), m_possible_truths (possible_truths),
3307 m_nonspec_possible_truths (nonspec_possible_truths),
3308 m_inline_param_summary (inline_param_summary),
3309 m_known_vals (known_vals),
3310 m_known_contexts (known_contexts),
3311 m_known_aggs (known_aggs)
3315 /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3317 void
3318 ipa_call_context::duplicate_from (const ipa_call_context &ctx)
3320 m_node = ctx.m_node;
3321 m_possible_truths = ctx.m_possible_truths;
3322 m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
3323 class ipa_node_params *params_summary = IPA_NODE_REF (m_node);
3324 unsigned int nargs = params_summary
3325 ? ipa_get_param_count (params_summary) : 0;
3327 m_inline_param_summary = vNULL;
3328 /* Copy the info only if there is at least one useful entry. */
3329 if (ctx.m_inline_param_summary.exists ())
3331 unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
3333 for (unsigned int i = 0; i < n; i++)
3334 if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
3335 && !ctx.m_inline_param_summary[i].useless_p ())
3337 m_inline_param_summary
3338 = ctx.m_inline_param_summary.copy ();
3339 break;
3342 m_known_vals = vNULL;
3343 if (ctx.m_known_vals.exists ())
3345 unsigned int n = MIN (ctx.m_known_vals.length (), nargs);
3347 for (unsigned int i = 0; i < n; i++)
3348 if (ipa_is_param_used_by_indirect_call (params_summary, i)
3349 && ctx.m_known_vals[i])
3351 m_known_vals = ctx.m_known_vals.copy ();
3352 break;
3356 m_known_contexts = vNULL;
3357 if (ctx.m_known_contexts.exists ())
3359 unsigned int n = MIN (ctx.m_known_contexts.length (), nargs);
3361 for (unsigned int i = 0; i < n; i++)
3362 if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3363 && !ctx.m_known_contexts[i].useless_p ())
3365 m_known_contexts = ctx.m_known_contexts.copy ();
3366 break;
3370 m_known_aggs = vNULL;
3371 if (ctx.m_known_aggs.exists ())
3373 unsigned int n = MIN (ctx.m_known_aggs.length (), nargs);
3375 for (unsigned int i = 0; i < n; i++)
3376 if (ipa_is_param_used_by_indirect_call (params_summary, i)
3377 && !ctx.m_known_aggs[i].is_empty ())
3379 m_known_aggs = ipa_copy_agg_values (ctx.m_known_aggs);
3380 break;
3385 /* Release memory used by known_vals/contexts/aggs vectors.
3386 If ALL is true release also inline_param_summary.
3387 This happens when context was previously duplicated to be stored
3388 into cache. */
3390 void
3391 ipa_call_context::release (bool all)
3393 /* See if context is initialized at first place. */
3394 if (!m_node)
3395 return;
3396 ipa_release_agg_values (m_known_aggs, all);
3397 if (all)
3399 m_known_vals.release ();
3400 m_known_contexts.release ();
3401 m_inline_param_summary.release ();
3405 /* Return true if CTX describes the same call context as THIS. */
3407 bool
3408 ipa_call_context::equal_to (const ipa_call_context &ctx)
3410 if (m_node != ctx.m_node
3411 || m_possible_truths != ctx.m_possible_truths
3412 || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
3413 return false;
3415 class ipa_node_params *params_summary = IPA_NODE_REF (m_node);
3416 unsigned int nargs = params_summary
3417 ? ipa_get_param_count (params_summary) : 0;
3419 if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
3421 for (unsigned int i = 0; i < nargs; i++)
3423 if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
3424 continue;
3425 if (i >= m_inline_param_summary.length ()
3426 || m_inline_param_summary[i].useless_p ())
3428 if (i < ctx.m_inline_param_summary.length ()
3429 && !ctx.m_inline_param_summary[i].useless_p ())
3430 return false;
3431 continue;
3433 if (i >= ctx.m_inline_param_summary.length ()
3434 || ctx.m_inline_param_summary[i].useless_p ())
3436 if (i < m_inline_param_summary.length ()
3437 && !m_inline_param_summary[i].useless_p ())
3438 return false;
3439 continue;
3441 if (!m_inline_param_summary[i].equal_to
3442 (ctx.m_inline_param_summary[i]))
3443 return false;
3446 if (m_known_vals.exists () || ctx.m_known_vals.exists ())
3448 for (unsigned int i = 0; i < nargs; i++)
3450 if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3451 continue;
3452 if (i >= m_known_vals.length () || !m_known_vals[i])
3454 if (i < ctx.m_known_vals.length () && ctx.m_known_vals[i])
3455 return false;
3456 continue;
3458 if (i >= ctx.m_known_vals.length () || !ctx.m_known_vals[i])
3460 if (i < m_known_vals.length () && m_known_vals[i])
3461 return false;
3462 continue;
3464 if (m_known_vals[i] != ctx.m_known_vals[i])
3465 return false;
3468 if (m_known_contexts.exists () || ctx.m_known_contexts.exists ())
3470 for (unsigned int i = 0; i < nargs; i++)
3472 if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
3473 continue;
3474 if (i >= m_known_contexts.length ()
3475 || m_known_contexts[i].useless_p ())
3477 if (i < ctx.m_known_contexts.length ()
3478 && !ctx.m_known_contexts[i].useless_p ())
3479 return false;
3480 continue;
3482 if (i >= ctx.m_known_contexts.length ()
3483 || ctx.m_known_contexts[i].useless_p ())
3485 if (i < m_known_contexts.length ()
3486 && !m_known_contexts[i].useless_p ())
3487 return false;
3488 continue;
3490 if (!m_known_contexts[i].equal_to
3491 (ctx.m_known_contexts[i]))
3492 return false;
3495 if (m_known_aggs.exists () || ctx.m_known_aggs.exists ())
3497 for (unsigned int i = 0; i < nargs; i++)
3499 if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3500 continue;
3501 if (i >= m_known_aggs.length () || m_known_aggs[i].is_empty ())
3503 if (i < ctx.m_known_aggs.length ()
3504 && !ctx.m_known_aggs[i].is_empty ())
3505 return false;
3506 continue;
3508 if (i >= ctx.m_known_aggs.length ()
3509 || ctx.m_known_aggs[i].is_empty ())
3511 if (i < m_known_aggs.length ()
3512 && !m_known_aggs[i].is_empty ())
3513 return false;
3514 continue;
3516 if (!m_known_aggs[i].equal_to (ctx.m_known_aggs[i]))
3517 return false;
3520 return true;
3523 /* Estimate size and time needed to execute call in the given context.
3524 Additionally determine hints determined by the context. Finally compute
3525 minimal size needed for the call that is independent on the call context and
3526 can be used for fast estimates. Return the values in RET_SIZE,
3527 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
3529 void
3530 ipa_call_context::estimate_size_and_time (int *ret_size,
3531 int *ret_min_size,
3532 sreal *ret_time,
3533 sreal *ret_nonspecialized_time,
3534 ipa_hints *ret_hints)
3536 class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
3537 size_time_entry *e;
3538 int size = 0;
3539 sreal time = 0;
3540 int min_size = 0;
3541 ipa_hints hints = 0;
3542 int i;
3544 if (dump_file && (dump_flags & TDF_DETAILS))
3546 bool found = false;
3547 fprintf (dump_file, " Estimating body: %s\n"
3548 " Known to be false: ", m_node->dump_name ());
3550 for (i = predicate::not_inlined_condition;
3551 i < (predicate::first_dynamic_condition
3552 + (int) vec_safe_length (info->conds)); i++)
3553 if (!(m_possible_truths & (1 << i)))
3555 if (found)
3556 fprintf (dump_file, ", ");
3557 found = true;
3558 dump_condition (dump_file, info->conds, i);
3562 if (m_node->callees || m_node->indirect_calls)
3563 estimate_calls_size_and_time (m_node, &size, &min_size,
3564 ret_time ? &time : NULL,
3565 ret_hints ? &hints : NULL, m_possible_truths,
3566 m_known_vals, m_known_contexts, m_known_aggs);
3568 sreal nonspecialized_time = time;
3570 min_size += (*info->size_time_table)[0].size;
3571 for (i = 0; vec_safe_iterate (info->size_time_table, i, &e); i++)
3573 bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
3575 /* Because predicates are conservative, it can happen that nonconst is 1
3576 but exec is 0. */
3577 if (exec)
3579 bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
3581 gcc_checking_assert (e->time >= 0);
3582 gcc_checking_assert (time >= 0);
3584 /* We compute specialized size only because size of nonspecialized
3585 copy is context independent.
3587 The difference between nonspecialized execution and specialized is
3588 that nonspecialized is not going to have optimized out computations
3589 known to be constant in a specialized setting. */
3590 if (nonconst)
3591 size += e->size;
3592 if (!ret_time)
3593 continue;
3594 nonspecialized_time += e->time;
3595 if (!nonconst)
3597 else if (!m_inline_param_summary.exists ())
3599 if (nonconst)
3600 time += e->time;
3602 else
3604 int prob = e->nonconst_predicate.probability
3605 (info->conds, m_possible_truths,
3606 m_inline_param_summary);
3607 gcc_checking_assert (prob >= 0);
3608 gcc_checking_assert (prob <= REG_BR_PROB_BASE);
3609 if (prob == REG_BR_PROB_BASE)
3610 time += e->time;
3611 else
3612 time += e->time * prob / REG_BR_PROB_BASE;
3614 gcc_checking_assert (time >= 0);
3617 gcc_checking_assert ((*info->size_time_table)[0].exec_predicate == true);
3618 gcc_checking_assert ((*info->size_time_table)[0].nonconst_predicate == true);
3619 gcc_checking_assert (min_size >= 0);
3620 gcc_checking_assert (size >= 0);
3621 gcc_checking_assert (time >= 0);
3622 /* nonspecialized_time should be always bigger than specialized time.
3623 Roundoff issues however may get into the way. */
3624 gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
3626 /* Roundoff issues may make specialized time bigger than nonspecialized
3627 time. We do not really want that to happen because some heuristics
3628 may get confused by seeing negative speedups. */
3629 if (time > nonspecialized_time)
3630 time = nonspecialized_time;
3632 if (ret_hints)
3634 if (info->loop_iterations
3635 && !info->loop_iterations->evaluate (m_possible_truths))
3636 hints |= INLINE_HINT_loop_iterations;
3637 if (info->loop_stride
3638 && !info->loop_stride->evaluate (m_possible_truths))
3639 hints |= INLINE_HINT_loop_stride;
3640 if (info->scc_no)
3641 hints |= INLINE_HINT_in_scc;
3642 if (DECL_DECLARED_INLINE_P (m_node->decl))
3643 hints |= INLINE_HINT_declared_inline;
3646 size = RDIV (size, ipa_fn_summary::size_scale);
3647 min_size = RDIV (min_size, ipa_fn_summary::size_scale);
3649 if (dump_file && (dump_flags & TDF_DETAILS))
3650 fprintf (dump_file, "\n size:%i time:%f nonspec time:%f\n", (int) size,
3651 time.to_double (), nonspecialized_time.to_double ());
3652 if (ret_time)
3653 *ret_time = time;
3654 if (ret_nonspecialized_time)
3655 *ret_nonspecialized_time = nonspecialized_time;
3656 if (ret_size)
3657 *ret_size = size;
3658 if (ret_min_size)
3659 *ret_min_size = min_size;
3660 if (ret_hints)
3661 *ret_hints = hints;
3662 return;
3666 /* Estimate size and time needed to execute callee of EDGE assuming that
3667 parameters known to be constant at caller of EDGE are propagated.
3668 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
3669 and types for parameters. */
3671 void
3672 estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
3673 vec<tree> known_vals,
3674 vec<ipa_polymorphic_call_context>
3675 known_contexts,
3676 vec<ipa_agg_value_set> known_aggs,
3677 int *ret_size, sreal *ret_time,
3678 sreal *ret_nonspec_time,
3679 ipa_hints *hints)
3681 clause_t clause, nonspec_clause;
3683 /* TODO: Also pass known value ranges. */
3684 evaluate_conditions_for_known_args (node, false, known_vals, vNULL,
3685 known_aggs, &clause, &nonspec_clause);
3686 ipa_call_context ctx (node, clause, nonspec_clause,
3687 known_vals, known_contexts,
3688 known_aggs, vNULL);
3689 ctx.estimate_size_and_time (ret_size, NULL, ret_time,
3690 ret_nonspec_time, hints);
3693 /* Return stack frame offset where frame of NODE is supposed to start inside
3694 of the function it is inlined to.
3695 Return 0 for functions that are not inlined. */
3697 HOST_WIDE_INT
3698 ipa_get_stack_frame_offset (struct cgraph_node *node)
3700 HOST_WIDE_INT offset = 0;
3701 if (!node->inlined_to)
3702 return 0;
3703 node = node->callers->caller;
3704 while (true)
3706 offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
3707 if (!node->inlined_to)
3708 return offset;
3709 node = node->callers->caller;
3714 /* Update summary information of inline clones after inlining.
3715 Compute peak stack usage. */
3717 static void
3718 inline_update_callee_summaries (struct cgraph_node *node, int depth)
3720 struct cgraph_edge *e;
3722 ipa_propagate_frequency (node);
3723 for (e = node->callees; e; e = e->next_callee)
3725 if (!e->inline_failed)
3726 inline_update_callee_summaries (e->callee, depth);
3727 else
3728 ipa_call_summaries->get (e)->loop_depth += depth;
3730 for (e = node->indirect_calls; e; e = e->next_callee)
3731 ipa_call_summaries->get (e)->loop_depth += depth;
3734 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3735 When function A is inlined in B and A calls C with parameter that
3736 changes with probability PROB1 and C is known to be passthrough
3737 of argument if B that change with probability PROB2, the probability
3738 of change is now PROB1*PROB2. */
3740 static void
3741 remap_edge_change_prob (struct cgraph_edge *inlined_edge,
3742 struct cgraph_edge *edge)
3744 if (ipa_node_params_sum)
3746 int i;
3747 class ipa_edge_args *args = IPA_EDGE_REF (edge);
3748 if (!args)
3749 return;
3750 class ipa_call_summary *es = ipa_call_summaries->get (edge);
3751 class ipa_call_summary *inlined_es
3752 = ipa_call_summaries->get (inlined_edge);
3754 if (es->param.length () == 0)
3755 return;
3757 for (i = 0; i < ipa_get_cs_argument_count (args); i++)
3759 struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
3760 if (jfunc->type == IPA_JF_PASS_THROUGH
3761 || jfunc->type == IPA_JF_ANCESTOR)
3763 int id = jfunc->type == IPA_JF_PASS_THROUGH
3764 ? ipa_get_jf_pass_through_formal_id (jfunc)
3765 : ipa_get_jf_ancestor_formal_id (jfunc);
3766 if (id < (int) inlined_es->param.length ())
3768 int prob1 = es->param[i].change_prob;
3769 int prob2 = inlined_es->param[id].change_prob;
3770 int prob = combine_probabilities (prob1, prob2);
3772 if (prob1 && prob2 && !prob)
3773 prob = 1;
3775 es->param[i].change_prob = prob;
3782 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3784 Remap predicates of callees of NODE. Rest of arguments match
3785 remap_predicate.
3787 Also update change probabilities. */
3789 static void
3790 remap_edge_summaries (struct cgraph_edge *inlined_edge,
3791 struct cgraph_node *node,
3792 class ipa_fn_summary *info,
3793 class ipa_node_params *params_summary,
3794 class ipa_fn_summary *callee_info,
3795 vec<int> operand_map,
3796 vec<int> offset_map,
3797 clause_t possible_truths,
3798 predicate *toplev_predicate)
3800 struct cgraph_edge *e, *next;
3801 for (e = node->callees; e; e = next)
3803 predicate p;
3804 next = e->next_callee;
3806 if (e->inline_failed)
3808 class ipa_call_summary *es = ipa_call_summaries->get (e);
3809 remap_edge_change_prob (inlined_edge, e);
3811 if (es->predicate)
3813 p = es->predicate->remap_after_inlining
3814 (info, params_summary,
3815 callee_info, operand_map,
3816 offset_map, possible_truths,
3817 *toplev_predicate);
3818 edge_set_predicate (e, &p);
3820 else
3821 edge_set_predicate (e, toplev_predicate);
3823 else
3824 remap_edge_summaries (inlined_edge, e->callee, info,
3825 params_summary, callee_info,
3826 operand_map, offset_map, possible_truths,
3827 toplev_predicate);
3829 for (e = node->indirect_calls; e; e = next)
3831 class ipa_call_summary *es = ipa_call_summaries->get (e);
3832 predicate p;
3833 next = e->next_callee;
3835 remap_edge_change_prob (inlined_edge, e);
3836 if (es->predicate)
3838 p = es->predicate->remap_after_inlining
3839 (info, params_summary,
3840 callee_info, operand_map, offset_map,
3841 possible_truths, *toplev_predicate);
3842 edge_set_predicate (e, &p);
3844 else
3845 edge_set_predicate (e, toplev_predicate);
3849 /* Same as remap_predicate, but set result into hint *HINT. */
3851 static void
3852 remap_hint_predicate (class ipa_fn_summary *info,
3853 class ipa_node_params *params_summary,
3854 class ipa_fn_summary *callee_info,
3855 predicate **hint,
3856 vec<int> operand_map,
3857 vec<int> offset_map,
3858 clause_t possible_truths,
3859 predicate *toplev_predicate)
3861 predicate p;
3863 if (!*hint)
3864 return;
3865 p = (*hint)->remap_after_inlining
3866 (info, params_summary, callee_info,
3867 operand_map, offset_map,
3868 possible_truths, *toplev_predicate);
3869 if (p != false && p != true)
3871 if (!*hint)
3872 set_hint_predicate (hint, p);
3873 else
3874 **hint &= p;
3878 /* We inlined EDGE. Update summary of the function we inlined into. */
3880 void
3881 ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
3883 ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
3884 struct cgraph_node *to = (edge->caller->inlined_to
3885 ? edge->caller->inlined_to : edge->caller);
3886 class ipa_fn_summary *info = ipa_fn_summaries->get (to);
3887 clause_t clause = 0; /* not_inline is known to be false. */
3888 size_time_entry *e;
3889 auto_vec<int, 8> operand_map;
3890 auto_vec<int, 8> offset_map;
3891 int i;
3892 predicate toplev_predicate;
3893 class ipa_call_summary *es = ipa_call_summaries->get (edge);
3894 class ipa_node_params *params_summary = (ipa_node_params_sum
3895 ? IPA_NODE_REF (to) : NULL);
3897 if (es->predicate)
3898 toplev_predicate = *es->predicate;
3899 else
3900 toplev_predicate = true;
3902 info->fp_expressions |= callee_info->fp_expressions;
3904 if (callee_info->conds)
3906 auto_vec<tree, 32> known_vals;
3907 auto_vec<ipa_agg_value_set, 32> known_aggs;
3908 evaluate_properties_for_edge (edge, true, &clause, NULL,
3909 &known_vals, NULL, &known_aggs);
3911 if (ipa_node_params_sum && callee_info->conds)
3913 class ipa_edge_args *args = IPA_EDGE_REF (edge);
3914 int count = args ? ipa_get_cs_argument_count (args) : 0;
3915 int i;
3917 if (count)
3919 operand_map.safe_grow_cleared (count);
3920 offset_map.safe_grow_cleared (count);
3922 for (i = 0; i < count; i++)
3924 struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
3925 int map = -1;
3927 /* TODO: handle non-NOPs when merging. */
3928 if (jfunc->type == IPA_JF_PASS_THROUGH)
3930 if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
3931 map = ipa_get_jf_pass_through_formal_id (jfunc);
3932 if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
3933 offset_map[i] = -1;
3935 else if (jfunc->type == IPA_JF_ANCESTOR)
3937 HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
3938 if (offset >= 0 && offset < INT_MAX)
3940 map = ipa_get_jf_ancestor_formal_id (jfunc);
3941 if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
3942 offset = -1;
3943 offset_map[i] = offset;
3946 operand_map[i] = map;
3947 gcc_assert (map < ipa_get_param_count (params_summary));
3950 sreal freq = edge->sreal_frequency ();
3951 for (i = 0; vec_safe_iterate (callee_info->size_time_table, i, &e); i++)
3953 predicate p;
3954 p = e->exec_predicate.remap_after_inlining
3955 (info, params_summary,
3956 callee_info, operand_map,
3957 offset_map, clause,
3958 toplev_predicate);
3959 predicate nonconstp;
3960 nonconstp = e->nonconst_predicate.remap_after_inlining
3961 (info, params_summary,
3962 callee_info, operand_map,
3963 offset_map, clause,
3964 toplev_predicate);
3965 if (p != false && nonconstp != false)
3967 sreal add_time = ((sreal)e->time * freq);
3968 int prob = e->nonconst_predicate.probability (callee_info->conds,
3969 clause, es->param);
3970 if (prob != REG_BR_PROB_BASE)
3971 add_time = add_time * prob / REG_BR_PROB_BASE;
3972 if (prob != REG_BR_PROB_BASE
3973 && dump_file && (dump_flags & TDF_DETAILS))
3975 fprintf (dump_file, "\t\tScaling time by probability:%f\n",
3976 (double) prob / REG_BR_PROB_BASE);
3978 info->account_size_time (e->size, add_time, p, nonconstp);
3981 remap_edge_summaries (edge, edge->callee, info, params_summary,
3982 callee_info, operand_map,
3983 offset_map, clause, &toplev_predicate);
3984 remap_hint_predicate (info, params_summary, callee_info,
3985 &callee_info->loop_iterations,
3986 operand_map, offset_map, clause, &toplev_predicate);
3987 remap_hint_predicate (info, params_summary, callee_info,
3988 &callee_info->loop_stride,
3989 operand_map, offset_map, clause, &toplev_predicate);
3991 HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
3992 HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
3994 if (info->estimated_stack_size < peak)
3995 info->estimated_stack_size = peak;
3997 inline_update_callee_summaries (edge->callee, es->loop_depth);
3998 if (info->call_size_time_table)
4000 int edge_size = 0;
4001 sreal edge_time = 0;
4003 estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, vNULL,
4004 vNULL, vNULL, 0);
4005 /* Unaccount size and time of the optimized out call. */
4006 info->account_size_time (-edge_size, -edge_time,
4007 es->predicate ? *es->predicate : true,
4008 es->predicate ? *es->predicate : true,
4009 true);
4010 /* Account new calls. */
4011 summarize_calls_size_and_time (edge->callee, info);
4014 /* Free summaries that are not maintained for inline clones/edges. */
4015 ipa_call_summaries->remove (edge);
4016 ipa_fn_summaries->remove (edge->callee);
4017 ipa_remove_from_growth_caches (edge);
4020 /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
4021 overall size and time. Recompute it.
4022 If RESET is true also recompute call_time_size_table. */
4024 void
4025 ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
4027 class ipa_fn_summary *info = ipa_fn_summaries->get (node);
4028 class ipa_size_summary *size_info = ipa_size_summaries->get (node);
4029 size_time_entry *e;
4030 int i;
4032 size_info->size = 0;
4033 info->time = 0;
4034 for (i = 0; vec_safe_iterate (info->size_time_table, i, &e); i++)
4036 size_info->size += e->size;
4037 info->time += e->time;
4039 info->min_size = (*info->size_time_table)[0].size;
4040 if (reset)
4041 vec_free (info->call_size_time_table);
4042 if (node->callees || node->indirect_calls)
4043 estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
4044 &info->time, NULL,
4045 ~(clause_t) (1 << predicate::false_condition),
4046 vNULL, vNULL, vNULL);
4047 size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
4048 info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
4052 /* This function performs intraprocedural analysis in NODE that is required to
4053 inline indirect calls. */
4055 static void
4056 inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
4058 ipa_analyze_node (node);
4059 if (dump_file && (dump_flags & TDF_DETAILS))
4061 ipa_print_node_params (dump_file, node);
4062 ipa_print_node_jump_functions (dump_file, node);
4067 /* Note function body size. */
4069 void
4070 inline_analyze_function (struct cgraph_node *node)
4072 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4074 if (dump_file)
4075 fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
4076 if (opt_for_fn (node->decl, optimize) && !node->thunk.thunk_p)
4077 inline_indirect_intraprocedural_analysis (node);
4078 compute_fn_summary (node, false);
4079 if (!optimize)
4081 struct cgraph_edge *e;
4082 for (e = node->callees; e; e = e->next_callee)
4083 e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4084 for (e = node->indirect_calls; e; e = e->next_callee)
4085 e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4088 pop_cfun ();
4092 /* Called when new function is inserted to callgraph late. */
4094 void
4095 ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
4097 inline_analyze_function (node);
4100 /* Note function body size. */
4102 static void
4103 ipa_fn_summary_generate (void)
4105 struct cgraph_node *node;
4107 FOR_EACH_DEFINED_FUNCTION (node)
4108 if (DECL_STRUCT_FUNCTION (node->decl))
4109 node->versionable = tree_versionable_function_p (node->decl);
4111 ipa_fn_summary_alloc ();
4113 ipa_fn_summaries->enable_insertion_hook ();
4115 ipa_register_cgraph_hooks ();
4117 FOR_EACH_DEFINED_FUNCTION (node)
4118 if (!node->alias
4119 && (flag_generate_lto || flag_generate_offload|| flag_wpa
4120 || opt_for_fn (node->decl, optimize)))
4121 inline_analyze_function (node);
4125 /* Write inline summary for edge E to OB. */
4127 static void
4128 read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
4129 bool prevails)
4131 class ipa_call_summary *es = prevails
4132 ? ipa_call_summaries->get_create (e) : NULL;
4133 predicate p;
4134 int length, i;
4136 int size = streamer_read_uhwi (ib);
4137 int time = streamer_read_uhwi (ib);
4138 int depth = streamer_read_uhwi (ib);
4140 if (es)
4142 es->call_stmt_size = size;
4143 es->call_stmt_time = time;
4144 es->loop_depth = depth;
4147 bitpack_d bp = streamer_read_bitpack (ib);
4148 if (es)
4149 es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
4150 else
4151 bp_unpack_value (&bp, 1);
4153 p.stream_in (ib);
4154 if (es)
4155 edge_set_predicate (e, &p);
4156 length = streamer_read_uhwi (ib);
4157 if (length && es && e->possibly_call_in_translation_unit_p ())
4159 es->param.safe_grow_cleared (length);
4160 for (i = 0; i < length; i++)
4161 es->param[i].change_prob = streamer_read_uhwi (ib);
4163 else
4165 for (i = 0; i < length; i++)
4166 streamer_read_uhwi (ib);
4171 /* Stream in inline summaries from the section. */
4173 static void
4174 inline_read_section (struct lto_file_decl_data *file_data, const char *data,
4175 size_t len)
4177 const struct lto_function_header *header =
4178 (const struct lto_function_header *) data;
4179 const int cfg_offset = sizeof (struct lto_function_header);
4180 const int main_offset = cfg_offset + header->cfg_size;
4181 const int string_offset = main_offset + header->main_size;
4182 class data_in *data_in;
4183 unsigned int i, count2, j;
4184 unsigned int f_count;
4186 lto_input_block ib ((const char *) data + main_offset, header->main_size,
4187 file_data->mode_table);
4189 data_in =
4190 lto_data_in_create (file_data, (const char *) data + string_offset,
4191 header->string_size, vNULL);
4192 f_count = streamer_read_uhwi (&ib);
4193 for (i = 0; i < f_count; i++)
4195 unsigned int index;
4196 struct cgraph_node *node;
4197 class ipa_fn_summary *info;
4198 class ipa_node_params *params_summary;
4199 class ipa_size_summary *size_info;
4200 lto_symtab_encoder_t encoder;
4201 struct bitpack_d bp;
4202 struct cgraph_edge *e;
4203 predicate p;
4205 index = streamer_read_uhwi (&ib);
4206 encoder = file_data->symtab_node_encoder;
4207 node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
4208 index));
4209 info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
4210 params_summary = node->prevailing_p () ? IPA_NODE_REF (node) : NULL;
4211 size_info = node->prevailing_p ()
4212 ? ipa_size_summaries->get_create (node) : NULL;
4214 int stack_size = streamer_read_uhwi (&ib);
4215 int size = streamer_read_uhwi (&ib);
4216 sreal time = sreal::stream_in (&ib);
4218 if (info)
4220 info->estimated_stack_size
4221 = size_info->estimated_self_stack_size = stack_size;
4222 size_info->size = size_info->self_size = size;
4223 info->time = time;
4226 bp = streamer_read_bitpack (&ib);
4227 if (info)
4229 info->inlinable = bp_unpack_value (&bp, 1);
4230 info->fp_expressions = bp_unpack_value (&bp, 1);
4232 else
4234 bp_unpack_value (&bp, 1);
4235 bp_unpack_value (&bp, 1);
4238 count2 = streamer_read_uhwi (&ib);
4239 gcc_assert (!info || !info->conds);
4240 if (info)
4241 vec_safe_reserve_exact (info->conds, count2);
4242 for (j = 0; j < count2; j++)
4244 struct condition c;
4245 unsigned int k, count3;
4246 c.operand_num = streamer_read_uhwi (&ib);
4247 c.code = (enum tree_code) streamer_read_uhwi (&ib);
4248 c.type = stream_read_tree (&ib, data_in);
4249 c.val = stream_read_tree (&ib, data_in);
4250 bp = streamer_read_bitpack (&ib);
4251 c.agg_contents = bp_unpack_value (&bp, 1);
4252 c.by_ref = bp_unpack_value (&bp, 1);
4253 if (c.agg_contents)
4254 c.offset = streamer_read_uhwi (&ib);
4255 count3 = streamer_read_uhwi (&ib);
4256 c.param_ops = NULL;
4257 if (info)
4258 vec_safe_reserve_exact (c.param_ops, count3);
4259 if (params_summary)
4260 ipa_set_param_used_by_ipa_predicates
4261 (params_summary, c.operand_num, true);
4262 for (k = 0; k < count3; k++)
4264 struct expr_eval_op op;
4265 enum gimple_rhs_class rhs_class;
4266 op.code = (enum tree_code) streamer_read_uhwi (&ib);
4267 op.type = stream_read_tree (&ib, data_in);
4268 switch (rhs_class = get_gimple_rhs_class (op.code))
4270 case GIMPLE_UNARY_RHS:
4271 op.index = 0;
4272 op.val[0] = NULL_TREE;
4273 op.val[1] = NULL_TREE;
4274 break;
4276 case GIMPLE_BINARY_RHS:
4277 case GIMPLE_TERNARY_RHS:
4278 bp = streamer_read_bitpack (&ib);
4279 op.index = bp_unpack_value (&bp, 2);
4280 op.val[0] = stream_read_tree (&ib, data_in);
4281 if (rhs_class == GIMPLE_BINARY_RHS)
4282 op.val[1] = NULL_TREE;
4283 else
4284 op.val[1] = stream_read_tree (&ib, data_in);
4285 break;
4287 default:
4288 fatal_error (UNKNOWN_LOCATION,
4289 "invalid fnsummary in LTO stream");
4291 if (info)
4292 c.param_ops->quick_push (op);
4294 if (info)
4295 info->conds->quick_push (c);
4297 count2 = streamer_read_uhwi (&ib);
4298 gcc_assert (!info || !info->size_time_table);
4299 if (info && count2)
4300 vec_safe_reserve_exact (info->size_time_table, count2);
4301 for (j = 0; j < count2; j++)
4303 class size_time_entry e;
4305 e.size = streamer_read_uhwi (&ib);
4306 e.time = sreal::stream_in (&ib);
4307 e.exec_predicate.stream_in (&ib);
4308 e.nonconst_predicate.stream_in (&ib);
4310 if (info)
4311 info->size_time_table->quick_push (e);
4314 p.stream_in (&ib);
4315 if (info)
4316 set_hint_predicate (&info->loop_iterations, p);
4317 p.stream_in (&ib);
4318 if (info)
4319 set_hint_predicate (&info->loop_stride, p);
4320 for (e = node->callees; e; e = e->next_callee)
4321 read_ipa_call_summary (&ib, e, info != NULL);
4322 for (e = node->indirect_calls; e; e = e->next_callee)
4323 read_ipa_call_summary (&ib, e, info != NULL);
4326 lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
4327 len);
4328 lto_data_in_delete (data_in);
4332 /* Read inline summary. Jump functions are shared among ipa-cp
4333 and inliner, so when ipa-cp is active, we don't need to write them
4334 twice. */
4336 static void
4337 ipa_fn_summary_read (void)
4339 struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
4340 struct lto_file_decl_data *file_data;
4341 unsigned int j = 0;
4343 ipa_fn_summary_alloc ();
4345 while ((file_data = file_data_vec[j++]))
4347 size_t len;
4348 const char *data
4349 = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
4350 &len);
4351 if (data)
4352 inline_read_section (file_data, data, len);
4353 else
4354 /* Fatal error here. We do not want to support compiling ltrans units
4355 with different version of compiler or different flags than the WPA
4356 unit, so this should never happen. */
4357 fatal_error (input_location,
4358 "ipa inline summary is missing in input file");
4360 ipa_register_cgraph_hooks ();
4361 if (!flag_ipa_cp)
4362 ipa_prop_read_jump_functions ();
4364 gcc_assert (ipa_fn_summaries);
4365 ipa_fn_summaries->enable_insertion_hook ();
4369 /* Write inline summary for edge E to OB. */
4371 static void
4372 write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
4374 class ipa_call_summary *es = ipa_call_summaries->get (e);
4375 int i;
4377 streamer_write_uhwi (ob, es->call_stmt_size);
4378 streamer_write_uhwi (ob, es->call_stmt_time);
4379 streamer_write_uhwi (ob, es->loop_depth);
4381 bitpack_d bp = bitpack_create (ob->main_stream);
4382 bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
4383 streamer_write_bitpack (&bp);
4385 if (es->predicate)
4386 es->predicate->stream_out (ob);
4387 else
4388 streamer_write_uhwi (ob, 0);
4389 streamer_write_uhwi (ob, es->param.length ());
4390 for (i = 0; i < (int) es->param.length (); i++)
4391 streamer_write_uhwi (ob, es->param[i].change_prob);
4395 /* Write inline summary for node in SET.
4396 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4397 active, we don't need to write them twice. */
4399 static void
4400 ipa_fn_summary_write (void)
4402 struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
4403 lto_symtab_encoder_iterator lsei;
4404 lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
4405 unsigned int count = 0;
4407 for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
4408 lsei_next_function_in_partition (&lsei))
4410 cgraph_node *cnode = lsei_cgraph_node (lsei);
4411 if (cnode->definition && !cnode->alias)
4412 count++;
4414 streamer_write_uhwi (ob, count);
4416 for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
4417 lsei_next_function_in_partition (&lsei))
4419 cgraph_node *cnode = lsei_cgraph_node (lsei);
4420 if (cnode->definition && !cnode->alias)
4422 class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
4423 class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
4424 struct bitpack_d bp;
4425 struct cgraph_edge *edge;
4426 int i;
4427 size_time_entry *e;
4428 struct condition *c;
4430 streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
4431 streamer_write_hwi (ob, size_info->estimated_self_stack_size);
4432 streamer_write_hwi (ob, size_info->self_size);
4433 info->time.stream_out (ob);
4434 bp = bitpack_create (ob->main_stream);
4435 bp_pack_value (&bp, info->inlinable, 1);
4436 bp_pack_value (&bp, false, 1);
4437 bp_pack_value (&bp, info->fp_expressions, 1);
4438 streamer_write_bitpack (&bp);
4439 streamer_write_uhwi (ob, vec_safe_length (info->conds));
4440 for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
4442 int j;
4443 struct expr_eval_op *op;
4445 streamer_write_uhwi (ob, c->operand_num);
4446 streamer_write_uhwi (ob, c->code);
4447 stream_write_tree (ob, c->type, true);
4448 stream_write_tree (ob, c->val, true);
4449 bp = bitpack_create (ob->main_stream);
4450 bp_pack_value (&bp, c->agg_contents, 1);
4451 bp_pack_value (&bp, c->by_ref, 1);
4452 streamer_write_bitpack (&bp);
4453 if (c->agg_contents)
4454 streamer_write_uhwi (ob, c->offset);
4455 streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
4456 for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
4458 streamer_write_uhwi (ob, op->code);
4459 stream_write_tree (ob, op->type, true);
4460 if (op->val[0])
4462 bp = bitpack_create (ob->main_stream);
4463 bp_pack_value (&bp, op->index, 2);
4464 streamer_write_bitpack (&bp);
4465 stream_write_tree (ob, op->val[0], true);
4466 if (op->val[1])
4467 stream_write_tree (ob, op->val[1], true);
4471 streamer_write_uhwi (ob, vec_safe_length (info->size_time_table));
4472 for (i = 0; vec_safe_iterate (info->size_time_table, i, &e); i++)
4474 streamer_write_uhwi (ob, e->size);
4475 e->time.stream_out (ob);
4476 e->exec_predicate.stream_out (ob);
4477 e->nonconst_predicate.stream_out (ob);
4479 if (info->loop_iterations)
4480 info->loop_iterations->stream_out (ob);
4481 else
4482 streamer_write_uhwi (ob, 0);
4483 if (info->loop_stride)
4484 info->loop_stride->stream_out (ob);
4485 else
4486 streamer_write_uhwi (ob, 0);
4487 for (edge = cnode->callees; edge; edge = edge->next_callee)
4488 write_ipa_call_summary (ob, edge);
4489 for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
4490 write_ipa_call_summary (ob, edge);
4493 streamer_write_char_stream (ob->main_stream, 0);
4494 produce_asm (ob, NULL);
4495 destroy_output_block (ob);
4497 if (!flag_ipa_cp)
4498 ipa_prop_write_jump_functions ();
4502 /* Release function summary. */
4504 void
4505 ipa_free_fn_summary (void)
4507 if (!ipa_call_summaries)
4508 return;
4509 ggc_delete (ipa_fn_summaries);
4510 ipa_fn_summaries = NULL;
4511 delete ipa_call_summaries;
4512 ipa_call_summaries = NULL;
4513 edge_predicate_pool.release ();
4514 /* During IPA this is one of largest datastructures to release. */
4515 if (flag_wpa)
4516 ggc_trim ();
4519 /* Release function summary. */
4521 void
4522 ipa_free_size_summary (void)
4524 if (!ipa_size_summaries)
4525 return;
4526 delete ipa_size_summaries;
4527 ipa_size_summaries = NULL;
4530 namespace {
4532 const pass_data pass_data_local_fn_summary =
4534 GIMPLE_PASS, /* type */
4535 "local-fnsummary", /* name */
4536 OPTGROUP_INLINE, /* optinfo_flags */
4537 TV_INLINE_PARAMETERS, /* tv_id */
4538 0, /* properties_required */
4539 0, /* properties_provided */
4540 0, /* properties_destroyed */
4541 0, /* todo_flags_start */
4542 0, /* todo_flags_finish */
4545 class pass_local_fn_summary : public gimple_opt_pass
4547 public:
4548 pass_local_fn_summary (gcc::context *ctxt)
4549 : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
4552 /* opt_pass methods: */
4553 opt_pass * clone () { return new pass_local_fn_summary (m_ctxt); }
4554 virtual unsigned int execute (function *)
4556 return compute_fn_summary_for_current ();
4559 }; // class pass_local_fn_summary
4561 } // anon namespace
4563 gimple_opt_pass *
4564 make_pass_local_fn_summary (gcc::context *ctxt)
4566 return new pass_local_fn_summary (ctxt);
4570 /* Free inline summary. */
4572 namespace {
4574 const pass_data pass_data_ipa_free_fn_summary =
4576 SIMPLE_IPA_PASS, /* type */
4577 "free-fnsummary", /* name */
4578 OPTGROUP_NONE, /* optinfo_flags */
4579 TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
4580 0, /* properties_required */
4581 0, /* properties_provided */
4582 0, /* properties_destroyed */
4583 0, /* todo_flags_start */
4584 0, /* todo_flags_finish */
4587 class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
4589 public:
4590 pass_ipa_free_fn_summary (gcc::context *ctxt)
4591 : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
4592 small_p (false)
4595 /* opt_pass methods: */
4596 opt_pass *clone () { return new pass_ipa_free_fn_summary (m_ctxt); }
4597 void set_pass_param (unsigned int n, bool param)
4599 gcc_assert (n == 0);
4600 small_p = param;
4602 virtual bool gate (function *) { return true; }
4603 virtual unsigned int execute (function *)
4605 ipa_free_fn_summary ();
4606 if (!flag_wpa)
4607 ipa_free_size_summary ();
4608 return 0;
4611 private:
4612 bool small_p;
4613 }; // class pass_ipa_free_fn_summary
4615 } // anon namespace
4617 simple_ipa_opt_pass *
4618 make_pass_ipa_free_fn_summary (gcc::context *ctxt)
4620 return new pass_ipa_free_fn_summary (ctxt);
4623 namespace {
4625 const pass_data pass_data_ipa_fn_summary =
4627 IPA_PASS, /* type */
4628 "fnsummary", /* name */
4629 OPTGROUP_INLINE, /* optinfo_flags */
4630 TV_IPA_FNSUMMARY, /* tv_id */
4631 0, /* properties_required */
4632 0, /* properties_provided */
4633 0, /* properties_destroyed */
4634 0, /* todo_flags_start */
4635 ( TODO_dump_symtab ), /* todo_flags_finish */
4638 class pass_ipa_fn_summary : public ipa_opt_pass_d
4640 public:
4641 pass_ipa_fn_summary (gcc::context *ctxt)
4642 : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
4643 ipa_fn_summary_generate, /* generate_summary */
4644 ipa_fn_summary_write, /* write_summary */
4645 ipa_fn_summary_read, /* read_summary */
4646 NULL, /* write_optimization_summary */
4647 NULL, /* read_optimization_summary */
4648 NULL, /* stmt_fixup */
4649 0, /* function_transform_todo_flags_start */
4650 NULL, /* function_transform */
4651 NULL) /* variable_transform */
4654 /* opt_pass methods: */
4655 virtual unsigned int execute (function *) { return 0; }
4657 }; // class pass_ipa_fn_summary
4659 } // anon namespace
4661 ipa_opt_pass_d *
4662 make_pass_ipa_fn_summary (gcc::context *ctxt)
4664 return new pass_ipa_fn_summary (ctxt);
4667 /* Reset all state within ipa-fnsummary.c so that we can rerun the compiler
4668 within the same process. For use by toplev::finalize. */
4670 void
4671 ipa_fnsummary_c_finalize (void)
4673 ipa_free_fn_summary ();