* omp-low.c (MASK_GANG, MASK_WORKER, MASK_VECTOR): Delete.
[official-gcc.git] / gcc / value-prof.c
blobe371c24195e442db826ad6d9d97258f96926bb2f
1 /* Transformations based on profile information for values.
2 Copyright (C) 2003-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "cfghooks.h"
25 #include "tree.h"
26 #include "gimple.h"
27 #include "rtl.h"
28 #include "ssa.h"
29 #include "alias.h"
30 #include "fold-const.h"
31 #include "tree-nested.h"
32 #include "calls.h"
33 #include "flags.h"
34 #include "insn-config.h"
35 #include "expmed.h"
36 #include "dojump.h"
37 #include "explow.h"
38 #include "emit-rtl.h"
39 #include "varasm.h"
40 #include "stmt.h"
41 #include "expr.h"
42 #include "value-prof.h"
43 #include "recog.h"
44 #include "insn-codes.h"
45 #include "optabs.h"
46 #include "regs.h"
47 #include "internal-fn.h"
48 #include "tree-eh.h"
49 #include "gimplify.h"
50 #include "gimple-iterator.h"
51 #include "tree-cfg.h"
52 #include "diagnostic.h"
53 #include "gimple-pretty-print.h"
54 #include "coverage.h"
55 #include "gcov-io.h"
56 #include "timevar.h"
57 #include "dumpfile.h"
58 #include "profile.h"
59 #include "cgraph.h"
60 #include "data-streamer.h"
61 #include "builtins.h"
62 #include "params.h"
63 #include "tree-chkp.h"
65 /* In this file value profile based optimizations are placed. Currently the
66 following optimizations are implemented (for more detailed descriptions
67 see comments at value_profile_transformations):
69 1) Division/modulo specialization. Provided that we can determine that the
70 operands of the division have some special properties, we may use it to
71 produce more effective code.
73 2) Indirect/virtual call specialization. If we can determine most
74 common function callee in indirect/virtual call. We can use this
75 information to improve code effectiveness (especially info for
76 the inliner).
78 3) Speculative prefetching. If we are able to determine that the difference
79 between addresses accessed by a memory reference is usually constant, we
80 may add the prefetch instructions.
81 FIXME: This transformation was removed together with RTL based value
82 profiling.
85 Value profiling internals
86 ==========================
88 Every value profiling transformation starts with defining what values
89 to profile. There are different histogram types (see HIST_TYPE_* in
90 value-prof.h) and each transformation can request one or more histogram
91 types per GIMPLE statement. The function gimple_find_values_to_profile()
92 collects the values to profile in a vec, and adds the number of counters
93 required for the different histogram types.
95 For a -fprofile-generate run, the statements for which values should be
96 recorded, are instrumented in instrument_values(). The instrumentation
97 is done by helper functions that can be found in tree-profile.c, where
98 new types of histograms can be added if necessary.
100 After a -fprofile-use, the value profiling data is read back in by
101 compute_value_histograms() that translates the collected data to
102 histograms and attaches them to the profiled statements via
103 gimple_add_histogram_value(). Histograms are stored in a hash table
104 that is attached to every intrumented function, see VALUE_HISTOGRAMS
105 in function.h.
107 The value-profile transformations driver is the function
108 gimple_value_profile_transformations(). It traverses all statements in
109 the to-be-transformed function, and looks for statements with one or
110 more histograms attached to it. If a statement has histograms, the
111 transformation functions are called on the statement.
113 Limitations / FIXME / TODO:
114 * Only one histogram of each type can be associated with a statement.
115 * Currently, HIST_TYPE_CONST_DELTA is not implemented.
116 (This type of histogram was originally used to implement a form of
117 stride profiling based speculative prefetching to improve SPEC2000
118 scores for memory-bound benchmarks, mcf and equake. However, this
119 was an RTL value-profiling transformation, and those have all been
120 removed.)
121 * Some value profile transformations are done in builtins.c (?!)
122 * Updating of histograms needs some TLC.
123 * The value profiling code could be used to record analysis results
124 from non-profiling (e.g. VRP).
125 * Adding new profilers should be simplified, starting with a cleanup
126 of what-happens-where andwith making gimple_find_values_to_profile
127 and gimple_value_profile_transformations table-driven, perhaps...
130 static tree gimple_divmod_fixed_value (gassign *, tree, int, gcov_type,
131 gcov_type);
132 static tree gimple_mod_pow2 (gassign *, int, gcov_type, gcov_type);
133 static tree gimple_mod_subtract (gassign *, int, int, int, gcov_type,
134 gcov_type, gcov_type);
135 static bool gimple_divmod_fixed_value_transform (gimple_stmt_iterator *);
136 static bool gimple_mod_pow2_value_transform (gimple_stmt_iterator *);
137 static bool gimple_mod_subtract_transform (gimple_stmt_iterator *);
138 static bool gimple_stringops_transform (gimple_stmt_iterator *);
139 static bool gimple_ic_transform (gimple_stmt_iterator *);
141 /* Allocate histogram value. */
143 histogram_value
144 gimple_alloc_histogram_value (struct function *fun ATTRIBUTE_UNUSED,
145 enum hist_type type, gimple *stmt, tree value)
147 histogram_value hist = (histogram_value) xcalloc (1, sizeof (*hist));
148 hist->hvalue.value = value;
149 hist->hvalue.stmt = stmt;
150 hist->type = type;
151 return hist;
154 /* Hash value for histogram. */
156 static hashval_t
157 histogram_hash (const void *x)
159 return htab_hash_pointer (((const_histogram_value)x)->hvalue.stmt);
162 /* Return nonzero if statement for histogram_value X is Y. */
164 static int
165 histogram_eq (const void *x, const void *y)
167 return ((const_histogram_value) x)->hvalue.stmt == (const gimple *) y;
170 /* Set histogram for STMT. */
172 static void
173 set_histogram_value (struct function *fun, gimple *stmt, histogram_value hist)
175 void **loc;
176 if (!hist && !VALUE_HISTOGRAMS (fun))
177 return;
178 if (!VALUE_HISTOGRAMS (fun))
179 VALUE_HISTOGRAMS (fun) = htab_create (1, histogram_hash,
180 histogram_eq, NULL);
181 loc = htab_find_slot_with_hash (VALUE_HISTOGRAMS (fun), stmt,
182 htab_hash_pointer (stmt),
183 hist ? INSERT : NO_INSERT);
184 if (!hist)
186 if (loc)
187 htab_clear_slot (VALUE_HISTOGRAMS (fun), loc);
188 return;
190 *loc = hist;
193 /* Get histogram list for STMT. */
195 histogram_value
196 gimple_histogram_value (struct function *fun, gimple *stmt)
198 if (!VALUE_HISTOGRAMS (fun))
199 return NULL;
200 return (histogram_value) htab_find_with_hash (VALUE_HISTOGRAMS (fun), stmt,
201 htab_hash_pointer (stmt));
204 /* Add histogram for STMT. */
206 void
207 gimple_add_histogram_value (struct function *fun, gimple *stmt,
208 histogram_value hist)
210 hist->hvalue.next = gimple_histogram_value (fun, stmt);
211 set_histogram_value (fun, stmt, hist);
212 hist->fun = fun;
215 /* Remove histogram HIST from STMT's histogram list. */
217 void
218 gimple_remove_histogram_value (struct function *fun, gimple *stmt,
219 histogram_value hist)
221 histogram_value hist2 = gimple_histogram_value (fun, stmt);
222 if (hist == hist2)
224 set_histogram_value (fun, stmt, hist->hvalue.next);
226 else
228 while (hist2->hvalue.next != hist)
229 hist2 = hist2->hvalue.next;
230 hist2->hvalue.next = hist->hvalue.next;
232 free (hist->hvalue.counters);
233 if (flag_checking)
234 memset (hist, 0xab, sizeof (*hist));
235 free (hist);
238 /* Lookup histogram of type TYPE in the STMT. */
240 histogram_value
241 gimple_histogram_value_of_type (struct function *fun, gimple *stmt,
242 enum hist_type type)
244 histogram_value hist;
245 for (hist = gimple_histogram_value (fun, stmt); hist;
246 hist = hist->hvalue.next)
247 if (hist->type == type)
248 return hist;
249 return NULL;
252 /* Dump information about HIST to DUMP_FILE. */
254 static void
255 dump_histogram_value (FILE *dump_file, histogram_value hist)
257 switch (hist->type)
259 case HIST_TYPE_INTERVAL:
260 fprintf (dump_file, "Interval counter range %d -- %d",
261 hist->hdata.intvl.int_start,
262 (hist->hdata.intvl.int_start
263 + hist->hdata.intvl.steps - 1));
264 if (hist->hvalue.counters)
266 unsigned int i;
267 fprintf (dump_file, " [");
268 for (i = 0; i < hist->hdata.intvl.steps; i++)
269 fprintf (dump_file, " %d:%" PRId64,
270 hist->hdata.intvl.int_start + i,
271 (int64_t) hist->hvalue.counters[i]);
272 fprintf (dump_file, " ] outside range:%" PRId64,
273 (int64_t) hist->hvalue.counters[i]);
275 fprintf (dump_file, ".\n");
276 break;
278 case HIST_TYPE_POW2:
279 fprintf (dump_file, "Pow2 counter ");
280 if (hist->hvalue.counters)
282 fprintf (dump_file, "pow2:%" PRId64
283 " nonpow2:%" PRId64,
284 (int64_t) hist->hvalue.counters[0],
285 (int64_t) hist->hvalue.counters[1]);
287 fprintf (dump_file, ".\n");
288 break;
290 case HIST_TYPE_SINGLE_VALUE:
291 fprintf (dump_file, "Single value ");
292 if (hist->hvalue.counters)
294 fprintf (dump_file, "value:%" PRId64
295 " match:%" PRId64
296 " wrong:%" PRId64,
297 (int64_t) hist->hvalue.counters[0],
298 (int64_t) hist->hvalue.counters[1],
299 (int64_t) hist->hvalue.counters[2]);
301 fprintf (dump_file, ".\n");
302 break;
304 case HIST_TYPE_AVERAGE:
305 fprintf (dump_file, "Average value ");
306 if (hist->hvalue.counters)
308 fprintf (dump_file, "sum:%" PRId64
309 " times:%" PRId64,
310 (int64_t) hist->hvalue.counters[0],
311 (int64_t) hist->hvalue.counters[1]);
313 fprintf (dump_file, ".\n");
314 break;
316 case HIST_TYPE_IOR:
317 fprintf (dump_file, "IOR value ");
318 if (hist->hvalue.counters)
320 fprintf (dump_file, "ior:%" PRId64,
321 (int64_t) hist->hvalue.counters[0]);
323 fprintf (dump_file, ".\n");
324 break;
326 case HIST_TYPE_CONST_DELTA:
327 fprintf (dump_file, "Constant delta ");
328 if (hist->hvalue.counters)
330 fprintf (dump_file, "value:%" PRId64
331 " match:%" PRId64
332 " wrong:%" PRId64,
333 (int64_t) hist->hvalue.counters[0],
334 (int64_t) hist->hvalue.counters[1],
335 (int64_t) hist->hvalue.counters[2]);
337 fprintf (dump_file, ".\n");
338 break;
339 case HIST_TYPE_INDIR_CALL:
340 fprintf (dump_file, "Indirect call ");
341 if (hist->hvalue.counters)
343 fprintf (dump_file, "value:%" PRId64
344 " match:%" PRId64
345 " all:%" PRId64,
346 (int64_t) hist->hvalue.counters[0],
347 (int64_t) hist->hvalue.counters[1],
348 (int64_t) hist->hvalue.counters[2]);
350 fprintf (dump_file, ".\n");
351 break;
352 case HIST_TYPE_TIME_PROFILE:
353 fprintf (dump_file, "Time profile ");
354 if (hist->hvalue.counters)
356 fprintf (dump_file, "time:%" PRId64,
357 (int64_t) hist->hvalue.counters[0]);
359 fprintf (dump_file, ".\n");
360 break;
361 case HIST_TYPE_INDIR_CALL_TOPN:
362 fprintf (dump_file, "Indirect call topn ");
363 if (hist->hvalue.counters)
365 int i;
367 fprintf (dump_file, "accu:%" PRId64, hist->hvalue.counters[0]);
368 for (i = 1; i < (GCOV_ICALL_TOPN_VAL << 2); i += 2)
370 fprintf (dump_file, " target:%" PRId64 " value:%" PRId64,
371 (int64_t) hist->hvalue.counters[i],
372 (int64_t) hist->hvalue.counters[i+1]);
375 fprintf (dump_file, ".\n");
376 break;
377 case HIST_TYPE_MAX:
378 gcc_unreachable ();
382 /* Dump information about HIST to DUMP_FILE. */
384 void
385 stream_out_histogram_value (struct output_block *ob, histogram_value hist)
387 struct bitpack_d bp;
388 unsigned int i;
390 bp = bitpack_create (ob->main_stream);
391 bp_pack_enum (&bp, hist_type, HIST_TYPE_MAX, hist->type);
392 bp_pack_value (&bp, hist->hvalue.next != NULL, 1);
393 streamer_write_bitpack (&bp);
394 switch (hist->type)
396 case HIST_TYPE_INTERVAL:
397 streamer_write_hwi (ob, hist->hdata.intvl.int_start);
398 streamer_write_uhwi (ob, hist->hdata.intvl.steps);
399 break;
400 default:
401 break;
403 for (i = 0; i < hist->n_counters; i++)
404 streamer_write_gcov_count (ob, hist->hvalue.counters[i]);
405 if (hist->hvalue.next)
406 stream_out_histogram_value (ob, hist->hvalue.next);
409 /* Dump information about HIST to DUMP_FILE. */
411 void
412 stream_in_histogram_value (struct lto_input_block *ib, gimple *stmt)
414 enum hist_type type;
415 unsigned int ncounters = 0;
416 struct bitpack_d bp;
417 unsigned int i;
418 histogram_value new_val;
419 bool next;
420 histogram_value *next_p = NULL;
424 bp = streamer_read_bitpack (ib);
425 type = bp_unpack_enum (&bp, hist_type, HIST_TYPE_MAX);
426 next = bp_unpack_value (&bp, 1);
427 new_val = gimple_alloc_histogram_value (cfun, type, stmt, NULL);
428 switch (type)
430 case HIST_TYPE_INTERVAL:
431 new_val->hdata.intvl.int_start = streamer_read_hwi (ib);
432 new_val->hdata.intvl.steps = streamer_read_uhwi (ib);
433 ncounters = new_val->hdata.intvl.steps + 2;
434 break;
436 case HIST_TYPE_POW2:
437 case HIST_TYPE_AVERAGE:
438 ncounters = 2;
439 break;
441 case HIST_TYPE_SINGLE_VALUE:
442 case HIST_TYPE_INDIR_CALL:
443 ncounters = 3;
444 break;
446 case HIST_TYPE_CONST_DELTA:
447 ncounters = 4;
448 break;
450 case HIST_TYPE_IOR:
451 case HIST_TYPE_TIME_PROFILE:
452 ncounters = 1;
453 break;
455 case HIST_TYPE_INDIR_CALL_TOPN:
456 ncounters = (GCOV_ICALL_TOPN_VAL << 2) + 1;
457 break;
459 case HIST_TYPE_MAX:
460 gcc_unreachable ();
462 new_val->hvalue.counters = XNEWVAR (gcov_type, sizeof (*new_val->hvalue.counters) * ncounters);
463 new_val->n_counters = ncounters;
464 for (i = 0; i < ncounters; i++)
465 new_val->hvalue.counters[i] = streamer_read_gcov_count (ib);
466 if (!next_p)
467 gimple_add_histogram_value (cfun, stmt, new_val);
468 else
469 *next_p = new_val;
470 next_p = &new_val->hvalue.next;
472 while (next);
475 /* Dump all histograms attached to STMT to DUMP_FILE. */
477 void
478 dump_histograms_for_stmt (struct function *fun, FILE *dump_file, gimple *stmt)
480 histogram_value hist;
481 for (hist = gimple_histogram_value (fun, stmt); hist; hist = hist->hvalue.next)
482 dump_histogram_value (dump_file, hist);
485 /* Remove all histograms associated with STMT. */
487 void
488 gimple_remove_stmt_histograms (struct function *fun, gimple *stmt)
490 histogram_value val;
491 while ((val = gimple_histogram_value (fun, stmt)) != NULL)
492 gimple_remove_histogram_value (fun, stmt, val);
495 /* Duplicate all histograms associates with OSTMT to STMT. */
497 void
498 gimple_duplicate_stmt_histograms (struct function *fun, gimple *stmt,
499 struct function *ofun, gimple *ostmt)
501 histogram_value val;
502 for (val = gimple_histogram_value (ofun, ostmt); val != NULL; val = val->hvalue.next)
504 histogram_value new_val = gimple_alloc_histogram_value (fun, val->type, NULL, NULL);
505 memcpy (new_val, val, sizeof (*val));
506 new_val->hvalue.stmt = stmt;
507 new_val->hvalue.counters = XNEWVAR (gcov_type, sizeof (*new_val->hvalue.counters) * new_val->n_counters);
508 memcpy (new_val->hvalue.counters, val->hvalue.counters, sizeof (*new_val->hvalue.counters) * new_val->n_counters);
509 gimple_add_histogram_value (fun, stmt, new_val);
513 /* Move all histograms associated with OSTMT to STMT. */
515 void
516 gimple_move_stmt_histograms (struct function *fun, gimple *stmt, gimple *ostmt)
518 histogram_value val = gimple_histogram_value (fun, ostmt);
519 if (val)
521 /* The following three statements can't be reordered,
522 because histogram hashtab relies on stmt field value
523 for finding the exact slot. */
524 set_histogram_value (fun, ostmt, NULL);
525 for (; val != NULL; val = val->hvalue.next)
526 val->hvalue.stmt = stmt;
527 set_histogram_value (fun, stmt, val);
531 static bool error_found = false;
533 /* Helper function for verify_histograms. For each histogram reachable via htab
534 walk verify that it was reached via statement walk. */
536 static int
537 visit_hist (void **slot, void *data)
539 hash_set<histogram_value> *visited = (hash_set<histogram_value> *) data;
540 histogram_value hist = *(histogram_value *) slot;
542 if (!visited->contains (hist)
543 && hist->type != HIST_TYPE_TIME_PROFILE)
545 error ("dead histogram");
546 dump_histogram_value (stderr, hist);
547 debug_gimple_stmt (hist->hvalue.stmt);
548 error_found = true;
550 return 1;
553 /* Verify sanity of the histograms. */
555 DEBUG_FUNCTION void
556 verify_histograms (void)
558 basic_block bb;
559 gimple_stmt_iterator gsi;
560 histogram_value hist;
562 error_found = false;
563 hash_set<histogram_value> visited_hists;
564 FOR_EACH_BB_FN (bb, cfun)
565 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
567 gimple *stmt = gsi_stmt (gsi);
569 for (hist = gimple_histogram_value (cfun, stmt); hist;
570 hist = hist->hvalue.next)
572 if (hist->hvalue.stmt != stmt)
574 error ("Histogram value statement does not correspond to "
575 "the statement it is associated with");
576 debug_gimple_stmt (stmt);
577 dump_histogram_value (stderr, hist);
578 error_found = true;
580 visited_hists.add (hist);
583 if (VALUE_HISTOGRAMS (cfun))
584 htab_traverse (VALUE_HISTOGRAMS (cfun), visit_hist, &visited_hists);
585 if (error_found)
586 internal_error ("verify_histograms failed");
589 /* Helper function for verify_histograms. For each histogram reachable via htab
590 walk verify that it was reached via statement walk. */
592 static int
593 free_hist (void **slot, void *data ATTRIBUTE_UNUSED)
595 histogram_value hist = *(histogram_value *) slot;
596 free (hist->hvalue.counters);
597 if (flag_checking)
598 memset (hist, 0xab, sizeof (*hist));
599 free (hist);
600 return 1;
603 void
604 free_histograms (struct function *fn)
606 if (VALUE_HISTOGRAMS (fn))
608 htab_traverse (VALUE_HISTOGRAMS (fn), free_hist, NULL);
609 htab_delete (VALUE_HISTOGRAMS (fn));
610 VALUE_HISTOGRAMS (fn) = NULL;
614 /* The overall number of invocations of the counter should match
615 execution count of basic block. Report it as error rather than
616 internal error as it might mean that user has misused the profile
617 somehow. */
619 static bool
620 check_counter (gimple *stmt, const char * name,
621 gcov_type *count, gcov_type *all, gcov_type bb_count)
623 if (*all != bb_count || *count > *all)
625 location_t locus;
626 locus = (stmt != NULL)
627 ? gimple_location (stmt)
628 : DECL_SOURCE_LOCATION (current_function_decl);
629 if (flag_profile_correction)
631 if (dump_enabled_p ())
632 dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus,
633 "correcting inconsistent value profile: %s "
634 "profiler overall count (%d) does not match BB "
635 "count (%d)\n", name, (int)*all, (int)bb_count);
636 *all = bb_count;
637 if (*count > *all)
638 *count = *all;
639 return false;
641 else
643 error_at (locus, "corrupted value profile: %s "
644 "profile counter (%d out of %d) inconsistent with "
645 "basic-block count (%d)",
646 name,
647 (int) *count,
648 (int) *all,
649 (int) bb_count);
650 return true;
654 return false;
657 /* GIMPLE based transformations. */
659 bool
660 gimple_value_profile_transformations (void)
662 basic_block bb;
663 gimple_stmt_iterator gsi;
664 bool changed = false;
665 FOR_EACH_BB_FN (bb, cfun)
667 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
669 gimple *stmt = gsi_stmt (gsi);
670 histogram_value th = gimple_histogram_value (cfun, stmt);
671 if (!th)
672 continue;
674 if (dump_file)
676 fprintf (dump_file, "Trying transformations on stmt ");
677 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
678 dump_histograms_for_stmt (cfun, dump_file, stmt);
681 /* Transformations: */
682 /* The order of things in this conditional controls which
683 transformation is used when more than one is applicable. */
684 /* It is expected that any code added by the transformations
685 will be added before the current statement, and that the
686 current statement remain valid (although possibly
687 modified) upon return. */
688 if (gimple_mod_subtract_transform (&gsi)
689 || gimple_divmod_fixed_value_transform (&gsi)
690 || gimple_mod_pow2_value_transform (&gsi)
691 || gimple_stringops_transform (&gsi)
692 || gimple_ic_transform (&gsi))
694 stmt = gsi_stmt (gsi);
695 changed = true;
696 /* Original statement may no longer be in the same block. */
697 if (bb != gimple_bb (stmt))
699 bb = gimple_bb (stmt);
700 gsi = gsi_for_stmt (stmt);
706 if (changed)
708 counts_to_freqs ();
711 return changed;
714 /* Generate code for transformation 1 (with parent gimple assignment
715 STMT and probability of taking the optimal path PROB, which is
716 equivalent to COUNT/ALL within roundoff error). This generates the
717 result into a temp and returns the temp; it does not replace or
718 alter the original STMT. */
720 static tree
721 gimple_divmod_fixed_value (gassign *stmt, tree value, int prob,
722 gcov_type count, gcov_type all)
724 gassign *stmt1, *stmt2;
725 gcond *stmt3;
726 tree tmp0, tmp1, tmp2;
727 gimple *bb1end, *bb2end, *bb3end;
728 basic_block bb, bb2, bb3, bb4;
729 tree optype, op1, op2;
730 edge e12, e13, e23, e24, e34;
731 gimple_stmt_iterator gsi;
733 gcc_assert (is_gimple_assign (stmt)
734 && (gimple_assign_rhs_code (stmt) == TRUNC_DIV_EXPR
735 || gimple_assign_rhs_code (stmt) == TRUNC_MOD_EXPR));
737 optype = TREE_TYPE (gimple_assign_lhs (stmt));
738 op1 = gimple_assign_rhs1 (stmt);
739 op2 = gimple_assign_rhs2 (stmt);
741 bb = gimple_bb (stmt);
742 gsi = gsi_for_stmt (stmt);
744 tmp0 = make_temp_ssa_name (optype, NULL, "PROF");
745 tmp1 = make_temp_ssa_name (optype, NULL, "PROF");
746 stmt1 = gimple_build_assign (tmp0, fold_convert (optype, value));
747 stmt2 = gimple_build_assign (tmp1, op2);
748 stmt3 = gimple_build_cond (NE_EXPR, tmp1, tmp0, NULL_TREE, NULL_TREE);
749 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
750 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
751 gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
752 bb1end = stmt3;
754 tmp2 = create_tmp_reg (optype, "PROF");
755 stmt1 = gimple_build_assign (tmp2, gimple_assign_rhs_code (stmt), op1, tmp0);
756 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
757 bb2end = stmt1;
759 stmt1 = gimple_build_assign (tmp2, gimple_assign_rhs_code (stmt), op1, op2);
760 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
761 bb3end = stmt1;
763 /* Fix CFG. */
764 /* Edge e23 connects bb2 to bb3, etc. */
765 e12 = split_block (bb, bb1end);
766 bb2 = e12->dest;
767 bb2->count = count;
768 e23 = split_block (bb2, bb2end);
769 bb3 = e23->dest;
770 bb3->count = all - count;
771 e34 = split_block (bb3, bb3end);
772 bb4 = e34->dest;
773 bb4->count = all;
775 e12->flags &= ~EDGE_FALLTHRU;
776 e12->flags |= EDGE_FALSE_VALUE;
777 e12->probability = prob;
778 e12->count = count;
780 e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
781 e13->probability = REG_BR_PROB_BASE - prob;
782 e13->count = all - count;
784 remove_edge (e23);
786 e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
787 e24->probability = REG_BR_PROB_BASE;
788 e24->count = count;
790 e34->probability = REG_BR_PROB_BASE;
791 e34->count = all - count;
793 return tmp2;
796 /* Do transform 1) on INSN if applicable. */
798 static bool
799 gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si)
801 histogram_value histogram;
802 enum tree_code code;
803 gcov_type val, count, all;
804 tree result, value, tree_val;
805 gcov_type prob;
806 gassign *stmt;
808 stmt = dyn_cast <gassign *> (gsi_stmt (*si));
809 if (!stmt)
810 return false;
812 if (!INTEGRAL_TYPE_P (TREE_TYPE (gimple_assign_lhs (stmt))))
813 return false;
815 code = gimple_assign_rhs_code (stmt);
817 if (code != TRUNC_DIV_EXPR && code != TRUNC_MOD_EXPR)
818 return false;
820 histogram = gimple_histogram_value_of_type (cfun, stmt,
821 HIST_TYPE_SINGLE_VALUE);
822 if (!histogram)
823 return false;
825 value = histogram->hvalue.value;
826 val = histogram->hvalue.counters[0];
827 count = histogram->hvalue.counters[1];
828 all = histogram->hvalue.counters[2];
829 gimple_remove_histogram_value (cfun, stmt, histogram);
831 /* We require that count is at least half of all; this means
832 that for the transformation to fire the value must be constant
833 at least 50% of time (and 75% gives the guarantee of usage). */
834 if (simple_cst_equal (gimple_assign_rhs2 (stmt), value) != 1
835 || 2 * count < all
836 || optimize_bb_for_size_p (gimple_bb (stmt)))
837 return false;
839 if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count))
840 return false;
842 /* Compute probability of taking the optimal path. */
843 if (all > 0)
844 prob = GCOV_COMPUTE_SCALE (count, all);
845 else
846 prob = 0;
848 if (sizeof (gcov_type) == sizeof (HOST_WIDE_INT))
849 tree_val = build_int_cst (get_gcov_type (), val);
850 else
852 HOST_WIDE_INT a[2];
853 a[0] = (unsigned HOST_WIDE_INT) val;
854 a[1] = val >> (HOST_BITS_PER_WIDE_INT - 1) >> 1;
856 tree_val = wide_int_to_tree (get_gcov_type (), wide_int::from_array (a, 2,
857 TYPE_PRECISION (get_gcov_type ()), false));
859 result = gimple_divmod_fixed_value (stmt, tree_val, prob, count, all);
861 if (dump_file)
863 fprintf (dump_file, "Div/mod by constant ");
864 print_generic_expr (dump_file, value, TDF_SLIM);
865 fprintf (dump_file, "=");
866 print_generic_expr (dump_file, tree_val, TDF_SLIM);
867 fprintf (dump_file, " transformation on insn ");
868 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
871 gimple_assign_set_rhs_from_tree (si, result);
872 update_stmt (gsi_stmt (*si));
874 return true;
877 /* Generate code for transformation 2 (with parent gimple assign STMT and
878 probability of taking the optimal path PROB, which is equivalent to COUNT/ALL
879 within roundoff error). This generates the result into a temp and returns
880 the temp; it does not replace or alter the original STMT. */
882 static tree
883 gimple_mod_pow2 (gassign *stmt, int prob, gcov_type count, gcov_type all)
885 gassign *stmt1, *stmt2, *stmt3;
886 gcond *stmt4;
887 tree tmp2, tmp3;
888 gimple *bb1end, *bb2end, *bb3end;
889 basic_block bb, bb2, bb3, bb4;
890 tree optype, op1, op2;
891 edge e12, e13, e23, e24, e34;
892 gimple_stmt_iterator gsi;
893 tree result;
895 gcc_assert (is_gimple_assign (stmt)
896 && gimple_assign_rhs_code (stmt) == TRUNC_MOD_EXPR);
898 optype = TREE_TYPE (gimple_assign_lhs (stmt));
899 op1 = gimple_assign_rhs1 (stmt);
900 op2 = gimple_assign_rhs2 (stmt);
902 bb = gimple_bb (stmt);
903 gsi = gsi_for_stmt (stmt);
905 result = create_tmp_reg (optype, "PROF");
906 tmp2 = make_temp_ssa_name (optype, NULL, "PROF");
907 tmp3 = make_temp_ssa_name (optype, NULL, "PROF");
908 stmt2 = gimple_build_assign (tmp2, PLUS_EXPR, op2,
909 build_int_cst (optype, -1));
910 stmt3 = gimple_build_assign (tmp3, BIT_AND_EXPR, tmp2, op2);
911 stmt4 = gimple_build_cond (NE_EXPR, tmp3, build_int_cst (optype, 0),
912 NULL_TREE, NULL_TREE);
913 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
914 gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
915 gsi_insert_before (&gsi, stmt4, GSI_SAME_STMT);
916 bb1end = stmt4;
918 /* tmp2 == op2-1 inherited from previous block. */
919 stmt1 = gimple_build_assign (result, BIT_AND_EXPR, op1, tmp2);
920 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
921 bb2end = stmt1;
923 stmt1 = gimple_build_assign (result, gimple_assign_rhs_code (stmt),
924 op1, op2);
925 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
926 bb3end = stmt1;
928 /* Fix CFG. */
929 /* Edge e23 connects bb2 to bb3, etc. */
930 e12 = split_block (bb, bb1end);
931 bb2 = e12->dest;
932 bb2->count = count;
933 e23 = split_block (bb2, bb2end);
934 bb3 = e23->dest;
935 bb3->count = all - count;
936 e34 = split_block (bb3, bb3end);
937 bb4 = e34->dest;
938 bb4->count = all;
940 e12->flags &= ~EDGE_FALLTHRU;
941 e12->flags |= EDGE_FALSE_VALUE;
942 e12->probability = prob;
943 e12->count = count;
945 e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
946 e13->probability = REG_BR_PROB_BASE - prob;
947 e13->count = all - count;
949 remove_edge (e23);
951 e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
952 e24->probability = REG_BR_PROB_BASE;
953 e24->count = count;
955 e34->probability = REG_BR_PROB_BASE;
956 e34->count = all - count;
958 return result;
961 /* Do transform 2) on INSN if applicable. */
963 static bool
964 gimple_mod_pow2_value_transform (gimple_stmt_iterator *si)
966 histogram_value histogram;
967 enum tree_code code;
968 gcov_type count, wrong_values, all;
969 tree lhs_type, result, value;
970 gcov_type prob;
971 gassign *stmt;
973 stmt = dyn_cast <gassign *> (gsi_stmt (*si));
974 if (!stmt)
975 return false;
977 lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
978 if (!INTEGRAL_TYPE_P (lhs_type))
979 return false;
981 code = gimple_assign_rhs_code (stmt);
983 if (code != TRUNC_MOD_EXPR || !TYPE_UNSIGNED (lhs_type))
984 return false;
986 histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_POW2);
987 if (!histogram)
988 return false;
990 value = histogram->hvalue.value;
991 wrong_values = histogram->hvalue.counters[0];
992 count = histogram->hvalue.counters[1];
994 gimple_remove_histogram_value (cfun, stmt, histogram);
996 /* We require that we hit a power of 2 at least half of all evaluations. */
997 if (simple_cst_equal (gimple_assign_rhs2 (stmt), value) != 1
998 || count < wrong_values
999 || optimize_bb_for_size_p (gimple_bb (stmt)))
1000 return false;
1002 if (dump_file)
1004 fprintf (dump_file, "Mod power of 2 transformation on insn ");
1005 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1008 /* Compute probability of taking the optimal path. */
1009 all = count + wrong_values;
1011 if (check_counter (stmt, "pow2", &count, &all, gimple_bb (stmt)->count))
1012 return false;
1014 if (all > 0)
1015 prob = GCOV_COMPUTE_SCALE (count, all);
1016 else
1017 prob = 0;
1019 result = gimple_mod_pow2 (stmt, prob, count, all);
1021 gimple_assign_set_rhs_from_tree (si, result);
1022 update_stmt (gsi_stmt (*si));
1024 return true;
1027 /* Generate code for transformations 3 and 4 (with parent gimple assign STMT, and
1028 NCOUNTS the number of cases to support. Currently only NCOUNTS==0 or 1 is
1029 supported and this is built into this interface. The probabilities of taking
1030 the optimal paths are PROB1 and PROB2, which are equivalent to COUNT1/ALL and
1031 COUNT2/ALL respectively within roundoff error). This generates the
1032 result into a temp and returns the temp; it does not replace or alter
1033 the original STMT. */
1034 /* FIXME: Generalize the interface to handle NCOUNTS > 1. */
1036 static tree
1037 gimple_mod_subtract (gassign *stmt, int prob1, int prob2, int ncounts,
1038 gcov_type count1, gcov_type count2, gcov_type all)
1040 gassign *stmt1;
1041 gimple *stmt2;
1042 gcond *stmt3;
1043 tree tmp1;
1044 gimple *bb1end, *bb2end = NULL, *bb3end;
1045 basic_block bb, bb2, bb3, bb4;
1046 tree optype, op1, op2;
1047 edge e12, e23 = 0, e24, e34, e14;
1048 gimple_stmt_iterator gsi;
1049 tree result;
1051 gcc_assert (is_gimple_assign (stmt)
1052 && gimple_assign_rhs_code (stmt) == TRUNC_MOD_EXPR);
1054 optype = TREE_TYPE (gimple_assign_lhs (stmt));
1055 op1 = gimple_assign_rhs1 (stmt);
1056 op2 = gimple_assign_rhs2 (stmt);
1058 bb = gimple_bb (stmt);
1059 gsi = gsi_for_stmt (stmt);
1061 result = create_tmp_reg (optype, "PROF");
1062 tmp1 = make_temp_ssa_name (optype, NULL, "PROF");
1063 stmt1 = gimple_build_assign (result, op1);
1064 stmt2 = gimple_build_assign (tmp1, op2);
1065 stmt3 = gimple_build_cond (LT_EXPR, result, tmp1, NULL_TREE, NULL_TREE);
1066 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
1067 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
1068 gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
1069 bb1end = stmt3;
1071 if (ncounts) /* Assumed to be 0 or 1 */
1073 stmt1 = gimple_build_assign (result, MINUS_EXPR, result, tmp1);
1074 stmt2 = gimple_build_cond (LT_EXPR, result, tmp1, NULL_TREE, NULL_TREE);
1075 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
1076 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
1077 bb2end = stmt2;
1080 /* Fallback case. */
1081 stmt1 = gimple_build_assign (result, gimple_assign_rhs_code (stmt),
1082 result, tmp1);
1083 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
1084 bb3end = stmt1;
1086 /* Fix CFG. */
1087 /* Edge e23 connects bb2 to bb3, etc. */
1088 /* However block 3 is optional; if it is not there, references
1089 to 3 really refer to block 2. */
1090 e12 = split_block (bb, bb1end);
1091 bb2 = e12->dest;
1092 bb2->count = all - count1;
1094 if (ncounts) /* Assumed to be 0 or 1. */
1096 e23 = split_block (bb2, bb2end);
1097 bb3 = e23->dest;
1098 bb3->count = all - count1 - count2;
1101 e34 = split_block (ncounts ? bb3 : bb2, bb3end);
1102 bb4 = e34->dest;
1103 bb4->count = all;
1105 e12->flags &= ~EDGE_FALLTHRU;
1106 e12->flags |= EDGE_FALSE_VALUE;
1107 e12->probability = REG_BR_PROB_BASE - prob1;
1108 e12->count = all - count1;
1110 e14 = make_edge (bb, bb4, EDGE_TRUE_VALUE);
1111 e14->probability = prob1;
1112 e14->count = count1;
1114 if (ncounts) /* Assumed to be 0 or 1. */
1116 e23->flags &= ~EDGE_FALLTHRU;
1117 e23->flags |= EDGE_FALSE_VALUE;
1118 e23->count = all - count1 - count2;
1119 e23->probability = REG_BR_PROB_BASE - prob2;
1121 e24 = make_edge (bb2, bb4, EDGE_TRUE_VALUE);
1122 e24->probability = prob2;
1123 e24->count = count2;
1126 e34->probability = REG_BR_PROB_BASE;
1127 e34->count = all - count1 - count2;
1129 return result;
1132 /* Do transforms 3) and 4) on the statement pointed-to by SI if applicable. */
1134 static bool
1135 gimple_mod_subtract_transform (gimple_stmt_iterator *si)
1137 histogram_value histogram;
1138 enum tree_code code;
1139 gcov_type count, wrong_values, all;
1140 tree lhs_type, result;
1141 gcov_type prob1, prob2;
1142 unsigned int i, steps;
1143 gcov_type count1, count2;
1144 gassign *stmt;
1145 stmt = dyn_cast <gassign *> (gsi_stmt (*si));
1146 if (!stmt)
1147 return false;
1149 lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
1150 if (!INTEGRAL_TYPE_P (lhs_type))
1151 return false;
1153 code = gimple_assign_rhs_code (stmt);
1155 if (code != TRUNC_MOD_EXPR || !TYPE_UNSIGNED (lhs_type))
1156 return false;
1158 histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_INTERVAL);
1159 if (!histogram)
1160 return false;
1162 all = 0;
1163 wrong_values = 0;
1164 for (i = 0; i < histogram->hdata.intvl.steps; i++)
1165 all += histogram->hvalue.counters[i];
1167 wrong_values += histogram->hvalue.counters[i];
1168 wrong_values += histogram->hvalue.counters[i+1];
1169 steps = histogram->hdata.intvl.steps;
1170 all += wrong_values;
1171 count1 = histogram->hvalue.counters[0];
1172 count2 = histogram->hvalue.counters[1];
1174 /* Compute probability of taking the optimal path. */
1175 if (check_counter (stmt, "interval", &count1, &all, gimple_bb (stmt)->count))
1177 gimple_remove_histogram_value (cfun, stmt, histogram);
1178 return false;
1181 if (flag_profile_correction && count1 + count2 > all)
1182 all = count1 + count2;
1184 gcc_assert (count1 + count2 <= all);
1186 /* We require that we use just subtractions in at least 50% of all
1187 evaluations. */
1188 count = 0;
1189 for (i = 0; i < histogram->hdata.intvl.steps; i++)
1191 count += histogram->hvalue.counters[i];
1192 if (count * 2 >= all)
1193 break;
1195 if (i == steps
1196 || optimize_bb_for_size_p (gimple_bb (stmt)))
1197 return false;
1199 gimple_remove_histogram_value (cfun, stmt, histogram);
1200 if (dump_file)
1202 fprintf (dump_file, "Mod subtract transformation on insn ");
1203 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1206 /* Compute probability of taking the optimal path(s). */
1207 if (all > 0)
1209 prob1 = GCOV_COMPUTE_SCALE (count1, all);
1210 prob2 = GCOV_COMPUTE_SCALE (count2, all);
1212 else
1214 prob1 = prob2 = 0;
1217 /* In practice, "steps" is always 2. This interface reflects this,
1218 and will need to be changed if "steps" can change. */
1219 result = gimple_mod_subtract (stmt, prob1, prob2, i, count1, count2, all);
1221 gimple_assign_set_rhs_from_tree (si, result);
1222 update_stmt (gsi_stmt (*si));
1224 return true;
1227 typedef int_hash <unsigned int, 0, UINT_MAX> profile_id_hash;
1229 static hash_map<profile_id_hash, cgraph_node *> *cgraph_node_map = 0;
1231 /* Returns true if node graph is initialized. This
1232 is used to test if profile_id has been created
1233 for cgraph_nodes. */
1235 bool
1236 coverage_node_map_initialized_p (void)
1238 return cgraph_node_map != 0;
1241 /* Initialize map from PROFILE_ID to CGRAPH_NODE.
1242 When LOCAL is true, the PROFILE_IDs are computed. when it is false we assume
1243 that the PROFILE_IDs was already assigned. */
1245 void
1246 init_node_map (bool local)
1248 struct cgraph_node *n;
1249 cgraph_node_map = new hash_map<profile_id_hash, cgraph_node *>;
1251 FOR_EACH_DEFINED_FUNCTION (n)
1252 if (n->has_gimple_body_p ())
1254 cgraph_node **val;
1255 if (local)
1257 n->profile_id = coverage_compute_profile_id (n);
1258 while ((val = cgraph_node_map->get (n->profile_id))
1259 || !n->profile_id)
1261 if (dump_file)
1262 fprintf (dump_file, "Local profile-id %i conflict"
1263 " with nodes %s/%i %s/%i\n",
1264 n->profile_id,
1265 n->name (),
1266 n->order,
1267 (*val)->name (),
1268 (*val)->order);
1269 n->profile_id = (n->profile_id + 1) & 0x7fffffff;
1272 else if (!n->profile_id)
1274 if (dump_file)
1275 fprintf (dump_file,
1276 "Node %s/%i has no profile-id"
1277 " (profile feedback missing?)\n",
1278 n->name (),
1279 n->order);
1280 continue;
1282 else if ((val = cgraph_node_map->get (n->profile_id)))
1284 if (dump_file)
1285 fprintf (dump_file,
1286 "Node %s/%i has IP profile-id %i conflict. "
1287 "Giving up.\n",
1288 n->name (),
1289 n->order,
1290 n->profile_id);
1291 *val = NULL;
1292 continue;
1294 cgraph_node_map->put (n->profile_id, n);
1298 /* Delete the CGRAPH_NODE_MAP. */
1300 void
1301 del_node_map (void)
1303 delete cgraph_node_map;
1306 /* Return cgraph node for function with pid */
1308 struct cgraph_node*
1309 find_func_by_profile_id (int profile_id)
1311 cgraph_node **val = cgraph_node_map->get (profile_id);
1312 if (val)
1313 return *val;
1314 else
1315 return NULL;
1318 /* Perform sanity check on the indirect call target. Due to race conditions,
1319 false function target may be attributed to an indirect call site. If the
1320 call expression type mismatches with the target function's type, expand_call
1321 may ICE. Here we only do very minimal sanity check just to make compiler happy.
1322 Returns true if TARGET is considered ok for call CALL_STMT. */
1324 bool
1325 check_ic_target (gcall *call_stmt, struct cgraph_node *target)
1327 location_t locus;
1328 if (gimple_check_call_matching_types (call_stmt, target->decl, true))
1329 return true;
1331 locus = gimple_location (call_stmt);
1332 if (dump_enabled_p ())
1333 dump_printf_loc (MSG_MISSED_OPTIMIZATION, locus,
1334 "Skipping target %s with mismatching types for icall\n",
1335 target->name ());
1336 return false;
1339 /* Do transformation
1341 if (actual_callee_address == address_of_most_common_function/method)
1342 do direct call
1343 else
1344 old call
1347 gcall *
1348 gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call,
1349 int prob, gcov_type count, gcov_type all)
1351 gcall *dcall_stmt;
1352 gassign *load_stmt;
1353 gcond *cond_stmt;
1354 gcall *iretbnd_stmt = NULL;
1355 tree tmp0, tmp1, tmp;
1356 basic_block cond_bb, dcall_bb, icall_bb, join_bb = NULL;
1357 tree optype = build_pointer_type (void_type_node);
1358 edge e_cd, e_ci, e_di, e_dj = NULL, e_ij;
1359 gimple_stmt_iterator gsi;
1360 int lp_nr, dflags;
1361 edge e_eh, e;
1362 edge_iterator ei;
1363 gimple_stmt_iterator psi;
1365 cond_bb = gimple_bb (icall_stmt);
1366 gsi = gsi_for_stmt (icall_stmt);
1368 if (gimple_call_with_bounds_p (icall_stmt) && gimple_call_lhs (icall_stmt))
1369 iretbnd_stmt = chkp_retbnd_call_by_val (gimple_call_lhs (icall_stmt));
1371 tmp0 = make_temp_ssa_name (optype, NULL, "PROF");
1372 tmp1 = make_temp_ssa_name (optype, NULL, "PROF");
1373 tmp = unshare_expr (gimple_call_fn (icall_stmt));
1374 load_stmt = gimple_build_assign (tmp0, tmp);
1375 gsi_insert_before (&gsi, load_stmt, GSI_SAME_STMT);
1377 tmp = fold_convert (optype, build_addr (direct_call->decl));
1378 load_stmt = gimple_build_assign (tmp1, tmp);
1379 gsi_insert_before (&gsi, load_stmt, GSI_SAME_STMT);
1381 cond_stmt = gimple_build_cond (EQ_EXPR, tmp1, tmp0, NULL_TREE, NULL_TREE);
1382 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
1384 if (TREE_CODE (gimple_vdef (icall_stmt)) == SSA_NAME)
1386 unlink_stmt_vdef (icall_stmt);
1387 release_ssa_name (gimple_vdef (icall_stmt));
1389 gimple_set_vdef (icall_stmt, NULL_TREE);
1390 gimple_set_vuse (icall_stmt, NULL_TREE);
1391 update_stmt (icall_stmt);
1392 dcall_stmt = as_a <gcall *> (gimple_copy (icall_stmt));
1393 gimple_call_set_fndecl (dcall_stmt, direct_call->decl);
1394 dflags = flags_from_decl_or_type (direct_call->decl);
1395 if ((dflags & ECF_NORETURN) != 0)
1396 gimple_call_set_lhs (dcall_stmt, NULL_TREE);
1397 gsi_insert_before (&gsi, dcall_stmt, GSI_SAME_STMT);
1399 /* Fix CFG. */
1400 /* Edge e_cd connects cond_bb to dcall_bb, etc; note the first letters. */
1401 e_cd = split_block (cond_bb, cond_stmt);
1402 dcall_bb = e_cd->dest;
1403 dcall_bb->count = count;
1405 e_di = split_block (dcall_bb, dcall_stmt);
1406 icall_bb = e_di->dest;
1407 icall_bb->count = all - count;
1409 /* Do not disturb existing EH edges from the indirect call. */
1410 if (!stmt_ends_bb_p (icall_stmt))
1411 e_ij = split_block (icall_bb, icall_stmt);
1412 else
1414 e_ij = find_fallthru_edge (icall_bb->succs);
1415 /* The indirect call might be noreturn. */
1416 if (e_ij != NULL)
1418 e_ij->probability = REG_BR_PROB_BASE;
1419 e_ij->count = all - count;
1420 e_ij = single_pred_edge (split_edge (e_ij));
1423 if (e_ij != NULL)
1425 join_bb = e_ij->dest;
1426 join_bb->count = all;
1429 e_cd->flags = (e_cd->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE;
1430 e_cd->probability = prob;
1431 e_cd->count = count;
1433 e_ci = make_edge (cond_bb, icall_bb, EDGE_FALSE_VALUE);
1434 e_ci->probability = REG_BR_PROB_BASE - prob;
1435 e_ci->count = all - count;
1437 remove_edge (e_di);
1439 if (e_ij != NULL)
1441 if ((dflags & ECF_NORETURN) != 0)
1442 e_ij->count = all;
1443 else
1445 e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU);
1446 e_dj->probability = REG_BR_PROB_BASE;
1447 e_dj->count = count;
1449 e_ij->count = all - count;
1451 e_ij->probability = REG_BR_PROB_BASE;
1454 /* Insert PHI node for the call result if necessary. */
1455 if (gimple_call_lhs (icall_stmt)
1456 && TREE_CODE (gimple_call_lhs (icall_stmt)) == SSA_NAME
1457 && (dflags & ECF_NORETURN) == 0)
1459 tree result = gimple_call_lhs (icall_stmt);
1460 gphi *phi = create_phi_node (result, join_bb);
1461 gimple_call_set_lhs (icall_stmt,
1462 duplicate_ssa_name (result, icall_stmt));
1463 add_phi_arg (phi, gimple_call_lhs (icall_stmt), e_ij, UNKNOWN_LOCATION);
1464 gimple_call_set_lhs (dcall_stmt,
1465 duplicate_ssa_name (result, dcall_stmt));
1466 add_phi_arg (phi, gimple_call_lhs (dcall_stmt), e_dj, UNKNOWN_LOCATION);
1468 /* If indirect call has following BUILT_IN_CHKP_BNDRET
1469 call then we need to make it's copy for the direct
1470 call. */
1471 if (iretbnd_stmt)
1473 if (gimple_call_lhs (iretbnd_stmt))
1475 gimple *copy;
1477 if (TREE_CODE (gimple_vdef (iretbnd_stmt)) == SSA_NAME)
1479 unlink_stmt_vdef (iretbnd_stmt);
1480 release_ssa_name (gimple_vdef (iretbnd_stmt));
1482 gimple_set_vdef (iretbnd_stmt, NULL_TREE);
1483 gimple_set_vuse (iretbnd_stmt, NULL_TREE);
1484 update_stmt (iretbnd_stmt);
1486 result = gimple_call_lhs (iretbnd_stmt);
1487 phi = create_phi_node (result, join_bb);
1489 copy = gimple_copy (iretbnd_stmt);
1490 gimple_call_set_arg (copy, 0,
1491 gimple_call_lhs (dcall_stmt));
1492 gimple_call_set_lhs (copy, duplicate_ssa_name (result, copy));
1493 gsi_insert_on_edge (e_dj, copy);
1494 add_phi_arg (phi, gimple_call_lhs (copy),
1495 e_dj, UNKNOWN_LOCATION);
1497 gimple_call_set_arg (iretbnd_stmt, 0,
1498 gimple_call_lhs (icall_stmt));
1499 gimple_call_set_lhs (iretbnd_stmt,
1500 duplicate_ssa_name (result, iretbnd_stmt));
1501 psi = gsi_for_stmt (iretbnd_stmt);
1502 gsi_remove (&psi, false);
1503 gsi_insert_on_edge (e_ij, iretbnd_stmt);
1504 add_phi_arg (phi, gimple_call_lhs (iretbnd_stmt),
1505 e_ij, UNKNOWN_LOCATION);
1507 gsi_commit_one_edge_insert (e_dj, NULL);
1508 gsi_commit_one_edge_insert (e_ij, NULL);
1510 else
1512 psi = gsi_for_stmt (iretbnd_stmt);
1513 gsi_remove (&psi, true);
1518 /* Build an EH edge for the direct call if necessary. */
1519 lp_nr = lookup_stmt_eh_lp (icall_stmt);
1520 if (lp_nr > 0 && stmt_could_throw_p (dcall_stmt))
1522 add_stmt_to_eh_lp (dcall_stmt, lp_nr);
1525 FOR_EACH_EDGE (e_eh, ei, icall_bb->succs)
1526 if (e_eh->flags & (EDGE_EH | EDGE_ABNORMAL))
1528 e = make_edge (dcall_bb, e_eh->dest, e_eh->flags);
1529 for (gphi_iterator psi = gsi_start_phis (e_eh->dest);
1530 !gsi_end_p (psi); gsi_next (&psi))
1532 gphi *phi = psi.phi ();
1533 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
1534 PHI_ARG_DEF_FROM_EDGE (phi, e_eh));
1537 if (!stmt_could_throw_p (dcall_stmt))
1538 gimple_purge_dead_eh_edges (dcall_bb);
1539 return dcall_stmt;
1543 For every checked indirect/virtual call determine if most common pid of
1544 function/class method has probability more than 50%. If yes modify code of
1545 this call to:
1548 static bool
1549 gimple_ic_transform (gimple_stmt_iterator *gsi)
1551 gcall *stmt;
1552 histogram_value histogram;
1553 gcov_type val, count, all, bb_all;
1554 struct cgraph_node *direct_call;
1556 stmt = dyn_cast <gcall *> (gsi_stmt (*gsi));
1557 if (!stmt)
1558 return false;
1560 if (gimple_call_fndecl (stmt) != NULL_TREE)
1561 return false;
1563 if (gimple_call_internal_p (stmt))
1564 return false;
1566 histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_INDIR_CALL);
1567 if (!histogram)
1568 return false;
1570 val = histogram->hvalue.counters [0];
1571 count = histogram->hvalue.counters [1];
1572 all = histogram->hvalue.counters [2];
1574 bb_all = gimple_bb (stmt)->count;
1575 /* The order of CHECK_COUNTER calls is important -
1576 since check_counter can correct the third parameter
1577 and we want to make count <= all <= bb_all. */
1578 if ( check_counter (stmt, "ic", &all, &bb_all, bb_all)
1579 || check_counter (stmt, "ic", &count, &all, all))
1581 gimple_remove_histogram_value (cfun, stmt, histogram);
1582 return false;
1585 if (4 * count <= 3 * all)
1586 return false;
1588 direct_call = find_func_by_profile_id ((int)val);
1590 if (direct_call == NULL)
1592 if (val)
1594 if (dump_file)
1596 fprintf (dump_file, "Indirect call -> direct call from other module");
1597 print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM);
1598 fprintf (dump_file, "=> %i (will resolve only with LTO)\n", (int)val);
1601 return false;
1604 if (!check_ic_target (stmt, direct_call))
1606 if (dump_file)
1608 fprintf (dump_file, "Indirect call -> direct call ");
1609 print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM);
1610 fprintf (dump_file, "=> ");
1611 print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
1612 fprintf (dump_file, " transformation skipped because of type mismatch");
1613 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1615 gimple_remove_histogram_value (cfun, stmt, histogram);
1616 return false;
1619 if (dump_file)
1621 fprintf (dump_file, "Indirect call -> direct call ");
1622 print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM);
1623 fprintf (dump_file, "=> ");
1624 print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
1625 fprintf (dump_file, " transformation on insn postponned to ipa-profile");
1626 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1627 fprintf (dump_file, "hist->count %" PRId64
1628 " hist->all %" PRId64"\n", count, all);
1631 return true;
1634 /* Return true if the stringop CALL shall be profiled. SIZE_ARG be
1635 set to the argument index for the size of the string operation. */
1637 static bool
1638 interesting_stringop_to_profile_p (gcall *call, int *size_arg)
1640 enum built_in_function fcode;
1642 fcode = DECL_FUNCTION_CODE (gimple_call_fndecl (call));
1643 if (fcode != BUILT_IN_MEMCPY && fcode != BUILT_IN_MEMPCPY
1644 && fcode != BUILT_IN_MEMSET && fcode != BUILT_IN_BZERO)
1645 return false;
1647 switch (fcode)
1649 case BUILT_IN_MEMCPY:
1650 case BUILT_IN_MEMPCPY:
1651 *size_arg = 2;
1652 return validate_gimple_arglist (call, POINTER_TYPE, POINTER_TYPE,
1653 INTEGER_TYPE, VOID_TYPE);
1654 case BUILT_IN_MEMSET:
1655 *size_arg = 2;
1656 return validate_gimple_arglist (call, POINTER_TYPE, INTEGER_TYPE,
1657 INTEGER_TYPE, VOID_TYPE);
1658 case BUILT_IN_BZERO:
1659 *size_arg = 1;
1660 return validate_gimple_arglist (call, POINTER_TYPE, INTEGER_TYPE,
1661 VOID_TYPE);
1662 default:
1663 gcc_unreachable ();
1667 /* Convert stringop (..., vcall_size)
1668 into
1669 if (vcall_size == icall_size)
1670 stringop (..., icall_size);
1671 else
1672 stringop (..., vcall_size);
1673 assuming we'll propagate a true constant into ICALL_SIZE later. */
1675 static void
1676 gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, int prob,
1677 gcov_type count, gcov_type all)
1679 gassign *tmp_stmt;
1680 gcond *cond_stmt;
1681 gcall *icall_stmt;
1682 tree tmp0, tmp1, vcall_size, optype;
1683 basic_block cond_bb, icall_bb, vcall_bb, join_bb;
1684 edge e_ci, e_cv, e_iv, e_ij, e_vj;
1685 gimple_stmt_iterator gsi;
1686 int size_arg;
1688 if (!interesting_stringop_to_profile_p (vcall_stmt, &size_arg))
1689 gcc_unreachable ();
1691 cond_bb = gimple_bb (vcall_stmt);
1692 gsi = gsi_for_stmt (vcall_stmt);
1694 vcall_size = gimple_call_arg (vcall_stmt, size_arg);
1695 optype = TREE_TYPE (vcall_size);
1697 tmp0 = make_temp_ssa_name (optype, NULL, "PROF");
1698 tmp1 = make_temp_ssa_name (optype, NULL, "PROF");
1699 tmp_stmt = gimple_build_assign (tmp0, fold_convert (optype, icall_size));
1700 gsi_insert_before (&gsi, tmp_stmt, GSI_SAME_STMT);
1702 tmp_stmt = gimple_build_assign (tmp1, vcall_size);
1703 gsi_insert_before (&gsi, tmp_stmt, GSI_SAME_STMT);
1705 cond_stmt = gimple_build_cond (EQ_EXPR, tmp1, tmp0, NULL_TREE, NULL_TREE);
1706 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
1708 if (TREE_CODE (gimple_vdef (vcall_stmt)) == SSA_NAME)
1710 unlink_stmt_vdef (vcall_stmt);
1711 release_ssa_name (gimple_vdef (vcall_stmt));
1713 gimple_set_vdef (vcall_stmt, NULL);
1714 gimple_set_vuse (vcall_stmt, NULL);
1715 update_stmt (vcall_stmt);
1716 icall_stmt = as_a <gcall *> (gimple_copy (vcall_stmt));
1717 gimple_call_set_arg (icall_stmt, size_arg, icall_size);
1718 gsi_insert_before (&gsi, icall_stmt, GSI_SAME_STMT);
1720 /* Fix CFG. */
1721 /* Edge e_ci connects cond_bb to icall_bb, etc. */
1722 e_ci = split_block (cond_bb, cond_stmt);
1723 icall_bb = e_ci->dest;
1724 icall_bb->count = count;
1726 e_iv = split_block (icall_bb, icall_stmt);
1727 vcall_bb = e_iv->dest;
1728 vcall_bb->count = all - count;
1730 e_vj = split_block (vcall_bb, vcall_stmt);
1731 join_bb = e_vj->dest;
1732 join_bb->count = all;
1734 e_ci->flags = (e_ci->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE;
1735 e_ci->probability = prob;
1736 e_ci->count = count;
1738 e_cv = make_edge (cond_bb, vcall_bb, EDGE_FALSE_VALUE);
1739 e_cv->probability = REG_BR_PROB_BASE - prob;
1740 e_cv->count = all - count;
1742 remove_edge (e_iv);
1744 e_ij = make_edge (icall_bb, join_bb, EDGE_FALLTHRU);
1745 e_ij->probability = REG_BR_PROB_BASE;
1746 e_ij->count = count;
1748 e_vj->probability = REG_BR_PROB_BASE;
1749 e_vj->count = all - count;
1751 /* Insert PHI node for the call result if necessary. */
1752 if (gimple_call_lhs (vcall_stmt)
1753 && TREE_CODE (gimple_call_lhs (vcall_stmt)) == SSA_NAME)
1755 tree result = gimple_call_lhs (vcall_stmt);
1756 gphi *phi = create_phi_node (result, join_bb);
1757 gimple_call_set_lhs (vcall_stmt,
1758 duplicate_ssa_name (result, vcall_stmt));
1759 add_phi_arg (phi, gimple_call_lhs (vcall_stmt), e_vj, UNKNOWN_LOCATION);
1760 gimple_call_set_lhs (icall_stmt,
1761 duplicate_ssa_name (result, icall_stmt));
1762 add_phi_arg (phi, gimple_call_lhs (icall_stmt), e_ij, UNKNOWN_LOCATION);
1765 /* Because these are all string op builtins, they're all nothrow. */
1766 gcc_assert (!stmt_could_throw_p (vcall_stmt));
1767 gcc_assert (!stmt_could_throw_p (icall_stmt));
1770 /* Find values inside STMT for that we want to measure histograms for
1771 division/modulo optimization. */
1773 static bool
1774 gimple_stringops_transform (gimple_stmt_iterator *gsi)
1776 gcall *stmt;
1777 tree blck_size;
1778 enum built_in_function fcode;
1779 histogram_value histogram;
1780 gcov_type count, all, val;
1781 tree dest, src;
1782 unsigned int dest_align, src_align;
1783 gcov_type prob;
1784 tree tree_val;
1785 int size_arg;
1787 stmt = dyn_cast <gcall *> (gsi_stmt (*gsi));
1788 if (!stmt)
1789 return false;
1791 if (!gimple_call_builtin_p (gsi_stmt (*gsi), BUILT_IN_NORMAL))
1792 return false;
1794 if (!interesting_stringop_to_profile_p (stmt, &size_arg))
1795 return false;
1797 blck_size = gimple_call_arg (stmt, size_arg);
1798 if (TREE_CODE (blck_size) == INTEGER_CST)
1799 return false;
1801 histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_SINGLE_VALUE);
1802 if (!histogram)
1803 return false;
1805 val = histogram->hvalue.counters[0];
1806 count = histogram->hvalue.counters[1];
1807 all = histogram->hvalue.counters[2];
1808 gimple_remove_histogram_value (cfun, stmt, histogram);
1810 /* We require that count is at least half of all; this means
1811 that for the transformation to fire the value must be constant
1812 at least 80% of time. */
1813 if ((6 * count / 5) < all || optimize_bb_for_size_p (gimple_bb (stmt)))
1814 return false;
1815 if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count))
1816 return false;
1817 if (all > 0)
1818 prob = GCOV_COMPUTE_SCALE (count, all);
1819 else
1820 prob = 0;
1822 dest = gimple_call_arg (stmt, 0);
1823 dest_align = get_pointer_alignment (dest);
1824 fcode = DECL_FUNCTION_CODE (gimple_call_fndecl (stmt));
1825 switch (fcode)
1827 case BUILT_IN_MEMCPY:
1828 case BUILT_IN_MEMPCPY:
1829 src = gimple_call_arg (stmt, 1);
1830 src_align = get_pointer_alignment (src);
1831 if (!can_move_by_pieces (val, MIN (dest_align, src_align)))
1832 return false;
1833 break;
1834 case BUILT_IN_MEMSET:
1835 if (!can_store_by_pieces (val, builtin_memset_read_str,
1836 gimple_call_arg (stmt, 1),
1837 dest_align, true))
1838 return false;
1839 break;
1840 case BUILT_IN_BZERO:
1841 if (!can_store_by_pieces (val, builtin_memset_read_str,
1842 integer_zero_node,
1843 dest_align, true))
1844 return false;
1845 break;
1846 default:
1847 gcc_unreachable ();
1850 if (sizeof (gcov_type) == sizeof (HOST_WIDE_INT))
1851 tree_val = build_int_cst (get_gcov_type (), val);
1852 else
1854 HOST_WIDE_INT a[2];
1855 a[0] = (unsigned HOST_WIDE_INT) val;
1856 a[1] = val >> (HOST_BITS_PER_WIDE_INT - 1) >> 1;
1858 tree_val = wide_int_to_tree (get_gcov_type (), wide_int::from_array (a, 2,
1859 TYPE_PRECISION (get_gcov_type ()), false));
1862 if (dump_file)
1864 fprintf (dump_file, "Single value %i stringop transformation on ",
1865 (int)val);
1866 print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1869 gimple_stringop_fixed_value (stmt, tree_val, prob, count, all);
1871 return true;
1874 void
1875 stringop_block_profile (gimple *stmt, unsigned int *expected_align,
1876 HOST_WIDE_INT *expected_size)
1878 histogram_value histogram;
1879 histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_AVERAGE);
1881 if (!histogram)
1882 *expected_size = -1;
1883 else if (!histogram->hvalue.counters[1])
1885 *expected_size = -1;
1886 gimple_remove_histogram_value (cfun, stmt, histogram);
1888 else
1890 gcov_type size;
1891 size = ((histogram->hvalue.counters[0]
1892 + histogram->hvalue.counters[1] / 2)
1893 / histogram->hvalue.counters[1]);
1894 /* Even if we can hold bigger value in SIZE, INT_MAX
1895 is safe "infinity" for code generation strategies. */
1896 if (size > INT_MAX)
1897 size = INT_MAX;
1898 *expected_size = size;
1899 gimple_remove_histogram_value (cfun, stmt, histogram);
1902 histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_IOR);
1904 if (!histogram)
1905 *expected_align = 0;
1906 else if (!histogram->hvalue.counters[0])
1908 gimple_remove_histogram_value (cfun, stmt, histogram);
1909 *expected_align = 0;
1911 else
1913 gcov_type count;
1914 int alignment;
1916 count = histogram->hvalue.counters[0];
1917 alignment = 1;
1918 while (!(count & alignment)
1919 && (alignment * 2 * BITS_PER_UNIT))
1920 alignment <<= 1;
1921 *expected_align = alignment * BITS_PER_UNIT;
1922 gimple_remove_histogram_value (cfun, stmt, histogram);
1927 /* Find values inside STMT for that we want to measure histograms for
1928 division/modulo optimization. */
1930 static void
1931 gimple_divmod_values_to_profile (gimple *stmt, histogram_values *values)
1933 tree lhs, divisor, op0, type;
1934 histogram_value hist;
1936 if (gimple_code (stmt) != GIMPLE_ASSIGN)
1937 return;
1939 lhs = gimple_assign_lhs (stmt);
1940 type = TREE_TYPE (lhs);
1941 if (!INTEGRAL_TYPE_P (type))
1942 return;
1944 switch (gimple_assign_rhs_code (stmt))
1946 case TRUNC_DIV_EXPR:
1947 case TRUNC_MOD_EXPR:
1948 divisor = gimple_assign_rhs2 (stmt);
1949 op0 = gimple_assign_rhs1 (stmt);
1951 values->reserve (3);
1953 if (TREE_CODE (divisor) == SSA_NAME)
1954 /* Check for the case where the divisor is the same value most
1955 of the time. */
1956 values->quick_push (gimple_alloc_histogram_value (cfun,
1957 HIST_TYPE_SINGLE_VALUE,
1958 stmt, divisor));
1960 /* For mod, check whether it is not often a noop (or replaceable by
1961 a few subtractions). */
1962 if (gimple_assign_rhs_code (stmt) == TRUNC_MOD_EXPR
1963 && TYPE_UNSIGNED (type))
1965 tree val;
1966 /* Check for a special case where the divisor is power of 2. */
1967 values->quick_push (gimple_alloc_histogram_value (cfun,
1968 HIST_TYPE_POW2,
1969 stmt, divisor));
1971 val = build2 (TRUNC_DIV_EXPR, type, op0, divisor);
1972 hist = gimple_alloc_histogram_value (cfun, HIST_TYPE_INTERVAL,
1973 stmt, val);
1974 hist->hdata.intvl.int_start = 0;
1975 hist->hdata.intvl.steps = 2;
1976 values->quick_push (hist);
1978 return;
1980 default:
1981 return;
1985 /* Find calls inside STMT for that we want to measure histograms for
1986 indirect/virtual call optimization. */
1988 static void
1989 gimple_indirect_call_to_profile (gimple *stmt, histogram_values *values)
1991 tree callee;
1993 if (gimple_code (stmt) != GIMPLE_CALL
1994 || gimple_call_internal_p (stmt)
1995 || gimple_call_fndecl (stmt) != NULL_TREE)
1996 return;
1998 callee = gimple_call_fn (stmt);
2000 values->reserve (3);
2002 values->quick_push (gimple_alloc_histogram_value (
2003 cfun,
2004 PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
2005 HIST_TYPE_INDIR_CALL_TOPN :
2006 HIST_TYPE_INDIR_CALL,
2007 stmt, callee));
2009 return;
2012 /* Find values inside STMT for that we want to measure histograms for
2013 string operations. */
2015 static void
2016 gimple_stringops_values_to_profile (gimple *gs, histogram_values *values)
2018 gcall *stmt;
2019 tree blck_size;
2020 tree dest;
2021 int size_arg;
2023 stmt = dyn_cast <gcall *> (gs);
2024 if (!stmt)
2025 return;
2027 if (!gimple_call_builtin_p (gs, BUILT_IN_NORMAL))
2028 return;
2030 if (!interesting_stringop_to_profile_p (stmt, &size_arg))
2031 return;
2033 dest = gimple_call_arg (stmt, 0);
2034 blck_size = gimple_call_arg (stmt, size_arg);
2036 if (TREE_CODE (blck_size) != INTEGER_CST)
2038 values->safe_push (gimple_alloc_histogram_value (cfun,
2039 HIST_TYPE_SINGLE_VALUE,
2040 stmt, blck_size));
2041 values->safe_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_AVERAGE,
2042 stmt, blck_size));
2045 if (TREE_CODE (blck_size) != INTEGER_CST)
2046 values->safe_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_IOR,
2047 stmt, dest));
2050 /* Find values inside STMT for that we want to measure histograms and adds
2051 them to list VALUES. */
2053 static void
2054 gimple_values_to_profile (gimple *stmt, histogram_values *values)
2056 gimple_divmod_values_to_profile (stmt, values);
2057 gimple_stringops_values_to_profile (stmt, values);
2058 gimple_indirect_call_to_profile (stmt, values);
2061 void
2062 gimple_find_values_to_profile (histogram_values *values)
2064 basic_block bb;
2065 gimple_stmt_iterator gsi;
2066 unsigned i;
2067 histogram_value hist = NULL;
2068 values->create (0);
2070 FOR_EACH_BB_FN (bb, cfun)
2071 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
2072 gimple_values_to_profile (gsi_stmt (gsi), values);
2074 values->safe_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_TIME_PROFILE, 0, 0));
2076 FOR_EACH_VEC_ELT (*values, i, hist)
2078 switch (hist->type)
2080 case HIST_TYPE_INTERVAL:
2081 hist->n_counters = hist->hdata.intvl.steps + 2;
2082 break;
2084 case HIST_TYPE_POW2:
2085 hist->n_counters = 2;
2086 break;
2088 case HIST_TYPE_SINGLE_VALUE:
2089 hist->n_counters = 3;
2090 break;
2092 case HIST_TYPE_CONST_DELTA:
2093 hist->n_counters = 4;
2094 break;
2096 case HIST_TYPE_INDIR_CALL:
2097 hist->n_counters = 3;
2098 break;
2100 case HIST_TYPE_TIME_PROFILE:
2101 hist->n_counters = 1;
2102 break;
2104 case HIST_TYPE_AVERAGE:
2105 hist->n_counters = 2;
2106 break;
2108 case HIST_TYPE_IOR:
2109 hist->n_counters = 1;
2110 break;
2112 case HIST_TYPE_INDIR_CALL_TOPN:
2113 hist->n_counters = GCOV_ICALL_TOPN_NCOUNTS;
2114 break;
2116 default:
2117 gcc_unreachable ();
2119 if (dump_file)
2121 fprintf (dump_file, "Stmt ");
2122 print_gimple_stmt (dump_file, hist->hvalue.stmt, 0, TDF_SLIM);
2123 dump_histogram_value (dump_file, hist);