1 /* Branch prediction routines for the GNU compiler.
2 Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2007, 2008
3 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 [1] "Branch Prediction for Free"
24 Ball and Larus; PLDI '93.
25 [2] "Static Branch Frequency and Program Profile Analysis"
26 Wu and Larus; MICRO-27.
27 [3] "Corpus-based Static Branch Prediction"
28 Calder, Grunwald, Lindsay, Martin, Mozer, and Zorn; PLDI '95. */
33 #include "coretypes.h"
38 #include "hard-reg-set.h"
39 #include "basic-block.h"
40 #include "insn-config.h"
55 #include "tree-flow.h"
57 #include "tree-dump.h"
58 #include "tree-pass.h"
60 #include "tree-scalar-evolution.h"
62 #include "pointer-set.h"
64 /* real constants: 0, 1, 1-1/REG_BR_PROB_BASE, REG_BR_PROB_BASE,
65 1/REG_BR_PROB_BASE, 0.5, BB_FREQ_MAX. */
66 static sreal real_zero
, real_one
, real_almost_one
, real_br_prob_base
,
67 real_inv_br_prob_base
, real_one_half
, real_bb_freq_max
;
69 /* Random guesstimation given names. */
70 #define PROB_VERY_UNLIKELY (REG_BR_PROB_BASE / 100 - 1)
71 #define PROB_EVEN (REG_BR_PROB_BASE / 2)
72 #define PROB_VERY_LIKELY (REG_BR_PROB_BASE - PROB_VERY_UNLIKELY)
73 #define PROB_ALWAYS (REG_BR_PROB_BASE)
75 static void combine_predictions_for_insn (rtx
, basic_block
);
76 static void dump_prediction (FILE *, enum br_predictor
, int, basic_block
, int);
77 static void predict_paths_leading_to (basic_block
, enum br_predictor
, enum prediction
);
78 static void compute_function_frequency (void);
79 static void choose_function_section (void);
80 static bool can_predict_insn_p (const_rtx
);
82 /* Information we hold about each branch predictor.
83 Filled using information from predict.def. */
87 const char *const name
; /* Name used in the debugging dumps. */
88 const int hitrate
; /* Expected hitrate used by
89 predict_insn_def call. */
93 /* Use given predictor without Dempster-Shaffer theory if it matches
94 using first_match heuristics. */
95 #define PRED_FLAG_FIRST_MATCH 1
97 /* Recompute hitrate in percent to our representation. */
99 #define HITRATE(VAL) ((int) ((VAL) * REG_BR_PROB_BASE + 50) / 100)
101 #define DEF_PREDICTOR(ENUM, NAME, HITRATE, FLAGS) {NAME, HITRATE, FLAGS},
102 static const struct predictor_info predictor_info
[]= {
103 #include "predict.def"
105 /* Upper bound on predictors. */
110 /* Return TRUE if frequency FREQ is considered to be hot. */
112 maybe_hot_frequency_p (int freq
)
114 if (!profile_info
|| !flag_branch_probabilities
)
116 if (cfun
->function_frequency
== FUNCTION_FREQUENCY_UNLIKELY_EXECUTED
)
118 if (cfun
->function_frequency
== FUNCTION_FREQUENCY_HOT
)
121 if (freq
< BB_FREQ_MAX
/ PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION
))
126 /* Return true in case BB can be CPU intensive and should be optimized
127 for maximal performance. */
130 maybe_hot_bb_p (const_basic_block bb
)
132 if (profile_info
&& flag_branch_probabilities
134 < profile_info
->sum_max
/ PARAM_VALUE (HOT_BB_COUNT_FRACTION
)))
136 return maybe_hot_frequency_p (bb
->frequency
);
139 /* Return true in case BB can be CPU intensive and should be optimized
140 for maximal performance. */
143 maybe_hot_edge_p (edge e
)
145 if (profile_info
&& flag_branch_probabilities
147 < profile_info
->sum_max
/ PARAM_VALUE (HOT_BB_COUNT_FRACTION
)))
149 return maybe_hot_frequency_p (EDGE_FREQUENCY (e
));
152 /* Return true in case BB is cold and should be optimized for size. */
155 probably_cold_bb_p (const_basic_block bb
)
157 if (profile_info
&& flag_branch_probabilities
159 < profile_info
->sum_max
/ PARAM_VALUE (HOT_BB_COUNT_FRACTION
)))
161 if ((!profile_info
|| !flag_branch_probabilities
)
162 && cfun
->function_frequency
== FUNCTION_FREQUENCY_UNLIKELY_EXECUTED
)
164 if (bb
->frequency
< BB_FREQ_MAX
/ PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION
))
169 /* Return true in case BB is probably never executed. */
171 probably_never_executed_bb_p (const_basic_block bb
)
173 if (profile_info
&& flag_branch_probabilities
)
174 return ((bb
->count
+ profile_info
->runs
/ 2) / profile_info
->runs
) == 0;
175 if ((!profile_info
|| !flag_branch_probabilities
)
176 && cfun
->function_frequency
== FUNCTION_FREQUENCY_UNLIKELY_EXECUTED
)
181 /* Return true if the one of outgoing edges is already predicted by
185 rtl_predicted_by_p (const_basic_block bb
, enum br_predictor predictor
)
188 if (!INSN_P (BB_END (bb
)))
190 for (note
= REG_NOTES (BB_END (bb
)); note
; note
= XEXP (note
, 1))
191 if (REG_NOTE_KIND (note
) == REG_BR_PRED
192 && INTVAL (XEXP (XEXP (note
, 0), 0)) == (int)predictor
)
197 /* This map contains for a basic block the list of predictions for the
200 static struct pointer_map_t
*bb_predictions
;
202 /* Return true if the one of outgoing edges is already predicted by
206 tree_predicted_by_p (const_basic_block bb
, enum br_predictor predictor
)
208 struct edge_prediction
*i
;
209 void **preds
= pointer_map_contains (bb_predictions
, bb
);
214 for (i
= (struct edge_prediction
*) *preds
; i
; i
= i
->ep_next
)
215 if (i
->ep_predictor
== predictor
)
220 /* Return true when the probability of edge is reliable.
222 The profile guessing code is good at predicting branch outcome (ie.
223 taken/not taken), that is predicted right slightly over 75% of time.
224 It is however notoriously poor on predicting the probability itself.
225 In general the profile appear a lot flatter (with probabilities closer
226 to 50%) than the reality so it is bad idea to use it to drive optimization
227 such as those disabling dynamic branch prediction for well predictable
230 There are two exceptions - edges leading to noreturn edges and edges
231 predicted by number of iterations heuristics are predicted well. This macro
232 should be able to distinguish those, but at the moment it simply check for
233 noreturn heuristic that is only one giving probability over 99% or bellow
234 1%. In future we might want to propagate reliability information across the
235 CFG if we find this information useful on multiple places. */
237 probability_reliable_p (int prob
)
239 return (profile_status
== PROFILE_READ
240 || (profile_status
== PROFILE_GUESSED
241 && (prob
<= HITRATE (1) || prob
>= HITRATE (99))));
244 /* Same predicate as above, working on edges. */
246 edge_probability_reliable_p (const_edge e
)
248 return probability_reliable_p (e
->probability
);
251 /* Same predicate as edge_probability_reliable_p, working on notes. */
253 br_prob_note_reliable_p (const_rtx note
)
255 gcc_assert (REG_NOTE_KIND (note
) == REG_BR_PROB
);
256 return probability_reliable_p (INTVAL (XEXP (note
, 0)));
260 predict_insn (rtx insn
, enum br_predictor predictor
, int probability
)
262 gcc_assert (any_condjump_p (insn
));
263 if (!flag_guess_branch_prob
)
266 add_reg_note (insn
, REG_BR_PRED
,
267 gen_rtx_CONCAT (VOIDmode
,
268 GEN_INT ((int) predictor
),
269 GEN_INT ((int) probability
)));
272 /* Predict insn by given predictor. */
275 predict_insn_def (rtx insn
, enum br_predictor predictor
,
276 enum prediction taken
)
278 int probability
= predictor_info
[(int) predictor
].hitrate
;
281 probability
= REG_BR_PROB_BASE
- probability
;
283 predict_insn (insn
, predictor
, probability
);
286 /* Predict edge E with given probability if possible. */
289 rtl_predict_edge (edge e
, enum br_predictor predictor
, int probability
)
292 last_insn
= BB_END (e
->src
);
294 /* We can store the branch prediction information only about
295 conditional jumps. */
296 if (!any_condjump_p (last_insn
))
299 /* We always store probability of branching. */
300 if (e
->flags
& EDGE_FALLTHRU
)
301 probability
= REG_BR_PROB_BASE
- probability
;
303 predict_insn (last_insn
, predictor
, probability
);
306 /* Predict edge E with the given PROBABILITY. */
308 tree_predict_edge (edge e
, enum br_predictor predictor
, int probability
)
310 gcc_assert (profile_status
!= PROFILE_GUESSED
);
311 if ((e
->src
!= ENTRY_BLOCK_PTR
&& EDGE_COUNT (e
->src
->succs
) > 1)
312 && flag_guess_branch_prob
&& optimize
)
314 struct edge_prediction
*i
= XNEW (struct edge_prediction
);
315 void **preds
= pointer_map_insert (bb_predictions
, e
->src
);
317 i
->ep_next
= (struct edge_prediction
*) *preds
;
319 i
->ep_probability
= probability
;
320 i
->ep_predictor
= predictor
;
325 /* Remove all predictions on given basic block that are attached
328 remove_predictions_associated_with_edge (edge e
)
335 preds
= pointer_map_contains (bb_predictions
, e
->src
);
339 struct edge_prediction
**prediction
= (struct edge_prediction
**) preds
;
340 struct edge_prediction
*next
;
344 if ((*prediction
)->ep_edge
== e
)
346 next
= (*prediction
)->ep_next
;
351 prediction
= &((*prediction
)->ep_next
);
356 /* Clears the list of predictions stored for BB. */
359 clear_bb_predictions (basic_block bb
)
361 void **preds
= pointer_map_contains (bb_predictions
, bb
);
362 struct edge_prediction
*pred
, *next
;
367 for (pred
= (struct edge_prediction
*) *preds
; pred
; pred
= next
)
369 next
= pred
->ep_next
;
375 /* Return true when we can store prediction on insn INSN.
376 At the moment we represent predictions only on conditional
377 jumps, not at computed jump or other complicated cases. */
379 can_predict_insn_p (const_rtx insn
)
381 return (JUMP_P (insn
)
382 && any_condjump_p (insn
)
383 && EDGE_COUNT (BLOCK_FOR_INSN (insn
)->succs
) >= 2);
386 /* Predict edge E by given predictor if possible. */
389 predict_edge_def (edge e
, enum br_predictor predictor
,
390 enum prediction taken
)
392 int probability
= predictor_info
[(int) predictor
].hitrate
;
395 probability
= REG_BR_PROB_BASE
- probability
;
397 predict_edge (e
, predictor
, probability
);
400 /* Invert all branch predictions or probability notes in the INSN. This needs
401 to be done each time we invert the condition used by the jump. */
404 invert_br_probabilities (rtx insn
)
408 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
409 if (REG_NOTE_KIND (note
) == REG_BR_PROB
)
410 XEXP (note
, 0) = GEN_INT (REG_BR_PROB_BASE
- INTVAL (XEXP (note
, 0)));
411 else if (REG_NOTE_KIND (note
) == REG_BR_PRED
)
412 XEXP (XEXP (note
, 0), 1)
413 = GEN_INT (REG_BR_PROB_BASE
- INTVAL (XEXP (XEXP (note
, 0), 1)));
416 /* Dump information about the branch prediction to the output file. */
419 dump_prediction (FILE *file
, enum br_predictor predictor
, int probability
,
420 basic_block bb
, int used
)
428 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
429 if (! (e
->flags
& EDGE_FALLTHRU
))
432 fprintf (file
, " %s heuristics%s: %.1f%%",
433 predictor_info
[predictor
].name
,
434 used
? "" : " (ignored)", probability
* 100.0 / REG_BR_PROB_BASE
);
438 fprintf (file
, " exec ");
439 fprintf (file
, HOST_WIDEST_INT_PRINT_DEC
, bb
->count
);
442 fprintf (file
, " hit ");
443 fprintf (file
, HOST_WIDEST_INT_PRINT_DEC
, e
->count
);
444 fprintf (file
, " (%.1f%%)", e
->count
* 100.0 / bb
->count
);
448 fprintf (file
, "\n");
451 /* We can not predict the probabilities of outgoing edges of bb. Set them
452 evenly and hope for the best. */
454 set_even_probabilities (basic_block bb
)
460 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
461 if (!(e
->flags
& (EDGE_EH
| EDGE_FAKE
)))
463 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
464 if (!(e
->flags
& (EDGE_EH
| EDGE_FAKE
)))
465 e
->probability
= (REG_BR_PROB_BASE
+ nedges
/ 2) / nedges
;
470 /* Combine all REG_BR_PRED notes into single probability and attach REG_BR_PROB
471 note if not already present. Remove now useless REG_BR_PRED notes. */
474 combine_predictions_for_insn (rtx insn
, basic_block bb
)
479 int best_probability
= PROB_EVEN
;
480 int best_predictor
= END_PREDICTORS
;
481 int combined_probability
= REG_BR_PROB_BASE
/ 2;
483 bool first_match
= false;
486 if (!can_predict_insn_p (insn
))
488 set_even_probabilities (bb
);
492 prob_note
= find_reg_note (insn
, REG_BR_PROB
, 0);
493 pnote
= ®_NOTES (insn
);
495 fprintf (dump_file
, "Predictions for insn %i bb %i\n", INSN_UID (insn
),
498 /* We implement "first match" heuristics and use probability guessed
499 by predictor with smallest index. */
500 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
501 if (REG_NOTE_KIND (note
) == REG_BR_PRED
)
503 int predictor
= INTVAL (XEXP (XEXP (note
, 0), 0));
504 int probability
= INTVAL (XEXP (XEXP (note
, 0), 1));
507 if (best_predictor
> predictor
)
508 best_probability
= probability
, best_predictor
= predictor
;
510 d
= (combined_probability
* probability
511 + (REG_BR_PROB_BASE
- combined_probability
)
512 * (REG_BR_PROB_BASE
- probability
));
514 /* Use FP math to avoid overflows of 32bit integers. */
516 /* If one probability is 0% and one 100%, avoid division by zero. */
517 combined_probability
= REG_BR_PROB_BASE
/ 2;
519 combined_probability
= (((double) combined_probability
) * probability
520 * REG_BR_PROB_BASE
/ d
+ 0.5);
523 /* Decide which heuristic to use. In case we didn't match anything,
524 use no_prediction heuristic, in case we did match, use either
525 first match or Dempster-Shaffer theory depending on the flags. */
527 if (predictor_info
[best_predictor
].flags
& PRED_FLAG_FIRST_MATCH
)
531 dump_prediction (dump_file
, PRED_NO_PREDICTION
,
532 combined_probability
, bb
, true);
535 dump_prediction (dump_file
, PRED_DS_THEORY
, combined_probability
,
537 dump_prediction (dump_file
, PRED_FIRST_MATCH
, best_probability
,
542 combined_probability
= best_probability
;
543 dump_prediction (dump_file
, PRED_COMBINED
, combined_probability
, bb
, true);
547 if (REG_NOTE_KIND (*pnote
) == REG_BR_PRED
)
549 int predictor
= INTVAL (XEXP (XEXP (*pnote
, 0), 0));
550 int probability
= INTVAL (XEXP (XEXP (*pnote
, 0), 1));
552 dump_prediction (dump_file
, predictor
, probability
, bb
,
553 !first_match
|| best_predictor
== predictor
);
554 *pnote
= XEXP (*pnote
, 1);
557 pnote
= &XEXP (*pnote
, 1);
562 add_reg_note (insn
, REG_BR_PROB
, GEN_INT (combined_probability
));
564 /* Save the prediction into CFG in case we are seeing non-degenerated
566 if (!single_succ_p (bb
))
568 BRANCH_EDGE (bb
)->probability
= combined_probability
;
569 FALLTHRU_EDGE (bb
)->probability
570 = REG_BR_PROB_BASE
- combined_probability
;
573 else if (!single_succ_p (bb
))
575 int prob
= INTVAL (XEXP (prob_note
, 0));
577 BRANCH_EDGE (bb
)->probability
= prob
;
578 FALLTHRU_EDGE (bb
)->probability
= REG_BR_PROB_BASE
- prob
;
581 single_succ_edge (bb
)->probability
= REG_BR_PROB_BASE
;
584 /* Combine predictions into single probability and store them into CFG.
585 Remove now useless prediction entries. */
588 combine_predictions_for_bb (basic_block bb
)
590 int best_probability
= PROB_EVEN
;
591 int best_predictor
= END_PREDICTORS
;
592 int combined_probability
= REG_BR_PROB_BASE
/ 2;
594 bool first_match
= false;
596 struct edge_prediction
*pred
;
598 edge e
, first
= NULL
, second
= NULL
;
602 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
603 if (!(e
->flags
& (EDGE_EH
| EDGE_FAKE
)))
606 if (first
&& !second
)
612 /* When there is no successor or only one choice, prediction is easy.
614 We are lazy for now and predict only basic blocks with two outgoing
615 edges. It is possible to predict generic case too, but we have to
616 ignore first match heuristics and do more involved combining. Implement
621 set_even_probabilities (bb
);
622 clear_bb_predictions (bb
);
624 fprintf (dump_file
, "%i edges in bb %i predicted to even probabilities\n",
630 fprintf (dump_file
, "Predictions for bb %i\n", bb
->index
);
632 preds
= pointer_map_contains (bb_predictions
, bb
);
635 /* We implement "first match" heuristics and use probability guessed
636 by predictor with smallest index. */
637 for (pred
= (struct edge_prediction
*) *preds
; pred
; pred
= pred
->ep_next
)
639 int predictor
= pred
->ep_predictor
;
640 int probability
= pred
->ep_probability
;
642 if (pred
->ep_edge
!= first
)
643 probability
= REG_BR_PROB_BASE
- probability
;
646 if (best_predictor
> predictor
)
647 best_probability
= probability
, best_predictor
= predictor
;
649 d
= (combined_probability
* probability
650 + (REG_BR_PROB_BASE
- combined_probability
)
651 * (REG_BR_PROB_BASE
- probability
));
653 /* Use FP math to avoid overflows of 32bit integers. */
655 /* If one probability is 0% and one 100%, avoid division by zero. */
656 combined_probability
= REG_BR_PROB_BASE
/ 2;
658 combined_probability
= (((double) combined_probability
)
660 * REG_BR_PROB_BASE
/ d
+ 0.5);
664 /* Decide which heuristic to use. In case we didn't match anything,
665 use no_prediction heuristic, in case we did match, use either
666 first match or Dempster-Shaffer theory depending on the flags. */
668 if (predictor_info
[best_predictor
].flags
& PRED_FLAG_FIRST_MATCH
)
672 dump_prediction (dump_file
, PRED_NO_PREDICTION
, combined_probability
, bb
, true);
675 dump_prediction (dump_file
, PRED_DS_THEORY
, combined_probability
, bb
,
677 dump_prediction (dump_file
, PRED_FIRST_MATCH
, best_probability
, bb
,
682 combined_probability
= best_probability
;
683 dump_prediction (dump_file
, PRED_COMBINED
, combined_probability
, bb
, true);
687 for (pred
= (struct edge_prediction
*) *preds
; pred
; pred
= pred
->ep_next
)
689 int predictor
= pred
->ep_predictor
;
690 int probability
= pred
->ep_probability
;
692 if (pred
->ep_edge
!= EDGE_SUCC (bb
, 0))
693 probability
= REG_BR_PROB_BASE
- probability
;
694 dump_prediction (dump_file
, predictor
, probability
, bb
,
695 !first_match
|| best_predictor
== predictor
);
698 clear_bb_predictions (bb
);
702 first
->probability
= combined_probability
;
703 second
->probability
= REG_BR_PROB_BASE
- combined_probability
;
707 /* Predict edge probabilities by exploiting loop structure. */
717 /* Try to predict out blocks in a loop that are not part of a
719 FOR_EACH_LOOP (li
, loop
, 0)
721 basic_block bb
, *bbs
;
723 VEC (edge
, heap
) *exits
;
724 struct tree_niter_desc niter_desc
;
727 exits
= get_loop_exit_edges (loop
);
728 n_exits
= VEC_length (edge
, exits
);
730 for (j
= 0; VEC_iterate (edge
, exits
, j
, ex
); j
++)
733 HOST_WIDE_INT nitercst
;
734 int max
= PARAM_VALUE (PARAM_MAX_PREDICTED_ITERATIONS
);
736 enum br_predictor predictor
;
738 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false))
739 niter
= niter_desc
.niter
;
740 if (!niter
|| TREE_CODE (niter_desc
.niter
) != INTEGER_CST
)
741 niter
= loop_niter_by_eval (loop
, ex
);
743 if (TREE_CODE (niter
) == INTEGER_CST
)
745 if (host_integerp (niter
, 1)
746 && compare_tree_int (niter
, max
-1) == -1)
747 nitercst
= tree_low_cst (niter
, 1) + 1;
750 predictor
= PRED_LOOP_ITERATIONS
;
752 /* If we have just one exit and we can derive some information about
753 the number of iterations of the loop from the statements inside
754 the loop, use it to predict this exit. */
755 else if (n_exits
== 1)
757 nitercst
= estimated_loop_iterations_int (loop
, false);
763 predictor
= PRED_LOOP_ITERATIONS_GUESSED
;
768 probability
= ((REG_BR_PROB_BASE
+ nitercst
/ 2) / nitercst
);
769 predict_edge (ex
, predictor
, probability
);
771 VEC_free (edge
, heap
, exits
);
773 bbs
= get_loop_body (loop
);
775 for (j
= 0; j
< loop
->num_nodes
; j
++)
777 int header_found
= 0;
783 /* Bypass loop heuristics on continue statement. These
784 statements construct loops via "non-loop" constructs
785 in the source language and are better to be handled
787 if (predicted_by_p (bb
, PRED_CONTINUE
))
790 /* Loop branch heuristics - predict an edge back to a
791 loop's head as taken. */
792 if (bb
== loop
->latch
)
794 e
= find_edge (loop
->latch
, loop
->header
);
798 predict_edge_def (e
, PRED_LOOP_BRANCH
, TAKEN
);
802 /* Loop exit heuristics - predict an edge exiting the loop if the
803 conditional has no loop header successors as not taken. */
805 /* If we already used more reliable loop exit predictors, do not
806 bother with PRED_LOOP_EXIT. */
807 && !predicted_by_p (bb
, PRED_LOOP_ITERATIONS_GUESSED
)
808 && !predicted_by_p (bb
, PRED_LOOP_ITERATIONS
))
810 /* For loop with many exits we don't want to predict all exits
811 with the pretty large probability, because if all exits are
812 considered in row, the loop would be predicted to iterate
813 almost never. The code to divide probability by number of
814 exits is very rough. It should compute the number of exits
815 taken in each patch through function (not the overall number
816 of exits that might be a lot higher for loops with wide switch
817 statements in them) and compute n-th square root.
819 We limit the minimal probability by 2% to avoid
820 EDGE_PROBABILITY_RELIABLE from trusting the branch prediction
821 as this was causing regression in perl benchmark containing such
824 int probability
= ((REG_BR_PROB_BASE
825 - predictor_info
[(int) PRED_LOOP_EXIT
].hitrate
)
827 if (probability
< HITRATE (2))
828 probability
= HITRATE (2);
829 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
830 if (e
->dest
->index
< NUM_FIXED_BLOCKS
831 || !flow_bb_inside_loop_p (loop
, e
->dest
))
832 predict_edge (e
, PRED_LOOP_EXIT
, probability
);
836 /* Free basic blocks from get_loop_body. */
843 /* Attempt to predict probabilities of BB outgoing edges using local
846 bb_estimate_probability_locally (basic_block bb
)
848 rtx last_insn
= BB_END (bb
);
851 if (! can_predict_insn_p (last_insn
))
853 cond
= get_condition (last_insn
, NULL
, false, false);
857 /* Try "pointer heuristic."
858 A comparison ptr == 0 is predicted as false.
859 Similarly, a comparison ptr1 == ptr2 is predicted as false. */
860 if (COMPARISON_P (cond
)
861 && ((REG_P (XEXP (cond
, 0)) && REG_POINTER (XEXP (cond
, 0)))
862 || (REG_P (XEXP (cond
, 1)) && REG_POINTER (XEXP (cond
, 1)))))
864 if (GET_CODE (cond
) == EQ
)
865 predict_insn_def (last_insn
, PRED_POINTER
, NOT_TAKEN
);
866 else if (GET_CODE (cond
) == NE
)
867 predict_insn_def (last_insn
, PRED_POINTER
, TAKEN
);
871 /* Try "opcode heuristic."
872 EQ tests are usually false and NE tests are usually true. Also,
873 most quantities are positive, so we can make the appropriate guesses
874 about signed comparisons against zero. */
875 switch (GET_CODE (cond
))
878 /* Unconditional branch. */
879 predict_insn_def (last_insn
, PRED_UNCONDITIONAL
,
880 cond
== const0_rtx
? NOT_TAKEN
: TAKEN
);
885 /* Floating point comparisons appears to behave in a very
886 unpredictable way because of special role of = tests in
888 if (FLOAT_MODE_P (GET_MODE (XEXP (cond
, 0))))
890 /* Comparisons with 0 are often used for booleans and there is
891 nothing useful to predict about them. */
892 else if (XEXP (cond
, 1) == const0_rtx
893 || XEXP (cond
, 0) == const0_rtx
)
896 predict_insn_def (last_insn
, PRED_OPCODE_NONEQUAL
, NOT_TAKEN
);
901 /* Floating point comparisons appears to behave in a very
902 unpredictable way because of special role of = tests in
904 if (FLOAT_MODE_P (GET_MODE (XEXP (cond
, 0))))
906 /* Comparisons with 0 are often used for booleans and there is
907 nothing useful to predict about them. */
908 else if (XEXP (cond
, 1) == const0_rtx
909 || XEXP (cond
, 0) == const0_rtx
)
912 predict_insn_def (last_insn
, PRED_OPCODE_NONEQUAL
, TAKEN
);
916 predict_insn_def (last_insn
, PRED_FPOPCODE
, TAKEN
);
920 predict_insn_def (last_insn
, PRED_FPOPCODE
, NOT_TAKEN
);
925 if (XEXP (cond
, 1) == const0_rtx
|| XEXP (cond
, 1) == const1_rtx
926 || XEXP (cond
, 1) == constm1_rtx
)
927 predict_insn_def (last_insn
, PRED_OPCODE_POSITIVE
, NOT_TAKEN
);
932 if (XEXP (cond
, 1) == const0_rtx
|| XEXP (cond
, 1) == const1_rtx
933 || XEXP (cond
, 1) == constm1_rtx
)
934 predict_insn_def (last_insn
, PRED_OPCODE_POSITIVE
, TAKEN
);
942 /* Set edge->probability for each successor edge of BB. */
944 guess_outgoing_edge_probabilities (basic_block bb
)
946 bb_estimate_probability_locally (bb
);
947 combine_predictions_for_insn (BB_END (bb
), bb
);
950 /* Return constant EXPR will likely have at execution time, NULL if unknown.
951 The function is used by builtin_expect branch predictor so the evidence
952 must come from this construct and additional possible constant folding.
954 We may want to implement more involved value guess (such as value range
955 propagation based prediction), but such tricks shall go to new
959 expr_expected_value (tree expr
, bitmap visited
)
961 if (TREE_CONSTANT (expr
))
963 else if (TREE_CODE (expr
) == SSA_NAME
)
965 tree def
= SSA_NAME_DEF_STMT (expr
);
967 /* If we were already here, break the infinite cycle. */
968 if (bitmap_bit_p (visited
, SSA_NAME_VERSION (expr
)))
970 bitmap_set_bit (visited
, SSA_NAME_VERSION (expr
));
972 if (TREE_CODE (def
) == PHI_NODE
)
974 /* All the arguments of the PHI node must have the same constant
977 tree val
= NULL
, new_val
;
979 for (i
= 0; i
< PHI_NUM_ARGS (def
); i
++)
981 tree arg
= PHI_ARG_DEF (def
, i
);
983 /* If this PHI has itself as an argument, we cannot
984 determine the string length of this argument. However,
985 if we can find an expected constant value for the other
986 PHI args then we can still be sure that this is
987 likely a constant. So be optimistic and just
988 continue with the next argument. */
989 if (arg
== PHI_RESULT (def
))
992 new_val
= expr_expected_value (arg
, visited
);
997 else if (!operand_equal_p (val
, new_val
, false))
1002 if (TREE_CODE (def
) != GIMPLE_MODIFY_STMT
1003 || GIMPLE_STMT_OPERAND (def
, 0) != expr
)
1005 return expr_expected_value (GIMPLE_STMT_OPERAND (def
, 1), visited
);
1007 else if (TREE_CODE (expr
) == CALL_EXPR
)
1009 tree decl
= get_callee_fndecl (expr
);
1012 if (DECL_BUILT_IN_CLASS (decl
) == BUILT_IN_NORMAL
1013 && DECL_FUNCTION_CODE (decl
) == BUILT_IN_EXPECT
)
1017 if (call_expr_nargs (expr
) != 2)
1019 val
= CALL_EXPR_ARG (expr
, 0);
1020 if (TREE_CONSTANT (val
))
1022 return CALL_EXPR_ARG (expr
, 1);
1025 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1028 op0
= expr_expected_value (TREE_OPERAND (expr
, 0), visited
);
1031 op1
= expr_expected_value (TREE_OPERAND (expr
, 1), visited
);
1034 res
= fold_build2 (TREE_CODE (expr
), TREE_TYPE (expr
), op0
, op1
);
1035 if (TREE_CONSTANT (res
))
1039 if (UNARY_CLASS_P (expr
))
1042 op0
= expr_expected_value (TREE_OPERAND (expr
, 0), visited
);
1045 res
= fold_build1 (TREE_CODE (expr
), TREE_TYPE (expr
), op0
);
1046 if (TREE_CONSTANT (res
))
1053 /* Get rid of all builtin_expect calls we no longer need. */
1055 strip_builtin_expect (void)
1060 block_stmt_iterator bi
;
1061 for (bi
= bsi_start (bb
); !bsi_end_p (bi
); bsi_next (&bi
))
1063 tree stmt
= bsi_stmt (bi
);
1067 if (TREE_CODE (stmt
) == GIMPLE_MODIFY_STMT
1068 && (call
= GIMPLE_STMT_OPERAND (stmt
, 1))
1069 && TREE_CODE (call
) == CALL_EXPR
1070 && (fndecl
= get_callee_fndecl (call
))
1071 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
1072 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_EXPECT
1073 && call_expr_nargs (call
) == 2)
1075 GIMPLE_STMT_OPERAND (stmt
, 1) = CALL_EXPR_ARG (call
, 0);
1082 /* Predict using opcode of the last statement in basic block. */
1084 tree_predict_by_opcode (basic_block bb
)
1086 tree stmt
= last_stmt (bb
);
1095 if (!stmt
|| TREE_CODE (stmt
) != COND_EXPR
)
1097 FOR_EACH_EDGE (then_edge
, ei
, bb
->succs
)
1098 if (then_edge
->flags
& EDGE_TRUE_VALUE
)
1100 cond
= TREE_OPERAND (stmt
, 0);
1101 if (!COMPARISON_CLASS_P (cond
))
1103 op0
= TREE_OPERAND (cond
, 0);
1104 type
= TREE_TYPE (op0
);
1105 visited
= BITMAP_ALLOC (NULL
);
1106 val
= expr_expected_value (cond
, visited
);
1107 BITMAP_FREE (visited
);
1110 if (integer_zerop (val
))
1111 predict_edge_def (then_edge
, PRED_BUILTIN_EXPECT
, NOT_TAKEN
);
1113 predict_edge_def (then_edge
, PRED_BUILTIN_EXPECT
, TAKEN
);
1116 /* Try "pointer heuristic."
1117 A comparison ptr == 0 is predicted as false.
1118 Similarly, a comparison ptr1 == ptr2 is predicted as false. */
1119 if (POINTER_TYPE_P (type
))
1121 if (TREE_CODE (cond
) == EQ_EXPR
)
1122 predict_edge_def (then_edge
, PRED_TREE_POINTER
, NOT_TAKEN
);
1123 else if (TREE_CODE (cond
) == NE_EXPR
)
1124 predict_edge_def (then_edge
, PRED_TREE_POINTER
, TAKEN
);
1128 /* Try "opcode heuristic."
1129 EQ tests are usually false and NE tests are usually true. Also,
1130 most quantities are positive, so we can make the appropriate guesses
1131 about signed comparisons against zero. */
1132 switch (TREE_CODE (cond
))
1136 /* Floating point comparisons appears to behave in a very
1137 unpredictable way because of special role of = tests in
1139 if (FLOAT_TYPE_P (type
))
1141 /* Comparisons with 0 are often used for booleans and there is
1142 nothing useful to predict about them. */
1143 else if (integer_zerop (op0
)
1144 || integer_zerop (TREE_OPERAND (cond
, 1)))
1147 predict_edge_def (then_edge
, PRED_TREE_OPCODE_NONEQUAL
, NOT_TAKEN
);
1152 /* Floating point comparisons appears to behave in a very
1153 unpredictable way because of special role of = tests in
1155 if (FLOAT_TYPE_P (type
))
1157 /* Comparisons with 0 are often used for booleans and there is
1158 nothing useful to predict about them. */
1159 else if (integer_zerop (op0
)
1160 || integer_zerop (TREE_OPERAND (cond
, 1)))
1163 predict_edge_def (then_edge
, PRED_TREE_OPCODE_NONEQUAL
, TAKEN
);
1167 predict_edge_def (then_edge
, PRED_TREE_FPOPCODE
, TAKEN
);
1170 case UNORDERED_EXPR
:
1171 predict_edge_def (then_edge
, PRED_TREE_FPOPCODE
, NOT_TAKEN
);
1176 if (integer_zerop (TREE_OPERAND (cond
, 1))
1177 || integer_onep (TREE_OPERAND (cond
, 1))
1178 || integer_all_onesp (TREE_OPERAND (cond
, 1))
1179 || real_zerop (TREE_OPERAND (cond
, 1))
1180 || real_onep (TREE_OPERAND (cond
, 1))
1181 || real_minus_onep (TREE_OPERAND (cond
, 1)))
1182 predict_edge_def (then_edge
, PRED_TREE_OPCODE_POSITIVE
, NOT_TAKEN
);
1187 if (integer_zerop (TREE_OPERAND (cond
, 1))
1188 || integer_onep (TREE_OPERAND (cond
, 1))
1189 || integer_all_onesp (TREE_OPERAND (cond
, 1))
1190 || real_zerop (TREE_OPERAND (cond
, 1))
1191 || real_onep (TREE_OPERAND (cond
, 1))
1192 || real_minus_onep (TREE_OPERAND (cond
, 1)))
1193 predict_edge_def (then_edge
, PRED_TREE_OPCODE_POSITIVE
, TAKEN
);
1201 /* Try to guess whether the value of return means error code. */
1202 static enum br_predictor
1203 return_prediction (tree val
, enum prediction
*prediction
)
1207 return PRED_NO_PREDICTION
;
1208 /* Different heuristics for pointers and scalars. */
1209 if (POINTER_TYPE_P (TREE_TYPE (val
)))
1211 /* NULL is usually not returned. */
1212 if (integer_zerop (val
))
1214 *prediction
= NOT_TAKEN
;
1215 return PRED_NULL_RETURN
;
1218 else if (INTEGRAL_TYPE_P (TREE_TYPE (val
)))
1220 /* Negative return values are often used to indicate
1222 if (TREE_CODE (val
) == INTEGER_CST
1223 && tree_int_cst_sgn (val
) < 0)
1225 *prediction
= NOT_TAKEN
;
1226 return PRED_NEGATIVE_RETURN
;
1228 /* Constant return values seems to be commonly taken.
1229 Zero/one often represent booleans so exclude them from the
1231 if (TREE_CONSTANT (val
)
1232 && (!integer_zerop (val
) && !integer_onep (val
)))
1234 *prediction
= TAKEN
;
1235 return PRED_CONST_RETURN
;
1238 return PRED_NO_PREDICTION
;
1241 /* Find the basic block with return expression and look up for possible
1242 return value trying to apply RETURN_PREDICTION heuristics. */
1244 apply_return_prediction (void)
1246 tree return_stmt
= NULL
;
1250 int phi_num_args
, i
;
1251 enum br_predictor pred
;
1252 enum prediction direction
;
1255 FOR_EACH_EDGE (e
, ei
, EXIT_BLOCK_PTR
->preds
)
1257 return_stmt
= last_stmt (e
->src
);
1259 && TREE_CODE (return_stmt
) == RETURN_EXPR
)
1264 return_val
= TREE_OPERAND (return_stmt
, 0);
1267 if (TREE_CODE (return_val
) == GIMPLE_MODIFY_STMT
)
1268 return_val
= GIMPLE_STMT_OPERAND (return_val
, 1);
1269 if (TREE_CODE (return_val
) != SSA_NAME
1270 || !SSA_NAME_DEF_STMT (return_val
)
1271 || TREE_CODE (SSA_NAME_DEF_STMT (return_val
)) != PHI_NODE
)
1273 for (phi
= SSA_NAME_DEF_STMT (return_val
); phi
; phi
= PHI_CHAIN (phi
))
1274 if (PHI_RESULT (phi
) == return_val
)
1278 phi_num_args
= PHI_NUM_ARGS (phi
);
1279 pred
= return_prediction (PHI_ARG_DEF (phi
, 0), &direction
);
1281 /* Avoid the degenerate case where all return values form the function
1282 belongs to same category (ie they are all positive constants)
1283 so we can hardly say something about them. */
1284 for (i
= 1; i
< phi_num_args
; i
++)
1285 if (pred
!= return_prediction (PHI_ARG_DEF (phi
, i
), &direction
))
1287 if (i
!= phi_num_args
)
1288 for (i
= 0; i
< phi_num_args
; i
++)
1290 pred
= return_prediction (PHI_ARG_DEF (phi
, i
), &direction
);
1291 if (pred
!= PRED_NO_PREDICTION
)
1292 predict_paths_leading_to (PHI_ARG_EDGE (phi
, i
)->src
, pred
,
1297 /* Look for basic block that contains unlikely to happen events
1298 (such as noreturn calls) and mark all paths leading to execution
1299 of this basic blocks as unlikely. */
1302 tree_bb_level_predictions (void)
1306 apply_return_prediction ();
1310 block_stmt_iterator bsi
= bsi_last (bb
);
1312 for (bsi
= bsi_start (bb
); !bsi_end_p (bsi
);)
1314 tree stmt
= bsi_stmt (bsi
);
1318 switch (TREE_CODE (stmt
))
1320 case GIMPLE_MODIFY_STMT
:
1321 if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt
, 1)) == CALL_EXPR
)
1323 stmt
= GIMPLE_STMT_OPERAND (stmt
, 1);
1329 if (call_expr_flags (stmt
) & ECF_NORETURN
)
1330 predict_paths_leading_to (bb
, PRED_NORETURN
,
1332 decl
= get_callee_fndecl (stmt
);
1334 && lookup_attribute ("cold",
1335 DECL_ATTRIBUTES (decl
)))
1336 predict_paths_leading_to (bb
, PRED_COLD_FUNCTION
,
1340 predict_paths_leading_to (bb
, PREDICT_EXPR_PREDICTOR (stmt
),
1341 PREDICT_EXPR_OUTCOME (stmt
));
1342 bsi_remove (&bsi
, true);
1354 #ifdef ENABLE_CHECKING
1356 /* Callback for pointer_map_traverse, asserts that the pointer map is
1360 assert_is_empty (const void *key ATTRIBUTE_UNUSED
, void **value
,
1361 void *data ATTRIBUTE_UNUSED
)
1363 gcc_assert (!*value
);
1368 /* Predict branch probabilities and estimate profile of the tree CFG. */
1370 tree_estimate_probability (void)
1374 loop_optimizer_init (0);
1375 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1376 flow_loops_dump (dump_file
, NULL
, 0);
1378 add_noreturn_fake_exit_edges ();
1379 connect_infinite_loops_to_exit ();
1380 /* We use loop_niter_by_eval, which requires that the loops have
1382 create_preheaders (CP_SIMPLE_PREHEADERS
);
1383 calculate_dominance_info (CDI_POST_DOMINATORS
);
1385 bb_predictions
= pointer_map_create ();
1386 tree_bb_level_predictions ();
1388 mark_irreducible_loops ();
1389 record_loop_exits ();
1390 if (number_of_loops () > 1)
1398 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1400 /* Predict early returns to be probable, as we've already taken
1401 care for error returns and other cases are often used for
1402 fast paths through function.
1404 Since we've already removed the return statements, we are
1405 looking for CFG like:
1415 if (e
->dest
!= bb
->next_bb
1416 && e
->dest
!= EXIT_BLOCK_PTR
1417 && single_succ_p (e
->dest
)
1418 && single_succ_edge (e
->dest
)->dest
== EXIT_BLOCK_PTR
1419 && TREE_CODE (last_stmt (e
->dest
)) == RETURN_EXPR
)
1424 if (single_succ_p (bb
))
1426 FOR_EACH_EDGE (e1
, ei1
, bb
->preds
)
1427 if (!predicted_by_p (e1
->src
, PRED_NULL_RETURN
)
1428 && !predicted_by_p (e1
->src
, PRED_CONST_RETURN
)
1429 && !predicted_by_p (e1
->src
, PRED_NEGATIVE_RETURN
))
1430 predict_edge_def (e1
, PRED_TREE_EARLY_RETURN
, NOT_TAKEN
);
1433 if (!predicted_by_p (e
->src
, PRED_NULL_RETURN
)
1434 && !predicted_by_p (e
->src
, PRED_CONST_RETURN
)
1435 && !predicted_by_p (e
->src
, PRED_NEGATIVE_RETURN
))
1436 predict_edge_def (e
, PRED_TREE_EARLY_RETURN
, NOT_TAKEN
);
1439 /* Look for block we are guarding (ie we dominate it,
1440 but it doesn't postdominate us). */
1441 if (e
->dest
!= EXIT_BLOCK_PTR
&& e
->dest
!= bb
1442 && dominated_by_p (CDI_DOMINATORS
, e
->dest
, e
->src
)
1443 && !dominated_by_p (CDI_POST_DOMINATORS
, e
->src
, e
->dest
))
1445 block_stmt_iterator bi
;
1447 /* The call heuristic claims that a guarded function call
1448 is improbable. This is because such calls are often used
1449 to signal exceptional situations such as printing error
1451 for (bi
= bsi_start (e
->dest
); !bsi_end_p (bi
);
1454 tree stmt
= bsi_stmt (bi
);
1455 if ((TREE_CODE (stmt
) == CALL_EXPR
1456 || (TREE_CODE (stmt
) == GIMPLE_MODIFY_STMT
1457 && TREE_CODE (GIMPLE_STMT_OPERAND (stmt
, 1))
1459 /* Constant and pure calls are hardly used to signalize
1460 something exceptional. */
1461 && TREE_SIDE_EFFECTS (stmt
))
1463 predict_edge_def (e
, PRED_CALL
, NOT_TAKEN
);
1469 tree_predict_by_opcode (bb
);
1472 combine_predictions_for_bb (bb
);
1474 #ifdef ENABLE_CHECKING
1475 pointer_map_traverse (bb_predictions
, assert_is_empty
, NULL
);
1477 pointer_map_destroy (bb_predictions
);
1478 bb_predictions
= NULL
;
1480 strip_builtin_expect ();
1481 estimate_bb_frequencies ();
1482 free_dominance_info (CDI_POST_DOMINATORS
);
1483 remove_fake_exit_edges ();
1484 loop_optimizer_finalize ();
1485 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1486 dump_tree_cfg (dump_file
, dump_flags
);
1487 if (profile_status
== PROFILE_ABSENT
)
1488 profile_status
= PROFILE_GUESSED
;
1492 /* Predict edges to successors of CUR whose sources are not postdominated by
1493 BB by PRED and recurse to all postdominators. */
1496 predict_paths_for_bb (basic_block cur
, basic_block bb
,
1497 enum br_predictor pred
,
1498 enum prediction taken
)
1504 /* We are looking for all edges forming edge cut induced by
1505 set of all blocks postdominated by BB. */
1506 FOR_EACH_EDGE (e
, ei
, cur
->preds
)
1507 if (e
->src
->index
>= NUM_FIXED_BLOCKS
1508 && !dominated_by_p (CDI_POST_DOMINATORS
, e
->src
, bb
))
1510 gcc_assert (bb
== cur
|| dominated_by_p (CDI_POST_DOMINATORS
, cur
, bb
));
1511 predict_edge_def (e
, pred
, taken
);
1513 for (son
= first_dom_son (CDI_POST_DOMINATORS
, cur
);
1515 son
= next_dom_son (CDI_POST_DOMINATORS
, son
))
1516 predict_paths_for_bb (son
, bb
, pred
, taken
);
1519 /* Sets branch probabilities according to PREDiction and
1523 predict_paths_leading_to (basic_block bb
, enum br_predictor pred
,
1524 enum prediction taken
)
1526 predict_paths_for_bb (bb
, bb
, pred
, taken
);
1529 /* This is used to carry information about basic blocks. It is
1530 attached to the AUX field of the standard CFG block. */
1532 typedef struct block_info_def
1534 /* Estimated frequency of execution of basic_block. */
1537 /* To keep queue of basic blocks to process. */
1540 /* Number of predecessors we need to visit first. */
1544 /* Similar information for edges. */
1545 typedef struct edge_info_def
1547 /* In case edge is a loopback edge, the probability edge will be reached
1548 in case header is. Estimated number of iterations of the loop can be
1549 then computed as 1 / (1 - back_edge_prob). */
1550 sreal back_edge_prob
;
1551 /* True if the edge is a loopback edge in the natural loop. */
1552 unsigned int back_edge
:1;
1555 #define BLOCK_INFO(B) ((block_info) (B)->aux)
1556 #define EDGE_INFO(E) ((edge_info) (E)->aux)
1558 /* Helper function for estimate_bb_frequencies.
1559 Propagate the frequencies in blocks marked in
1560 TOVISIT, starting in HEAD. */
1563 propagate_freq (basic_block head
, bitmap tovisit
)
1572 /* For each basic block we need to visit count number of his predecessors
1573 we need to visit first. */
1574 EXECUTE_IF_SET_IN_BITMAP (tovisit
, 0, i
, bi
)
1579 /* The outermost "loop" includes the exit block, which we can not
1580 look up via BASIC_BLOCK. Detect this and use EXIT_BLOCK_PTR
1581 directly. Do the same for the entry block. */
1582 bb
= BASIC_BLOCK (i
);
1584 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1586 bool visit
= bitmap_bit_p (tovisit
, e
->src
->index
);
1588 if (visit
&& !(e
->flags
& EDGE_DFS_BACK
))
1590 else if (visit
&& dump_file
&& !EDGE_INFO (e
)->back_edge
)
1592 "Irreducible region hit, ignoring edge to %i->%i\n",
1593 e
->src
->index
, bb
->index
);
1595 BLOCK_INFO (bb
)->npredecessors
= count
;
1598 memcpy (&BLOCK_INFO (head
)->frequency
, &real_one
, sizeof (real_one
));
1600 for (bb
= head
; bb
; bb
= nextbb
)
1603 sreal cyclic_probability
, frequency
;
1605 memcpy (&cyclic_probability
, &real_zero
, sizeof (real_zero
));
1606 memcpy (&frequency
, &real_zero
, sizeof (real_zero
));
1608 nextbb
= BLOCK_INFO (bb
)->next
;
1609 BLOCK_INFO (bb
)->next
= NULL
;
1611 /* Compute frequency of basic block. */
1614 #ifdef ENABLE_CHECKING
1615 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1616 gcc_assert (!bitmap_bit_p (tovisit
, e
->src
->index
)
1617 || (e
->flags
& EDGE_DFS_BACK
));
1620 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1621 if (EDGE_INFO (e
)->back_edge
)
1623 sreal_add (&cyclic_probability
, &cyclic_probability
,
1624 &EDGE_INFO (e
)->back_edge_prob
);
1626 else if (!(e
->flags
& EDGE_DFS_BACK
))
1630 /* frequency += (e->probability
1631 * BLOCK_INFO (e->src)->frequency /
1632 REG_BR_PROB_BASE); */
1634 sreal_init (&tmp
, e
->probability
, 0);
1635 sreal_mul (&tmp
, &tmp
, &BLOCK_INFO (e
->src
)->frequency
);
1636 sreal_mul (&tmp
, &tmp
, &real_inv_br_prob_base
);
1637 sreal_add (&frequency
, &frequency
, &tmp
);
1640 if (sreal_compare (&cyclic_probability
, &real_zero
) == 0)
1642 memcpy (&BLOCK_INFO (bb
)->frequency
, &frequency
,
1643 sizeof (frequency
));
1647 if (sreal_compare (&cyclic_probability
, &real_almost_one
) > 0)
1649 memcpy (&cyclic_probability
, &real_almost_one
,
1650 sizeof (real_almost_one
));
1653 /* BLOCK_INFO (bb)->frequency = frequency
1654 / (1 - cyclic_probability) */
1656 sreal_sub (&cyclic_probability
, &real_one
, &cyclic_probability
);
1657 sreal_div (&BLOCK_INFO (bb
)->frequency
,
1658 &frequency
, &cyclic_probability
);
1662 bitmap_clear_bit (tovisit
, bb
->index
);
1664 e
= find_edge (bb
, head
);
1669 /* EDGE_INFO (e)->back_edge_prob
1670 = ((e->probability * BLOCK_INFO (bb)->frequency)
1671 / REG_BR_PROB_BASE); */
1673 sreal_init (&tmp
, e
->probability
, 0);
1674 sreal_mul (&tmp
, &tmp
, &BLOCK_INFO (bb
)->frequency
);
1675 sreal_mul (&EDGE_INFO (e
)->back_edge_prob
,
1676 &tmp
, &real_inv_br_prob_base
);
1679 /* Propagate to successor blocks. */
1680 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1681 if (!(e
->flags
& EDGE_DFS_BACK
)
1682 && BLOCK_INFO (e
->dest
)->npredecessors
)
1684 BLOCK_INFO (e
->dest
)->npredecessors
--;
1685 if (!BLOCK_INFO (e
->dest
)->npredecessors
)
1690 BLOCK_INFO (last
)->next
= e
->dest
;
1698 /* Estimate probabilities of loopback edges in loops at same nest level. */
1701 estimate_loops_at_level (struct loop
*first_loop
)
1705 for (loop
= first_loop
; loop
; loop
= loop
->next
)
1710 bitmap tovisit
= BITMAP_ALLOC (NULL
);
1712 estimate_loops_at_level (loop
->inner
);
1714 /* Find current loop back edge and mark it. */
1715 e
= loop_latch_edge (loop
);
1716 EDGE_INFO (e
)->back_edge
= 1;
1718 bbs
= get_loop_body (loop
);
1719 for (i
= 0; i
< loop
->num_nodes
; i
++)
1720 bitmap_set_bit (tovisit
, bbs
[i
]->index
);
1722 propagate_freq (loop
->header
, tovisit
);
1723 BITMAP_FREE (tovisit
);
1727 /* Propagates frequencies through structure of loops. */
1730 estimate_loops (void)
1732 bitmap tovisit
= BITMAP_ALLOC (NULL
);
1735 /* Start by estimating the frequencies in the loops. */
1736 if (number_of_loops () > 1)
1737 estimate_loops_at_level (current_loops
->tree_root
->inner
);
1739 /* Now propagate the frequencies through all the blocks. */
1742 bitmap_set_bit (tovisit
, bb
->index
);
1744 propagate_freq (ENTRY_BLOCK_PTR
, tovisit
);
1745 BITMAP_FREE (tovisit
);
1748 /* Convert counts measured by profile driven feedback to frequencies.
1749 Return nonzero iff there was any nonzero execution count. */
1752 counts_to_freqs (void)
1754 gcov_type count_max
, true_count_max
= 0;
1758 true_count_max
= MAX (bb
->count
, true_count_max
);
1760 count_max
= MAX (true_count_max
, 1);
1761 FOR_BB_BETWEEN (bb
, ENTRY_BLOCK_PTR
, NULL
, next_bb
)
1762 bb
->frequency
= (bb
->count
* BB_FREQ_MAX
+ count_max
/ 2) / count_max
;
1764 return true_count_max
;
1767 /* Return true if function is likely to be expensive, so there is no point to
1768 optimize performance of prologue, epilogue or do inlining at the expense
1769 of code size growth. THRESHOLD is the limit of number of instructions
1770 function can execute at average to be still considered not expensive. */
1773 expensive_function_p (int threshold
)
1775 unsigned int sum
= 0;
1779 /* We can not compute accurately for large thresholds due to scaled
1781 gcc_assert (threshold
<= BB_FREQ_MAX
);
1783 /* Frequencies are out of range. This either means that function contains
1784 internal loop executing more than BB_FREQ_MAX times or profile feedback
1785 is available and function has not been executed at all. */
1786 if (ENTRY_BLOCK_PTR
->frequency
== 0)
1789 /* Maximally BB_FREQ_MAX^2 so overflow won't happen. */
1790 limit
= ENTRY_BLOCK_PTR
->frequency
* threshold
;
1795 for (insn
= BB_HEAD (bb
); insn
!= NEXT_INSN (BB_END (bb
));
1796 insn
= NEXT_INSN (insn
))
1797 if (active_insn_p (insn
))
1799 sum
+= bb
->frequency
;
1808 /* Estimate basic blocks frequency by given branch probabilities. */
1811 estimate_bb_frequencies (void)
1816 if (!flag_branch_probabilities
|| !counts_to_freqs ())
1818 static int real_values_initialized
= 0;
1820 if (!real_values_initialized
)
1822 real_values_initialized
= 1;
1823 sreal_init (&real_zero
, 0, 0);
1824 sreal_init (&real_one
, 1, 0);
1825 sreal_init (&real_br_prob_base
, REG_BR_PROB_BASE
, 0);
1826 sreal_init (&real_bb_freq_max
, BB_FREQ_MAX
, 0);
1827 sreal_init (&real_one_half
, 1, -1);
1828 sreal_div (&real_inv_br_prob_base
, &real_one
, &real_br_prob_base
);
1829 sreal_sub (&real_almost_one
, &real_one
, &real_inv_br_prob_base
);
1832 mark_dfs_back_edges ();
1834 single_succ_edge (ENTRY_BLOCK_PTR
)->probability
= REG_BR_PROB_BASE
;
1836 /* Set up block info for each basic block. */
1837 alloc_aux_for_blocks (sizeof (struct block_info_def
));
1838 alloc_aux_for_edges (sizeof (struct edge_info_def
));
1839 FOR_BB_BETWEEN (bb
, ENTRY_BLOCK_PTR
, NULL
, next_bb
)
1844 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1846 sreal_init (&EDGE_INFO (e
)->back_edge_prob
, e
->probability
, 0);
1847 sreal_mul (&EDGE_INFO (e
)->back_edge_prob
,
1848 &EDGE_INFO (e
)->back_edge_prob
,
1849 &real_inv_br_prob_base
);
1853 /* First compute probabilities locally for each loop from innermost
1854 to outermost to examine probabilities for back edges. */
1857 memcpy (&freq_max
, &real_zero
, sizeof (real_zero
));
1859 if (sreal_compare (&freq_max
, &BLOCK_INFO (bb
)->frequency
) < 0)
1860 memcpy (&freq_max
, &BLOCK_INFO (bb
)->frequency
, sizeof (freq_max
));
1862 sreal_div (&freq_max
, &real_bb_freq_max
, &freq_max
);
1863 FOR_BB_BETWEEN (bb
, ENTRY_BLOCK_PTR
, NULL
, next_bb
)
1867 sreal_mul (&tmp
, &BLOCK_INFO (bb
)->frequency
, &freq_max
);
1868 sreal_add (&tmp
, &tmp
, &real_one_half
);
1869 bb
->frequency
= sreal_to_int (&tmp
);
1872 free_aux_for_blocks ();
1873 free_aux_for_edges ();
1875 compute_function_frequency ();
1876 if (flag_reorder_functions
)
1877 choose_function_section ();
1880 /* Decide whether function is hot, cold or unlikely executed. */
1882 compute_function_frequency (void)
1886 if (!profile_info
|| !flag_branch_probabilities
)
1888 if (lookup_attribute ("cold", DECL_ATTRIBUTES (current_function_decl
))
1890 cfun
->function_frequency
= FUNCTION_FREQUENCY_UNLIKELY_EXECUTED
;
1891 else if (lookup_attribute ("hot", DECL_ATTRIBUTES (current_function_decl
))
1893 cfun
->function_frequency
= FUNCTION_FREQUENCY_HOT
;
1896 cfun
->function_frequency
= FUNCTION_FREQUENCY_UNLIKELY_EXECUTED
;
1899 if (maybe_hot_bb_p (bb
))
1901 cfun
->function_frequency
= FUNCTION_FREQUENCY_HOT
;
1904 if (!probably_never_executed_bb_p (bb
))
1905 cfun
->function_frequency
= FUNCTION_FREQUENCY_NORMAL
;
1909 /* Choose appropriate section for the function. */
1911 choose_function_section (void)
1913 if (DECL_SECTION_NAME (current_function_decl
)
1914 || !targetm
.have_named_sections
1915 /* Theoretically we can split the gnu.linkonce text section too,
1916 but this requires more work as the frequency needs to match
1917 for all generated objects so we need to merge the frequency
1918 of all instances. For now just never set frequency for these. */
1919 || DECL_ONE_ONLY (current_function_decl
))
1922 /* If we are doing the partitioning optimization, let the optimization
1923 choose the correct section into which to put things. */
1925 if (flag_reorder_blocks_and_partition
)
1928 if (cfun
->function_frequency
== FUNCTION_FREQUENCY_HOT
)
1929 DECL_SECTION_NAME (current_function_decl
) =
1930 build_string (strlen (HOT_TEXT_SECTION_NAME
), HOT_TEXT_SECTION_NAME
);
1931 if (cfun
->function_frequency
== FUNCTION_FREQUENCY_UNLIKELY_EXECUTED
)
1932 DECL_SECTION_NAME (current_function_decl
) =
1933 build_string (strlen (UNLIKELY_EXECUTED_TEXT_SECTION_NAME
),
1934 UNLIKELY_EXECUTED_TEXT_SECTION_NAME
);
1938 gate_estimate_probability (void)
1940 return flag_guess_branch_prob
;
1943 /* Build PREDICT_EXPR. */
1945 build_predict_expr (enum br_predictor predictor
, enum prediction taken
)
1947 tree t
= build1 (PREDICT_EXPR
, void_type_node
,
1948 build_int_cst (NULL
, predictor
));
1949 PREDICT_EXPR_OUTCOME (t
) = taken
;
1954 predictor_name (enum br_predictor predictor
)
1956 return predictor_info
[predictor
].name
;
1959 struct gimple_opt_pass pass_profile
=
1963 "profile", /* name */
1964 gate_estimate_probability
, /* gate */
1965 tree_estimate_probability
, /* execute */
1968 0, /* static_pass_number */
1969 TV_BRANCH_PROB
, /* tv_id */
1970 PROP_cfg
, /* properties_required */
1971 0, /* properties_provided */
1972 0, /* properties_destroyed */
1973 0, /* todo_flags_start */
1974 TODO_ggc_collect
| TODO_verify_ssa
/* todo_flags_finish */