1 /* Profile counter container type.
2 Copyright (C) 2017 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #ifndef GCC_PROFILE_COUNT_H
22 #define GCC_PROFILE_COUNT_H
24 /* Quality of the profile count. Because gengtype does not support enums
25 inside of classes, this is in global namespace. */
26 enum profile_quality
{
27 /* Profile is based on static branch prediction heuristics. It may or may
28 not reflect the reality. */
30 /* Profile was determined by autofdo. */
32 /* Profile was originally based on feedback but it was adjusted
33 by code duplicating optimization. It may not precisely reflect the
34 particular code path. */
36 /* Profile was read from profile feedback or determined by accurate static
41 /* The base value for branch probability notes and edge probabilities. */
42 #define REG_BR_PROB_BASE 10000
44 #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
46 /* Data type to hold probabilities. It implements fixed point arithmetics
47 with capping so probability is always in range [0,1] and scaling requiring
48 values greater than 1 needs to be represented otherwise.
50 In addition to actual value the quality of profile is tracked and propagated
51 through all operations. Special value UNINITIALIZED is used for probabilities
52 that has not been determined yet (for example bacause of
53 -fno-guess-branch-probability)
55 Typically probabilities are derived from profile feedback (via
56 probability_in_gcov_type), autoFDO or guessed statically and then propagated
57 thorough the compilation.
59 Named probabilities are available:
60 - never (0 probability)
62 - very_unlikely (1/2000 probability)
63 - unlikely (1/5 probablity)
64 - even (1/2 probability)
65 - likely (4/5 probability)
66 - very_likely (1999/2000 probability)
70 Named probabilities except for never/always are assumed to be statically
71 guessed and thus not necessarily accurate. The difference between never
72 and guessed_never is that the first one should be used only in case that
73 well behaving program will very likely not execute the "never" path.
74 For example if the path is going to abort () call or it exception handling.
76 Always and guessed_always probabilities are symmetric.
78 For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
79 integer arithmetics. Once the code is converted to branch probabilities,
80 these conversions will probably go away because they are lossy.
83 class GTY((user
)) profile_probability
85 /* For now use values in range 0...REG_BR_PROB_BASE. Later we can use full
86 precision of 30 bits available. */
88 static const int n_bits
= 30;
89 static const uint32_t max_probability
= REG_BR_PROB_BASE
;
90 static const uint32_t uninitialized_probability
= ((uint32_t) 1 << n_bits
) - 1;
93 enum profile_quality m_quality
: 2;
95 friend class profile_count
;
98 /* Named probabilities. */
99 static profile_probability
never ()
101 profile_probability ret
;
103 ret
.m_quality
= profile_precise
;
106 static profile_probability
guessed_never ()
108 profile_probability ret
;
110 ret
.m_quality
= profile_guessed
;
113 static profile_probability
very_unlikely ()
115 /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
116 profile_probability r
117 = profile_probability::always ().apply_scale (1, 2000);
121 static profile_probability
unlikely ()
123 /* Be consistent with PROB_VERY_LIKELY in predict.h. */
124 profile_probability r
125 = profile_probability::always ().apply_scale (1, 5);
129 static profile_probability
even ()
131 return profile_probability::always ().apply_scale (1, 2);
133 static profile_probability
very_likely ()
135 return profile_probability::always () - very_unlikely ();
137 static profile_probability
likely ()
139 return profile_probability::always () - unlikely ();
141 static profile_probability
guessed_always ()
143 profile_probability ret
;
144 ret
.m_val
= max_probability
;
145 ret
.m_quality
= profile_guessed
;
148 static profile_probability
always ()
150 profile_probability ret
;
151 ret
.m_val
= max_probability
;
152 ret
.m_quality
= profile_precise
;
155 /* Probabilities which has not been initialized. Either because
156 initialization did not happen yet or because profile is unknown. */
157 static profile_probability
uninitialized ()
159 profile_probability c
;
160 c
.m_val
= uninitialized_probability
;
161 c
.m_quality
= profile_guessed
;
166 /* Return true if value has been initialized. */
167 bool initialized_p () const
169 return m_val
!= uninitialized_probability
;
171 /* Return true if value can be trusted. */
172 bool reliable_p () const
174 return initialized_p ();
177 /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
178 this is mostly to support legacy code and should go away. */
179 static profile_probability
from_reg_br_prob_base (int v
)
181 profile_probability ret
;
182 gcc_checking_assert (v
>= 0 && v
<= REG_BR_PROB_BASE
);
183 ret
.m_val
= RDIV (v
* max_probability
, REG_BR_PROB_BASE
);
184 ret
.m_quality
= profile_guessed
;
187 int to_reg_br_prob_base () const
189 gcc_checking_assert (initialized_p ());
190 return RDIV (m_val
* REG_BR_PROB_BASE
, max_probability
);
193 /* Return VAL1/VAL2. */
194 static profile_probability probability_in_gcov_type
195 (gcov_type val1
, gcov_type val2
)
197 profile_probability ret
;
198 gcc_checking_assert (val1
>= 0 && val2
> 0);
200 ret
.m_val
= max_probability
;
202 ret
.m_val
= RDIV (val1
* max_probability
, val2
);
203 ret
.m_quality
= profile_precise
;
207 /* Basic operations. */
208 bool operator== (const profile_probability
&other
) const
210 return m_val
== other
.m_val
&& m_quality
== other
.m_quality
;
212 profile_probability
operator+ (const profile_probability
&other
) const
214 if (other
== profile_probability::never ())
216 if (*this == profile_probability::never ())
218 if (!initialized_p () || !other
.initialized_p ())
219 return profile_probability::uninitialized ();
221 profile_probability ret
;
222 ret
.m_val
= MIN ((uint32_t)(m_val
+ other
.m_val
), max_probability
);
223 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
226 profile_probability
&operator+= (const profile_probability
&other
)
228 if (other
== profile_probability::never ())
230 if (*this == profile_probability::never ())
235 if (!initialized_p () || !other
.initialized_p ())
236 return *this = profile_probability::uninitialized ();
239 m_val
= MIN ((uint32_t)(m_val
+ other
.m_val
), max_probability
);
240 m_quality
= MIN (m_quality
, other
.m_quality
);
244 profile_probability
operator- (const profile_probability
&other
) const
246 if (*this == profile_probability::never ()
247 || other
== profile_probability::never ())
249 if (!initialized_p () || !other
.initialized_p ())
250 return profile_probability::uninitialized ();
251 profile_probability ret
;
252 ret
.m_val
= m_val
>= other
.m_val
? m_val
- other
.m_val
: 0;
253 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
256 profile_probability
&operator-= (const profile_probability
&other
)
258 if (*this == profile_probability::never ()
259 || other
== profile_probability::never ())
261 if (!initialized_p () || !other
.initialized_p ())
262 return *this = profile_probability::uninitialized ();
265 m_val
= m_val
>= other
.m_val
? m_val
- other
.m_val
: 0;
266 m_quality
= MIN (m_quality
, other
.m_quality
);
270 profile_probability
operator* (const profile_probability
&other
) const
272 if (*this == profile_probability::never ()
273 || other
== profile_probability::never ())
274 return profile_probability::never ();
275 if (!initialized_p () || !other
.initialized_p ())
276 return profile_probability::uninitialized ();
277 profile_probability ret
;
278 ret
.m_val
= RDIV ((uint64_t)m_val
* other
.m_val
, max_probability
);
279 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
282 profile_probability
&operator*= (const profile_probability
&other
)
284 if (*this == profile_probability::never ()
285 || other
== profile_probability::never ())
286 return *this = profile_probability::never ();
287 if (!initialized_p () || !other
.initialized_p ())
288 return *this = profile_probability::uninitialized ();
291 m_val
= RDIV ((uint64_t)m_val
* other
.m_val
, max_probability
);
292 m_quality
= MIN (m_quality
, other
.m_quality
);
296 profile_probability
operator/ (const profile_probability
&other
) const
298 if (*this == profile_probability::never ())
299 return profile_probability::never ();
300 if (!initialized_p () || !other
.initialized_p ())
301 return profile_probability::uninitialized ();
302 profile_probability ret
;
303 if (m_val
>= other
.m_val
)
304 ret
.m_val
= max_probability
;
309 gcc_checking_assert (other
.m_val
);
310 ret
.m_val
= MIN (RDIV ((uint64_t)m_val
* max_probability
,
314 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
317 profile_probability
&operator/= (const profile_probability
&other
)
319 if (*this == profile_probability::never ())
320 return *this = profile_probability::never ();
321 if (!initialized_p () || !other
.initialized_p ())
322 return *this = profile_probability::uninitialized ();
325 if (m_val
> other
.m_val
)
326 m_val
= max_probability
;
331 gcc_checking_assert (other
.m_val
);
332 m_val
= MIN (RDIV ((uint64_t)m_val
* max_probability
,
336 m_quality
= MIN (m_quality
, other
.m_quality
);
341 gcov_type
apply (gcov_type val
) const
343 if (*this == profile_probability::uninitialized ())
345 return RDIV (val
* m_val
, max_probability
);
348 /* Return 1-*THIS. */
349 profile_probability
invert () const
351 return profile_probability::always() - *this;
354 /* Return THIS with quality dropped to GUESSED. */
355 profile_probability
guessed () const
357 profile_probability ret
= *this;
358 ret
.m_quality
= profile_guessed
;
362 /* Return THIS with quality dropped to AFDO. */
363 profile_probability
afdo () const
365 profile_probability ret
= *this;
366 ret
.m_quality
= profile_afdo
;
370 profile_probability
combine_with_freq (int freq1
, profile_probability other
,
373 profile_probability ret
;
375 if (*this == profile_probability::uninitialized ()
376 || other
== profile_probability::uninitialized ())
377 return profile_probability::uninitialized ();
379 gcc_checking_assert (freq1
>= 0 && freq2
>= 0);
380 if (!freq1
&& !freq2
)
382 ret
.m_val
= (m_val
+ other
.m_val
) / 2;
385 ret
.m_val
= RDIV (m_val
* (uint64_t) freq1
386 + other
.m_val
* (uint64_t) freq2
, freq1
+ freq2
);
387 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
391 /* Return *THIS * NUM / DEN. */
392 profile_probability
apply_scale (int64_t num
, int64_t den
) const
394 if (*this == profile_probability::never ())
396 if (!initialized_p ())
397 return profile_probability::uninitialized ();
398 profile_probability ret
;
399 ret
.m_val
= MIN (RDIV (m_val
* num
, den
),
401 ret
.m_quality
= MIN (m_quality
, profile_adjusted
);
405 /* Return true when the probability of edge is reliable.
407 The profile guessing code is good at predicting branch outcome (ie.
408 taken/not taken), that is predicted right slightly over 75% of time.
409 It is however notoriously poor on predicting the probability itself.
410 In general the profile appear a lot flatter (with probabilities closer
411 to 50%) than the reality so it is bad idea to use it to drive optimization
412 such as those disabling dynamic branch prediction for well predictable
415 There are two exceptions - edges leading to noreturn edges and edges
416 predicted by number of iterations heuristics are predicted well. This macro
417 should be able to distinguish those, but at the moment it simply check for
418 noreturn heuristic that is only one giving probability over 99% or bellow
419 1%. In future we might want to propagate reliability information across the
420 CFG if we find this information useful on multiple places. */
422 bool probably_reliable_p () const
424 if (m_quality
>= profile_adjusted
)
426 if (!initialized_p ())
428 return m_val
< max_probability
/ 100
429 || m_val
> max_probability
- max_probability
/ 100;
432 /* Return false if profile_probability is bogus. */
435 if (m_val
== uninitialized_probability
)
436 return m_quality
== profile_guessed
;
438 return m_val
<= REG_BR_PROB_BASE
;
441 /* Comparsions are three-state and conservative. False is returned if
442 the inequality can not be decided. */
443 bool operator< (const profile_probability
&other
) const
445 return initialized_p () && other
.initialized_p () && m_val
< other
.m_val
;
447 bool operator> (const profile_probability
&other
) const
449 return initialized_p () && other
.initialized_p () && m_val
> other
.m_val
;
452 bool operator<= (const profile_probability
&other
) const
454 return initialized_p () && other
.initialized_p () && m_val
<= other
.m_val
;
456 bool operator>= (const profile_probability
&other
) const
458 return initialized_p () && other
.initialized_p () && m_val
>= other
.m_val
;
461 /* Output THIS to F. */
462 void dump (FILE *f
) const;
464 /* Print THIS to stderr. */
467 /* Return true if THIS is known to differ significantly from OTHER. */
468 bool differs_from_p (profile_probability other
) const;
469 /* Return if difference is greater than 50%. */
470 bool differs_lot_from_p (profile_probability other
) const;
472 /* LTO streaming support. */
473 static profile_probability
stream_in (struct lto_input_block
*);
474 void stream_out (struct output_block
*);
475 void stream_out (struct lto_output_stream
*);
478 /* Main data type to hold profile counters in GCC. In most cases profile
479 counts originate from profile feedback. They are 64bit integers
480 representing number of executions during the train run.
481 As the profile is maintained during the compilation, many adjustments are
482 made. Not all transformations can be made precisely, most importantly
483 when code is being duplicated. It also may happen that part of CFG has
484 profile counts known while other do not - for example when LTO optimizing
485 partly profiled program or when profile was lost due to COMDAT merging.
487 For this reason profile_count tracks more information than
488 just unsigned integer and it is also ready for profile mismatches.
489 The API of this data type represent operations that are natural
490 on profile counts - sum, difference and operation with scales and
491 probabilities. All operations are safe by never getting negative counts
492 and they do end up in uninitialized scale if any of the parameters is
495 All comparsions that are three state and handling of probabilities. Thus
496 a < b is not equal to !(a >= b).
498 The following pre-defined counts are available:
500 profile_count::zero () for code that is known to execute zero times at
501 runtime (this can be detected statically i.e. for paths leading to
503 profile_count::one () for code that is known to execute once (such as
505 profile_count::uninitialized () for unknown execution count.
509 class GTY(()) profile_count
511 /* Use 62bit to hold basic block counters. Should be at least
512 64bit. Although a counter cannot be negative, we use a signed
513 type to hold various extra stages. */
515 static const int n_bits
= 62;
516 static const uint64_t max_count
= ((uint64_t) 1 << n_bits
) - 2;
517 static const uint64_t uninitialized_count
= ((uint64_t) 1 << n_bits
) - 1;
519 uint64_t m_val
: n_bits
;
520 enum profile_quality m_quality
: 2;
522 /* Assume numbers smaller than this to multiply. This is set to make
523 testsuite pass, in future we may implement precise multiplication in higer
525 static const uint64_t max_safe_multiplier
= 131072;
528 /* Used for counters which are expected to be never executed. */
529 static profile_count
zero ()
531 return from_gcov_type (0);
533 static profile_count
guessed_zero ()
537 c
.m_quality
= profile_guessed
;
540 static profile_count
one ()
542 return from_gcov_type (1);
544 /* Value of counters which has not been initialized. Either because
545 initialization did not happen yet or because profile is unknown. */
546 static profile_count
uninitialized ()
549 c
.m_val
= uninitialized_count
;
550 c
.m_quality
= profile_guessed
;
554 /* The profiling runtime uses gcov_type, which is usually 64bit integer.
555 Conversions back and forth are used to read the coverage and get it
556 into internal representation. */
557 static profile_count
from_gcov_type (gcov_type v
)
560 gcc_checking_assert (v
>= 0 && (uint64_t) v
<= max_count
);
562 ret
.m_quality
= profile_precise
;
566 /* Conversion to gcov_type is lossy. */
567 gcov_type
to_gcov_type () const
569 gcc_checking_assert (initialized_p ());
573 /* Return true if value has been initialized. */
574 bool initialized_p () const
576 return m_val
!= uninitialized_count
;
578 /* Return true if value can be trusted. */
579 bool reliable_p () const
581 return initialized_p ();
584 /* When merging basic blocks, the two different profile counts are unified.
585 Return true if this can be done without losing info about profile.
586 The only case we care about here is when first BB contains something
587 that makes it terminate in a way not visible in CFG. */
588 bool ok_for_merging (profile_count other
) const
590 if (m_quality
< profile_adjusted
591 || other
.m_quality
< profile_adjusted
)
593 return !(other
< *this);
596 /* When merging two BBs with different counts, pick common count that looks
597 most representative. */
598 profile_count
merge (profile_count other
) const
600 if (*this == other
|| !other
.initialized_p ()
601 || m_quality
> other
.m_quality
)
603 if (other
.m_quality
> m_quality
609 /* Basic operations. */
610 bool operator== (const profile_count
&other
) const
612 return m_val
== other
.m_val
&& m_quality
== other
.m_quality
;
614 profile_count
operator+ (const profile_count
&other
) const
616 if (other
== profile_count::zero ())
618 if (*this == profile_count::zero ())
620 if (!initialized_p () || !other
.initialized_p ())
621 return profile_count::uninitialized ();
624 ret
.m_val
= m_val
+ other
.m_val
;
625 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
628 profile_count
&operator+= (const profile_count
&other
)
630 if (other
== profile_count::zero ())
632 if (*this == profile_count::zero ())
637 if (!initialized_p () || !other
.initialized_p ())
638 return *this = profile_count::uninitialized ();
641 m_val
+= other
.m_val
;
642 m_quality
= MIN (m_quality
, other
.m_quality
);
646 profile_count
operator- (const profile_count
&other
) const
648 if (*this == profile_count::zero () || other
== profile_count::zero ())
650 if (!initialized_p () || !other
.initialized_p ())
651 return profile_count::uninitialized ();
653 ret
.m_val
= m_val
>= other
.m_val
? m_val
- other
.m_val
: 0;
654 ret
.m_quality
= MIN (m_quality
, other
.m_quality
);
657 profile_count
&operator-= (const profile_count
&other
)
659 if (*this == profile_count::zero () || other
== profile_count::zero ())
661 if (!initialized_p () || !other
.initialized_p ())
662 return *this = profile_count::uninitialized ();
665 m_val
= m_val
>= other
.m_val
? m_val
- other
.m_val
: 0;
666 m_quality
= MIN (m_quality
, other
.m_quality
);
671 /* Return false if profile_count is bogus. */
674 return m_val
!= uninitialized_count
|| m_quality
== profile_guessed
;
677 /* Comparsions are three-state and conservative. False is returned if
678 the inequality can not be decided. */
679 bool operator< (const profile_count
&other
) const
681 return initialized_p () && other
.initialized_p () && m_val
< other
.m_val
;
683 bool operator> (const profile_count
&other
) const
685 return initialized_p () && other
.initialized_p () && m_val
> other
.m_val
;
687 bool operator< (const gcov_type other
) const
689 gcc_checking_assert (other
>= 0);
690 return initialized_p () && m_val
< (uint64_t) other
;
692 bool operator> (const gcov_type other
) const
694 gcc_checking_assert (other
>= 0);
695 return initialized_p () && m_val
> (uint64_t) other
;
698 bool operator<= (const profile_count
&other
) const
700 return initialized_p () && other
.initialized_p () && m_val
<= other
.m_val
;
702 bool operator>= (const profile_count
&other
) const
704 return initialized_p () && other
.initialized_p () && m_val
>= other
.m_val
;
706 bool operator<= (const gcov_type other
) const
708 gcc_checking_assert (other
>= 0);
709 return initialized_p () && m_val
<= (uint64_t) other
;
711 bool operator>= (const gcov_type other
) const
713 gcc_checking_assert (other
>= 0);
714 return initialized_p () && m_val
>= (uint64_t) other
;
717 /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
719 profile_count
apply_probability (int prob
) const
721 gcc_checking_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
724 if (!initialized_p ())
725 return profile_count::uninitialized ();
727 ret
.m_val
= RDIV (m_val
* prob
, REG_BR_PROB_BASE
);
728 ret
.m_quality
= MIN (m_quality
, profile_adjusted
);
732 /* Scale counter according to PROB. */
733 profile_count
apply_probability (profile_probability prob
) const
735 if (*this == profile_count::zero ())
737 if (prob
== profile_probability::never ())
738 return profile_count::zero ();
739 if (!initialized_p ())
740 return profile_count::uninitialized ();
742 ret
.m_val
= RDIV (m_val
* prob
.m_val
,
743 profile_probability::max_probability
);
744 ret
.m_quality
= MIN (m_quality
, prob
.m_quality
);
747 /* Return *THIS * NUM / DEN. */
748 profile_count
apply_scale (int64_t num
, int64_t den
) const
752 if (!initialized_p ())
753 return profile_count::uninitialized ();
755 gcc_checking_assert (num
>= 0 && den
> 0);
756 /* FIXME: shrink wrapping violates this sanity check. */
757 gcc_checking_assert ((num
<= REG_BR_PROB_BASE
758 || den
<= REG_BR_PROB_BASE
) || 1);
759 ret
.m_val
= RDIV (m_val
* num
, den
);
760 ret
.m_quality
= MIN (m_quality
, profile_adjusted
);
763 profile_count
apply_scale (profile_count num
, profile_count den
) const
769 if (!initialized_p () || !num
.initialized_p () || !den
.initialized_p ())
770 return profile_count::uninitialized ();
771 gcc_checking_assert (den
> 0);
776 /* Take care for overflows! */
777 if (num
.m_val
< max_safe_multiplier
|| m_val
< max_safe_multiplier
)
778 ret
.m_val
= RDIV (m_val
* num
.m_val
, den
.m_val
);
780 ret
.m_val
= RDIV (m_val
* RDIV (num
.m_val
* max_safe_multiplier
,
781 den
.m_val
), max_safe_multiplier
);
782 ret
.m_quality
= MIN (m_quality
, profile_adjusted
);
786 /* Return THIS with quality dropped to GUESSED. */
787 profile_count
guessed () const
789 profile_count ret
= *this;
790 ret
.m_quality
= profile_guessed
;
794 /* Return THIS with quality dropped to AFDO. */
795 profile_count
afdo () const
797 profile_count ret
= *this;
798 ret
.m_quality
= profile_afdo
;
802 /* Return probability of event with counter THIS within event with counter
804 profile_probability
probability_in (const profile_count overall
) const
807 return profile_probability::never ();
808 if (!initialized_p () || !overall
.initialized_p ()
810 return profile_probability::uninitialized ();
811 profile_probability ret
;
813 ret
.m_val
= profile_probability::max_probability
;
815 ret
.m_val
= RDIV (m_val
* profile_probability::max_probability
,
817 ret
.m_quality
= MIN (m_quality
, overall
.m_quality
);
821 /* Output THIS to F. */
822 void dump (FILE *f
) const;
824 /* Print THIS to stderr. */
827 /* Return true if THIS is known to differ significantly from OTHER. */
828 bool differs_from_p (profile_count other
) const;
830 /* LTO streaming support. */
831 static profile_count
stream_in (struct lto_input_block
*);
832 void stream_out (struct output_block
*);
833 void stream_out (struct lto_output_stream
*);