1 /* Calculate branch probabilities, and basic block execution counts.
2 Copyright (C) 2012. Free Software Foundation, Inc.
3 Contributed by Dehao Chen (dehao@google.com)
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Read and annotate call graph profile from the auto profile data
31 #include "coretypes.h"
33 #include "flags.h" /* for auto_profile_file. */
34 #include "basic-block.h" /* for gcov_type. */
35 #include "diagnostic-core.h" /* for inform (). */
36 #include "gcov-io.h" /* for gcov_read_unsigned (). */
37 #include "input.h" /* for expanded_location. */
38 #include "profile.h" /* for profile_info. */
39 #include "langhooks.h" /* for langhooks. */
40 #include "opts.h" /* for in_fnames. */
41 #include "tree-pass.h" /* for ipa pass. */
42 #include "cfgloop.h" /* for loop_optimizer_init. */
45 #include "tree-flow.h"
46 #include "value-prof.h"
50 #include "ipa-utils.h"
51 #include "ipa-inline.h"
53 #include "dwarf2asm.h"
54 #include "auto-profile.h"
56 /* The following routines implements AutoFDO optimization.
58 This optimization uses sampling profiles to annotate basic block counts
59 and uses heuristics to estimate branch probabilities.
61 There are three phases in AutoFDO:
63 Phase 1: Read profile from the profile data file.
64 The following info is read from the profile datafile:
65 * function_name_map: a map between function name and its index.
66 * autofdo_source_profile: a map from function_instance name to
67 function_instance. This is represented as a forest of
69 * autofdo_module_profile: a map from module name to its
70 compilation/aux-module info.
71 * WorkingSet: a histogram of how many instructions are covered for a
72 given percentage of total cycles.
74 Phase 2: Early inline.
75 Early inline uses autofdo_source_profile to find if a callsite is:
76 * inlined in the profiled binary.
77 * callee body is hot in the profiling run.
78 If both condition satisfies, early inline will inline the callsite
79 regardless of the code growth.
81 Phase 3: Annotate control flow graph.
82 AutoFDO uses a separate pass to:
83 * Annotate basic block count
84 * Estimate branch probability
86 After the above 3 phases, all profile is readily annotated on the GCC IR.
87 AutoFDO tries to reuse all FDO infrastructure as much as possible to make
88 use of the profile. E.g. it uses existing mechanism to calculate the basic
89 block/edge frequency, as well as the cgraph node/edge count.
92 #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
96 /* Represent a source location: (function_decl, lineno). */
97 typedef std::pair
<tree
, unsigned> decl_lineno
;
98 /* Represent an inline stack. vector[0] is the leaf node. */
99 typedef std::vector
<decl_lineno
> inline_stack
;
100 /* String array that stores function names. */
101 typedef std::vector
<const char *> string_vector
;
102 /* Map from function name's index in function_name_map to target's
104 typedef std::map
<unsigned, gcov_type
> icall_target_map
;
106 /* Set of inline_stack. Used to track if the profile is already used to
107 annotate the program. */
108 typedef std::set
<inline_stack
> location_set
;
110 /* Set of gimple stmts. Used to track if the stmt has already been promoted
112 typedef std::set
<gimple
> stmt_set
;
117 icall_target_map targets
;
121 struct string_compare
123 bool operator() (const char *a
, const char *b
) const
124 { return strcmp (a
, b
) < 0; }
127 /* Store a string array, indexed by string position in the array. */
128 class function_name_map
{
130 static function_name_map
*create ();
132 /* For a given string, returns its index. */
133 int get_index (const char *name
) const;
134 /* For a given decl, returns the index of the decl name. */
135 int get_index_by_decl (tree decl
) const;
136 /* For a given index, returns the string. */
137 const char *get_name (int index
) const;
140 function_name_map () {}
143 typedef std::map
<const char *, unsigned, string_compare
> string_index_map
;
144 string_vector vector_
;
145 string_index_map map_
;
148 /* Profile of a function copy:
149 1. total_count of the copy.
150 2. head_count of the copy (only valid when the copy is a top-level
151 function_instance, i.e. it is the original copy instead of the
153 3. map from source location (decl_lineno) of the inlined callsite to
154 profile (count_info).
155 4. map from callsite to callee function_instance. */
156 class function_instance
{
158 typedef std::vector
<function_instance
*> function_instance_stack
;
160 /* Read the profile and create a function_instance with head count as
161 HEAD_COUNT. Recursively read callsites to create nested function_instances
162 too. STACK is used to track the recursive creation process. */
163 static function_instance
*read_function_instance (
164 function_instance_stack
*stack
, gcov_type head_count
);
166 /* Recursively deallocate all callsites (nested function_instances). */
167 ~function_instance ();
170 unsigned name () const { return name_
; }
171 gcov_type
total_count () const { return total_count_
; }
172 gcov_type
head_count () const { return head_count_
; }
174 /* Recursively traverse STACK starting from LEVEL to find the corresponding
175 function_instance. */
176 function_instance
*get_function_instance (const inline_stack
&stack
,
179 /* Store the profile info for LOC in INFO. Return TRUE if profile info
181 bool get_count_info (location_t loc
, count_info
*info
) const;
183 /* Read the inlinied indirect call target profile for STMT and store it in
184 MAP, return the total count for all inlined indirect calls. */
185 gcov_type
find_icall_target_map (gimple stmt
, icall_target_map
*map
) const;
187 /* Total number of counts that is used during annotation. */
188 gcov_type
total_annotated_count () const;
190 /* Mark LOC as annotated. */
191 void mark_annotated (location_t loc
);
194 function_instance (unsigned name
, gcov_type head_count
)
195 : name_(name
), total_count_(0), head_count_(head_count
) {}
197 /* Traverse callsites of the current function_instance to find one at the
198 location of LINENO and callee name represented in DECL. */
199 function_instance
*get_function_instance_by_decl (unsigned lineno
, tree decl
);
201 /* Map from callsite decl_lineno (lineno in higher 16 bits, discriminator
202 in lower 16 bits) to callee function_instance. */
203 typedef std::map
<unsigned, function_instance
*> callsite_map
;
204 /* Map from source location (decl_lineno) to profile (count_info). */
205 typedef std::map
<unsigned, count_info
> position_count_map
;
207 /* function_instance name index in the function_name_map. */
209 /* The total sampled count. */
210 gcov_type total_count_
;
211 /* The total sampled count in the head bb. */
212 gcov_type head_count_
;
213 /* Map from callsite location to callee function_instance. */
214 callsite_map callsites
;
215 /* Map from source location to count and instruction number. */
216 position_count_map pos_counts
;
219 /* Profile for all functions. */
220 class autofdo_source_profile
{
222 static autofdo_source_profile
*create ()
224 autofdo_source_profile
*map
= new autofdo_source_profile ();
230 ~autofdo_source_profile ();
231 /* For a given DECL, returns the top-level function_instance. */
232 function_instance
*get_function_instance_by_decl (tree decl
);
233 /* Find profile info for a given gimple STMT. If found, and if the location
234 of STMT does not exist in ANNOTATED, store the profile info in INFO, and
235 return true; otherwise return false. */
236 bool get_count_info (gimple stmt
, count_info
*info
) const;
237 /* Find total count of the callee of EDGE. */
238 gcov_type
get_callsite_total_count (struct cgraph_edge
*edge
) const;
240 /* Update value profile INFO for STMT from the inlined indirect callsite.
241 Return true if INFO is updated. */
242 bool update_inlined_ind_target (gimple stmt
, count_info
*info
);
244 /* Mark LOCUS as annotated. */
245 void mark_annotated (location_t locus
);
247 /* Writes the profile annotation status for each function in an elf
249 void write_annotated_count () const;
252 /* Map from function_instance name index (in function_name_map) to
253 function_instance. */
254 typedef std::map
<unsigned, function_instance
*>
255 name_function_instance_map
;
257 autofdo_source_profile () {}
259 /* Return the function_instance in the profile that correspond to the
261 function_instance
*get_function_instance_by_inline_stack (
262 const inline_stack
&stack
) const;
264 name_function_instance_map map_
;
267 /* Module profile. */
268 class autofdo_module_profile
{
270 static autofdo_module_profile
*create ()
272 autofdo_module_profile
*map
= new autofdo_module_profile ();
279 /* For a given module NAME, returns this module's gcov_module_info. */
280 gcov_module_info
*get_module(const char *name
) const
282 name_target_map::const_iterator iter
= map_
.find (name
);
283 return iter
== map_
.end() ? NULL
: iter
->second
.second
;
286 /* For a given module NAME, returns this module's aux-modules. */
287 const string_vector
*get_aux_modules(const char *name
) const
289 name_target_map::const_iterator iter
= map_
.find (name
);
290 return iter
== map_
.end() ? NULL
: &iter
->second
.first
;
294 autofdo_module_profile () {}
297 typedef std::pair
<string_vector
, gcov_module_info
*> AuxInfo
;
298 typedef std::map
<const char *, AuxInfo
, string_compare
> name_target_map
;
299 /* Map from module name to (aux_modules, gcov_module_info). */
300 name_target_map map_
;
304 /* Store the strings read from the profile data file. */
305 static function_name_map
*afdo_function_name_map
;
306 static autofdo_source_profile
*afdo_source_profile
;
307 static autofdo_module_profile
*afdo_module_profile
;
309 /* gcov_ctr_summary structure to store the profile_info. */
310 static struct gcov_ctr_summary
*afdo_profile_info
;
313 /* Helper functions. */
315 /* Return the original name of NAME: strip the suffix that starts
318 static const char *get_original_name (const char *name
)
320 char *ret
= xstrdup (name
);
321 char *find
= strchr (ret
, '.');
327 /* Return the combined location, which is a 32bit integer in which
328 higher 16 bits stores the line offset of LOC to the start lineno
329 of DECL, The lower 16 bits stores the discrimnator. */
332 get_combined_location (location_t loc
, tree decl
)
334 return ((LOCATION_LINE (loc
) - DECL_SOURCE_LINE (decl
)) << 16)
335 | get_discriminator_from_locus (loc
);
338 /* Return the function decl of a given lexical BLOCK. */
341 get_function_decl_from_block (tree block
)
345 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
) == UNKNOWN_LOCATION
))
348 for (decl
= BLOCK_ABSTRACT_ORIGIN (block
);
349 decl
&& (TREE_CODE (decl
) == BLOCK
);
350 decl
= BLOCK_ABSTRACT_ORIGIN (decl
))
351 if (TREE_CODE (decl
) == FUNCTION_DECL
)
356 /* Store inline stack for STMT in STACK. */
359 get_inline_stack (location_t locus
, inline_stack
*stack
)
361 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
364 tree block
= LOCATION_BLOCK (locus
);
365 if (block
&& TREE_CODE (block
) == BLOCK
)
368 for (block
= BLOCK_SUPERCONTEXT (block
);
369 block
&& (TREE_CODE (block
) == BLOCK
);
370 block
= BLOCK_SUPERCONTEXT (block
))
372 location_t tmp_locus
= BLOCK_SOURCE_LOCATION (block
);
373 if (LOCATION_LOCUS (tmp_locus
) == UNKNOWN_LOCATION
)
376 tree decl
= get_function_decl_from_block (block
);
377 stack
->push_back (std::make_pair (
378 decl
, get_combined_location (locus
, decl
)));
383 stack
->push_back (std::make_pair (
384 current_function_decl
,
385 get_combined_location (locus
, current_function_decl
)));
388 /* Return STMT's combined location, which is a 32bit integer in which
389 higher 16 bits stores the line offset of LOC to the start lineno
390 of DECL, The lower 16 bits stores the discrimnator. */
393 get_relative_location_for_stmt (gimple stmt
)
395 location_t locus
= gimple_location (stmt
);
396 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
397 return UNKNOWN_LOCATION
;
399 for (tree block
= gimple_block (stmt
);
400 block
&& (TREE_CODE (block
) == BLOCK
);
401 block
= BLOCK_SUPERCONTEXT (block
))
402 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
)) != UNKNOWN_LOCATION
)
403 return get_combined_location (
404 locus
, get_function_decl_from_block (block
));
405 return get_combined_location (locus
, current_function_decl
);
408 /* Return true if BB contains indirect call. */
411 has_indirect_call (basic_block bb
)
413 gimple_stmt_iterator gsi
;
415 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
417 gimple stmt
= gsi_stmt (gsi
);
418 if (gimple_code (stmt
) == GIMPLE_CALL
419 && TREE_CODE (gimple_call_fn (stmt
)) != FUNCTION_DECL
)
425 /* Member functions for function_name_map. */
427 function_name_map
*function_name_map::create ()
429 function_name_map
*map
= new function_name_map();
436 int function_name_map::get_index (const char *name
) const
440 string_index_map::const_iterator iter
= map_
.find (name
);
441 if (iter
== map_
.end())
447 int function_name_map::get_index_by_decl (tree decl
) const
449 const char *name
= get_original_name (
450 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)));
451 int ret
= get_index (name
);
454 ret
= get_index (lang_hooks
.dwarf_name (decl
, 0));
457 if (DECL_ABSTRACT_ORIGIN (decl
))
458 return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl
));
463 const char *function_name_map::get_name (int index
) const
465 gcc_assert (index
> 0 && index
< (int) vector_
.size());
466 return vector_
[index
];
469 bool function_name_map::read ()
471 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES
)
473 /* Skip the length of the section. */
474 gcov_read_unsigned ();
475 /* Read in the file name table. */
476 unsigned string_num
= gcov_read_unsigned ();
477 for (unsigned i
= 0; i
< string_num
; i
++)
479 vector_
.push_back (get_original_name (gcov_read_string ()));
480 map_
[vector_
.back()] = i
;
486 /* Member functions for function_instance. */
488 function_instance::~function_instance ()
490 for (callsite_map::iterator iter
= callsites
.begin();
491 iter
!= callsites
.end(); ++iter
)
495 /* Traverse callsites of the current function_instance to find one at the
496 location of LINENO and callee name represented in DECL. */
498 function_instance
*function_instance::get_function_instance_by_decl (
499 unsigned lineno
, tree decl
)
501 int func_name_idx
= afdo_function_name_map
->get_index_by_decl (decl
);
502 if (func_name_idx
!= -1)
504 callsite_map::iterator ret
= callsites
.find (lineno
);
505 if (ret
!= callsites
.end ())
508 func_name_idx
= afdo_function_name_map
->get_index (
509 lang_hooks
.dwarf_name (decl
, 0));
510 if (func_name_idx
!= -1)
512 callsite_map::iterator ret
= callsites
.find (lineno
);
513 if (ret
!= callsites
.end ())
516 if (DECL_ABSTRACT_ORIGIN (decl
))
517 return get_function_instance_by_decl (lineno
, DECL_ABSTRACT_ORIGIN (decl
));
522 /* Recursively traverse STACK starting from LEVEL to find the corresponding
523 function_instance. */
525 function_instance
*function_instance::get_function_instance (
526 const inline_stack
&stack
, unsigned level
)
530 function_instance
*s
=
531 get_function_instance_by_decl (stack
[level
].second
, stack
[level
- 1].first
);
533 return s
->get_function_instance (stack
, level
- 1);
538 /* Store the profile info for LOC in INFO. Return TRUE if profile info
541 bool function_instance::get_count_info (location_t loc
, count_info
*info
) const
543 position_count_map::const_iterator iter
= pos_counts
.find (loc
);
544 if (iter
== pos_counts
.end ())
546 *info
= iter
->second
;
550 void function_instance::mark_annotated (location_t loc
)
552 position_count_map::iterator iter
= pos_counts
.find (loc
);
553 if (iter
== pos_counts
.end ())
555 iter
->second
.annotated
= true;
558 /* Read the inlinied indirect call target profile for STMT and store it in
559 MAP, return the total count for all inlined indirect calls. */
562 function_instance::find_icall_target_map (
563 gimple stmt
, icall_target_map
*map
) const
566 unsigned stmt_offset
= get_relative_location_for_stmt (stmt
);
568 for (callsite_map::const_iterator iter
= callsites
.begin();
569 iter
!= callsites
.end(); ++iter
)
571 unsigned callee
= iter
->second
->name();
572 /* Check if callsite location match the stmt. */
573 if (iter
->first
!= stmt_offset
)
575 struct cgraph_node
*node
= find_func_by_global_id (
576 (unsigned long long) afdo_function_name_map
->get_name (callee
), true);
579 if (!check_ic_target (stmt
, node
))
581 (*map
)[callee
] = iter
->second
->total_count ();
582 ret
+= iter
->second
->total_count ();
587 /* Read the profile and create a function_instance with head count as
588 HEAD_COUNT. Recursively read callsites to create nested function_instances
589 too. STACK is used to track the recursive creation process. */
591 function_instance
*function_instance::read_function_instance (
592 function_instance_stack
*stack
, gcov_type head_count
)
594 unsigned name
= gcov_read_unsigned ();
595 unsigned num_pos_counts
= gcov_read_unsigned ();
596 unsigned num_callsites
= gcov_read_unsigned ();
597 function_instance
*s
= new function_instance (name
, head_count
);
600 for (unsigned i
= 0; i
< num_pos_counts
; i
++)
602 unsigned offset
= gcov_read_unsigned ();
603 unsigned num_targets
= gcov_read_unsigned ();
604 gcov_type count
= gcov_read_counter ();
605 s
->pos_counts
[offset
].count
= count
;
606 for (unsigned j
= 0; j
< stack
->size(); j
++)
607 (*stack
)[j
]->total_count_
+= count
;
608 for (unsigned j
= 0; j
< num_targets
; j
++)
610 /* Only indirect call target histogram is supported now. */
611 gcov_read_unsigned ();
612 gcov_type target_idx
= gcov_read_counter ();
613 s
->pos_counts
[offset
].targets
[target_idx
] =
614 gcov_read_counter ();
617 for (unsigned i
= 0; i
< num_callsites
; i
++) {
618 unsigned offset
= gcov_read_unsigned ();
619 s
->callsites
[offset
] = read_function_instance (stack
, 0);
625 gcov_type
function_instance::total_annotated_count () const
628 for (callsite_map::const_iterator iter
= callsites
.begin();
629 iter
!= callsites
.end(); ++iter
)
630 ret
+= iter
->second
->total_annotated_count ();
631 for (position_count_map::const_iterator iter
= pos_counts
.begin();
632 iter
!= pos_counts
.end(); ++iter
)
633 if (iter
->second
.annotated
)
634 ret
+= iter
->second
.count
;
638 void autofdo_source_profile::write_annotated_count () const
640 /* We store the annotation info as a string in the format of:
642 function_name:total_count:annotated_count
644 Because different modules may output the annotation info for a same
645 function, we set the section as SECTION_MERGE so that we don't have
646 replicated info in the final binary. */
647 switch_to_section (get_section (
648 ".gnu.switches.text.annotation",
649 SECTION_DEBUG
| SECTION_MERGE
| SECTION_STRINGS
| (SECTION_ENTSIZE
& 1),
651 for (name_function_instance_map::const_iterator iter
= map_
.begin ();
652 iter
!= map_
.end (); ++iter
)
653 if (iter
->second
->total_count () > 0)
657 "%s:"HOST_WIDEST_INT_PRINT_DEC
":"HOST_WIDEST_INT_PRINT_DEC
,
658 afdo_function_name_map
->get_name (iter
->first
),
659 iter
->second
->total_count (),
660 iter
->second
->total_annotated_count ());
661 dw2_asm_output_nstring (buf
, (size_t)-1, NULL
);
666 /* Member functions for autofdo_source_profile. */
668 autofdo_source_profile::~autofdo_source_profile ()
670 for (name_function_instance_map::const_iterator iter
= map_
.begin ();
671 iter
!= map_
.end (); ++iter
)
675 /* For a given DECL, returns the top-level function_instance. */
677 function_instance
*autofdo_source_profile::get_function_instance_by_decl (
680 int index
= afdo_function_name_map
->get_index_by_decl (decl
);
683 name_function_instance_map::const_iterator ret
= map_
.find (index
);
684 return ret
== map_
.end() ? NULL
: ret
->second
;
687 /* Find profile info for a given gimple STMT. If found, and if the location
688 of STMT does not exist in ANNOTATED, store the profile info in INFO, and
689 return true; otherwise return false. */
691 bool autofdo_source_profile::get_count_info (
692 gimple stmt
, count_info
*info
) const
694 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
698 get_inline_stack (gimple_location (stmt
), &stack
);
699 if (stack
.size () == 0)
701 const function_instance
*s
= get_function_instance_by_inline_stack (stack
);
704 return s
->get_count_info (stack
[0].second
, info
);
707 void autofdo_source_profile::mark_annotated (location_t locus
) {
709 get_inline_stack (locus
, &stack
);
710 if (stack
.size () == 0)
712 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
715 s
->mark_annotated (stack
[0].second
);
718 /* Update value profile INFO for STMT from the inlined indirect callsite.
719 Return true if INFO is updated. */
722 autofdo_source_profile::update_inlined_ind_target (
723 gimple stmt
, count_info
*info
)
725 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
729 get_count_info (stmt
, &old_info
);
731 for (icall_target_map::const_iterator iter
= old_info
.targets
.begin();
732 iter
!= old_info
.targets
.end(); ++iter
)
733 total
+= iter
->second
;
735 /* Program behavior changed, original promoted (and inlined) target is not
736 hot any more. Will avoid promote the original target.
738 To check if original promoted target is still hot, we check the total
739 count of the unpromoted targets (stored in old_info). If it is no less
740 than half of the callsite count (stored in INFO), the original promoted
741 target is considered not hot any more. */
742 if (total
>= info
->count
* 0.5)
746 get_inline_stack (gimple_location (stmt
), &stack
);
747 if (stack
.size () == 0)
749 const function_instance
*s
= get_function_instance_by_inline_stack (stack
);
752 icall_target_map map
;
753 if (s
->find_icall_target_map (stmt
, &map
) == 0)
755 for (icall_target_map::const_iterator iter
= map
.begin();
756 iter
!= map
.end(); ++iter
)
757 info
->targets
[iter
->first
] = iter
->second
;
761 /* Find total count of the callee of EDGE. */
763 gcov_type
autofdo_source_profile::get_callsite_total_count (
764 struct cgraph_edge
*edge
) const
767 stack
.push_back (std::make_pair(edge
->callee
->symbol
.decl
, 0));
768 get_inline_stack (gimple_location (edge
->call_stmt
), &stack
);
770 const function_instance
*s
= get_function_instance_by_inline_stack (stack
);
774 return s
->total_count ();
777 /* Read source profile. */
779 bool autofdo_source_profile::read ()
781 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION
)
783 inform (0, "Not expected TAG.");
787 /* Skip the length of the section. */
788 gcov_read_unsigned ();
790 /* Read in the function/callsite profile, and store it in local
792 unsigned function_num
= gcov_read_unsigned ();
793 for (unsigned i
= 0; i
< function_num
; i
++)
795 function_instance::function_instance_stack stack
;
796 function_instance
*s
= function_instance::read_function_instance (
797 &stack
, gcov_read_counter ());
798 afdo_profile_info
->sum_all
+= s
->total_count ();
799 map_
[s
->name ()] = s
;
804 /* Return the function_instance in the profile that correspond to the
808 autofdo_source_profile::get_function_instance_by_inline_stack (
809 const inline_stack
&stack
) const
811 name_function_instance_map::const_iterator iter
= map_
.find (
812 afdo_function_name_map
->get_index_by_decl (
813 stack
[stack
.size() - 1].first
));
814 return iter
== map_
.end()
815 ? NULL
: iter
->second
->get_function_instance (stack
, stack
.size() - 1);
819 /* Member functions for autofdo_module_profile. */
821 bool autofdo_module_profile::read ()
823 /* Read in the module info. */
824 if (gcov_read_unsigned () != GCOV_TAG_AFDO_MODULE_GROUPING
)
826 inform (0, "Not expected TAG.");
829 /* Skip the length of the section. */
830 gcov_read_unsigned ();
832 /* Read in the file name table. */
833 unsigned total_module_num
= gcov_read_unsigned ();
834 for (unsigned i
= 0; i
< total_module_num
; i
++)
836 char *name
= xstrdup (gcov_read_string ());
837 unsigned total_num
= 0;
838 unsigned num_array
[7];
839 unsigned exported
= gcov_read_unsigned ();
840 unsigned lang
= gcov_read_unsigned ();
841 unsigned ggc_memory
= gcov_read_unsigned ();
842 for (unsigned j
= 0; j
< 7; j
++)
844 num_array
[j
] = gcov_read_unsigned ();
845 total_num
+= num_array
[j
];
847 gcov_module_info
*module
= XCNEWVAR (
849 sizeof (gcov_module_info
) + sizeof (char *) * total_num
);
851 std::pair
<name_target_map::iterator
, bool> ret
= map_
.insert(
852 name_target_map::value_type (name
, AuxInfo()));
853 gcc_assert (ret
.second
);
854 ret
.first
->second
.second
= module
;
855 module
->ident
= i
+ 1;
857 module
->ggc_memory
= ggc_memory
;
858 module
->num_quote_paths
= num_array
[1];
859 module
->num_bracket_paths
= num_array
[2];
860 module
->num_system_paths
= num_array
[3];
861 module
->num_cpp_defines
= num_array
[4];
862 module
->num_cpp_includes
= num_array
[5];
863 module
->num_cl_args
= num_array
[6];
864 module
->source_filename
= name
;
865 module
->is_primary
= strcmp (name
, in_fnames
[0]) == 0;
866 module
->flags
= module
->is_primary
? exported
: 1;
867 for (unsigned j
= 0; j
< num_array
[0]; j
++)
868 ret
.first
->second
.first
.push_back (xstrdup (gcov_read_string ()));
869 for (unsigned j
= 0; j
< total_num
- num_array
[0]; j
++)
870 module
->string_array
[j
] = xstrdup (gcov_read_string ());
875 /* Read the profile from the profile file. */
880 if (gcov_open (auto_profile_file
, 1) == 0)
881 error ("Cannot open profile file %s.", auto_profile_file
);
883 if (gcov_read_unsigned () != GCOV_DATA_MAGIC
)
884 error ("AutoFDO profile magic number does not mathch.");
886 /* Skip the version number. */
887 gcov_read_unsigned ();
889 /* Skip the empty integer. */
890 gcov_read_unsigned ();
892 /* function_name_map. */
893 afdo_function_name_map
= function_name_map::create ();
894 if (afdo_function_name_map
== NULL
)
895 error ("Cannot read string table from %s.", auto_profile_file
);
897 /* autofdo_source_profile. */
898 afdo_source_profile
= autofdo_source_profile::create ();
899 if (afdo_source_profile
== NULL
)
900 error ("Cannot read function profile from %s.", auto_profile_file
);
902 /* autofdo_module_profile. */
903 afdo_module_profile
= autofdo_module_profile::create ();
904 if (afdo_module_profile
== NULL
)
905 error ("Cannot read module profile from %s.", auto_profile_file
);
907 /* Read in the working set. */
908 if (gcov_read_unsigned () != GCOV_TAG_AFDO_WORKING_SET
)
909 error ("Cannot read working set from %s.", auto_profile_file
);
911 /* Skip the length of the section. */
912 gcov_read_unsigned ();
913 gcov_working_set_t set
[128];
914 for (unsigned i
= 0; i
< 128; i
++)
916 set
[i
].num_counters
= gcov_read_unsigned ();
917 set
[i
].min_counter
= gcov_read_counter ();
919 add_working_set (set
);
922 /* Read in the auxiliary modules for the current primary module. */
925 read_aux_modules (void)
927 gcov_module_info
*module
= afdo_module_profile
->get_module (in_fnames
[0]);
931 const string_vector
*aux_modules
=
932 afdo_module_profile
->get_aux_modules (in_fnames
[0]);
933 unsigned num_aux_modules
= aux_modules
? aux_modules
->size() : 0;
935 module_infos
= XCNEWVEC (gcov_module_info
*, num_aux_modules
+ 1);
936 module_infos
[0] = module
;
937 primary_module_id
= module
->ident
;
938 if (aux_modules
== NULL
)
940 unsigned curr_module
= 1, max_group
= PARAM_VALUE (PARAM_MAX_LIPO_GROUP
);
941 for (string_vector::const_iterator iter
= aux_modules
->begin();
942 iter
!= aux_modules
->end(); ++iter
)
944 gcov_module_info
*aux_module
= afdo_module_profile
->get_module (*iter
);
945 if (aux_module
== module
)
947 if (aux_module
== NULL
)
950 inform (0, "aux module %s cannot be found.", *iter
);
953 if ((aux_module
->lang
& GCOV_MODULE_LANG_MASK
) !=
954 (module
->lang
& GCOV_MODULE_LANG_MASK
))
957 inform (0, "Not importing %s: source language"
958 " different from primary module's source language", *iter
);
961 if ((aux_module
->lang
& GCOV_MODULE_ASM_STMTS
)
962 && flag_ripa_disallow_asm_modules
)
965 inform (0, "Not importing %s: contains "
966 "assembler statements", *iter
);
969 if (max_group
!= 0 && curr_module
>= max_group
)
972 inform (0, "Not importing %s: maximum group size reached", *iter
);
975 if (incompatible_cl_args (module
, aux_module
))
978 inform (0, "Not importing %s: command-line"
979 " arguments not compatible with primary module", *iter
);
982 module_infos
[curr_module
++] = aux_module
;
983 add_input_filename (*iter
);
987 /* From AutoFDO profiles, find values inside STMT for that we want to measure
988 histograms for indirect-call optimization. */
991 afdo_indirect_call (gimple stmt
, const icall_target_map
&map
)
995 if (map
.size() == 0 || gimple_code (stmt
) != GIMPLE_CALL
996 || gimple_call_fndecl (stmt
) != NULL_TREE
)
999 callee
= gimple_call_fn (stmt
);
1001 histogram_value hist
= gimple_alloc_histogram_value (
1002 cfun
, HIST_TYPE_INDIR_CALL_TOPN
, stmt
, callee
);
1003 hist
->n_counters
= (GCOV_ICALL_TOPN_VAL
<< 2) + 1;
1004 hist
->hvalue
.counters
= XNEWVEC (gcov_type
, hist
->n_counters
);
1005 gimple_add_histogram_value (cfun
, stmt
, hist
);
1007 gcov_type total
= 0;
1008 icall_target_map::const_iterator max_iter1
= map
.end();
1009 icall_target_map::const_iterator max_iter2
= map
.end();
1011 for (icall_target_map::const_iterator iter
= map
.begin();
1012 iter
!= map
.end(); ++iter
)
1014 total
+= iter
->second
;
1015 if (max_iter1
== map
.end() || max_iter1
->second
< iter
->second
)
1017 max_iter2
= max_iter1
;
1020 else if (max_iter2
== map
.end() || max_iter2
->second
< iter
->second
)
1024 hist
->hvalue
.counters
[0] = total
;
1025 hist
->hvalue
.counters
[1] = (unsigned long long)
1026 afdo_function_name_map
->get_name (max_iter1
->first
);
1027 hist
->hvalue
.counters
[2] = max_iter1
->second
;
1028 if (max_iter2
!= map
.end())
1030 hist
->hvalue
.counters
[3] = (unsigned long long)
1031 afdo_function_name_map
->get_name (max_iter2
->first
);
1032 hist
->hvalue
.counters
[4] = max_iter2
->second
;
1036 hist
->hvalue
.counters
[3] = 0;
1037 hist
->hvalue
.counters
[4] = 0;
1041 /* From AutoFDO profiles, find values inside STMT for that we want to measure
1042 histograms and adds them to list VALUES. */
1045 afdo_vpt (gimple stmt
, const icall_target_map
&map
)
1047 afdo_indirect_call (stmt
, map
);
1050 /* For a given BB, return its execution count. Add the location of annotated
1051 stmt to ANNOTATED. Attach value profile if a stmt is not in PROMOTED,
1052 because we only want to promot an indirect call once. */
1055 afdo_get_bb_count (basic_block bb
, const stmt_set
&promoted
)
1057 gimple_stmt_iterator gsi
;
1060 gcov_type max_count
= 0;
1061 bool has_annotated
= false;
1063 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1066 gimple stmt
= gsi_stmt (gsi
);
1067 if (afdo_source_profile
->get_count_info (stmt
, &info
))
1071 if (info
.count
> max_count
)
1072 max_count
= info
.count
;
1073 has_annotated
= true;
1074 if (info
.targets
.size() > 0 && promoted
.find (stmt
) == promoted
.end ())
1075 afdo_vpt (stmt
, info
.targets
);
1082 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1083 afdo_source_profile
->mark_annotated (gimple_location (gsi_stmt (gsi
)));
1084 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1086 gimple phi
= gsi_stmt (gsi
);
1088 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
1089 afdo_source_profile
->mark_annotated (gimple_phi_arg_location (phi
, i
));
1091 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1092 afdo_source_profile
->mark_annotated (e
->goto_locus
);
1094 bb
->flags
|= BB_ANNOTATED
;
1098 /* BB1 and BB2 are in an equivalent class iff:
1099 1. BB1 dominates BB2.
1100 2. BB2 post-dominates BB1.
1101 3. BB1 and BB2 are in the same loop nest.
1102 This function finds the equivalent class for each basic block, and
1103 stores a pointer to the first BB in its equivalent class. Meanwhile,
1104 set bb counts for the same equivalent class to be idenical. */
1107 afdo_find_equiv_class (void)
1116 vec
<basic_block
> dom_bbs
;
1120 if (bb
->aux
!= NULL
)
1123 dom_bbs
= get_all_dominated_blocks (CDI_DOMINATORS
, bb
);
1124 FOR_EACH_VEC_ELT (dom_bbs
, i
, bb1
)
1125 if (bb1
->aux
== NULL
1126 && dominated_by_p (CDI_POST_DOMINATORS
, bb
, bb1
)
1127 && bb1
->loop_father
== bb
->loop_father
)
1130 if (bb1
->count
> bb
->count
&& (bb1
->flags
& BB_ANNOTATED
) != 0)
1132 bb
->count
= MAX (bb
->count
, bb1
->count
);
1133 bb
->flags
|= BB_ANNOTATED
;
1136 dom_bbs
= get_all_dominated_blocks (CDI_POST_DOMINATORS
, bb
);
1137 FOR_EACH_VEC_ELT (dom_bbs
, i
, bb1
)
1138 if (bb1
->aux
== NULL
1139 && dominated_by_p (CDI_DOMINATORS
, bb
, bb1
)
1140 && bb1
->loop_father
== bb
->loop_father
)
1143 if (bb1
->count
> bb
->count
&& (bb1
->flags
& BB_ANNOTATED
) != 0)
1145 bb
->count
= MAX (bb
->count
, bb1
->count
);
1146 bb
->flags
|= BB_ANNOTATED
;
1152 /* If a basic block's count is known, and only one of its in/out edges' count
1153 is unknown, its count can be calculated.
1154 Meanwhile, if all of the in/out edges' counts are known, then the basic
1155 block's unknown count can also be calculated.
1156 IS_SUCC is true if out edges of a basic blocks are examined.
1157 Return TRUE if any basic block/edge count is changed. */
1160 afdo_propagate_edge (bool is_succ
)
1163 bool changed
= false;
1167 edge e
, unknown_edge
= NULL
;
1169 int num_unknown_edge
= 0;
1170 gcov_type total_known_count
= 0;
1172 FOR_EACH_EDGE (e
, ei
, is_succ
? bb
->succs
: bb
->preds
)
1173 if ((e
->flags
& EDGE_ANNOTATED
) == 0)
1174 num_unknown_edge
++, unknown_edge
= e
;
1176 total_known_count
+= e
->count
;
1178 if (num_unknown_edge
== 0)
1180 if (total_known_count
> bb
->count
)
1182 bb
->count
= total_known_count
;
1185 if ((bb
->flags
& BB_ANNOTATED
) == 0)
1187 bb
->flags
|= BB_ANNOTATED
;
1191 else if (num_unknown_edge
== 1
1192 && (bb
->flags
& BB_ANNOTATED
) != 0)
1194 if (bb
->count
>= total_known_count
)
1195 unknown_edge
->count
= bb
->count
- total_known_count
;
1197 unknown_edge
->count
= 0;
1198 unknown_edge
->flags
|= EDGE_ANNOTATED
;
1205 /* Special propagation for circuit expressions. Because GCC translates
1206 control flow into data flow for circuit expressions. E.g.
1213 will be translated into:
1228 tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
1234 In this case, we need to propagate through PHI to determine the edge
1235 count of BB1->BB.t1, BB.t1->BB.t2. */
1238 afdo_propagate_circuit (void)
1244 tree cmp_rhs
, cmp_lhs
;
1245 gimple cmp_stmt
= last_stmt (bb
);
1249 if (!cmp_stmt
|| gimple_code (cmp_stmt
) != GIMPLE_COND
)
1251 cmp_rhs
= gimple_cond_rhs (cmp_stmt
);
1252 cmp_lhs
= gimple_cond_lhs (cmp_stmt
);
1253 if (!TREE_CONSTANT (cmp_rhs
)
1254 || !(integer_zerop (cmp_rhs
) || integer_onep (cmp_rhs
)))
1256 if (TREE_CODE (cmp_lhs
) != SSA_NAME
)
1258 if ((bb
->flags
& BB_ANNOTATED
) == 0)
1260 phi_stmt
= SSA_NAME_DEF_STMT (cmp_lhs
);
1261 while (phi_stmt
&& gimple_code (phi_stmt
) == GIMPLE_ASSIGN
1262 && gimple_assign_single_p (phi_stmt
)
1263 && TREE_CODE (gimple_assign_rhs1 (phi_stmt
)) == SSA_NAME
)
1264 phi_stmt
= SSA_NAME_DEF_STMT (gimple_assign_rhs1 (phi_stmt
));
1265 if (!phi_stmt
|| gimple_code (phi_stmt
) != GIMPLE_PHI
)
1267 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1269 unsigned i
, total
= 0;
1271 bool check_value_one
= (((integer_onep (cmp_rhs
))
1272 ^ (gimple_cond_code (cmp_stmt
) == EQ_EXPR
))
1273 ^ ((e
->flags
& EDGE_TRUE_VALUE
) != 0));
1274 if ((e
->flags
& EDGE_ANNOTATED
) == 0)
1276 for (i
= 0; i
< gimple_phi_num_args (phi_stmt
); i
++)
1278 tree val
= gimple_phi_arg_def (phi_stmt
, i
);
1279 edge ep
= gimple_phi_arg_edge (phi_stmt
, i
);
1281 if (!TREE_CONSTANT (val
) || !(integer_zerop (val
)
1282 || integer_onep (val
)))
1284 if (check_value_one
^ integer_onep (val
))
1288 if (e
->probability
== 0 && (e
->flags
& EDGE_ANNOTATED
) == 0)
1290 ep
->probability
= 0;
1292 ep
->flags
|= EDGE_ANNOTATED
;
1295 if (total
== 1 && (only_one
->flags
& EDGE_ANNOTATED
) == 0)
1297 only_one
->probability
= e
->probability
;
1298 only_one
->count
= e
->count
;
1299 only_one
->flags
|= EDGE_ANNOTATED
;
1305 /* Propagate the basic block count and edge count on the control flow
1306 graph. We do the propagation iteratively until stablize. */
1309 afdo_propagate (void)
1312 bool changed
= true;
1317 bb
->count
= ((basic_block
) bb
->aux
)->count
;
1318 if ((((basic_block
) bb
->aux
)->flags
& BB_ANNOTATED
) != 0)
1319 bb
->flags
|= BB_ANNOTATED
;
1322 while (changed
&& i
++ < PARAM_VALUE (PARAM_AUTOFDO_MAX_PROPAGATE_ITERATIONS
))
1326 if (afdo_propagate_edge (true))
1328 if (afdo_propagate_edge (false))
1330 afdo_propagate_circuit ();
1334 /* Propagate counts on control flow graph and calculate branch
1338 afdo_calculate_branch_prob (void)
1341 bool has_sample
= false;
1350 calculate_dominance_info (CDI_POST_DOMINATORS
);
1351 calculate_dominance_info (CDI_DOMINATORS
);
1352 loop_optimizer_init (0);
1354 afdo_find_equiv_class ();
1361 int num_unknown_succ
= 0;
1362 gcov_type total_count
= 0;
1364 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1366 if ((e
->flags
& EDGE_ANNOTATED
) == 0)
1367 num_unknown_succ
++;
1369 total_count
+= e
->count
;
1371 if (num_unknown_succ
== 0 && total_count
> 0)
1373 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1375 (double) e
->count
* REG_BR_PROB_BASE
/ total_count
;
1383 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1385 (double) bb
->count
* e
->probability
/ REG_BR_PROB_BASE
;
1389 loop_optimizer_finalize ();
1390 free_dominance_info (CDI_DOMINATORS
);
1391 free_dominance_info (CDI_POST_DOMINATORS
);
1394 /* Perform value profile transformation using AutoFDO profile. Add the
1395 promoted stmts to PROMOTED_STMTS. Return TRUE if there is any
1396 indirect call promoted. */
1399 afdo_vpt_for_early_inline (stmt_set
*promoted_stmts
)
1402 if (afdo_source_profile
->get_function_instance_by_decl (
1403 current_function_decl
) == NULL
)
1406 bool has_vpt
= false;
1409 if (!has_indirect_call (bb
))
1411 gimple_stmt_iterator gsi
;
1413 gcov_type bb_count
= 0;
1414 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1417 gimple stmt
= gsi_stmt (gsi
);
1418 if (afdo_source_profile
->get_count_info (stmt
, &info
))
1419 bb_count
= MAX (bb_count
, info
.count
);
1422 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1424 gimple stmt
= gsi_stmt (gsi
);
1425 /* IC_promotion and early_inline_2 is done in multiple iterations.
1426 No need to promoted the stmt if its in promoted_stmts (means
1427 it is already been promoted in the previous iterations). */
1428 if (gimple_code (stmt
) != GIMPLE_CALL
1429 || TREE_CODE (gimple_call_fn (stmt
)) == FUNCTION_DECL
1430 || promoted_stmts
->find (stmt
) != promoted_stmts
->end ())
1434 afdo_source_profile
->get_count_info (stmt
, &info
);
1435 info
.count
= bb_count
;
1436 if (afdo_source_profile
->update_inlined_ind_target (stmt
, &info
))
1438 /* Promote the indirect call and update the promoted_stmts. */
1439 promoted_stmts
->insert (stmt
);
1440 afdo_vpt (stmt
, info
.targets
);
1445 if (has_vpt
&& gimple_value_profile_transformations ())
1447 free_dominance_info (CDI_DOMINATORS
);
1448 free_dominance_info (CDI_POST_DOMINATORS
);
1449 calculate_dominance_info (CDI_POST_DOMINATORS
);
1450 calculate_dominance_info (CDI_DOMINATORS
);
1451 rebuild_cgraph_edges ();
1458 /* Annotate auto profile to the control flow graph. Do not annotate value
1459 profile for stmts in PROMOTED_STMTS. */
1462 afdo_annotate_cfg (const stmt_set
&promoted_stmts
)
1465 const function_instance
*s
=
1466 afdo_source_profile
->get_function_instance_by_decl (
1467 current_function_decl
);
1471 cgraph_get_node (current_function_decl
)->count
= s
->head_count ();
1472 ENTRY_BLOCK_PTR
->count
= s
->head_count ();
1473 gcov_type max_count
= ENTRY_BLOCK_PTR
->count
;
1481 bb
->flags
&= (~BB_ANNOTATED
);
1482 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1485 e
->flags
&= (~EDGE_ANNOTATED
);
1488 bb
->count
= afdo_get_bb_count (bb
, promoted_stmts
);
1489 if (bb
->count
> max_count
)
1490 max_count
= bb
->count
;
1492 if (ENTRY_BLOCK_PTR
->count
> ENTRY_BLOCK_PTR
->next_bb
->count
)
1494 ENTRY_BLOCK_PTR
->next_bb
->count
= ENTRY_BLOCK_PTR
->count
;
1495 ENTRY_BLOCK_PTR
->next_bb
->flags
|= BB_ANNOTATED
;
1497 if (ENTRY_BLOCK_PTR
->count
> EXIT_BLOCK_PTR
->prev_bb
->count
)
1499 EXIT_BLOCK_PTR
->prev_bb
->count
= ENTRY_BLOCK_PTR
->count
;
1500 EXIT_BLOCK_PTR
->prev_bb
->flags
|= BB_ANNOTATED
;
1502 afdo_source_profile
->mark_annotated (
1503 DECL_SOURCE_LOCATION (current_function_decl
));
1504 afdo_source_profile
->mark_annotated (cfun
->function_start_locus
);
1505 afdo_source_profile
->mark_annotated (cfun
->function_end_locus
);
1508 afdo_calculate_branch_prob ();
1510 profile_status
= PROFILE_READ
;
1512 if (flag_value_profile_transformations
)
1513 gimple_value_profile_transformations ();
1515 } /* namespace autofdo. */
1517 static void early_inline ()
1519 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
1520 unsigned todo
= early_inliner ();
1521 if (todo
& TODO_update_ssa_any
)
1522 update_ssa (TODO_update_ssa
);
1525 /* Use AutoFDO profile to annoate the control flow graph.
1526 Return the todo flag. */
1531 struct cgraph_node
*node
;
1533 if (cgraph_state
== CGRAPH_STATE_FINISHED
)
1537 profile_info
= autofdo::afdo_profile_info
;
1539 cgraph_pre_profiling_inlining_done
= true;
1540 cgraph_process_module_scope_statics ();
1541 /* Now perform link to allow cross module inlining. */
1544 cgraph_unify_type_alias_sets ();
1546 FOR_EACH_FUNCTION (node
)
1548 if (!gimple_has_body_p (node
->symbol
.decl
))
1551 /* Don't profile functions produced for builtin stuff. */
1552 if (DECL_SOURCE_LOCATION (node
->symbol
.decl
) == BUILTINS_LOCATION
)
1555 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
1557 if (L_IPO_COMP_MODE
)
1562 gimple_stmt_iterator gsi
;
1563 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1565 gimple stmt
= gsi_stmt (gsi
);
1566 if (is_gimple_call (stmt
))
1567 lipo_fixup_cgraph_edge_call_target (stmt
);
1571 rebuild_cgraph_edges ();
1575 FOR_EACH_FUNCTION (node
)
1577 if (!gimple_has_body_p (node
->symbol
.decl
))
1580 /* Don't profile functions produced for builtin stuff. */
1581 if (DECL_SOURCE_LOCATION (node
->symbol
.decl
) == BUILTINS_LOCATION
)
1584 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
1586 /* First do indirect call promotion and early inline to make the
1587 IR match the profiled binary before actual annotation.
1589 This is needed because an indirect call might have been promoted
1590 and inlined in the profiled binary. If we do not promote and
1591 inline these indirect calls before annotation, the profile for
1592 these promoted functions will be lost.
1594 e.g. foo() --indirect_call--> bar()
1595 In profiled binary, the callsite is promoted and inlined, making
1596 the profile look like:
1606 Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
1607 If we perform annotation on it, the profile inside bar@loc_foo2
1610 To avoid this, we promote loc_foo_2 and inline the promoted bar
1611 function before annotation, so the profile inside bar@loc_foo2
1613 autofdo::stmt_set promoted_stmts
;
1614 for (int i
= 0; i
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
); i
++)
1616 if (!flag_value_profile_transformations
1617 || !autofdo::afdo_vpt_for_early_inline (&promoted_stmts
))
1623 autofdo::afdo_annotate_cfg (promoted_stmts
);
1624 compute_function_frequency ();
1625 update_ssa (TODO_update_ssa
);
1627 /* Local pure-const may imply need to fixup the cfg. */
1628 if (execute_fixup_cfg () & TODO_cleanup_cfg
)
1629 cleanup_tree_cfg ();
1631 free_dominance_info (CDI_DOMINATORS
);
1632 free_dominance_info (CDI_POST_DOMINATORS
);
1633 rebuild_cgraph_edges ();
1637 autofdo::afdo_source_profile
->write_annotated_count ();
1642 gate_auto_profile_ipa (void)
1644 return flag_auto_profile
;
1647 /* Read the profile from the profile data file. */
1650 init_auto_profile (void)
1652 if (auto_profile_file
== NULL
)
1653 auto_profile_file
= DEFAULT_AUTO_PROFILE_FILE
;
1655 autofdo::afdo_profile_info
= (struct gcov_ctr_summary
*)
1656 xcalloc (1, sizeof (struct gcov_ctr_summary
));
1657 autofdo::afdo_profile_info
->runs
= 1;
1658 autofdo::afdo_profile_info
->sum_max
= 0;
1659 autofdo::afdo_profile_info
->sum_all
= 0;
1661 /* Read the profile from the profile file. */
1662 autofdo::read_profile ();
1665 autofdo::read_aux_modules ();
1668 /* Free the resources. */
1671 end_auto_profile (void)
1673 delete autofdo::afdo_source_profile
;
1674 delete autofdo::afdo_function_name_map
;
1675 delete autofdo::afdo_module_profile
;
1676 profile_info
= NULL
;
1679 /* Returns TRUE if EDGE is hot enough to be inlined early. */
1682 afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge
*edge
)
1685 autofdo::afdo_source_profile
->get_callsite_total_count (edge
);
1689 const struct gcov_ctr_summary
*saved_profile_info
= profile_info
;
1690 /* At earling inline stage, profile_info is not set yet. We need to
1691 temporarily set it to afdo_profile_info to calculate hotness. */
1692 profile_info
= autofdo::afdo_profile_info
;
1693 is_hot
= maybe_hot_count_p (NULL
, count
);
1694 profile_info
= saved_profile_info
;
1701 struct simple_ipa_opt_pass pass_ipa_auto_profile
=
1706 OPTGROUP_NONE
, /* optinfo_flags */
1707 gate_auto_profile_ipa
, /* gate */
1708 auto_profile
, /* execute */
1711 0, /* static_pass_number */
1712 TV_IPA_AUTOFDO
, /* tv_id */
1713 0, /* properties_required */
1714 0, /* properties_provided */
1715 0, /* properties_destroyed */
1716 0, /* todo_flags_start */
1717 0 /* todo_flags_finish */