1 /* Calculate branch probabilities, and basic block execution counts.
2 Copyright (C) 2012. Free Software Foundation, Inc.
3 Contributed by Dehao Chen (dehao@google.com)
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Read and annotate call graph profile from the auto profile data
31 #include "coretypes.h"
33 #include "flags.h" /* for auto_profile_file. */
34 #include "basic-block.h" /* for gcov_type. */
35 #include "diagnostic-core.h" /* for inform (). */
36 #include "gcov-io.h" /* for gcov_read_unsigned (). */
37 #include "input.h" /* for expanded_location. */
38 #include "profile.h" /* for profile_info. */
39 #include "langhooks.h" /* for langhooks. */
40 #include "opts.h" /* for in_fnames. */
41 #include "tree-pass.h" /* for ipa pass. */
42 #include "cfgloop.h" /* for loop_optimizer_init. */
45 #include "tree-flow.h"
46 #include "value-prof.h"
50 #include "ipa-utils.h"
51 #include "ipa-inline.h"
52 #include "auto-profile.h"
54 /* The following routines implements AutoFDO optimization.
56 This optimization uses sampling profiles to annotate basic block counts
57 and uses heuristics to estimate branch probabilities.
59 There are three phases in AutoFDO:
61 Phase 1: Read profile from the profile data file.
62 The following info is read from the profile datafile:
63 * function_name_map: a map between function name and its index.
64 * autofdo_source_profile: a map from function_instance name to
65 function_instance. This is represented as a forest of
67 * autofdo_module_profile: a map from module name to its
68 compilation/aux-module info.
69 * WorkingSet: a histogram of how many instructions are covered for a
70 given percentage of total cycles.
72 Phase 2: Early inline.
73 Early inline uses autofdo_source_profile to find if a callsite is:
74 * inlined in the profiled binary.
75 * callee body is hot in the profiling run.
76 If both condition satisfies, early inline will inline the callsite
77 regardless of the code growth.
79 Phase 3: Annotate control flow graph.
80 AutoFDO uses a separate pass to:
81 * Annotate basic block count
82 * Estimate branch probability
84 After the above 3 phases, all profile is readily annotated on the GCC IR.
85 AutoFDO tries to reuse all FDO infrastructure as much as possible to make
86 use of the profile. E.g. it uses existing mechanism to calculate the basic
87 block/edge frequency, as well as the cgraph node/edge count.
90 #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
94 /* Represent a source location: (function_decl, lineno). */
95 typedef std::pair
<tree
, unsigned> decl_lineno
;
96 /* Represent an inline stack. vector[0] is the leaf node. */
97 typedef std::vector
<decl_lineno
> inline_stack
;
98 /* String array that stores function names. */
99 typedef std::vector
<const char *> string_vector
;
100 /* Map from function name's index in function_name_map to target's
102 typedef std::map
<unsigned, gcov_type
> icall_target_map
;
103 /* Represent profile count of an inline stack, profile count is represented as
104 (execution_count, value_profile_histogram). */
105 typedef std::pair
<gcov_type
, icall_target_map
> count_info
;
107 /* Set of inline_stack. Used to track if the profile is already used to
108 annotate the program. */
109 typedef std::set
<inline_stack
> location_set
;
111 /* Set of gimple stmts. Used to track if the stmt has already been promoted
113 typedef std::set
<gimple
> stmt_set
;
115 struct string_compare
117 bool operator() (const char *a
, const char *b
) const
118 { return strcmp (a
, b
) < 0; }
121 /* Store a string array, indexed by string position in the array. */
122 class function_name_map
{
124 static function_name_map
*create ();
126 /* For a given string, returns its index. */
127 int get_index (const char *name
) const;
128 /* For a given decl, returns the index of the decl name. */
129 int get_index_by_decl (tree decl
) const;
130 /* For a given index, returns the string. */
131 const char *get_name (int index
) const;
134 function_name_map () {}
137 typedef std::map
<const char *, unsigned, string_compare
> string_index_map
;
138 string_vector vector_
;
139 string_index_map map_
;
142 /* Profile of a function copy:
143 1. total_count of the copy.
144 2. head_count of the copy (only valid when the copy is a top-level
145 function_instance, i.e. it is the original copy instead of the
147 3. map from source location (decl_lineno) of the inlined callsite to
148 profile (count_info).
149 4. map from callsite to callee function_instance. */
150 class function_instance
{
152 typedef std::vector
<function_instance
*> function_instance_stack
;
154 /* Read the profile and create a function_instance with head count as
155 HEAD_COUNT. Recursively read callsites to create nested function_instances
156 too. STACK is used to track the recursive creation process. */
157 static const function_instance
*read_function_instance (
158 function_instance_stack
*stack
, gcov_type head_count
);
160 /* Recursively deallocate all callsites (nested function_instances). */
161 ~function_instance ();
164 unsigned name () const { return name_
; }
165 gcov_type
total_count () const { return total_count_
; }
166 gcov_type
head_count () const { return head_count_
; }
168 /* Recursively traverse STACK starting from LEVEL to find the corresponding
169 function_instance. */
170 const function_instance
*get_function_instance (const inline_stack
&stack
,
171 unsigned level
) const;
173 /* Store the profile info for LOC in INFO. Return TRUE if profile info
175 bool get_count_info (location_t loc
, count_info
*info
) const;
177 /* Read the inlinied indirect call target profile for STMT and store it in
178 MAP, return the total count for all inlined indirect calls. */
179 gcov_type
find_icall_target_map (gimple stmt
, icall_target_map
*map
) const;
182 function_instance (unsigned name
, gcov_type head_count
)
183 : name_(name
), total_count_(0), head_count_(head_count
) {}
185 /* Traverse callsites of the current function_instance to find one at the
186 location of LINENO and callee name represented in DECL. */
187 const function_instance
*get_function_instance_by_decl (unsigned lineno
,
190 /* Map from callsite decl_lineno (lineno in higher 16 bits, discriminator
191 in lower 16 bits) to callee function_instance. */
192 typedef std::map
<unsigned, const function_instance
*> callsite_map
;
193 /* Map from source location (decl_lineno) to profile (count_info). */
194 typedef std::map
<unsigned, count_info
> position_count_map
;
196 /* function_instance name index in the function_name_map. */
198 /* The total sampled count. */
199 gcov_type total_count_
;
200 /* The total sampled count in the head bb. */
201 gcov_type head_count_
;
202 /* Map from callsite location to callee function_instance. */
203 callsite_map callsites
;
204 /* Map from source location to count and instruction number. */
205 position_count_map pos_counts
;
208 /* Profile for all functions. */
209 class autofdo_source_profile
{
211 static autofdo_source_profile
*create ()
213 autofdo_source_profile
*map
= new autofdo_source_profile ();
219 ~autofdo_source_profile ();
220 /* For a given DECL, returns the top-level function_instance. */
221 const function_instance
*get_function_instance_by_decl (tree decl
) const;
222 /* Find profile info for a given gimple STMT. If found, and if the location
223 of STMT does not exist in ANNOTATED, store the profile info in INFO, and
224 return true; otherwise return false. */
225 bool get_count_info (gimple stmt
, count_info
*info
,
226 const location_set
*annotated
) const;
227 /* Find total count of the callee of EDGE. */
228 gcov_type
get_callsite_total_count (struct cgraph_edge
*edge
) const;
230 /* Update value profile INFO for STMT from the inlined indirect callsite.
231 Return true if INFO is updated. */
232 bool update_inlined_ind_target (gimple stmt
, count_info
*info
);
235 /* Map from function_instance name index (in function_name_map) to
236 function_instance. */
237 typedef std::map
<unsigned, const function_instance
*>
238 name_function_instance_map
;
240 autofdo_source_profile () {}
242 /* Return the function_instance in the profile that correspond to the
244 const function_instance
*get_function_instance_by_inline_stack (
245 const inline_stack
&stack
) const;
247 name_function_instance_map map_
;
250 /* Module profile. */
251 class autofdo_module_profile
{
253 static autofdo_module_profile
*create ()
255 autofdo_module_profile
*map
= new autofdo_module_profile ();
262 /* For a given module NAME, returns this module's gcov_module_info. */
263 gcov_module_info
*get_module(const char *name
) const
265 name_target_map::const_iterator iter
= map_
.find (name
);
266 return iter
== map_
.end() ? NULL
: iter
->second
.second
;
269 /* For a given module NAME, returns this module's aux-modules. */
270 const string_vector
*get_aux_modules(const char *name
) const
272 name_target_map::const_iterator iter
= map_
.find (name
);
273 return iter
== map_
.end() ? NULL
: &iter
->second
.first
;
277 autofdo_module_profile () {}
280 typedef std::pair
<string_vector
, gcov_module_info
*> AuxInfo
;
281 typedef std::map
<const char *, AuxInfo
, string_compare
> name_target_map
;
282 /* Map from module name to (aux_modules, gcov_module_info). */
283 name_target_map map_
;
287 /* Store the strings read from the profile data file. */
288 static function_name_map
*afdo_function_name_map
;
289 static autofdo_source_profile
*afdo_source_profile
;
290 static autofdo_module_profile
*afdo_module_profile
;
292 /* gcov_ctr_summary structure to store the profile_info. */
293 static struct gcov_ctr_summary
*afdo_profile_info
;
296 /* Helper functions. */
298 /* Return the original name of NAME: strip the suffix that starts
301 static const char *get_original_name (const char *name
)
303 char *ret
= xstrdup (name
);
304 char *find
= strchr (ret
, '.');
310 /* Return the combined location, which is a 32bit integer in which
311 higher 16 bits stores the line offset of LOC to the start lineno
312 of DECL, The lower 16 bits stores the discrimnator. */
315 get_combined_location (location_t loc
, tree decl
)
317 return ((LOCATION_LINE (loc
) - DECL_SOURCE_LINE (decl
)) << 16)
318 | get_discriminator_from_locus (loc
);
321 /* Return the function decl of a given lexical BLOCK. */
324 get_function_decl_from_block (tree block
)
328 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
) == UNKNOWN_LOCATION
))
331 for (decl
= BLOCK_ABSTRACT_ORIGIN (block
);
332 decl
&& (TREE_CODE (decl
) == BLOCK
);
333 decl
= BLOCK_ABSTRACT_ORIGIN (decl
))
334 if (TREE_CODE (decl
) == FUNCTION_DECL
)
339 /* Store inline stack for STMT in STACK. */
342 get_inline_stack (gimple stmt
, inline_stack
*stack
)
344 location_t locus
= gimple_location (stmt
);
345 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
348 tree block
= gimple_block (stmt
);
349 if (!block
|| TREE_CODE (block
) != BLOCK
)
353 for (block
= BLOCK_SUPERCONTEXT (block
);
354 block
&& (TREE_CODE (block
) == BLOCK
);
355 block
= BLOCK_SUPERCONTEXT (block
))
357 location_t tmp_locus
= BLOCK_SOURCE_LOCATION (block
);
358 if (LOCATION_LOCUS (tmp_locus
) == UNKNOWN_LOCATION
)
361 tree decl
= get_function_decl_from_block (block
);
362 stack
->push_back (std::make_pair (
363 decl
, get_combined_location (locus
, decl
)));
367 stack
->push_back (std::make_pair (
368 current_function_decl
,
369 get_combined_location (locus
, current_function_decl
)));
372 /* Return STMT's combined location, which is a 32bit integer in which
373 higher 16 bits stores the line offset of LOC to the start lineno
374 of DECL, The lower 16 bits stores the discrimnator. */
377 get_relative_location_for_stmt (gimple stmt
)
379 location_t locus
= gimple_location (stmt
);
380 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
381 return UNKNOWN_LOCATION
;
383 for (tree block
= gimple_block (stmt
);
384 block
&& (TREE_CODE (block
) == BLOCK
);
385 block
= BLOCK_SUPERCONTEXT (block
))
386 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
)) != UNKNOWN_LOCATION
)
387 return get_combined_location (
388 locus
, get_function_decl_from_block (block
));
389 return get_combined_location (locus
, current_function_decl
);
392 /* Return true if BB contains indirect call. */
395 has_indirect_call (basic_block bb
)
397 gimple_stmt_iterator gsi
;
399 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
401 gimple stmt
= gsi_stmt (gsi
);
402 if (gimple_code (stmt
) == GIMPLE_CALL
403 && TREE_CODE (gimple_call_fn (stmt
)) != FUNCTION_DECL
)
409 /* Member functions for function_name_map. */
411 function_name_map
*function_name_map::create ()
413 function_name_map
*map
= new function_name_map();
420 int function_name_map::get_index (const char *name
) const
424 string_index_map::const_iterator iter
= map_
.find (name
);
425 if (iter
== map_
.end())
431 int function_name_map::get_index_by_decl (tree decl
) const
433 const char *name
= get_original_name (
434 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)));
435 int ret
= get_index (name
);
438 ret
= get_index (lang_hooks
.dwarf_name (decl
, 0));
441 if (DECL_ABSTRACT_ORIGIN (decl
))
442 return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl
));
447 const char *function_name_map::get_name (int index
) const
449 gcc_assert (index
> 0 && index
< (int) vector_
.size());
450 return vector_
[index
];
453 bool function_name_map::read ()
455 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES
)
457 /* Skip the length of the section. */
458 gcov_read_unsigned ();
459 /* Read in the file name table. */
460 unsigned string_num
= gcov_read_unsigned ();
461 for (unsigned i
= 0; i
< string_num
; i
++)
463 vector_
.push_back (get_original_name (gcov_read_string ()));
464 map_
[vector_
.back()] = i
;
470 /* Member functions for function_instance. */
472 function_instance::~function_instance ()
474 for (callsite_map::iterator iter
= callsites
.begin();
475 iter
!= callsites
.end(); ++iter
)
479 /* Traverse callsites of the current function_instance to find one at the
480 location of LINENO and callee name represented in DECL. */
482 const function_instance
*function_instance::get_function_instance_by_decl (
483 unsigned lineno
, tree decl
) const
485 int func_name_idx
= afdo_function_name_map
->get_index_by_decl (decl
);
486 if (func_name_idx
!= -1)
488 callsite_map::const_iterator ret
= callsites
.find (lineno
);
489 if (ret
!= callsites
.end ())
492 func_name_idx
= afdo_function_name_map
->get_index (
493 lang_hooks
.dwarf_name (decl
, 0));
494 if (func_name_idx
!= -1)
496 callsite_map::const_iterator ret
= callsites
.find (lineno
);
497 if (ret
!= callsites
.end ())
500 if (DECL_ABSTRACT_ORIGIN (decl
))
501 return get_function_instance_by_decl (lineno
, DECL_ABSTRACT_ORIGIN (decl
));
506 /* Recursively traverse STACK starting from LEVEL to find the corresponding
507 function_instance. */
509 const function_instance
*function_instance::get_function_instance (
510 const inline_stack
&stack
, unsigned level
) const
514 const function_instance
*s
=
515 get_function_instance_by_decl (stack
[level
].second
, stack
[level
- 1].first
);
517 return s
->get_function_instance (stack
, level
- 1);
522 /* Store the profile info for LOC in INFO. Return TRUE if profile info
525 bool function_instance::get_count_info (location_t loc
, count_info
*info
) const
527 position_count_map::const_iterator iter
= pos_counts
.find (loc
);
528 if (iter
== pos_counts
.end ())
530 *info
= iter
->second
;
534 /* Read the inlinied indirect call target profile for STMT and store it in
535 MAP, return the total count for all inlined indirect calls. */
538 function_instance::find_icall_target_map (
539 gimple stmt
, icall_target_map
*map
) const
542 unsigned stmt_offset
= get_relative_location_for_stmt (stmt
);
544 for (callsite_map::const_iterator iter
= callsites
.begin();
545 iter
!= callsites
.end(); ++iter
)
547 unsigned callee
= iter
->second
->name();
548 /* Check if callsite location match the stmt. */
549 if (iter
->first
!= stmt_offset
)
551 struct cgraph_node
*node
= find_func_by_global_id (
552 (unsigned long long) afdo_function_name_map
->get_name (callee
), true);
555 if (!check_ic_target (stmt
, node
))
557 (*map
)[callee
] = iter
->second
->total_count ();
558 ret
+= iter
->second
->total_count ();
563 /* Read the profile and create a function_instance with head count as
564 HEAD_COUNT. Recursively read callsites to create nested function_instances
565 too. STACK is used to track the recursive creation process. */
567 const function_instance
*function_instance::read_function_instance (
568 function_instance_stack
*stack
, gcov_type head_count
)
570 unsigned name
= gcov_read_unsigned ();
571 unsigned num_pos_counts
= gcov_read_unsigned ();
572 unsigned num_callsites
= gcov_read_unsigned ();
573 function_instance
*s
= new function_instance (name
, head_count
);
576 for (unsigned i
= 0; i
< num_pos_counts
; i
++)
578 unsigned offset
= gcov_read_unsigned ();
579 unsigned num_targets
= gcov_read_unsigned ();
580 gcov_type count
= gcov_read_counter ();
581 s
->pos_counts
[offset
].first
= count
;
582 for (unsigned j
= 0; j
< stack
->size(); j
++)
583 (*stack
)[j
]->total_count_
+= count
;
584 for (unsigned j
= 0; j
< num_targets
; j
++)
586 /* Only indirect call target histogram is supported now. */
587 gcov_read_unsigned ();
588 gcov_type target_idx
= gcov_read_counter ();
589 s
->pos_counts
[offset
].second
[target_idx
] =
590 gcov_read_counter ();
593 for (unsigned i
= 0; i
< num_callsites
; i
++) {
594 unsigned offset
= gcov_read_unsigned ();
595 s
->callsites
[offset
] = read_function_instance (stack
, 0);
602 /* Member functions for autofdo_source_profile. */
604 autofdo_source_profile::~autofdo_source_profile ()
606 for (name_function_instance_map::const_iterator iter
= map_
.begin ();
607 iter
!= map_
.end (); ++iter
)
611 /* For a given DECL, returns the top-level function_instance. */
613 const function_instance
*autofdo_source_profile::get_function_instance_by_decl (
616 int index
= afdo_function_name_map
->get_index_by_decl (decl
);
619 name_function_instance_map::const_iterator ret
= map_
.find (index
);
620 return ret
== map_
.end() ? NULL
: ret
->second
;
623 /* Find profile info for a given gimple STMT. If found, and if the location
624 of STMT does not exist in ANNOTATED, store the profile info in INFO, and
625 return true; otherwise return false. */
627 bool autofdo_source_profile::get_count_info (
628 gimple stmt
, count_info
*info
, const location_set
*annotated
) const
630 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
634 get_inline_stack (stmt
, &stack
);
635 if (annotated
&& annotated
->find(stack
) != annotated
->end())
637 if (stack
.size () == 0)
639 const function_instance
*s
= get_function_instance_by_inline_stack (stack
);
642 return s
->get_count_info (stack
[0].second
, info
);
645 /* Update value profile INFO for STMT from the inlined indirect callsite.
646 Return true if INFO is updated. */
649 autofdo_source_profile::update_inlined_ind_target (
650 gimple stmt
, count_info
*info
)
652 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
656 get_count_info (stmt
, &old_info
, NULL
);
658 for (icall_target_map::const_iterator iter
= old_info
.second
.begin();
659 iter
!= old_info
.second
.end(); ++iter
)
660 total
+= iter
->second
;
662 /* Program behavior changed, original promoted (and inlined) target is not
663 hot any more. Will avoid promote the original target.
665 To check if original promoted target is still hot, we check the total
666 count of the unpromoted targets (stored in old_info). If it is no less
667 than half of the callsite count (stored in INFO), the original promoted
668 target is considered not hot any more. */
669 if (total
>= info
->first
* 0.5)
673 get_inline_stack (stmt
, &stack
);
674 if (stack
.size () == 0)
676 const function_instance
*s
= get_function_instance_by_inline_stack (stack
);
679 icall_target_map map
;
680 if (s
->find_icall_target_map (stmt
, &map
) == 0)
682 for (icall_target_map::const_iterator iter
= map
.begin();
683 iter
!= map
.end(); ++iter
)
684 info
->second
[iter
->first
] = iter
->second
;
688 /* Find total count of the callee of EDGE. */
690 gcov_type
autofdo_source_profile::get_callsite_total_count (
691 struct cgraph_edge
*edge
) const
694 stack
.push_back (std::make_pair(edge
->callee
->symbol
.decl
, 0));
695 get_inline_stack (edge
->call_stmt
, &stack
);
697 const function_instance
*s
= get_function_instance_by_inline_stack (stack
);
701 return s
->total_count ();
704 /* Read source profile. */
706 bool autofdo_source_profile::read ()
708 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION
)
710 inform (0, "Not expected TAG.");
714 /* Skip the length of the section. */
715 gcov_read_unsigned ();
717 /* Read in the function/callsite profile, and store it in local
719 unsigned function_num
= gcov_read_unsigned ();
720 for (unsigned i
= 0; i
< function_num
; i
++)
722 function_instance::function_instance_stack stack
;
723 const function_instance
*s
= function_instance::read_function_instance (
724 &stack
, gcov_read_counter ());
725 afdo_profile_info
->sum_all
+= s
->total_count ();
726 map_
[s
->name ()] = s
;
731 /* Return the function_instance in the profile that correspond to the
734 const function_instance
*
735 autofdo_source_profile::get_function_instance_by_inline_stack (
736 const inline_stack
&stack
) const
738 name_function_instance_map::const_iterator iter
= map_
.find (
739 afdo_function_name_map
->get_index_by_decl (
740 stack
[stack
.size() - 1].first
));
741 return iter
== map_
.end()
742 ? NULL
: iter
->second
->get_function_instance (stack
, stack
.size() - 1);
746 /* Member functions for autofdo_module_profile. */
748 bool autofdo_module_profile::read ()
750 /* Read in the module info. */
751 if (gcov_read_unsigned () != GCOV_TAG_AFDO_MODULE_GROUPING
)
753 inform (0, "Not expected TAG.");
756 /* Skip the length of the section. */
757 gcov_read_unsigned ();
759 /* Read in the file name table. */
760 unsigned total_module_num
= gcov_read_unsigned ();
761 for (unsigned i
= 0; i
< total_module_num
; i
++)
763 char *name
= xstrdup (gcov_read_string ());
764 unsigned total_num
= 0;
765 unsigned num_array
[7];
766 unsigned exported
= gcov_read_unsigned ();
767 unsigned lang
= gcov_read_unsigned ();
768 unsigned ggc_memory
= gcov_read_unsigned ();
769 for (unsigned j
= 0; j
< 7; j
++)
771 num_array
[j
] = gcov_read_unsigned ();
772 total_num
+= num_array
[j
];
774 gcov_module_info
*module
= XCNEWVAR (
776 sizeof (gcov_module_info
) + sizeof (char *) * total_num
);
778 std::pair
<name_target_map::iterator
, bool> ret
= map_
.insert(
779 name_target_map::value_type (name
, AuxInfo()));
780 gcc_assert (ret
.second
);
781 ret
.first
->second
.second
= module
;
782 module
->ident
= i
+ 1;
784 module
->ggc_memory
= ggc_memory
;
785 module
->num_quote_paths
= num_array
[1];
786 module
->num_bracket_paths
= num_array
[2];
787 module
->num_system_paths
= num_array
[3];
788 module
->num_cpp_defines
= num_array
[4];
789 module
->num_cpp_includes
= num_array
[5];
790 module
->num_cl_args
= num_array
[6];
791 module
->source_filename
= name
;
792 module
->is_primary
= strcmp (name
, in_fnames
[0]) == 0;
793 module
->flags
= module
->is_primary
? exported
: 1;
794 for (unsigned j
= 0; j
< num_array
[0]; j
++)
795 ret
.first
->second
.first
.push_back (xstrdup (gcov_read_string ()));
796 for (unsigned j
= 0; j
< total_num
- num_array
[0]; j
++)
797 module
->string_array
[j
] = xstrdup (gcov_read_string ());
802 /* Read the profile from the profile file. */
807 if (gcov_open (auto_profile_file
, 1) == 0)
808 error ("Cannot open profile file %s.", auto_profile_file
);
810 if (gcov_read_unsigned () != GCOV_DATA_MAGIC
)
811 error ("AutoFDO profile magic number does not mathch.");
813 /* Skip the version number. */
814 gcov_read_unsigned ();
816 /* Skip the empty integer. */
817 gcov_read_unsigned ();
819 /* function_name_map. */
820 afdo_function_name_map
= function_name_map::create ();
821 if (afdo_function_name_map
== NULL
)
822 error ("Cannot read string table from %s.", auto_profile_file
);
824 /* autofdo_source_profile. */
825 afdo_source_profile
= autofdo_source_profile::create ();
826 if (afdo_source_profile
== NULL
)
827 error ("Cannot read function profile from %s.", auto_profile_file
);
829 /* autofdo_module_profile. */
830 afdo_module_profile
= autofdo_module_profile::create ();
831 if (afdo_module_profile
== NULL
)
832 error ("Cannot read module profile from %s.", auto_profile_file
);
834 /* Read in the working set. */
835 if (gcov_read_unsigned () != GCOV_TAG_AFDO_WORKING_SET
)
836 error ("Cannot read working set from %s.", auto_profile_file
);
838 /* Skip the length of the section. */
839 gcov_read_unsigned ();
840 gcov_working_set_t set
[128];
841 for (unsigned i
= 0; i
< 128; i
++)
843 set
[i
].num_counters
= gcov_read_unsigned ();
844 set
[i
].min_counter
= gcov_read_counter ();
846 add_working_set (set
);
849 /* Read in the auxiliary modules for the current primary module. */
852 read_aux_modules (void)
854 gcov_module_info
*module
= afdo_module_profile
->get_module (in_fnames
[0]);
858 const string_vector
*aux_modules
=
859 afdo_module_profile
->get_aux_modules (in_fnames
[0]);
860 unsigned num_aux_modules
= aux_modules
? aux_modules
->size() : 0;
862 module_infos
= XCNEWVEC (gcov_module_info
*, num_aux_modules
+ 1);
863 module_infos
[0] = module
;
864 primary_module_id
= module
->ident
;
865 if (aux_modules
== NULL
)
867 unsigned curr_module
= 1, max_group
= PARAM_VALUE (PARAM_MAX_LIPO_GROUP
);
868 for (string_vector::const_iterator iter
= aux_modules
->begin();
869 iter
!= aux_modules
->end(); ++iter
)
871 gcov_module_info
*aux_module
= afdo_module_profile
->get_module (*iter
);
872 if (aux_module
== module
)
874 if (aux_module
== NULL
)
877 inform (0, "aux module %s cannot be found.", *iter
);
880 if ((aux_module
->lang
& GCOV_MODULE_LANG_MASK
) !=
881 (module
->lang
& GCOV_MODULE_LANG_MASK
))
884 inform (0, "Not importing %s: source language"
885 " different from primary module's source language", *iter
);
888 if ((aux_module
->lang
& GCOV_MODULE_ASM_STMTS
)
889 && flag_ripa_disallow_asm_modules
)
892 inform (0, "Not importing %s: contains "
893 "assembler statements", *iter
);
896 if (max_group
!= 0 && curr_module
>= max_group
)
899 inform (0, "Not importing %s: maximum group size reached", *iter
);
901 if (incompatible_cl_args (module
, aux_module
))
904 inform (0, "Not importing %s: command-line"
905 " arguments not compatible with primary module", *iter
);
908 module_infos
[curr_module
++] = aux_module
;
909 add_input_filename (*iter
);
913 /* From AutoFDO profiles, find values inside STMT for that we want to measure
914 histograms for indirect-call optimization. */
917 afdo_indirect_call (gimple stmt
, const icall_target_map
&map
)
921 if (map
.size() == 0 || gimple_code (stmt
) != GIMPLE_CALL
922 || gimple_call_fndecl (stmt
) != NULL_TREE
)
925 callee
= gimple_call_fn (stmt
);
927 histogram_value hist
= gimple_alloc_histogram_value (
928 cfun
, HIST_TYPE_INDIR_CALL_TOPN
, stmt
, callee
);
929 hist
->n_counters
= (GCOV_ICALL_TOPN_VAL
<< 2) + 1;
930 hist
->hvalue
.counters
= XNEWVEC (gcov_type
, hist
->n_counters
);
931 gimple_add_histogram_value (cfun
, stmt
, hist
);
934 icall_target_map::const_iterator max_iter1
= map
.end();
935 icall_target_map::const_iterator max_iter2
= map
.end();
937 for (icall_target_map::const_iterator iter
= map
.begin();
938 iter
!= map
.end(); ++iter
)
940 total
+= iter
->second
;
941 if (max_iter1
== map
.end() || max_iter1
->second
< iter
->second
)
943 max_iter2
= max_iter1
;
946 else if (max_iter2
== map
.end() || max_iter2
->second
< iter
->second
)
950 hist
->hvalue
.counters
[0] = total
;
951 hist
->hvalue
.counters
[1] = (unsigned long long)
952 afdo_function_name_map
->get_name (max_iter1
->first
);
953 hist
->hvalue
.counters
[2] = max_iter1
->second
;
954 if (max_iter2
!= map
.end())
956 hist
->hvalue
.counters
[3] = (unsigned long long)
957 afdo_function_name_map
->get_name (max_iter2
->first
);
958 hist
->hvalue
.counters
[4] = max_iter2
->second
;
962 hist
->hvalue
.counters
[3] = 0;
963 hist
->hvalue
.counters
[4] = 0;
967 /* From AutoFDO profiles, find values inside STMT for that we want to measure
968 histograms and adds them to list VALUES. */
971 afdo_vpt (gimple stmt
, const icall_target_map
&map
)
973 afdo_indirect_call (stmt
, map
);
976 /* For a given BB, return its execution count. Add the location of annotated
977 stmt to ANNOTATED. Attach value profile if a stmt is not in PROMOTED,
978 because we only want to promot an indirect call once. */
981 afdo_get_bb_count (basic_block bb
, location_set
*annotated
,
982 const stmt_set
&promoted
)
984 gimple_stmt_iterator gsi
;
985 gcov_type max_count
= 0;
986 bool has_annotated
= false;
988 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
991 gimple stmt
= gsi_stmt (gsi
);
992 if (afdo_source_profile
->get_count_info (stmt
, &info
, annotated
))
994 if (info
.first
> max_count
)
995 max_count
= info
.first
;
996 has_annotated
= true;
997 if (info
.second
.size() > 0 && promoted
.find (stmt
) == promoted
.end ())
998 afdo_vpt (stmt
, info
.second
);
1002 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1005 get_inline_stack (gsi_stmt (gsi
), &stack
);
1006 if (stack
.size() > 0)
1007 annotated
->insert(stack
);
1011 bb
->flags
|= BB_ANNOTATED
;
1018 /* BB1 and BB2 are in an equivalent class iff:
1019 1. BB1 dominates BB2.
1020 2. BB2 post-dominates BB1.
1021 3. BB1 and BB2 are in the same loop nest.
1022 This function finds the equivalent class for each basic block, and
1023 stores a pointer to the first BB in its equivalent class. Meanwhile,
1024 set bb counts for the same equivalent class to be idenical. */
1027 afdo_find_equiv_class (void)
1036 vec
<basic_block
> dom_bbs
;
1040 if (bb
->aux
!= NULL
)
1043 dom_bbs
= get_all_dominated_blocks (CDI_DOMINATORS
, bb
);
1044 FOR_EACH_VEC_ELT (dom_bbs
, i
, bb1
)
1045 if (bb1
->aux
== NULL
1046 && dominated_by_p (CDI_POST_DOMINATORS
, bb
, bb1
)
1047 && bb1
->loop_father
== bb
->loop_father
)
1050 if (bb1
->count
> bb
->count
&& (bb1
->flags
& BB_ANNOTATED
) != 0)
1052 bb
->count
= MAX (bb
->count
, bb1
->count
);
1053 bb
->flags
|= BB_ANNOTATED
;
1056 dom_bbs
= get_all_dominated_blocks (CDI_POST_DOMINATORS
, bb
);
1057 FOR_EACH_VEC_ELT (dom_bbs
, i
, bb1
)
1058 if (bb1
->aux
== NULL
1059 && dominated_by_p (CDI_DOMINATORS
, bb
, bb1
)
1060 && bb1
->loop_father
== bb
->loop_father
)
1063 if (bb1
->count
> bb
->count
&& (bb1
->flags
& BB_ANNOTATED
) != 0)
1065 bb
->count
= MAX (bb
->count
, bb1
->count
);
1066 bb
->flags
|= BB_ANNOTATED
;
1072 /* If a baisk block only has one in/out edge, then the bb count and he
1073 edge count should be the same.
1074 IS_SUCC is true if the out edge of the basic block is examined.
1075 Return TRUE if any basic block/edge count is changed. */
1078 afdo_propagate_single_edge (bool is_succ
)
1081 bool changed
= false;
1084 if (is_succ
? single_succ_p (bb
) : single_pred_p (bb
))
1086 edge e
= is_succ
? single_succ_edge (bb
) : single_pred_edge (bb
);
1087 if (((e
->flags
& EDGE_ANNOTATED
) == 0)
1088 && ((bb
->flags
& BB_ANNOTATED
) != 0))
1090 e
->count
= bb
->count
;
1091 e
->flags
|= EDGE_ANNOTATED
;
1094 else if (((e
->flags
& EDGE_ANNOTATED
) != 0)
1095 && ((bb
->flags
& BB_ANNOTATED
) == 0))
1097 bb
->count
= e
->count
;
1098 bb
->flags
|= BB_ANNOTATED
;
1101 else if (bb
->count
!= e
->count
)
1103 e
->count
= bb
->count
= MAX (bb
->count
, e
->count
);
1110 /* If a basic block's count is known, and only one of its in/out edges' count
1111 is unknown, its count can be calculated.
1112 Meanwhile, if all of the in/out edges' counts are known, then the basic
1113 block's unknown count can also be calculated.
1114 IS_SUCC is true if out edges of a basic blocks are examined.
1115 Return TRUE if any basic block/edge count is changed. */
1118 afdo_propagate_multi_edge (bool is_succ
)
1121 bool changed
= false;
1125 edge e
, unknown_edge
= NULL
;
1127 int num_unknown_edge
= 0;
1128 gcov_type total_known_count
= 0;
1130 FOR_EACH_EDGE (e
, ei
, is_succ
? bb
->succs
: bb
->preds
)
1131 if ((e
->flags
& EDGE_ANNOTATED
) == 0)
1132 num_unknown_edge
++, unknown_edge
= e
;
1134 total_known_count
+= e
->count
;
1136 if (num_unknown_edge
== 0)
1138 if (total_known_count
> bb
->count
)
1140 bb
->count
= total_known_count
;
1143 if ((bb
->flags
& BB_ANNOTATED
) == 0)
1145 bb
->flags
|= BB_ANNOTATED
;
1149 else if (num_unknown_edge
== 1
1150 && (bb
->flags
& BB_ANNOTATED
) != 0)
1152 if (bb
->count
>= total_known_count
)
1153 unknown_edge
->count
= bb
->count
- total_known_count
;
1155 unknown_edge
->count
= 0;
1156 unknown_edge
->flags
|= EDGE_ANNOTATED
;
1163 /* Special propagation for circuit expressions. Because GCC translates
1164 control flow into data flow for circuit expressions. E.g.
1171 will be translated into:
1186 tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
1192 In this case, we need to propagate through PHI to determine the edge
1193 count of BB1->BB.t1, BB.t1->BB.t2. */
1196 afdo_propagate_circuit (void)
1202 tree cmp_rhs
, cmp_lhs
;
1203 gimple cmp_stmt
= last_stmt (bb
);
1207 if (!cmp_stmt
|| gimple_code (cmp_stmt
) != GIMPLE_COND
)
1209 cmp_rhs
= gimple_cond_rhs (cmp_stmt
);
1210 cmp_lhs
= gimple_cond_lhs (cmp_stmt
);
1211 if (!TREE_CONSTANT (cmp_rhs
)
1212 || !(integer_zerop (cmp_rhs
) || integer_onep (cmp_rhs
)))
1214 if (TREE_CODE (cmp_lhs
) != SSA_NAME
)
1216 if ((bb
->flags
& BB_ANNOTATED
) == 0)
1218 phi_stmt
= SSA_NAME_DEF_STMT (cmp_lhs
);
1219 while (phi_stmt
&& gimple_code (phi_stmt
) == GIMPLE_ASSIGN
1220 && gimple_assign_single_p (phi_stmt
)
1221 && TREE_CODE (gimple_assign_rhs1 (phi_stmt
)) == SSA_NAME
)
1222 phi_stmt
= SSA_NAME_DEF_STMT (gimple_assign_rhs1 (phi_stmt
));
1223 if (!phi_stmt
|| gimple_code (phi_stmt
) != GIMPLE_PHI
)
1225 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1227 unsigned i
, total
= 0;
1229 bool check_value_one
= (((integer_onep (cmp_rhs
))
1230 ^ (gimple_cond_code (cmp_stmt
) == EQ_EXPR
))
1231 ^ ((e
->flags
& EDGE_TRUE_VALUE
) != 0));
1232 if ((e
->flags
& EDGE_ANNOTATED
) == 0)
1234 for (i
= 0; i
< gimple_phi_num_args (phi_stmt
); i
++)
1236 tree val
= gimple_phi_arg_def (phi_stmt
, i
);
1237 edge ep
= gimple_phi_arg_edge (phi_stmt
, i
);
1239 if (!TREE_CONSTANT (val
) || !(integer_zerop (val
)
1240 || integer_onep (val
)))
1242 if (check_value_one
^ integer_onep (val
))
1246 if (e
->probability
== 0 && (e
->flags
& EDGE_ANNOTATED
) == 0)
1248 ep
->probability
= 0;
1250 ep
->flags
|= EDGE_ANNOTATED
;
1253 if (total
== 1 && (only_one
->flags
& EDGE_ANNOTATED
) == 0)
1255 only_one
->probability
= e
->probability
;
1256 only_one
->count
= e
->count
;
1257 only_one
->flags
|= EDGE_ANNOTATED
;
1263 /* Propagate the basic block count and edge count on the control flow
1264 graph. We do the propagation iteratively until stablize. */
1267 afdo_propagate (void)
1270 bool changed
= true;
1275 bb
->count
= ((basic_block
) bb
->aux
)->count
;
1276 if ((((basic_block
) bb
->aux
)->flags
& BB_ANNOTATED
) != 0)
1277 bb
->flags
|= BB_ANNOTATED
;
1280 while (changed
&& i
++ < PARAM_VALUE (PARAM_AUTOFDO_MAX_PROPAGATE_ITERATIONS
))
1284 if (afdo_propagate_single_edge (true))
1286 if (afdo_propagate_single_edge (false))
1288 if (afdo_propagate_multi_edge (true))
1290 if (afdo_propagate_multi_edge (false))
1292 afdo_propagate_circuit ();
1296 /* Propagate counts on control flow graph and calculate branch
1300 afdo_calculate_branch_prob (void)
1303 bool has_sample
= false;
1312 calculate_dominance_info (CDI_POST_DOMINATORS
);
1313 calculate_dominance_info (CDI_DOMINATORS
);
1314 loop_optimizer_init (0);
1316 afdo_find_equiv_class ();
1323 int num_unknown_succ
= 0;
1324 gcov_type total_count
= 0;
1326 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1328 if ((e
->flags
& EDGE_ANNOTATED
) == 0)
1329 num_unknown_succ
++;
1331 total_count
+= e
->count
;
1333 if (num_unknown_succ
== 0 && total_count
> 0)
1335 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1337 (double) e
->count
* REG_BR_PROB_BASE
/ total_count
;
1345 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1347 (double) bb
->count
* e
->probability
/ REG_BR_PROB_BASE
;
1351 loop_optimizer_finalize ();
1352 free_dominance_info (CDI_DOMINATORS
);
1353 free_dominance_info (CDI_POST_DOMINATORS
);
1356 /* Perform value profile transformation using AutoFDO profile. Add the
1357 promoted stmts to PROMOTED_STMTS. Return TRUE if there is any
1358 indirect call promoted. */
1361 afdo_vpt_for_early_inline (stmt_set
*promoted_stmts
)
1364 if (afdo_source_profile
->get_function_instance_by_decl (
1365 current_function_decl
) == NULL
)
1368 bool has_vpt
= false;
1371 if (!has_indirect_call (bb
))
1373 gimple_stmt_iterator gsi
;
1375 gcov_type bb_count
= 0;
1376 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1379 gimple stmt
= gsi_stmt (gsi
);
1380 if (afdo_source_profile
->get_count_info (stmt
, &info
, NULL
))
1381 bb_count
= MAX (bb_count
, info
.first
);
1384 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1386 gimple stmt
= gsi_stmt (gsi
);
1387 /* IC_promotion and early_inline_2 is done in multiple iterations.
1388 No need to promoted the stmt if its in promoted_stmts (means
1389 it is already been promoted in the previous iterations). */
1390 if (gimple_code (stmt
) != GIMPLE_CALL
1391 || TREE_CODE (gimple_call_fn (stmt
)) == FUNCTION_DECL
1392 || promoted_stmts
->find (stmt
) != promoted_stmts
->end ())
1396 afdo_source_profile
->get_count_info (stmt
, &info
, NULL
);
1397 info
.first
= bb_count
;
1398 if (afdo_source_profile
->update_inlined_ind_target (stmt
, &info
))
1400 /* Promote the indirect call and update the promoted_stmts. */
1401 promoted_stmts
->insert (stmt
);
1402 afdo_vpt (stmt
, info
.second
);
1407 if (has_vpt
&& gimple_value_profile_transformations ())
1409 free_dominance_info (CDI_DOMINATORS
);
1410 free_dominance_info (CDI_POST_DOMINATORS
);
1411 calculate_dominance_info (CDI_POST_DOMINATORS
);
1412 calculate_dominance_info (CDI_DOMINATORS
);
1413 rebuild_cgraph_edges ();
1414 update_ssa (TODO_update_ssa
);
1415 compute_inline_parameters (cgraph_get_node (current_function_decl
),
1423 /* Annotate auto profile to the control flow graph. Do not annotate value
1424 profile for stmts in PROMOTED_STMTS. */
1427 afdo_annotate_cfg (const stmt_set
&promoted_stmts
)
1430 const function_instance
*s
=
1431 afdo_source_profile
->get_function_instance_by_decl (
1432 current_function_decl
);
1436 cgraph_get_node (current_function_decl
)->count
= s
->head_count ();
1437 ENTRY_BLOCK_PTR
->count
= s
->head_count ();
1438 gcov_type max_count
= ENTRY_BLOCK_PTR
->count
;
1440 location_set annotated_locs
;
1448 bb
->flags
&= (~BB_ANNOTATED
);
1449 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1452 e
->flags
&= (~EDGE_ANNOTATED
);
1455 bb
->count
= afdo_get_bb_count (bb
, &annotated_locs
, promoted_stmts
);
1456 if (bb
->count
> max_count
)
1457 max_count
= bb
->count
;
1459 if (ENTRY_BLOCK_PTR
->count
> ENTRY_BLOCK_PTR
->next_bb
->count
)
1461 ENTRY_BLOCK_PTR
->next_bb
->count
= ENTRY_BLOCK_PTR
->count
;
1462 ENTRY_BLOCK_PTR
->next_bb
->flags
|= BB_ANNOTATED
;
1464 if (ENTRY_BLOCK_PTR
->count
> EXIT_BLOCK_PTR
->prev_bb
->count
)
1466 EXIT_BLOCK_PTR
->prev_bb
->count
= ENTRY_BLOCK_PTR
->count
;
1467 EXIT_BLOCK_PTR
->prev_bb
->flags
|= BB_ANNOTATED
;
1471 afdo_calculate_branch_prob ();
1473 profile_status
= PROFILE_READ
;
1475 if (flag_value_profile_transformations
)
1476 gimple_value_profile_transformations ();
1478 } /* namespace autofdo. */
1480 /* Use AutoFDO profile to annoate the control flow graph.
1481 Return the todo flag. */
1486 struct cgraph_node
*node
;
1488 if (cgraph_state
== CGRAPH_STATE_FINISHED
)
1492 profile_info
= autofdo::afdo_profile_info
;
1494 FOR_EACH_FUNCTION (node
)
1496 if (!gimple_has_body_p (node
->symbol
.decl
))
1499 /* Don't profile functions produced for builtin stuff. */
1500 if (DECL_SOURCE_LOCATION (node
->symbol
.decl
) == BUILTINS_LOCATION
)
1503 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
1505 /* First do indirect call promotion and early inline to make the
1506 IR match the profiled binary before actual annotation.
1508 This is needed because an indirect call might have been promoted
1509 and inlined in the profiled binary. If we do not promote and
1510 inline these indirect calls before annotation, the profile for
1511 these promoted functions will be lost.
1513 e.g. foo() --indirect_call--> bar()
1514 In profiled binary, the callsite is promoted and inlined, making
1515 the profile look like:
1525 Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
1526 If we perform annotation on it, the profile inside bar@loc_foo2
1529 To avoid this, we promote loc_foo_2 and inline the promoted bar
1530 function before annotation, so the profile inside bar@loc_foo2
1532 autofdo::stmt_set promoted_stmts
;
1533 for (int i
= 0; i
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
); i
++)
1535 if (!flag_value_profile_transformations
1536 || !autofdo::afdo_vpt_for_early_inline (&promoted_stmts
))
1541 autofdo::afdo_annotate_cfg (promoted_stmts
);
1542 compute_function_frequency ();
1543 update_ssa (TODO_update_ssa
);
1547 cgraph_pre_profiling_inlining_done
= true;
1548 cgraph_process_module_scope_statics ();
1549 /* Now perform link to allow cross module inlining. */
1552 cgraph_unify_type_alias_sets ();
1554 FOR_EACH_FUNCTION (node
)
1556 if (!gimple_has_body_p (node
->symbol
.decl
))
1559 /* Don't profile functions produced for builtin stuff. */
1560 if (DECL_SOURCE_LOCATION (node
->symbol
.decl
) == BUILTINS_LOCATION
)
1563 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
1565 if (L_IPO_COMP_MODE
)
1570 gimple_stmt_iterator gsi
;
1571 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1573 gimple stmt
= gsi_stmt (gsi
);
1574 if (is_gimple_call (stmt
))
1575 lipo_fixup_cgraph_edge_call_target (stmt
);
1579 /* Local pure-const may imply need to fixup the cfg. */
1580 if (execute_fixup_cfg () & TODO_cleanup_cfg
)
1581 cleanup_tree_cfg ();
1583 rebuild_cgraph_edges ();
1590 gate_auto_profile_ipa (void)
1592 return flag_auto_profile
;
1595 /* Read the profile from the profile data file. */
1598 init_auto_profile (void)
1600 if (auto_profile_file
== NULL
)
1601 auto_profile_file
= DEFAULT_AUTO_PROFILE_FILE
;
1603 autofdo::afdo_profile_info
= (struct gcov_ctr_summary
*)
1604 xcalloc (1, sizeof (struct gcov_ctr_summary
));
1605 autofdo::afdo_profile_info
->runs
= 1;
1606 autofdo::afdo_profile_info
->sum_max
= 0;
1607 autofdo::afdo_profile_info
->sum_all
= 0;
1609 /* Read the profile from the profile file. */
1610 autofdo::read_profile ();
1613 autofdo::read_aux_modules ();
1616 /* Free the resources. */
1619 end_auto_profile (void)
1621 delete autofdo::afdo_source_profile
;
1622 delete autofdo::afdo_function_name_map
;
1623 delete autofdo::afdo_module_profile
;
1624 profile_info
= NULL
;
1627 /* Returns TRUE if EDGE is hot enough to be inlined early. */
1630 afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge
*edge
)
1633 autofdo::afdo_source_profile
->get_callsite_total_count (edge
);
1637 const struct gcov_ctr_summary
*saved_profile_info
= profile_info
;
1638 /* At earling inline stage, profile_info is not set yet. We need to
1639 temporarily set it to afdo_profile_info to calculate hotness. */
1640 profile_info
= autofdo::afdo_profile_info
;
1641 is_hot
= maybe_hot_count_p (NULL
, count
);
1642 profile_info
= saved_profile_info
;
1649 struct simple_ipa_opt_pass pass_ipa_auto_profile
=
1654 OPTGROUP_NONE
, /* optinfo_flags */
1655 gate_auto_profile_ipa
, /* gate */
1656 auto_profile
, /* execute */
1659 0, /* static_pass_number */
1660 TV_IPA_AUTOFDO
, /* tv_id */
1661 0, /* properties_required */
1662 0, /* properties_provided */
1663 0, /* properties_destroyed */
1664 0, /* todo_flags_start */
1665 0 /* todo_flags_finish */