1 /* Read and annotate call graph profile from the auto profile data file.
2 Copyright (C) 2014-2017 Free Software Foundation, Inc.
3 Contributed by Dehao Chen (dehao@google.com)
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "alloc-pool.h"
31 #include "tree-pass.h"
35 #include "diagnostic-core.h"
37 #include "langhooks.h"
40 #include "tree-cfgcleanup.h"
41 #include "tree-into-ssa.h"
42 #include "gimple-iterator.h"
43 #include "value-prof.h"
45 #include "symbol-summary.h"
47 #include "ipa-fnsummary.h"
48 #include "ipa-inline.h"
49 #include "tree-inline.h"
50 #include "auto-profile.h"
51 #include "tree-pretty-print.h"
52 #include "gimple-pretty-print.h"
54 /* The following routines implements AutoFDO optimization.
56 This optimization uses sampling profiles to annotate basic block counts
57 and uses heuristics to estimate branch probabilities.
59 There are three phases in AutoFDO:
61 Phase 1: Read profile from the profile data file.
62 The following info is read from the profile datafile:
63 * string_table: a map between function name and its index.
64 * autofdo_source_profile: a map from function_instance name to
65 function_instance. This is represented as a forest of
67 * WorkingSet: a histogram of how many instructions are covered for a
68 given percentage of total cycles. This is describing the binary
69 level information (not source level). This info is used to help
70 decide if we want aggressive optimizations that could increase
71 code footprint (e.g. loop unroll etc.)
72 A function instance is an instance of function that could either be a
73 standalone symbol, or a clone of a function that is inlined into another
76 Phase 2: Early inline + value profile transformation.
77 Early inline uses autofdo_source_profile to find if a callsite is:
78 * inlined in the profiled binary.
79 * callee body is hot in the profiling run.
80 If both condition satisfies, early inline will inline the callsite
81 regardless of the code growth.
82 Phase 2 is an iterative process. During each iteration, we also check
83 if an indirect callsite is promoted and inlined in the profiling run.
84 If yes, vpt will happen to force promote it and in the next iteration,
85 einline will inline the promoted callsite in the next iteration.
87 Phase 3: Annotate control flow graph.
88 AutoFDO uses a separate pass to:
89 * Annotate basic block count
90 * Estimate branch probability
92 After the above 3 phases, all profile is readily annotated on the GCC IR.
93 AutoFDO tries to reuse all FDO infrastructure as much as possible to make
94 use of the profile. E.g. it uses existing mechanism to calculate the basic
95 block/edge frequency, as well as the cgraph node/edge count.
98 #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
99 #define AUTO_PROFILE_VERSION 1
104 /* Represent a source location: (function_decl, lineno). */
105 typedef std::pair
<tree
, unsigned> decl_lineno
;
107 /* Represent an inline stack. vector[0] is the leaf node. */
108 typedef auto_vec
<decl_lineno
> inline_stack
;
110 /* String array that stores function names. */
111 typedef auto_vec
<char *> string_vector
;
113 /* Map from function name's index in string_table to target's
115 typedef std::map
<unsigned, gcov_type
> icall_target_map
;
117 /* Set of gimple stmts. Used to track if the stmt has already been promoted
119 typedef std::set
<gimple
*> stmt_set
;
121 /* Represent count info of an inline stack. */
124 /* Sampled count of the inline stack. */
127 /* Map from indirect call target to its sample count. */
128 icall_target_map targets
;
130 /* Whether this inline stack is already used in annotation.
132 Each inline stack should only be used to annotate IR once.
133 This will be enforced when instruction-level discriminator
138 /* operator< for "const char *". */
139 struct string_compare
141 bool operator()(const char *a
, const char *b
) const
143 return strcmp (a
, b
) < 0;
147 /* Store a string array, indexed by string position in the array. */
156 /* For a given string, returns its index. */
157 int get_index (const char *name
) const;
159 /* For a given decl, returns the index of the decl name. */
160 int get_index_by_decl (tree decl
) const;
162 /* For a given index, returns the string. */
163 const char *get_name (int index
) const;
165 /* Read profile, return TRUE on success. */
169 typedef std::map
<const char *, unsigned, string_compare
> string_index_map
;
170 string_vector vector_
;
171 string_index_map map_
;
174 /* Profile of a function instance:
175 1. total_count of the function.
176 2. head_count (entry basic block count) of the function (only valid when
177 function is a top-level function_instance, i.e. it is the original copy
178 instead of the inlined copy).
179 3. map from source location (decl_lineno) to profile (count_info).
180 4. map from callsite to callee function_instance. */
181 class function_instance
184 typedef auto_vec
<function_instance
*> function_instance_stack
;
186 /* Read the profile and return a function_instance with head count as
187 HEAD_COUNT. Recursively read callsites to create nested function_instances
188 too. STACK is used to track the recursive creation process. */
189 static function_instance
*
190 read_function_instance (function_instance_stack
*stack
,
191 gcov_type head_count
);
193 /* Recursively deallocate all callsites (nested function_instances). */
194 ~function_instance ();
213 /* Traverse callsites of the current function_instance to find one at the
214 location of LINENO and callee name represented in DECL. */
215 function_instance
*get_function_instance_by_decl (unsigned lineno
,
218 /* Store the profile info for LOC in INFO. Return TRUE if profile info
220 bool get_count_info (location_t loc
, count_info
*info
) const;
222 /* Read the inlined indirect call target profile for STMT and store it in
223 MAP, return the total count for all inlined indirect calls. */
224 gcov_type
find_icall_target_map (gcall
*stmt
, icall_target_map
*map
) const;
226 /* Sum of counts that is used during annotation. */
227 gcov_type
total_annotated_count () const;
229 /* Mark LOC as annotated. */
230 void mark_annotated (location_t loc
);
233 /* Callsite, represented as (decl_lineno, callee_function_name_index). */
234 typedef std::pair
<unsigned, unsigned> callsite
;
236 /* Map from callsite to callee function_instance. */
237 typedef std::map
<callsite
, function_instance
*> callsite_map
;
239 function_instance (unsigned name
, gcov_type head_count
)
240 : name_ (name
), total_count_ (0), head_count_ (head_count
)
244 /* Map from source location (decl_lineno) to profile (count_info). */
245 typedef std::map
<unsigned, count_info
> position_count_map
;
247 /* function_instance name index in the string_table. */
250 /* Total sample count. */
251 gcov_type total_count_
;
253 /* Entry BB's sample count. */
254 gcov_type head_count_
;
256 /* Map from callsite location to callee function_instance. */
257 callsite_map callsites
;
259 /* Map from source location to count_info. */
260 position_count_map pos_counts
;
263 /* Profile for all functions. */
264 class autofdo_source_profile
267 static autofdo_source_profile
*
270 autofdo_source_profile
*map
= new autofdo_source_profile ();
278 ~autofdo_source_profile ();
280 /* For a given DECL, returns the top-level function_instance. */
281 function_instance
*get_function_instance_by_decl (tree decl
) const;
283 /* Find count_info for a given gimple STMT. If found, store the count_info
284 in INFO and return true; otherwise return false. */
285 bool get_count_info (gimple
*stmt
, count_info
*info
) const;
287 /* Find total count of the callee of EDGE. */
288 gcov_type
get_callsite_total_count (struct cgraph_edge
*edge
) const;
290 /* Update value profile INFO for STMT from the inlined indirect callsite.
291 Return true if INFO is updated. */
292 bool update_inlined_ind_target (gcall
*stmt
, count_info
*info
);
294 /* Mark LOC as annotated. */
295 void mark_annotated (location_t loc
);
298 /* Map from function_instance name index (in string_table) to
299 function_instance. */
300 typedef std::map
<unsigned, function_instance
*> name_function_instance_map
;
302 autofdo_source_profile () {}
304 /* Read AutoFDO profile and returns TRUE on success. */
307 /* Return the function_instance in the profile that correspond to the
310 get_function_instance_by_inline_stack (const inline_stack
&stack
) const;
312 name_function_instance_map map_
;
315 /* Store the strings read from the profile data file. */
316 static string_table
*afdo_string_table
;
318 /* Store the AutoFDO source profile. */
319 static autofdo_source_profile
*afdo_source_profile
;
321 /* gcov_ctr_summary structure to store the profile_info. */
322 static struct gcov_ctr_summary
*afdo_profile_info
;
324 /* Helper functions. */
326 /* Return the original name of NAME: strip the suffix that starts
327 with '.' Caller is responsible for freeing RET. */
330 get_original_name (const char *name
)
332 char *ret
= xstrdup (name
);
333 char *find
= strchr (ret
, '.');
339 /* Return the combined location, which is a 32bit integer in which
340 higher 16 bits stores the line offset of LOC to the start lineno
341 of DECL, The lower 16 bits stores the discriminator. */
344 get_combined_location (location_t loc
, tree decl
)
346 /* TODO: allow more bits for line and less bits for discriminator. */
347 if (LOCATION_LINE (loc
) - DECL_SOURCE_LINE (decl
) >= (1<<16))
348 warning_at (loc
, OPT_Woverflow
, "offset exceeds 16 bytes");
349 return ((LOCATION_LINE (loc
) - DECL_SOURCE_LINE (decl
)) << 16);
352 /* Return the function decl of a given lexical BLOCK. */
355 get_function_decl_from_block (tree block
)
359 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
) == UNKNOWN_LOCATION
))
362 for (decl
= BLOCK_ABSTRACT_ORIGIN (block
);
363 decl
&& (TREE_CODE (decl
) == BLOCK
);
364 decl
= BLOCK_ABSTRACT_ORIGIN (decl
))
365 if (TREE_CODE (decl
) == FUNCTION_DECL
)
370 /* Store inline stack for STMT in STACK. */
373 get_inline_stack (location_t locus
, inline_stack
*stack
)
375 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
378 tree block
= LOCATION_BLOCK (locus
);
379 if (block
&& TREE_CODE (block
) == BLOCK
)
382 for (block
= BLOCK_SUPERCONTEXT (block
);
383 block
&& (TREE_CODE (block
) == BLOCK
);
384 block
= BLOCK_SUPERCONTEXT (block
))
386 location_t tmp_locus
= BLOCK_SOURCE_LOCATION (block
);
387 if (LOCATION_LOCUS (tmp_locus
) == UNKNOWN_LOCATION
)
390 tree decl
= get_function_decl_from_block (block
);
392 std::make_pair (decl
, get_combined_location (locus
, decl
)));
398 std::make_pair (current_function_decl
,
399 get_combined_location (locus
, current_function_decl
)));
402 /* Return STMT's combined location, which is a 32bit integer in which
403 higher 16 bits stores the line offset of LOC to the start lineno
404 of DECL, The lower 16 bits stores the discriminator. */
407 get_relative_location_for_stmt (gimple
*stmt
)
409 location_t locus
= gimple_location (stmt
);
410 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
411 return UNKNOWN_LOCATION
;
413 for (tree block
= gimple_block (stmt
); block
&& (TREE_CODE (block
) == BLOCK
);
414 block
= BLOCK_SUPERCONTEXT (block
))
415 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
)) != UNKNOWN_LOCATION
)
416 return get_combined_location (locus
,
417 get_function_decl_from_block (block
));
418 return get_combined_location (locus
, current_function_decl
);
421 /* Return true if BB contains indirect call. */
424 has_indirect_call (basic_block bb
)
426 gimple_stmt_iterator gsi
;
428 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
430 gimple
*stmt
= gsi_stmt (gsi
);
431 if (gimple_code (stmt
) == GIMPLE_CALL
&& !gimple_call_internal_p (stmt
)
432 && (gimple_call_fn (stmt
) == NULL
433 || TREE_CODE (gimple_call_fn (stmt
)) != FUNCTION_DECL
))
439 /* Member functions for string_table. */
443 string_table::~string_table ()
445 for (unsigned i
= 0; i
< vector_
.length (); i
++)
450 /* Return the index of a given function NAME. Return -1 if NAME is not
451 found in string table. */
454 string_table::get_index (const char *name
) const
458 string_index_map::const_iterator iter
= map_
.find (name
);
459 if (iter
== map_
.end ())
465 /* Return the index of a given function DECL. Return -1 if DECL is not
466 found in string table. */
469 string_table::get_index_by_decl (tree decl
) const
472 = get_original_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)));
473 int ret
= get_index (name
);
477 ret
= get_index (lang_hooks
.dwarf_name (decl
, 0));
480 if (DECL_ABSTRACT_ORIGIN (decl
))
481 return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl
));
486 /* Return the function name of a given INDEX. */
489 string_table::get_name (int index
) const
491 gcc_assert (index
> 0 && index
< (int)vector_
.length ());
492 return vector_
[index
];
495 /* Read the string table. Return TRUE if reading is successful. */
498 string_table::read ()
500 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES
)
502 /* Skip the length of the section. */
503 gcov_read_unsigned ();
504 /* Read in the file name table. */
505 unsigned string_num
= gcov_read_unsigned ();
506 for (unsigned i
= 0; i
< string_num
; i
++)
508 vector_
.safe_push (get_original_name (gcov_read_string ()));
509 map_
[vector_
.last ()] = i
;
514 /* Member functions for function_instance. */
516 function_instance::~function_instance ()
518 for (callsite_map::iterator iter
= callsites
.begin ();
519 iter
!= callsites
.end (); ++iter
)
523 /* Traverse callsites of the current function_instance to find one at the
524 location of LINENO and callee name represented in DECL. */
527 function_instance::get_function_instance_by_decl (unsigned lineno
,
530 int func_name_idx
= afdo_string_table
->get_index_by_decl (decl
);
531 if (func_name_idx
!= -1)
533 callsite_map::const_iterator ret
534 = callsites
.find (std::make_pair (lineno
, func_name_idx
));
535 if (ret
!= callsites
.end ())
539 = afdo_string_table
->get_index (lang_hooks
.dwarf_name (decl
, 0));
540 if (func_name_idx
!= -1)
542 callsite_map::const_iterator ret
543 = callsites
.find (std::make_pair (lineno
, func_name_idx
));
544 if (ret
!= callsites
.end ())
547 if (DECL_ABSTRACT_ORIGIN (decl
))
548 return get_function_instance_by_decl (lineno
, DECL_ABSTRACT_ORIGIN (decl
));
553 /* Store the profile info for LOC in INFO. Return TRUE if profile info
557 function_instance::get_count_info (location_t loc
, count_info
*info
) const
559 position_count_map::const_iterator iter
= pos_counts
.find (loc
);
560 if (iter
== pos_counts
.end ())
562 *info
= iter
->second
;
566 /* Mark LOC as annotated. */
569 function_instance::mark_annotated (location_t loc
)
571 position_count_map::iterator iter
= pos_counts
.find (loc
);
572 if (iter
== pos_counts
.end ())
574 iter
->second
.annotated
= true;
577 /* Read the inlined indirect call target profile for STMT and store it in
578 MAP, return the total count for all inlined indirect calls. */
581 function_instance::find_icall_target_map (gcall
*stmt
,
582 icall_target_map
*map
) const
585 unsigned stmt_offset
= get_relative_location_for_stmt (stmt
);
587 for (callsite_map::const_iterator iter
= callsites
.begin ();
588 iter
!= callsites
.end (); ++iter
)
590 unsigned callee
= iter
->second
->name ();
591 /* Check if callsite location match the stmt. */
592 if (iter
->first
.first
!= stmt_offset
)
594 struct cgraph_node
*node
= cgraph_node::get_for_asmname (
595 get_identifier (afdo_string_table
->get_name (callee
)));
598 if (!check_ic_target (stmt
, node
))
600 (*map
)[callee
] = iter
->second
->total_count ();
601 ret
+= iter
->second
->total_count ();
606 /* Read the profile and create a function_instance with head count as
607 HEAD_COUNT. Recursively read callsites to create nested function_instances
608 too. STACK is used to track the recursive creation process. */
610 /* function instance profile format:
614 NUM_POS_COUNTS: 4 bytes
615 NUM_CALLSITES: 4 byte
617 POS_1_OFFSET: 4 bytes
621 VALUE_PROFILE_TYPE: 4 bytes
631 CALLSITE_1_OFFSET: 4 bytes
632 FUNCTION_INSTANCE_PROFILE (nested)
638 function_instance::read_function_instance (function_instance_stack
*stack
,
639 gcov_type head_count
)
641 unsigned name
= gcov_read_unsigned ();
642 unsigned num_pos_counts
= gcov_read_unsigned ();
643 unsigned num_callsites
= gcov_read_unsigned ();
644 function_instance
*s
= new function_instance (name
, head_count
);
645 stack
->safe_push (s
);
647 for (unsigned i
= 0; i
< num_pos_counts
; i
++)
649 unsigned offset
= gcov_read_unsigned () & 0xffff0000;
650 unsigned num_targets
= gcov_read_unsigned ();
651 gcov_type count
= gcov_read_counter ();
652 s
->pos_counts
[offset
].count
= count
;
653 for (unsigned j
= 0; j
< stack
->length (); j
++)
654 (*stack
)[j
]->total_count_
+= count
;
655 for (unsigned j
= 0; j
< num_targets
; j
++)
657 /* Only indirect call target histogram is supported now. */
658 gcov_read_unsigned ();
659 gcov_type target_idx
= gcov_read_counter ();
660 s
->pos_counts
[offset
].targets
[target_idx
] = gcov_read_counter ();
663 for (unsigned i
= 0; i
< num_callsites
; i
++)
665 unsigned offset
= gcov_read_unsigned ();
666 function_instance
*callee_function_instance
667 = read_function_instance (stack
, 0);
668 s
->callsites
[std::make_pair (offset
, callee_function_instance
->name ())]
669 = callee_function_instance
;
675 /* Sum of counts that is used during annotation. */
678 function_instance::total_annotated_count () const
681 for (callsite_map::const_iterator iter
= callsites
.begin ();
682 iter
!= callsites
.end (); ++iter
)
683 ret
+= iter
->second
->total_annotated_count ();
684 for (position_count_map::const_iterator iter
= pos_counts
.begin ();
685 iter
!= pos_counts
.end (); ++iter
)
686 if (iter
->second
.annotated
)
687 ret
+= iter
->second
.count
;
691 /* Member functions for autofdo_source_profile. */
693 autofdo_source_profile::~autofdo_source_profile ()
695 for (name_function_instance_map::const_iterator iter
= map_
.begin ();
696 iter
!= map_
.end (); ++iter
)
700 /* For a given DECL, returns the top-level function_instance. */
703 autofdo_source_profile::get_function_instance_by_decl (tree decl
) const
705 int index
= afdo_string_table
->get_index_by_decl (decl
);
708 name_function_instance_map::const_iterator ret
= map_
.find (index
);
709 return ret
== map_
.end () ? NULL
: ret
->second
;
712 /* Find count_info for a given gimple STMT. If found, store the count_info
713 in INFO and return true; otherwise return false. */
716 autofdo_source_profile::get_count_info (gimple
*stmt
, count_info
*info
) const
718 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
722 get_inline_stack (gimple_location (stmt
), &stack
);
723 if (stack
.length () == 0)
725 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
728 return s
->get_count_info (stack
[0].second
, info
);
731 /* Mark LOC as annotated. */
734 autofdo_source_profile::mark_annotated (location_t loc
)
737 get_inline_stack (loc
, &stack
);
738 if (stack
.length () == 0)
740 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
743 s
->mark_annotated (stack
[0].second
);
746 /* Update value profile INFO for STMT from the inlined indirect callsite.
747 Return true if INFO is updated. */
750 autofdo_source_profile::update_inlined_ind_target (gcall
*stmt
,
755 fprintf (dump_file
, "Checking indirect call -> direct call ");
756 print_gimple_stmt (dump_file
, stmt
, 0, TDF_SLIM
);
759 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
762 fprintf (dump_file
, " good locus\n");
767 get_count_info (stmt
, &old_info
);
769 for (icall_target_map::const_iterator iter
= old_info
.targets
.begin ();
770 iter
!= old_info
.targets
.end (); ++iter
)
771 total
+= iter
->second
;
773 /* Program behavior changed, original promoted (and inlined) target is not
774 hot any more. Will avoid promote the original target.
776 To check if original promoted target is still hot, we check the total
777 count of the unpromoted targets (stored in old_info). If it is no less
778 than half of the callsite count (stored in INFO), the original promoted
779 target is considered not hot any more. */
780 if (total
>= info
->count
/ 2)
783 fprintf (dump_file
, " not hot anymore %ld >= %ld",
785 (long)info
->count
/2);
790 get_inline_stack (gimple_location (stmt
), &stack
);
791 if (stack
.length () == 0)
794 fprintf (dump_file
, " no inline stack\n");
797 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
801 fprintf (dump_file
, " function not found in inline stack\n");
804 icall_target_map map
;
805 if (s
->find_icall_target_map (stmt
, &map
) == 0)
808 fprintf (dump_file
, " no target map\n");
811 for (icall_target_map::const_iterator iter
= map
.begin ();
812 iter
!= map
.end (); ++iter
)
813 info
->targets
[iter
->first
] = iter
->second
;
815 fprintf (dump_file
, " looks good\n");
819 /* Find total count of the callee of EDGE. */
822 autofdo_source_profile::get_callsite_total_count (
823 struct cgraph_edge
*edge
) const
826 stack
.safe_push (std::make_pair (edge
->callee
->decl
, 0));
827 get_inline_stack (gimple_location (edge
->call_stmt
), &stack
);
829 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
831 || afdo_string_table
->get_index (IDENTIFIER_POINTER (
832 DECL_ASSEMBLER_NAME (edge
->callee
->decl
))) != s
->name ())
835 return s
->total_count ();
838 /* Read AutoFDO profile and returns TRUE on success. */
840 /* source profile format:
842 GCOV_TAG_AFDO_FUNCTION: 4 bytes
844 NUM_FUNCTIONS: 4 bytes
848 FUNCTION_INSTANCE_N. */
851 autofdo_source_profile::read ()
853 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION
)
855 inform (0, "Not expected TAG.");
859 /* Skip the length of the section. */
860 gcov_read_unsigned ();
862 /* Read in the function/callsite profile, and store it in local
864 unsigned function_num
= gcov_read_unsigned ();
865 for (unsigned i
= 0; i
< function_num
; i
++)
867 function_instance::function_instance_stack stack
;
868 function_instance
*s
= function_instance::read_function_instance (
869 &stack
, gcov_read_counter ());
870 afdo_profile_info
->sum_all
+= s
->total_count ();
871 map_
[s
->name ()] = s
;
876 /* Return the function_instance in the profile that correspond to the
880 autofdo_source_profile::get_function_instance_by_inline_stack (
881 const inline_stack
&stack
) const
883 name_function_instance_map::const_iterator iter
= map_
.find (
884 afdo_string_table
->get_index_by_decl (stack
[stack
.length () - 1].first
));
885 if (iter
== map_
.end())
887 function_instance
*s
= iter
->second
;
888 for (unsigned i
= stack
.length() - 1; i
> 0; i
--)
890 s
= s
->get_function_instance_by_decl (
891 stack
[i
].second
, stack
[i
- 1].first
);
898 /* Module profile is only used by LIPO. Here we simply ignore it. */
901 fake_read_autofdo_module_profile ()
903 /* Read in the module info. */
904 gcov_read_unsigned ();
906 /* Skip the length of the section. */
907 gcov_read_unsigned ();
909 /* Read in the file name table. */
910 unsigned total_module_num
= gcov_read_unsigned ();
911 gcc_assert (total_module_num
== 0);
914 /* Read data from profile data file. */
919 if (gcov_open (auto_profile_file
, 1) == 0)
921 error ("cannot open profile file %s", auto_profile_file
);
925 if (gcov_read_unsigned () != GCOV_DATA_MAGIC
)
927 error ("AutoFDO profile magic number does not match");
931 /* Skip the version number. */
932 unsigned version
= gcov_read_unsigned ();
933 if (version
!= AUTO_PROFILE_VERSION
)
935 error ("AutoFDO profile version %u does match %u",
936 version
, AUTO_PROFILE_VERSION
);
940 /* Skip the empty integer. */
941 gcov_read_unsigned ();
944 afdo_string_table
= new string_table ();
945 if (!afdo_string_table
->read())
947 error ("cannot read string table from %s", auto_profile_file
);
951 /* autofdo_source_profile. */
952 afdo_source_profile
= autofdo_source_profile::create ();
953 if (afdo_source_profile
== NULL
)
955 error ("cannot read function profile from %s", auto_profile_file
);
959 /* autofdo_module_profile. */
960 fake_read_autofdo_module_profile ();
962 /* Read in the working set. */
963 if (gcov_read_unsigned () != GCOV_TAG_AFDO_WORKING_SET
)
965 error ("cannot read working set from %s", auto_profile_file
);
969 /* Skip the length of the section. */
970 gcov_read_unsigned ();
971 gcov_working_set_t set
[128];
972 for (unsigned i
= 0; i
< 128; i
++)
974 set
[i
].num_counters
= gcov_read_unsigned ();
975 set
[i
].min_counter
= gcov_read_counter ();
977 add_working_set (set
);
980 /* From AutoFDO profiles, find values inside STMT for that we want to measure
981 histograms for indirect-call optimization.
983 This function is actually served for 2 purposes:
984 * before annotation, we need to mark histogram, promote and inline
985 * after annotation, we just need to mark, and let follow-up logic to
986 decide if it needs to promote and inline. */
989 afdo_indirect_call (gimple_stmt_iterator
*gsi
, const icall_target_map
&map
,
992 gimple
*gs
= gsi_stmt (*gsi
);
995 if (map
.size () == 0)
997 gcall
*stmt
= dyn_cast
<gcall
*> (gs
);
998 if ((!stmt
) || gimple_call_fndecl (stmt
) != NULL_TREE
)
1001 callee
= gimple_call_fn (stmt
);
1003 histogram_value hist
= gimple_alloc_histogram_value (
1004 cfun
, HIST_TYPE_INDIR_CALL
, stmt
, callee
);
1005 hist
->n_counters
= 3;
1006 hist
->hvalue
.counters
= XNEWVEC (gcov_type
, hist
->n_counters
);
1007 gimple_add_histogram_value (cfun
, stmt
, hist
);
1009 gcov_type total
= 0;
1010 icall_target_map::const_iterator max_iter
= map
.end ();
1012 for (icall_target_map::const_iterator iter
= map
.begin ();
1013 iter
!= map
.end (); ++iter
)
1015 total
+= iter
->second
;
1016 if (max_iter
== map
.end () || max_iter
->second
< iter
->second
)
1020 hist
->hvalue
.counters
[0]
1021 = (unsigned long long)afdo_string_table
->get_name (max_iter
->first
);
1022 hist
->hvalue
.counters
[1] = max_iter
->second
;
1023 hist
->hvalue
.counters
[2] = total
;
1028 struct cgraph_edge
*indirect_edge
1029 = cgraph_node::get (current_function_decl
)->get_edge (stmt
);
1030 struct cgraph_node
*direct_call
= cgraph_node::get_for_asmname (
1031 get_identifier ((const char *) hist
->hvalue
.counters
[0]));
1035 fprintf (dump_file
, "Indirect call -> direct call ");
1036 print_generic_expr (dump_file
, callee
, TDF_SLIM
);
1037 fprintf (dump_file
, " => ");
1038 print_generic_expr (dump_file
, direct_call
->decl
, TDF_SLIM
);
1041 if (direct_call
== NULL
|| !check_ic_target (stmt
, direct_call
))
1044 fprintf (dump_file
, " not transforming\n");
1047 if (DECL_STRUCT_FUNCTION (direct_call
->decl
) == NULL
)
1050 fprintf (dump_file
, " no declaration\n");
1056 fprintf (dump_file
, " transformation on insn ");
1057 print_gimple_stmt (dump_file
, stmt
, 0, TDF_SLIM
);
1058 fprintf (dump_file
, "\n");
1061 struct cgraph_edge
*new_edge
1062 = indirect_edge
->make_speculative (direct_call
, 0, 0);
1063 new_edge
->redirect_call_stmt_to_callee ();
1064 gimple_remove_histogram_value (cfun
, stmt
, hist
);
1065 inline_call (new_edge
, true, NULL
, NULL
, false);
1068 /* From AutoFDO profiles, find values inside STMT for that we want to measure
1069 histograms and adds them to list VALUES. */
1072 afdo_vpt (gimple_stmt_iterator
*gsi
, const icall_target_map
&map
,
1075 afdo_indirect_call (gsi
, map
, transform
);
1078 typedef std::set
<basic_block
> bb_set
;
1079 typedef std::set
<edge
> edge_set
;
1082 is_bb_annotated (const basic_block bb
, const bb_set
&annotated
)
1084 return annotated
.find (bb
) != annotated
.end ();
1088 set_bb_annotated (basic_block bb
, bb_set
*annotated
)
1090 annotated
->insert (bb
);
1094 is_edge_annotated (const edge e
, const edge_set
&annotated
)
1096 return annotated
.find (e
) != annotated
.end ();
1100 set_edge_annotated (edge e
, edge_set
*annotated
)
1102 annotated
->insert (e
);
1105 /* For a given BB, set its execution count. Attach value profile if a stmt
1106 is not in PROMOTED, because we only want to promote an indirect call once.
1107 Return TRUE if BB is annotated. */
1110 afdo_set_bb_count (basic_block bb
, const stmt_set
&promoted
)
1112 gimple_stmt_iterator gsi
;
1115 gcov_type max_count
= 0;
1116 bool has_annotated
= false;
1118 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1121 gimple
*stmt
= gsi_stmt (gsi
);
1122 if (gimple_clobber_p (stmt
) || is_gimple_debug (stmt
))
1124 if (afdo_source_profile
->get_count_info (stmt
, &info
))
1126 if (info
.count
> max_count
)
1127 max_count
= info
.count
;
1128 has_annotated
= true;
1129 if (info
.targets
.size () > 0
1130 && promoted
.find (stmt
) == promoted
.end ())
1131 afdo_vpt (&gsi
, info
.targets
, false);
1138 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1139 afdo_source_profile
->mark_annotated (gimple_location (gsi_stmt (gsi
)));
1140 for (gphi_iterator gpi
= gsi_start_phis (bb
);
1144 gphi
*phi
= gpi
.phi ();
1146 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
1147 afdo_source_profile
->mark_annotated (gimple_phi_arg_location (phi
, i
));
1149 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1150 afdo_source_profile
->mark_annotated (e
->goto_locus
);
1152 bb
->count
= max_count
;
1156 /* BB1 and BB2 are in an equivalent class iff:
1157 1. BB1 dominates BB2.
1158 2. BB2 post-dominates BB1.
1159 3. BB1 and BB2 are in the same loop nest.
1160 This function finds the equivalent class for each basic block, and
1161 stores a pointer to the first BB in its equivalent class. Meanwhile,
1162 set bb counts for the same equivalent class to be idenical. Update
1163 ANNOTATED_BB for the first BB in its equivalent class. */
1166 afdo_find_equiv_class (bb_set
*annotated_bb
)
1170 FOR_ALL_BB_FN (bb
, cfun
)
1173 FOR_ALL_BB_FN (bb
, cfun
)
1175 vec
<basic_block
> dom_bbs
;
1179 if (bb
->aux
!= NULL
)
1182 dom_bbs
= get_dominated_by (CDI_DOMINATORS
, bb
);
1183 FOR_EACH_VEC_ELT (dom_bbs
, i
, bb1
)
1184 if (bb1
->aux
== NULL
&& dominated_by_p (CDI_POST_DOMINATORS
, bb
, bb1
)
1185 && bb1
->loop_father
== bb
->loop_father
)
1188 if (bb1
->count
> bb
->count
&& is_bb_annotated (bb1
, *annotated_bb
))
1190 bb
->count
= bb1
->count
;
1191 set_bb_annotated (bb
, annotated_bb
);
1194 dom_bbs
= get_dominated_by (CDI_POST_DOMINATORS
, bb
);
1195 FOR_EACH_VEC_ELT (dom_bbs
, i
, bb1
)
1196 if (bb1
->aux
== NULL
&& dominated_by_p (CDI_DOMINATORS
, bb
, bb1
)
1197 && bb1
->loop_father
== bb
->loop_father
)
1200 if (bb1
->count
> bb
->count
&& is_bb_annotated (bb1
, *annotated_bb
))
1202 bb
->count
= bb1
->count
;
1203 set_bb_annotated (bb
, annotated_bb
);
1209 /* If a basic block's count is known, and only one of its in/out edges' count
1210 is unknown, its count can be calculated. Meanwhile, if all of the in/out
1211 edges' counts are known, then the basic block's unknown count can also be
1213 IS_SUCC is true if out edges of a basic blocks are examined.
1214 Update ANNOTATED_BB and ANNOTATED_EDGE accordingly.
1215 Return TRUE if any basic block/edge count is changed. */
1218 afdo_propagate_edge (bool is_succ
, bb_set
*annotated_bb
,
1219 edge_set
*annotated_edge
)
1222 bool changed
= false;
1224 FOR_EACH_BB_FN (bb
, cfun
)
1226 edge e
, unknown_edge
= NULL
;
1228 int num_unknown_edge
= 0;
1229 gcov_type total_known_count
= 0;
1231 FOR_EACH_EDGE (e
, ei
, is_succ
? bb
->succs
: bb
->preds
)
1232 if (!is_edge_annotated (e
, *annotated_edge
))
1233 num_unknown_edge
++, unknown_edge
= e
;
1235 total_known_count
+= e
->count
;
1237 if (num_unknown_edge
== 0)
1239 if (total_known_count
> bb
->count
)
1241 bb
->count
= total_known_count
;
1244 if (!is_bb_annotated (bb
, *annotated_bb
))
1246 set_bb_annotated (bb
, annotated_bb
);
1250 else if (num_unknown_edge
== 1 && is_bb_annotated (bb
, *annotated_bb
))
1252 if (bb
->count
>= total_known_count
)
1253 unknown_edge
->count
= bb
->count
- total_known_count
;
1255 unknown_edge
->count
= 0;
1256 set_edge_annotated (unknown_edge
, annotated_edge
);
1263 /* Special propagation for circuit expressions. Because GCC translates
1264 control flow into data flow for circuit expressions. E.g.
1271 will be translated into:
1286 tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
1292 In this case, we need to propagate through PHI to determine the edge
1293 count of BB1->BB.t1, BB.t1->BB.t2.
1294 Update ANNOTATED_EDGE accordingly. */
1297 afdo_propagate_circuit (const bb_set
&annotated_bb
, edge_set
*annotated_edge
)
1300 FOR_ALL_BB_FN (bb
, cfun
)
1303 tree cmp_rhs
, cmp_lhs
;
1304 gimple
*cmp_stmt
= last_stmt (bb
);
1308 if (!cmp_stmt
|| gimple_code (cmp_stmt
) != GIMPLE_COND
)
1310 cmp_rhs
= gimple_cond_rhs (cmp_stmt
);
1311 cmp_lhs
= gimple_cond_lhs (cmp_stmt
);
1312 if (!TREE_CONSTANT (cmp_rhs
)
1313 || !(integer_zerop (cmp_rhs
) || integer_onep (cmp_rhs
)))
1315 if (TREE_CODE (cmp_lhs
) != SSA_NAME
)
1317 if (!is_bb_annotated (bb
, annotated_bb
))
1319 def_stmt
= SSA_NAME_DEF_STMT (cmp_lhs
);
1320 while (def_stmt
&& gimple_code (def_stmt
) == GIMPLE_ASSIGN
1321 && gimple_assign_single_p (def_stmt
)
1322 && TREE_CODE (gimple_assign_rhs1 (def_stmt
)) == SSA_NAME
)
1323 def_stmt
= SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt
));
1326 gphi
*phi_stmt
= dyn_cast
<gphi
*> (def_stmt
);
1329 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1331 unsigned i
, total
= 0;
1333 bool check_value_one
= (((integer_onep (cmp_rhs
))
1334 ^ (gimple_cond_code (cmp_stmt
) == EQ_EXPR
))
1335 ^ ((e
->flags
& EDGE_TRUE_VALUE
) != 0));
1336 if (!is_edge_annotated (e
, *annotated_edge
))
1338 for (i
= 0; i
< gimple_phi_num_args (phi_stmt
); i
++)
1340 tree val
= gimple_phi_arg_def (phi_stmt
, i
);
1341 edge ep
= gimple_phi_arg_edge (phi_stmt
, i
);
1343 if (!TREE_CONSTANT (val
)
1344 || !(integer_zerop (val
) || integer_onep (val
)))
1346 if (check_value_one
^ integer_onep (val
))
1350 if (e
->probability
== 0 && !is_edge_annotated (ep
, *annotated_edge
))
1352 ep
->probability
= 0;
1354 set_edge_annotated (ep
, annotated_edge
);
1357 if (total
== 1 && !is_edge_annotated (only_one
, *annotated_edge
))
1359 only_one
->probability
= e
->probability
;
1360 only_one
->count
= e
->count
;
1361 set_edge_annotated (only_one
, annotated_edge
);
1367 /* Propagate the basic block count and edge count on the control flow
1368 graph. We do the propagation iteratively until stablize. */
1371 afdo_propagate (bb_set
*annotated_bb
, edge_set
*annotated_edge
)
1374 bool changed
= true;
1377 FOR_ALL_BB_FN (bb
, cfun
)
1379 bb
->count
= ((basic_block
)bb
->aux
)->count
;
1380 if (is_bb_annotated ((basic_block
)bb
->aux
, *annotated_bb
))
1381 set_bb_annotated (bb
, annotated_bb
);
1384 while (changed
&& i
++ < 10)
1388 if (afdo_propagate_edge (true, annotated_bb
, annotated_edge
))
1390 if (afdo_propagate_edge (false, annotated_bb
, annotated_edge
))
1392 afdo_propagate_circuit (*annotated_bb
, annotated_edge
);
1396 /* Propagate counts on control flow graph and calculate branch
1400 afdo_calculate_branch_prob (bb_set
*annotated_bb
, edge_set
*annotated_edge
)
1403 bool has_sample
= false;
1405 FOR_EACH_BB_FN (bb
, cfun
)
1417 calculate_dominance_info (CDI_POST_DOMINATORS
);
1418 calculate_dominance_info (CDI_DOMINATORS
);
1419 loop_optimizer_init (0);
1421 afdo_find_equiv_class (annotated_bb
);
1422 afdo_propagate (annotated_bb
, annotated_edge
);
1424 FOR_EACH_BB_FN (bb
, cfun
)
1428 int num_unknown_succ
= 0;
1429 gcov_type total_count
= 0;
1431 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1433 if (!is_edge_annotated (e
, *annotated_edge
))
1436 total_count
+= e
->count
;
1438 if (num_unknown_succ
== 0 && total_count
> 0)
1440 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1441 e
->probability
= (double)e
->count
* REG_BR_PROB_BASE
/ total_count
;
1444 FOR_ALL_BB_FN (bb
, cfun
)
1449 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1450 e
->count
= (double)bb
->count
* e
->probability
/ REG_BR_PROB_BASE
;
1454 loop_optimizer_finalize ();
1455 free_dominance_info (CDI_DOMINATORS
);
1456 free_dominance_info (CDI_POST_DOMINATORS
);
1459 /* Perform value profile transformation using AutoFDO profile. Add the
1460 promoted stmts to PROMOTED_STMTS. Return TRUE if there is any
1461 indirect call promoted. */
1464 afdo_vpt_for_early_inline (stmt_set
*promoted_stmts
)
1467 if (afdo_source_profile
->get_function_instance_by_decl (
1468 current_function_decl
) == NULL
)
1471 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
1473 bool has_vpt
= false;
1474 FOR_EACH_BB_FN (bb
, cfun
)
1476 if (!has_indirect_call (bb
))
1478 gimple_stmt_iterator gsi
;
1480 gcov_type bb_count
= 0;
1481 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1484 gimple
*stmt
= gsi_stmt (gsi
);
1485 if (afdo_source_profile
->get_count_info (stmt
, &info
))
1486 bb_count
= MAX (bb_count
, info
.count
);
1489 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1491 gcall
*stmt
= dyn_cast
<gcall
*> (gsi_stmt (gsi
));
1492 /* IC_promotion and early_inline_2 is done in multiple iterations.
1493 No need to promoted the stmt if its in promoted_stmts (means
1494 it is already been promoted in the previous iterations). */
1495 if ((!stmt
) || gimple_call_fn (stmt
) == NULL
1496 || TREE_CODE (gimple_call_fn (stmt
)) == FUNCTION_DECL
1497 || promoted_stmts
->find (stmt
) != promoted_stmts
->end ())
1501 afdo_source_profile
->get_count_info (stmt
, &info
);
1502 info
.count
= bb_count
;
1503 if (afdo_source_profile
->update_inlined_ind_target (stmt
, &info
))
1505 /* Promote the indirect call and update the promoted_stmts. */
1506 promoted_stmts
->insert (stmt
);
1507 afdo_vpt (&gsi
, info
.targets
, true);
1515 unsigned todo
= optimize_inline_calls (current_function_decl
);
1516 if (todo
& TODO_update_ssa_any
)
1517 update_ssa (TODO_update_ssa
);
1524 /* Annotate auto profile to the control flow graph. Do not annotate value
1525 profile for stmts in PROMOTED_STMTS. */
1528 afdo_annotate_cfg (const stmt_set
&promoted_stmts
)
1531 bb_set annotated_bb
;
1532 edge_set annotated_edge
;
1533 const function_instance
*s
1534 = afdo_source_profile
->get_function_instance_by_decl (
1535 current_function_decl
);
1539 cgraph_node::get (current_function_decl
)->count
= s
->head_count ();
1540 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
= s
->head_count ();
1541 gcov_type max_count
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1543 FOR_EACH_BB_FN (bb
, cfun
)
1549 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1552 if (afdo_set_bb_count (bb
, promoted_stmts
))
1553 set_bb_annotated (bb
, &annotated_bb
);
1554 if (bb
->count
> max_count
)
1555 max_count
= bb
->count
;
1557 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
1558 > ENTRY_BLOCK_PTR_FOR_FN (cfun
)->next_bb
->count
)
1560 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->next_bb
->count
1561 = ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1562 set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->next_bb
, &annotated_bb
);
1564 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
1565 > EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
->count
)
1567 EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
->count
1568 = ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1569 set_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
, &annotated_bb
);
1571 afdo_source_profile
->mark_annotated (
1572 DECL_SOURCE_LOCATION (current_function_decl
));
1573 afdo_source_profile
->mark_annotated (cfun
->function_start_locus
);
1574 afdo_source_profile
->mark_annotated (cfun
->function_end_locus
);
1577 afdo_calculate_branch_prob (&annotated_bb
, &annotated_edge
);
1579 profile_status_for_fn (cfun
) = PROFILE_READ
;
1581 if (flag_value_profile_transformations
)
1583 gimple_value_profile_transformations ();
1584 free_dominance_info (CDI_DOMINATORS
);
1585 free_dominance_info (CDI_POST_DOMINATORS
);
1586 update_ssa (TODO_update_ssa
);
1590 /* Wrapper function to invoke early inliner. */
1595 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
1596 unsigned todo
= early_inliner (cfun
);
1597 if (todo
& TODO_update_ssa_any
)
1598 update_ssa (TODO_update_ssa
);
1601 /* Use AutoFDO profile to annoate the control flow graph.
1602 Return the todo flag. */
1607 struct cgraph_node
*node
;
1609 if (symtab
->state
== FINISHED
)
1612 init_node_map (true);
1613 profile_info
= autofdo::afdo_profile_info
;
1615 FOR_EACH_FUNCTION (node
)
1617 if (!gimple_has_body_p (node
->decl
))
1620 /* Don't profile functions produced for builtin stuff. */
1621 if (DECL_SOURCE_LOCATION (node
->decl
) == BUILTINS_LOCATION
)
1624 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1626 /* First do indirect call promotion and early inline to make the
1627 IR match the profiled binary before actual annotation.
1629 This is needed because an indirect call might have been promoted
1630 and inlined in the profiled binary. If we do not promote and
1631 inline these indirect calls before annotation, the profile for
1632 these promoted functions will be lost.
1634 e.g. foo() --indirect_call--> bar()
1635 In profiled binary, the callsite is promoted and inlined, making
1636 the profile look like:
1646 Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
1647 If we perform annotation on it, the profile inside bar@loc_foo2
1650 To avoid this, we promote loc_foo_2 and inline the promoted bar
1651 function before annotation, so the profile inside bar@loc_foo2
1653 autofdo::stmt_set promoted_stmts
;
1654 for (int i
= 0; i
< PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS
); i
++)
1656 if (!flag_value_profile_transformations
1657 || !autofdo::afdo_vpt_for_early_inline (&promoted_stmts
))
1663 autofdo::afdo_annotate_cfg (promoted_stmts
);
1664 compute_function_frequency ();
1666 /* Local pure-const may imply need to fixup the cfg. */
1667 if (execute_fixup_cfg () & TODO_cleanup_cfg
)
1668 cleanup_tree_cfg ();
1670 free_dominance_info (CDI_DOMINATORS
);
1671 free_dominance_info (CDI_POST_DOMINATORS
);
1672 cgraph_edge::rebuild_edges ();
1673 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
1677 return TODO_rebuild_cgraph_edges
;
1679 } /* namespace autofdo. */
1681 /* Read the profile from the profile data file. */
1684 read_autofdo_file (void)
1686 if (auto_profile_file
== NULL
)
1687 auto_profile_file
= DEFAULT_AUTO_PROFILE_FILE
;
1689 autofdo::afdo_profile_info
= (struct gcov_ctr_summary
*)xcalloc (
1690 1, sizeof (struct gcov_ctr_summary
));
1691 autofdo::afdo_profile_info
->runs
= 1;
1692 autofdo::afdo_profile_info
->sum_max
= 0;
1693 autofdo::afdo_profile_info
->sum_all
= 0;
1695 /* Read the profile from the profile file. */
1696 autofdo::read_profile ();
1699 /* Free the resources. */
1702 end_auto_profile (void)
1704 delete autofdo::afdo_source_profile
;
1705 delete autofdo::afdo_string_table
;
1706 profile_info
= NULL
;
1709 /* Returns TRUE if EDGE is hot enough to be inlined early. */
1712 afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge
*edge
)
1715 = autofdo::afdo_source_profile
->get_callsite_total_count (edge
);
1720 const struct gcov_ctr_summary
*saved_profile_info
= profile_info
;
1721 /* At early inline stage, profile_info is not set yet. We need to
1722 temporarily set it to afdo_profile_info to calculate hotness. */
1723 profile_info
= autofdo::afdo_profile_info
;
1724 is_hot
= maybe_hot_count_p (NULL
, count
);
1725 profile_info
= saved_profile_info
;
1735 const pass_data pass_data_ipa_auto_profile
= {
1736 SIMPLE_IPA_PASS
, "afdo", /* name */
1737 OPTGROUP_NONE
, /* optinfo_flags */
1738 TV_IPA_AUTOFDO
, /* tv_id */
1739 0, /* properties_required */
1740 0, /* properties_provided */
1741 0, /* properties_destroyed */
1742 0, /* todo_flags_start */
1743 0, /* todo_flags_finish */
1746 class pass_ipa_auto_profile
: public simple_ipa_opt_pass
1749 pass_ipa_auto_profile (gcc::context
*ctxt
)
1750 : simple_ipa_opt_pass (pass_data_ipa_auto_profile
, ctxt
)
1754 /* opt_pass methods: */
1758 return flag_auto_profile
;
1760 virtual unsigned int
1761 execute (function
*)
1763 return autofdo::auto_profile ();
1765 }; // class pass_ipa_auto_profile
1769 simple_ipa_opt_pass
*
1770 make_pass_ipa_auto_profile (gcc::context
*ctxt
)
1772 return new pass_ipa_auto_profile (ctxt
);