gcc/tree-profile.cc

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2024 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.cc for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "stringpool.h"
  53 #include "attribs.h"
  54 #include "tree-pretty-print.h"
  55 #include "langhooks.h"
  56 #include "stor-layout.h"
  57 #include "xregex.h"
  58 #include "alloc-pool.h"
  59 #include "symbol-summary.h"
  60 #include "symtab-thunks.h"
  61
  62 static GTY(()) tree gcov_type_node;
  63 static GTY(()) tree tree_interval_profiler_fn;
  64 static GTY(()) tree tree_pow2_profiler_fn;
  65 static GTY(()) tree tree_topn_values_profiler_fn;
  66 static GTY(()) tree tree_indirect_call_profiler_fn;
  67 static GTY(()) tree tree_average_profiler_fn;
  68 static GTY(()) tree tree_ior_profiler_fn;
  69 static GTY(()) tree tree_time_profiler_counter;
  70
  71
  72 static GTY(()) tree ic_tuple_var;
  73 static GTY(()) tree ic_tuple_counters_field;
  74 static GTY(()) tree ic_tuple_callee_field;
  75
  76 /* Types of counter update methods.
  77
  78    By default, the counter updates are done for a single threaded system
  79    (COUNTER_UPDATE_SINGLE_THREAD).
  80
  81    If the user selected atomic profile counter updates
  82    (-fprofile-update=atomic), then the counter updates will be done atomically
  83    on a best-effort basis.  One of three methods to do the counter updates is
  84    selected according to the target capabilities.
  85
  86    Ideally, the counter updates are done through atomic operations in hardware
  87    (COUNTER_UPDATE_ATOMIC_BUILTIN).
  88
  89    If the target supports only 32-bit atomic increments and gcov_type_node is a
  90    64-bit integer type, then for the profile edge counters the increment is
  91    performed through two separate 32-bit atomic increments
  92    (COUNTER_UPDATE_ATOMIC_SPLIT or COUNTER_UPDATE_ATOMIC_PARTIAL).  If the
  93    target supports libatomic (targetm.have_libatomic), then other counter
  94    updates are carried out by libatomic calls (COUNTER_UPDATE_ATOMIC_SPLIT).
  95    If the target does not support libatomic, then the other counter updates are
  96    not done atomically (COUNTER_UPDATE_ATOMIC_PARTIAL) and a warning is
  97    issued.
  98
  99    If the target does not support atomic operations in hardware, however,  it
 100    supports libatomic, then all updates are carried out by libatomic calls
 101    (COUNTER_UPDATE_ATOMIC_BUILTIN).  */
 102 enum counter_update_method {
 103   COUNTER_UPDATE_SINGLE_THREAD,
 104   COUNTER_UPDATE_ATOMIC_BUILTIN,
 105   COUNTER_UPDATE_ATOMIC_SPLIT,
 106   COUNTER_UPDATE_ATOMIC_PARTIAL
 107 };
 108
 109 static counter_update_method counter_update = COUNTER_UPDATE_SINGLE_THREAD;
 110
 111 /* Do initialization work for the edge profiler.  */
 112
 113 /* Add code:
 114    __thread gcov*       __gcov_indirect_call.counters; // pointer to actual counter
 115    __thread void*       __gcov_indirect_call.callee; // actual callee address
 116    __thread int __gcov_function_counter; // time profiler function counter
 117 */
 118 static void
 119 init_ic_make_global_vars (void)
 120 {
 121   tree gcov_type_ptr;
 122
 123   gcov_type_ptr = build_pointer_type (get_gcov_type ());
 124
 125   tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
 126
 127   /* callee */
 128   ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
 129                                       ptr_type_node);
 130
 131   /* counters */
 132   ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
 133                                         NULL_TREE, gcov_type_ptr);
 134   DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
 135
 136   finish_builtin_struct (tuple_type, "indirect_call_tuple",
 137                          ic_tuple_counters_field, NULL_TREE);
 138
 139   ic_tuple_var
 140     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 141                   get_identifier ("__gcov_indirect_call"), tuple_type);
 142   TREE_PUBLIC (ic_tuple_var) = 1;
 143   DECL_ARTIFICIAL (ic_tuple_var) = 1;
 144   DECL_INITIAL (ic_tuple_var) = NULL;
 145   DECL_EXTERNAL (ic_tuple_var) = 1;
 146   if (targetm.have_tls)
 147     set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
 148 }
 149
 150 /* Create the type and function decls for the interface with gcov.  */
 151
 152 void
 153 gimple_init_gcov_profiler (void)
 154 {
 155   tree interval_profiler_fn_type;
 156   tree pow2_profiler_fn_type;
 157   tree topn_values_profiler_fn_type;
 158   tree gcov_type_ptr;
 159   tree ic_profiler_fn_type;
 160   tree average_profiler_fn_type;
 161   const char *fn_name;
 162
 163   if (!gcov_type_node)
 164     {
 165       const char *fn_suffix
 166         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 167
 168       gcov_type_node = get_gcov_type ();
 169       gcov_type_ptr = build_pointer_type (gcov_type_node);
 170
 171       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 172       interval_profiler_fn_type
 173               = build_function_type_list (void_type_node,
 174                                           gcov_type_ptr, gcov_type_node,
 175                                           integer_type_node,
 176                                           unsigned_type_node, NULL_TREE);
 177       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 178       tree_interval_profiler_fn = build_fn_decl (fn_name,
 179                                                  interval_profiler_fn_type);
 180       free (CONST_CAST (char *, fn_name));
 181       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 182       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 183         = tree_cons (get_identifier ("leaf"), NULL,
 184                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 185
 186       /* void (*) (gcov_type *, gcov_type)  */
 187       pow2_profiler_fn_type
 188               = build_function_type_list (void_type_node,
 189                                           gcov_type_ptr, gcov_type_node,
 190                                           NULL_TREE);
 191       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 192       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 193       free (CONST_CAST (char *, fn_name));
 194       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 195       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 196         = tree_cons (get_identifier ("leaf"), NULL,
 197                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 198
 199       /* void (*) (gcov_type *, gcov_type)  */
 200       topn_values_profiler_fn_type
 201               = build_function_type_list (void_type_node,
 202                                           gcov_type_ptr, gcov_type_node,
 203                                           NULL_TREE);
 204       fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
 205       tree_topn_values_profiler_fn
 206         = build_fn_decl (fn_name, topn_values_profiler_fn_type);
 207       free (CONST_CAST (char *, fn_name));
 208
 209       TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
 210       DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
 211         = tree_cons (get_identifier ("leaf"), NULL,
 212                      DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
 213
 214       init_ic_make_global_vars ();
 215
 216       /* void (*) (gcov_type, void *)  */
 217       ic_profiler_fn_type
 218                = build_function_type_list (void_type_node,
 219                                           gcov_type_node,
 220                                           ptr_type_node,
 221                                           NULL_TREE);
 222       fn_name = concat ("__gcov_indirect_call_profiler_v4", fn_suffix, NULL);
 223       tree_indirect_call_profiler_fn
 224         = build_fn_decl (fn_name, ic_profiler_fn_type);
 225       free (CONST_CAST (char *, fn_name));
 226
 227       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 228       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 229         = tree_cons (get_identifier ("leaf"), NULL,
 230                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 231
 232       tree_time_profiler_counter
 233         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 234                       get_identifier ("__gcov_time_profiler_counter"),
 235                       get_gcov_type ());
 236       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 237       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 238       TREE_STATIC (tree_time_profiler_counter) = 1;
 239       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 240       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 241
 242       /* void (*) (gcov_type *, gcov_type)  */
 243       average_profiler_fn_type
 244               = build_function_type_list (void_type_node,
 245                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 246       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 247       tree_average_profiler_fn = build_fn_decl (fn_name,
 248                                                 average_profiler_fn_type);
 249       free (CONST_CAST (char *, fn_name));
 250       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 251       DECL_ATTRIBUTES (tree_average_profiler_fn)
 252         = tree_cons (get_identifier ("leaf"), NULL,
 253                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 254       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 255       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 256       free (CONST_CAST (char *, fn_name));
 257       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 258       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 259         = tree_cons (get_identifier ("leaf"), NULL,
 260                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 261
 262       /* LTO streamer needs assembler names.  Because we create these decls
 263          late, we need to initialize them by hand.  */
 264       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 265       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 266       DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
 267       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 268       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 269       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 270     }
 271 }
 272
 273 /* If RESULT is not null, then output instructions as GIMPLE trees to assign
 274    the updated counter from CALL of FUNC to RESULT.  Insert the CALL and the
 275    optional assignment instructions to GSI.  Use NAME for temporary values.  */
 276
 277 static inline void
 278 gen_assign_counter_update (gimple_stmt_iterator *gsi, gcall *call, tree func,
 279                            tree result, const char *name)
 280 {
 281   if (result)
 282     {
 283       tree result_type = TREE_TYPE (TREE_TYPE (func));
 284       tree tmp1 = make_temp_ssa_name (result_type, NULL, name);
 285       gimple_set_lhs (call, tmp1);
 286       gsi_insert_after (gsi, call, GSI_NEW_STMT);
 287       tree tmp2 = make_temp_ssa_name (TREE_TYPE (result), NULL, name);
 288       gassign *assign = gimple_build_assign (tmp2, NOP_EXPR, tmp1);
 289       gsi_insert_after (gsi, assign, GSI_NEW_STMT);
 290       assign = gimple_build_assign (result, tmp2);
 291       gsi_insert_after (gsi, assign, GSI_NEW_STMT);
 292     }
 293   else
 294     gsi_insert_after (gsi, call, GSI_NEW_STMT);
 295 }
 296
 297 /* Output instructions as GIMPLE trees to increment the COUNTER.  If RESULT is
 298    not null, then assign the updated counter value to RESULT.  Insert the
 299    instructions to GSI.  Use NAME for temporary values.  */
 300
 301 static inline void
 302 gen_counter_update (gimple_stmt_iterator *gsi, tree counter, tree result,
 303                     const char *name)
 304 {
 305   tree type = gcov_type_node;
 306   tree addr = build_fold_addr_expr (counter);
 307   tree one = build_int_cst (type, 1);
 308   tree relaxed = build_int_cst (integer_type_node, MEMMODEL_RELAXED);
 309
 310   if (counter_update == COUNTER_UPDATE_ATOMIC_BUILTIN
 311       || (result && counter_update == COUNTER_UPDATE_ATOMIC_SPLIT))
 312     {
 313       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 314       tree f = builtin_decl_explicit (TYPE_PRECISION (type) > 32
 315                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8
 316                                       : BUILT_IN_ATOMIC_ADD_FETCH_4);
 317       gcall *call = gimple_build_call (f, 3, addr, one, relaxed);
 318       gen_assign_counter_update (gsi, call, f, result, name);
 319     }
 320   else if (!result && (counter_update == COUNTER_UPDATE_ATOMIC_SPLIT
 321                        || counter_update == COUNTER_UPDATE_ATOMIC_PARTIAL))
 322     {
 323       /* low = __atomic_add_fetch_4 (addr, 1, MEMMODEL_RELAXED);
 324          high_inc = low == 0 ? 1 : 0;
 325          __atomic_add_fetch_4 (addr_high, high_inc, MEMMODEL_RELAXED); */
 326       tree zero32 = build_zero_cst (uint32_type_node);
 327       tree one32 = build_one_cst (uint32_type_node);
 328       tree addr_high = make_temp_ssa_name (TREE_TYPE (addr), NULL, name);
 329       tree four = build_int_cst (size_type_node, 4);
 330       gassign *assign1 = gimple_build_assign (addr_high, POINTER_PLUS_EXPR,
 331                                               addr, four);
 332       gsi_insert_after (gsi, assign1, GSI_NEW_STMT);
 333       if (WORDS_BIG_ENDIAN)
 334         std::swap (addr, addr_high);
 335       tree f = builtin_decl_explicit (BUILT_IN_ATOMIC_ADD_FETCH_4);
 336       gcall *call1 = gimple_build_call (f, 3, addr, one, relaxed);
 337       tree low = make_temp_ssa_name (uint32_type_node, NULL, name);
 338       gimple_call_set_lhs (call1, low);
 339       gsi_insert_after (gsi, call1, GSI_NEW_STMT);
 340       tree is_zero = make_temp_ssa_name (boolean_type_node, NULL, name);
 341       gassign *assign2 = gimple_build_assign (is_zero, EQ_EXPR, low,
 342                                               zero32);
 343       gsi_insert_after (gsi, assign2, GSI_NEW_STMT);
 344       tree high_inc = make_temp_ssa_name (uint32_type_node, NULL, name);
 345       gassign *assign3 = gimple_build_assign (high_inc, COND_EXPR,
 346                                               is_zero, one32, zero32);
 347       gsi_insert_after (gsi, assign3, GSI_NEW_STMT);
 348       gcall *call2 = gimple_build_call (f, 3, addr_high, high_inc,
 349                                         relaxed);
 350       gsi_insert_after (gsi, call2, GSI_NEW_STMT);
 351     }
 352   else
 353     {
 354       tree tmp1 = make_temp_ssa_name (type, NULL, name);
 355       gassign *assign1 = gimple_build_assign (tmp1, counter);
 356       gsi_insert_after (gsi, assign1, GSI_NEW_STMT);
 357       tree tmp2 = make_temp_ssa_name (type, NULL, name);
 358       gassign *assign2 = gimple_build_assign (tmp2, PLUS_EXPR, tmp1, one);
 359       gsi_insert_after (gsi, assign2, GSI_NEW_STMT);
 360       gassign *assign3 = gimple_build_assign (unshare_expr (counter), tmp2);
 361       gsi_insert_after (gsi, assign3, GSI_NEW_STMT);
 362       if (result)
 363         {
 364           gassign *assign4 = gimple_build_assign (result, tmp2);
 365           gsi_insert_after (gsi, assign4, GSI_NEW_STMT);
 366         }
 367     }
 368 }
 369
 370 /* Output instructions as GIMPLE trees to increment the edge
 371    execution count, and insert them on E.  */
 372
 373 void
 374 gimple_gen_edge_profiler (int edgeno, edge e)
 375 {
 376   gimple_stmt_iterator gsi = gsi_last (PENDING_STMT (e));
 377   tree counter = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 378   gen_counter_update (&gsi, counter, NULL_TREE, "PROF_edge_counter");
 379 }
 380
 381 /* Emits code to get VALUE to instrument at GSI, and returns the
 382    variable containing the value.  */
 383
 384 static tree
 385 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 386 {
 387   tree val = value->hvalue.value;
 388   if (POINTER_TYPE_P (TREE_TYPE (val)))
 389     val = fold_convert (build_nonstandard_integer_type
 390                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 391   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 392                                    true, NULL_TREE, true, GSI_SAME_STMT);
 393 }
 394
 395 /* Output instructions as GIMPLE trees to increment the interval histogram
 396    counter.  VALUE is the expression whose value is profiled.  TAG is the
 397    tag of the section for counters, BASE is offset of the counter position.  */
 398
 399 void
 400 gimple_gen_interval_profiler (histogram_value value, unsigned tag)
 401 {
 402   gimple *stmt = value->hvalue.stmt;
 403   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 404   tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr;
 405   gcall *call;
 406   tree val;
 407   tree start = build_int_cst_type (integer_type_node,
 408                                    value->hdata.intvl.int_start);
 409   tree steps = build_int_cst_type (unsigned_type_node,
 410                                    value->hdata.intvl.steps);
 411
 412   ref_ptr = force_gimple_operand_gsi (&gsi,
 413                                       build_addr (ref),
 414                                       true, NULL_TREE, true, GSI_SAME_STMT);
 415   val = prepare_instrumented_value (&gsi, value);
 416   call = gimple_build_call (tree_interval_profiler_fn, 4,
 417                             ref_ptr, val, start, steps);
 418   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 419 }
 420
 421 /* Output instructions as GIMPLE trees to increment the power of two histogram
 422    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 423    of the section for counters.  */
 424
 425 void
 426 gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
 427 {
 428   gimple *stmt = value->hvalue.stmt;
 429   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 430   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 431   gcall *call;
 432   tree val;
 433
 434   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 435                                       true, NULL_TREE, true, GSI_SAME_STMT);
 436   val = prepare_instrumented_value (&gsi, value);
 437   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 438   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 439 }
 440
 441 /* Output instructions as GIMPLE trees for code to find the most N common
 442    values.  VALUE is the expression whose value is profiled.  TAG is the tag
 443    of the section for counters.  */
 444
 445 void
 446 gimple_gen_topn_values_profiler (histogram_value value, unsigned tag)
 447 {
 448   gimple *stmt = value->hvalue.stmt;
 449   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 450   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 451   gcall *call;
 452   tree val;
 453
 454   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 455                                       true, NULL_TREE, true, GSI_SAME_STMT);
 456   val = prepare_instrumented_value (&gsi, value);
 457   call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
 458   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 459 }
 460
 461
 462 /* Output instructions as GIMPLE trees for code to find the most
 463    common called function in indirect call.
 464    VALUE is the call expression whose indirect callee is profiled.
 465    TAG is the tag of the section for counters.  */
 466
 467 void
 468 gimple_gen_ic_profiler (histogram_value value, unsigned tag)
 469 {
 470   tree tmp1;
 471   gassign *stmt1, *stmt2, *stmt3;
 472   gimple *stmt = value->hvalue.stmt;
 473   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 474   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 475
 476   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 477                                       true, NULL_TREE, true, GSI_SAME_STMT);
 478
 479   /* Insert code:
 480
 481     stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
 482     stmt2: tmp1 = (void *) (indirect call argument value)
 483     stmt3: __gcov_indirect_call.callee = tmp1;
 484
 485     Example:
 486       f_1 = foo;
 487       __gcov_indirect_call.counters = &__gcov4.main[0];
 488       PROF_fn_9 = f_1;
 489       __gcov_indirect_call.callee = PROF_fn_9;
 490       _4 = f_1 ();
 491    */
 492
 493   tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
 494
 495   tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
 496                              ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
 497
 498   stmt1 = gimple_build_assign (counter_ref, ref_ptr);
 499   tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF_fn");
 500   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 501   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 502                              ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 503   stmt3 = gimple_build_assign (callee_ref, tmp1);
 504
 505   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 506   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 507   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 508 }
 509
 510
 511 /* Output instructions as GIMPLE trees for code to find the most
 512    common called function in indirect call. Insert instructions at the
 513    beginning of every possible called function.
 514   */
 515
 516 void
 517 gimple_gen_ic_func_profiler (void)
 518 {
 519   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 520   gcall *stmt1;
 521   tree tree_uid, cur_func, void0;
 522
 523   if (c_node->only_called_directly_p ())
 524     return;
 525
 526   gimple_init_gcov_profiler ();
 527
 528   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 529   basic_block cond_bb = split_edge (single_succ_edge (entry));
 530   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 531
 532   /* We need to do an extra split in order to not create an input
 533      for a possible PHI node.  */
 534   split_edge (single_succ_edge (update_bb));
 535
 536   edge true_edge = single_succ_edge (cond_bb);
 537   true_edge->flags = EDGE_TRUE_VALUE;
 538
 539   profile_probability probability;
 540   if (DECL_VIRTUAL_P (current_function_decl))
 541     probability = profile_probability::very_likely ();
 542   else
 543     probability = profile_probability::unlikely ();
 544
 545   true_edge->probability = probability;
 546   edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
 547                       EDGE_FALSE_VALUE);
 548   e->probability = true_edge->probability.invert ();
 549
 550   /* Insert code:
 551
 552      if (__gcov_indirect_call.callee != NULL)
 553        __gcov_indirect_call_profiler_v3 (profile_id, &current_function_decl);
 554
 555      The function __gcov_indirect_call_profiler_v3 is responsible for
 556      resetting __gcov_indirect_call.callee to NULL.  */
 557
 558   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 559   void0 = build_int_cst (ptr_type_node, 0);
 560
 561   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 562                             ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 563
 564   tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
 565                                        true, GSI_SAME_STMT);
 566
 567   gcond *cond = gimple_build_cond (NE_EXPR, ref,
 568                                    void0, NULL, NULL);
 569   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 570
 571   gsi = gsi_after_labels (update_bb);
 572
 573   cur_func = force_gimple_operand_gsi (&gsi,
 574                                        build_addr (current_function_decl),
 575                                        true, NULL_TREE,
 576                                        true, GSI_SAME_STMT);
 577   tree_uid = build_int_cst
 578               (gcov_type_node,
 579                cgraph_node::get (current_function_decl)->profile_id);
 580   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 581                              tree_uid, cur_func);
 582   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 583 }
 584
 585 /* Output instructions as GIMPLE tree at the beginning for each function.
 586    TAG is the tag of the section for counters, BASE is offset of the
 587    counter position and GSI is the iterator we place the counter.  */
 588
 589 void
 590 gimple_gen_time_profiler (unsigned tag)
 591 {
 592   tree type = get_gcov_type ();
 593   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 594   basic_block cond_bb = split_edge (single_succ_edge (entry));
 595   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 596
 597   /* We need to do an extra split in order to not create an input
 598      for a possible PHI node.  */
 599   split_edge (single_succ_edge (update_bb));
 600
 601   edge true_edge = single_succ_edge (cond_bb);
 602   true_edge->flags = EDGE_TRUE_VALUE;
 603   true_edge->probability = profile_probability::unlikely ();
 604   edge e
 605     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 606   e->probability = true_edge->probability.invert ();
 607
 608   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 609   tree original_ref = tree_coverage_counter_ref (tag, 0);
 610   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 611                                        true, GSI_SAME_STMT);
 612
 613   /* Emit: if (counters[0] != 0).  */
 614   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 615                                    NULL, NULL);
 616   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 617
 618   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 619   gsi = gsi_start_bb (update_bb);
 620   gen_counter_update (&gsi, tree_time_profiler_counter, original_ref,
 621                       "PROF_time_profile");
 622 }
 623
 624 /* Output instructions as GIMPLE trees to increment the average histogram
 625    counter.  VALUE is the expression whose value is profiled.  TAG is the
 626    tag of the section for counters, BASE is offset of the counter position.  */
 627
 628 void
 629 gimple_gen_average_profiler (histogram_value value, unsigned tag)
 630 {
 631   gimple *stmt = value->hvalue.stmt;
 632   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 633   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 634   gcall *call;
 635   tree val;
 636
 637   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 638                                       true, NULL_TREE,
 639                                       true, GSI_SAME_STMT);
 640   val = prepare_instrumented_value (&gsi, value);
 641   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 642   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 643 }
 644
 645 /* Output instructions as GIMPLE trees to increment the ior histogram
 646    counter.  VALUE is the expression whose value is profiled.  TAG is the
 647    tag of the section for counters, BASE is offset of the counter position.  */
 648
 649 void
 650 gimple_gen_ior_profiler (histogram_value value, unsigned tag)
 651 {
 652   gimple *stmt = value->hvalue.stmt;
 653   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 654   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 655   gcall *call;
 656   tree val;
 657
 658   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 659                                       true, NULL_TREE, true, GSI_SAME_STMT);
 660   val = prepare_instrumented_value (&gsi, value);
 661   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 662   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 663 }
 664
 665 static vec<regex_t> profile_filter_files;
 666 static vec<regex_t> profile_exclude_files;
 667
 668 /* Parse list of provided REGEX (separated with semi-collon) and
 669    create expressions (of type regex_t) and save them into V vector.
 670    If there is a regular expression parsing error, error message is
 671    printed for FLAG_NAME.  */
 672
 673 static void
 674 parse_profile_filter (const char *regex, vec<regex_t> *v,
 675                       const char *flag_name)
 676 {
 677   v->create (4);
 678   if (regex != NULL)
 679     {
 680       char *str = xstrdup (regex);
 681       for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
 682         {
 683           regex_t r;
 684           if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
 685             {
 686               error ("invalid regular expression %qs in %qs",
 687                      p, flag_name);
 688               return;
 689             }
 690
 691           v->safe_push (r);
 692         }
 693     }
 694 }
 695
 696 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files
 697    options.  */
 698
 699 static void
 700 parse_profile_file_filtering ()
 701 {
 702   parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
 703                         "-fprofile-filter-files");
 704   parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
 705                         "-fprofile-exclude-files");
 706 }
 707
 708 /* Parse vectors of regular expressions.  */
 709
 710 static void
 711 release_profile_file_filtering ()
 712 {
 713   profile_filter_files.release ();
 714   profile_exclude_files.release ();
 715 }
 716
 717 /* Return true when FILENAME should be instrumented based on
 718    -fprofile-filter-files and -fprofile-exclude-files options.  */
 719
 720 static bool
 721 include_source_file_for_profile (const char *filename)
 722 {
 723   /* First check whether file is included in flag_profile_exclude_files.  */
 724   for (unsigned i = 0; i < profile_exclude_files.length (); i++)
 725     if (regexec (&profile_exclude_files[i],
 726                  filename, 0, NULL, 0) == REG_NOERROR)
 727       return false;
 728
 729   /* For non-empty flag_profile_filter_files include only files matching a
 730      regex in the flag.  */
 731   if (profile_filter_files.is_empty ())
 732     return true;
 733
 734   for (unsigned i = 0; i < profile_filter_files.length (); i++)
 735     if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
 736       return true;
 737
 738   return false;
 739 }
 740
 741 #ifndef HAVE_sync_compare_and_swapsi
 742 #define HAVE_sync_compare_and_swapsi 0
 743 #endif
 744 #ifndef HAVE_atomic_compare_and_swapsi
 745 #define HAVE_atomic_compare_and_swapsi 0
 746 #endif
 747
 748 #ifndef HAVE_sync_compare_and_swapdi
 749 #define HAVE_sync_compare_and_swapdi 0
 750 #endif
 751 #ifndef HAVE_atomic_compare_and_swapdi
 752 #define HAVE_atomic_compare_and_swapdi 0
 753 #endif
 754
 755 /* Profile all functions in the callgraph.  */
 756
 757 static unsigned int
 758 tree_profiling (void)
 759 {
 760   struct cgraph_node *node;
 761
 762   /* Verify whether we can utilize atomic update operations.  */
 763   bool can_support_atomic = targetm.have_libatomic;
 764   unsigned HOST_WIDE_INT gcov_type_size
 765     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 766   bool have_atomic_4
 767     = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 768   bool have_atomic_8
 769     = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 770   bool needs_split = gcov_type_size == 8 && !have_atomic_8 && have_atomic_4;
 771   if (!can_support_atomic)
 772     {
 773       if (gcov_type_size == 4)
 774         can_support_atomic = have_atomic_4;
 775       else if (gcov_type_size == 8)
 776         can_support_atomic = have_atomic_8;
 777     }
 778
 779   if (flag_profile_update != PROFILE_UPDATE_SINGLE && needs_split)
 780     counter_update = COUNTER_UPDATE_ATOMIC_PARTIAL;
 781
 782   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 783       && !can_support_atomic)
 784     {
 785       warning (0, "target does not support atomic profile update, "
 786                "single mode is selected");
 787       flag_profile_update = PROFILE_UPDATE_SINGLE;
 788     }
 789   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 790     flag_profile_update
 791       = can_support_atomic ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 792
 793   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 794     {
 795       if (needs_split)
 796         counter_update = COUNTER_UPDATE_ATOMIC_SPLIT;
 797       else
 798         counter_update = COUNTER_UPDATE_ATOMIC_BUILTIN;
 799     }
 800
 801   /* This is a small-ipa pass that gets called only once, from
 802      cgraphunit.cc:ipa_passes().  */
 803   gcc_assert (symtab->state == IPA_SSA);
 804
 805   init_node_map (true);
 806   parse_profile_file_filtering ();
 807
 808   FOR_EACH_DEFINED_FUNCTION (node)
 809     {
 810       bool thunk = false;
 811       if (!gimple_has_body_p (node->decl) && !node->thunk)
 812         continue;
 813
 814       /* Don't profile functions produced for builtin stuff.  */
 815       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 816         continue;
 817
 818       if (lookup_attribute ("no_profile_instrument_function",
 819                             DECL_ATTRIBUTES (node->decl)))
 820         continue;
 821       /* Do not instrument extern inline functions when testing coverage.
 822          While this is not perfectly consistent (early inlined extern inlines
 823          will get acocunted), testsuite expects that.  */
 824       if (DECL_EXTERNAL (node->decl)
 825           && flag_test_coverage)
 826         continue;
 827
 828       const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
 829       if (!include_source_file_for_profile (file))
 830         continue;
 831
 832       if (node->thunk)
 833         {
 834           /* We cannot expand variadic thunks to Gimple.  */
 835           if (stdarg_p (TREE_TYPE (node->decl)))
 836             continue;
 837           thunk = true;
 838           /* When generate profile, expand thunk to gimple so it can be
 839              instrumented same way as other functions.  */
 840           if (profile_arc_flag)
 841             expand_thunk (node, false, true);
 842           /* Read cgraph profile but keep function as thunk at profile-use
 843              time.  */
 844           else
 845             {
 846               read_thunk_profile (node);
 847               continue;
 848             }
 849         }
 850
 851       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 852
 853       if (dump_file)
 854         dump_function_header (dump_file, cfun->decl, dump_flags);
 855
 856       /* Local pure-const may imply need to fixup the cfg.  */
 857       if (gimple_has_body_p (node->decl)
 858           && (execute_fixup_cfg () & TODO_cleanup_cfg))
 859         cleanup_tree_cfg ();
 860
 861       branch_prob (thunk);
 862
 863       if (! flag_branch_probabilities
 864           && flag_profile_values)
 865         gimple_gen_ic_func_profiler ();
 866
 867       if (flag_branch_probabilities
 868           && !thunk
 869           && flag_profile_values
 870           && flag_value_profile_transformations
 871           && profile_status_for_fn (cfun) == PROFILE_READ)
 872         gimple_value_profile_transformations ();
 873
 874       /* The above could hose dominator info.  Currently there is
 875          none coming in, this is a safety valve.  It should be
 876          easy to adjust it, if and when there is some.  */
 877       free_dominance_info (CDI_DOMINATORS);
 878       free_dominance_info (CDI_POST_DOMINATORS);
 879       pop_cfun ();
 880     }
 881
 882   release_profile_file_filtering ();
 883
 884   /* Drop pure/const flags from instrumented functions.  */
 885   if (profile_arc_flag || flag_test_coverage)
 886     FOR_EACH_DEFINED_FUNCTION (node)
 887       {
 888         if (!gimple_has_body_p (node->decl)
 889             || !(!node->clone_of
 890                  || node->decl != node->clone_of->decl))
 891           continue;
 892
 893         /* Don't profile functions produced for builtin stuff.  */
 894         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 895           continue;
 896
 897         node->set_const_flag (false, false);
 898         node->set_pure_flag (false, false);
 899       }
 900
 901   /* Update call statements and rebuild the cgraph.  */
 902   FOR_EACH_DEFINED_FUNCTION (node)
 903     {
 904       basic_block bb;
 905
 906       if (!gimple_has_body_p (node->decl)
 907           || !(!node->clone_of
 908           || node->decl != node->clone_of->decl))
 909         continue;
 910
 911       /* Don't profile functions produced for builtin stuff.  */
 912       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 913         continue;
 914
 915       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 916
 917       if (profile_arc_flag || flag_test_coverage)
 918         FOR_EACH_BB_FN (bb, cfun)
 919           {
 920             gimple_stmt_iterator gsi;
 921             for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 922               {
 923                 gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
 924                 if (!call || gimple_call_internal_p (call))
 925                   continue;
 926
 927                 /* We do not clear pure/const on decls without body.  */
 928                 tree fndecl = gimple_call_fndecl (call);
 929                 cgraph_node *callee;
 930                 if (fndecl
 931                     && (callee = cgraph_node::get (fndecl))
 932                     && callee->get_availability (node) == AVAIL_NOT_AVAILABLE)
 933                   continue;
 934
 935                 /* Drop the const attribute from the call type (the pure
 936                    attribute is not available on types).  */
 937                 tree fntype = gimple_call_fntype (call);
 938                 if (fntype && TYPE_READONLY (fntype))
 939                   {
 940                     int quals = TYPE_QUALS (fntype) & ~TYPE_QUAL_CONST;
 941                     fntype = build_qualified_type (fntype, quals);
 942                     gimple_call_set_fntype (call, fntype);
 943                   }
 944
 945                 /* Update virtual operands of calls to no longer const/pure
 946                    functions.  */
 947                 update_stmt (call);
 948               }
 949           }
 950
 951       /* re-merge split blocks.  */
 952       cleanup_tree_cfg ();
 953       update_ssa (TODO_update_ssa);
 954
 955       cgraph_edge::rebuild_edges ();
 956
 957       pop_cfun ();
 958     }
 959
 960   handle_missing_profiles ();
 961
 962   del_node_map ();
 963   return 0;
 964 }
 965
 966 namespace {
 967
 968 const pass_data pass_data_ipa_tree_profile =
 969 {
 970   SIMPLE_IPA_PASS, /* type */
 971   "profile", /* name */
 972   OPTGROUP_NONE, /* optinfo_flags */
 973   TV_IPA_PROFILE, /* tv_id */
 974   0, /* properties_required */
 975   0, /* properties_provided */
 976   0, /* properties_destroyed */
 977   0, /* todo_flags_start */
 978   TODO_dump_symtab, /* todo_flags_finish */
 979 };
 980
 981 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 982 {
 983 public:
 984   pass_ipa_tree_profile (gcc::context *ctxt)
 985     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 986   {}
 987
 988   /* opt_pass methods: */
 989   bool gate (function *) final override;
 990   unsigned int execute (function *) final override { return tree_profiling (); }
 991
 992 }; // class pass_ipa_tree_profile
 993
 994 bool
 995 pass_ipa_tree_profile::gate (function *)
 996 {
 997   /* When profile instrumentation, use or test coverage shall be performed.
 998      But for AutoFDO, this there is no instrumentation, thus this pass is
 999      disabled.  */
1000   return (!in_lto_p && !flag_auto_profile
1001           && (flag_branch_probabilities || flag_test_coverage
1002               || profile_arc_flag));
1003 }
1004
1005 } // anon namespace
1006
1007 simple_ipa_opt_pass *
1008 make_pass_ipa_tree_profile (gcc::context *ctxt)
1009 {
1010   return new pass_ipa_tree_profile (ctxt);
1011 }
1012
1013 #include "gt-tree-profile.h"