gcc/tree-profile.cc

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2023 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.cc for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "stringpool.h"
  53 #include "attribs.h"
  54 #include "tree-pretty-print.h"
  55 #include "langhooks.h"
  56 #include "stor-layout.h"
  57 #include "xregex.h"
  58 #include "alloc-pool.h"
  59 #include "symbol-summary.h"
  60 #include "symtab-thunks.h"
  61
  62 static GTY(()) tree gcov_type_node;
  63 static GTY(()) tree tree_interval_profiler_fn;
  64 static GTY(()) tree tree_pow2_profiler_fn;
  65 static GTY(()) tree tree_topn_values_profiler_fn;
  66 static GTY(()) tree tree_indirect_call_profiler_fn;
  67 static GTY(()) tree tree_average_profiler_fn;
  68 static GTY(()) tree tree_ior_profiler_fn;
  69 static GTY(()) tree tree_time_profiler_counter;
  70
  71
  72 static GTY(()) tree ic_tuple_var;
  73 static GTY(()) tree ic_tuple_counters_field;
  74 static GTY(()) tree ic_tuple_callee_field;
  75
  76 /* Do initialization work for the edge profiler.  */
  77
  78 /* Add code:
  79    __thread gcov*       __gcov_indirect_call.counters; // pointer to actual counter
  80    __thread void*       __gcov_indirect_call.callee; // actual callee address
  81    __thread int __gcov_function_counter; // time profiler function counter
  82 */
  83 static void
  84 init_ic_make_global_vars (void)
  85 {
  86   tree gcov_type_ptr;
  87
  88   gcov_type_ptr = build_pointer_type (get_gcov_type ());
  89
  90   tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
  91
  92   /* callee */
  93   ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
  94                                       ptr_type_node);
  95
  96   /* counters */
  97   ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
  98                                         NULL_TREE, gcov_type_ptr);
  99   DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
 100
 101   finish_builtin_struct (tuple_type, "indirect_call_tuple",
 102                          ic_tuple_counters_field, NULL_TREE);
 103
 104   ic_tuple_var
 105     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 106                   get_identifier ("__gcov_indirect_call"), tuple_type);
 107   TREE_PUBLIC (ic_tuple_var) = 1;
 108   DECL_ARTIFICIAL (ic_tuple_var) = 1;
 109   DECL_INITIAL (ic_tuple_var) = NULL;
 110   DECL_EXTERNAL (ic_tuple_var) = 1;
 111   if (targetm.have_tls)
 112     set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
 113 }
 114
 115 /* Create the type and function decls for the interface with gcov.  */
 116
 117 void
 118 gimple_init_gcov_profiler (void)
 119 {
 120   tree interval_profiler_fn_type;
 121   tree pow2_profiler_fn_type;
 122   tree topn_values_profiler_fn_type;
 123   tree gcov_type_ptr;
 124   tree ic_profiler_fn_type;
 125   tree average_profiler_fn_type;
 126   const char *fn_name;
 127
 128   if (!gcov_type_node)
 129     {
 130       const char *fn_suffix
 131         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 132
 133       gcov_type_node = get_gcov_type ();
 134       gcov_type_ptr = build_pointer_type (gcov_type_node);
 135
 136       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 137       interval_profiler_fn_type
 138               = build_function_type_list (void_type_node,
 139                                           gcov_type_ptr, gcov_type_node,
 140                                           integer_type_node,
 141                                           unsigned_type_node, NULL_TREE);
 142       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 143       tree_interval_profiler_fn = build_fn_decl (fn_name,
 144                                                  interval_profiler_fn_type);
 145       free (CONST_CAST (char *, fn_name));
 146       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 147       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 148         = tree_cons (get_identifier ("leaf"), NULL,
 149                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 150
 151       /* void (*) (gcov_type *, gcov_type)  */
 152       pow2_profiler_fn_type
 153               = build_function_type_list (void_type_node,
 154                                           gcov_type_ptr, gcov_type_node,
 155                                           NULL_TREE);
 156       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 157       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 158       free (CONST_CAST (char *, fn_name));
 159       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 160       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 161         = tree_cons (get_identifier ("leaf"), NULL,
 162                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 163
 164       /* void (*) (gcov_type *, gcov_type)  */
 165       topn_values_profiler_fn_type
 166               = build_function_type_list (void_type_node,
 167                                           gcov_type_ptr, gcov_type_node,
 168                                           NULL_TREE);
 169       fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
 170       tree_topn_values_profiler_fn
 171         = build_fn_decl (fn_name, topn_values_profiler_fn_type);
 172       free (CONST_CAST (char *, fn_name));
 173
 174       TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
 175       DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
 176         = tree_cons (get_identifier ("leaf"), NULL,
 177                      DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
 178
 179       init_ic_make_global_vars ();
 180
 181       /* void (*) (gcov_type, void *)  */
 182       ic_profiler_fn_type
 183                = build_function_type_list (void_type_node,
 184                                           gcov_type_node,
 185                                           ptr_type_node,
 186                                           NULL_TREE);
 187       fn_name = concat ("__gcov_indirect_call_profiler_v4", fn_suffix, NULL);
 188       tree_indirect_call_profiler_fn
 189         = build_fn_decl (fn_name, ic_profiler_fn_type);
 190       free (CONST_CAST (char *, fn_name));
 191
 192       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 193       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 194         = tree_cons (get_identifier ("leaf"), NULL,
 195                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 196
 197       tree_time_profiler_counter
 198         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 199                       get_identifier ("__gcov_time_profiler_counter"),
 200                       get_gcov_type ());
 201       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 202       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 203       TREE_STATIC (tree_time_profiler_counter) = 1;
 204       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 205       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 206
 207       /* void (*) (gcov_type *, gcov_type)  */
 208       average_profiler_fn_type
 209               = build_function_type_list (void_type_node,
 210                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 211       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 212       tree_average_profiler_fn = build_fn_decl (fn_name,
 213                                                 average_profiler_fn_type);
 214       free (CONST_CAST (char *, fn_name));
 215       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 216       DECL_ATTRIBUTES (tree_average_profiler_fn)
 217         = tree_cons (get_identifier ("leaf"), NULL,
 218                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 219       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 220       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 221       free (CONST_CAST (char *, fn_name));
 222       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 223       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 224         = tree_cons (get_identifier ("leaf"), NULL,
 225                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 226
 227       /* LTO streamer needs assembler names.  Because we create these decls
 228          late, we need to initialize them by hand.  */
 229       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 230       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 231       DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
 232       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 233       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 234       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 235     }
 236 }
 237
 238 /* Output instructions as GIMPLE trees to increment the edge
 239    execution count, and insert them on E.  We rely on
 240    gsi_insert_on_edge to preserve the order.  */
 241
 242 void
 243 gimple_gen_edge_profiler (int edgeno, edge e)
 244 {
 245   tree one;
 246
 247   one = build_int_cst (gcov_type_node, 1);
 248
 249   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 250     {
 251       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 252       tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
 253       tree f = builtin_decl_explicit (TYPE_PRECISION (gcov_type_node) > 32
 254                                       ? BUILT_IN_ATOMIC_FETCH_ADD_8:
 255                                       BUILT_IN_ATOMIC_FETCH_ADD_4);
 256       gcall *stmt = gimple_build_call (f, 3, addr, one,
 257                                        build_int_cst (integer_type_node,
 258                                                       MEMMODEL_RELAXED));
 259       gsi_insert_on_edge (e, stmt);
 260     }
 261   else
 262     {
 263       tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 264       tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 265                                                    NULL, "PROF_edge_counter");
 266       gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
 267       gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 268                                               NULL, "PROF_edge_counter");
 269       gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
 270                                             gimple_assign_lhs (stmt1), one);
 271       gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
 272                                             gimple_assign_lhs (stmt2));
 273       gsi_insert_on_edge (e, stmt1);
 274       gsi_insert_on_edge (e, stmt2);
 275       gsi_insert_on_edge (e, stmt3);
 276     }
 277 }
 278
 279 /* Emits code to get VALUE to instrument at GSI, and returns the
 280    variable containing the value.  */
 281
 282 static tree
 283 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 284 {
 285   tree val = value->hvalue.value;
 286   if (POINTER_TYPE_P (TREE_TYPE (val)))
 287     val = fold_convert (build_nonstandard_integer_type
 288                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 289   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 290                                    true, NULL_TREE, true, GSI_SAME_STMT);
 291 }
 292
 293 /* Output instructions as GIMPLE trees to increment the interval histogram
 294    counter.  VALUE is the expression whose value is profiled.  TAG is the
 295    tag of the section for counters, BASE is offset of the counter position.  */
 296
 297 void
 298 gimple_gen_interval_profiler (histogram_value value, unsigned tag)
 299 {
 300   gimple *stmt = value->hvalue.stmt;
 301   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 302   tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr;
 303   gcall *call;
 304   tree val;
 305   tree start = build_int_cst_type (integer_type_node,
 306                                    value->hdata.intvl.int_start);
 307   tree steps = build_int_cst_type (unsigned_type_node,
 308                                    value->hdata.intvl.steps);
 309
 310   ref_ptr = force_gimple_operand_gsi (&gsi,
 311                                       build_addr (ref),
 312                                       true, NULL_TREE, true, GSI_SAME_STMT);
 313   val = prepare_instrumented_value (&gsi, value);
 314   call = gimple_build_call (tree_interval_profiler_fn, 4,
 315                             ref_ptr, val, start, steps);
 316   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 317 }
 318
 319 /* Output instructions as GIMPLE trees to increment the power of two histogram
 320    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 321    of the section for counters.  */
 322
 323 void
 324 gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
 325 {
 326   gimple *stmt = value->hvalue.stmt;
 327   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 328   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 329   gcall *call;
 330   tree val;
 331
 332   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 333                                       true, NULL_TREE, true, GSI_SAME_STMT);
 334   val = prepare_instrumented_value (&gsi, value);
 335   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 336   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 337 }
 338
 339 /* Output instructions as GIMPLE trees for code to find the most N common
 340    values.  VALUE is the expression whose value is profiled.  TAG is the tag
 341    of the section for counters.  */
 342
 343 void
 344 gimple_gen_topn_values_profiler (histogram_value value, unsigned tag)
 345 {
 346   gimple *stmt = value->hvalue.stmt;
 347   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 348   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 349   gcall *call;
 350   tree val;
 351
 352   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 353                                       true, NULL_TREE, true, GSI_SAME_STMT);
 354   val = prepare_instrumented_value (&gsi, value);
 355   call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
 356   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 357 }
 358
 359
 360 /* Output instructions as GIMPLE trees for code to find the most
 361    common called function in indirect call.
 362    VALUE is the call expression whose indirect callee is profiled.
 363    TAG is the tag of the section for counters.  */
 364
 365 void
 366 gimple_gen_ic_profiler (histogram_value value, unsigned tag)
 367 {
 368   tree tmp1;
 369   gassign *stmt1, *stmt2, *stmt3;
 370   gimple *stmt = value->hvalue.stmt;
 371   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 372   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 373
 374   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 375                                       true, NULL_TREE, true, GSI_SAME_STMT);
 376
 377   /* Insert code:
 378
 379     stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
 380     stmt2: tmp1 = (void *) (indirect call argument value)
 381     stmt3: __gcov_indirect_call.callee = tmp1;
 382
 383     Example:
 384       f_1 = foo;
 385       __gcov_indirect_call.counters = &__gcov4.main[0];
 386       PROF_fn_9 = f_1;
 387       __gcov_indirect_call.callee = PROF_fn_9;
 388       _4 = f_1 ();
 389    */
 390
 391   tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
 392
 393   tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
 394                              ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
 395
 396   stmt1 = gimple_build_assign (counter_ref, ref_ptr);
 397   tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF_fn");
 398   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 399   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 400                              ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 401   stmt3 = gimple_build_assign (callee_ref, tmp1);
 402
 403   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 404   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 405   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 406 }
 407
 408
 409 /* Output instructions as GIMPLE trees for code to find the most
 410    common called function in indirect call. Insert instructions at the
 411    beginning of every possible called function.
 412   */
 413
 414 void
 415 gimple_gen_ic_func_profiler (void)
 416 {
 417   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 418   gcall *stmt1;
 419   tree tree_uid, cur_func, void0;
 420
 421   if (c_node->only_called_directly_p ())
 422     return;
 423
 424   gimple_init_gcov_profiler ();
 425
 426   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 427   basic_block cond_bb = split_edge (single_succ_edge (entry));
 428   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 429
 430   /* We need to do an extra split in order to not create an input
 431      for a possible PHI node.  */
 432   split_edge (single_succ_edge (update_bb));
 433
 434   edge true_edge = single_succ_edge (cond_bb);
 435   true_edge->flags = EDGE_TRUE_VALUE;
 436
 437   profile_probability probability;
 438   if (DECL_VIRTUAL_P (current_function_decl))
 439     probability = profile_probability::very_likely ();
 440   else
 441     probability = profile_probability::unlikely ();
 442
 443   true_edge->probability = probability;
 444   edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
 445                       EDGE_FALSE_VALUE);
 446   e->probability = true_edge->probability.invert ();
 447
 448   /* Insert code:
 449
 450      if (__gcov_indirect_call.callee != NULL)
 451        __gcov_indirect_call_profiler_v3 (profile_id, &current_function_decl);
 452
 453      The function __gcov_indirect_call_profiler_v3 is responsible for
 454      resetting __gcov_indirect_call.callee to NULL.  */
 455
 456   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 457   void0 = build_int_cst (ptr_type_node, 0);
 458
 459   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 460                             ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 461
 462   tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
 463                                        true, GSI_SAME_STMT);
 464
 465   gcond *cond = gimple_build_cond (NE_EXPR, ref,
 466                                    void0, NULL, NULL);
 467   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 468
 469   gsi = gsi_after_labels (update_bb);
 470
 471   cur_func = force_gimple_operand_gsi (&gsi,
 472                                        build_addr (current_function_decl),
 473                                        true, NULL_TREE,
 474                                        true, GSI_SAME_STMT);
 475   tree_uid = build_int_cst
 476               (gcov_type_node,
 477                cgraph_node::get (current_function_decl)->profile_id);
 478   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 479                              tree_uid, cur_func);
 480   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 481 }
 482
 483 /* Output instructions as GIMPLE tree at the beginning for each function.
 484    TAG is the tag of the section for counters, BASE is offset of the
 485    counter position and GSI is the iterator we place the counter.  */
 486
 487 void
 488 gimple_gen_time_profiler (unsigned tag)
 489 {
 490   tree type = get_gcov_type ();
 491   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 492   basic_block cond_bb = split_edge (single_succ_edge (entry));
 493   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 494
 495   /* We need to do an extra split in order to not create an input
 496      for a possible PHI node.  */
 497   split_edge (single_succ_edge (update_bb));
 498
 499   edge true_edge = single_succ_edge (cond_bb);
 500   true_edge->flags = EDGE_TRUE_VALUE;
 501   true_edge->probability = profile_probability::unlikely ();
 502   edge e
 503     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 504   e->probability = true_edge->probability.invert ();
 505
 506   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 507   tree original_ref = tree_coverage_counter_ref (tag, 0);
 508   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 509                                        true, GSI_SAME_STMT);
 510   tree one = build_int_cst (type, 1);
 511
 512   /* Emit: if (counters[0] != 0).  */
 513   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 514                                    NULL, NULL);
 515   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 516
 517   gsi = gsi_start_bb (update_bb);
 518
 519   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 520   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 521     {
 522       tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
 523                                      "PROF_time_profiler_counter_ptr");
 524       tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
 525                           tree_time_profiler_counter);
 526       gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
 527       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 528       tree f = builtin_decl_explicit (TYPE_PRECISION (gcov_type_node) > 32
 529                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8:
 530                                       BUILT_IN_ATOMIC_ADD_FETCH_4);
 531       gcall *stmt = gimple_build_call (f, 3, ptr, one,
 532                                        build_int_cst (integer_type_node,
 533                                                       MEMMODEL_RELAXED));
 534       tree result_type = TREE_TYPE (TREE_TYPE (f));
 535       tree tmp = make_temp_ssa_name (result_type, NULL, "PROF_time_profile");
 536       gimple_set_lhs (stmt, tmp);
 537       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
 538       tmp = make_temp_ssa_name (type, NULL, "PROF_time_profile");
 539       assign = gimple_build_assign (tmp, NOP_EXPR,
 540                                     gimple_call_lhs (stmt));
 541       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 542       assign = gimple_build_assign (original_ref, tmp);
 543       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 544     }
 545   else
 546     {
 547       tree tmp = make_temp_ssa_name (type, NULL, "PROF_time_profile");
 548       gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
 549       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 550
 551       tmp = make_temp_ssa_name (type, NULL, "PROF_time_profile");
 552       assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
 553                                     one);
 554       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 555       assign = gimple_build_assign (original_ref, tmp);
 556       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 557       assign = gimple_build_assign (tree_time_profiler_counter, tmp);
 558       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 559     }
 560 }
 561
 562 /* Output instructions as GIMPLE trees to increment the average histogram
 563    counter.  VALUE is the expression whose value is profiled.  TAG is the
 564    tag of the section for counters, BASE is offset of the counter position.  */
 565
 566 void
 567 gimple_gen_average_profiler (histogram_value value, unsigned tag)
 568 {
 569   gimple *stmt = value->hvalue.stmt;
 570   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 571   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 572   gcall *call;
 573   tree val;
 574
 575   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 576                                       true, NULL_TREE,
 577                                       true, GSI_SAME_STMT);
 578   val = prepare_instrumented_value (&gsi, value);
 579   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 580   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 581 }
 582
 583 /* Output instructions as GIMPLE trees to increment the ior histogram
 584    counter.  VALUE is the expression whose value is profiled.  TAG is the
 585    tag of the section for counters, BASE is offset of the counter position.  */
 586
 587 void
 588 gimple_gen_ior_profiler (histogram_value value, unsigned tag)
 589 {
 590   gimple *stmt = value->hvalue.stmt;
 591   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 592   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 593   gcall *call;
 594   tree val;
 595
 596   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 597                                       true, NULL_TREE, true, GSI_SAME_STMT);
 598   val = prepare_instrumented_value (&gsi, value);
 599   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 600   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 601 }
 602
 603 static vec<regex_t> profile_filter_files;
 604 static vec<regex_t> profile_exclude_files;
 605
 606 /* Parse list of provided REGEX (separated with semi-collon) and
 607    create expressions (of type regex_t) and save them into V vector.
 608    If there is a regular expression parsing error, error message is
 609    printed for FLAG_NAME.  */
 610
 611 static void
 612 parse_profile_filter (const char *regex, vec<regex_t> *v,
 613                       const char *flag_name)
 614 {
 615   v->create (4);
 616   if (regex != NULL)
 617     {
 618       char *str = xstrdup (regex);
 619       for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
 620         {
 621           regex_t r;
 622           if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
 623             {
 624               error ("invalid regular expression %qs in %qs",
 625                      p, flag_name);
 626               return;
 627             }
 628
 629           v->safe_push (r);
 630         }
 631     }
 632 }
 633
 634 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files
 635    options.  */
 636
 637 static void
 638 parse_profile_file_filtering ()
 639 {
 640   parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
 641                         "-fprofile-filter-files");
 642   parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
 643                         "-fprofile-exclude-files");
 644 }
 645
 646 /* Parse vectors of regular expressions.  */
 647
 648 static void
 649 release_profile_file_filtering ()
 650 {
 651   profile_filter_files.release ();
 652   profile_exclude_files.release ();
 653 }
 654
 655 /* Return true when FILENAME should be instrumented based on
 656    -fprofile-filter-files and -fprofile-exclude-files options.  */
 657
 658 static bool
 659 include_source_file_for_profile (const char *filename)
 660 {
 661   /* First check whether file is included in flag_profile_exclude_files.  */
 662   for (unsigned i = 0; i < profile_exclude_files.length (); i++)
 663     if (regexec (&profile_exclude_files[i],
 664                  filename, 0, NULL, 0) == REG_NOERROR)
 665       return false;
 666
 667   /* For non-empty flag_profile_filter_files include only files matching a
 668      regex in the flag.  */
 669   if (profile_filter_files.is_empty ())
 670     return true;
 671
 672   for (unsigned i = 0; i < profile_filter_files.length (); i++)
 673     if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
 674       return true;
 675
 676   return false;
 677 }
 678
 679 #ifndef HAVE_sync_compare_and_swapsi
 680 #define HAVE_sync_compare_and_swapsi 0
 681 #endif
 682 #ifndef HAVE_atomic_compare_and_swapsi
 683 #define HAVE_atomic_compare_and_swapsi 0
 684 #endif
 685
 686 #ifndef HAVE_sync_compare_and_swapdi
 687 #define HAVE_sync_compare_and_swapdi 0
 688 #endif
 689 #ifndef HAVE_atomic_compare_and_swapdi
 690 #define HAVE_atomic_compare_and_swapdi 0
 691 #endif
 692
 693 /* Profile all functions in the callgraph.  */
 694
 695 static unsigned int
 696 tree_profiling (void)
 697 {
 698   struct cgraph_node *node;
 699
 700   /* Verify whether we can utilize atomic update operations.  */
 701   bool can_support_atomic = false;
 702   unsigned HOST_WIDE_INT gcov_type_size
 703     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 704   if (gcov_type_size == 4)
 705     can_support_atomic
 706       = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 707   else if (gcov_type_size == 8)
 708     can_support_atomic
 709       = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 710
 711   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 712       && !can_support_atomic)
 713     {
 714       warning (0, "target does not support atomic profile update, "
 715                "single mode is selected");
 716       flag_profile_update = PROFILE_UPDATE_SINGLE;
 717     }
 718   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 719     flag_profile_update = can_support_atomic
 720       ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 721
 722   /* This is a small-ipa pass that gets called only once, from
 723      cgraphunit.cc:ipa_passes().  */
 724   gcc_assert (symtab->state == IPA_SSA);
 725
 726   init_node_map (true);
 727   parse_profile_file_filtering ();
 728
 729   FOR_EACH_DEFINED_FUNCTION (node)
 730     {
 731       bool thunk = false;
 732       if (!gimple_has_body_p (node->decl) && !node->thunk)
 733         continue;
 734
 735       /* Don't profile functions produced for builtin stuff.  */
 736       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 737         continue;
 738
 739       if (lookup_attribute ("no_profile_instrument_function",
 740                             DECL_ATTRIBUTES (node->decl)))
 741         continue;
 742       /* Do not instrument extern inline functions when testing coverage.
 743          While this is not perfectly consistent (early inlined extern inlines
 744          will get acocunted), testsuite expects that.  */
 745       if (DECL_EXTERNAL (node->decl)
 746           && flag_test_coverage)
 747         continue;
 748
 749       const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
 750       if (!include_source_file_for_profile (file))
 751         continue;
 752
 753       if (node->thunk)
 754         {
 755           /* We cannot expand variadic thunks to Gimple.  */
 756           if (stdarg_p (TREE_TYPE (node->decl)))
 757             continue;
 758           thunk = true;
 759           /* When generate profile, expand thunk to gimple so it can be
 760              instrumented same way as other functions.  */
 761           if (profile_arc_flag)
 762             expand_thunk (node, false, true);
 763           /* Read cgraph profile but keep function as thunk at profile-use
 764              time.  */
 765           else
 766             {
 767               read_thunk_profile (node);
 768               continue;
 769             }
 770         }
 771
 772       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 773
 774       if (dump_file)
 775         dump_function_header (dump_file, cfun->decl, dump_flags);
 776
 777       /* Local pure-const may imply need to fixup the cfg.  */
 778       if (gimple_has_body_p (node->decl)
 779           && (execute_fixup_cfg () & TODO_cleanup_cfg))
 780         cleanup_tree_cfg ();
 781
 782       branch_prob (thunk);
 783
 784       if (! flag_branch_probabilities
 785           && flag_profile_values)
 786         gimple_gen_ic_func_profiler ();
 787
 788       if (flag_branch_probabilities
 789           && !thunk
 790           && flag_profile_values
 791           && flag_value_profile_transformations
 792           && profile_status_for_fn (cfun) == PROFILE_READ)
 793         gimple_value_profile_transformations ();
 794
 795       /* The above could hose dominator info.  Currently there is
 796          none coming in, this is a safety valve.  It should be
 797          easy to adjust it, if and when there is some.  */
 798       free_dominance_info (CDI_DOMINATORS);
 799       free_dominance_info (CDI_POST_DOMINATORS);
 800       pop_cfun ();
 801     }
 802
 803   release_profile_file_filtering ();
 804
 805   /* Drop pure/const flags from instrumented functions.  */
 806   if (profile_arc_flag || flag_test_coverage)
 807     FOR_EACH_DEFINED_FUNCTION (node)
 808       {
 809         if (!gimple_has_body_p (node->decl)
 810             || !(!node->clone_of
 811             || node->decl != node->clone_of->decl))
 812           continue;
 813
 814         /* Don't profile functions produced for builtin stuff.  */
 815         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 816           continue;
 817
 818         node->set_const_flag (false, false);
 819         node->set_pure_flag (false, false);
 820       }
 821
 822   /* Update call statements and rebuild the cgraph.  */
 823   FOR_EACH_DEFINED_FUNCTION (node)
 824     {
 825       basic_block bb;
 826
 827       if (!gimple_has_body_p (node->decl)
 828           || !(!node->clone_of
 829           || node->decl != node->clone_of->decl))
 830         continue;
 831
 832       /* Don't profile functions produced for builtin stuff.  */
 833       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 834         continue;
 835
 836       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 837
 838       FOR_EACH_BB_FN (bb, cfun)
 839         {
 840           gimple_stmt_iterator gsi;
 841           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 842             {
 843               gimple *stmt = gsi_stmt (gsi);
 844               if (is_gimple_call (stmt))
 845                 update_stmt (stmt);
 846             }
 847         }
 848
 849       /* re-merge split blocks.  */
 850       cleanup_tree_cfg ();
 851       update_ssa (TODO_update_ssa);
 852
 853       cgraph_edge::rebuild_edges ();
 854
 855       pop_cfun ();
 856     }
 857
 858   handle_missing_profiles ();
 859
 860   del_node_map ();
 861   return 0;
 862 }
 863
 864 namespace {
 865
 866 const pass_data pass_data_ipa_tree_profile =
 867 {
 868   SIMPLE_IPA_PASS, /* type */
 869   "profile", /* name */
 870   OPTGROUP_NONE, /* optinfo_flags */
 871   TV_IPA_PROFILE, /* tv_id */
 872   0, /* properties_required */
 873   0, /* properties_provided */
 874   0, /* properties_destroyed */
 875   0, /* todo_flags_start */
 876   TODO_dump_symtab, /* todo_flags_finish */
 877 };
 878
 879 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 880 {
 881 public:
 882   pass_ipa_tree_profile (gcc::context *ctxt)
 883     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 884   {}
 885
 886   /* opt_pass methods: */
 887   bool gate (function *) final override;
 888   unsigned int execute (function *) final override { return tree_profiling (); }
 889
 890 }; // class pass_ipa_tree_profile
 891
 892 bool
 893 pass_ipa_tree_profile::gate (function *)
 894 {
 895   /* When profile instrumentation, use or test coverage shall be performed.
 896      But for AutoFDO, this there is no instrumentation, thus this pass is
 897      disabled.  */
 898   return (!in_lto_p && !flag_auto_profile
 899           && (flag_branch_probabilities || flag_test_coverage
 900               || profile_arc_flag));
 901 }
 902
 903 } // anon namespace
 904
 905 simple_ipa_opt_pass *
 906 make_pass_ipa_tree_profile (gcc::context *ctxt)
 907 {
 908   return new pass_ipa_tree_profile (ctxt);
 909 }
 910
 911 #include "gt-tree-profile.h"