gcc/tree-profile.c

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2019 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.c for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "params.h"
  53 #include "stringpool.h"
  54 #include "attribs.h"
  55 #include "tree-pretty-print.h"
  56 #include "langhooks.h"
  57 #include "stor-layout.h"
  58 #include "xregex.h"
  59
  60 static GTY(()) tree gcov_type_node;
  61 static GTY(()) tree tree_interval_profiler_fn;
  62 static GTY(()) tree tree_pow2_profiler_fn;
  63 static GTY(()) tree tree_topn_values_profiler_fn;
  64 static GTY(()) tree tree_indirect_call_profiler_fn;
  65 static GTY(()) tree tree_average_profiler_fn;
  66 static GTY(()) tree tree_ior_profiler_fn;
  67 static GTY(()) tree tree_time_profiler_counter;
  68
  69
  70 static GTY(()) tree ic_tuple_var;
  71 static GTY(()) tree ic_tuple_counters_field;
  72 static GTY(()) tree ic_tuple_callee_field;
  73
  74 /* Do initialization work for the edge profiler.  */
  75
  76 /* Add code:
  77    __thread gcov*       __gcov_indirect_call_counters; // pointer to actual counter
  78    __thread void*       __gcov_indirect_call_callee; // actual callee address
  79    __thread int __gcov_function_counter; // time profiler function counter
  80 */
  81 static void
  82 init_ic_make_global_vars (void)
  83 {
  84   tree gcov_type_ptr;
  85
  86   gcov_type_ptr = build_pointer_type (get_gcov_type ());
  87
  88   tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
  89
  90   /* callee */
  91   ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
  92                                       ptr_type_node);
  93
  94   /* counters */
  95   ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
  96                                         NULL_TREE, gcov_type_ptr);
  97   DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
  98
  99   finish_builtin_struct (tuple_type, "indirect_call_tuple",
 100                          ic_tuple_counters_field, NULL_TREE);
 101
 102   ic_tuple_var
 103     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 104                   get_identifier ("__gcov_indirect_call"), tuple_type);
 105   TREE_PUBLIC (ic_tuple_var) = 1;
 106   DECL_ARTIFICIAL (ic_tuple_var) = 1;
 107   DECL_INITIAL (ic_tuple_var) = NULL;
 108   DECL_EXTERNAL (ic_tuple_var) = 1;
 109   if (targetm.have_tls)
 110     set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
 111 }
 112
 113 /* Create the type and function decls for the interface with gcov.  */
 114
 115 void
 116 gimple_init_gcov_profiler (void)
 117 {
 118   tree interval_profiler_fn_type;
 119   tree pow2_profiler_fn_type;
 120   tree topn_values_profiler_fn_type;
 121   tree gcov_type_ptr;
 122   tree ic_profiler_fn_type;
 123   tree average_profiler_fn_type;
 124   const char *profiler_fn_name;
 125   const char *fn_name;
 126
 127   if (!gcov_type_node)
 128     {
 129       const char *fn_suffix
 130         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 131
 132       gcov_type_node = get_gcov_type ();
 133       gcov_type_ptr = build_pointer_type (gcov_type_node);
 134
 135       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 136       interval_profiler_fn_type
 137               = build_function_type_list (void_type_node,
 138                                           gcov_type_ptr, gcov_type_node,
 139                                           integer_type_node,
 140                                           unsigned_type_node, NULL_TREE);
 141       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 142       tree_interval_profiler_fn = build_fn_decl (fn_name,
 143                                                  interval_profiler_fn_type);
 144       free (CONST_CAST (char *, fn_name));
 145       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 146       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 147         = tree_cons (get_identifier ("leaf"), NULL,
 148                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 149
 150       /* void (*) (gcov_type *, gcov_type)  */
 151       pow2_profiler_fn_type
 152               = build_function_type_list (void_type_node,
 153                                           gcov_type_ptr, gcov_type_node,
 154                                           NULL_TREE);
 155       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 156       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 157       free (CONST_CAST (char *, fn_name));
 158       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 159       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 160         = tree_cons (get_identifier ("leaf"), NULL,
 161                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 162
 163       /* void (*) (gcov_type *, gcov_type)  */
 164       topn_values_profiler_fn_type
 165               = build_function_type_list (void_type_node,
 166                                           gcov_type_ptr, gcov_type_node,
 167                                           NULL_TREE);
 168       fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
 169       tree_topn_values_profiler_fn
 170         = build_fn_decl (fn_name, topn_values_profiler_fn_type);
 171
 172       TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
 173       DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
 174         = tree_cons (get_identifier ("leaf"), NULL,
 175                      DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
 176
 177       init_ic_make_global_vars ();
 178
 179       /* void (*) (gcov_type, void *)  */
 180       ic_profiler_fn_type
 181                = build_function_type_list (void_type_node,
 182                                           gcov_type_node,
 183                                           ptr_type_node,
 184                                           NULL_TREE);
 185       profiler_fn_name = "__gcov_indirect_call_profiler_v4";
 186
 187       tree_indirect_call_profiler_fn
 188               = build_fn_decl (profiler_fn_name, ic_profiler_fn_type);
 189
 190       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 191       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 192         = tree_cons (get_identifier ("leaf"), NULL,
 193                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 194
 195       tree_time_profiler_counter
 196         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 197                       get_identifier ("__gcov_time_profiler_counter"),
 198                       get_gcov_type ());
 199       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 200       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 201       TREE_STATIC (tree_time_profiler_counter) = 1;
 202       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 203       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 204
 205       /* void (*) (gcov_type *, gcov_type)  */
 206       average_profiler_fn_type
 207               = build_function_type_list (void_type_node,
 208                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 209       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 210       tree_average_profiler_fn = build_fn_decl (fn_name,
 211                                                 average_profiler_fn_type);
 212       free (CONST_CAST (char *, fn_name));
 213       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 214       DECL_ATTRIBUTES (tree_average_profiler_fn)
 215         = tree_cons (get_identifier ("leaf"), NULL,
 216                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 217       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 218       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 219       free (CONST_CAST (char *, fn_name));
 220       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 221       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 222         = tree_cons (get_identifier ("leaf"), NULL,
 223                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 224
 225       /* LTO streamer needs assembler names.  Because we create these decls
 226          late, we need to initialize them by hand.  */
 227       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 228       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 229       DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
 230       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 231       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 232       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 233     }
 234 }
 235
 236 /* Output instructions as GIMPLE trees to increment the edge
 237    execution count, and insert them on E.  We rely on
 238    gsi_insert_on_edge to preserve the order.  */
 239
 240 void
 241 gimple_gen_edge_profiler (int edgeno, edge e)
 242 {
 243   tree one;
 244
 245   one = build_int_cst (gcov_type_node, 1);
 246
 247   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 248     {
 249       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 250       tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
 251       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 252                                       ? BUILT_IN_ATOMIC_FETCH_ADD_8:
 253                                       BUILT_IN_ATOMIC_FETCH_ADD_4);
 254       gcall *stmt = gimple_build_call (f, 3, addr, one,
 255                                        build_int_cst (integer_type_node,
 256                                                       MEMMODEL_RELAXED));
 257       gsi_insert_on_edge (e, stmt);
 258     }
 259   else
 260     {
 261       tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 262       tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 263                                                    NULL, "PROF_edge_counter");
 264       gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
 265       gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 266                                               NULL, "PROF_edge_counter");
 267       gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
 268                                             gimple_assign_lhs (stmt1), one);
 269       gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
 270                                             gimple_assign_lhs (stmt2));
 271       gsi_insert_on_edge (e, stmt1);
 272       gsi_insert_on_edge (e, stmt2);
 273       gsi_insert_on_edge (e, stmt3);
 274     }
 275 }
 276
 277 /* Emits code to get VALUE to instrument at GSI, and returns the
 278    variable containing the value.  */
 279
 280 static tree
 281 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 282 {
 283   tree val = value->hvalue.value;
 284   if (POINTER_TYPE_P (TREE_TYPE (val)))
 285     val = fold_convert (build_nonstandard_integer_type
 286                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 287   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 288                                    true, NULL_TREE, true, GSI_SAME_STMT);
 289 }
 290
 291 /* Output instructions as GIMPLE trees to increment the interval histogram
 292    counter.  VALUE is the expression whose value is profiled.  TAG is the
 293    tag of the section for counters, BASE is offset of the counter position.  */
 294
 295 void
 296 gimple_gen_interval_profiler (histogram_value value, unsigned tag, unsigned base)
 297 {
 298   gimple *stmt = value->hvalue.stmt;
 299   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 300   tree ref = tree_coverage_counter_ref (tag, base), ref_ptr;
 301   gcall *call;
 302   tree val;
 303   tree start = build_int_cst_type (integer_type_node,
 304                                    value->hdata.intvl.int_start);
 305   tree steps = build_int_cst_type (unsigned_type_node,
 306                                    value->hdata.intvl.steps);
 307
 308   ref_ptr = force_gimple_operand_gsi (&gsi,
 309                                       build_addr (ref),
 310                                       true, NULL_TREE, true, GSI_SAME_STMT);
 311   val = prepare_instrumented_value (&gsi, value);
 312   call = gimple_build_call (tree_interval_profiler_fn, 4,
 313                             ref_ptr, val, start, steps);
 314   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 315 }
 316
 317 /* Output instructions as GIMPLE trees to increment the power of two histogram
 318    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 319    of the section for counters, BASE is offset of the counter position.  */
 320
 321 void
 322 gimple_gen_pow2_profiler (histogram_value value, unsigned tag, unsigned base)
 323 {
 324   gimple *stmt = value->hvalue.stmt;
 325   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 326   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 327   gcall *call;
 328   tree val;
 329
 330   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 331                                       true, NULL_TREE, true, GSI_SAME_STMT);
 332   val = prepare_instrumented_value (&gsi, value);
 333   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 334   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 335 }
 336
 337 /* Output instructions as GIMPLE trees for code to find the most N common
 338    values.  VALUE is the expression whose value is profiled.  TAG is the tag
 339    of the section for counters, BASE is offset of the counter position.  */
 340
 341 void
 342 gimple_gen_topn_values_profiler (histogram_value value, unsigned tag,
 343                                  unsigned base)
 344 {
 345   gimple *stmt = value->hvalue.stmt;
 346   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 347   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 348   gcall *call;
 349   tree val;
 350
 351   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 352                                       true, NULL_TREE, true, GSI_SAME_STMT);
 353   val = prepare_instrumented_value (&gsi, value);
 354   call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
 355   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 356 }
 357
 358
 359 /* Output instructions as GIMPLE trees for code to find the most
 360    common called function in indirect call.
 361    VALUE is the call expression whose indirect callee is profiled.
 362    TAG is the tag of the section for counters, BASE is offset of the
 363    counter position.  */
 364
 365 void
 366 gimple_gen_ic_profiler (histogram_value value, unsigned tag, unsigned base)
 367 {
 368   tree tmp1;
 369   gassign *stmt1, *stmt2, *stmt3;
 370   gimple *stmt = value->hvalue.stmt;
 371   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 372   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 373
 374   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 375                                       true, NULL_TREE, true, GSI_SAME_STMT);
 376
 377   /* Insert code:
 378
 379     stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
 380     stmt2: tmp1 = (void *) (indirect call argument value)
 381     stmt3: __gcov_indirect_call.callee = tmp1;
 382
 383     Example:
 384       f_1 = foo;
 385       __gcov_indirect_call.counters = &__gcov4.main[0];
 386       PROF_9 = f_1;
 387       __gcov_indirect_call_callee = PROF_9;
 388       _4 = f_1 ();
 389    */
 390
 391   tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
 392
 393   tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
 394                              ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
 395
 396   stmt1 = gimple_build_assign (counter_ref, ref_ptr);
 397   tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF");
 398   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 399   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 400                              ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 401   stmt3 = gimple_build_assign (callee_ref, tmp1);
 402
 403   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 404   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 405   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 406 }
 407
 408
 409 /* Output instructions as GIMPLE trees for code to find the most
 410    common called function in indirect call. Insert instructions at the
 411    beginning of every possible called function.
 412   */
 413
 414 void
 415 gimple_gen_ic_func_profiler (void)
 416 {
 417   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 418   gcall *stmt1;
 419   tree tree_uid, cur_func, void0;
 420
 421   if (c_node->only_called_directly_p ())
 422     return;
 423
 424   gimple_init_gcov_profiler ();
 425
 426   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 427   basic_block cond_bb = split_edge (single_succ_edge (entry));
 428   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 429
 430   /* We need to do an extra split in order to not create an input
 431      for a possible PHI node.  */
 432   split_edge (single_succ_edge (update_bb));
 433
 434   edge true_edge = single_succ_edge (cond_bb);
 435   true_edge->flags = EDGE_TRUE_VALUE;
 436
 437   profile_probability probability;
 438   if (DECL_VIRTUAL_P (current_function_decl))
 439     probability = profile_probability::very_likely ();
 440   else
 441     probability = profile_probability::unlikely ();
 442
 443   true_edge->probability = probability;
 444   edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
 445                       EDGE_FALSE_VALUE);
 446   e->probability = true_edge->probability.invert ();
 447
 448   /* Insert code:
 449
 450      if (__gcov_indirect_call_callee != NULL)
 451        __gcov_indirect_call_profiler_v3 (profile_id, &current_function_decl);
 452
 453      The function __gcov_indirect_call_profiler_v3 is responsible for
 454      resetting __gcov_indirect_call_callee to NULL.  */
 455
 456   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 457   void0 = build_int_cst (ptr_type_node, 0);
 458
 459   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 460                             ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 461
 462   tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
 463                                        true, GSI_SAME_STMT);
 464
 465   gcond *cond = gimple_build_cond (NE_EXPR, ref,
 466                                    void0, NULL, NULL);
 467   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 468
 469   gsi = gsi_after_labels (update_bb);
 470
 471   cur_func = force_gimple_operand_gsi (&gsi,
 472                                        build_addr (current_function_decl),
 473                                        true, NULL_TREE,
 474                                        true, GSI_SAME_STMT);
 475   tree_uid = build_int_cst
 476               (gcov_type_node,
 477                cgraph_node::get (current_function_decl)->profile_id);
 478   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 479                              tree_uid, cur_func);
 480   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 481 }
 482
 483 /* Output instructions as GIMPLE tree at the beginning for each function.
 484    TAG is the tag of the section for counters, BASE is offset of the
 485    counter position and GSI is the iterator we place the counter.  */
 486
 487 void
 488 gimple_gen_time_profiler (unsigned tag, unsigned base)
 489 {
 490   tree type = get_gcov_type ();
 491   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 492   basic_block cond_bb = split_edge (single_succ_edge (entry));
 493   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 494
 495   /* We need to do an extra split in order to not create an input
 496      for a possible PHI node.  */
 497   split_edge (single_succ_edge (update_bb));
 498
 499   edge true_edge = single_succ_edge (cond_bb);
 500   true_edge->flags = EDGE_TRUE_VALUE;
 501   true_edge->probability = profile_probability::unlikely ();
 502   edge e
 503     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 504   e->probability = true_edge->probability.invert ();
 505
 506   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 507   tree original_ref = tree_coverage_counter_ref (tag, base);
 508   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 509                                        true, GSI_SAME_STMT);
 510   tree one = build_int_cst (type, 1);
 511
 512   /* Emit: if (counters[0] != 0).  */
 513   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 514                                    NULL, NULL);
 515   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 516
 517   gsi = gsi_start_bb (update_bb);
 518
 519   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 520   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 521     {
 522       tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
 523                                      "time_profiler_counter_ptr");
 524       tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
 525                           tree_time_profiler_counter);
 526       gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
 527       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 528       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 529                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8:
 530                                       BUILT_IN_ATOMIC_ADD_FETCH_4);
 531       gcall *stmt = gimple_build_call (f, 3, ptr, one,
 532                                        build_int_cst (integer_type_node,
 533                                                       MEMMODEL_RELAXED));
 534       tree result_type = TREE_TYPE (TREE_TYPE (f));
 535       tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
 536       gimple_set_lhs (stmt, tmp);
 537       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
 538       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 539       assign = gimple_build_assign (tmp, NOP_EXPR,
 540                                     gimple_call_lhs (stmt));
 541       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 542       assign = gimple_build_assign (original_ref, tmp);
 543       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 544     }
 545   else
 546     {
 547       tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
 548       gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
 549       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 550
 551       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 552       assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
 553                                     one);
 554       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 555       assign = gimple_build_assign (original_ref, tmp);
 556       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 557       assign = gimple_build_assign (tree_time_profiler_counter, tmp);
 558       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 559     }
 560 }
 561
 562 /* Output instructions as GIMPLE trees to increment the average histogram
 563    counter.  VALUE is the expression whose value is profiled.  TAG is the
 564    tag of the section for counters, BASE is offset of the counter position.  */
 565
 566 void
 567 gimple_gen_average_profiler (histogram_value value, unsigned tag, unsigned base)
 568 {
 569   gimple *stmt = value->hvalue.stmt;
 570   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 571   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 572   gcall *call;
 573   tree val;
 574
 575   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 576                                       true, NULL_TREE,
 577                                       true, GSI_SAME_STMT);
 578   val = prepare_instrumented_value (&gsi, value);
 579   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 580   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 581 }
 582
 583 /* Output instructions as GIMPLE trees to increment the ior histogram
 584    counter.  VALUE is the expression whose value is profiled.  TAG is the
 585    tag of the section for counters, BASE is offset of the counter position.  */
 586
 587 void
 588 gimple_gen_ior_profiler (histogram_value value, unsigned tag, unsigned base)
 589 {
 590   gimple *stmt = value->hvalue.stmt;
 591   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 592   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 593   gcall *call;
 594   tree val;
 595
 596   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 597                                       true, NULL_TREE, true, GSI_SAME_STMT);
 598   val = prepare_instrumented_value (&gsi, value);
 599   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 600   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 601 }
 602
 603 static vec<regex_t> profile_filter_files;
 604 static vec<regex_t> profile_exclude_files;
 605
 606 /* Parse list of provided REGEX (separated with semi-collon) and
 607    create expressions (of type regex_t) and save them into V vector.
 608    If there is a regular expression parsing error, error message is
 609    printed for FLAG_NAME.  */
 610
 611 static void
 612 parse_profile_filter (const char *regex, vec<regex_t> *v,
 613                       const char *flag_name)
 614 {
 615   v->create (4);
 616   if (regex != NULL)
 617     {
 618       char *str = xstrdup (regex);
 619       for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
 620         {
 621           regex_t r;
 622           if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
 623             {
 624               error ("invalid regular expression %qs in %qs",
 625                      p, flag_name);
 626               return;
 627             }
 628
 629           v->safe_push (r);
 630         }
 631     }
 632 }
 633
 634 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files
 635    options.  */
 636
 637 static void
 638 parse_profile_file_filtering ()
 639 {
 640   parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
 641                         "-fprofile-filter-files");
 642   parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
 643                         "-fprofile-exclude-files");
 644 }
 645
 646 /* Parse vectors of regular expressions.  */
 647
 648 static void
 649 release_profile_file_filtering ()
 650 {
 651   profile_filter_files.release ();
 652   profile_exclude_files.release ();
 653 }
 654
 655 /* Return true when FILENAME should be instrumented based on
 656    -fprofile-filter-files and -fprofile-exclude-files options.  */
 657
 658 static bool
 659 include_source_file_for_profile (const char *filename)
 660 {
 661   /* First check whether file is included in flag_profile_exclude_files.  */
 662   for (unsigned i = 0; i < profile_exclude_files.length (); i++)
 663     if (regexec (&profile_exclude_files[i],
 664                  filename, 0, NULL, 0) == REG_NOERROR)
 665       return false;
 666
 667   /* For non-empty flag_profile_filter_files include only files matching a
 668      regex in the flag.  */
 669   if (profile_filter_files.is_empty ())
 670     return true;
 671
 672   for (unsigned i = 0; i < profile_filter_files.length (); i++)
 673     if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
 674       return true;
 675
 676   return false;
 677 }
 678
 679 #ifndef HAVE_sync_compare_and_swapsi
 680 #define HAVE_sync_compare_and_swapsi 0
 681 #endif
 682 #ifndef HAVE_atomic_compare_and_swapsi
 683 #define HAVE_atomic_compare_and_swapsi 0
 684 #endif
 685
 686 #ifndef HAVE_sync_compare_and_swapdi
 687 #define HAVE_sync_compare_and_swapdi 0
 688 #endif
 689 #ifndef HAVE_atomic_compare_and_swapdi
 690 #define HAVE_atomic_compare_and_swapdi 0
 691 #endif
 692
 693 /* Profile all functions in the callgraph.  */
 694
 695 static unsigned int
 696 tree_profiling (void)
 697 {
 698   struct cgraph_node *node;
 699
 700   /* Verify whether we can utilize atomic update operations.  */
 701   bool can_support_atomic = false;
 702   unsigned HOST_WIDE_INT gcov_type_size
 703     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 704   if (gcov_type_size == 4)
 705     can_support_atomic
 706       = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 707   else if (gcov_type_size == 8)
 708     can_support_atomic
 709       = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 710
 711   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 712       && !can_support_atomic)
 713     {
 714       warning (0, "target does not support atomic profile update, "
 715                "single mode is selected");
 716       flag_profile_update = PROFILE_UPDATE_SINGLE;
 717     }
 718   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 719     flag_profile_update = can_support_atomic
 720       ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 721
 722   /* This is a small-ipa pass that gets called only once, from
 723      cgraphunit.c:ipa_passes().  */
 724   gcc_assert (symtab->state == IPA_SSA);
 725
 726   init_node_map (true);
 727   parse_profile_file_filtering ();
 728
 729   FOR_EACH_DEFINED_FUNCTION (node)
 730     {
 731       bool thunk = false;
 732       if (!gimple_has_body_p (node->decl) && !node->thunk.thunk_p)
 733         continue;
 734
 735       /* Don't profile functions produced for builtin stuff.  */
 736       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 737         continue;
 738
 739       if (lookup_attribute ("no_profile_instrument_function",
 740                             DECL_ATTRIBUTES (node->decl)))
 741         continue;
 742       /* Do not instrument extern inline functions when testing coverage.
 743          While this is not perfectly consistent (early inlined extern inlines
 744          will get acocunted), testsuite expects that.  */
 745       if (DECL_EXTERNAL (node->decl)
 746           && flag_test_coverage)
 747         continue;
 748
 749       const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
 750       if (!include_source_file_for_profile (file))
 751         continue;
 752
 753       if (node->thunk.thunk_p)
 754         {
 755           /* We cannot expand variadic thunks to Gimple.  */
 756           if (stdarg_p (TREE_TYPE (node->decl)))
 757             continue;
 758           thunk = true;
 759           /* When generate profile, expand thunk to gimple so it can be
 760              instrumented same way as other functions.  */
 761           if (profile_arc_flag)
 762             node->expand_thunk (false, true);
 763           /* Read cgraph profile but keep function as thunk at profile-use
 764              time.  */
 765           else
 766             {
 767               read_thunk_profile (node);
 768               continue;
 769             }
 770         }
 771
 772       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 773
 774       if (dump_file)
 775         dump_function_header (dump_file, cfun->decl, dump_flags);
 776
 777       /* Local pure-const may imply need to fixup the cfg.  */
 778       if (gimple_has_body_p (node->decl)
 779           && (execute_fixup_cfg () & TODO_cleanup_cfg))
 780         cleanup_tree_cfg ();
 781
 782       branch_prob (thunk);
 783
 784       if (! flag_branch_probabilities
 785           && flag_profile_values)
 786         gimple_gen_ic_func_profiler ();
 787
 788       if (flag_branch_probabilities
 789           && !thunk
 790           && flag_profile_values
 791           && flag_value_profile_transformations)
 792         gimple_value_profile_transformations ();
 793
 794       /* The above could hose dominator info.  Currently there is
 795          none coming in, this is a safety valve.  It should be
 796          easy to adjust it, if and when there is some.  */
 797       free_dominance_info (CDI_DOMINATORS);
 798       free_dominance_info (CDI_POST_DOMINATORS);
 799       pop_cfun ();
 800     }
 801
 802   release_profile_file_filtering ();
 803
 804   /* Drop pure/const flags from instrumented functions.  */
 805   if (profile_arc_flag || flag_test_coverage)
 806     FOR_EACH_DEFINED_FUNCTION (node)
 807       {
 808         if (!gimple_has_body_p (node->decl)
 809             || !(!node->clone_of
 810             || node->decl != node->clone_of->decl))
 811           continue;
 812
 813         /* Don't profile functions produced for builtin stuff.  */
 814         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 815           continue;
 816
 817         node->set_const_flag (false, false);
 818         node->set_pure_flag (false, false);
 819       }
 820
 821   /* Update call statements and rebuild the cgraph.  */
 822   FOR_EACH_DEFINED_FUNCTION (node)
 823     {
 824       basic_block bb;
 825
 826       if (!gimple_has_body_p (node->decl)
 827           || !(!node->clone_of
 828           || node->decl != node->clone_of->decl))
 829         continue;
 830
 831       /* Don't profile functions produced for builtin stuff.  */
 832       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 833         continue;
 834
 835       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 836
 837       FOR_EACH_BB_FN (bb, cfun)
 838         {
 839           gimple_stmt_iterator gsi;
 840           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 841             {
 842               gimple *stmt = gsi_stmt (gsi);
 843               if (is_gimple_call (stmt))
 844                 update_stmt (stmt);
 845             }
 846         }
 847
 848       /* re-merge split blocks.  */
 849       cleanup_tree_cfg ();
 850       update_ssa (TODO_update_ssa);
 851
 852       cgraph_edge::rebuild_edges ();
 853
 854       pop_cfun ();
 855     }
 856
 857   handle_missing_profiles ();
 858
 859   del_node_map ();
 860   return 0;
 861 }
 862
 863 namespace {
 864
 865 const pass_data pass_data_ipa_tree_profile =
 866 {
 867   SIMPLE_IPA_PASS, /* type */
 868   "profile", /* name */
 869   OPTGROUP_NONE, /* optinfo_flags */
 870   TV_IPA_PROFILE, /* tv_id */
 871   0, /* properties_required */
 872   0, /* properties_provided */
 873   0, /* properties_destroyed */
 874   0, /* todo_flags_start */
 875   TODO_dump_symtab, /* todo_flags_finish */
 876 };
 877
 878 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 879 {
 880 public:
 881   pass_ipa_tree_profile (gcc::context *ctxt)
 882     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 883   {}
 884
 885   /* opt_pass methods: */
 886   virtual bool gate (function *);
 887   virtual unsigned int execute (function *) { return tree_profiling (); }
 888
 889 }; // class pass_ipa_tree_profile
 890
 891 bool
 892 pass_ipa_tree_profile::gate (function *)
 893 {
 894   /* When profile instrumentation, use or test coverage shall be performed.
 895      But for AutoFDO, this there is no instrumentation, thus this pass is
 896      diabled.  */
 897   return (!in_lto_p && !flag_auto_profile
 898           && (flag_branch_probabilities || flag_test_coverage
 899               || profile_arc_flag));
 900 }
 901
 902 } // anon namespace
 903
 904 simple_ipa_opt_pass *
 905 make_pass_ipa_tree_profile (gcc::context *ctxt)
 906 {
 907   return new pass_ipa_tree_profile (ctxt);
 908 }
 909
 910 #include "gt-tree-profile.h"