gcc/tree-profile.c

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2020 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.c for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "stringpool.h"
  53 #include "attribs.h"
  54 #include "tree-pretty-print.h"
  55 #include "langhooks.h"
  56 #include "stor-layout.h"
  57 #include "xregex.h"
  58
  59 static GTY(()) tree gcov_type_node;
  60 static GTY(()) tree tree_interval_profiler_fn;
  61 static GTY(()) tree tree_pow2_profiler_fn;
  62 static GTY(()) tree tree_topn_values_profiler_fn;
  63 static GTY(()) tree tree_indirect_call_profiler_fn;
  64 static GTY(()) tree tree_average_profiler_fn;
  65 static GTY(()) tree tree_ior_profiler_fn;
  66 static GTY(()) tree tree_time_profiler_counter;
  67
  68
  69 static GTY(()) tree ic_tuple_var;
  70 static GTY(()) tree ic_tuple_counters_field;
  71 static GTY(()) tree ic_tuple_callee_field;
  72
  73 /* Do initialization work for the edge profiler.  */
  74
  75 /* Add code:
  76    __thread gcov*       __gcov_indirect_call.counters; // pointer to actual counter
  77    __thread void*       __gcov_indirect_call.callee; // actual callee address
  78    __thread int __gcov_function_counter; // time profiler function counter
  79 */
  80 static void
  81 init_ic_make_global_vars (void)
  82 {
  83   tree gcov_type_ptr;
  84
  85   gcov_type_ptr = build_pointer_type (get_gcov_type ());
  86
  87   tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
  88
  89   /* callee */
  90   ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
  91                                       ptr_type_node);
  92
  93   /* counters */
  94   ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
  95                                         NULL_TREE, gcov_type_ptr);
  96   DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
  97
  98   finish_builtin_struct (tuple_type, "indirect_call_tuple",
  99                          ic_tuple_counters_field, NULL_TREE);
 100
 101   ic_tuple_var
 102     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 103                   get_identifier ("__gcov_indirect_call"), tuple_type);
 104   TREE_PUBLIC (ic_tuple_var) = 1;
 105   DECL_ARTIFICIAL (ic_tuple_var) = 1;
 106   DECL_INITIAL (ic_tuple_var) = NULL;
 107   DECL_EXTERNAL (ic_tuple_var) = 1;
 108   if (targetm.have_tls)
 109     set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
 110 }
 111
 112 /* Create the type and function decls for the interface with gcov.  */
 113
 114 void
 115 gimple_init_gcov_profiler (void)
 116 {
 117   tree interval_profiler_fn_type;
 118   tree pow2_profiler_fn_type;
 119   tree topn_values_profiler_fn_type;
 120   tree gcov_type_ptr;
 121   tree ic_profiler_fn_type;
 122   tree average_profiler_fn_type;
 123   const char *fn_name;
 124
 125   if (!gcov_type_node)
 126     {
 127       const char *fn_suffix
 128         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 129
 130       gcov_type_node = get_gcov_type ();
 131       gcov_type_ptr = build_pointer_type (gcov_type_node);
 132
 133       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 134       interval_profiler_fn_type
 135               = build_function_type_list (void_type_node,
 136                                           gcov_type_ptr, gcov_type_node,
 137                                           integer_type_node,
 138                                           unsigned_type_node, NULL_TREE);
 139       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 140       tree_interval_profiler_fn = build_fn_decl (fn_name,
 141                                                  interval_profiler_fn_type);
 142       free (CONST_CAST (char *, fn_name));
 143       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 144       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 145         = tree_cons (get_identifier ("leaf"), NULL,
 146                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 147
 148       /* void (*) (gcov_type *, gcov_type)  */
 149       pow2_profiler_fn_type
 150               = build_function_type_list (void_type_node,
 151                                           gcov_type_ptr, gcov_type_node,
 152                                           NULL_TREE);
 153       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 154       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 155       free (CONST_CAST (char *, fn_name));
 156       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 157       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 158         = tree_cons (get_identifier ("leaf"), NULL,
 159                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 160
 161       /* void (*) (gcov_type *, gcov_type)  */
 162       topn_values_profiler_fn_type
 163               = build_function_type_list (void_type_node,
 164                                           gcov_type_ptr, gcov_type_node,
 165                                           NULL_TREE);
 166       fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
 167       tree_topn_values_profiler_fn
 168         = build_fn_decl (fn_name, topn_values_profiler_fn_type);
 169       free (CONST_CAST (char *, fn_name));
 170
 171       TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
 172       DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
 173         = tree_cons (get_identifier ("leaf"), NULL,
 174                      DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
 175
 176       init_ic_make_global_vars ();
 177
 178       /* void (*) (gcov_type, void *)  */
 179       ic_profiler_fn_type
 180                = build_function_type_list (void_type_node,
 181                                           gcov_type_node,
 182                                           ptr_type_node,
 183                                           NULL_TREE);
 184       fn_name = concat ("__gcov_indirect_call_profiler_v4", fn_suffix, NULL);
 185       tree_indirect_call_profiler_fn
 186         = build_fn_decl (fn_name, ic_profiler_fn_type);
 187       free (CONST_CAST (char *, fn_name));
 188
 189       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 190       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 191         = tree_cons (get_identifier ("leaf"), NULL,
 192                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 193
 194       tree_time_profiler_counter
 195         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 196                       get_identifier ("__gcov_time_profiler_counter"),
 197                       get_gcov_type ());
 198       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 199       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 200       TREE_STATIC (tree_time_profiler_counter) = 1;
 201       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 202       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 203
 204       /* void (*) (gcov_type *, gcov_type)  */
 205       average_profiler_fn_type
 206               = build_function_type_list (void_type_node,
 207                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 208       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 209       tree_average_profiler_fn = build_fn_decl (fn_name,
 210                                                 average_profiler_fn_type);
 211       free (CONST_CAST (char *, fn_name));
 212       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 213       DECL_ATTRIBUTES (tree_average_profiler_fn)
 214         = tree_cons (get_identifier ("leaf"), NULL,
 215                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 216       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 217       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 218       free (CONST_CAST (char *, fn_name));
 219       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 220       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 221         = tree_cons (get_identifier ("leaf"), NULL,
 222                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 223
 224       /* LTO streamer needs assembler names.  Because we create these decls
 225          late, we need to initialize them by hand.  */
 226       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 227       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 228       DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
 229       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 230       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 231       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 232     }
 233 }
 234
 235 /* Output instructions as GIMPLE trees to increment the edge
 236    execution count, and insert them on E.  We rely on
 237    gsi_insert_on_edge to preserve the order.  */
 238
 239 void
 240 gimple_gen_edge_profiler (int edgeno, edge e)
 241 {
 242   tree one;
 243
 244   one = build_int_cst (gcov_type_node, 1);
 245
 246   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 247     {
 248       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 249       tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
 250       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 251                                       ? BUILT_IN_ATOMIC_FETCH_ADD_8:
 252                                       BUILT_IN_ATOMIC_FETCH_ADD_4);
 253       gcall *stmt = gimple_build_call (f, 3, addr, one,
 254                                        build_int_cst (integer_type_node,
 255                                                       MEMMODEL_RELAXED));
 256       gsi_insert_on_edge (e, stmt);
 257     }
 258   else
 259     {
 260       tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 261       tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 262                                                    NULL, "PROF_edge_counter");
 263       gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
 264       gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 265                                               NULL, "PROF_edge_counter");
 266       gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
 267                                             gimple_assign_lhs (stmt1), one);
 268       gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
 269                                             gimple_assign_lhs (stmt2));
 270       gsi_insert_on_edge (e, stmt1);
 271       gsi_insert_on_edge (e, stmt2);
 272       gsi_insert_on_edge (e, stmt3);
 273     }
 274 }
 275
 276 /* Emits code to get VALUE to instrument at GSI, and returns the
 277    variable containing the value.  */
 278
 279 static tree
 280 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 281 {
 282   tree val = value->hvalue.value;
 283   if (POINTER_TYPE_P (TREE_TYPE (val)))
 284     val = fold_convert (build_nonstandard_integer_type
 285                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 286   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 287                                    true, NULL_TREE, true, GSI_SAME_STMT);
 288 }
 289
 290 /* Output instructions as GIMPLE trees to increment the interval histogram
 291    counter.  VALUE is the expression whose value is profiled.  TAG is the
 292    tag of the section for counters, BASE is offset of the counter position.  */
 293
 294 void
 295 gimple_gen_interval_profiler (histogram_value value, unsigned tag)
 296 {
 297   gimple *stmt = value->hvalue.stmt;
 298   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 299   tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr;
 300   gcall *call;
 301   tree val;
 302   tree start = build_int_cst_type (integer_type_node,
 303                                    value->hdata.intvl.int_start);
 304   tree steps = build_int_cst_type (unsigned_type_node,
 305                                    value->hdata.intvl.steps);
 306
 307   ref_ptr = force_gimple_operand_gsi (&gsi,
 308                                       build_addr (ref),
 309                                       true, NULL_TREE, true, GSI_SAME_STMT);
 310   val = prepare_instrumented_value (&gsi, value);
 311   call = gimple_build_call (tree_interval_profiler_fn, 4,
 312                             ref_ptr, val, start, steps);
 313   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 314 }
 315
 316 /* Output instructions as GIMPLE trees to increment the power of two histogram
 317    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 318    of the section for counters.  */
 319
 320 void
 321 gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
 322 {
 323   gimple *stmt = value->hvalue.stmt;
 324   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 325   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 326   gcall *call;
 327   tree val;
 328
 329   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 330                                       true, NULL_TREE, true, GSI_SAME_STMT);
 331   val = prepare_instrumented_value (&gsi, value);
 332   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 333   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 334 }
 335
 336 /* Output instructions as GIMPLE trees for code to find the most N common
 337    values.  VALUE is the expression whose value is profiled.  TAG is the tag
 338    of the section for counters.  */
 339
 340 void
 341 gimple_gen_topn_values_profiler (histogram_value value, unsigned tag)
 342 {
 343   gimple *stmt = value->hvalue.stmt;
 344   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 345   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 346   gcall *call;
 347   tree val;
 348
 349   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 350                                       true, NULL_TREE, true, GSI_SAME_STMT);
 351   val = prepare_instrumented_value (&gsi, value);
 352   call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
 353   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 354 }
 355
 356
 357 /* Output instructions as GIMPLE trees for code to find the most
 358    common called function in indirect call.
 359    VALUE is the call expression whose indirect callee is profiled.
 360    TAG is the tag of the section for counters.  */
 361
 362 void
 363 gimple_gen_ic_profiler (histogram_value value, unsigned tag)
 364 {
 365   tree tmp1;
 366   gassign *stmt1, *stmt2, *stmt3;
 367   gimple *stmt = value->hvalue.stmt;
 368   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 369   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 370
 371   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 372                                       true, NULL_TREE, true, GSI_SAME_STMT);
 373
 374   /* Insert code:
 375
 376     stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
 377     stmt2: tmp1 = (void *) (indirect call argument value)
 378     stmt3: __gcov_indirect_call.callee = tmp1;
 379
 380     Example:
 381       f_1 = foo;
 382       __gcov_indirect_call.counters = &__gcov4.main[0];
 383       PROF_9 = f_1;
 384       __gcov_indirect_call.callee = PROF_9;
 385       _4 = f_1 ();
 386    */
 387
 388   tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
 389
 390   tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
 391                              ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
 392
 393   stmt1 = gimple_build_assign (counter_ref, ref_ptr);
 394   tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF");
 395   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 396   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 397                              ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 398   stmt3 = gimple_build_assign (callee_ref, tmp1);
 399
 400   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 401   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 402   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 403 }
 404
 405
 406 /* Output instructions as GIMPLE trees for code to find the most
 407    common called function in indirect call. Insert instructions at the
 408    beginning of every possible called function.
 409   */
 410
 411 void
 412 gimple_gen_ic_func_profiler (void)
 413 {
 414   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 415   gcall *stmt1;
 416   tree tree_uid, cur_func, void0;
 417
 418   if (c_node->only_called_directly_p ())
 419     return;
 420
 421   gimple_init_gcov_profiler ();
 422
 423   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 424   basic_block cond_bb = split_edge (single_succ_edge (entry));
 425   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 426
 427   /* We need to do an extra split in order to not create an input
 428      for a possible PHI node.  */
 429   split_edge (single_succ_edge (update_bb));
 430
 431   edge true_edge = single_succ_edge (cond_bb);
 432   true_edge->flags = EDGE_TRUE_VALUE;
 433
 434   profile_probability probability;
 435   if (DECL_VIRTUAL_P (current_function_decl))
 436     probability = profile_probability::very_likely ();
 437   else
 438     probability = profile_probability::unlikely ();
 439
 440   true_edge->probability = probability;
 441   edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
 442                       EDGE_FALSE_VALUE);
 443   e->probability = true_edge->probability.invert ();
 444
 445   /* Insert code:
 446
 447      if (__gcov_indirect_call.callee != NULL)
 448        __gcov_indirect_call_profiler_v3 (profile_id, &current_function_decl);
 449
 450      The function __gcov_indirect_call_profiler_v3 is responsible for
 451      resetting __gcov_indirect_call.callee to NULL.  */
 452
 453   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 454   void0 = build_int_cst (ptr_type_node, 0);
 455
 456   tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
 457                             ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
 458
 459   tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
 460                                        true, GSI_SAME_STMT);
 461
 462   gcond *cond = gimple_build_cond (NE_EXPR, ref,
 463                                    void0, NULL, NULL);
 464   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 465
 466   gsi = gsi_after_labels (update_bb);
 467
 468   cur_func = force_gimple_operand_gsi (&gsi,
 469                                        build_addr (current_function_decl),
 470                                        true, NULL_TREE,
 471                                        true, GSI_SAME_STMT);
 472   tree_uid = build_int_cst
 473               (gcov_type_node,
 474                cgraph_node::get (current_function_decl)->profile_id);
 475   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 476                              tree_uid, cur_func);
 477   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 478 }
 479
 480 /* Output instructions as GIMPLE tree at the beginning for each function.
 481    TAG is the tag of the section for counters, BASE is offset of the
 482    counter position and GSI is the iterator we place the counter.  */
 483
 484 void
 485 gimple_gen_time_profiler (unsigned tag)
 486 {
 487   tree type = get_gcov_type ();
 488   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 489   basic_block cond_bb = split_edge (single_succ_edge (entry));
 490   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 491
 492   /* We need to do an extra split in order to not create an input
 493      for a possible PHI node.  */
 494   split_edge (single_succ_edge (update_bb));
 495
 496   edge true_edge = single_succ_edge (cond_bb);
 497   true_edge->flags = EDGE_TRUE_VALUE;
 498   true_edge->probability = profile_probability::unlikely ();
 499   edge e
 500     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 501   e->probability = true_edge->probability.invert ();
 502
 503   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 504   tree original_ref = tree_coverage_counter_ref (tag, 0);
 505   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 506                                        true, GSI_SAME_STMT);
 507   tree one = build_int_cst (type, 1);
 508
 509   /* Emit: if (counters[0] != 0).  */
 510   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 511                                    NULL, NULL);
 512   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 513
 514   gsi = gsi_start_bb (update_bb);
 515
 516   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 517   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 518     {
 519       tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
 520                                      "time_profiler_counter_ptr");
 521       tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
 522                           tree_time_profiler_counter);
 523       gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
 524       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 525       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 526                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8:
 527                                       BUILT_IN_ATOMIC_ADD_FETCH_4);
 528       gcall *stmt = gimple_build_call (f, 3, ptr, one,
 529                                        build_int_cst (integer_type_node,
 530                                                       MEMMODEL_RELAXED));
 531       tree result_type = TREE_TYPE (TREE_TYPE (f));
 532       tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
 533       gimple_set_lhs (stmt, tmp);
 534       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
 535       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 536       assign = gimple_build_assign (tmp, NOP_EXPR,
 537                                     gimple_call_lhs (stmt));
 538       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 539       assign = gimple_build_assign (original_ref, tmp);
 540       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 541     }
 542   else
 543     {
 544       tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
 545       gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
 546       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 547
 548       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 549       assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
 550                                     one);
 551       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 552       assign = gimple_build_assign (original_ref, tmp);
 553       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 554       assign = gimple_build_assign (tree_time_profiler_counter, tmp);
 555       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 556     }
 557 }
 558
 559 /* Output instructions as GIMPLE trees to increment the average histogram
 560    counter.  VALUE is the expression whose value is profiled.  TAG is the
 561    tag of the section for counters, BASE is offset of the counter position.  */
 562
 563 void
 564 gimple_gen_average_profiler (histogram_value value, unsigned tag)
 565 {
 566   gimple *stmt = value->hvalue.stmt;
 567   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 568   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 569   gcall *call;
 570   tree val;
 571
 572   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 573                                       true, NULL_TREE,
 574                                       true, GSI_SAME_STMT);
 575   val = prepare_instrumented_value (&gsi, value);
 576   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 577   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 578 }
 579
 580 /* Output instructions as GIMPLE trees to increment the ior histogram
 581    counter.  VALUE is the expression whose value is profiled.  TAG is the
 582    tag of the section for counters, BASE is offset of the counter position.  */
 583
 584 void
 585 gimple_gen_ior_profiler (histogram_value value, unsigned tag)
 586 {
 587   gimple *stmt = value->hvalue.stmt;
 588   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 589   tree ref_ptr = tree_coverage_counter_addr (tag, 0);
 590   gcall *call;
 591   tree val;
 592
 593   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 594                                       true, NULL_TREE, true, GSI_SAME_STMT);
 595   val = prepare_instrumented_value (&gsi, value);
 596   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 597   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 598 }
 599
 600 static vec<regex_t> profile_filter_files;
 601 static vec<regex_t> profile_exclude_files;
 602
 603 /* Parse list of provided REGEX (separated with semi-collon) and
 604    create expressions (of type regex_t) and save them into V vector.
 605    If there is a regular expression parsing error, error message is
 606    printed for FLAG_NAME.  */
 607
 608 static void
 609 parse_profile_filter (const char *regex, vec<regex_t> *v,
 610                       const char *flag_name)
 611 {
 612   v->create (4);
 613   if (regex != NULL)
 614     {
 615       char *str = xstrdup (regex);
 616       for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
 617         {
 618           regex_t r;
 619           if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
 620             {
 621               error ("invalid regular expression %qs in %qs",
 622                      p, flag_name);
 623               return;
 624             }
 625
 626           v->safe_push (r);
 627         }
 628     }
 629 }
 630
 631 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files
 632    options.  */
 633
 634 static void
 635 parse_profile_file_filtering ()
 636 {
 637   parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
 638                         "-fprofile-filter-files");
 639   parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
 640                         "-fprofile-exclude-files");
 641 }
 642
 643 /* Parse vectors of regular expressions.  */
 644
 645 static void
 646 release_profile_file_filtering ()
 647 {
 648   profile_filter_files.release ();
 649   profile_exclude_files.release ();
 650 }
 651
 652 /* Return true when FILENAME should be instrumented based on
 653    -fprofile-filter-files and -fprofile-exclude-files options.  */
 654
 655 static bool
 656 include_source_file_for_profile (const char *filename)
 657 {
 658   /* First check whether file is included in flag_profile_exclude_files.  */
 659   for (unsigned i = 0; i < profile_exclude_files.length (); i++)
 660     if (regexec (&profile_exclude_files[i],
 661                  filename, 0, NULL, 0) == REG_NOERROR)
 662       return false;
 663
 664   /* For non-empty flag_profile_filter_files include only files matching a
 665      regex in the flag.  */
 666   if (profile_filter_files.is_empty ())
 667     return true;
 668
 669   for (unsigned i = 0; i < profile_filter_files.length (); i++)
 670     if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
 671       return true;
 672
 673   return false;
 674 }
 675
 676 #ifndef HAVE_sync_compare_and_swapsi
 677 #define HAVE_sync_compare_and_swapsi 0
 678 #endif
 679 #ifndef HAVE_atomic_compare_and_swapsi
 680 #define HAVE_atomic_compare_and_swapsi 0
 681 #endif
 682
 683 #ifndef HAVE_sync_compare_and_swapdi
 684 #define HAVE_sync_compare_and_swapdi 0
 685 #endif
 686 #ifndef HAVE_atomic_compare_and_swapdi
 687 #define HAVE_atomic_compare_and_swapdi 0
 688 #endif
 689
 690 /* Profile all functions in the callgraph.  */
 691
 692 static unsigned int
 693 tree_profiling (void)
 694 {
 695   struct cgraph_node *node;
 696
 697   /* Verify whether we can utilize atomic update operations.  */
 698   bool can_support_atomic = false;
 699   unsigned HOST_WIDE_INT gcov_type_size
 700     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 701   if (gcov_type_size == 4)
 702     can_support_atomic
 703       = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 704   else if (gcov_type_size == 8)
 705     can_support_atomic
 706       = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 707
 708   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 709       && !can_support_atomic)
 710     {
 711       warning (0, "target does not support atomic profile update, "
 712                "single mode is selected");
 713       flag_profile_update = PROFILE_UPDATE_SINGLE;
 714     }
 715   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 716     flag_profile_update = can_support_atomic
 717       ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 718
 719   /* This is a small-ipa pass that gets called only once, from
 720      cgraphunit.c:ipa_passes().  */
 721   gcc_assert (symtab->state == IPA_SSA);
 722
 723   init_node_map (true);
 724   parse_profile_file_filtering ();
 725
 726   FOR_EACH_DEFINED_FUNCTION (node)
 727     {
 728       bool thunk = false;
 729       if (!gimple_has_body_p (node->decl) && !node->thunk.thunk_p)
 730         continue;
 731
 732       /* Don't profile functions produced for builtin stuff.  */
 733       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 734         continue;
 735
 736       if (lookup_attribute ("no_profile_instrument_function",
 737                             DECL_ATTRIBUTES (node->decl)))
 738         continue;
 739       /* Do not instrument extern inline functions when testing coverage.
 740          While this is not perfectly consistent (early inlined extern inlines
 741          will get acocunted), testsuite expects that.  */
 742       if (DECL_EXTERNAL (node->decl)
 743           && flag_test_coverage)
 744         continue;
 745
 746       const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
 747       if (!include_source_file_for_profile (file))
 748         continue;
 749
 750       if (node->thunk.thunk_p)
 751         {
 752           /* We cannot expand variadic thunks to Gimple.  */
 753           if (stdarg_p (TREE_TYPE (node->decl)))
 754             continue;
 755           thunk = true;
 756           /* When generate profile, expand thunk to gimple so it can be
 757              instrumented same way as other functions.  */
 758           if (profile_arc_flag)
 759             node->expand_thunk (false, true);
 760           /* Read cgraph profile but keep function as thunk at profile-use
 761              time.  */
 762           else
 763             {
 764               read_thunk_profile (node);
 765               continue;
 766             }
 767         }
 768
 769       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 770
 771       if (dump_file)
 772         dump_function_header (dump_file, cfun->decl, dump_flags);
 773
 774       /* Local pure-const may imply need to fixup the cfg.  */
 775       if (gimple_has_body_p (node->decl)
 776           && (execute_fixup_cfg () & TODO_cleanup_cfg))
 777         cleanup_tree_cfg ();
 778
 779       branch_prob (thunk);
 780
 781       if (! flag_branch_probabilities
 782           && flag_profile_values)
 783         gimple_gen_ic_func_profiler ();
 784
 785       if (flag_branch_probabilities
 786           && !thunk
 787           && flag_profile_values
 788           && flag_value_profile_transformations
 789           && profile_status_for_fn (cfun) == PROFILE_READ)
 790         gimple_value_profile_transformations ();
 791
 792       /* The above could hose dominator info.  Currently there is
 793          none coming in, this is a safety valve.  It should be
 794          easy to adjust it, if and when there is some.  */
 795       free_dominance_info (CDI_DOMINATORS);
 796       free_dominance_info (CDI_POST_DOMINATORS);
 797       pop_cfun ();
 798     }
 799
 800   release_profile_file_filtering ();
 801
 802   /* Drop pure/const flags from instrumented functions.  */
 803   if (profile_arc_flag || flag_test_coverage)
 804     FOR_EACH_DEFINED_FUNCTION (node)
 805       {
 806         if (!gimple_has_body_p (node->decl)
 807             || !(!node->clone_of
 808             || node->decl != node->clone_of->decl))
 809           continue;
 810
 811         /* Don't profile functions produced for builtin stuff.  */
 812         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 813           continue;
 814
 815         node->set_const_flag (false, false);
 816         node->set_pure_flag (false, false);
 817       }
 818
 819   /* Update call statements and rebuild the cgraph.  */
 820   FOR_EACH_DEFINED_FUNCTION (node)
 821     {
 822       basic_block bb;
 823
 824       if (!gimple_has_body_p (node->decl)
 825           || !(!node->clone_of
 826           || node->decl != node->clone_of->decl))
 827         continue;
 828
 829       /* Don't profile functions produced for builtin stuff.  */
 830       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 831         continue;
 832
 833       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 834
 835       FOR_EACH_BB_FN (bb, cfun)
 836         {
 837           gimple_stmt_iterator gsi;
 838           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 839             {
 840               gimple *stmt = gsi_stmt (gsi);
 841               if (is_gimple_call (stmt))
 842                 update_stmt (stmt);
 843             }
 844         }
 845
 846       /* re-merge split blocks.  */
 847       cleanup_tree_cfg ();
 848       update_ssa (TODO_update_ssa);
 849
 850       cgraph_edge::rebuild_edges ();
 851
 852       pop_cfun ();
 853     }
 854
 855   handle_missing_profiles ();
 856
 857   del_node_map ();
 858   return 0;
 859 }
 860
 861 namespace {
 862
 863 const pass_data pass_data_ipa_tree_profile =
 864 {
 865   SIMPLE_IPA_PASS, /* type */
 866   "profile", /* name */
 867   OPTGROUP_NONE, /* optinfo_flags */
 868   TV_IPA_PROFILE, /* tv_id */
 869   0, /* properties_required */
 870   0, /* properties_provided */
 871   0, /* properties_destroyed */
 872   0, /* todo_flags_start */
 873   TODO_dump_symtab, /* todo_flags_finish */
 874 };
 875
 876 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 877 {
 878 public:
 879   pass_ipa_tree_profile (gcc::context *ctxt)
 880     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 881   {}
 882
 883   /* opt_pass methods: */
 884   virtual bool gate (function *);
 885   virtual unsigned int execute (function *) { return tree_profiling (); }
 886
 887 }; // class pass_ipa_tree_profile
 888
 889 bool
 890 pass_ipa_tree_profile::gate (function *)
 891 {
 892   /* When profile instrumentation, use or test coverage shall be performed.
 893      But for AutoFDO, this there is no instrumentation, thus this pass is
 894      disabled.  */
 895   return (!in_lto_p && !flag_auto_profile
 896           && (flag_branch_probabilities || flag_test_coverage
 897               || profile_arc_flag));
 898 }
 899
 900 } // anon namespace
 901
 902 simple_ipa_opt_pass *
 903 make_pass_ipa_tree_profile (gcc::context *ctxt)
 904 {
 905   return new pass_ipa_tree_profile (ctxt);
 906 }
 907
 908 #include "gt-tree-profile.h"