gcc/tree-profile.c

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2017 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.c for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "params.h"
  53 #include "stringpool.h"
  54 #include "attribs.h"
  55
  56 static GTY(()) tree gcov_type_node;
  57 static GTY(()) tree tree_interval_profiler_fn;
  58 static GTY(()) tree tree_pow2_profiler_fn;
  59 static GTY(()) tree tree_one_value_profiler_fn;
  60 static GTY(()) tree tree_indirect_call_profiler_fn;
  61 static GTY(()) tree tree_average_profiler_fn;
  62 static GTY(()) tree tree_ior_profiler_fn;
  63 static GTY(()) tree tree_time_profiler_counter;
  64
  65
  66 static GTY(()) tree ic_void_ptr_var;
  67 static GTY(()) tree ic_gcov_type_ptr_var;
  68 static GTY(()) tree ptr_void;
  69
  70 /* Do initialization work for the edge profiler.  */
  71
  72 /* Add code:
  73    __thread gcov*       __gcov_indirect_call_counters; // pointer to actual counter
  74    __thread void*       __gcov_indirect_call_callee; // actual callee address
  75    __thread int __gcov_function_counter; // time profiler function counter
  76 */
  77 static void
  78 init_ic_make_global_vars (void)
  79 {
  80   tree gcov_type_ptr;
  81
  82   ptr_void = build_pointer_type (void_type_node);
  83
  84   ic_void_ptr_var
  85     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
  86                   get_identifier (
  87                           (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
  88                            "__gcov_indirect_call_topn_callee" :
  89                            "__gcov_indirect_call_callee")),
  90                   ptr_void);
  91   TREE_PUBLIC (ic_void_ptr_var) = 1;
  92   DECL_EXTERNAL (ic_void_ptr_var) = 1;
  93   TREE_STATIC (ic_void_ptr_var) = 1;
  94   DECL_ARTIFICIAL (ic_void_ptr_var) = 1;
  95   DECL_INITIAL (ic_void_ptr_var) = NULL;
  96   if (targetm.have_tls)
  97     set_decl_tls_model (ic_void_ptr_var, decl_default_tls_model (ic_void_ptr_var));
  98
  99   gcov_type_ptr = build_pointer_type (get_gcov_type ());
 100
 101   ic_gcov_type_ptr_var
 102     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 103                   get_identifier (
 104                           (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
 105                            "__gcov_indirect_call_topn_counters" :
 106                            "__gcov_indirect_call_counters")),
 107                   gcov_type_ptr);
 108   TREE_PUBLIC (ic_gcov_type_ptr_var) = 1;
 109   DECL_EXTERNAL (ic_gcov_type_ptr_var) = 1;
 110   TREE_STATIC (ic_gcov_type_ptr_var) = 1;
 111   DECL_ARTIFICIAL (ic_gcov_type_ptr_var) = 1;
 112   DECL_INITIAL (ic_gcov_type_ptr_var) = NULL;
 113   if (targetm.have_tls)
 114     set_decl_tls_model (ic_gcov_type_ptr_var, decl_default_tls_model (ic_gcov_type_ptr_var));
 115 }
 116
 117 /* Create the type and function decls for the interface with gcov.  */
 118
 119 void
 120 gimple_init_gcov_profiler (void)
 121 {
 122   tree interval_profiler_fn_type;
 123   tree pow2_profiler_fn_type;
 124   tree one_value_profiler_fn_type;
 125   tree gcov_type_ptr;
 126   tree ic_profiler_fn_type;
 127   tree average_profiler_fn_type;
 128   const char *profiler_fn_name;
 129   const char *fn_name;
 130
 131   if (!gcov_type_node)
 132     {
 133       const char *fn_suffix
 134         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 135
 136       gcov_type_node = get_gcov_type ();
 137       gcov_type_ptr = build_pointer_type (gcov_type_node);
 138
 139       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 140       interval_profiler_fn_type
 141               = build_function_type_list (void_type_node,
 142                                           gcov_type_ptr, gcov_type_node,
 143                                           integer_type_node,
 144                                           unsigned_type_node, NULL_TREE);
 145       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 146       tree_interval_profiler_fn = build_fn_decl (fn_name,
 147                                                  interval_profiler_fn_type);
 148       free (CONST_CAST (char *, fn_name));
 149       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 150       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 151         = tree_cons (get_identifier ("leaf"), NULL,
 152                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 153
 154       /* void (*) (gcov_type *, gcov_type)  */
 155       pow2_profiler_fn_type
 156               = build_function_type_list (void_type_node,
 157                                           gcov_type_ptr, gcov_type_node,
 158                                           NULL_TREE);
 159       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 160       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 161       free (CONST_CAST (char *, fn_name));
 162       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 163       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 164         = tree_cons (get_identifier ("leaf"), NULL,
 165                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 166
 167       /* void (*) (gcov_type *, gcov_type)  */
 168       one_value_profiler_fn_type
 169               = build_function_type_list (void_type_node,
 170                                           gcov_type_ptr, gcov_type_node,
 171                                           NULL_TREE);
 172       fn_name = concat ("__gcov_one_value_profiler", fn_suffix, NULL);
 173       tree_one_value_profiler_fn = build_fn_decl (fn_name,
 174                                                   one_value_profiler_fn_type);
 175       free (CONST_CAST (char *, fn_name));
 176       TREE_NOTHROW (tree_one_value_profiler_fn) = 1;
 177       DECL_ATTRIBUTES (tree_one_value_profiler_fn)
 178         = tree_cons (get_identifier ("leaf"), NULL,
 179                      DECL_ATTRIBUTES (tree_one_value_profiler_fn));
 180
 181       init_ic_make_global_vars ();
 182
 183       /* void (*) (gcov_type, void *)  */
 184       ic_profiler_fn_type
 185                = build_function_type_list (void_type_node,
 186                                           gcov_type_node,
 187                                           ptr_void,
 188                                           NULL_TREE);
 189       profiler_fn_name = "__gcov_indirect_call_profiler_v2";
 190       if (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE))
 191         profiler_fn_name = "__gcov_indirect_call_topn_profiler";
 192
 193       tree_indirect_call_profiler_fn
 194               = build_fn_decl (profiler_fn_name, ic_profiler_fn_type);
 195
 196       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 197       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 198         = tree_cons (get_identifier ("leaf"), NULL,
 199                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 200
 201       tree_time_profiler_counter
 202         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 203                       get_identifier ("__gcov_time_profiler_counter"),
 204                       get_gcov_type ());
 205       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 206       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 207       TREE_STATIC (tree_time_profiler_counter) = 1;
 208       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 209       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 210
 211       /* void (*) (gcov_type *, gcov_type)  */
 212       average_profiler_fn_type
 213               = build_function_type_list (void_type_node,
 214                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 215       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 216       tree_average_profiler_fn = build_fn_decl (fn_name,
 217                                                 average_profiler_fn_type);
 218       free (CONST_CAST (char *, fn_name));
 219       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 220       DECL_ATTRIBUTES (tree_average_profiler_fn)
 221         = tree_cons (get_identifier ("leaf"), NULL,
 222                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 223       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 224       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 225       free (CONST_CAST (char *, fn_name));
 226       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 227       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 228         = tree_cons (get_identifier ("leaf"), NULL,
 229                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 230
 231       /* LTO streamer needs assembler names.  Because we create these decls
 232          late, we need to initialize them by hand.  */
 233       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 234       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 235       DECL_ASSEMBLER_NAME (tree_one_value_profiler_fn);
 236       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 237       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 238       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 239     }
 240 }
 241
 242 /* Output instructions as GIMPLE trees to increment the edge
 243    execution count, and insert them on E.  We rely on
 244    gsi_insert_on_edge to preserve the order.  */
 245
 246 void
 247 gimple_gen_edge_profiler (int edgeno, edge e)
 248 {
 249   tree one;
 250
 251   one = build_int_cst (gcov_type_node, 1);
 252
 253   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 254     {
 255       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 256       tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
 257       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 258                                       ? BUILT_IN_ATOMIC_FETCH_ADD_8:
 259                                       BUILT_IN_ATOMIC_FETCH_ADD_4);
 260       gcall *stmt = gimple_build_call (f, 3, addr, one,
 261                                        build_int_cst (integer_type_node,
 262                                                       MEMMODEL_RELAXED));
 263       gsi_insert_on_edge (e, stmt);
 264     }
 265   else
 266     {
 267       tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 268       tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 269                                                    NULL, "PROF_edge_counter");
 270       gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
 271       gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 272                                               NULL, "PROF_edge_counter");
 273       gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
 274                                             gimple_assign_lhs (stmt1), one);
 275       gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
 276                                             gimple_assign_lhs (stmt2));
 277       gsi_insert_on_edge (e, stmt1);
 278       gsi_insert_on_edge (e, stmt2);
 279       gsi_insert_on_edge (e, stmt3);
 280     }
 281 }
 282
 283 /* Emits code to get VALUE to instrument at GSI, and returns the
 284    variable containing the value.  */
 285
 286 static tree
 287 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 288 {
 289   tree val = value->hvalue.value;
 290   if (POINTER_TYPE_P (TREE_TYPE (val)))
 291     val = fold_convert (build_nonstandard_integer_type
 292                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 293   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 294                                    true, NULL_TREE, true, GSI_SAME_STMT);
 295 }
 296
 297 /* Output instructions as GIMPLE trees to increment the interval histogram
 298    counter.  VALUE is the expression whose value is profiled.  TAG is the
 299    tag of the section for counters, BASE is offset of the counter position.  */
 300
 301 void
 302 gimple_gen_interval_profiler (histogram_value value, unsigned tag, unsigned base)
 303 {
 304   gimple *stmt = value->hvalue.stmt;
 305   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 306   tree ref = tree_coverage_counter_ref (tag, base), ref_ptr;
 307   gcall *call;
 308   tree val;
 309   tree start = build_int_cst_type (integer_type_node,
 310                                    value->hdata.intvl.int_start);
 311   tree steps = build_int_cst_type (unsigned_type_node,
 312                                    value->hdata.intvl.steps);
 313
 314   ref_ptr = force_gimple_operand_gsi (&gsi,
 315                                       build_addr (ref),
 316                                       true, NULL_TREE, true, GSI_SAME_STMT);
 317   val = prepare_instrumented_value (&gsi, value);
 318   call = gimple_build_call (tree_interval_profiler_fn, 4,
 319                             ref_ptr, val, start, steps);
 320   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 321 }
 322
 323 /* Output instructions as GIMPLE trees to increment the power of two histogram
 324    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 325    of the section for counters, BASE is offset of the counter position.  */
 326
 327 void
 328 gimple_gen_pow2_profiler (histogram_value value, unsigned tag, unsigned base)
 329 {
 330   gimple *stmt = value->hvalue.stmt;
 331   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 332   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 333   gcall *call;
 334   tree val;
 335
 336   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 337                                       true, NULL_TREE, true, GSI_SAME_STMT);
 338   val = prepare_instrumented_value (&gsi, value);
 339   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 340   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 341 }
 342
 343 /* Output instructions as GIMPLE trees for code to find the most common value.
 344    VALUE is the expression whose value is profiled.  TAG is the tag of the
 345    section for counters, BASE is offset of the counter position.  */
 346
 347 void
 348 gimple_gen_one_value_profiler (histogram_value value, unsigned tag, unsigned base)
 349 {
 350   gimple *stmt = value->hvalue.stmt;
 351   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 352   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 353   gcall *call;
 354   tree val;
 355
 356   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 357                                       true, NULL_TREE, true, GSI_SAME_STMT);
 358   val = prepare_instrumented_value (&gsi, value);
 359   call = gimple_build_call (tree_one_value_profiler_fn, 2, ref_ptr, val);
 360   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 361 }
 362
 363
 364 /* Output instructions as GIMPLE trees for code to find the most
 365    common called function in indirect call.
 366    VALUE is the call expression whose indirect callee is profiled.
 367    TAG is the tag of the section for counters, BASE is offset of the
 368    counter position.  */
 369
 370 void
 371 gimple_gen_ic_profiler (histogram_value value, unsigned tag, unsigned base)
 372 {
 373   tree tmp1;
 374   gassign *stmt1, *stmt2, *stmt3;
 375   gimple *stmt = value->hvalue.stmt;
 376   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 377   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 378
 379   if ( (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) &&
 380         tag == GCOV_COUNTER_V_INDIR) ||
 381        (!PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) &&
 382         tag == GCOV_COUNTER_ICALL_TOPNV))
 383     return;
 384
 385   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 386                                       true, NULL_TREE, true, GSI_SAME_STMT);
 387
 388   /* Insert code:
 389
 390     stmt1: __gcov_indirect_call_counters = get_relevant_counter_ptr ();
 391     stmt2: tmp1 = (void *) (indirect call argument value)
 392     stmt3: __gcov_indirect_call_callee = tmp1;
 393
 394     Example:
 395       f_1 = foo;
 396       __gcov_indirect_call_counters = &__gcov4.main[0];
 397       PROF_9 = f_1;
 398       __gcov_indirect_call_callee = PROF_9;
 399       _4 = f_1 ();
 400    */
 401
 402   stmt1 = gimple_build_assign (ic_gcov_type_ptr_var, ref_ptr);
 403   tmp1 = make_temp_ssa_name (ptr_void, NULL, "PROF");
 404   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 405   stmt3 = gimple_build_assign (ic_void_ptr_var, gimple_assign_lhs (stmt2));
 406
 407   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 408   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 409   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 410 }
 411
 412
 413 /* Output instructions as GIMPLE trees for code to find the most
 414    common called function in indirect call. Insert instructions at the
 415    beginning of every possible called function.
 416   */
 417
 418 void
 419 gimple_gen_ic_func_profiler (void)
 420 {
 421   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 422   gcall *stmt1;
 423   tree tree_uid, cur_func, void0;
 424
 425   if (c_node->only_called_directly_p ())
 426     return;
 427
 428   gimple_init_gcov_profiler ();
 429
 430   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 431   basic_block cond_bb = split_edge (single_succ_edge (entry));
 432   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 433
 434   /* We need to do an extra split in order to not create an input
 435      for a possible PHI node.  */
 436   split_edge (single_succ_edge (update_bb));
 437
 438   edge true_edge = single_succ_edge (cond_bb);
 439   true_edge->flags = EDGE_TRUE_VALUE;
 440
 441   profile_probability probability;
 442   if (DECL_VIRTUAL_P (current_function_decl))
 443     probability = profile_probability::very_likely ();
 444   else
 445     probability = profile_probability::unlikely ();
 446
 447   true_edge->probability = probability;
 448   edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
 449                       EDGE_FALSE_VALUE);
 450   e->probability = true_edge->probability.invert ();
 451
 452   /* Insert code:
 453
 454      if (__gcov_indirect_call_callee != NULL)
 455        __gcov_indirect_call_profiler_v2 (profile_id, &current_function_decl);
 456
 457      The function __gcov_indirect_call_profiler_v2 is responsible for
 458      resetting __gcov_indirect_call_callee to NULL.  */
 459
 460   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 461   void0 = build_int_cst (build_pointer_type (void_type_node), 0);
 462
 463   tree ref = force_gimple_operand_gsi (&gsi, ic_void_ptr_var, true, NULL_TREE,
 464                                        true, GSI_SAME_STMT);
 465
 466   gcond *cond = gimple_build_cond (NE_EXPR, ref,
 467                                    void0, NULL, NULL);
 468   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 469
 470   gsi = gsi_after_labels (update_bb);
 471
 472   cur_func = force_gimple_operand_gsi (&gsi,
 473                                        build_addr (current_function_decl),
 474                                        true, NULL_TREE,
 475                                        true, GSI_SAME_STMT);
 476   tree_uid = build_int_cst
 477               (gcov_type_node,
 478                cgraph_node::get (current_function_decl)->profile_id);
 479   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 480                              tree_uid, cur_func);
 481   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 482 }
 483
 484 /* Output instructions as GIMPLE tree at the beginning for each function.
 485    TAG is the tag of the section for counters, BASE is offset of the
 486    counter position and GSI is the iterator we place the counter.  */
 487
 488 void
 489 gimple_gen_time_profiler (unsigned tag, unsigned base)
 490 {
 491   tree type = get_gcov_type ();
 492   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 493   basic_block cond_bb = split_edge (single_succ_edge (entry));
 494   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 495
 496   /* We need to do an extra split in order to not create an input
 497      for a possible PHI node.  */
 498   split_edge (single_succ_edge (update_bb));
 499
 500   edge true_edge = single_succ_edge (cond_bb);
 501   true_edge->flags = EDGE_TRUE_VALUE;
 502   true_edge->probability = profile_probability::unlikely ();
 503   edge e
 504     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 505   e->probability = true_edge->probability.invert ();
 506
 507   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 508   tree original_ref = tree_coverage_counter_ref (tag, base);
 509   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 510                                        true, GSI_SAME_STMT);
 511   tree one = build_int_cst (type, 1);
 512
 513   /* Emit: if (counters[0] != 0).  */
 514   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 515                                    NULL, NULL);
 516   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 517
 518   gsi = gsi_start_bb (update_bb);
 519
 520   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 521   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 522     {
 523       tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
 524                                      "time_profiler_counter_ptr");
 525       tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
 526                           tree_time_profiler_counter);
 527       gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
 528       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 529       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 530                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8:
 531                                       BUILT_IN_ATOMIC_ADD_FETCH_4);
 532       gcall *stmt = gimple_build_call (f, 3, ptr, one,
 533                                        build_int_cst (integer_type_node,
 534                                                       MEMMODEL_RELAXED));
 535       tree result_type = TREE_TYPE (TREE_TYPE (f));
 536       tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
 537       gimple_set_lhs (stmt, tmp);
 538       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
 539       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 540       assign = gimple_build_assign (tmp, NOP_EXPR,
 541                                     gimple_call_lhs (stmt));
 542       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 543       assign = gimple_build_assign (original_ref, tmp);
 544       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 545     }
 546   else
 547     {
 548       tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
 549       gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
 550       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 551
 552       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 553       assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
 554                                     one);
 555       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 556       assign = gimple_build_assign (original_ref, tmp);
 557       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 558       assign = gimple_build_assign (tree_time_profiler_counter, tmp);
 559       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 560     }
 561 }
 562
 563 /* Output instructions as GIMPLE trees to increment the average histogram
 564    counter.  VALUE is the expression whose value is profiled.  TAG is the
 565    tag of the section for counters, BASE is offset of the counter position.  */
 566
 567 void
 568 gimple_gen_average_profiler (histogram_value value, unsigned tag, unsigned base)
 569 {
 570   gimple *stmt = value->hvalue.stmt;
 571   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 572   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 573   gcall *call;
 574   tree val;
 575
 576   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 577                                       true, NULL_TREE,
 578                                       true, GSI_SAME_STMT);
 579   val = prepare_instrumented_value (&gsi, value);
 580   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 581   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 582 }
 583
 584 /* Output instructions as GIMPLE trees to increment the ior histogram
 585    counter.  VALUE is the expression whose value is profiled.  TAG is the
 586    tag of the section for counters, BASE is offset of the counter position.  */
 587
 588 void
 589 gimple_gen_ior_profiler (histogram_value value, unsigned tag, unsigned base)
 590 {
 591   gimple *stmt = value->hvalue.stmt;
 592   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 593   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 594   gcall *call;
 595   tree val;
 596
 597   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 598                                       true, NULL_TREE, true, GSI_SAME_STMT);
 599   val = prepare_instrumented_value (&gsi, value);
 600   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 601   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 602 }
 603
 604 #ifndef HAVE_sync_compare_and_swapsi
 605 #define HAVE_sync_compare_and_swapsi 0
 606 #endif
 607 #ifndef HAVE_atomic_compare_and_swapsi
 608 #define HAVE_atomic_compare_and_swapsi 0
 609 #endif
 610
 611 #ifndef HAVE_sync_compare_and_swapdi
 612 #define HAVE_sync_compare_and_swapdi 0
 613 #endif
 614 #ifndef HAVE_atomic_compare_and_swapdi
 615 #define HAVE_atomic_compare_and_swapdi 0
 616 #endif
 617
 618 /* Profile all functions in the callgraph.  */
 619
 620 static unsigned int
 621 tree_profiling (void)
 622 {
 623   struct cgraph_node *node;
 624
 625   /* Verify whether we can utilize atomic update operations.  */
 626   bool can_support_atomic = false;
 627   unsigned HOST_WIDE_INT gcov_type_size
 628     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 629   if (gcov_type_size == 4)
 630     can_support_atomic
 631       = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 632   else if (gcov_type_size == 8)
 633     can_support_atomic
 634       = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 635
 636   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 637       && !can_support_atomic)
 638     {
 639       warning (0, "target does not support atomic profile update, "
 640                "single mode is selected");
 641       flag_profile_update = PROFILE_UPDATE_SINGLE;
 642     }
 643   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 644     flag_profile_update = can_support_atomic
 645       ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 646
 647   /* This is a small-ipa pass that gets called only once, from
 648      cgraphunit.c:ipa_passes().  */
 649   gcc_assert (symtab->state == IPA_SSA);
 650
 651   init_node_map (true);
 652
 653   FOR_EACH_DEFINED_FUNCTION (node)
 654     {
 655       if (!gimple_has_body_p (node->decl))
 656         continue;
 657
 658       /* Don't profile functions produced for builtin stuff.  */
 659       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 660         continue;
 661
 662       if (lookup_attribute ("no_profile_instrument_function",
 663                             DECL_ATTRIBUTES (node->decl)))
 664         continue;
 665       /* Do not instrument extern inline functions when testing coverage.
 666          While this is not perfectly consistent (early inlined extern inlines
 667          will get acocunted), testsuite expects that.  */
 668       if (DECL_EXTERNAL (node->decl)
 669           && flag_test_coverage)
 670         continue;
 671
 672       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 673
 674       /* Local pure-const may imply need to fixup the cfg.  */
 675       if (execute_fixup_cfg () & TODO_cleanup_cfg)
 676         cleanup_tree_cfg ();
 677
 678       branch_prob ();
 679
 680       if (! flag_branch_probabilities
 681           && flag_profile_values)
 682         gimple_gen_ic_func_profiler ();
 683
 684       if (flag_branch_probabilities
 685           && flag_profile_values
 686           && flag_value_profile_transformations)
 687         gimple_value_profile_transformations ();
 688
 689       /* The above could hose dominator info.  Currently there is
 690          none coming in, this is a safety valve.  It should be
 691          easy to adjust it, if and when there is some.  */
 692       free_dominance_info (CDI_DOMINATORS);
 693       free_dominance_info (CDI_POST_DOMINATORS);
 694       pop_cfun ();
 695     }
 696
 697   /* Drop pure/const flags from instrumented functions.  */
 698   if (profile_arc_flag || flag_test_coverage)
 699     FOR_EACH_DEFINED_FUNCTION (node)
 700       {
 701         if (!gimple_has_body_p (node->decl)
 702             || !(!node->clone_of
 703             || node->decl != node->clone_of->decl))
 704           continue;
 705
 706         /* Don't profile functions produced for builtin stuff.  */
 707         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 708           continue;
 709
 710         node->set_const_flag (false, false);
 711         node->set_pure_flag (false, false);
 712       }
 713
 714   /* Update call statements and rebuild the cgraph.  */
 715   FOR_EACH_DEFINED_FUNCTION (node)
 716     {
 717       basic_block bb;
 718
 719       if (!gimple_has_body_p (node->decl)
 720           || !(!node->clone_of
 721           || node->decl != node->clone_of->decl))
 722         continue;
 723
 724       /* Don't profile functions produced for builtin stuff.  */
 725       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 726         continue;
 727
 728       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 729
 730       FOR_EACH_BB_FN (bb, cfun)
 731         {
 732           gimple_stmt_iterator gsi;
 733           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 734             {
 735               gimple *stmt = gsi_stmt (gsi);
 736               if (is_gimple_call (stmt))
 737                 update_stmt (stmt);
 738             }
 739         }
 740
 741       /* re-merge split blocks.  */
 742       cleanup_tree_cfg ();
 743       update_ssa (TODO_update_ssa);
 744
 745       cgraph_edge::rebuild_edges ();
 746
 747       pop_cfun ();
 748     }
 749
 750   handle_missing_profiles ();
 751
 752   del_node_map ();
 753   return 0;
 754 }
 755
 756 namespace {
 757
 758 const pass_data pass_data_ipa_tree_profile =
 759 {
 760   SIMPLE_IPA_PASS, /* type */
 761   "profile", /* name */
 762   OPTGROUP_NONE, /* optinfo_flags */
 763   TV_IPA_PROFILE, /* tv_id */
 764   0, /* properties_required */
 765   0, /* properties_provided */
 766   0, /* properties_destroyed */
 767   0, /* todo_flags_start */
 768   TODO_dump_symtab, /* todo_flags_finish */
 769 };
 770
 771 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 772 {
 773 public:
 774   pass_ipa_tree_profile (gcc::context *ctxt)
 775     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 776   {}
 777
 778   /* opt_pass methods: */
 779   virtual bool gate (function *);
 780   virtual unsigned int execute (function *) { return tree_profiling (); }
 781
 782 }; // class pass_ipa_tree_profile
 783
 784 bool
 785 pass_ipa_tree_profile::gate (function *)
 786 {
 787   /* When profile instrumentation, use or test coverage shall be performed.
 788      But for AutoFDO, this there is no instrumentation, thus this pass is
 789      diabled.  */
 790   return (!in_lto_p && !flag_auto_profile
 791           && (flag_branch_probabilities || flag_test_coverage
 792               || profile_arc_flag));
 793 }
 794
 795 } // anon namespace
 796
 797 simple_ipa_opt_pass *
 798 make_pass_ipa_tree_profile (gcc::context *ctxt)
 799 {
 800   return new pass_ipa_tree_profile (ctxt);
 801 }
 802
 803 #include "gt-tree-profile.h"