gcc/tree-profile.c

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2017 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.c for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "params.h"
  53
  54 static GTY(()) tree gcov_type_node;
  55 static GTY(()) tree tree_interval_profiler_fn;
  56 static GTY(()) tree tree_pow2_profiler_fn;
  57 static GTY(()) tree tree_one_value_profiler_fn;
  58 static GTY(()) tree tree_indirect_call_profiler_fn;
  59 static GTY(()) tree tree_average_profiler_fn;
  60 static GTY(()) tree tree_ior_profiler_fn;
  61 static GTY(()) tree tree_time_profiler_counter;
  62
  63
  64 static GTY(()) tree ic_void_ptr_var;
  65 static GTY(()) tree ic_gcov_type_ptr_var;
  66 static GTY(()) tree ptr_void;
  67
  68 /* Do initialization work for the edge profiler.  */
  69
  70 /* Add code:
  71    __thread gcov*       __gcov_indirect_call_counters; // pointer to actual counter
  72    __thread void*       __gcov_indirect_call_callee; // actual callee address
  73    __thread int __gcov_function_counter; // time profiler function counter
  74 */
  75 static void
  76 init_ic_make_global_vars (void)
  77 {
  78   tree gcov_type_ptr;
  79
  80   ptr_void = build_pointer_type (void_type_node);
  81
  82   ic_void_ptr_var
  83     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
  84                   get_identifier (
  85                           (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
  86                            "__gcov_indirect_call_topn_callee" :
  87                            "__gcov_indirect_call_callee")),
  88                   ptr_void);
  89   TREE_PUBLIC (ic_void_ptr_var) = 1;
  90   DECL_EXTERNAL (ic_void_ptr_var) = 1;
  91   TREE_STATIC (ic_void_ptr_var) = 1;
  92   DECL_ARTIFICIAL (ic_void_ptr_var) = 1;
  93   DECL_INITIAL (ic_void_ptr_var) = NULL;
  94   if (targetm.have_tls)
  95     set_decl_tls_model (ic_void_ptr_var, decl_default_tls_model (ic_void_ptr_var));
  96
  97   gcov_type_ptr = build_pointer_type (get_gcov_type ());
  98
  99   ic_gcov_type_ptr_var
 100     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 101                   get_identifier (
 102                           (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
 103                            "__gcov_indirect_call_topn_counters" :
 104                            "__gcov_indirect_call_counters")),
 105                   gcov_type_ptr);
 106   TREE_PUBLIC (ic_gcov_type_ptr_var) = 1;
 107   DECL_EXTERNAL (ic_gcov_type_ptr_var) = 1;
 108   TREE_STATIC (ic_gcov_type_ptr_var) = 1;
 109   DECL_ARTIFICIAL (ic_gcov_type_ptr_var) = 1;
 110   DECL_INITIAL (ic_gcov_type_ptr_var) = NULL;
 111   if (targetm.have_tls)
 112     set_decl_tls_model (ic_gcov_type_ptr_var, decl_default_tls_model (ic_gcov_type_ptr_var));
 113 }
 114
 115 /* Create the type and function decls for the interface with gcov.  */
 116
 117 void
 118 gimple_init_gcov_profiler (void)
 119 {
 120   tree interval_profiler_fn_type;
 121   tree pow2_profiler_fn_type;
 122   tree one_value_profiler_fn_type;
 123   tree gcov_type_ptr;
 124   tree ic_profiler_fn_type;
 125   tree average_profiler_fn_type;
 126   const char *profiler_fn_name;
 127   const char *fn_name;
 128
 129   if (!gcov_type_node)
 130     {
 131       const char *fn_suffix
 132         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 133
 134       gcov_type_node = get_gcov_type ();
 135       gcov_type_ptr = build_pointer_type (gcov_type_node);
 136
 137       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 138       interval_profiler_fn_type
 139               = build_function_type_list (void_type_node,
 140                                           gcov_type_ptr, gcov_type_node,
 141                                           integer_type_node,
 142                                           unsigned_type_node, NULL_TREE);
 143       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 144       tree_interval_profiler_fn = build_fn_decl (fn_name,
 145                                                  interval_profiler_fn_type);
 146       free (CONST_CAST (char *, fn_name));
 147       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 148       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 149         = tree_cons (get_identifier ("leaf"), NULL,
 150                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 151
 152       /* void (*) (gcov_type *, gcov_type)  */
 153       pow2_profiler_fn_type
 154               = build_function_type_list (void_type_node,
 155                                           gcov_type_ptr, gcov_type_node,
 156                                           NULL_TREE);
 157       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 158       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 159       free (CONST_CAST (char *, fn_name));
 160       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 161       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 162         = tree_cons (get_identifier ("leaf"), NULL,
 163                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 164
 165       /* void (*) (gcov_type *, gcov_type)  */
 166       one_value_profiler_fn_type
 167               = build_function_type_list (void_type_node,
 168                                           gcov_type_ptr, gcov_type_node,
 169                                           NULL_TREE);
 170       fn_name = concat ("__gcov_one_value_profiler", fn_suffix, NULL);
 171       tree_one_value_profiler_fn = build_fn_decl (fn_name,
 172                                                   one_value_profiler_fn_type);
 173       free (CONST_CAST (char *, fn_name));
 174       TREE_NOTHROW (tree_one_value_profiler_fn) = 1;
 175       DECL_ATTRIBUTES (tree_one_value_profiler_fn)
 176         = tree_cons (get_identifier ("leaf"), NULL,
 177                      DECL_ATTRIBUTES (tree_one_value_profiler_fn));
 178
 179       init_ic_make_global_vars ();
 180
 181       /* void (*) (gcov_type, void *)  */
 182       ic_profiler_fn_type
 183                = build_function_type_list (void_type_node,
 184                                           gcov_type_node,
 185                                           ptr_void,
 186                                           NULL_TREE);
 187       profiler_fn_name = "__gcov_indirect_call_profiler_v2";
 188       if (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE))
 189         profiler_fn_name = "__gcov_indirect_call_topn_profiler";
 190
 191       tree_indirect_call_profiler_fn
 192               = build_fn_decl (profiler_fn_name, ic_profiler_fn_type);
 193
 194       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 195       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 196         = tree_cons (get_identifier ("leaf"), NULL,
 197                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 198
 199       tree_time_profiler_counter
 200         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 201                       get_identifier ("__gcov_time_profiler_counter"),
 202                       get_gcov_type ());
 203       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 204       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 205       TREE_STATIC (tree_time_profiler_counter) = 1;
 206       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 207       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 208
 209       /* void (*) (gcov_type *, gcov_type)  */
 210       average_profiler_fn_type
 211               = build_function_type_list (void_type_node,
 212                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 213       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 214       tree_average_profiler_fn = build_fn_decl (fn_name,
 215                                                 average_profiler_fn_type);
 216       free (CONST_CAST (char *, fn_name));
 217       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 218       DECL_ATTRIBUTES (tree_average_profiler_fn)
 219         = tree_cons (get_identifier ("leaf"), NULL,
 220                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 221       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 222       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 223       free (CONST_CAST (char *, fn_name));
 224       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 225       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 226         = tree_cons (get_identifier ("leaf"), NULL,
 227                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 228
 229       /* LTO streamer needs assembler names.  Because we create these decls
 230          late, we need to initialize them by hand.  */
 231       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 232       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 233       DECL_ASSEMBLER_NAME (tree_one_value_profiler_fn);
 234       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 235       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 236       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 237     }
 238 }
 239
 240 /* Output instructions as GIMPLE trees to increment the edge
 241    execution count, and insert them on E.  We rely on
 242    gsi_insert_on_edge to preserve the order.  */
 243
 244 void
 245 gimple_gen_edge_profiler (int edgeno, edge e)
 246 {
 247   tree one;
 248
 249   one = build_int_cst (gcov_type_node, 1);
 250
 251   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 252     {
 253       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 254       tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
 255       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 256                                       ? BUILT_IN_ATOMIC_FETCH_ADD_8:
 257                                       BUILT_IN_ATOMIC_FETCH_ADD_4);
 258       gcall *stmt = gimple_build_call (f, 3, addr, one,
 259                                        build_int_cst (integer_type_node,
 260                                                       MEMMODEL_RELAXED));
 261       gsi_insert_on_edge (e, stmt);
 262     }
 263   else
 264     {
 265       tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 266       tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 267                                                    NULL, "PROF_edge_counter");
 268       gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
 269       gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 270                                               NULL, "PROF_edge_counter");
 271       gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
 272                                             gimple_assign_lhs (stmt1), one);
 273       gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
 274                                             gimple_assign_lhs (stmt2));
 275       gsi_insert_on_edge (e, stmt1);
 276       gsi_insert_on_edge (e, stmt2);
 277       gsi_insert_on_edge (e, stmt3);
 278     }
 279 }
 280
 281 /* Emits code to get VALUE to instrument at GSI, and returns the
 282    variable containing the value.  */
 283
 284 static tree
 285 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 286 {
 287   tree val = value->hvalue.value;
 288   if (POINTER_TYPE_P (TREE_TYPE (val)))
 289     val = fold_convert (build_nonstandard_integer_type
 290                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 291   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 292                                    true, NULL_TREE, true, GSI_SAME_STMT);
 293 }
 294
 295 /* Output instructions as GIMPLE trees to increment the interval histogram
 296    counter.  VALUE is the expression whose value is profiled.  TAG is the
 297    tag of the section for counters, BASE is offset of the counter position.  */
 298
 299 void
 300 gimple_gen_interval_profiler (histogram_value value, unsigned tag, unsigned base)
 301 {
 302   gimple *stmt = value->hvalue.stmt;
 303   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 304   tree ref = tree_coverage_counter_ref (tag, base), ref_ptr;
 305   gcall *call;
 306   tree val;
 307   tree start = build_int_cst_type (integer_type_node,
 308                                    value->hdata.intvl.int_start);
 309   tree steps = build_int_cst_type (unsigned_type_node,
 310                                    value->hdata.intvl.steps);
 311
 312   ref_ptr = force_gimple_operand_gsi (&gsi,
 313                                       build_addr (ref),
 314                                       true, NULL_TREE, true, GSI_SAME_STMT);
 315   val = prepare_instrumented_value (&gsi, value);
 316   call = gimple_build_call (tree_interval_profiler_fn, 4,
 317                             ref_ptr, val, start, steps);
 318   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 319 }
 320
 321 /* Output instructions as GIMPLE trees to increment the power of two histogram
 322    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 323    of the section for counters, BASE is offset of the counter position.  */
 324
 325 void
 326 gimple_gen_pow2_profiler (histogram_value value, unsigned tag, unsigned base)
 327 {
 328   gimple *stmt = value->hvalue.stmt;
 329   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 330   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 331   gcall *call;
 332   tree val;
 333
 334   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 335                                       true, NULL_TREE, true, GSI_SAME_STMT);
 336   val = prepare_instrumented_value (&gsi, value);
 337   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 338   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 339 }
 340
 341 /* Output instructions as GIMPLE trees for code to find the most common value.
 342    VALUE is the expression whose value is profiled.  TAG is the tag of the
 343    section for counters, BASE is offset of the counter position.  */
 344
 345 void
 346 gimple_gen_one_value_profiler (histogram_value value, unsigned tag, unsigned base)
 347 {
 348   gimple *stmt = value->hvalue.stmt;
 349   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 350   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 351   gcall *call;
 352   tree val;
 353
 354   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 355                                       true, NULL_TREE, true, GSI_SAME_STMT);
 356   val = prepare_instrumented_value (&gsi, value);
 357   call = gimple_build_call (tree_one_value_profiler_fn, 2, ref_ptr, val);
 358   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 359 }
 360
 361
 362 /* Output instructions as GIMPLE trees for code to find the most
 363    common called function in indirect call.
 364    VALUE is the call expression whose indirect callee is profiled.
 365    TAG is the tag of the section for counters, BASE is offset of the
 366    counter position.  */
 367
 368 void
 369 gimple_gen_ic_profiler (histogram_value value, unsigned tag, unsigned base)
 370 {
 371   tree tmp1;
 372   gassign *stmt1, *stmt2, *stmt3;
 373   gimple *stmt = value->hvalue.stmt;
 374   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 375   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 376
 377   if ( (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) &&
 378         tag == GCOV_COUNTER_V_INDIR) ||
 379        (!PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) &&
 380         tag == GCOV_COUNTER_ICALL_TOPNV))
 381     return;
 382
 383   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 384                                       true, NULL_TREE, true, GSI_SAME_STMT);
 385
 386   /* Insert code:
 387
 388     stmt1: __gcov_indirect_call_counters = get_relevant_counter_ptr ();
 389     stmt2: tmp1 = (void *) (indirect call argument value)
 390     stmt3: __gcov_indirect_call_callee = tmp1;
 391
 392     Example:
 393       f_1 = foo;
 394       __gcov_indirect_call_counters = &__gcov4.main[0];
 395       PROF_9 = f_1;
 396       __gcov_indirect_call_callee = PROF_9;
 397       _4 = f_1 ();
 398    */
 399
 400   stmt1 = gimple_build_assign (ic_gcov_type_ptr_var, ref_ptr);
 401   tmp1 = make_temp_ssa_name (ptr_void, NULL, "PROF");
 402   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 403   stmt3 = gimple_build_assign (ic_void_ptr_var, gimple_assign_lhs (stmt2));
 404
 405   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 406   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 407   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 408 }
 409
 410
 411 /* Output instructions as GIMPLE trees for code to find the most
 412    common called function in indirect call. Insert instructions at the
 413    beginning of every possible called function.
 414   */
 415
 416 void
 417 gimple_gen_ic_func_profiler (void)
 418 {
 419   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 420   gcall *stmt1;
 421   tree tree_uid, cur_func, void0;
 422
 423   if (c_node->only_called_directly_p ())
 424     return;
 425
 426   gimple_init_gcov_profiler ();
 427
 428   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 429   basic_block cond_bb = split_edge (single_succ_edge (entry));
 430   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 431
 432   /* We need to do an extra split in order to not create an input
 433      for a possible PHI node.  */
 434   split_edge (single_succ_edge (update_bb));
 435
 436   edge true_edge = single_succ_edge (cond_bb);
 437   true_edge->flags = EDGE_TRUE_VALUE;
 438
 439   profile_probability probability;
 440   if (DECL_VIRTUAL_P (current_function_decl))
 441     probability = profile_probability::very_likely ();
 442   else
 443     probability = profile_probability::unlikely ();
 444
 445   true_edge->probability = probability;
 446   edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
 447                       EDGE_FALSE_VALUE);
 448   e->probability = true_edge->probability.invert ();
 449
 450   /* Insert code:
 451
 452      if (__gcov_indirect_call_callee != NULL)
 453        __gcov_indirect_call_profiler_v2 (profile_id, &current_function_decl);
 454
 455      The function __gcov_indirect_call_profiler_v2 is responsible for
 456      resetting __gcov_indirect_call_callee to NULL.  */
 457
 458   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 459   void0 = build_int_cst (build_pointer_type (void_type_node), 0);
 460
 461   tree ref = force_gimple_operand_gsi (&gsi, ic_void_ptr_var, true, NULL_TREE,
 462                                        true, GSI_SAME_STMT);
 463
 464   gcond *cond = gimple_build_cond (NE_EXPR, ref,
 465                                    void0, NULL, NULL);
 466   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 467
 468   gsi = gsi_after_labels (update_bb);
 469
 470   cur_func = force_gimple_operand_gsi (&gsi,
 471                                        build_addr (current_function_decl),
 472                                        true, NULL_TREE,
 473                                        true, GSI_SAME_STMT);
 474   tree_uid = build_int_cst
 475               (gcov_type_node,
 476                cgraph_node::get (current_function_decl)->profile_id);
 477   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 478                              tree_uid, cur_func);
 479   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 480 }
 481
 482 /* Output instructions as GIMPLE tree at the beginning for each function.
 483    TAG is the tag of the section for counters, BASE is offset of the
 484    counter position and GSI is the iterator we place the counter.  */
 485
 486 void
 487 gimple_gen_time_profiler (unsigned tag, unsigned base)
 488 {
 489   tree type = get_gcov_type ();
 490   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 491   basic_block cond_bb = split_edge (single_succ_edge (entry));
 492   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 493
 494   /* We need to do an extra split in order to not create an input
 495      for a possible PHI node.  */
 496   split_edge (single_succ_edge (update_bb));
 497
 498   edge true_edge = single_succ_edge (cond_bb);
 499   true_edge->flags = EDGE_TRUE_VALUE;
 500   true_edge->probability = profile_probability::unlikely ();
 501   edge e
 502     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 503   e->probability = true_edge->probability.invert ();
 504
 505   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 506   tree original_ref = tree_coverage_counter_ref (tag, base);
 507   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 508                                        true, GSI_SAME_STMT);
 509   tree one = build_int_cst (type, 1);
 510
 511   /* Emit: if (counters[0] != 0).  */
 512   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 513                                    NULL, NULL);
 514   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 515
 516   gsi = gsi_start_bb (update_bb);
 517
 518   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 519   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 520     {
 521       tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
 522                                      "time_profiler_counter_ptr");
 523       tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
 524                           tree_time_profiler_counter);
 525       gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
 526       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 527       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 528                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8:
 529                                       BUILT_IN_ATOMIC_ADD_FETCH_4);
 530       gcall *stmt = gimple_build_call (f, 3, ptr, one,
 531                                        build_int_cst (integer_type_node,
 532                                                       MEMMODEL_RELAXED));
 533       tree result_type = TREE_TYPE (TREE_TYPE (f));
 534       tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
 535       gimple_set_lhs (stmt, tmp);
 536       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
 537       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 538       assign = gimple_build_assign (tmp, NOP_EXPR,
 539                                     gimple_call_lhs (stmt));
 540       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 541       assign = gimple_build_assign (original_ref, tmp);
 542       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 543     }
 544   else
 545     {
 546       tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
 547       gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
 548       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 549
 550       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 551       assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
 552                                     one);
 553       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 554       assign = gimple_build_assign (original_ref, tmp);
 555       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 556       assign = gimple_build_assign (tree_time_profiler_counter, tmp);
 557       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 558     }
 559 }
 560
 561 /* Output instructions as GIMPLE trees to increment the average histogram
 562    counter.  VALUE is the expression whose value is profiled.  TAG is the
 563    tag of the section for counters, BASE is offset of the counter position.  */
 564
 565 void
 566 gimple_gen_average_profiler (histogram_value value, unsigned tag, unsigned base)
 567 {
 568   gimple *stmt = value->hvalue.stmt;
 569   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 570   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 571   gcall *call;
 572   tree val;
 573
 574   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 575                                       true, NULL_TREE,
 576                                       true, GSI_SAME_STMT);
 577   val = prepare_instrumented_value (&gsi, value);
 578   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 579   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 580 }
 581
 582 /* Output instructions as GIMPLE trees to increment the ior histogram
 583    counter.  VALUE is the expression whose value is profiled.  TAG is the
 584    tag of the section for counters, BASE is offset of the counter position.  */
 585
 586 void
 587 gimple_gen_ior_profiler (histogram_value value, unsigned tag, unsigned base)
 588 {
 589   gimple *stmt = value->hvalue.stmt;
 590   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 591   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 592   gcall *call;
 593   tree val;
 594
 595   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 596                                       true, NULL_TREE, true, GSI_SAME_STMT);
 597   val = prepare_instrumented_value (&gsi, value);
 598   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 599   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 600 }
 601
 602 #ifndef HAVE_sync_compare_and_swapsi
 603 #define HAVE_sync_compare_and_swapsi 0
 604 #endif
 605 #ifndef HAVE_atomic_compare_and_swapsi
 606 #define HAVE_atomic_compare_and_swapsi 0
 607 #endif
 608
 609 #ifndef HAVE_sync_compare_and_swapdi
 610 #define HAVE_sync_compare_and_swapdi 0
 611 #endif
 612 #ifndef HAVE_atomic_compare_and_swapdi
 613 #define HAVE_atomic_compare_and_swapdi 0
 614 #endif
 615
 616 /* Profile all functions in the callgraph.  */
 617
 618 static unsigned int
 619 tree_profiling (void)
 620 {
 621   struct cgraph_node *node;
 622
 623   /* Verify whether we can utilize atomic update operations.  */
 624   bool can_support_atomic = false;
 625   unsigned HOST_WIDE_INT gcov_type_size
 626     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 627   if (gcov_type_size == 4)
 628     can_support_atomic
 629       = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 630   else if (gcov_type_size == 8)
 631     can_support_atomic
 632       = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 633
 634   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 635       && !can_support_atomic)
 636     {
 637       warning (0, "target does not support atomic profile update, "
 638                "single mode is selected");
 639       flag_profile_update = PROFILE_UPDATE_SINGLE;
 640     }
 641   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 642     flag_profile_update = can_support_atomic
 643       ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 644
 645   /* This is a small-ipa pass that gets called only once, from
 646      cgraphunit.c:ipa_passes().  */
 647   gcc_assert (symtab->state == IPA_SSA);
 648
 649   init_node_map (true);
 650
 651   FOR_EACH_DEFINED_FUNCTION (node)
 652     {
 653       if (!gimple_has_body_p (node->decl))
 654         continue;
 655
 656       /* Don't profile functions produced for builtin stuff.  */
 657       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 658         continue;
 659
 660       if (lookup_attribute ("no_profile_instrument_function",
 661                             DECL_ATTRIBUTES (node->decl)))
 662         continue;
 663       /* Do not instrument extern inline functions when testing coverage.
 664          While this is not perfectly consistent (early inlined extern inlines
 665          will get acocunted), testsuite expects that.  */
 666       if (DECL_EXTERNAL (node->decl)
 667           && flag_test_coverage)
 668         continue;
 669
 670       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 671
 672       /* Local pure-const may imply need to fixup the cfg.  */
 673       if (execute_fixup_cfg () & TODO_cleanup_cfg)
 674         cleanup_tree_cfg ();
 675
 676       branch_prob ();
 677
 678       if (! flag_branch_probabilities
 679           && flag_profile_values)
 680         gimple_gen_ic_func_profiler ();
 681
 682       if (flag_branch_probabilities
 683           && flag_profile_values
 684           && flag_value_profile_transformations)
 685         gimple_value_profile_transformations ();
 686
 687       /* The above could hose dominator info.  Currently there is
 688          none coming in, this is a safety valve.  It should be
 689          easy to adjust it, if and when there is some.  */
 690       free_dominance_info (CDI_DOMINATORS);
 691       free_dominance_info (CDI_POST_DOMINATORS);
 692       pop_cfun ();
 693     }
 694
 695   /* Drop pure/const flags from instrumented functions.  */
 696   if (profile_arc_flag || flag_test_coverage)
 697     FOR_EACH_DEFINED_FUNCTION (node)
 698       {
 699         if (!gimple_has_body_p (node->decl)
 700             || !(!node->clone_of
 701             || node->decl != node->clone_of->decl))
 702           continue;
 703
 704         /* Don't profile functions produced for builtin stuff.  */
 705         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 706           continue;
 707
 708         node->set_const_flag (false, false);
 709         node->set_pure_flag (false, false);
 710       }
 711
 712   /* Update call statements and rebuild the cgraph.  */
 713   FOR_EACH_DEFINED_FUNCTION (node)
 714     {
 715       basic_block bb;
 716
 717       if (!gimple_has_body_p (node->decl)
 718           || !(!node->clone_of
 719           || node->decl != node->clone_of->decl))
 720         continue;
 721
 722       /* Don't profile functions produced for builtin stuff.  */
 723       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 724         continue;
 725
 726       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 727
 728       FOR_EACH_BB_FN (bb, cfun)
 729         {
 730           gimple_stmt_iterator gsi;
 731           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 732             {
 733               gimple *stmt = gsi_stmt (gsi);
 734               if (is_gimple_call (stmt))
 735                 update_stmt (stmt);
 736             }
 737         }
 738
 739       /* re-merge split blocks.  */
 740       cleanup_tree_cfg ();
 741       update_ssa (TODO_update_ssa);
 742
 743       cgraph_edge::rebuild_edges ();
 744
 745       pop_cfun ();
 746     }
 747
 748   handle_missing_profiles ();
 749
 750   del_node_map ();
 751   return 0;
 752 }
 753
 754 namespace {
 755
 756 const pass_data pass_data_ipa_tree_profile =
 757 {
 758   SIMPLE_IPA_PASS, /* type */
 759   "profile", /* name */
 760   OPTGROUP_NONE, /* optinfo_flags */
 761   TV_IPA_PROFILE, /* tv_id */
 762   0, /* properties_required */
 763   0, /* properties_provided */
 764   0, /* properties_destroyed */
 765   0, /* todo_flags_start */
 766   TODO_dump_symtab, /* todo_flags_finish */
 767 };
 768
 769 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 770 {
 771 public:
 772   pass_ipa_tree_profile (gcc::context *ctxt)
 773     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 774   {}
 775
 776   /* opt_pass methods: */
 777   virtual bool gate (function *);
 778   virtual unsigned int execute (function *) { return tree_profiling (); }
 779
 780 }; // class pass_ipa_tree_profile
 781
 782 bool
 783 pass_ipa_tree_profile::gate (function *)
 784 {
 785   /* When profile instrumentation, use or test coverage shall be performed.
 786      But for AutoFDO, this there is no instrumentation, thus this pass is
 787      diabled.  */
 788   return (!in_lto_p && !flag_auto_profile
 789           && (flag_branch_probabilities || flag_test_coverage
 790               || profile_arc_flag));
 791 }
 792
 793 } // anon namespace
 794
 795 simple_ipa_opt_pass *
 796 make_pass_ipa_tree_profile (gcc::context *ctxt)
 797 {
 798   return new pass_ipa_tree_profile (ctxt);
 799 }
 800
 801 #include "gt-tree-profile.h"