gcc/tree-profile.c

   1 /* Calculate branch probabilities, and basic block execution counts.
   2    Copyright (C) 1990-2017 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
   4    based on some ideas from Dain Samples of UC Berkeley.
   5    Further mangling by Bob Manson, Cygnus Support.
   6    Converted to use trees by Dale Johannesen, Apple Computer.
   7
   8 This file is part of GCC.
   9
  10 GCC is free software; you can redistribute it and/or modify it under
  11 the terms of the GNU General Public License as published by the Free
  12 Software Foundation; either version 3, or (at your option) any later
  13 version.
  14
  15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  18 for more details.
  19
  20 You should have received a copy of the GNU General Public License
  21 along with GCC; see the file COPYING3.  If not see
  22 <http://www.gnu.org/licenses/>.  */
  23
  24 /* Generate basic block profile instrumentation and auxiliary files.
  25    Tree-based version.  See profile.c for overview.  */
  26
  27 #include "config.h"
  28 #include "system.h"
  29 #include "coretypes.h"
  30 #include "memmodel.h"
  31 #include "backend.h"
  32 #include "target.h"
  33 #include "tree.h"
  34 #include "gimple.h"
  35 #include "cfghooks.h"
  36 #include "tree-pass.h"
  37 #include "ssa.h"
  38 #include "cgraph.h"
  39 #include "coverage.h"
  40 #include "diagnostic-core.h"
  41 #include "fold-const.h"
  42 #include "varasm.h"
  43 #include "tree-nested.h"
  44 #include "gimplify.h"
  45 #include "gimple-iterator.h"
  46 #include "gimplify-me.h"
  47 #include "tree-cfg.h"
  48 #include "tree-into-ssa.h"
  49 #include "value-prof.h"
  50 #include "profile.h"
  51 #include "tree-cfgcleanup.h"
  52 #include "params.h"
  53
  54 static GTY(()) tree gcov_type_node;
  55 static GTY(()) tree tree_interval_profiler_fn;
  56 static GTY(()) tree tree_pow2_profiler_fn;
  57 static GTY(()) tree tree_one_value_profiler_fn;
  58 static GTY(()) tree tree_indirect_call_profiler_fn;
  59 static GTY(()) tree tree_average_profiler_fn;
  60 static GTY(()) tree tree_ior_profiler_fn;
  61 static GTY(()) tree tree_time_profiler_counter;
  62
  63
  64 static GTY(()) tree ic_void_ptr_var;
  65 static GTY(()) tree ic_gcov_type_ptr_var;
  66 static GTY(()) tree ptr_void;
  67
  68 /* Do initialization work for the edge profiler.  */
  69
  70 /* Add code:
  71    __thread gcov*       __gcov_indirect_call_counters; // pointer to actual counter
  72    __thread void*       __gcov_indirect_call_callee; // actual callee address
  73    __thread int __gcov_function_counter; // time profiler function counter
  74 */
  75 static void
  76 init_ic_make_global_vars (void)
  77 {
  78   tree gcov_type_ptr;
  79
  80   ptr_void = build_pointer_type (void_type_node);
  81
  82   ic_void_ptr_var
  83     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
  84                   get_identifier (
  85                           (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
  86                            "__gcov_indirect_call_topn_callee" :
  87                            "__gcov_indirect_call_callee")),
  88                   ptr_void);
  89   TREE_PUBLIC (ic_void_ptr_var) = 1;
  90   DECL_EXTERNAL (ic_void_ptr_var) = 1;
  91   TREE_STATIC (ic_void_ptr_var) = 1;
  92   DECL_ARTIFICIAL (ic_void_ptr_var) = 1;
  93   DECL_INITIAL (ic_void_ptr_var) = NULL;
  94   if (targetm.have_tls)
  95     set_decl_tls_model (ic_void_ptr_var, decl_default_tls_model (ic_void_ptr_var));
  96
  97   varpool_node::finalize_decl (ic_void_ptr_var);
  98
  99   gcov_type_ptr = build_pointer_type (get_gcov_type ());
 100
 101   ic_gcov_type_ptr_var
 102     = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 103                   get_identifier (
 104                           (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ?
 105                            "__gcov_indirect_call_topn_counters" :
 106                            "__gcov_indirect_call_counters")),
 107                   gcov_type_ptr);
 108   TREE_PUBLIC (ic_gcov_type_ptr_var) = 1;
 109   DECL_EXTERNAL (ic_gcov_type_ptr_var) = 1;
 110   TREE_STATIC (ic_gcov_type_ptr_var) = 1;
 111   DECL_ARTIFICIAL (ic_gcov_type_ptr_var) = 1;
 112   DECL_INITIAL (ic_gcov_type_ptr_var) = NULL;
 113   if (targetm.have_tls)
 114     set_decl_tls_model (ic_gcov_type_ptr_var, decl_default_tls_model (ic_gcov_type_ptr_var));
 115
 116   varpool_node::finalize_decl (ic_gcov_type_ptr_var);
 117 }
 118
 119 /* Create the type and function decls for the interface with gcov.  */
 120
 121 void
 122 gimple_init_gcov_profiler (void)
 123 {
 124   tree interval_profiler_fn_type;
 125   tree pow2_profiler_fn_type;
 126   tree one_value_profiler_fn_type;
 127   tree gcov_type_ptr;
 128   tree ic_profiler_fn_type;
 129   tree average_profiler_fn_type;
 130   const char *profiler_fn_name;
 131   const char *fn_name;
 132
 133   if (!gcov_type_node)
 134     {
 135       const char *fn_suffix
 136         = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
 137
 138       gcov_type_node = get_gcov_type ();
 139       gcov_type_ptr = build_pointer_type (gcov_type_node);
 140
 141       /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
 142       interval_profiler_fn_type
 143               = build_function_type_list (void_type_node,
 144                                           gcov_type_ptr, gcov_type_node,
 145                                           integer_type_node,
 146                                           unsigned_type_node, NULL_TREE);
 147       fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
 148       tree_interval_profiler_fn = build_fn_decl (fn_name,
 149                                                  interval_profiler_fn_type);
 150       free (CONST_CAST (char *, fn_name));
 151       TREE_NOTHROW (tree_interval_profiler_fn) = 1;
 152       DECL_ATTRIBUTES (tree_interval_profiler_fn)
 153         = tree_cons (get_identifier ("leaf"), NULL,
 154                      DECL_ATTRIBUTES (tree_interval_profiler_fn));
 155
 156       /* void (*) (gcov_type *, gcov_type)  */
 157       pow2_profiler_fn_type
 158               = build_function_type_list (void_type_node,
 159                                           gcov_type_ptr, gcov_type_node,
 160                                           NULL_TREE);
 161       fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
 162       tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
 163       free (CONST_CAST (char *, fn_name));
 164       TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
 165       DECL_ATTRIBUTES (tree_pow2_profiler_fn)
 166         = tree_cons (get_identifier ("leaf"), NULL,
 167                      DECL_ATTRIBUTES (tree_pow2_profiler_fn));
 168
 169       /* void (*) (gcov_type *, gcov_type)  */
 170       one_value_profiler_fn_type
 171               = build_function_type_list (void_type_node,
 172                                           gcov_type_ptr, gcov_type_node,
 173                                           NULL_TREE);
 174       fn_name = concat ("__gcov_one_value_profiler", fn_suffix, NULL);
 175       tree_one_value_profiler_fn = build_fn_decl (fn_name,
 176                                                   one_value_profiler_fn_type);
 177       free (CONST_CAST (char *, fn_name));
 178       TREE_NOTHROW (tree_one_value_profiler_fn) = 1;
 179       DECL_ATTRIBUTES (tree_one_value_profiler_fn)
 180         = tree_cons (get_identifier ("leaf"), NULL,
 181                      DECL_ATTRIBUTES (tree_one_value_profiler_fn));
 182
 183       init_ic_make_global_vars ();
 184
 185       /* void (*) (gcov_type, void *)  */
 186       ic_profiler_fn_type
 187                = build_function_type_list (void_type_node,
 188                                           gcov_type_node,
 189                                           ptr_void,
 190                                           NULL_TREE);
 191       profiler_fn_name = "__gcov_indirect_call_profiler_v2";
 192       if (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE))
 193         profiler_fn_name = "__gcov_indirect_call_topn_profiler";
 194
 195       tree_indirect_call_profiler_fn
 196               = build_fn_decl (profiler_fn_name, ic_profiler_fn_type);
 197
 198       TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
 199       DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
 200         = tree_cons (get_identifier ("leaf"), NULL,
 201                      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
 202
 203       tree_time_profiler_counter
 204         = build_decl (UNKNOWN_LOCATION, VAR_DECL,
 205                       get_identifier ("__gcov_time_profiler_counter"),
 206                       get_gcov_type ());
 207       TREE_PUBLIC (tree_time_profiler_counter) = 1;
 208       DECL_EXTERNAL (tree_time_profiler_counter) = 1;
 209       TREE_STATIC (tree_time_profiler_counter) = 1;
 210       DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
 211       DECL_INITIAL (tree_time_profiler_counter) = NULL;
 212
 213       varpool_node::finalize_decl (tree_time_profiler_counter);
 214
 215       /* void (*) (gcov_type *, gcov_type)  */
 216       average_profiler_fn_type
 217               = build_function_type_list (void_type_node,
 218                                           gcov_type_ptr, gcov_type_node, NULL_TREE);
 219       fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
 220       tree_average_profiler_fn = build_fn_decl (fn_name,
 221                                                 average_profiler_fn_type);
 222       free (CONST_CAST (char *, fn_name));
 223       TREE_NOTHROW (tree_average_profiler_fn) = 1;
 224       DECL_ATTRIBUTES (tree_average_profiler_fn)
 225         = tree_cons (get_identifier ("leaf"), NULL,
 226                      DECL_ATTRIBUTES (tree_average_profiler_fn));
 227       fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
 228       tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
 229       free (CONST_CAST (char *, fn_name));
 230       TREE_NOTHROW (tree_ior_profiler_fn) = 1;
 231       DECL_ATTRIBUTES (tree_ior_profiler_fn)
 232         = tree_cons (get_identifier ("leaf"), NULL,
 233                      DECL_ATTRIBUTES (tree_ior_profiler_fn));
 234
 235       /* LTO streamer needs assembler names.  Because we create these decls
 236          late, we need to initialize them by hand.  */
 237       DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
 238       DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
 239       DECL_ASSEMBLER_NAME (tree_one_value_profiler_fn);
 240       DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
 241       DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
 242       DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
 243     }
 244 }
 245
 246 /* Output instructions as GIMPLE trees to increment the edge
 247    execution count, and insert them on E.  We rely on
 248    gsi_insert_on_edge to preserve the order.  */
 249
 250 void
 251 gimple_gen_edge_profiler (int edgeno, edge e)
 252 {
 253   tree one;
 254
 255   one = build_int_cst (gcov_type_node, 1);
 256
 257   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 258     {
 259       /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
 260       tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
 261       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 262                                       ? BUILT_IN_ATOMIC_FETCH_ADD_8:
 263                                       BUILT_IN_ATOMIC_FETCH_ADD_4);
 264       gcall *stmt = gimple_build_call (f, 3, addr, one,
 265                                        build_int_cst (integer_type_node,
 266                                                       MEMMODEL_RELAXED));
 267       gsi_insert_on_edge (e, stmt);
 268     }
 269   else
 270     {
 271       tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
 272       tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 273                                                    NULL, "PROF_edge_counter");
 274       gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
 275       gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
 276                                               NULL, "PROF_edge_counter");
 277       gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
 278                                             gimple_assign_lhs (stmt1), one);
 279       gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
 280                                             gimple_assign_lhs (stmt2));
 281       gsi_insert_on_edge (e, stmt1);
 282       gsi_insert_on_edge (e, stmt2);
 283       gsi_insert_on_edge (e, stmt3);
 284     }
 285 }
 286
 287 /* Emits code to get VALUE to instrument at GSI, and returns the
 288    variable containing the value.  */
 289
 290 static tree
 291 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
 292 {
 293   tree val = value->hvalue.value;
 294   if (POINTER_TYPE_P (TREE_TYPE (val)))
 295     val = fold_convert (build_nonstandard_integer_type
 296                           (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
 297   return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
 298                                    true, NULL_TREE, true, GSI_SAME_STMT);
 299 }
 300
 301 /* Output instructions as GIMPLE trees to increment the interval histogram
 302    counter.  VALUE is the expression whose value is profiled.  TAG is the
 303    tag of the section for counters, BASE is offset of the counter position.  */
 304
 305 void
 306 gimple_gen_interval_profiler (histogram_value value, unsigned tag, unsigned base)
 307 {
 308   gimple *stmt = value->hvalue.stmt;
 309   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 310   tree ref = tree_coverage_counter_ref (tag, base), ref_ptr;
 311   gcall *call;
 312   tree val;
 313   tree start = build_int_cst_type (integer_type_node,
 314                                    value->hdata.intvl.int_start);
 315   tree steps = build_int_cst_type (unsigned_type_node,
 316                                    value->hdata.intvl.steps);
 317
 318   ref_ptr = force_gimple_operand_gsi (&gsi,
 319                                       build_addr (ref),
 320                                       true, NULL_TREE, true, GSI_SAME_STMT);
 321   val = prepare_instrumented_value (&gsi, value);
 322   call = gimple_build_call (tree_interval_profiler_fn, 4,
 323                             ref_ptr, val, start, steps);
 324   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 325 }
 326
 327 /* Output instructions as GIMPLE trees to increment the power of two histogram
 328    counter.  VALUE is the expression whose value is profiled.  TAG is the tag
 329    of the section for counters, BASE is offset of the counter position.  */
 330
 331 void
 332 gimple_gen_pow2_profiler (histogram_value value, unsigned tag, unsigned base)
 333 {
 334   gimple *stmt = value->hvalue.stmt;
 335   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 336   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 337   gcall *call;
 338   tree val;
 339
 340   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 341                                       true, NULL_TREE, true, GSI_SAME_STMT);
 342   val = prepare_instrumented_value (&gsi, value);
 343   call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
 344   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 345 }
 346
 347 /* Output instructions as GIMPLE trees for code to find the most common value.
 348    VALUE is the expression whose value is profiled.  TAG is the tag of the
 349    section for counters, BASE is offset of the counter position.  */
 350
 351 void
 352 gimple_gen_one_value_profiler (histogram_value value, unsigned tag, unsigned base)
 353 {
 354   gimple *stmt = value->hvalue.stmt;
 355   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 356   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 357   gcall *call;
 358   tree val;
 359
 360   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 361                                       true, NULL_TREE, true, GSI_SAME_STMT);
 362   val = prepare_instrumented_value (&gsi, value);
 363   call = gimple_build_call (tree_one_value_profiler_fn, 2, ref_ptr, val);
 364   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 365 }
 366
 367
 368 /* Output instructions as GIMPLE trees for code to find the most
 369    common called function in indirect call.
 370    VALUE is the call expression whose indirect callee is profiled.
 371    TAG is the tag of the section for counters, BASE is offset of the
 372    counter position.  */
 373
 374 void
 375 gimple_gen_ic_profiler (histogram_value value, unsigned tag, unsigned base)
 376 {
 377   tree tmp1;
 378   gassign *stmt1, *stmt2, *stmt3;
 379   gimple *stmt = value->hvalue.stmt;
 380   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 381   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 382
 383   if ( (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) &&
 384         tag == GCOV_COUNTER_V_INDIR) ||
 385        (!PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) &&
 386         tag == GCOV_COUNTER_ICALL_TOPNV))
 387     return;
 388
 389   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 390                                       true, NULL_TREE, true, GSI_SAME_STMT);
 391
 392   /* Insert code:
 393
 394     stmt1: __gcov_indirect_call_counters = get_relevant_counter_ptr ();
 395     stmt2: tmp1 = (void *) (indirect call argument value)
 396     stmt3: __gcov_indirect_call_callee = tmp1;
 397    */
 398
 399   stmt1 = gimple_build_assign (ic_gcov_type_ptr_var, ref_ptr);
 400   tmp1 = make_temp_ssa_name (ptr_void, NULL, "PROF");
 401   stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
 402   stmt3 = gimple_build_assign (ic_void_ptr_var, gimple_assign_lhs (stmt2));
 403
 404   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 405   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 406   gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
 407 }
 408
 409
 410 /* Output instructions as GIMPLE trees for code to find the most
 411    common called function in indirect call. Insert instructions at the
 412    beginning of every possible called function.
 413   */
 414
 415 void
 416 gimple_gen_ic_func_profiler (void)
 417 {
 418   struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
 419   gimple_stmt_iterator gsi;
 420   gcall *stmt1;
 421   gassign *stmt2;
 422   tree tree_uid, cur_func, void0;
 423
 424   if (c_node->only_called_directly_p ())
 425     return;
 426
 427   gimple_init_gcov_profiler ();
 428
 429   /* Insert code:
 430
 431     stmt1: __gcov_indirect_call_profiler_v2 (profile_id,
 432                                              &current_function_decl)
 433    */
 434   gsi = gsi_after_labels (split_edge (single_succ_edge
 435                                          (ENTRY_BLOCK_PTR_FOR_FN (cfun))));
 436
 437   cur_func = force_gimple_operand_gsi (&gsi,
 438                                        build_addr (current_function_decl),
 439                                        true, NULL_TREE,
 440                                        true, GSI_SAME_STMT);
 441   tree_uid = build_int_cst
 442               (gcov_type_node,
 443                cgraph_node::get (current_function_decl)->profile_id);
 444   stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
 445                              tree_uid, cur_func);
 446   gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
 447
 448   /* Set __gcov_indirect_call_callee to 0,
 449      so that calls from other modules won't get misattributed
 450      to the last caller of the current callee. */
 451   void0 = build_int_cst (build_pointer_type (void_type_node), 0);
 452   stmt2 = gimple_build_assign (ic_void_ptr_var, void0);
 453   gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
 454 }
 455
 456 /* Output instructions as GIMPLE tree at the beginning for each function.
 457    TAG is the tag of the section for counters, BASE is offset of the
 458    counter position and GSI is the iterator we place the counter.  */
 459
 460 void
 461 gimple_gen_time_profiler (unsigned tag, unsigned base)
 462 {
 463   tree type = get_gcov_type ();
 464   basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
 465   basic_block cond_bb = split_edge (single_succ_edge (entry));
 466   basic_block update_bb = split_edge (single_succ_edge (cond_bb));
 467   split_edge (single_succ_edge (update_bb));
 468
 469   edge true_edge = single_succ_edge (cond_bb);
 470   true_edge->flags = EDGE_TRUE_VALUE;
 471   true_edge->probability = PROB_VERY_UNLIKELY;
 472   edge e
 473     = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
 474   e->probability = REG_BR_PROB_BASE - true_edge->probability;
 475
 476   gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
 477   tree original_ref = tree_coverage_counter_ref (tag, base);
 478   tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
 479                                        true, GSI_SAME_STMT);
 480   tree one = build_int_cst (type, 1);
 481
 482   /* Emit: if (counters[0] != 0).  */
 483   gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
 484                                    NULL, NULL);
 485   gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
 486
 487   gsi = gsi_start_bb (update_bb);
 488
 489   /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
 490   if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
 491     {
 492       tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
 493                                      "time_profiler_counter_ptr");
 494       tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
 495                           tree_time_profiler_counter);
 496       gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
 497       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 498       tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
 499                                       ? BUILT_IN_ATOMIC_ADD_FETCH_8:
 500                                       BUILT_IN_ATOMIC_ADD_FETCH_4);
 501       gcall *stmt = gimple_build_call (f, 3, ptr, one,
 502                                        build_int_cst (integer_type_node,
 503                                                       MEMMODEL_RELAXED));
 504       tree result_type = TREE_TYPE (TREE_TYPE (f));
 505       tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
 506       gimple_set_lhs (stmt, tmp);
 507       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
 508       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 509       assign = gimple_build_assign (tmp, NOP_EXPR,
 510                                     gimple_call_lhs (stmt));
 511       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 512       assign = gimple_build_assign (original_ref, tmp);
 513       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 514     }
 515   else
 516     {
 517       tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
 518       gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
 519       gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
 520
 521       tmp = make_temp_ssa_name (type, NULL, "time_profile");
 522       assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
 523                                     one);
 524       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 525       assign = gimple_build_assign (original_ref, tmp);
 526       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 527       assign = gimple_build_assign (tree_time_profiler_counter, tmp);
 528       gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
 529     }
 530 }
 531
 532 /* Output instructions as GIMPLE trees to increment the average histogram
 533    counter.  VALUE is the expression whose value is profiled.  TAG is the
 534    tag of the section for counters, BASE is offset of the counter position.  */
 535
 536 void
 537 gimple_gen_average_profiler (histogram_value value, unsigned tag, unsigned base)
 538 {
 539   gimple *stmt = value->hvalue.stmt;
 540   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 541   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 542   gcall *call;
 543   tree val;
 544
 545   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 546                                       true, NULL_TREE,
 547                                       true, GSI_SAME_STMT);
 548   val = prepare_instrumented_value (&gsi, value);
 549   call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
 550   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 551 }
 552
 553 /* Output instructions as GIMPLE trees to increment the ior histogram
 554    counter.  VALUE is the expression whose value is profiled.  TAG is the
 555    tag of the section for counters, BASE is offset of the counter position.  */
 556
 557 void
 558 gimple_gen_ior_profiler (histogram_value value, unsigned tag, unsigned base)
 559 {
 560   gimple *stmt = value->hvalue.stmt;
 561   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 562   tree ref_ptr = tree_coverage_counter_addr (tag, base);
 563   gcall *call;
 564   tree val;
 565
 566   ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
 567                                       true, NULL_TREE, true, GSI_SAME_STMT);
 568   val = prepare_instrumented_value (&gsi, value);
 569   call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
 570   gsi_insert_before (&gsi, call, GSI_NEW_STMT);
 571 }
 572
 573 #ifndef HAVE_sync_compare_and_swapsi
 574 #define HAVE_sync_compare_and_swapsi 0
 575 #endif
 576 #ifndef HAVE_atomic_compare_and_swapsi
 577 #define HAVE_atomic_compare_and_swapsi 0
 578 #endif
 579
 580 #ifndef HAVE_sync_compare_and_swapdi
 581 #define HAVE_sync_compare_and_swapdi 0
 582 #endif
 583 #ifndef HAVE_atomic_compare_and_swapdi
 584 #define HAVE_atomic_compare_and_swapdi 0
 585 #endif
 586
 587 /* Profile all functions in the callgraph.  */
 588
 589 static unsigned int
 590 tree_profiling (void)
 591 {
 592   struct cgraph_node *node;
 593
 594   /* Verify whether we can utilize atomic update operations.  */
 595   bool can_support_atomic = false;
 596   unsigned HOST_WIDE_INT gcov_type_size
 597     = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
 598   if (gcov_type_size == 4)
 599     can_support_atomic
 600       = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
 601   else if (gcov_type_size == 8)
 602     can_support_atomic
 603       = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
 604
 605   if (flag_profile_update == PROFILE_UPDATE_ATOMIC
 606       && !can_support_atomic)
 607     {
 608       warning (0, "target does not support atomic profile update, "
 609                "single mode is selected");
 610       flag_profile_update = PROFILE_UPDATE_SINGLE;
 611     }
 612   else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
 613     flag_profile_update = can_support_atomic
 614       ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
 615
 616   /* This is a small-ipa pass that gets called only once, from
 617      cgraphunit.c:ipa_passes().  */
 618   gcc_assert (symtab->state == IPA_SSA);
 619
 620   init_node_map (true);
 621
 622   FOR_EACH_DEFINED_FUNCTION (node)
 623     {
 624       if (!gimple_has_body_p (node->decl))
 625         continue;
 626
 627       /* Don't profile functions produced for builtin stuff.  */
 628       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 629         continue;
 630
 631       if (lookup_attribute ("no_profile_instrument_function",
 632                             DECL_ATTRIBUTES (node->decl)))
 633         continue;
 634       /* Do not instrument extern inline functions when testing coverage.
 635          While this is not perfectly consistent (early inlined extern inlines
 636          will get acocunted), testsuite expects that.  */
 637       if (DECL_EXTERNAL (node->decl)
 638           && flag_test_coverage)
 639         continue;
 640
 641       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 642
 643       /* Local pure-const may imply need to fixup the cfg.  */
 644       if (execute_fixup_cfg () & TODO_cleanup_cfg)
 645         cleanup_tree_cfg ();
 646
 647       branch_prob ();
 648
 649       if (! flag_branch_probabilities
 650           && flag_profile_values)
 651         gimple_gen_ic_func_profiler ();
 652
 653       if (flag_branch_probabilities
 654           && flag_profile_values
 655           && flag_value_profile_transformations)
 656         gimple_value_profile_transformations ();
 657
 658       /* The above could hose dominator info.  Currently there is
 659          none coming in, this is a safety valve.  It should be
 660          easy to adjust it, if and when there is some.  */
 661       free_dominance_info (CDI_DOMINATORS);
 662       free_dominance_info (CDI_POST_DOMINATORS);
 663       pop_cfun ();
 664     }
 665
 666   /* Drop pure/const flags from instrumented functions.  */
 667   if (profile_arc_flag || flag_test_coverage)
 668     FOR_EACH_DEFINED_FUNCTION (node)
 669       {
 670         if (!gimple_has_body_p (node->decl)
 671             || !(!node->clone_of
 672             || node->decl != node->clone_of->decl))
 673           continue;
 674
 675         /* Don't profile functions produced for builtin stuff.  */
 676         if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 677           continue;
 678
 679         node->set_const_flag (false, false);
 680         node->set_pure_flag (false, false);
 681       }
 682
 683   /* Update call statements and rebuild the cgraph.  */
 684   FOR_EACH_DEFINED_FUNCTION (node)
 685     {
 686       basic_block bb;
 687
 688       if (!gimple_has_body_p (node->decl)
 689           || !(!node->clone_of
 690           || node->decl != node->clone_of->decl))
 691         continue;
 692
 693       /* Don't profile functions produced for builtin stuff.  */
 694       if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
 695         continue;
 696
 697       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
 698
 699       FOR_EACH_BB_FN (bb, cfun)
 700         {
 701           gimple_stmt_iterator gsi;
 702           for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 703             {
 704               gimple *stmt = gsi_stmt (gsi);
 705               if (is_gimple_call (stmt))
 706                 update_stmt (stmt);
 707             }
 708         }
 709
 710       /* re-merge split blocks.  */
 711       cleanup_tree_cfg ();
 712       update_ssa (TODO_update_ssa);
 713
 714       cgraph_edge::rebuild_edges ();
 715
 716       pop_cfun ();
 717     }
 718
 719   handle_missing_profiles ();
 720
 721   del_node_map ();
 722   return 0;
 723 }
 724
 725 namespace {
 726
 727 const pass_data pass_data_ipa_tree_profile =
 728 {
 729   SIMPLE_IPA_PASS, /* type */
 730   "profile", /* name */
 731   OPTGROUP_NONE, /* optinfo_flags */
 732   TV_IPA_PROFILE, /* tv_id */
 733   0, /* properties_required */
 734   0, /* properties_provided */
 735   0, /* properties_destroyed */
 736   0, /* todo_flags_start */
 737   TODO_dump_symtab, /* todo_flags_finish */
 738 };
 739
 740 class pass_ipa_tree_profile : public simple_ipa_opt_pass
 741 {
 742 public:
 743   pass_ipa_tree_profile (gcc::context *ctxt)
 744     : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
 745   {}
 746
 747   /* opt_pass methods: */
 748   virtual bool gate (function *);
 749   virtual unsigned int execute (function *) { return tree_profiling (); }
 750
 751 }; // class pass_ipa_tree_profile
 752
 753 bool
 754 pass_ipa_tree_profile::gate (function *)
 755 {
 756   /* When profile instrumentation, use or test coverage shall be performed.
 757      But for AutoFDO, this there is no instrumentation, thus this pass is
 758      diabled.  */
 759   return (!in_lto_p && !flag_auto_profile
 760           && (flag_branch_probabilities || flag_test_coverage
 761               || profile_arc_flag));
 762 }
 763
 764 } // anon namespace
 765
 766 simple_ipa_opt_pass *
 767 make_pass_ipa_tree_profile (gcc::context *ctxt)
 768 {
 769   return new pass_ipa_tree_profile (ctxt);
 770 }
 771
 772 #include "gt-tree-profile.h"