contrib/analyze_brprob.py

   1 #!/usr/bin/env python3
   2 #
   3 # Script to analyze results of our branch prediction heuristics
   4 #
   5 # This file is part of GCC.
   6 #
   7 # GCC is free software; you can redistribute it and/or modify it under
   8 # the terms of the GNU General Public License as published by the Free
   9 # Software Foundation; either version 3, or (at your option) any later
  10 # version.
  11 #
  12 # GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 # for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with GCC; see the file COPYING3.  If not see
  19 # <http://www.gnu.org/licenses/>.  */
  20 #
  21 #
  22 #
  23 # This script is used to calculate two basic properties of the branch prediction
  24 # heuristics - coverage and hitrate.  Coverage is number of executions
  25 # of a given branch matched by the heuristics and hitrate is probability
  26 # that once branch is predicted as taken it is really taken.
  27 #
  28 # These values are useful to determine the quality of given heuristics.
  29 # Hitrate may be directly used in predict.def.
  30 #
  31 # Usage:
  32 #  Step 1: Compile and profile your program.  You need to use -fprofile-generate
  33 #    flag to get the profiles.
  34 #  Step 2: Make a reference run of the intrumented application.
  35 #  Step 3: Compile the program with collected profile and dump IPA profiles
  36 #          (-fprofile-use -fdump-ipa-profile-details)
  37 #  Step 4: Collect all generated dump files:
  38 #          find . -name '*.profile' | xargs cat > dump_file
  39 #  Step 5: Run the script:
  40 #          ./analyze_brprob.py dump_file
  41 #          and read results.  Basically the following table is printed:
  42 #
  43 # HEURISTICS                           BRANCHES  (REL)  HITRATE                COVERAGE  (REL)
  44 # early return (on trees)                     3   0.2%  35.83% /  93.64%          66360   0.0%
  45 # guess loop iv compare                       8   0.6%  53.35% /  53.73%       11183344   0.0%
  46 # call                                       18   1.4%  31.95% /  69.95%       51880179   0.2%
  47 # loop guard                                 23   1.8%  84.13% /  84.85%    13749065956  42.2%
  48 # opcode values positive (on trees)          42   3.3%  15.71% /  84.81%     6771097902  20.8%
  49 # opcode values nonequal (on trees)         226  17.6%  72.48% /  72.84%      844753864   2.6%
  50 # loop exit                                 231  18.0%  86.97% /  86.98%     8952666897  27.5%
  51 # loop iterations                           239  18.6%  91.10% /  91.10%     3062707264   9.4%
  52 # DS theory                                 281  21.9%  82.08% /  83.39%     7787264075  23.9%
  53 # no prediction                             293  22.9%  46.92% /  70.70%     2293267840   7.0%
  54 # guessed loop iterations                   313  24.4%  76.41% /  76.41%    10782750177  33.1%
  55 # first match                               708  55.2%  82.30% /  82.31%    22489588691  69.0%
  56 # combined                                 1282 100.0%  79.76% /  81.75%    32570120606 100.0%
  57 #
  58 #
  59 #  The heuristics called "first match" is a heuristics used by GCC branch
  60 #  prediction pass and it predicts 55.2% branches correctly. As you can,
  61 #  the heuristics has very good covertage (69.05%).  On the other hand,
  62 #  "opcode values nonequal (on trees)" heuristics has good hirate, but poor
  63 #  coverage.
  64
  65 import sys
  66 import os
  67 import re
  68 import argparse
  69
  70 from math import *
  71
  72 counter_aggregates = set(['combined', 'first match', 'DS theory',
  73     'no prediction'])
  74 hot_threshold = 10
  75
  76 def percentage(a, b):
  77     return 100.0 * a / b
  78
  79 def average(values):
  80     return 1.0 * sum(values) / len(values)
  81
  82 def average_cutoff(values, cut):
  83     l = len(values)
  84     skip = floor(l * cut / 2)
  85     if skip > 0:
  86         values.sort()
  87         values = values[skip:-skip]
  88     return average(values)
  89
  90 def median(values):
  91     values.sort()
  92     return values[int(len(values) / 2)]
  93
  94 class PredictDefFile:
  95     def __init__(self, path):
  96         self.path = path
  97         self.predictors = {}
  98
  99     def parse_and_modify(self, heuristics, write_def_file):
 100         lines = [x.rstrip() for x in open(self.path).readlines()]
 101
 102         p = None
 103         modified_lines = []
 104         for l in lines:
 105             if l.startswith('DEF_PREDICTOR'):
 106                 m = re.match('.*"(.*)".*', l)
 107                 p = m.group(1)
 108             elif l == '':
 109                 p = None
 110
 111             if p != None:
 112                 heuristic = [x for x in heuristics if x.name == p]
 113                 heuristic = heuristic[0] if len(heuristic) == 1 else None
 114
 115                 m = re.match('.*HITRATE \(([^)]*)\).*', l)
 116                 if (m != None):
 117                     self.predictors[p] = int(m.group(1))
 118
 119                     # modify the line
 120                     if heuristic != None:
 121                         new_line = (l[:m.start(1)]
 122                             + str(round(heuristic.get_hitrate()))
 123                             + l[m.end(1):])
 124                         l = new_line
 125                     p = None
 126                 elif 'PROB_VERY_LIKELY' in l:
 127                     self.predictors[p] = 100
 128             modified_lines.append(l)
 129
 130         # save the file
 131         if write_def_file:
 132             with open(self.path, 'w+') as f:
 133                 for l in modified_lines:
 134                     f.write(l + '\n')
 135 class Heuristics:
 136     def __init__(self, count, hits, fits):
 137         self.count = count
 138         self.hits = hits
 139         self.fits = fits
 140
 141 class Summary:
 142     def __init__(self, name):
 143         self.name = name
 144         self.edges= []
 145
 146     def branches(self):
 147         return len(self.edges)
 148
 149     def hits(self):
 150         return sum([x.hits for x in self.edges])
 151
 152     def fits(self):
 153         return sum([x.fits for x in self.edges])
 154
 155     def count(self):
 156         return sum([x.count for x in self.edges])
 157
 158     def successfull_branches(self):
 159         return len([x for x in self.edges if 2 * x.hits >= x.count])
 160
 161     def get_hitrate(self):
 162         return 100.0 * self.hits() / self.count()
 163
 164     def get_branch_hitrate(self):
 165         return 100.0 * self.successfull_branches() / self.branches()
 166
 167     def count_formatted(self):
 168         v = self.count()
 169         for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']:
 170             if v < 1000:
 171                 return "%3.2f%s" % (v, unit)
 172             v /= 1000.0
 173         return "%.1f%s" % (v, 'Y')
 174
 175     def count(self):
 176         return sum([x.count for x in self.edges])
 177
 178     def print(self, branches_max, count_max, predict_def):
 179         # filter out most hot edges (if requested)
 180         self.edges = sorted(self.edges, reverse = True, key = lambda x: x.count)
 181         if args.coverage_threshold != None:
 182             threshold = args.coverage_threshold * self.count() / 100
 183             edges = [x for x in self.edges if x.count < threshold]
 184             if len(edges) != 0:
 185                 self.edges = edges
 186
 187         predicted_as = None
 188         if predict_def != None and self.name in predict_def.predictors:
 189             predicted_as = predict_def.predictors[self.name]
 190
 191         print('%-40s %8i %5.1f%% %11.2f%% %7.2f%% / %6.2f%% %14i %8s %5.1f%%' %
 192             (self.name, self.branches(),
 193                 percentage(self.branches(), branches_max),
 194                 self.get_branch_hitrate(),
 195                 self.get_hitrate(),
 196                 percentage(self.fits(), self.count()),
 197                 self.count(), self.count_formatted(),
 198                 percentage(self.count(), count_max)), end = '')
 199
 200         if predicted_as != None:
 201             print('%12i%% %5.1f%%' % (predicted_as,
 202                 self.get_hitrate() - predicted_as), end = '')
 203         else:
 204             print(' ' * 20, end = '')
 205
 206         # print details about the most important edges
 207         if args.coverage_threshold == None:
 208             edges = [x for x in self.edges[:100] if x.count * hot_threshold > self.count()]
 209             if args.verbose:
 210                 for c in edges:
 211                     r = 100.0 * c.count / self.count()
 212                     print(' %.0f%%:%d' % (r, c.count), end = '')
 213             elif len(edges) > 0:
 214                 print(' %0.0f%%:%d' % (100.0 * sum([x.count for x in edges]) / self.count(), len(edges)), end = '')
 215
 216         print()
 217
 218 class Profile:
 219     def __init__(self, filename):
 220         self.filename = filename
 221         self.heuristics = {}
 222         self.niter_vector = []
 223
 224     def add(self, name, prediction, count, hits):
 225         if not name in self.heuristics:
 226             self.heuristics[name] = Summary(name)
 227
 228         s = self.heuristics[name]
 229
 230         if prediction < 50:
 231             hits = count - hits
 232         remaining = count - hits
 233         fits = max(hits, remaining)
 234
 235         s.edges.append(Heuristics(count, hits, fits))
 236
 237     def add_loop_niter(self, niter):
 238         if niter > 0:
 239             self.niter_vector.append(niter)
 240
 241     def branches_max(self):
 242         return max([v.branches() for k, v in self.heuristics.items()])
 243
 244     def count_max(self):
 245         return max([v.count() for k, v in self.heuristics.items()])
 246
 247     def print_group(self, sorting, group_name, heuristics, predict_def):
 248         count_max = self.count_max()
 249         branches_max = self.branches_max()
 250
 251         sorter = lambda x: x.branches()
 252         if sorting == 'branch-hitrate':
 253             sorter = lambda x: x.get_branch_hitrate()
 254         elif sorting == 'hitrate':
 255             sorter = lambda x: x.get_hitrate()
 256         elif sorting == 'coverage':
 257             sorter = lambda x: x.count
 258         elif sorting == 'name':
 259             sorter = lambda x: x.name.lower()
 260
 261         print('%-40s %8s %6s %12s %18s %14s %8s %6s %12s %6s %s' %
 262             ('HEURISTICS', 'BRANCHES', '(REL)',
 263             'BR. HITRATE', 'HITRATE', 'COVERAGE', 'COVERAGE', '(REL)',
 264             'predict.def', '(REL)', 'HOT branches (>%d%%)' % hot_threshold))
 265         for h in sorted(heuristics, key = sorter):
 266             h.print(branches_max, count_max, predict_def)
 267
 268     def dump(self, sorting):
 269         heuristics = self.heuristics.values()
 270         if len(heuristics) == 0:
 271             print('No heuristics available')
 272             return
 273
 274         predict_def = None
 275         if args.def_file != None:
 276             predict_def = PredictDefFile(args.def_file)
 277             predict_def.parse_and_modify(heuristics, args.write_def_file)
 278
 279         special = list(filter(lambda x: x.name in counter_aggregates,
 280             heuristics))
 281         normal = list(filter(lambda x: x.name not in counter_aggregates,
 282             heuristics))
 283
 284         self.print_group(sorting, 'HEURISTICS', normal, predict_def)
 285         print()
 286         self.print_group(sorting, 'HEURISTIC AGGREGATES', special, predict_def)
 287
 288         if len(self.niter_vector) > 0:
 289             print ('\nLoop count: %d' % len(self.niter_vector)),
 290             print('  avg. # of iter: %.2f' % average(self.niter_vector))
 291             print('  median # of iter: %.2f' % median(self.niter_vector))
 292             for v in [1, 5, 10, 20, 30]:
 293                 cut = 0.01 * v
 294                 print('  avg. (%d%% cutoff) # of iter: %.2f'
 295                     % (v, average_cutoff(self.niter_vector, cut)))
 296
 297 parser = argparse.ArgumentParser()
 298 parser.add_argument('dump_file', metavar = 'dump_file',
 299     help = 'IPA profile dump file')
 300 parser.add_argument('-s', '--sorting', dest = 'sorting',
 301     choices = ['branches', 'branch-hitrate', 'hitrate', 'coverage', 'name'],
 302     default = 'branches')
 303 parser.add_argument('-d', '--def-file', help = 'path to predict.def')
 304 parser.add_argument('-w', '--write-def-file', action = 'store_true',
 305     help = 'Modify predict.def file in order to set new numbers')
 306 parser.add_argument('-c', '--coverage-threshold', type = int,
 307     help = 'Ignore edges that have percentage coverage >= coverage-threshold')
 308 parser.add_argument('-v', '--verbose', action = 'store_true', help = 'Print verbose informations')
 309
 310 args = parser.parse_args()
 311
 312 profile = Profile(args.dump_file)
 313 loop_niter_str = ';;  profile-based iteration count: '
 314
 315 for l in open(args.dump_file):
 316     if l.startswith(';;heuristics;'):
 317         parts = l.strip().split(';')
 318         assert len(parts) == 8
 319         name = parts[3]
 320         prediction = float(parts[6])
 321         count = int(parts[4])
 322         hits = int(parts[5])
 323
 324         profile.add(name, prediction, count, hits)
 325     elif l.startswith(loop_niter_str):
 326         v = int(l[len(loop_niter_str):])
 327         profile.add_loop_niter(v)
 328
 329 profile.dump(args.sorting)