tools/auto_bisect/bisect_printer.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This file contains printing-related functionality of the bisect."""
   6
   7 import datetime
   8 import re
   9
  10 from bisect_results import BisectResults
  11 import bisect_utils
  12 import source_control
  13
  14
  15 # The perf dashboard looks for a string like "Estimated Confidence: 95%"
  16 # to decide whether or not to cc the author(s). If you change this, please
  17 # update the perf dashboard as well.
  18 RESULTS_BANNER = """
  19 ===== BISECT JOB RESULTS =====
  20 Status: %(status)s
  21
  22 Test Command: %(command)s
  23 Test Metric: %(metric)s
  24 Relative Change: %(change)s
  25 Estimated Confidence: %(confidence).02f%%
  26 Retested CL with revert: %(retest)s"""
  27
  28 # When the bisect was aborted without a bisect failure the following template
  29 # is used.
  30 ABORT_REASON_TEMPLATE = """
  31 ===== BISECTION ABORTED =====
  32 The bisect was aborted because %(abort_reason)s
  33 Please contact the the team (see below) if you believe this is in error.
  34
  35 Bug ID: %(bug_id)s
  36
  37 Test Command: %(command)s
  38 Test Metric: %(metric)s
  39 Good revision: %(good_revision)s
  40 Bad revision: %(bad_revision)s """
  41
  42 # The perf dashboard specifically looks for the string
  43 # "Author  : " to parse out who to cc on a bug. If you change the
  44 # formatting here, please update the perf dashboard as well.
  45 RESULTS_REVISION_INFO = """
  46 ===== SUSPECTED CL(s) =====
  47 Subject : %(subject)s
  48 Author  : %(author)s%(commit_info)s
  49 Commit  : %(cl)s
  50 Date    : %(cl_date)s"""
  51
  52 RESULTS_THANKYOU = """
  53 | O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq
  54 |  X  | for more information addressing perf regression bugs. For feedback,
  55 | / \\ | file a bug with label Cr-Tests-AutoBisect.  Thank you!"""
  56
  57
  58 class BisectPrinter(object):
  59
  60   def __init__(self, opts, depot_registry=None):
  61     self.opts = opts
  62     self.depot_registry = depot_registry
  63
  64   def FormatAndPrintResults(self, bisect_results):
  65     """Prints the results from a bisection run in a readable format.
  66
  67     Also prints annotations creating buildbot step "Results".
  68
  69     Args:
  70       bisect_results: BisectResult object containing results to be printed.
  71     """
  72     if bisect_results.abort_reason:
  73       self._PrintAbortResults(bisect_results.abort_reason)
  74       return
  75
  76     if self.opts.output_buildbot_annotations:
  77       bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
  78
  79     print
  80     print 'Full results of bisection:'
  81     for revision_state in bisect_results.state.GetRevisionStates():
  82       build_status = revision_state.passed
  83
  84       if type(build_status) is bool:
  85         if build_status:
  86           build_status = 'Good'
  87         else:
  88           build_status = 'Bad'
  89
  90       print '  %20s  %40s  %s' % (revision_state.depot,
  91                                   revision_state.revision,
  92                                   build_status)
  93     print
  94
  95     if self.opts.output_buildbot_annotations:
  96       bisect_utils.OutputAnnotationStepClosed()
  97       # The perf dashboard scrapes the "results" step in order to comment on
  98       # bugs. If you change this, please update the perf dashboard as well.
  99       bisect_utils.OutputAnnotationStepStart('Results')
 100
 101     self._PrintBanner(bisect_results)
 102     self._PrintWarnings(bisect_results.warnings)
 103
 104     if bisect_results.culprit_revisions and bisect_results.confidence:
 105       for culprit in bisect_results.culprit_revisions:
 106         cl, info, depot = culprit
 107         self._PrintRevisionInfo(cl, info, depot)
 108     self._PrintRetestResults(bisect_results)
 109     self._PrintTestedCommitsTable(bisect_results.state.GetRevisionStates(),
 110                                   bisect_results.first_working_revision,
 111                                   bisect_results.last_broken_revision,
 112                                   bisect_results.confidence,
 113                                   final_step=True)
 114     self._PrintStepTime(bisect_results.state.GetRevisionStates())
 115     self._PrintThankYou()
 116     if self.opts.output_buildbot_annotations:
 117       bisect_utils.OutputAnnotationStepClosed()
 118
 119   def PrintPartialResults(self, bisect_state):
 120     revision_states = bisect_state.GetRevisionStates()
 121     first_working_rev, last_broken_rev = BisectResults.FindBreakingRevRange(
 122         revision_states)
 123     self._PrintTestedCommitsTable(revision_states, first_working_rev,
 124                                   last_broken_rev, 100, final_step=False)
 125
 126   def _PrintAbortResults(self, abort_reason):
 127     if self.opts.output_buildbot_annotations:
 128       bisect_utils.OutputAnnotationStepStart('Results')
 129
 130     # Metric string in config is not split in case of return code mode.
 131     if (self.opts.metric and
 132         self.opts.bisect_mode != bisect_utils.BISECT_MODE_RETURN_CODE):
 133       metric = '/'.join(self.opts.metric)
 134     else:
 135       metric = self.opts.metric
 136
 137     print ABORT_REASON_TEMPLATE % {
 138         'abort_reason': abort_reason,
 139         'bug_id': self.opts.bug_id or 'NOT SPECIFIED',
 140         'command': self.opts.command,
 141         'metric': metric,
 142         'good_revision': self.opts.good_revision,
 143         'bad_revision': self.opts.bad_revision,
 144     }
 145     self._PrintThankYou()
 146     if self.opts.output_buildbot_annotations:
 147       bisect_utils.OutputAnnotationStepClosed()
 148
 149   @staticmethod
 150   def _PrintThankYou():
 151     print RESULTS_THANKYOU
 152
 153   @staticmethod
 154   def _PrintStepTime(revision_states):
 155     """Prints information about how long various steps took.
 156
 157     Args:
 158       revision_states: Ordered list of revision states."""
 159     step_perf_time_avg = 0.0
 160     step_build_time_avg = 0.0
 161     step_count = 0.0
 162     for revision_state in revision_states:
 163       if revision_state.value:
 164         step_perf_time_avg += revision_state.perf_time
 165         step_build_time_avg += revision_state.build_time
 166         step_count += 1
 167     if step_count:
 168       step_perf_time_avg = step_perf_time_avg / step_count
 169       step_build_time_avg = step_build_time_avg / step_count
 170     print
 171     print 'Average build time : %s' % datetime.timedelta(
 172         seconds=int(step_build_time_avg))
 173     print 'Average test time  : %s' % datetime.timedelta(
 174         seconds=int(step_perf_time_avg))
 175
 176   @staticmethod
 177   def _GetViewVCLinkFromDepotAndHash(git_revision, depot):
 178     """Gets link to the repository browser."""
 179     if depot and 'viewvc' in bisect_utils.DEPOT_DEPS_NAME[depot]:
 180       return bisect_utils.DEPOT_DEPS_NAME[depot]['viewvc'] + git_revision
 181     return ''
 182
 183   def _PrintRevisionInfo(self, cl, info, depot=None):
 184     commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
 185     if commit_link:
 186       commit_link = '\nLink    : %s' % commit_link
 187     else:
 188       commit_link = ('\Description:\n%s' % info['body'])
 189     print RESULTS_REVISION_INFO % {
 190         'subject': info['subject'],
 191         'author': info['email'],
 192         'commit_info': commit_link,
 193         'cl': cl,
 194         'cl_date': info['date']
 195     }
 196
 197   @staticmethod
 198   def _PrintTableRow(column_widths, row_data):
 199     """Prints out a row in a formatted table that has columns aligned.
 200
 201     Args:
 202       column_widths: A list of column width numbers.
 203       row_data: A list of items for each column in this row.
 204     """
 205     assert len(column_widths) == len(row_data)
 206     text = ''
 207     for i in xrange(len(column_widths)):
 208       current_row_data = row_data[i].center(column_widths[i], ' ')
 209       text += ('%%%ds' % column_widths[i]) % current_row_data
 210     print text
 211
 212   def _PrintTestedCommitsHeader(self):
 213     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
 214       self._PrintTableRow(
 215           [20, 12, 70, 14, 12, 13],
 216           ['Depot', 'Position', 'SHA', 'Mean', 'Std. Error', 'State'])
 217     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
 218       self._PrintTableRow(
 219           [20, 12, 70, 14, 12, 13],
 220           ['Depot', 'Position', 'SHA', 'Std. Error', 'Mean', 'State'])
 221     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 222       self._PrintTableRow(
 223           [20, 12, 70, 14, 13],
 224           ['Depot', 'Position', 'SHA', 'Return Code', 'State'])
 225     else:
 226       assert False, 'Invalid bisect_mode specified.'
 227
 228   def _PrintTestedCommitsEntry(self, revision_state, commit_position, cl_link,
 229                                state_str):
 230     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_MEAN:
 231       std_error = '+-%.02f' % revision_state.value['std_err']
 232       mean = '%.02f' % revision_state.value['mean']
 233       self._PrintTableRow(
 234           [20, 12, 70, 12, 14, 13],
 235           [revision_state.depot, commit_position, cl_link, mean, std_error,
 236            state_str])
 237     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_STD_DEV:
 238       std_error = '+-%.02f' % revision_state.value['std_err']
 239       mean = '%.02f' % revision_state.value['mean']
 240       self._PrintTableRow(
 241           [20, 12, 70, 12, 14, 13],
 242           [revision_state.depot, commit_position, cl_link, std_error, mean,
 243            state_str])
 244     elif self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 245       mean = '%d' % revision_state.value['mean']
 246       self._PrintTableRow(
 247           [20, 12, 70, 14, 13],
 248           [revision_state.depot, commit_position, cl_link, mean,
 249            state_str])
 250
 251   def _PrintTestedCommitsTable(
 252       self, revision_states, first_working_revision, last_broken_revision,
 253       confidence, final_step=True):
 254     print
 255     if final_step:
 256       print '===== TESTED COMMITS ====='
 257     else:
 258       print '===== PARTIAL RESULTS ====='
 259     self._PrintTestedCommitsHeader()
 260     state = 0
 261     for revision_state in revision_states:
 262       if revision_state.value:
 263         if (revision_state == last_broken_revision or
 264             revision_state == first_working_revision):
 265           # If confidence is too low, don't add this empty line since it's
 266           # used to put focus on a suspected CL.
 267           if confidence and final_step:
 268             print
 269           state += 1
 270           if state == 2 and not final_step:
 271             # Just want a separation between "bad" and "good" cl's.
 272             print
 273
 274         state_str = 'Bad'
 275         if state == 1 and final_step:
 276           state_str = 'Suspected CL'
 277         elif state == 2:
 278           state_str = 'Good'
 279
 280         # If confidence is too low, don't bother outputting good/bad.
 281         if not confidence:
 282           state_str = ''
 283         state_str = state_str.center(13, ' ')
 284         commit_position = source_control.GetCommitPosition(
 285             revision_state.revision,
 286             self.depot_registry.GetDepotDir(revision_state.depot))
 287         display_commit_pos = ''
 288         if commit_position:
 289           display_commit_pos = str(commit_position)
 290         self._PrintTestedCommitsEntry(revision_state,
 291                                       display_commit_pos,
 292                                       revision_state.revision,
 293                                       state_str)
 294
 295   def _PrintRetestResults(self, bisect_results):
 296     if (not bisect_results.retest_results_tot or
 297         not bisect_results.retest_results_reverted):
 298       return
 299     print
 300     print '===== RETEST RESULTS ====='
 301     self._PrintTestedCommitsEntry(
 302         bisect_results.retest_results_tot, '', '', '')
 303     self._PrintTestedCommitsEntry(
 304         bisect_results.retest_results_reverted, '', '', '')
 305
 306   def _PrintBanner(self, bisect_results):
 307     if self.opts.bisect_mode == bisect_utils.BISECT_MODE_RETURN_CODE:
 308       metric = 'N/A'
 309       change = 'Yes'
 310     else:
 311       metric = '/'.join(self.opts.metric)
 312       change = '%.02f%% (+/-%.02f%%)' % (
 313           bisect_results.regression_size, bisect_results.regression_std_err)
 314     if not bisect_results.culprit_revisions:
 315       change = 'No significant change reproduced.'
 316
 317     print RESULTS_BANNER % {
 318         'status': self._StatusMessage(bisect_results),
 319         'command': self.opts.command,
 320         'metric': metric,
 321         'change': change,
 322         'confidence': bisect_results.confidence,
 323         'retest': 'Yes' if bisect_results.retest_results_tot else 'No',
 324     }
 325
 326   @staticmethod
 327   def _StatusMessage(bisect_results):
 328     if bisect_results.confidence >= bisect_utils.HIGH_CONFIDENCE:
 329       return 'Positive: Reproduced a change.'
 330     elif bisect_results.culprit_revisions:
 331       return 'Negative: Found possible suspect(s), but with low confidence.'
 332     return 'Negative: Did not reproduce a change.'
 333
 334   @staticmethod
 335   def _PrintWarnings(warnings):
 336     """Prints a list of warning strings if there are any."""
 337     if not warnings:
 338       return
 339     print
 340     print 'WARNINGS:'
 341     for w in set(warnings):
 342       print '  ! %s' % w