sdk/platform-tools/systrace/catapult/telemetry/telemetry/benchmark.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import optparse
   6
   7 from telemetry import decorators
   8 from telemetry.internal import story_runner
   9 from telemetry.internal.util import command_line
  10 from telemetry.page import legacy_page_test
  11 from telemetry.web_perf import timeline_based_measurement
  12
  13 Disabled = decorators.Disabled
  14 Enabled = decorators.Enabled
  15 Owner = decorators.Owner
  16
  17 class InvalidOptionsError(Exception):
  18   """Raised for invalid benchmark options."""
  19   pass
  20
  21
  22 class BenchmarkMetadata(object):
  23   def __init__(self, name, description='', rerun_options=None):
  24     self._name = name
  25     self._description = description
  26     self._rerun_options = rerun_options
  27
  28   @property
  29   def name(self):
  30     return self._name
  31
  32   @property
  33   def description(self):
  34     return self._description
  35
  36   @property
  37   def rerun_options(self):
  38     return self._rerun_options
  39
  40   def AsDict(self):
  41     return {
  42       'type': 'telemetry_benchmark',
  43       'name': self._name,
  44       'description': self._description,
  45       'rerun_options': self._rerun_options,
  46     }
  47
  48
  49 class Benchmark(command_line.Command):
  50   """Base class for a Telemetry benchmark.
  51
  52   A benchmark packages a measurement and a PageSet together.
  53   Benchmarks default to using TBM unless you override the value of
  54   Benchmark.test, or override the CreatePageTest method.
  55
  56   New benchmarks should override CreateStorySet.
  57   """
  58   options = {}
  59   page_set = None
  60   test = timeline_based_measurement.TimelineBasedMeasurement
  61
  62   def __init__(self, max_failures=None):
  63     """Creates a new Benchmark.
  64
  65     Args:
  66       max_failures: The number of story run's failures before bailing
  67           from executing subsequent page runs. If None, we never bail.
  68     """
  69     self._max_failures = max_failures
  70     self._has_original_tbm_options = (
  71         self.CreateTimelineBasedMeasurementOptions.__func__ ==
  72         Benchmark.CreateTimelineBasedMeasurementOptions.__func__)
  73     has_original_create_page_test = (
  74         self.CreatePageTest.__func__ == Benchmark.CreatePageTest.__func__)
  75     assert self._has_original_tbm_options or has_original_create_page_test, (
  76         'Cannot override both CreatePageTest and '
  77         'CreateTimelineBasedMeasurementOptions.')
  78
  79   # pylint: disable=unused-argument
  80   @classmethod
  81   def ShouldDisable(cls, possible_browser):
  82     """Override this method to disable a benchmark under specific conditions.
  83
  84      Supports logic too complex for simple Enabled and Disabled decorators.
  85      Decorators are still respected in cases where this function returns False.
  86      """
  87     return False
  88
  89   def Run(self, finder_options):
  90     """Do not override this method."""
  91     return story_runner.RunBenchmark(self, finder_options)
  92
  93   @property
  94   def max_failures(self):
  95     return self._max_failures
  96
  97   @classmethod
  98   def Name(cls):
  99     return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
 100
 101   @classmethod
 102   def ShouldTearDownStateAfterEachStoryRun(cls):
 103     """Override to specify whether to tear down state after each story run.
 104
 105     Tearing down all states after each story run, e.g., clearing profiles,
 106     stopping the browser, stopping local server, etc. So the browser will not be
 107     reused among multiple stories. This is particularly useful to get the
 108     startup part of launching the browser in each story.
 109
 110     This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
 111     not by PageTest based benchmarks.
 112     """
 113     return True
 114
 115   # NOTE: this is a temporary workaround for crbug.com/645329, do not rely on
 116   # this as a stable public API as we may remove this without public notice.
 117   @classmethod
 118   def IsShouldTearDownStateAfterEachStoryRunOverriden(cls):
 119     return (cls.ShouldTearDownStateAfterEachStoryRun.__func__ !=
 120             Benchmark.ShouldTearDownStateAfterEachStoryRun.__func__)
 121
 122   @classmethod
 123   def ShouldTearDownStateAfterEachStorySetRun(cls):
 124     """Override to specify whether to tear down state after each story set run.
 125
 126     Defaults to True in order to reset the state and make individual story set
 127     repeats more independent of each other. The intended effect is to average
 128     out noise in measurements between repeats.
 129
 130     Long running benchmarks willing to stess test the browser and have it run
 131     for long periods of time may switch this value to False.
 132
 133     This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
 134     not by PageTest based benchmarks.
 135     """
 136     return True
 137
 138   @classmethod
 139   def AddCommandLineArgs(cls, parser):
 140     group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
 141     cls.AddBenchmarkCommandLineArgs(group)
 142
 143     if cls.HasTraceRerunDebugOption():
 144       group.add_option(
 145           '--rerun-with-debug-trace',
 146           action='store_true',
 147           help='Rerun option that enables more extensive tracing.')
 148
 149     if group.option_list:
 150       parser.add_option_group(group)
 151
 152   @classmethod
 153   def AddBenchmarkCommandLineArgs(cls, group):
 154     del group  # unused
 155
 156   @classmethod
 157   def HasTraceRerunDebugOption(cls):
 158     return False
 159
 160   def GetTraceRerunCommands(self):
 161     if self.HasTraceRerunDebugOption():
 162       return [['Debug Trace', '--rerun-with-debug-trace']]
 163     return []
 164
 165   def SetupTraceRerunOptions(self, browser_options, tbm_options):
 166     if self.HasTraceRerunDebugOption():
 167       if browser_options.rerun_with_debug_trace:
 168         self.SetupBenchmarkDebugTraceRerunOptions(tbm_options)
 169       else:
 170         self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options)
 171
 172   def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
 173     """Setup tracing categories associated with default trace option."""
 174
 175   def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
 176     """Setup tracing categories associated with debug trace option."""
 177
 178   @classmethod
 179   def SetArgumentDefaults(cls, parser):
 180     default_values = parser.get_default_values()
 181     invalid_options = [
 182         o for o in cls.options if not hasattr(default_values, o)]
 183     if invalid_options:
 184       raise InvalidOptionsError('Invalid benchmark options: %s',
 185                                 ', '.join(invalid_options))
 186     parser.set_defaults(**cls.options)
 187
 188   @classmethod
 189   def ProcessCommandLineArgs(cls, parser, args):
 190     pass
 191
 192   # pylint: disable=unused-argument
 193   @classmethod
 194   def ValueCanBeAddedPredicate(cls, value, is_first_result):
 195     """Returns whether |value| can be added to the test results.
 196
 197     Override this method to customize the logic of adding values to test
 198     results.
 199
 200     Args:
 201       value: a value.Value instance (except failure.FailureValue,
 202         skip.SkipValue or trace.TraceValue which will always be added).
 203       is_first_result: True if |value| is the first result for its
 204           corresponding story.
 205
 206     Returns:
 207       True if |value| should be added to the test results.
 208       Otherwise, it returns False.
 209     """
 210     return True
 211
 212   def CustomizeBrowserOptions(self, options):
 213     """Add browser options that are required by this benchmark."""
 214
 215   def GetMetadata(self):
 216     return BenchmarkMetadata(
 217         self.Name(), self.__doc__, self.GetTraceRerunCommands())
 218
 219   def CreateTimelineBasedMeasurementOptions(self):
 220     """Return the TimelineBasedMeasurementOptions for this Benchmark.
 221
 222     Override this method to configure a TimelineBasedMeasurement benchmark.
 223     Otherwise, override CreatePageTest for PageTest tests. Do not override
 224     both methods.
 225     """
 226     return timeline_based_measurement.Options()
 227
 228   def CreatePageTest(self, options):  # pylint: disable=unused-argument
 229     """Return the PageTest for this Benchmark.
 230
 231     Override this method for PageTest tests.
 232     Override, override CreateTimelineBasedMeasurementOptions to configure
 233     TimelineBasedMeasurement tests. Do not override both methods.
 234
 235     Args:
 236       options: a browser_options.BrowserFinderOptions instance
 237     Returns:
 238       |test()| if |test| is a PageTest class.
 239       Otherwise, a TimelineBasedMeasurement instance.
 240     """
 241     is_page_test = issubclass(self.test, legacy_page_test.LegacyPageTest)
 242     is_tbm = self.test == timeline_based_measurement.TimelineBasedMeasurement
 243     if not is_page_test and not is_tbm:
 244       raise TypeError('"%s" is not a PageTest or a TimelineBasedMeasurement.' %
 245                       self.test.__name__)
 246     if is_page_test:
 247       assert self._has_original_tbm_options, (
 248           'Cannot override CreateTimelineBasedMeasurementOptions '
 249           'with a PageTest.')
 250       return self.test()  # pylint: disable=no-value-for-parameter
 251
 252     opts = self.CreateTimelineBasedMeasurementOptions()
 253     self.SetupTraceRerunOptions(options, opts)
 254     return timeline_based_measurement.TimelineBasedMeasurement(opts)
 255
 256   def CreateStorySet(self, options):
 257     """Creates the instance of StorySet used to run the benchmark.
 258
 259     Can be overridden by subclasses.
 260     """
 261     del options  # unused
 262     # TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with
 263     # story_set.
 264     if not self.page_set:
 265       raise NotImplementedError('This test has no "page_set" attribute.')
 266     return self.page_set()  # pylint: disable=not-callable
 267
 268
 269 def AddCommandLineArgs(parser):
 270   story_runner.AddCommandLineArgs(parser)
 271
 272
 273 def ProcessCommandLineArgs(parser, args):
 274   story_runner.ProcessCommandLineArgs(parser, args)