sdk/platform-tools/systrace/catapult/telemetry/telemetry/web_perf/timeline_based_page_test_unittest.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 from telemetry import decorators
   6 from telemetry.page import page as page_module
   7 from telemetry.testing import browser_test_case
   8 from telemetry.testing import options_for_unittests
   9 from telemetry.testing import page_test_test_case
  10 from telemetry.timeline import chrome_trace_category_filter
  11 from telemetry.util import wpr_modes
  12 from telemetry.web_perf import timeline_based_measurement as tbm_module
  13 from telemetry.web_perf.metrics import gpu_timeline
  14 from telemetry.web_perf.metrics import smoothness
  15
  16 class TestTimelinebasedMeasurementPage(page_module.Page):
  17
  18   def __init__(self, ps, base_dir, trigger_animation=False,
  19                trigger_jank=False, trigger_slow=False,
  20                trigger_scroll_gesture=False):
  21     super(TestTimelinebasedMeasurementPage, self).__init__(
  22         'file://interaction_enabled_page.html', ps, base_dir)
  23     self._trigger_animation = trigger_animation
  24     self._trigger_jank = trigger_jank
  25     self._trigger_slow = trigger_slow
  26     self._trigger_scroll_gesture = trigger_scroll_gesture
  27
  28   def RunPageInteractions(self, action_runner):
  29     if self._trigger_animation:
  30       action_runner.TapElement('#animating-button')
  31       action_runner.WaitForJavaScriptCondition('window.animationDone')
  32     if self._trigger_jank:
  33       action_runner.TapElement('#jank-button')
  34       action_runner.WaitForJavaScriptCondition('window.jankScriptDone')
  35     if self._trigger_slow:
  36       action_runner.TapElement('#slow-button')
  37       action_runner.WaitForJavaScriptCondition('window.slowScriptDone')
  38     if self._trigger_scroll_gesture:
  39       with action_runner.CreateGestureInteraction('Scroll'):
  40         action_runner.ScrollPage()
  41
  42
  43 class TimelineBasedPageTestTest(page_test_test_case.PageTestTestCase):
  44
  45   def setUp(self):
  46     browser_test_case.teardown_browser()
  47     self._options = options_for_unittests.GetCopy()
  48     self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF
  49
  50   # This test is flaky when run in parallel on the mac: crbug.com/426676
  51   # Also, fails on android: crbug.com/437057, and chromeos: crbug.com/483212
  52   @decorators.Disabled('android', 'mac', 'chromeos')
  53   @decorators.Disabled('win')  # catapult/issues/2282
  54   @decorators.Isolated  # Needed because of py_trace_event
  55   def testSmoothnessTimelineBasedMeasurementForSmoke(self):
  56     ps = self.CreateEmptyPageSet()
  57     ps.AddStory(TestTimelinebasedMeasurementPage(
  58         ps, ps.base_dir, trigger_animation=True))
  59
  60     options = tbm_module.Options()
  61     options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()])
  62     tbm = tbm_module.TimelineBasedMeasurement(options)
  63     results = self.RunMeasurement(tbm, ps, options=self._options)
  64
  65     self.assertEquals(0, len(results.failures))
  66     v = results.FindAllPageSpecificValuesFromIRNamed(
  67         'CenterAnimation', 'frame_time_discrepancy')
  68     self.assertEquals(len(v), 1)
  69     v = results.FindAllPageSpecificValuesFromIRNamed(
  70         'DrawerAnimation', 'frame_time_discrepancy')
  71     self.assertEquals(len(v), 1)
  72
  73   # This test should eventually work on all platforms, but currently this
  74   # this metric is flaky on desktop: crbug.com/453131
  75   # https://github.com/catapult-project/catapult/issues/3099 (Android)
  76   @decorators.Disabled('all')
  77   def testGPUTimesTimelineBasedMeasurementForSmoke(self):
  78     ps = self.CreateEmptyPageSet()
  79     ps.AddStory(TestTimelinebasedMeasurementPage(
  80         ps, ps.base_dir, trigger_animation=True))
  81
  82     cat_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
  83         'disabled-by-default-gpu.service')
  84     tbm_option = tbm_module.Options(overhead_level=cat_filter)
  85     tbm_option.SetLegacyTimelineBasedMetrics([gpu_timeline.GPUTimelineMetric()])
  86     tbm = tbm_module.TimelineBasedMeasurement(tbm_option)
  87     results = self.RunMeasurement(tbm, ps, options=self._options)
  88
  89     self.assertEquals(0, len(results.failures))
  90     v = results.FindAllPageSpecificValuesFromIRNamed(
  91         'CenterAnimation', 'browser_compositor_max_cpu_time')
  92     self.assertEquals(len(v), 1)
  93     self.assertGreater(v[0].value, 0)
  94     v = results.FindAllPageSpecificValuesFromIRNamed(
  95         'DrawerAnimation', 'browser_compositor_max_cpu_time')
  96     self.assertEquals(len(v), 1)
  97     self.assertGreater(v[0].value, 0)
  98
  99   # win: crbug.com/520781, chromeos: crbug.com/483212.
 100   @decorators.Disabled('win', 'chromeos')
 101   @decorators.Isolated  # Needed because of py_trace_event
 102   def testTimelineBasedMeasurementGestureAdjustmentSmoke(self):
 103     ps = self.CreateEmptyPageSet()
 104     ps.AddStory(TestTimelinebasedMeasurementPage(
 105         ps, ps.base_dir, trigger_scroll_gesture=True))
 106
 107     options = tbm_module.Options()
 108     options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()])
 109     tbm = tbm_module.TimelineBasedMeasurement(options)
 110     results = self.RunMeasurement(tbm, ps, options=self._options)
 111
 112     self.assertEquals(0, len(results.failures))
 113     v = results.FindAllPageSpecificValuesFromIRNamed(
 114         'Gesture_Scroll', 'frame_time_discrepancy')
 115     self.assertEquals(len(v), 1)
 116
 117   # Fails on chromeos: crbug.com/483212
 118   @decorators.Disabled('chromeos')
 119   @decorators.Isolated
 120   def testTBM2ForSmoke(self):
 121     ps = self.CreateEmptyPageSet()
 122     ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir))
 123
 124     options = tbm_module.Options()
 125     options.config.enable_chrome_trace = True
 126     options.SetTimelineBasedMetrics(['sampleMetric'])
 127
 128     tbm = tbm_module.TimelineBasedMeasurement(options)
 129     results = self.RunMeasurement(tbm, ps, self._options)
 130
 131     self.assertEquals(0, len(results.failures))
 132     self.assertEquals(2, len(results.value_set))
 133     diagnostics = results.value_set[1]['diagnostics']
 134     self.assertEquals(1, len(diagnostics))
 135     telemetry_info = results.value_set[0]
 136     self.assertEquals(telemetry_info['guid'], diagnostics['telemetry'])
 137     self.assertEqual('TelemetryInfo', telemetry_info['type'])
 138     self.assertEqual('', telemetry_info['benchmarkName'])
 139     self.assertEqual('interaction_enabled_page.html',
 140                      telemetry_info['storyDisplayName'])
 141     self.assertNotIn('storyGroupingKeys', telemetry_info)
 142     self.assertEqual(0, telemetry_info['storysetRepeatCounter'])
 143     v_foo = results.FindAllPageSpecificValuesNamed('foo_avg')
 144     self.assertEquals(len(v_foo), 1)
 145     self.assertEquals(v_foo[0].value, 50)
 146     self.assertIsNotNone(v_foo[0].page)
 147
 148   @decorators.Disabled('chromeos')
 149   def testFirstPaintMetricSmoke(self):
 150     ps = self.CreateEmptyPageSet()
 151     ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir))
 152
 153     cat_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
 154         filter_string='*,blink.console,navigation,blink.user_timing,loading,' +
 155         'devtools.timeline,disabled-by-default-blink.debug.layout')
 156
 157     options = tbm_module.Options(overhead_level=cat_filter)
 158     options.SetTimelineBasedMetrics(['loadingMetric'])
 159
 160     tbm = tbm_module.TimelineBasedMeasurement(options)
 161     results = self.RunMeasurement(tbm, ps, self._options)
 162
 163     self.assertEquals(0, len(results.failures), results.failures)
 164     v_ttfcp_max = results.FindAllPageSpecificValuesNamed(
 165         'timeToFirstContentfulPaint_max')
 166     self.assertEquals(len(v_ttfcp_max), 1)
 167     self.assertIsNotNone(v_ttfcp_max[0].page)
 168     # TODO(kouhei): enable this once the reference build of telemetry is
 169     # updated.
 170     #  self.assertGreater(v_ttfcp_max[0].value, 0)
 171
 172     v_ttfmp_max = results.FindAllPageSpecificValuesNamed(
 173        'timeToFirstMeaningfulPaint_max')
 174     self.assertEquals(len(v_ttfmp_max), 1)
 175     # TODO(ksakamoto): enable this once the reference build of telemetry is
 176     # updated.
 177     # self.assertIsNotNone(v_ttfmp_max[0].page)