sdk/platform-tools/systrace/catapult/telemetry/telemetry/web_perf/metrics/smoothness.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6
   7 from telemetry.util import perf_tests_helper
   8 from telemetry.util import statistics
   9 from telemetry.value import improvement_direction
  10 from telemetry.value import list_of_scalar_values
  11 from telemetry.value import scalar
  12 from telemetry.web_perf.metrics import rendering_stats
  13 from telemetry.web_perf.metrics import timeline_based_metric
  14
  15
  16 NOT_ENOUGH_FRAMES_MESSAGE = (
  17   'Not enough frames for smoothness metrics (at least two are required).\n'
  18   'Issues that have caused this in the past:\n'
  19   '- Browser bugs that prevents the page from redrawing\n'
  20   '- Bugs in the synthetic gesture code\n'
  21   '- Page and benchmark out of sync (e.g. clicked element was renamed)\n'
  22   '- Pages that render extremely slow\n'
  23   '- Pages that can\'t be scrolled')
  24
  25
  26 class SmoothnessMetric(timeline_based_metric.TimelineBasedMetric):
  27   """Computes metrics that measure smoothness of animations over given ranges.
  28
  29   Animations are typically considered smooth if the frame rates are close to
  30   60 frames per second (fps) and uniformly distributed over the sequence. To
  31   determine if a timeline range contains a smooth animation, we update the
  32   results object with several representative metrics:
  33
  34     frame_times: A list of raw frame times
  35     mean_frame_time: The arithmetic mean of frame times
  36     percentage_smooth: Percentage of frames that were hitting 60 FPS.
  37     frame_time_discrepancy: The absolute discrepancy of frame timestamps
  38     mean_pixels_approximated: The mean percentage of pixels approximated
  39     queueing_durations: The queueing delay between compositor & main threads
  40
  41   Note that if any of the interaction records provided to AddResults have less
  42   than 2 frames, we will return telemetry values with None values for each of
  43   the smoothness metrics. Similarly, older browsers without support for
  44   tracking the BeginMainFrame events will report a ListOfScalarValues with a
  45   None value for the queueing duration metric.
  46   """
  47
  48   def __init__(self):
  49     super(SmoothnessMetric, self).__init__()
  50
  51   def AddResults(self, model, renderer_thread, interaction_records, results):
  52     self.VerifyNonOverlappedRecords(interaction_records)
  53     renderer_process = renderer_thread.parent
  54     stats = rendering_stats.RenderingStats(
  55       renderer_process, model.browser_process, model.surface_flinger_process,
  56       model.gpu_process, [r.GetBounds() for r in interaction_records])
  57     has_surface_flinger_stats = model.surface_flinger_process is not None
  58     self._PopulateResultsFromStats(results, stats, has_surface_flinger_stats)
  59
  60   def _PopulateResultsFromStats(self, results, stats,
  61                                 has_surface_flinger_stats):
  62     page = results.current_page
  63     values = [
  64         self._ComputeQueueingDuration(page, stats),
  65         self._ComputeFrameTimeDiscrepancy(page, stats),
  66         self._ComputeMeanPixelsApproximated(page, stats),
  67         self._ComputeMeanPixelsCheckerboarded(page, stats)
  68     ]
  69     values += self._ComputeLatencyMetric(page, stats, 'input_event_latency',
  70                                          stats.input_event_latency)
  71     values += self._ComputeLatencyMetric(page, stats,
  72                                          'main_thread_scroll_latency',
  73                                          stats.main_thread_scroll_latency)
  74     values.append(self._ComputeFirstGestureScrollUpdateLatencies(page, stats))
  75     values += self._ComputeFrameTimeMetric(page, stats)
  76     if has_surface_flinger_stats:
  77       values += self._ComputeSurfaceFlingerMetric(page, stats)
  78
  79     for v in values:
  80       results.AddValue(v)
  81
  82   def _HasEnoughFrames(self, list_of_frame_timestamp_lists):
  83     """Whether we have collected at least two frames in every timestamp list."""
  84     return all(len(s) >= 2 for s in list_of_frame_timestamp_lists)
  85
  86   @staticmethod
  87   def _GetNormalizedDeltas(data, refresh_period, min_normalized_delta=None):
  88     deltas = [t2 - t1 for t1, t2 in zip(data, data[1:])]
  89     if min_normalized_delta != None:
  90       deltas = [d for d in deltas
  91                 if d / refresh_period >= min_normalized_delta]
  92     return (deltas, [delta / refresh_period for delta in deltas])
  93
  94   @staticmethod
  95   def _JoinTimestampRanges(frame_timestamps):
  96     """Joins ranges of timestamps, adjusting timestamps to remove deltas
  97     between the start of a range and the end of the prior range.
  98     """
  99     timestamps = []
 100     for timestamp_range in frame_timestamps:
 101       if len(timestamps) == 0:
 102         timestamps.extend(timestamp_range)
 103       else:
 104         for i in range(1, len(timestamp_range)):
 105           timestamps.append(timestamps[-1] +
 106               timestamp_range[i] - timestamp_range[i-1])
 107     return timestamps
 108
 109   def _ComputeSurfaceFlingerMetric(self, page, stats):
 110     jank_count = None
 111     avg_surface_fps = None
 112     max_frame_delay = None
 113     frame_lengths = None
 114     none_value_reason = None
 115     if self._HasEnoughFrames(stats.frame_timestamps):
 116       timestamps = self._JoinTimestampRanges(stats.frame_timestamps)
 117       frame_count = len(timestamps)
 118       milliseconds = timestamps[-1] - timestamps[0]
 119       min_normalized_frame_length = 0.5
 120
 121       frame_lengths, normalized_frame_lengths = \
 122           self._GetNormalizedDeltas(timestamps, stats.refresh_period,
 123                                     min_normalized_frame_length)
 124       if len(frame_lengths) < frame_count - 1:
 125         logging.warning('Skipping frame lengths that are too short.')
 126         frame_count = len(frame_lengths) + 1
 127       if len(frame_lengths) == 0:
 128         raise Exception('No valid frames lengths found.')
 129       _, normalized_changes = \
 130           self._GetNormalizedDeltas(frame_lengths, stats.refresh_period)
 131       jankiness = [max(0, round(change)) for change in normalized_changes]
 132       pause_threshold = 20
 133       jank_count = sum(1 for change in jankiness
 134                        if change > 0 and change < pause_threshold)
 135       avg_surface_fps = int(round((frame_count - 1) * 1000.0 / milliseconds))
 136       max_frame_delay = round(max(normalized_frame_lengths))
 137       frame_lengths = normalized_frame_lengths
 138     else:
 139       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 140
 141     return (
 142         scalar.ScalarValue(
 143             page, 'avg_surface_fps', 'fps', avg_surface_fps,
 144             description='Average frames per second as measured by the '
 145                         'platform\'s SurfaceFlinger.',
 146             none_value_reason=none_value_reason,
 147             improvement_direction=improvement_direction.UP),
 148         scalar.ScalarValue(
 149             page, 'jank_count', 'janks', jank_count,
 150             description='Number of changes in frame rate as measured by the '
 151                         'platform\'s SurfaceFlinger.',
 152             none_value_reason=none_value_reason,
 153             improvement_direction=improvement_direction.DOWN),
 154         scalar.ScalarValue(
 155             page, 'max_frame_delay', 'vsyncs', max_frame_delay,
 156             description='Largest frame time as measured by the platform\'s '
 157                         'SurfaceFlinger.',
 158             none_value_reason=none_value_reason,
 159             improvement_direction=improvement_direction.DOWN),
 160         list_of_scalar_values.ListOfScalarValues(
 161             page, 'frame_lengths', 'vsyncs', frame_lengths,
 162             description='Frame time in vsyncs as measured by the platform\'s '
 163                         'SurfaceFlinger.',
 164             none_value_reason=none_value_reason,
 165             improvement_direction=improvement_direction.DOWN)
 166     )
 167
 168   def _ComputeLatencyMetric(self, page, stats, name, list_of_latency_lists):
 169     """Returns Values for the mean and discrepancy for given latency stats."""
 170     mean_latency = None
 171     latency_discrepancy = None
 172     none_value_reason = None
 173     if self._HasEnoughFrames(stats.frame_timestamps):
 174       latency_list = perf_tests_helper.FlattenList(list_of_latency_lists)
 175       if len(latency_list) == 0:
 176         return ()
 177       mean_latency = round(statistics.ArithmeticMean(latency_list), 3)
 178       latency_discrepancy = (
 179           round(statistics.DurationsDiscrepancy(latency_list), 4))
 180     else:
 181       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 182     return (
 183       scalar.ScalarValue(
 184           page, 'mean_%s' % name, 'ms', mean_latency,
 185           description='Arithmetic mean of the raw %s values' % name,
 186           none_value_reason=none_value_reason,
 187           improvement_direction=improvement_direction.DOWN),
 188       scalar.ScalarValue(
 189           page, '%s_discrepancy' % name, 'ms', latency_discrepancy,
 190           description='Discrepancy of the raw %s values' % name,
 191           none_value_reason=none_value_reason,
 192           improvement_direction=improvement_direction.DOWN)
 193     )
 194
 195   def _ComputeFirstGestureScrollUpdateLatencies(self, page, stats):
 196     """Returns a ListOfScalarValuesValues of gesture scroll update latencies.
 197
 198     Returns a Value for the first gesture scroll update latency for each
 199     interaction record in |stats|.
 200     """
 201     none_value_reason = None
 202     first_gesture_scroll_update_latencies = [round(latencies[0], 4)
 203         for latencies in stats.gesture_scroll_update_latency
 204         if len(latencies)]
 205     if (not self._HasEnoughFrames(stats.frame_timestamps) or
 206         not first_gesture_scroll_update_latencies):
 207       first_gesture_scroll_update_latencies = None
 208       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 209     return list_of_scalar_values.ListOfScalarValues(
 210         page, 'first_gesture_scroll_update_latency', 'ms',
 211         first_gesture_scroll_update_latencies,
 212         description='First gesture scroll update latency measures the time it '
 213                     'takes to process the very first gesture scroll update '
 214                     'input event. The first scroll gesture can often get '
 215                     'delayed by work related to page loading.',
 216         none_value_reason=none_value_reason,
 217         improvement_direction=improvement_direction.DOWN)
 218
 219   def _ComputeQueueingDuration(self, page, stats):
 220     """Returns a Value for the frame queueing durations."""
 221     queueing_durations = None
 222     none_value_reason = None
 223     if 'frame_queueing_durations' in stats.errors:
 224       none_value_reason = stats.errors['frame_queueing_durations']
 225     elif self._HasEnoughFrames(stats.frame_timestamps):
 226       queueing_durations = perf_tests_helper.FlattenList(
 227           stats.frame_queueing_durations)
 228       if len(queueing_durations) == 0:
 229         queueing_durations = None
 230         none_value_reason = 'No frame queueing durations recorded.'
 231     else:
 232       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 233     return list_of_scalar_values.ListOfScalarValues(
 234         page, 'queueing_durations', 'ms', queueing_durations,
 235         description='The frame queueing duration quantifies how out of sync '
 236                     'the compositor and renderer threads are. It is the amount '
 237                     'of wall time that elapses between a '
 238                     'ScheduledActionSendBeginMainFrame event in the compositor '
 239                     'thread and the corresponding BeginMainFrame event in the '
 240                     'main thread.',
 241         none_value_reason=none_value_reason,
 242         improvement_direction=improvement_direction.DOWN)
 243
 244   def _ComputeFrameTimeMetric(self, page, stats):
 245     """Returns Values for the frame time metrics.
 246
 247     This includes the raw and mean frame times, as well as the percentage of
 248     frames that were hitting 60 fps.
 249     """
 250     frame_times = None
 251     mean_frame_time = None
 252     percentage_smooth = None
 253     none_value_reason = None
 254     if self._HasEnoughFrames(stats.frame_timestamps):
 255       frame_times = perf_tests_helper.FlattenList(stats.frame_times)
 256       mean_frame_time = round(statistics.ArithmeticMean(frame_times), 3)
 257       # We use 17ms as a somewhat looser threshold, instead of 1000.0/60.0.
 258       smooth_threshold = 17.0
 259       smooth_count = sum(1 for t in frame_times if t < smooth_threshold)
 260       percentage_smooth = float(smooth_count) / len(frame_times) * 100.0
 261     else:
 262       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 263     return (
 264         list_of_scalar_values.ListOfScalarValues(
 265             page, 'frame_times', 'ms', frame_times,
 266             description='List of raw frame times, helpful to understand the '
 267                         'other metrics.',
 268             none_value_reason=none_value_reason,
 269             improvement_direction=improvement_direction.DOWN),
 270         scalar.ScalarValue(
 271             page, 'mean_frame_time', 'ms', mean_frame_time,
 272             description='Arithmetic mean of frame times.',
 273             none_value_reason=none_value_reason,
 274             improvement_direction=improvement_direction.DOWN),
 275         scalar.ScalarValue(
 276             page, 'percentage_smooth', 'score', percentage_smooth,
 277             description='Percentage of frames that were hitting 60 fps.',
 278             none_value_reason=none_value_reason,
 279             improvement_direction=improvement_direction.UP)
 280     )
 281
 282   def _ComputeFrameTimeDiscrepancy(self, page, stats):
 283     """Returns a Value for the absolute discrepancy of frame time stamps."""
 284
 285     frame_discrepancy = None
 286     none_value_reason = None
 287     if self._HasEnoughFrames(stats.frame_timestamps):
 288       frame_discrepancy = round(statistics.TimestampsDiscrepancy(
 289           stats.frame_timestamps), 4)
 290     else:
 291       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 292     return scalar.ScalarValue(
 293         page, 'frame_time_discrepancy', 'ms', frame_discrepancy,
 294         description='Absolute discrepancy of frame time stamps, where '
 295                     'discrepancy is a measure of irregularity. It quantifies '
 296                     'the worst jank. For a single pause, discrepancy '
 297                     'corresponds to the length of this pause in milliseconds. '
 298                     'Consecutive pauses increase the discrepancy. This metric '
 299                     'is important because even if the mean and 95th '
 300                     'percentile are good, one long pause in the middle of an '
 301                     'interaction is still bad.',
 302         none_value_reason=none_value_reason,
 303         improvement_direction=improvement_direction.DOWN)
 304
 305   def _ComputeMeanPixelsApproximated(self, page, stats):
 306     """Add the mean percentage of pixels approximated.
 307
 308     This looks at tiles which are missing or of low or non-ideal resolution.
 309     """
 310     mean_pixels_approximated = None
 311     none_value_reason = None
 312     if self._HasEnoughFrames(stats.frame_timestamps):
 313       mean_pixels_approximated = round(statistics.ArithmeticMean(
 314           perf_tests_helper.FlattenList(
 315               stats.approximated_pixel_percentages)), 3)
 316     else:
 317       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 318     return scalar.ScalarValue(
 319         page, 'mean_pixels_approximated', 'percent', mean_pixels_approximated,
 320         description='Percentage of pixels that were approximated '
 321                     '(checkerboarding, low-resolution tiles, etc.).',
 322         none_value_reason=none_value_reason,
 323         improvement_direction=improvement_direction.DOWN)
 324
 325   def _ComputeMeanPixelsCheckerboarded(self, page, stats):
 326     """Add the mean percentage of pixels checkerboarded.
 327
 328     This looks at tiles which are only missing.
 329     It does not take into consideration tiles which are of low or
 330     non-ideal resolution.
 331     """
 332     mean_pixels_checkerboarded = None
 333     none_value_reason = None
 334     if self._HasEnoughFrames(stats.frame_timestamps):
 335       if rendering_stats.CHECKERBOARDED_PIXEL_ERROR in stats.errors:
 336         none_value_reason = stats.errors[
 337             rendering_stats.CHECKERBOARDED_PIXEL_ERROR]
 338       else:
 339         mean_pixels_checkerboarded = round(statistics.ArithmeticMean(
 340             perf_tests_helper.FlattenList(
 341                 stats.checkerboarded_pixel_percentages)), 3)
 342     else:
 343       none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE
 344     return scalar.ScalarValue(
 345         page, 'mean_pixels_checkerboarded', 'percent',
 346         mean_pixels_checkerboarded,
 347         description='Percentage of pixels that were checkerboarded.',
 348         none_value_reason=none_value_reason,
 349         improvement_direction=improvement_direction.DOWN)