sdk/platform-tools/systrace/catapult/telemetry/telemetry/internal/browser/browser_options.py

   1 # Copyright 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import copy
   6 import logging
   7 import optparse
   8 import os
   9 import shlex
  10 import socket
  11 import sys
  12
  13 from py_utils import cloud_storage  # pylint: disable=import-error
  14
  15 from telemetry.core import platform
  16 from telemetry.core import util
  17 from telemetry.internal.browser import browser_finder
  18 from telemetry.internal.browser import browser_finder_exceptions
  19 from telemetry.internal.browser import profile_types
  20 from telemetry.internal.platform import device_finder
  21 from telemetry.internal.platform import remote_platform_options
  22 from telemetry.internal.platform.profiler import profiler_finder
  23 from telemetry.internal.util import binary_manager
  24 from telemetry.util import wpr_modes
  25
  26
  27 class BrowserFinderOptions(optparse.Values):
  28   """Options to be used for discovering a browser."""
  29
  30   def __init__(self, browser_type=None):
  31     optparse.Values.__init__(self)
  32
  33     self.browser_type = browser_type
  34     self.browser_executable = None
  35     self.chrome_root = None  # Path to src/
  36     self.chromium_output_dir = None  # E.g.: out/Debug
  37     self.device = None
  38     self.cros_ssh_identity = None
  39
  40     self.cros_remote = None
  41
  42     self.profiler = None
  43     self.verbosity = 0
  44
  45     self.browser_options = BrowserOptions()
  46     self.output_file = None
  47
  48     self.remote_platform_options = None
  49
  50     self.no_performance_mode = False
  51
  52   def __repr__(self):
  53     return str(sorted(self.__dict__.items()))
  54
  55   def Copy(self):
  56     return copy.deepcopy(self)
  57
  58   def CreateParser(self, *args, **kwargs):
  59     parser = optparse.OptionParser(*args, **kwargs)
  60
  61     # Selection group
  62     group = optparse.OptionGroup(parser, 'Which browser to use')
  63     group.add_option('--browser',
  64         dest='browser_type',
  65         default=None,
  66         help='Browser type to run, '
  67              'in order of priority. Supported values: list,%s' %
  68              ','.join(browser_finder.FindAllBrowserTypes(self)))
  69     group.add_option('--browser-executable',
  70         dest='browser_executable',
  71         help='The exact browser to run.')
  72     group.add_option('--chrome-root',
  73         dest='chrome_root',
  74         help='Where to look for chrome builds. '
  75              'Defaults to searching parent dirs by default.')
  76     group.add_option('--chromium-output-directory',
  77         dest='chromium_output_dir',
  78         help='Where to look for build artifacts. '
  79              'Can also be specified by setting environment variable '
  80              'CHROMIUM_OUTPUT_DIR.')
  81     group.add_option(
  82         '--remote',
  83         dest='cros_remote',
  84         help='The hostname of a remote ChromeOS device to use.')
  85     group.add_option(
  86         '--remote-ssh-port',
  87         type=int,
  88         default=socket.getservbyname('ssh'),
  89         dest='cros_remote_ssh_port',
  90         help='The SSH port of the remote ChromeOS device (requires --remote).')
  91     identity = None
  92     testing_rsa = os.path.join(
  93         util.GetTelemetryThirdPartyDir(), 'chromite', 'ssh_keys', 'testing_rsa')
  94     if os.path.exists(testing_rsa):
  95       identity = testing_rsa
  96     group.add_option('--identity',
  97         dest='cros_ssh_identity',
  98         default=identity,
  99         help='The identity file to use when ssh\'ing into the ChromeOS device')
 100     parser.add_option_group(group)
 101
 102     # Debugging options
 103     group = optparse.OptionGroup(parser, 'When things go wrong')
 104     profiler_choices = profiler_finder.GetAllAvailableProfilers()
 105     group.add_option(
 106         '--profiler', default=None, type='choice',
 107         choices=profiler_choices,
 108         help='Record profiling data using this tool. Supported values: %s. '
 109              '(Notice: this flag cannot be used for Timeline Based Measurement '
 110              'benchmarks.)' % ', '.join(profiler_choices))
 111     group.add_option(
 112         '-v', '--verbose', action='count', dest='verbosity',
 113         help='Increase verbosity level (repeat as needed)')
 114     group.add_option('--print-bootstrap-deps',
 115                      action='store_true',
 116                      help='Output bootstrap deps list.')
 117     parser.add_option_group(group)
 118
 119     # Platform options
 120     group = optparse.OptionGroup(parser, 'Platform options')
 121     group.add_option('--no-performance-mode', action='store_true',
 122         help='Some platforms run on "full performance mode" where the '
 123         'test is executed at maximum CPU speed in order to minimize noise '
 124         '(specially important for dashboards / continuous builds). '
 125         'This option prevents Telemetry from tweaking such platform settings.')
 126     parser.add_option_group(group)
 127
 128     # Remote platform options
 129     group = optparse.OptionGroup(parser, 'Remote platform options')
 130     group.add_option('--android-blacklist-file',
 131                      help='Device blacklist JSON file.')
 132     group.add_option('--device',
 133     help='The device ID to use. '
 134          'If not specified, only 0 or 1 connected devices are supported. '
 135          'If specified as "android", all available Android devices are '
 136          'used.')
 137     parser.add_option_group(group)
 138
 139     # Browser options.
 140     self.browser_options.AddCommandLineArgs(parser)
 141
 142     real_parse = parser.parse_args
 143     def ParseArgs(args=None):
 144       defaults = parser.get_default_values()
 145       for k, v in defaults.__dict__.items():
 146         if k in self.__dict__ and self.__dict__[k] != None:
 147           continue
 148         self.__dict__[k] = v
 149       ret = real_parse(args, self) # pylint: disable=E1121
 150
 151       if self.verbosity >= 2:
 152         logging.getLogger().setLevel(logging.DEBUG)
 153       elif self.verbosity:
 154         logging.getLogger().setLevel(logging.INFO)
 155       else:
 156         logging.getLogger().setLevel(logging.WARNING)
 157
 158       if self.chromium_output_dir:
 159         os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir
 160
 161       # Parse remote platform options.
 162       self.BuildRemotePlatformOptions()
 163
 164       if self.remote_platform_options.device == 'list':
 165         if binary_manager.NeedsInit():
 166           binary_manager.InitDependencyManager([])
 167         devices = device_finder.GetDevicesMatchingOptions(self)
 168         print 'Available devices:'
 169         for device in devices:
 170           print ' ', device.name
 171         sys.exit(0)
 172
 173       if self.browser_executable and not self.browser_type:
 174         self.browser_type = 'exact'
 175       if self.browser_type == 'list':
 176         if binary_manager.NeedsInit():
 177           binary_manager.InitDependencyManager([])
 178         devices = device_finder.GetDevicesMatchingOptions(self)
 179         if not devices:
 180           sys.exit(0)
 181         browser_types = {}
 182         for device in devices:
 183           try:
 184             possible_browsers = browser_finder.GetAllAvailableBrowsers(self,
 185                                                                        device)
 186             browser_types[device.name] = sorted(
 187               [browser.browser_type for browser in possible_browsers])
 188           except browser_finder_exceptions.BrowserFinderException as ex:
 189             print >> sys.stderr, 'ERROR: ', ex
 190             sys.exit(1)
 191         print 'Available browsers:'
 192         if len(browser_types) == 0:
 193           print '  No devices were found.'
 194         for device_name in sorted(browser_types.keys()):
 195           print '  ', device_name
 196           for browser_type in browser_types[device_name]:
 197             print '    ', browser_type
 198         sys.exit(0)
 199
 200       # Parse browser options.
 201       self.browser_options.UpdateFromParseResults(self)
 202
 203       return ret
 204     parser.parse_args = ParseArgs
 205     return parser
 206
 207   # TODO(eakuefner): Factor this out into OptionBuilder pattern
 208   def BuildRemotePlatformOptions(self):
 209     if self.device or self.android_blacklist_file:
 210       self.remote_platform_options = (
 211           remote_platform_options.AndroidPlatformOptions(
 212               self.device, self.android_blacklist_file))
 213
 214       # We delete these options because they should live solely in the
 215       # AndroidPlatformOptions instance belonging to this class.
 216       if self.device:
 217         del self.device
 218       if self.android_blacklist_file:
 219         del self.android_blacklist_file
 220     else:
 221       self.remote_platform_options = (
 222           remote_platform_options.AndroidPlatformOptions())
 223
 224   def AppendExtraBrowserArgs(self, args):
 225     self.browser_options.AppendExtraBrowserArgs(args)
 226
 227   def MergeDefaultValues(self, defaults):
 228     for k, v in defaults.__dict__.items():
 229       self.ensure_value(k, v)
 230
 231 class BrowserOptions(object):
 232   """Options to be used for launching a browser."""
 233
 234   # Levels of browser logging.
 235   NO_LOGGING = 'none'
 236   NON_VERBOSE_LOGGING = 'non-verbose'
 237   VERBOSE_LOGGING = 'verbose'
 238
 239   _LOGGING_LEVELS = (NO_LOGGING, NON_VERBOSE_LOGGING, VERBOSE_LOGGING)
 240   _DEFAULT_LOGGING_LEVEL = NO_LOGGING
 241
 242   def __init__(self):
 243     self.browser_type = None
 244     self.show_stdout = False
 245
 246     self.extensions_to_load = []
 247
 248     # If set, copy the generated profile to this path on exit.
 249     self.output_profile_path = None
 250
 251     # When set to True, the browser will use the default profile.  Telemetry
 252     # will not provide an alternate profile directory.
 253     self.dont_override_profile = False
 254     self.profile_dir = None
 255     self.profile_type = None
 256     self._extra_browser_args = set()
 257     self.extra_wpr_args = []
 258     self.wpr_mode = wpr_modes.WPR_OFF
 259     self.full_performance_mode = True
 260
 261     # The amount of time Telemetry should wait for the browser to start.
 262     # This property is not exposed as a command line option.
 263     self._browser_startup_timeout = 60
 264
 265     self.disable_background_networking = True
 266     self.browser_user_agent_type = None
 267
 268     self.clear_sytem_cache_for_browser_and_profile_on_start = False
 269     self.startup_url = 'about:blank'
 270
 271     # Background pages of built-in component extensions can interfere with
 272     # performance measurements.
 273     self.disable_component_extensions_with_background_pages = True
 274     # Disable default apps.
 275     self.disable_default_apps = True
 276
 277     self.logging_verbosity = self._DEFAULT_LOGGING_LEVEL
 278
 279     # The cloud storage bucket & path for uploading logs data produced by the
 280     # browser to.
 281     # If logs_cloud_remote_path is None, a random remote path is generated every
 282     # time the logs data is uploaded.
 283     self.logs_cloud_bucket = cloud_storage.TELEMETRY_OUTPUT
 284     self.logs_cloud_remote_path = None
 285
 286     # TODO(danduong): Find a way to store target_os here instead of
 287     # finder_options.
 288     self._finder_options = None
 289
 290     # Whether to take screen shot for failed page & put them in telemetry's
 291     # profiling results.
 292     self.take_screenshot_for_failed_page = False
 293
 294   def __repr__(self):
 295     # This works around the infinite loop caused by the introduction of a
 296     # circular reference with _finder_options.
 297     obj = self.__dict__.copy()
 298     del obj['_finder_options']
 299     return str(sorted(obj.items()))
 300
 301   def IsCrosBrowserOptions(self):
 302     return False
 303
 304   @classmethod
 305   def AddCommandLineArgs(cls, parser):
 306
 307     ############################################################################
 308     # Please do not add any more options here without first discussing with    #
 309     # a telemetry owner. This is not the right place for platform-specific     #
 310     # options.                                                                 #
 311     ############################################################################
 312
 313     group = optparse.OptionGroup(parser, 'Browser options')
 314     profile_choices = profile_types.GetProfileTypes()
 315     group.add_option('--profile-type',
 316         dest='profile_type',
 317         type='choice',
 318         default='clean',
 319         choices=profile_choices,
 320         help=('The user profile to use. A clean profile is used by default. '
 321               'Supported values: ' + ', '.join(profile_choices)))
 322     group.add_option('--profile-dir',
 323         dest='profile_dir',
 324         help='Profile directory to launch the browser with. '
 325              'A clean profile is used by default')
 326     group.add_option('--extra-browser-args',
 327         dest='extra_browser_args_as_string',
 328         help='Additional arguments to pass to the browser when it starts')
 329     group.add_option('--extra-wpr-args',
 330         dest='extra_wpr_args_as_string',
 331         help=('Additional arguments to pass to Web Page Replay. '
 332               'See third_party/web-page-replay/replay.py for usage.'))
 333     group.add_option('--show-stdout',
 334         action='store_true',
 335         help='When possible, will display the stdout of the process')
 336
 337     group.add_option('--browser-logging-verbosity',
 338         dest='logging_verbosity',
 339         type='choice',
 340         choices=cls._LOGGING_LEVELS,
 341         help=('Browser logging verbosity. The log file is saved in temp '
 342               "directory. Note that logging affects the browser's "
 343               'performance. Supported values: %s. Defaults to %s.' % (
 344                   ', '.join(cls._LOGGING_LEVELS), cls._DEFAULT_LOGGING_LEVEL)))
 345     parser.add_option_group(group)
 346
 347     group = optparse.OptionGroup(parser, 'Compatibility options')
 348     group.add_option('--gtest_output',
 349         help='Ignored argument for compatibility with runtest.py harness')
 350     parser.add_option_group(group)
 351
 352   def UpdateFromParseResults(self, finder_options):
 353     """Copies our options from finder_options"""
 354     browser_options_list = [
 355         'extra_browser_args_as_string',
 356         'extra_wpr_args_as_string',
 357         'profile_dir',
 358         'profile_type',
 359         'show_stdout',
 360         ]
 361     for o in browser_options_list:
 362       a = getattr(finder_options, o, None)
 363       if a is not None:
 364         setattr(self, o, a)
 365         delattr(finder_options, o)
 366
 367     self.browser_type = finder_options.browser_type
 368     self._finder_options = finder_options
 369
 370     if hasattr(self, 'extra_browser_args_as_string'):
 371       tmp = shlex.split(
 372         self.extra_browser_args_as_string)
 373       self.AppendExtraBrowserArgs(tmp)
 374       delattr(self, 'extra_browser_args_as_string')
 375     if hasattr(self, 'extra_wpr_args_as_string'):
 376       tmp = shlex.split(
 377         self.extra_wpr_args_as_string)
 378       self.extra_wpr_args.extend(tmp)
 379       delattr(self, 'extra_wpr_args_as_string')
 380     if self.profile_type == 'default':
 381       self.dont_override_profile = True
 382
 383     if self.profile_dir and self.profile_type != 'clean':
 384       logging.critical(
 385           "It's illegal to specify both --profile-type and --profile-dir.\n"
 386           "For more information see: http://goo.gl/ngdGD5")
 387       sys.exit(1)
 388
 389     if self.profile_dir and not os.path.isdir(self.profile_dir):
 390       logging.critical(
 391           "Directory specified by --profile-dir (%s) doesn't exist "
 392           "or isn't a directory.\n"
 393           "For more information see: http://goo.gl/ngdGD5" % self.profile_dir)
 394       sys.exit(1)
 395
 396     if not self.profile_dir:
 397       self.profile_dir = profile_types.GetProfileDir(self.profile_type)
 398
 399     if getattr(finder_options, 'logging_verbosity'):
 400       self.logging_verbosity = finder_options.logging_verbosity
 401       delattr(finder_options, 'logging_verbosity')
 402
 403     # This deferred import is necessary because browser_options is imported in
 404     # telemetry/telemetry/__init__.py.
 405     finder_options.browser_options = CreateChromeBrowserOptions(self)
 406
 407   @property
 408   def finder_options(self):
 409     return self._finder_options
 410
 411   @property
 412   def extra_browser_args(self):
 413     return self._extra_browser_args
 414
 415   @property
 416   def browser_startup_timeout(self):
 417     return self._browser_startup_timeout
 418
 419   @browser_startup_timeout.setter
 420   def browser_startup_timeout(self, value):
 421     self._browser_startup_timeout = value
 422
 423   def AppendExtraBrowserArgs(self, args):
 424     if isinstance(args, list):
 425       self._extra_browser_args.update(args)
 426     else:
 427       self._extra_browser_args.add(args)
 428
 429
 430 def CreateChromeBrowserOptions(br_options):
 431   browser_type = br_options.browser_type
 432
 433   if (platform.GetHostPlatform().GetOSName() == 'chromeos' or
 434       (browser_type and browser_type.startswith('cros'))):
 435     return CrosBrowserOptions(br_options)
 436
 437   return br_options
 438
 439
 440 class ChromeBrowserOptions(BrowserOptions):
 441   """Chrome-specific browser options."""
 442
 443   def __init__(self, br_options):
 444     super(ChromeBrowserOptions, self).__init__()
 445     # Copy to self.
 446     self.__dict__.update(br_options.__dict__)
 447
 448
 449 class CrosBrowserOptions(ChromeBrowserOptions):
 450   """ChromeOS-specific browser options."""
 451
 452   def __init__(self, br_options):
 453     super(CrosBrowserOptions, self).__init__(br_options)
 454     # Create a browser with oobe property.
 455     self.create_browser_with_oobe = False
 456     # Clear enterprise policy before logging in.
 457     self.clear_enterprise_policy = True
 458     # Disable GAIA/enterprise services.
 459     self.disable_gaia_services = True
 460
 461     self.auto_login = True
 462     self.gaia_login = False
 463     self.username = 'test@test.test'
 464     self.password = ''
 465     self.gaia_id = '12345'
 466     # For non-accelerated QEMU VMs.
 467     self.browser_startup_timeout = 240
 468
 469   def IsCrosBrowserOptions(self):
 470     return True