testing/marionette/client/marionette_driver/geckoinstance.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/
   4
   5 # ALL CHANGES TO THIS FILE MUST HAVE REVIEW FROM A MARIONETTE PEER!
   6 #
   7 # The Marionette Python client is used out-of-tree with various builds of
   8 # Firefox. Removing a preference from this file will cause regressions,
   9 # so please be careful and get review from a Testing :: Marionette peer
  10 # before you make any changes to this file.
  11
  12 from __future__ import absolute_import
  13
  14 import os
  15 import sys
  16 import tempfile
  17 import time
  18 import traceback
  19
  20 from copy import deepcopy
  21
  22 import mozversion
  23
  24 from mozprofile import Profile
  25 from mozrunner import Runner, FennecEmulatorRunner
  26 from six import reraise
  27
  28 from . import errors
  29
  30
  31 class GeckoInstance(object):
  32     required_prefs = {
  33         # Make sure Shield doesn't hit the network.
  34         "app.normandy.api_url": "",
  35
  36         # Increase the APZ content response timeout in tests to 1 minute.
  37         # This is to accommodate the fact that test environments tends to be slower
  38         # than production environments (with the b2g emulator being the slowest of them
  39         # all), resulting in the production timeout value sometimes being exceeded
  40         # and causing false-positive test failures. See bug 1176798, bug 1177018,
  41         # bug 1210465.
  42         "apz.content_response_timeout": 60000,
  43
  44         # Do not send Firefox health reports to the production server
  45         # removed in Firefox 59
  46         "datareporting.healthreport.about.reportUrl": "http://%(server)s/dummy/abouthealthreport/",
  47         "datareporting.healthreport.documentServerURI": "http://%(server)s/dummy/healthreport/",
  48
  49         # Do not show datareporting policy notifications which can interfer with tests
  50         "datareporting.policy.dataSubmissionPolicyBypassNotification": True,
  51
  52         # Automatically unload beforeunload alerts
  53         "dom.disable_beforeunload": True,
  54
  55         # Disable the ProcessHangMonitor
  56         "dom.ipc.reportProcessHangs": False,
  57
  58         # No slow script dialogs
  59         "dom.max_chrome_script_run_time": 0,
  60         "dom.max_script_run_time": 0,
  61
  62         # DOM Push
  63         "dom.push.connection.enabled": False,
  64
  65         # Only load extensions from the application and user profile
  66         # AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION
  67         "extensions.autoDisableScopes": 0,
  68         "extensions.enabledScopes": 5,
  69         # Disable metadata caching for installed add-ons by default
  70         "extensions.getAddons.cache.enabled": False,
  71         # Disable intalling any distribution add-ons
  72         "extensions.installDistroAddons": False,
  73         # Make sure Shield doesn't hit the network.
  74         # Removed in Firefox 60.
  75         "extensions.shield-recipe-client.api_url": "",
  76         # Disable extensions compatibility dialogue.
  77         # Removed in Firefox 61.
  78         "extensions.showMismatchUI": False,
  79         # Turn off extension updates so they don't bother tests
  80         "extensions.update.enabled": False,
  81         "extensions.update.notifyUser": False,
  82         # Make sure opening about:addons won"t hit the network
  83         "extensions.webservice.discoverURL": "http://%(server)s/dummy/discoveryURL",
  84
  85         # Allow the application to have focus even it runs in the background
  86         "focusmanager.testmode": True,
  87
  88         # Disable useragent updates
  89         "general.useragent.updates.enabled": False,
  90
  91         # Always use network provider for geolocation tests
  92         # so we bypass the OSX dialog raised by the corelocation provider
  93         "geo.provider.testing": True,
  94         # Do not scan Wifi
  95         "geo.wifi.scan": False,
  96
  97         "javascript.options.showInConsole": True,
  98
  99         # Enable Marionette component
 100         "marionette.enabled": True,
 101         # (deprecated and can be removed when Firefox 60 ships)
 102         "marionette.defaultPrefs.enabled": True,
 103
 104         # Disable recommended automation prefs in CI
 105         "marionette.prefs.recommended": False,
 106
 107         # Disable download and usage of OpenH264, and Widevine plugins
 108         "media.gmp-manager.updateEnabled": False,
 109
 110         "media.volume_scale": "0.01",
 111
 112         # Do not prompt for temporary redirects
 113         "network.http.prompt-temp-redirect": False,
 114         # Disable speculative connections so they aren"t reported as leaking when they"re
 115         # hanging around
 116         "network.http.speculative-parallel-limit": 0,
 117         # Do not automatically switch between offline and online
 118         "network.manage-offline-status": False,
 119         # Make sure SNTP requests don't hit the network
 120         "network.sntp.pools": "%(server)s",
 121
 122         # Privacy and Tracking Protection
 123         "privacy.trackingprotection.enabled": False,
 124
 125         # Don't do network connections for mitm priming
 126         "security.certerrors.mitm.priming.enabled": False,
 127
 128         # Tests don't wait for the notification button security delay
 129         "security.notification_enable_delay": 0,
 130
 131         # Ensure blocklist updates don't hit the network
 132         "services.settings.server": "http://%(server)s/dummy/blocklist/",
 133
 134         # Disable password capture, so that tests that include forms aren"t
 135         # influenced by the presence of the persistent doorhanger notification
 136         "signon.rememberSignons": False,
 137
 138         # Prevent starting into safe mode after application crashes
 139         "toolkit.startup.max_resumed_crashes": -1,
 140
 141         # We want to collect telemetry, but we don't want to send in the results
 142         "toolkit.telemetry.server": "https://%(server)s/dummy/telemetry/",
 143
 144         # Enabling the support for File object creation in the content process.
 145         "dom.file.createInChild": True,
 146     }
 147
 148     def __init__(self, host=None, port=None, bin=None, profile=None, addons=None,
 149                  app_args=None, symbols_path=None, gecko_log=None, prefs=None,
 150                  workspace=None, verbose=0, headless=False):
 151         self.runner_class = Runner
 152         self.app_args = app_args or []
 153         self.runner = None
 154         self.symbols_path = symbols_path
 155         self.binary = bin
 156
 157         self.marionette_host = host
 158         self.marionette_port = port
 159         self.addons = addons
 160         self.prefs = prefs
 161         self.required_prefs = deepcopy(self.required_prefs)
 162         if prefs:
 163             self.required_prefs.update(prefs)
 164
 165         self._gecko_log_option = gecko_log
 166         self._gecko_log = None
 167         self.verbose = verbose
 168         self.headless = headless
 169
 170         # keep track of errors to decide whether instance is unresponsive
 171         self.unresponsive_count = 0
 172
 173         # Alternative to default temporary directory
 174         self.workspace = workspace
 175
 176         # Don't use the 'profile' property here, because sub-classes could add
 177         # further preferences and data, which would not be included in the new
 178         # profile
 179         self._profile = profile
 180
 181     @property
 182     def gecko_log(self):
 183         if self._gecko_log:
 184             return self._gecko_log
 185
 186         path = self._gecko_log_option
 187         if path != "-":
 188             if path is None:
 189                 path = "gecko.log"
 190             elif os.path.isdir(path):
 191                 fname = "gecko-{}.log".format(time.time())
 192                 path = os.path.join(path, fname)
 193
 194             path = os.path.realpath(path)
 195             if os.access(path, os.F_OK):
 196                 os.remove(path)
 197
 198         self._gecko_log = path
 199         return self._gecko_log
 200
 201     @property
 202     def profile(self):
 203         return self._profile
 204
 205     @profile.setter
 206     def profile(self, value):
 207         self._update_profile(value)
 208
 209     def _update_profile(self, profile=None, profile_name=None):
 210         """Check if the profile has to be created, or replaced
 211
 212         :param profile: A Profile instance to be used.
 213         :param name: Profile name to be used in the path.
 214         """
 215         if self.runner and self.runner.is_running():
 216             raise errors.MarionetteException("The current profile can only be updated "
 217                                              "when the instance is not running")
 218
 219         if isinstance(profile, Profile):
 220             # Only replace the profile if it is not the current one
 221             if hasattr(self, "_profile") and profile is self._profile:
 222                 return
 223
 224         else:
 225             profile_args = self.profile_args
 226             profile_path = profile
 227
 228             # If a path to a profile is given then clone it
 229             if isinstance(profile_path, basestring):
 230                 profile_args["path_from"] = profile_path
 231                 profile_args["path_to"] = tempfile.mkdtemp(
 232                     suffix=u".{}".format(profile_name or os.path.basename(profile_path)),
 233                     dir=self.workspace)
 234                 # The target must not exist yet
 235                 os.rmdir(profile_args["path_to"])
 236
 237                 profile = Profile.clone(**profile_args)
 238
 239             # Otherwise create a new profile
 240             else:
 241                 profile_args["profile"] = tempfile.mkdtemp(
 242                     suffix=u".{}".format(profile_name or "mozrunner"),
 243                     dir=self.workspace)
 244                 profile = Profile(**profile_args)
 245                 profile.create_new = True
 246
 247         if isinstance(self.profile, Profile):
 248             self.profile.cleanup()
 249
 250         self._profile = profile
 251
 252     def switch_profile(self, profile_name=None, clone_from=None):
 253         """Switch the profile by using the given name, and optionally clone it.
 254
 255         Compared to :attr:`profile` this method allows to switch the profile
 256         by giving control over the profile name as used for the new profile. It
 257         also always creates a new blank profile, or as clone of an existent one.
 258
 259         :param profile_name: Optional, name of the profile, which will be used
 260             as part of the profile path (folder name containing the profile).
 261         :clone_from: Optional, if specified the new profile will be cloned
 262             based on the given profile. This argument can be an instance of
 263             ``mozprofile.Profile``, or the path of the profile.
 264         """
 265         if isinstance(clone_from, Profile):
 266             clone_from = clone_from.profile
 267
 268         self._update_profile(clone_from, profile_name=profile_name)
 269
 270     @property
 271     def profile_args(self):
 272         args = {"preferences": deepcopy(self.required_prefs)}
 273         args["preferences"]["marionette.port"] = self.marionette_port
 274         args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
 275
 276         if self.prefs:
 277             args["preferences"].update(self.prefs)
 278
 279         if self.verbose:
 280             level = "Trace" if self.verbose >= 2 else "Debug"
 281             args["preferences"]["marionette.log.level"] = level
 282             args["preferences"]["marionette.logging"] = level
 283
 284         if "-jsdebugger" in self.app_args:
 285             args["preferences"].update({
 286                 "devtools.browsertoolbox.panel": "jsdebugger",
 287                 "devtools.debugger.remote-enabled": True,
 288                 "devtools.chrome.enabled": True,
 289                 "devtools.debugger.prompt-connection": False,
 290                 "marionette.debugging.clicktostart": True,
 291             })
 292
 293         if self.addons:
 294             args["addons"] = self.addons
 295
 296         return args
 297
 298     @classmethod
 299     def create(cls, app=None, *args, **kwargs):
 300         try:
 301             if not app and kwargs["bin"] is not None:
 302                 app_id = mozversion.get_version(binary=kwargs["bin"])["application_id"]
 303                 app = app_ids[app_id]
 304
 305             instance_class = apps[app]
 306         except (IOError, KeyError):
 307             exc, val, tb = sys.exc_info()
 308             msg = 'Application "{0}" unknown (should be one of {1})'
 309             reraise(NotImplementedError, msg.format(app, apps.keys()), tb)
 310
 311         return instance_class(*args, **kwargs)
 312
 313     def start(self):
 314         self._update_profile(self.profile)
 315         self.runner = self.runner_class(**self._get_runner_args())
 316         self.runner.start()
 317
 318     def _get_runner_args(self):
 319         process_args = {
 320             "processOutputLine": [NullOutput()],
 321         }
 322
 323         if self.gecko_log == "-":
 324             process_args["stream"] = sys.stdout
 325         else:
 326             process_args["logfile"] = self.gecko_log
 327
 328         env = os.environ.copy()
 329
 330         if self.headless:
 331             env["MOZ_HEADLESS"] = "1"
 332             env["DISPLAY"] = "77"  # Set a fake display.
 333
 334         # environment variables needed for crashreporting
 335         # https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
 336         env.update({"MOZ_CRASHREPORTER": "1",
 337                     "MOZ_CRASHREPORTER_NO_REPORT": "1",
 338                     "MOZ_CRASHREPORTER_SHUTDOWN": "1",
 339                     })
 340
 341         return {
 342             "binary": self.binary,
 343             "profile": self.profile,
 344             "cmdargs": ["-no-remote", "-marionette"] + self.app_args,
 345             "env": env,
 346             "symbols_path": self.symbols_path,
 347             "process_args": process_args
 348         }
 349
 350     def close(self, clean=False):
 351         """
 352         Close the managed Gecko process.
 353
 354         Depending on self.runner_class, setting `clean` to True may also kill
 355         the emulator process in which this instance is running.
 356
 357         :param clean: If True, also perform runner cleanup.
 358         """
 359         if self.runner:
 360             self.runner.stop()
 361             if clean:
 362                 self.runner.cleanup()
 363
 364         if clean:
 365             if isinstance(self.profile, Profile):
 366                 self.profile.cleanup()
 367             self.profile = None
 368
 369     def restart(self, prefs=None, clean=True):
 370         """
 371         Close then start the managed Gecko process.
 372
 373         :param prefs: Dictionary of preference names and values.
 374         :param clean: If True, reset the profile before starting.
 375         """
 376         if prefs:
 377             self.prefs = prefs
 378         else:
 379             self.prefs = None
 380
 381         self.close(clean=clean)
 382         self.start()
 383
 384
 385 class FennecInstance(GeckoInstance):
 386     fennec_prefs = {
 387         # Enable output for dump() and chrome console API
 388         "browser.dom.window.dump.enabled": True,
 389         "devtools.console.stdout.chrome": True,
 390
 391         # Disable safebrowsing components
 392         "browser.safebrowsing.blockedURIs.enabled": False,
 393         "browser.safebrowsing.downloads.enabled": False,
 394         "browser.safebrowsing.passwords.enabled": False,
 395         "browser.safebrowsing.malware.enabled": False,
 396         "browser.safebrowsing.phishing.enabled": False,
 397
 398         # Do not restore the last open set of tabs if the browser has crashed
 399         "browser.sessionstore.resume_from_crash": False,
 400
 401         # Disable e10s by default
 402         "browser.tabs.remote.autostart": False,
 403
 404         # Do not allow background tabs to be zombified, otherwise for tests that
 405         # open additional tabs, the test harness tab itself might get unloaded
 406         "browser.tabs.disableBackgroundZombification": True,
 407     }
 408
 409     def __init__(self, emulator_binary=None, avd_home=None, avd=None,
 410                  adb_path=None, serial=None, connect_to_running_emulator=False,
 411                  package_name=None, env=None, *args, **kwargs):
 412         required_prefs = deepcopy(FennecInstance.fennec_prefs)
 413         required_prefs.update(kwargs.get("prefs", {}))
 414
 415         super(FennecInstance, self).__init__(*args, **kwargs)
 416         self.required_prefs.update(required_prefs)
 417
 418         self.runner_class = FennecEmulatorRunner
 419         # runner args
 420         self._package_name = package_name
 421         self.emulator_binary = emulator_binary
 422         self.avd_home = avd_home
 423         self.adb_path = adb_path
 424         self.avd = avd
 425         self.env = env
 426         self.serial = serial
 427         self.connect_to_running_emulator = connect_to_running_emulator
 428
 429     @property
 430     def package_name(self):
 431         """
 432         Name of app to run on emulator.
 433
 434         Note that FennecInstance does not use self.binary
 435         """
 436         if self._package_name is None:
 437             self._package_name = "org.mozilla.fennec"
 438             user = os.getenv("USER")
 439             if user:
 440                 self._package_name += "_" + user
 441         return self._package_name
 442
 443     def start(self):
 444         self._update_profile(self.profile)
 445         self.runner = self.runner_class(**self._get_runner_args())
 446         try:
 447             if self.connect_to_running_emulator:
 448                 self.runner.device.connect()
 449             self.runner.start()
 450         except Exception as e:
 451             exc, val, tb = sys.exc_info()
 452             message = "Error possibly due to runner or device args: {}"
 453             reraise(exc, message.format(e.message), tb)
 454
 455         # forward marionette port
 456         self.runner.device.device.forward(
 457             local="tcp:{}".format(self.marionette_port),
 458             remote="tcp:{}".format(self.marionette_port))
 459
 460     def _get_runner_args(self):
 461         process_args = {
 462             "processOutputLine": [NullOutput()],
 463         }
 464
 465         runner_args = {
 466             "app": self.package_name,
 467             "avd_home": self.avd_home,
 468             "adb_path": self.adb_path,
 469             "binary": self.emulator_binary,
 470             "env": self.env,
 471             "profile": self.profile,
 472             "cmdargs": ["-marionette"] + self.app_args,
 473             "symbols_path": self.symbols_path,
 474             "process_args": process_args,
 475             "logdir": self.workspace or os.getcwd(),
 476             "serial": self.serial,
 477         }
 478         if self.avd:
 479             runner_args["avd"] = self.avd
 480
 481         return runner_args
 482
 483     def close(self, clean=False):
 484         """
 485         Close the managed Gecko process.
 486
 487         If `clean` is True and the Fennec instance is running in an
 488         emulator managed by mozrunner, this will stop the emulator.
 489
 490         :param clean: If True, also perform runner cleanup.
 491         """
 492         super(FennecInstance, self).close(clean)
 493         if clean and self.runner and self.runner.device.connected:
 494             try:
 495                 self.runner.device.device.remove_forwards(
 496                     "tcp:{}".format(self.marionette_port))
 497                 self.unresponsive_count = 0
 498             except Exception:
 499                 self.unresponsive_count += 1
 500                 traceback.print_exception(*sys.exc_info())
 501
 502
 503 class DesktopInstance(GeckoInstance):
 504     desktop_prefs = {
 505         # Disable Firefox old build background check
 506         "app.update.checkInstallTime": False,
 507
 508         # Disable automatically upgrading Firefox
 509         #
 510         # Note: Possible update tests could reset or flip the value to allow
 511         # updates to be downloaded and applied.
 512         "app.update.disabledForTesting": True,
 513         # !!! For backward compatibility up to Firefox 64. Only remove
 514         # when this Firefox version is no longer supported by the client !!!
 515         "app.update.auto": False,
 516
 517         # Don't show the content blocking introduction panel
 518         # We use a larger number than the default 22 to have some buffer
 519         "browser.contentblocking.introCount": 99,
 520
 521         # Enable output for dump() and chrome console API
 522         "browser.dom.window.dump.enabled": True,
 523         "devtools.console.stdout.chrome": True,
 524
 525         # Indicate that the download panel has been shown once so that whichever
 526         # download test runs first doesn"t show the popup inconsistently
 527         "browser.download.panel.shown": True,
 528
 529         # Do not show the EULA notification which can interfer with tests
 530         "browser.EULA.override": True,
 531
 532         # Always display a blank page
 533         "browser.newtabpage.enabled": False,
 534
 535         # Background thumbnails in particular cause grief, and disabling thumbnails
 536         # in general can"t hurt - we re-enable them when tests need them
 537         "browser.pagethumbnails.capturing_disabled": True,
 538
 539         # Disable safebrowsing components
 540         "browser.safebrowsing.blockedURIs.enabled": False,
 541         "browser.safebrowsing.downloads.enabled": False,
 542         "browser.safebrowsing.passwords.enabled": False,
 543         "browser.safebrowsing.malware.enabled": False,
 544         "browser.safebrowsing.phishing.enabled": False,
 545
 546         # Disable updates to search engines
 547         "browser.search.update": False,
 548
 549         # Do not restore the last open set of tabs if the browser has crashed
 550         "browser.sessionstore.resume_from_crash": False,
 551
 552         # Don't check for the default web browser during startup
 553         "browser.shell.checkDefaultBrowser": False,
 554
 555         # Disable e10s by default
 556         "browser.tabs.remote.autostart": False,
 557
 558         # Needed for branded builds to prevent opening a second tab on startup
 559         "browser.startup.homepage_override.mstone": "ignore",
 560         # Start with a blank page by default
 561         "browser.startup.page": 0,
 562
 563         # Disable browser animations
 564         "toolkit.cosmeticAnimations.enabled": False,
 565
 566         # Do not warn when closing all open tabs
 567         "browser.tabs.warnOnClose": False,
 568         # Do not warn when closing all other open tabs
 569         "browser.tabs.warnOnCloseOtherTabs": False,
 570         # Do not warn when multiple tabs will be opened
 571         "browser.tabs.warnOnOpen": False,
 572
 573         # Disable the UI tour
 574         "browser.uitour.enabled": False,
 575
 576         # Turn off search suggestions in the location bar so as not to trigger network
 577         # connections.
 578         "browser.urlbar.suggest.searches": False,
 579
 580         # Turn off the location bar search suggestions opt-in.  It interferes with
 581         # tests that don't expect it to be there.
 582         "browser.urlbar.userMadeSearchSuggestionsChoice": True,
 583
 584         # Don't warn when exiting the browser
 585         "browser.warnOnQuit": False,
 586
 587         # Disable first-run welcome page
 588         "startup.homepage_welcome_url": "about:blank",
 589         "startup.homepage_welcome_url.additional": "",
 590     }
 591
 592     def __init__(self, *args, **kwargs):
 593         required_prefs = deepcopy(DesktopInstance.desktop_prefs)
 594         required_prefs.update(kwargs.get("prefs", {}))
 595
 596         super(DesktopInstance, self).__init__(*args, **kwargs)
 597         self.required_prefs.update(required_prefs)
 598
 599
 600 class ThunderbirdInstance(GeckoInstance):
 601     def __init__(self, *args, **kwargs):
 602         super(ThunderbirdInstance, self).__init__(*args, **kwargs)
 603         try:
 604             # Copied alongside in the test archive
 605             from .thunderbirdinstance import thunderbird_prefs
 606         except ImportError:
 607             try:
 608                 # Coming from source tree through virtualenv
 609                 from thunderbirdinstance import thunderbird_prefs
 610             except ImportError:
 611                 thunderbird_prefs = {}
 612         self.required_prefs.update(thunderbird_prefs)
 613
 614
 615 class NullOutput(object):
 616     def __call__(self, line):
 617         pass
 618
 619
 620 apps = {
 621     'fennec': FennecInstance,
 622     'fxdesktop': DesktopInstance,
 623     'thunderbird': ThunderbirdInstance,
 624 }
 625
 626 app_ids = {
 627     '{aa3c5121-dab2-40e2-81ca-7ea25febc110}': 'fennec',
 628     '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}': 'fxdesktop',
 629     '{3550f703-e582-4d05-9a08-453d09bdfdc6}': 'thunderbird',
 630 }