testing/mozharness/scripts/web_platform_tests.py

   1 #!/usr/bin/env python
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6 # ***** END LICENSE BLOCK *****
   7 import copy
   8 import glob
   9 import json
  10 import os
  11 import sys
  12
  13 # load modules from parent dir
  14 sys.path.insert(1, os.path.dirname(sys.path[0]))
  15
  16 from mozharness.base.errors import BaseErrorList
  17 from mozharness.base.script import PreScriptAction
  18 from mozharness.base.vcs.vcsbase import MercurialScript
  19 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
  20 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options, TOOLTOOL_PLATFORM_DIR
  21 from mozharness.mozilla.testing.codecoverage import (
  22     CodeCoverageMixin,
  23     code_coverage_config_options
  24 )
  25 from mozharness.mozilla.testing.errors import HarnessErrorList
  26
  27 from mozharness.mozilla.structuredlog import StructuredOutputParser
  28 from mozharness.base.log import INFO
  29
  30 class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin, CodeCoverageMixin):
  31     config_options = [
  32         [['--test-type'], {
  33             "action": "extend",
  34             "dest": "test_type",
  35             "help": "Specify the test types to run."}
  36          ],
  37         [['--e10s'], {
  38             "action": "store_true",
  39             "dest": "e10s",
  40             "default": False,
  41             "help": "Run with e10s enabled"}
  42          ],
  43         [["--total-chunks"], {
  44             "action": "store",
  45             "dest": "total_chunks",
  46             "help": "Number of total chunks"}
  47          ],
  48         [["--this-chunk"], {
  49             "action": "store",
  50             "dest": "this_chunk",
  51             "help": "Number of this chunk"}
  52          ],
  53         [["--allow-software-gl-layers"], {
  54             "action": "store_true",
  55             "dest": "allow_software_gl_layers",
  56             "default": False,
  57             "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."}
  58          ],
  59         [["--enable-webrender"], {
  60             "action": "store_true",
  61             "dest": "enable_webrender",
  62             "default": False,
  63             "help": "Tries to enable the WebRender compositor."}
  64          ],
  65         [["--parallel-stylo-traversal"], {
  66             "action": "store_true",
  67             "dest": "parallel_stylo_traversal",
  68             "default": False,
  69             "help": "Forcibly enable parallel traversal in Stylo with STYLO_THREADS=4"}
  70          ],
  71         [["--enable-stylo"], {
  72             "action": "store_true",
  73             "dest": "enable_stylo",
  74             "default": False,
  75             "help": "Run tests with Stylo enabled"}
  76          ],
  77     ] + copy.deepcopy(testing_config_options) + \
  78         copy.deepcopy(blobupload_config_options) + \
  79         copy.deepcopy(code_coverage_config_options)
  80
  81     def __init__(self, require_config_file=True):
  82         super(WebPlatformTest, self).__init__(
  83             config_options=self.config_options,
  84             all_actions=[
  85                 'clobber',
  86                 'read-buildbot-config',
  87                 'download-and-extract',
  88                 'create-virtualenv',
  89                 'pull',
  90                 'install',
  91                 'run-tests',
  92             ],
  93             require_config_file=require_config_file,
  94             config={'require_test_zip': True})
  95
  96         # Surely this should be in the superclass
  97         c = self.config
  98         self.installer_url = c.get('installer_url')
  99         self.test_url = c.get('test_url')
 100         self.test_packages_url = c.get('test_packages_url')
 101         self.installer_path = c.get('installer_path')
 102         self.binary_path = c.get('binary_path')
 103         self.abs_app_dir = None
 104
 105     def query_abs_app_dir(self):
 106         """We can't set this in advance, because OSX install directories
 107         change depending on branding and opt/debug.
 108         """
 109         if self.abs_app_dir:
 110             return self.abs_app_dir
 111         if not self.binary_path:
 112             self.fatal("Can't determine abs_app_dir (binary_path not set!)")
 113         self.abs_app_dir = os.path.dirname(self.binary_path)
 114         return self.abs_app_dir
 115
 116     def query_abs_dirs(self):
 117         if self.abs_dirs:
 118             return self.abs_dirs
 119         abs_dirs = super(WebPlatformTest, self).query_abs_dirs()
 120
 121         dirs = {}
 122         dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
 123         dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
 124         dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
 125         dirs["abs_wpttest_dir"] = os.path.join(dirs['abs_test_install_dir'], "web-platform")
 126         dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
 127
 128         abs_dirs.update(dirs)
 129         self.abs_dirs = abs_dirs
 130
 131         return self.abs_dirs
 132
 133     @PreScriptAction('create-virtualenv')
 134     def _pre_create_virtualenv(self, action):
 135         dirs = self.query_abs_dirs()
 136
 137         requirements = os.path.join(dirs['abs_test_install_dir'],
 138                                     'config',
 139                                     'marionette_requirements.txt')
 140
 141         self.register_virtualenv_module(requirements=[requirements],
 142                                         two_pass=True)
 143
 144     def _query_cmd(self):
 145         if not self.binary_path:
 146             self.fatal("Binary path could not be determined")
 147             #And exit
 148
 149         c = self.config
 150         dirs = self.query_abs_dirs()
 151         abs_app_dir = self.query_abs_app_dir()
 152         run_file_name = "runtests.py"
 153
 154         cmd = [self.query_python_path('python'), '-u']
 155         cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
 156
 157         # Make sure that the logging directory exists
 158         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
 159             self.fatal("Could not create blobber upload directory")
 160             # Exit
 161
 162         cmd += ["--log-raw=-",
 163                 "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
 164                                               "wpt_raw.log"),
 165                 "--log-errorsummary=%s" % os.path.join(dirs["abs_blob_upload_dir"],
 166                                                        "wpt_errorsummary.log"),
 167                 "--binary=%s" % self.binary_path,
 168                 "--symbols-path=%s" % self.query_symbols_url(),
 169                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),
 170                 "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin"),
 171                 "--run-by-dir=3"]
 172
 173         if not sys.platform.startswith("linux"):
 174             cmd += ["--exclude=css"]
 175
 176         # Let wptrunner determine the test type when --try-test-paths is used
 177         wpt_test_paths = self.try_test_paths.get("web-platform-tests")
 178         if not wpt_test_paths:
 179             for test_type in c.get("test_type", []):
 180                 cmd.append("--test-type=%s" % test_type)
 181
 182         if not c["e10s"]:
 183             cmd.append("--disable-e10s")
 184
 185         if c["parallel_stylo_traversal"]:
 186             cmd.append("--stylo-threads=4")
 187
 188         for opt in ["total_chunks", "this_chunk"]:
 189             val = c.get(opt)
 190             if val:
 191                 cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
 192
 193         if wpt_test_paths or "wdspec" in c.get("test_type", []):
 194             geckodriver_path = os.path.join(dirs["abs_test_bin_dir"], "geckodriver")
 195             if not os.path.isfile(geckodriver_path):
 196                 self.fatal("Unable to find geckodriver binary "
 197                            "in common test package: %s" % geckodriver_path)
 198             cmd.append("--webdriver-binary=%s" % geckodriver_path)
 199
 200         options = list(c.get("options", []))
 201
 202         str_format_values = {
 203             'binary_path': self.binary_path,
 204             'test_path': dirs["abs_wpttest_dir"],
 205             'test_install_path': dirs["abs_test_install_dir"],
 206             'abs_app_dir': abs_app_dir,
 207             'abs_work_dir': dirs["abs_work_dir"]
 208             }
 209
 210         try_options, try_tests = self.try_args("web-platform-tests")
 211
 212         cmd.extend(self.query_options(options,
 213                                       try_options,
 214                                       str_format_values=str_format_values))
 215         cmd.extend(self.query_tests_args(try_tests,
 216                                          str_format_values=str_format_values))
 217
 218         return cmd
 219
 220     def download_and_extract(self):
 221         super(WebPlatformTest, self).download_and_extract(
 222             extract_dirs=["mach",
 223                           "bin/*",
 224                           "config/*",
 225                           "mozbase/*",
 226                           "marionette/*",
 227                           "tools/*",
 228                           "web-platform/*"],
 229             suite_categories=["web-platform"])
 230
 231     def _install_fonts(self):
 232         # Ensure the Ahem font is available
 233         dirs = self.query_abs_dirs()
 234
 235         if not sys.platform.startswith("darwin"):
 236             font_path = os.path.join(os.path.dirname(self.binary_path), "fonts")
 237         else:
 238             font_path = os.path.join(os.path.dirname(self.binary_path), os.pardir, "Resources", "res", "fonts")
 239         if not os.path.exists(font_path):
 240             os.makedirs(font_path)
 241         ahem_src = os.path.join(dirs["abs_wpttest_dir"], "tests", "fonts", "Ahem.ttf")
 242         ahem_dest = os.path.join(font_path, "Ahem.ttf")
 243         with open(ahem_src) as src, open(ahem_dest, "w") as dest:
 244             dest.write(src.read())
 245
 246     def run_tests(self):
 247         dirs = self.query_abs_dirs()
 248         cmd = self._query_cmd()
 249
 250         self._install_fonts()
 251
 252         parser = StructuredOutputParser(config=self.config,
 253                                         log_obj=self.log_obj,
 254                                         log_compact=True,
 255                                         error_list=BaseErrorList + HarnessErrorList)
 256
 257         env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
 258         env['RUST_BACKTRACE'] = '1'
 259
 260         if self.config['allow_software_gl_layers']:
 261             env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
 262         if self.config['enable_webrender']:
 263             env['MOZ_WEBRENDER'] = '1'
 264
 265         env['STYLO_THREADS'] = '4' if self.config['parallel_stylo_traversal'] else '1'
 266         if self.config['enable_stylo']:
 267             env['STYLO_FORCE_ENABLED'] = '1'
 268
 269         env = self.query_env(partial_env=env, log_level=INFO)
 270
 271         return_code = self.run_command(cmd,
 272                                        cwd=dirs['abs_work_dir'],
 273                                        output_timeout=1000,
 274                                        output_parser=parser,
 275                                        env=env)
 276
 277         tbpl_status, log_level = parser.evaluate_parser(return_code)
 278
 279         self.buildbot_status(tbpl_status, level=log_level)
 280
 281
 282 # main {{{1
 283 if __name__ == '__main__':
 284     web_platform_tests = WebPlatformTest()
 285     web_platform_tests.run_and_exit()