testing/awsy/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import logging
   7 import os
   8 import sys
   9
  10 import mozinfo
  11 import six
  12 from mach.decorators import Command, CommandArgument, CommandArgumentGroup
  13 from mozbuild.base import BinaryNotFoundException
  14 from mozbuild.base import MachCommandConditions as conditions
  15
  16
  17 def setup_awsy_argument_parser():
  18     from marionette_harness.runtests import MarionetteArguments
  19     from mozlog.structured import commandline
  20
  21     parser = MarionetteArguments()
  22     commandline.add_logging_group(parser)
  23
  24     return parser
  25
  26
  27 from awsy import ITERATIONS, MAX_TABS, PER_TAB_PAUSE, SETTLE_WAIT_TIME
  28
  29
  30 def run_awsy(command_context, tests, binary=None, **kwargs):
  31     import json
  32
  33     from marionette_harness.runtests import MarionetteHarness, MarionetteTestRunner
  34     from mozlog.structured import commandline
  35
  36     parser = setup_awsy_argument_parser()
  37
  38     awsy_source_dir = os.path.join(command_context.topsrcdir, "testing", "awsy")
  39     if not tests:
  40         if kwargs["base"]:
  41             filename = "test_base_memory_usage.py"
  42         else:
  43             filename = "test_memory_usage.py"
  44         tests = [os.path.join(awsy_source_dir, "awsy", filename)]
  45
  46     args = argparse.Namespace(tests=tests)
  47
  48     args.binary = binary
  49
  50     if kwargs["quick"]:
  51         kwargs["entities"] = 3
  52         kwargs["iterations"] = 1
  53         kwargs["perTabPause"] = 1
  54         kwargs["settleWaitTime"] = 1
  55
  56     runtime_testvars = {}
  57     for arg in (
  58         "webRootDir",
  59         "pageManifest",
  60         "resultsDir",
  61         "entities",
  62         "iterations",
  63         "perTabPause",
  64         "settleWaitTime",
  65         "maxTabs",
  66         "dmd",
  67         "tp6",
  68     ):
  69         if arg in kwargs and kwargs[arg] is not None:
  70             runtime_testvars[arg] = kwargs[arg]
  71
  72     if "webRootDir" not in runtime_testvars:
  73         awsy_tests_dir = os.path.join(command_context.topobjdir, "_tests", "awsy")
  74         web_root_dir = os.path.join(awsy_tests_dir, "html")
  75         runtime_testvars["webRootDir"] = web_root_dir
  76     else:
  77         web_root_dir = runtime_testvars["webRootDir"]
  78         awsy_tests_dir = os.path.dirname(web_root_dir)
  79
  80     if "resultsDir" not in runtime_testvars:
  81         runtime_testvars["resultsDir"] = os.path.join(awsy_tests_dir, "results")
  82
  83     runtime_testvars["bin"] = binary
  84     runtime_testvars["run_local"] = True
  85
  86     page_load_test_dir = os.path.join(web_root_dir, "page_load_test")
  87     if not os.path.isdir(page_load_test_dir):
  88         os.makedirs(page_load_test_dir)
  89
  90     if not os.path.isdir(runtime_testvars["resultsDir"]):
  91         os.makedirs(runtime_testvars["resultsDir"])
  92
  93     runtime_testvars_path = os.path.join(awsy_tests_dir, "runtime-testvars.json")
  94     if kwargs["testvars"]:
  95         kwargs["testvars"].append(runtime_testvars_path)
  96     else:
  97         kwargs["testvars"] = [runtime_testvars_path]
  98
  99     runtime_testvars_file = open(runtime_testvars_path, "wb" if six.PY2 else "w")
 100     runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2))
 101     runtime_testvars_file.close()
 102
 103     manifest_file = os.path.join(awsy_source_dir, "tp5n-pageset.manifest")
 104     tooltool_args = {
 105         "args": [
 106             sys.executable,
 107             os.path.join(command_context.topsrcdir, "mach"),
 108             "artifact",
 109             "toolchain",
 110             "-v",
 111             "--tooltool-manifest=%s" % manifest_file,
 112             "--cache-dir=%s"
 113             % os.path.join(command_context._mach_context.state_dir, "tooltool-cache"),
 114         ]
 115     }
 116     command_context.run_process(cwd=page_load_test_dir, **tooltool_args)
 117     tp5nzip = os.path.join(page_load_test_dir, "tp5n.zip")
 118     tp5nmanifest = os.path.join(page_load_test_dir, "tp5n", "tp5n.manifest")
 119     if not os.path.exists(tp5nmanifest):
 120         unzip_args = {"args": ["unzip", "-q", "-o", tp5nzip, "-d", page_load_test_dir]}
 121         try:
 122             command_context.run_process(**unzip_args)
 123         except Exception as exc:
 124             troubleshoot = ""
 125             if mozinfo.os == "win":
 126                 troubleshoot = (
 127                     " Try using --web-root to specify a "
 128                     "directory closer to the drive root."
 129                 )
 130
 131             command_context.log(
 132                 logging.ERROR,
 133                 "awsy",
 134                 {"directory": page_load_test_dir, "exception": exc},
 135                 "Failed to unzip `tp5n.zip` into "
 136                 "`{directory}` with `{exception}`." + troubleshoot,
 137             )
 138             raise exc
 139
 140     # If '--preferences' was not specified supply our default set.
 141     if not kwargs["prefs_files"]:
 142         kwargs["prefs_files"] = [os.path.join(awsy_source_dir, "conf", "prefs.json")]
 143
 144     # Setup DMD env vars if necessary.
 145     if kwargs["dmd"]:
 146         bin_dir = os.path.dirname(binary)
 147
 148         if "DMD" not in os.environ:
 149             os.environ["DMD"] = "1"
 150
 151         # Work around a startup crash with DMD on windows
 152         if mozinfo.os == "win":
 153             kwargs["pref"] = "security.sandbox.content.level:0"
 154             command_context.log(
 155                 logging.WARNING,
 156                 "awsy",
 157                 {},
 158                 "Forcing 'security.sandbox.content.level' = 0 because DMD is enabled.",
 159             )
 160         elif mozinfo.os == "mac":
 161             # On mac binary is in MacOS and dmd.py is in Resources, ie:
 162             #   Name.app/Contents/MacOS/libdmd.dylib
 163             #   Name.app/Contents/Resources/dmd.py
 164             bin_dir = os.path.join(bin_dir, "../Resources/")
 165
 166         # Also add the bin dir to the python path so we can use dmd.py
 167         if bin_dir not in sys.path:
 168             sys.path.append(bin_dir)
 169
 170     for k, v in six.iteritems(kwargs):
 171         setattr(args, k, v)
 172
 173     parser.verify_usage(args)
 174
 175     args.logger = commandline.setup_logging(
 176         "Are We Slim Yet Tests", args, {"mach": sys.stdout}
 177     )
 178     failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
 179     if failed > 0:
 180         return 1
 181     else:
 182         return 0
 183
 184
 185 @Command(
 186     "awsy-test",
 187     category="testing",
 188     description="Run Are We Slim Yet (AWSY) memory usage testing using marionette.",
 189     parser=setup_awsy_argument_parser,
 190 )
 191 @CommandArgumentGroup("AWSY")
 192 @CommandArgument(
 193     "--web-root",
 194     group="AWSY",
 195     action="store",
 196     type=str,
 197     dest="webRootDir",
 198     help="Path to web server root directory. If not specified, "
 199     "defaults to topobjdir/_tests/awsy/html.",
 200 )
 201 @CommandArgument(
 202     "--page-manifest",
 203     group="AWSY",
 204     action="store",
 205     type=str,
 206     dest="pageManifest",
 207     help="Path to page manifest text file containing a list "
 208     "of urls to test. The urls must be served from localhost. If not "
 209     "specified, defaults to page_load_test/tp5n/tp5n.manifest under "
 210     "the web root.",
 211 )
 212 @CommandArgument(
 213     "--results",
 214     group="AWSY",
 215     action="store",
 216     type=str,
 217     dest="resultsDir",
 218     help="Path to results directory. If not specified, defaults "
 219     "to the parent directory of the web root.",
 220 )
 221 @CommandArgument(
 222     "--quick",
 223     group="AWSY",
 224     action="store_true",
 225     dest="quick",
 226     default=False,
 227     help="Set --entities=3, --iterations=1, --per-tab-pause=1, "
 228     "--settle-wait-time=1 for a quick test. Overrides any explicit "
 229     "argument settings.",
 230 )
 231 @CommandArgument(
 232     "--entities",
 233     group="AWSY",
 234     action="store",
 235     type=int,
 236     dest="entities",
 237     help="Number of urls to load. Defaults to the total number of urls.",
 238 )
 239 @CommandArgument(
 240     "--max-tabs",
 241     group="AWSY",
 242     action="store",
 243     type=int,
 244     dest="maxTabs",
 245     help="Maximum number of tabs to open. Defaults to %s." % MAX_TABS,
 246 )
 247 @CommandArgument(
 248     "--iterations",
 249     group="AWSY",
 250     action="store",
 251     type=int,
 252     dest="iterations",
 253     help="Number of times to run through the test suite. "
 254     "Defaults to %s." % ITERATIONS,
 255 )
 256 @CommandArgument(
 257     "--per-tab-pause",
 258     group="AWSY",
 259     action="store",
 260     type=int,
 261     dest="perTabPause",
 262     help="Seconds to wait in between opening tabs. Defaults to %s." % PER_TAB_PAUSE,
 263 )
 264 @CommandArgument(
 265     "--settle-wait-time",
 266     group="AWSY",
 267     action="store",
 268     type=int,
 269     dest="settleWaitTime",
 270     help="Seconds to wait for things to settled down. "
 271     "Defaults to %s." % SETTLE_WAIT_TIME,
 272 )
 273 @CommandArgument(
 274     "--dmd",
 275     group="AWSY",
 276     action="store_true",
 277     dest="dmd",
 278     default=False,
 279     help="Enable DMD during testing. Requires a DMD-enabled build.",
 280 )
 281 @CommandArgument(
 282     "--tp6",
 283     group="AWSY",
 284     action="store_true",
 285     dest="tp6",
 286     default=False,
 287     help="Use the tp6 pageset during testing.",
 288 )
 289 @CommandArgument(
 290     "--base",
 291     group="AWSY",
 292     action="store_true",
 293     dest="base",
 294     default=False,
 295     help="Run base memory usage tests.",
 296 )
 297 def run_awsy_test(command_context, tests, **kwargs):
 298     """mach awsy-test runs the in-tree version of the Are We Slim Yet
 299     (AWSY) tests.
 300
 301     awsy-test is implemented as a marionette test and marionette
 302     test arguments also apply although they are not necessary
 303     since reasonable defaults will be chosen.
 304
 305     The AWSY specific arguments can be found in the Command
 306     Arguments for AWSY section below.
 307
 308     awsy-test will automatically download the tp5n.zip talos
 309     pageset from tooltool and install it under
 310     topobjdir/_tests/awsy/html. You can specify your own page set
 311     by specifying --web-root and --page-manifest.
 312
 313     The results of the test will be placed in the results
 314     directory specified by the --results argument.
 315
 316     On Windows, you may experience problems due to path length
 317     errors when extracting the tp5n.zip file containing the
 318     test pages or when attempting to write checkpoints to the
 319     results directory. In that case, you should specify both
 320     the --web-root and --results arguments pointing to a location
 321     with a short path. For example:
 322
 323     --web-root=c:\\\\tmp\\\\html --results=c:\\\\tmp\\\\results
 324
 325     Note that the double backslashes are required.
 326     """
 327     kwargs["logger_name"] = "Awsy Tests"
 328     if "test_objects" in kwargs:
 329         tests = []
 330         for obj in kwargs["test_objects"]:
 331             tests.append(obj["file_relpath"])
 332         del kwargs["test_objects"]
 333
 334     if not kwargs.get("binary") and conditions.is_firefox(command_context):
 335         try:
 336             kwargs["binary"] = command_context.get_binary_path("app")
 337         except BinaryNotFoundException as e:
 338             command_context.log(
 339                 logging.ERROR, "awsy", {"error": str(e)}, "ERROR: {error}"
 340             )
 341             command_context.log(logging.INFO, "awsy", {"help": e.help()}, "{help}")
 342             return 1
 343     return run_awsy(command_context, tests, **kwargs)