testing/awsy/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, unicode_literals
   6
   7 import argparse
   8 import logging
   9 import os
  10 import sys
  11
  12 import six
  13
  14 from mozbuild.base import (
  15     MachCommandConditions as conditions,
  16     BinaryNotFoundException,
  17 )
  18
  19 from mach.decorators import (
  20     CommandArgument,
  21     CommandArgumentGroup,
  22     Command,
  23 )
  24
  25 import mozinfo
  26
  27
  28 def setup_awsy_argument_parser():
  29     from marionette_harness.runtests import MarionetteArguments
  30     from mozlog.structured import commandline
  31
  32     parser = MarionetteArguments()
  33     commandline.add_logging_group(parser)
  34
  35     return parser
  36
  37
  38 from awsy import ITERATIONS, PER_TAB_PAUSE, SETTLE_WAIT_TIME, MAX_TABS
  39
  40
  41 def run_awsy(command_context, tests, binary=None, **kwargs):
  42     import json
  43     from mozlog.structured import commandline
  44
  45     from marionette_harness.runtests import MarionetteTestRunner, MarionetteHarness
  46
  47     parser = setup_awsy_argument_parser()
  48
  49     awsy_source_dir = os.path.join(command_context.topsrcdir, "testing", "awsy")
  50     if not tests:
  51         tests = [os.path.join(awsy_source_dir, "awsy", "test_memory_usage.py")]
  52
  53     args = argparse.Namespace(tests=tests)
  54
  55     args.binary = binary
  56
  57     if kwargs["quick"]:
  58         kwargs["entities"] = 3
  59         kwargs["iterations"] = 1
  60         kwargs["perTabPause"] = 1
  61         kwargs["settleWaitTime"] = 1
  62
  63     if "single_stylo_traversal" in kwargs and kwargs["single_stylo_traversal"]:
  64         os.environ["STYLO_THREADS"] = "1"
  65     else:
  66         os.environ["STYLO_THREADS"] = "4"
  67
  68     runtime_testvars = {}
  69     for arg in (
  70         "webRootDir",
  71         "pageManifest",
  72         "resultsDir",
  73         "entities",
  74         "iterations",
  75         "perTabPause",
  76         "settleWaitTime",
  77         "maxTabs",
  78         "dmd",
  79         "tp6",
  80     ):
  81         if arg in kwargs and kwargs[arg] is not None:
  82             runtime_testvars[arg] = kwargs[arg]
  83
  84     if "webRootDir" not in runtime_testvars:
  85         awsy_tests_dir = os.path.join(command_context.topobjdir, "_tests", "awsy")
  86         web_root_dir = os.path.join(awsy_tests_dir, "html")
  87         runtime_testvars["webRootDir"] = web_root_dir
  88     else:
  89         web_root_dir = runtime_testvars["webRootDir"]
  90         awsy_tests_dir = os.path.dirname(web_root_dir)
  91
  92     if "resultsDir" not in runtime_testvars:
  93         runtime_testvars["resultsDir"] = os.path.join(awsy_tests_dir, "results")
  94
  95     runtime_testvars["bin"] = binary
  96     runtime_testvars["run_local"] = True
  97
  98     page_load_test_dir = os.path.join(web_root_dir, "page_load_test")
  99     if not os.path.isdir(page_load_test_dir):
 100         os.makedirs(page_load_test_dir)
 101
 102     if not os.path.isdir(runtime_testvars["resultsDir"]):
 103         os.makedirs(runtime_testvars["resultsDir"])
 104
 105     runtime_testvars_path = os.path.join(awsy_tests_dir, "runtime-testvars.json")
 106     if kwargs["testvars"]:
 107         kwargs["testvars"].append(runtime_testvars_path)
 108     else:
 109         kwargs["testvars"] = [runtime_testvars_path]
 110
 111     runtime_testvars_file = open(runtime_testvars_path, "wb" if six.PY2 else "w")
 112     runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2))
 113     runtime_testvars_file.close()
 114
 115     manifest_file = os.path.join(awsy_source_dir, "tp5n-pageset.manifest")
 116     tooltool_args = {
 117         "args": [
 118             sys.executable,
 119             os.path.join(command_context.topsrcdir, "mach"),
 120             "artifact",
 121             "toolchain",
 122             "-v",
 123             "--tooltool-manifest=%s" % manifest_file,
 124             "--cache-dir=%s"
 125             % os.path.join(command_context.topsrcdir, "tooltool-cache"),
 126         ]
 127     }
 128     command_context.run_process(cwd=page_load_test_dir, **tooltool_args)
 129     tp5nzip = os.path.join(page_load_test_dir, "tp5n.zip")
 130     tp5nmanifest = os.path.join(page_load_test_dir, "tp5n", "tp5n.manifest")
 131     if not os.path.exists(tp5nmanifest):
 132         unzip_args = {"args": ["unzip", "-q", "-o", tp5nzip, "-d", page_load_test_dir]}
 133         try:
 134             command_context.run_process(**unzip_args)
 135         except Exception as exc:
 136             troubleshoot = ""
 137             if mozinfo.os == "win":
 138                 troubleshoot = (
 139                     " Try using --web-root to specify a "
 140                     "directory closer to the drive root."
 141                 )
 142
 143             command_context.log(
 144                 logging.ERROR,
 145                 "awsy",
 146                 {"directory": page_load_test_dir, "exception": exc},
 147                 "Failed to unzip `tp5n.zip` into "
 148                 "`{directory}` with `{exception}`." + troubleshoot,
 149             )
 150             raise exc
 151
 152     # If '--preferences' was not specified supply our default set.
 153     if not kwargs["prefs_files"]:
 154         kwargs["prefs_files"] = [os.path.join(awsy_source_dir, "conf", "prefs.json")]
 155
 156     # Setup DMD env vars if necessary.
 157     if kwargs["dmd"]:
 158         bin_dir = os.path.dirname(binary)
 159
 160         if "DMD" not in os.environ:
 161             os.environ["DMD"] = "1"
 162
 163         # Work around a startup crash with DMD on windows
 164         if mozinfo.os == "win":
 165             kwargs["pref"] = "security.sandbox.content.level:0"
 166             command_context.log(
 167                 logging.WARNING,
 168                 "awsy",
 169                 {},
 170                 "Forcing 'security.sandbox.content.level' = 0 because DMD is enabled.",
 171             )
 172         elif mozinfo.os == "mac":
 173             # On mac binary is in MacOS and dmd.py is in Resources, ie:
 174             #   Name.app/Contents/MacOS/libdmd.dylib
 175             #   Name.app/Contents/Resources/dmd.py
 176             bin_dir = os.path.join(bin_dir, "../Resources/")
 177
 178         # Also add the bin dir to the python path so we can use dmd.py
 179         if bin_dir not in sys.path:
 180             sys.path.append(bin_dir)
 181
 182     for k, v in six.iteritems(kwargs):
 183         setattr(args, k, v)
 184
 185     parser.verify_usage(args)
 186
 187     args.logger = commandline.setup_logging(
 188         "Are We Slim Yet Tests", args, {"mach": sys.stdout}
 189     )
 190     failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
 191     if failed > 0:
 192         return 1
 193     else:
 194         return 0
 195
 196
 197 @Command(
 198     "awsy-test",
 199     category="testing",
 200     description="Run Are We Slim Yet (AWSY) memory usage testing using marionette.",
 201     parser=setup_awsy_argument_parser,
 202 )
 203 @CommandArgumentGroup("AWSY")
 204 @CommandArgument(
 205     "--web-root",
 206     group="AWSY",
 207     action="store",
 208     type=str,
 209     dest="webRootDir",
 210     help="Path to web server root directory. If not specified, "
 211     "defaults to topobjdir/_tests/awsy/html.",
 212 )
 213 @CommandArgument(
 214     "--page-manifest",
 215     group="AWSY",
 216     action="store",
 217     type=str,
 218     dest="pageManifest",
 219     help="Path to page manifest text file containing a list "
 220     "of urls to test. The urls must be served from localhost. If not "
 221     "specified, defaults to page_load_test/tp5n/tp5n.manifest under "
 222     "the web root.",
 223 )
 224 @CommandArgument(
 225     "--results",
 226     group="AWSY",
 227     action="store",
 228     type=str,
 229     dest="resultsDir",
 230     help="Path to results directory. If not specified, defaults "
 231     "to the parent directory of the web root.",
 232 )
 233 @CommandArgument(
 234     "--quick",
 235     group="AWSY",
 236     action="store_true",
 237     dest="quick",
 238     default=False,
 239     help="Set --entities=3, --iterations=1, --per-tab-pause=1, "
 240     "--settle-wait-time=1 for a quick test. Overrides any explicit "
 241     "argument settings.",
 242 )
 243 @CommandArgument(
 244     "--entities",
 245     group="AWSY",
 246     action="store",
 247     type=int,
 248     dest="entities",
 249     help="Number of urls to load. Defaults to the total number of urls.",
 250 )
 251 @CommandArgument(
 252     "--max-tabs",
 253     group="AWSY",
 254     action="store",
 255     type=int,
 256     dest="maxTabs",
 257     help="Maximum number of tabs to open. Defaults to %s." % MAX_TABS,
 258 )
 259 @CommandArgument(
 260     "--iterations",
 261     group="AWSY",
 262     action="store",
 263     type=int,
 264     dest="iterations",
 265     help="Number of times to run through the test suite. "
 266     "Defaults to %s." % ITERATIONS,
 267 )
 268 @CommandArgument(
 269     "--per-tab-pause",
 270     group="AWSY",
 271     action="store",
 272     type=int,
 273     dest="perTabPause",
 274     help="Seconds to wait in between opening tabs. Defaults to %s." % PER_TAB_PAUSE,
 275 )
 276 @CommandArgument(
 277     "--settle-wait-time",
 278     group="AWSY",
 279     action="store",
 280     type=int,
 281     dest="settleWaitTime",
 282     help="Seconds to wait for things to settled down. "
 283     "Defaults to %s." % SETTLE_WAIT_TIME,
 284 )
 285 @CommandArgument(
 286     "--dmd",
 287     group="AWSY",
 288     action="store_true",
 289     dest="dmd",
 290     default=False,
 291     help="Enable DMD during testing. Requires a DMD-enabled build.",
 292 )
 293 @CommandArgument(
 294     "--tp6",
 295     group="AWSY",
 296     action="store_true",
 297     dest="tp6",
 298     default=False,
 299     help="Use the tp6 pageset during testing.",
 300 )
 301 def run_awsy_test(command_context, tests, **kwargs):
 302     """mach awsy-test runs the in-tree version of the Are We Slim Yet
 303     (AWSY) tests.
 304
 305     awsy-test is implemented as a marionette test and marionette
 306     test arguments also apply although they are not necessary
 307     since reasonable defaults will be chosen.
 308
 309     The AWSY specific arguments can be found in the Command
 310     Arguments for AWSY section below.
 311
 312     awsy-test will automatically download the tp5n.zip talos
 313     pageset from tooltool and install it under
 314     topobjdir/_tests/awsy/html. You can specify your own page set
 315     by specifying --web-root and --page-manifest.
 316
 317     The results of the test will be placed in the results
 318     directory specified by the --results argument.
 319
 320     On Windows, you may experience problems due to path length
 321     errors when extracting the tp5n.zip file containing the
 322     test pages or when attempting to write checkpoints to the
 323     results directory. In that case, you should specify both
 324     the --web-root and --results arguments pointing to a location
 325     with a short path. For example:
 326
 327     --web-root=c:\\\\tmp\\\\html --results=c:\\\\tmp\\\\results
 328
 329     Note that the double backslashes are required.
 330     """
 331     kwargs["logger_name"] = "Awsy Tests"
 332     if "test_objects" in kwargs:
 333         tests = []
 334         for obj in kwargs["test_objects"]:
 335             tests.append(obj["file_relpath"])
 336         del kwargs["test_objects"]
 337
 338     if not kwargs.get("binary") and conditions.is_firefox(command_context):
 339         try:
 340             kwargs["binary"] = command_context.get_binary_path("app")
 341         except BinaryNotFoundException as e:
 342             command_context.log(
 343                 logging.ERROR, "awsy", {"error": str(e)}, "ERROR: {error}"
 344             )
 345             command_context.log(logging.INFO, "awsy", {"help": e.help()}, "{help}")
 346             return 1
 347     return run_awsy(command_context, tests, **kwargs)