testing/awsy/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, unicode_literals
   6
   7 import argparse
   8 import logging
   9 import os
  10 import sys
  11
  12 from mozbuild.base import (
  13     MachCommandBase,
  14     MachCommandConditions as conditions,
  15 )
  16
  17 from mach.decorators import (
  18     CommandArgument,
  19     CommandArgumentGroup,
  20     CommandProvider,
  21     Command,
  22 )
  23
  24 import mozinfo
  25
  26
  27 def setup_awsy_argument_parser():
  28     from marionette_harness.runtests import MarionetteArguments
  29     from mozlog.structured import commandline
  30
  31     parser = MarionetteArguments()
  32     commandline.add_logging_group(parser)
  33
  34     return parser
  35
  36
  37 @CommandProvider
  38 class MachCommands(MachCommandBase):
  39     AWSY_PATH = os.path.dirname(os.path.realpath(__file__))
  40     if AWSY_PATH not in sys.path:
  41         sys.path.append(AWSY_PATH)
  42     from awsy import ITERATIONS, PER_TAB_PAUSE, SETTLE_WAIT_TIME, MAX_TABS
  43
  44     def run_awsy(self, tests, binary=None, **kwargs):
  45         import json
  46         from mozlog.structured import commandline
  47
  48         from marionette_harness.runtests import (
  49             MarionetteTestRunner,
  50             MarionetteHarness
  51         )
  52
  53         parser = setup_awsy_argument_parser()
  54
  55         awsy_source_dir = os.path.join(self.topsrcdir, 'testing', 'awsy')
  56         if not tests:
  57             tests = [os.path.join(awsy_source_dir,
  58                                   'awsy',
  59                                   'test_memory_usage.py')]
  60
  61         args = argparse.Namespace(tests=tests)
  62
  63         args.binary = binary
  64
  65         if kwargs['quick']:
  66             kwargs['entities'] = 3
  67             kwargs['iterations'] = 1
  68             kwargs['perTabPause'] = 1
  69             kwargs['settleWaitTime'] = 1
  70
  71         if 'single_stylo_traversal' in kwargs and kwargs['single_stylo_traversal']:
  72             os.environ['STYLO_THREADS'] = '1'
  73         else:
  74             os.environ['STYLO_THREADS'] = '4'
  75
  76         runtime_testvars = {}
  77         for arg in ('webRootDir', 'pageManifest', 'resultsDir', 'entities', 'iterations',
  78                     'perTabPause', 'settleWaitTime', 'maxTabs', 'dmd', 'tp6'):
  79             if arg in kwargs and kwargs[arg] is not None:
  80                 runtime_testvars[arg] = kwargs[arg]
  81
  82         if 'webRootDir' not in runtime_testvars:
  83             awsy_tests_dir = os.path.join(self.topobjdir, '_tests', 'awsy')
  84             web_root_dir = os.path.join(awsy_tests_dir, 'html')
  85             runtime_testvars['webRootDir'] = web_root_dir
  86         else:
  87             web_root_dir = runtime_testvars['webRootDir']
  88             awsy_tests_dir = os.path.dirname(web_root_dir)
  89
  90         if 'resultsDir' not in runtime_testvars:
  91             runtime_testvars['resultsDir'] = os.path.join(awsy_tests_dir,
  92                                                           'results')
  93
  94         runtime_testvars['bin'] = binary
  95         runtime_testvars['run_local'] = True
  96
  97         page_load_test_dir = os.path.join(web_root_dir, 'page_load_test')
  98         if not os.path.isdir(page_load_test_dir):
  99             os.makedirs(page_load_test_dir)
 100
 101         if not os.path.isdir(runtime_testvars['resultsDir']):
 102             os.makedirs(runtime_testvars['resultsDir'])
 103
 104         runtime_testvars_path = os.path.join(awsy_tests_dir, 'runtime-testvars.json')
 105         if kwargs['testvars']:
 106             kwargs['testvars'].append(runtime_testvars_path)
 107         else:
 108             kwargs['testvars'] = [runtime_testvars_path]
 109
 110         runtime_testvars_file = open(runtime_testvars_path, 'wb')
 111         runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2))
 112         runtime_testvars_file.close()
 113
 114         manifest_file = os.path.join(awsy_source_dir,
 115                                      'tp5n-pageset.manifest')
 116         tooltool_args = {'args': [
 117             sys.executable,
 118             os.path.join(self.topsrcdir, 'mach'),
 119             'artifact', 'toolchain', '-v',
 120             '--tooltool-manifest=%s' % manifest_file,
 121             '--cache-dir=%s' % os.path.join(self.topsrcdir, 'tooltool-cache'),
 122         ]}
 123         self.run_process(cwd=page_load_test_dir, **tooltool_args)
 124         tp5nzip = os.path.join(page_load_test_dir, 'tp5n.zip')
 125         tp5nmanifest = os.path.join(page_load_test_dir, 'tp5n', 'tp5n.manifest')
 126         if not os.path.exists(tp5nmanifest):
 127             unzip_args = {'args': [
 128                 'unzip',
 129                 '-q',
 130                 '-o',
 131                 tp5nzip,
 132                 '-d',
 133                 page_load_test_dir]}
 134             try:
 135                 self.run_process(**unzip_args)
 136             except Exception as exc:
 137                 troubleshoot = ''
 138                 if mozinfo.os == 'win':
 139                     troubleshoot = ' Try using --web-root to specify a ' \
 140                                    'directory closer to the drive root.'
 141
 142                 self.log(logging.ERROR, 'awsy',
 143                          {'directory': page_load_test_dir, 'exception': exc},
 144                          'Failed to unzip `tp5n.zip` into '
 145                          '`{directory}` with `{exception}`.' + troubleshoot)
 146                 raise exc
 147
 148         # If '--preferences' was not specified supply our default set.
 149         if not kwargs['prefs_files']:
 150             kwargs['prefs_files'] = [os.path.join(awsy_source_dir, 'conf', 'prefs.json')]
 151
 152         # Setup DMD env vars if necessary.
 153         if kwargs['dmd']:
 154             bin_dir = os.path.dirname(binary)
 155
 156             if 'DMD' not in os.environ:
 157                 os.environ['DMD'] = '1'
 158
 159             # Work around a startup crash with DMD on windows
 160             if mozinfo.os == 'win':
 161                 kwargs['pref'] = 'security.sandbox.content.level:0'
 162                 self.log(logging.WARNING, 'awsy', {},
 163                          'Forcing \'security.sandbox.content.level\' = 0 because DMD is enabled.')
 164             elif mozinfo.os == 'mac':
 165                 # On mac binary is in MacOS and dmd.py is in Resources, ie:
 166                 #   Name.app/Contents/MacOS/libdmd.dylib
 167                 #   Name.app/Contents/Resources/dmd.py
 168                 bin_dir = os.path.join(bin_dir, "../Resources/")
 169
 170             # Also add the bin dir to the python path so we can use dmd.py
 171             if bin_dir not in sys.path:
 172                 sys.path.append(bin_dir)
 173
 174         for k, v in kwargs.iteritems():
 175             setattr(args, k, v)
 176
 177         parser.verify_usage(args)
 178
 179         args.logger = commandline.setup_logging('Are We Slim Yet Tests',
 180                                                 args,
 181                                                 {'mach': sys.stdout})
 182         failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
 183         if failed > 0:
 184             return 1
 185         else:
 186             return 0
 187
 188     @Command('awsy-test', category='testing',
 189              description='Run Are We Slim Yet (AWSY) memory usage testing using marionette.',
 190              parser=setup_awsy_argument_parser,
 191              )
 192     @CommandArgumentGroup('AWSY')
 193     @CommandArgument('--web-root', group='AWSY', action='store', type=str,
 194                      dest='webRootDir',
 195                      help='Path to web server root directory. If not specified, '
 196                      'defaults to topobjdir/_tests/awsy/html.')
 197     @CommandArgument('--page-manifest', group='AWSY', action='store', type=str,
 198                      dest='pageManifest',
 199                      help='Path to page manifest text file containing a list '
 200                      'of urls to test. The urls must be served from localhost. If not '
 201                      'specified, defaults to page_load_test/tp5n/tp5n.manifest under '
 202                      'the web root.')
 203     @CommandArgument('--results', group='AWSY', action='store', type=str,
 204                      dest='resultsDir',
 205                      help='Path to results directory. If not specified, defaults '
 206                      'to the parent directory of the web root.')
 207     @CommandArgument('--quick', group='AWSY', action='store_true',
 208                      dest='quick', default=False,
 209                      help='Set --entities=3, --iterations=1, --per-tab-pause=1, '
 210                      '--settle-wait-time=1 for a quick test. Overrides any explicit '
 211                      'argument settings.')
 212     @CommandArgument('--entities', group='AWSY', action='store', type=int,
 213                      dest='entities',
 214                      help='Number of urls to load. Defaults to the total number of '
 215                      'urls.')
 216     @CommandArgument('--max-tabs', group='AWSY', action='store', type=int,
 217                      dest='maxTabs',
 218                      help='Maximum number of tabs to open. '
 219                      'Defaults to %s.' % MAX_TABS)
 220     @CommandArgument('--iterations', group='AWSY', action='store', type=int,
 221                      dest='iterations',
 222                      help='Number of times to run through the test suite. '
 223                      'Defaults to %s.' % ITERATIONS)
 224     @CommandArgument('--per-tab-pause', group='AWSY', action='store', type=int,
 225                      dest='perTabPause',
 226                      help='Seconds to wait in between opening tabs. '
 227                      'Defaults to %s.' % PER_TAB_PAUSE)
 228     @CommandArgument('--settle-wait-time', group='AWSY', action='store', type=int,
 229                      dest='settleWaitTime',
 230                      help='Seconds to wait for things to settled down. '
 231                      'Defaults to %s.' % SETTLE_WAIT_TIME)
 232     @CommandArgument('--single-stylo-traversal', group='AWSY', action='store_true',
 233                      dest='single_stylo_traversal', default=False,
 234                      help='Set STYLO_THREADS=1.')
 235     @CommandArgument('--dmd', group='AWSY', action='store_true',
 236                      dest='dmd', default=False,
 237                      help='Enable DMD during testing. Requires a DMD-enabled build.')
 238     @CommandArgument('--tp6', group='AWSY', action='store_true',
 239                      dest='tp6', default=False,
 240                      help='Use the tp6 pageset during testing.')
 241     def run_awsy_test(self, tests, **kwargs):
 242         """mach awsy-test runs the in-tree version of the Are We Slim Yet
 243         (AWSY) tests.
 244
 245         awsy-test is implemented as a marionette test and marionette
 246         test arguments also apply although they are not necessary
 247         since reasonable defaults will be chosen.
 248
 249         The AWSY specific arguments can be found in the Command
 250         Arguments for AWSY section below.
 251
 252         awsy-test will automatically download the tp5n.zip talos
 253         pageset from tooltool and install it under
 254         topobjdir/_tests/awsy/html. You can specify your own page set
 255         by specifying --web-root and --page-manifest.
 256
 257         The results of the test will be placed in the results
 258         directory specified by the --results argument.
 259
 260         On Windows, you may experience problems due to path length
 261         errors when extracting the tp5n.zip file containing the
 262         test pages or when attempting to write checkpoints to the
 263         results directory. In that case, you should specify both
 264         the --web-root and --results arguments pointing to a location
 265         with a short path. For example:
 266
 267         --web-root=c:\\\\tmp\\\\html --results=c:\\\\tmp\\\\results
 268
 269         Note that the double backslashes are required.
 270         """
 271         kwargs['logger_name'] = 'Awsy Tests'
 272         if 'test_objects' in kwargs:
 273             tests = []
 274             for obj in kwargs['test_objects']:
 275                 tests.append(obj['file_relpath'])
 276             del kwargs['test_objects']
 277
 278         if not kwargs.get('binary') and conditions.is_firefox(self):
 279             kwargs['binary'] = self.get_binary_path('app')
 280         return self.run_awsy(tests, **kwargs)