testing/runtimes/writeruntimes

   1 #!/bin/sh
   2 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
   3 # vim: set filetype=python:
   4
   5 # This Source Code Form is subject to the terms of the Mozilla Public
   6 # License, v. 2.0. If a copy of the MPL was not distributed with this
   7 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8
   9 # The beginning of this script is both valid shell and valid python,
  10 # such that the script starts with the shell and is reexecuted python
  11 ''':'
  12 which mach > /dev/null 2>&1 && exec mach python "$0" "$@" ||
  13 echo "mach not found, either add it to your \$PATH or run this script via ./mach python testing/runtimes/writeruntimes"; exit  # noqa
  14 '''
  15
  16 from __future__ import absolute_import, print_function
  17
  18 import datetime
  19 import json
  20 import os
  21 import sys
  22 import time
  23 from argparse import ArgumentParser
  24 from collections import defaultdict
  25
  26 import requests
  27
  28 from moztest.resolve import (
  29     TestManifestLoader,
  30     TestResolver,
  31     TEST_SUITES,
  32 )
  33
  34 here = os.path.abspath(os.path.dirname(__file__))
  35 ACTIVE_DATA_URL = "https://activedata.allizom.org/query"
  36 EXCEED_LIMIT = [
  37     # Suites that exceed 10,000 ActiveData result limit will be defined here.
  38     'web-platform-tests',
  39     'web-platform-tests-reftest',
  40 ]
  41 MAX_RETRIES = 10
  42 RETRY_INTERVAL = 10
  43
  44
  45 def construct_query(suite, platform):
  46     if platform in ('windows', 'android'):
  47         platform_clause = '{"find":{"run.machine.platform": "%s"}}' % platform
  48     else:
  49         # Bundle macosx and linux results together - they are not too different.
  50         platform_clause = '''
  51             {
  52                 "not": {
  53                     "or": [
  54                         {"find":{"run.machine.platform": "windows"}},
  55                         {"find":{"run.machine.platform": "android"}}
  56                     ]
  57                 }
  58             }
  59         '''
  60
  61     # Only use this if the suite being queried exceeeds 10,000 results.
  62     output_clause = '"destination": "url",\n"format": "list",' if suite in EXCEED_LIMIT else ''
  63
  64     query = """
  65 {
  66     "from":"unittest",
  67     "limit":200000,
  68     "groupby":["result.test"],
  69     "select":{"value":"result.duration","aggregate":"median"},
  70     %s
  71     "where":{"and":[
  72         {"eq":{"repo.branch.name": "mozilla-central"}},
  73         {"in":{"result.status": ["OK", "PASS", "FAIL"]}},
  74         {"gt":{"run.timestamp": {"date": "today-week"}}},
  75         {"eq":{"run.suite.fullname":"%s"}},
  76         %s
  77     ]}
  78 }
  79 """ % (output_clause, suite, platform_clause)
  80
  81     return query
  82
  83
  84 def query_activedata(suite, platform):
  85     query = construct_query(suite, platform)
  86     print("Querying ActiveData for '{}' tests on '{}' platforms.. "
  87             .format(suite, platform), end='')
  88     sys.stdout.flush()
  89     response = requests.post(ACTIVE_DATA_URL,
  90                              data=query,
  91                              stream=True)
  92     response.raise_for_status()
  93
  94     # Presence of destination clause in the query requires additional processing
  95     # to produce the dataset that can be used.
  96     if suite in EXCEED_LIMIT:
  97         # The output_url is where result of the query will be stored.
  98         output_url = response.json()["url"]
  99
 100         tried = 0
 101         while tried < MAX_RETRIES:
 102             # Use the requests.Session object to manage requests, since the output_url
 103             # can often return 403 Forbidden.
 104             session = requests.Session()
 105             response = session.get(output_url)
 106             if response.status_code == 200:
 107                 break
 108             # A non-200 status code means we should retry after some wait.
 109             time.sleep(RETRY_INTERVAL)
 110             tried += 1
 111
 112         # Data returned from destination is in format of:
 113         # {data: [result: {test: test_name, duration: duration}]}
 114         # Normalize it to the format expected by compute_manifest_runtimes.
 115         raw_data = response.json()["data"]
 116         data = dict([[item['result']['test'], item['result']['duration']] for item in raw_data])
 117     else:
 118         data = dict(response.json()["data"])
 119
 120     print("{} found".format(len(data)))
 121     return data
 122
 123
 124 def write_runtimes(manifest_runtimes, platform, suite, outdir=here):
 125     if not os.path.exists(outdir):
 126         os.makedirs(outdir)
 127
 128     outfilename = os.path.join(outdir, "manifest-runtimes-{}.json".format(platform))
 129     # If file is not present, initialize a file with empty JSON object.
 130     if not os.path.exists(outfilename):
 131         with open(outfilename, 'w+') as f:
 132             json.dump({}, f)
 133
 134     # Load the entire file.
 135     with open(outfilename, 'r') as f:
 136         data = json.load(f)
 137
 138     # Update the specific suite with the new runtime information and write to file.
 139     data[suite] = manifest_runtimes
 140     with open(outfilename, 'w') as f:
 141         json.dump(data, f, indent=2, sort_keys=True)
 142
 143
 144 def compute_manifest_runtimes(suite, platform):
 145     resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
 146
 147     crashtest_prefixes = {
 148         'http': '/tests/',
 149         'chrome': '/reftest/content/',
 150         'file': '/reftest/tests/',
 151     }
 152     manifest_runtimes = defaultdict(float)
 153     data = query_activedata(suite, platform)
 154
 155     if "web-platform-tests" in suite:
 156         wpt_groups = {t["name"]: t["manifest"]
 157                         for t in resolver.resolve_tests(flavor="web-platform-tests")}
 158
 159     for path, duration in data.items():
 160         # Returned data did not contain a test path, so go to next result.
 161         if not path:
 162             continue
 163
 164         if suite in ('reftest', 'crashtest') and ' ' in path:
 165             path = path.split()[0]
 166
 167         if suite == 'crashtest' and '://' in path:
 168             # Crashtest paths are URLs with various schemes and prefixes.
 169             # Normalize it to become relative to mozilla-central.
 170             scheme = path[:path.index('://')]
 171             if ':' in scheme:
 172                 scheme = scheme.split(':')[-1]
 173             prefix = crashtest_prefixes[scheme]
 174             path = path.split(prefix, 1)[-1]
 175         elif suite == 'xpcshell' and ':' in path:
 176             path = path.split(':', 1)[-1]
 177
 178         if "web-platform-tests" in suite:
 179             if path in wpt_groups:
 180                 manifest_runtimes[wpt_groups[path]] += duration
 181             continue
 182
 183         if path not in resolver.tests_by_path:
 184             continue
 185
 186         for test in resolver.tests_by_path[path]:
 187             manifest = test.get('ancestor_manifest') or test['manifest_relpath']
 188             manifest_runtimes[manifest] += duration
 189
 190     manifest_runtimes = {k: round(v, 2) for k, v in manifest_runtimes.items()}
 191     return manifest_runtimes
 192
 193
 194 def cli(args=sys.argv[1:]):
 195     default_suites = [suite for suite, obj in TEST_SUITES.items() if 'build_flavor' in obj]
 196     default_platforms = ['android', 'windows', 'unix']
 197
 198     parser = ArgumentParser()
 199     parser.add_argument('-o', '--output-directory', dest='outdir', default=here,
 200                         help="Directory to save runtime data.")
 201     parser.add_argument('-s', '--suite', dest='suites', action='append',
 202                         default=None, choices=default_suites,
 203                         help="Suite(s) to include in the data set (default: all)")
 204     parser.add_argument('-p', '--platform', dest='platforms', action='append',
 205                         default=None, choices=default_platforms,
 206                         help="Platform(s) to gather runtime information on "
 207                              "(default: all).")
 208     args = parser.parse_args(args)
 209
 210     # If a suite was specified, use that. Otherwise, use the full default set.
 211     suites = args.suites or default_suites
 212     # Same as above, but for the platform clause.
 213     platforms = args.platforms or default_platforms
 214
 215     for platform in platforms:
 216         for suite in suites:
 217             runtimes = compute_manifest_runtimes(suite, platform)
 218             if not runtimes:
 219                 print("Not writing runtime data for '{}' for '{}' as no data was found".format(suite, platform))
 220                 continue
 221
 222             write_runtimes(runtimes, platform, suite, outdir=args.outdir)
 223
 224
 225 if __name__ == "__main__":
 226     sys.exit(cli())