testing/runtimes/writeruntimes

   1 #!/bin/sh
   2 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
   3 # vim: set filetype=python:
   4
   5 # This Source Code Form is subject to the terms of the Mozilla Public
   6 # License, v. 2.0. If a copy of the MPL was not distributed with this
   7 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8
   9 # The beginning of this script is both valid shell and valid python,
  10 # such that the script starts with the shell and is reexecuted python
  11 ''':'
  12 which mach > /dev/null 2>&1 && exec mach python "$0" "$@" ||
  13 echo "mach not found, either add it to your \$PATH or run this script via ./mach python testing/runtimes/writeruntimes"; exit  # noqa
  14 '''
  15
  16 import datetime
  17 import json
  18 import os
  19 import sys
  20 import time
  21 from argparse import ArgumentParser
  22 from collections import defaultdict
  23
  24 import requests
  25
  26 from moztest.resolve import (
  27     TestManifestLoader,
  28     TestResolver,
  29     TEST_SUITES,
  30 )
  31
  32 here = os.path.abspath(os.path.dirname(__file__))
  33 ACTIVE_DATA_URL = "https://activedata.allizom.org/query"
  34 EXCEED_LIMIT = [
  35     # Suites that exceed 10,000 ActiveData result limit will be defined here.
  36     'web-platform-tests',
  37     'web-platform-tests-reftest',
  38 ]
  39 MAX_RETRIES = 10
  40 RETRY_INTERVAL = 10
  41
  42
  43 def construct_query(suite, platform):
  44     if platform in ('windows', 'android'):
  45         platform_clause = '{"find":{"run.machine.platform": "%s"}}' % platform
  46     else:
  47         # Bundle macosx and linux results together - they are not too different.
  48         platform_clause = '''
  49             {
  50                 "not": {
  51                     "or": [
  52                         {"find":{"run.machine.platform": "windows"}},
  53                         {"find":{"run.machine.platform": "android"}}
  54                     ]
  55                 }
  56             }
  57         '''
  58
  59     # Only use this if the suite being queried exceeeds 10,000 results.
  60     output_clause = '"destination": "url",\n"format": "list",' if suite in EXCEED_LIMIT else ''
  61
  62     query = """
  63 {
  64     "from":"unittest",
  65     "limit":200000,
  66     "groupby":["result.test"],
  67     "select":{"value":"result.duration","aggregate":"median"},
  68     %s
  69     "where":{"and":[
  70         {"eq":{"repo.branch.name": "mozilla-central"}},
  71         {"in":{"result.status": ["OK", "PASS", "FAIL"]}},
  72         {"gt":{"run.timestamp": {"date": "today-week"}}},
  73         {"eq":{"run.suite.fullname":"%s"}},
  74         %s
  75     ]}
  76 }
  77 """ % (output_clause, suite, platform_clause)
  78
  79     return query
  80
  81
  82 def query_activedata(suite, platform):
  83     query = construct_query(suite, platform)
  84     print("Querying ActiveData for '{}' tests on '{}' platforms.. "
  85             .format(suite, platform), end='')
  86     sys.stdout.flush()
  87     response = requests.post(ACTIVE_DATA_URL,
  88                              data=query,
  89                              stream=True)
  90     response.raise_for_status()
  91
  92     # Presence of destination clause in the query requires additional processing
  93     # to produce the dataset that can be used.
  94     if suite in EXCEED_LIMIT:
  95         # The output_url is where result of the query will be stored.
  96         output_url = response.json()["url"]
  97
  98         tried = 0
  99         while tried < MAX_RETRIES:
 100             # Use the requests.Session object to manage requests, since the output_url
 101             # can often return 403 Forbidden.
 102             session = requests.Session()
 103             response = session.get(output_url)
 104             if response.status_code == 200:
 105                 break
 106             # A non-200 status code means we should retry after some wait.
 107             time.sleep(RETRY_INTERVAL)
 108             tried += 1
 109
 110         # Data returned from destination is in format of:
 111         # {data: [result: {test: test_name, duration: duration}]}
 112         # Normalize it to the format expected by compute_manifest_runtimes.
 113         raw_data = response.json()["data"]
 114         data = dict([[item['result']['test'], item['result']['duration']] for item in raw_data])
 115     else:
 116         data = dict(response.json()["data"])
 117
 118     print("{} found".format(len(data)))
 119     return data
 120
 121
 122 def write_runtimes(manifest_runtimes, platform, suite, outdir=here):
 123     if not os.path.exists(outdir):
 124         os.makedirs(outdir)
 125
 126     outfilename = os.path.join(outdir, "manifest-runtimes-{}.json".format(platform))
 127     # If file is not present, initialize a file with empty JSON object.
 128     if not os.path.exists(outfilename):
 129         with open(outfilename, 'w+') as f:
 130             json.dump({}, f)
 131
 132     # Load the entire file.
 133     with open(outfilename, 'r') as f:
 134         data = json.load(f)
 135
 136     # Update the specific suite with the new runtime information and write to file.
 137     data[suite] = manifest_runtimes
 138     with open(outfilename, 'w') as f:
 139         json.dump(data, f, indent=2, sort_keys=True)
 140
 141
 142 def compute_manifest_runtimes(suite, platform):
 143     resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
 144
 145     crashtest_prefixes = {
 146         'http': '/tests/',
 147         'chrome': '/reftest/content/',
 148         'file': '/reftest/tests/',
 149     }
 150     manifest_runtimes = defaultdict(float)
 151     data = query_activedata(suite, platform)
 152
 153     if "web-platform-tests" in suite:
 154         wpt_groups = {t["name"]: t["manifest"]
 155                         for t in resolver.resolve_tests(flavor="web-platform-tests")}
 156
 157     for path, duration in data.items():
 158         # Returned data did not contain a test path, so go to next result.
 159         if not path:
 160             continue
 161
 162         if suite in ('reftest', 'crashtest') and ' ' in path:
 163             path = path.split()[0]
 164
 165         if suite == 'crashtest' and '://' in path:
 166             # Crashtest paths are URLs with various schemes and prefixes.
 167             # Normalize it to become relative to mozilla-central.
 168             scheme = path[:path.index('://')]
 169             if ':' in scheme:
 170                 scheme = scheme.split(':')[-1]
 171             prefix = crashtest_prefixes[scheme]
 172             path = path.split(prefix, 1)[-1]
 173         elif suite == 'xpcshell' and ':' in path:
 174             path = path.split(':', 1)[-1]
 175
 176         if "web-platform-tests" in suite:
 177             if path in wpt_groups:
 178                 manifest_runtimes[wpt_groups[path]] += duration
 179             continue
 180
 181         if path not in resolver.tests_by_path:
 182             continue
 183
 184         for test in resolver.tests_by_path[path]:
 185             manifest = test.get('ancestor_manifest') or test['manifest_relpath']
 186             manifest_runtimes[manifest] += duration
 187
 188     manifest_runtimes = {k: round(v, 2) for k, v in manifest_runtimes.items()}
 189     return manifest_runtimes
 190
 191
 192 def cli(args=sys.argv[1:]):
 193     default_suites = [suite for suite, obj in TEST_SUITES.items() if 'build_flavor' in obj]
 194     default_platforms = ['android', 'windows', 'unix']
 195
 196     parser = ArgumentParser()
 197     parser.add_argument('-o', '--output-directory', dest='outdir', default=here,
 198                         help="Directory to save runtime data.")
 199     parser.add_argument('-s', '--suite', dest='suites', action='append',
 200                         default=None, choices=default_suites,
 201                         help="Suite(s) to include in the data set (default: all)")
 202     parser.add_argument('-p', '--platform', dest='platforms', action='append',
 203                         default=None, choices=default_platforms,
 204                         help="Platform(s) to gather runtime information on "
 205                              "(default: all).")
 206     args = parser.parse_args(args)
 207
 208     # If a suite was specified, use that. Otherwise, use the full default set.
 209     suites = args.suites or default_suites
 210     # Same as above, but for the platform clause.
 211     platforms = args.platforms or default_platforms
 212
 213     for platform in platforms:
 214         for suite in suites:
 215             runtimes = compute_manifest_runtimes(suite, platform)
 216             if not runtimes:
 217                 print("Not writing runtime data for '{}' for '{}' as no data was found".format(suite, platform))
 218                 continue
 219
 220             write_runtimes(runtimes, platform, suite, outdir=args.outdir)
 221
 222
 223 if __name__ == "__main__":
 224     sys.exit(cli())