testing/web-platform/metasummary.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import json
   7 import logging
   8 import os
   9 import re
  10 from urllib import parse as urlparse
  11 from collections import defaultdict
  12
  13 import manifestupdate
  14
  15 from wptrunner import expected
  16 from wptrunner.wptmanifest.serializer import serialize
  17 from wptrunner.wptmanifest.backends import base
  18
  19 here = os.path.dirname(__file__)
  20 logger = logging.getLogger(__name__)
  21 yaml = None
  22
  23
  24 class Compiler(base.Compiler):
  25     def visit_KeyValueNode(self, node):
  26         key_name = node.data
  27         values = []
  28         for child in node.children:
  29             values.append(self.visit(child))
  30
  31         self.output_node.set(key_name, values)
  32
  33     def visit_ConditionalNode(self, node):
  34         assert len(node.children) == 2
  35         # For conditional nodes, just return the subtree
  36         return node.children[0], self.visit(node.children[1])
  37
  38     def visit_UnaryExpressionNode(self, node):
  39         raise NotImplementedError
  40
  41     def visit_BinaryExpressionNode(self, node):
  42         raise NotImplementedError
  43
  44     def visit_UnaryOperatorNode(self, node):
  45         raise NotImplementedError
  46
  47     def visit_BinaryOperatorNode(self, node):
  48         raise NotImplementedError
  49
  50
  51 class ExpectedManifest(base.ManifestItem):
  52     def __init__(self, node, test_path, url_base):
  53         """Object representing all the tests in a particular manifest
  54
  55         :param name: Name of the AST Node associated with this object.
  56                      Should always be None since this should always be associated with
  57                      the root node of the AST.
  58         :param test_path: Path of the test file associated with this manifest.
  59         :param url_base: Base url for serving the tests in this manifest
  60         """
  61         if test_path is None:
  62             raise ValueError("ExpectedManifest requires a test path")
  63         if url_base is None:
  64             raise ValueError("ExpectedManifest requires a base url")
  65         base.ManifestItem.__init__(self, node)
  66         self.child_map = {}
  67         self.test_path = test_path
  68         self.url_base = url_base
  69
  70     def append(self, child):
  71         """Add a test to the manifest"""
  72         base.ManifestItem.append(self, child)
  73         self.child_map[child.id] = child
  74
  75     @property
  76     def url(self):
  77         return urlparse.urljoin(
  78             self.url_base, "/".join(self.test_path.split(os.path.sep))
  79         )
  80
  81
  82 class DirectoryManifest(base.ManifestItem):
  83     pass
  84
  85
  86 class TestManifestItem(base.ManifestItem):
  87     def __init__(self, node, **kwargs):
  88         """Tree node associated with a particular test in a manifest
  89
  90         :param name: name of the test"""
  91         base.ManifestItem.__init__(self, node)
  92         self.subtests = {}
  93
  94     @property
  95     def id(self):
  96         return urlparse.urljoin(self.parent.url, self.name)
  97
  98     def append(self, node):
  99         """Add a subtest to the current test
 100
 101         :param node: AST Node associated with the subtest"""
 102         child = base.ManifestItem.append(self, node)
 103         self.subtests[child.name] = child
 104
 105     def get_subtest(self, name):
 106         """Get the SubtestNode corresponding to a particular subtest, by name
 107
 108         :param name: Name of the node to return"""
 109         if name in self.subtests:
 110             return self.subtests[name]
 111         return None
 112
 113
 114 class SubtestManifestItem(TestManifestItem):
 115     pass
 116
 117
 118 def data_cls_getter(output_node, visited_node):
 119     # visited_node is intentionally unused
 120     if output_node is None:
 121         return ExpectedManifest
 122     if isinstance(output_node, ExpectedManifest):
 123         return TestManifestItem
 124     if isinstance(output_node, TestManifestItem):
 125         return SubtestManifestItem
 126     raise ValueError
 127
 128
 129 def get_manifest(metadata_root, test_path, url_base):
 130     """Get the ExpectedManifest for a particular test path, or None if there is no
 131     metadata stored for that test path.
 132
 133     :param metadata_root: Absolute path to the root of the metadata directory
 134     :param test_path: Path to the test(s) relative to the test root
 135     :param url_base: Base url for serving the tests in this manifest
 136     :param run_info: Dictionary of properties of the test run for which the expectation
 137                      values should be computed.
 138     """
 139     manifest_path = expected.expected_path(metadata_root, test_path)
 140     try:
 141         with open(manifest_path, "rb") as f:
 142             return compile(
 143                 f,
 144                 data_cls_getter=data_cls_getter,
 145                 test_path=test_path,
 146                 url_base=url_base,
 147             )
 148     except IOError:
 149         return None
 150
 151
 152 def get_dir_manifest(path):
 153     """Get the ExpectedManifest for a particular test path, or None if there is no
 154     metadata stored for that test path.
 155
 156     :param path: Full path to the ini file
 157     :param run_info: Dictionary of properties of the test run for which the expectation
 158                      values should be computed.
 159     """
 160     try:
 161         with open(path, "rb") as f:
 162             return compile(f, data_cls_getter=lambda x, y: DirectoryManifest)
 163     except IOError:
 164         return None
 165
 166
 167 def compile(stream, data_cls_getter=None, **kwargs):
 168     return base.compile(Compiler, stream, data_cls_getter=data_cls_getter, **kwargs)
 169
 170
 171 def create_parser():
 172     parser = argparse.ArgumentParser()
 173     parser.add_argument("--out-dir", help="Directory to store output files")
 174     parser.add_argument(
 175         "--meta-dir", help="Directory containing wpt-metadata " "checkout to update."
 176     )
 177     return parser
 178
 179
 180 def run(src_root, obj_root, logger_=None, **kwargs):
 181     logger_obj = logger_ if logger_ is not None else logger
 182
 183     manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs)
 184
 185     rv = {}
 186     dirs_seen = set()
 187
 188     for meta_root, test_path, test_metadata in iter_tests(manifests):
 189         for dir_path in get_dir_paths(meta_root, test_path):
 190             if dir_path not in dirs_seen:
 191                 dirs_seen.add(dir_path)
 192                 dir_manifest = get_dir_manifest(dir_path)
 193                 rel_path = os.path.relpath(dir_path, meta_root)
 194                 if dir_manifest:
 195                     add_manifest(rv, rel_path, dir_manifest)
 196             else:
 197                 break
 198         add_manifest(rv, test_path, test_metadata)
 199
 200     if kwargs["out_dir"]:
 201         if not os.path.exists(kwargs["out_dir"]):
 202             os.makedirs(kwargs["out_dir"])
 203         out_path = os.path.join(kwargs["out_dir"], "summary.json")
 204         with open(out_path, "w") as f:
 205             json.dump(rv, f)
 206     else:
 207         print(json.dumps(rv, indent=2))
 208
 209     if kwargs["meta_dir"]:
 210         update_wpt_meta(logger_obj, kwargs["meta_dir"], rv)
 211
 212
 213 def get_dir_paths(test_root, test_path):
 214     if not os.path.isabs(test_path):
 215         test_path = os.path.join(test_root, test_path)
 216     dir_path = os.path.dirname(test_path)
 217     while dir_path != test_root:
 218         yield os.path.join(dir_path, "__dir__.ini")
 219         dir_path = os.path.dirname(dir_path)
 220         assert len(dir_path) >= len(test_root)
 221
 222
 223 def iter_tests(manifests):
 224     for manifest in manifests.keys():
 225         for test_type, test_path, tests in manifest:
 226             url_base = manifests[manifest]["url_base"]
 227             metadata_base = manifests[manifest]["metadata_path"]
 228             expected_manifest = get_manifest(metadata_base, test_path, url_base)
 229             if expected_manifest:
 230                 yield metadata_base, test_path, expected_manifest
 231
 232
 233 def add_manifest(target, path, metadata):
 234     dir_name = os.path.dirname(path)
 235     key = [dir_name]
 236
 237     add_metadata(target, key, metadata)
 238
 239     key.append("_tests")
 240
 241     for test_metadata in metadata.children:
 242         key.append(test_metadata.name)
 243         add_metadata(target, key, test_metadata)
 244         key.append("_subtests")
 245         for subtest_metadata in test_metadata.children:
 246             key.append(subtest_metadata.name)
 247             add_metadata(target, key, subtest_metadata)
 248             key.pop()
 249         key.pop()
 250         key.pop()
 251
 252
 253 simple_props = [
 254     "disabled",
 255     "min-asserts",
 256     "max-asserts",
 257     "lsan-allowed",
 258     "leak-allowed",
 259     "bug",
 260 ]
 261 statuses = set(["CRASH"])
 262
 263
 264 def add_metadata(target, key, metadata):
 265     if not is_interesting(metadata):
 266         return
 267
 268     for part in key:
 269         if part not in target:
 270             target[part] = {}
 271         target = target[part]
 272
 273     for prop in simple_props:
 274         if metadata.has_key(prop):  # noqa W601
 275             target[prop] = get_condition_value_list(metadata, prop)
 276
 277     if metadata.has_key("expected"):  # noqa W601
 278         intermittent = []
 279         values = metadata.get("expected")
 280         by_status = defaultdict(list)
 281         for item in values:
 282             if isinstance(item, tuple):
 283                 condition, status = item
 284             else:
 285                 condition = None
 286                 status = item
 287             if isinstance(status, list):
 288                 intermittent.append((condition, status))
 289                 expected_status = status[0]
 290             else:
 291                 expected_status = status
 292             by_status[expected_status].append(condition)
 293         for status in statuses:
 294             if status in by_status:
 295                 target["expected_%s" % status] = [
 296                     serialize(item) if item else None for item in by_status[status]
 297                 ]
 298         if intermittent:
 299             target["intermittent"] = [
 300                 [serialize(cond) if cond else None, intermittent_statuses]
 301                 for cond, intermittent_statuses in intermittent
 302             ]
 303
 304
 305 def get_condition_value_list(metadata, key):
 306     conditions = []
 307     for item in metadata.get(key):
 308         if isinstance(item, tuple):
 309             assert len(item) == 2
 310             conditions.append((serialize(item[0]), item[1]))
 311         else:
 312             conditions.append((None, item))
 313     return conditions
 314
 315
 316 def is_interesting(metadata):
 317     if any(metadata.has_key(prop) for prop in simple_props):  # noqa W601
 318         return True
 319
 320     if metadata.has_key("expected"):  # noqa W601
 321         for expected_value in metadata.get("expected"):
 322             # Include both expected and known intermittent values
 323             if isinstance(expected_value, tuple):
 324                 expected_value = expected_value[1]
 325             if isinstance(expected_value, list):
 326                 return True
 327             if expected_value in statuses:
 328                 return True
 329             return True
 330     return False
 331
 332
 333 def update_wpt_meta(logger, meta_root, data):
 334     global yaml
 335     import yaml
 336
 337     if not os.path.exists(meta_root) or not os.path.isdir(meta_root):
 338         raise ValueError("%s is not a directory" % (meta_root,))
 339
 340     with WptMetaCollection(meta_root) as wpt_meta:
 341         for dir_path, dir_data in sorted(data.items()):
 342             for test, test_data in dir_data.get("_tests", {}).items():
 343                 add_test_data(logger, wpt_meta, dir_path, test, None, test_data)
 344                 for subtest, subtest_data in test_data.get("_subtests", {}).items():
 345                     add_test_data(
 346                         logger, wpt_meta, dir_path, test, subtest, subtest_data
 347                     )
 348
 349
 350 def add_test_data(logger, wpt_meta, dir_path, test, subtest, test_data):
 351     triage_keys = ["bug"]
 352
 353     for key in triage_keys:
 354         if key in test_data:
 355             value = test_data[key]
 356             for cond_value in value:
 357                 if cond_value[0] is not None:
 358                     logger.info("Skipping conditional metadata")
 359                     continue
 360                 cond_value = cond_value[1]
 361                 if not isinstance(cond_value, list):
 362                     cond_value = [cond_value]
 363                 for bug_value in cond_value:
 364                     bug_link = get_bug_link(bug_value)
 365                     if bug_link is None:
 366                         logger.info("Could not extract bug: %s" % value)
 367                         continue
 368                     meta = wpt_meta.get(dir_path)
 369                     meta.set(test, subtest, product="firefox", bug_url=bug_link)
 370
 371
 372 bugzilla_re = re.compile("https://bugzilla\.mozilla\.org/show_bug\.cgi\?id=\d+")
 373 bug_re = re.compile("(?:[Bb][Uu][Gg])?\s*(\d+)")
 374
 375
 376 def get_bug_link(value):
 377     value = value.strip()
 378     m = bugzilla_re.match(value)
 379     if m:
 380         return m.group(0)
 381     m = bug_re.match(value)
 382     if m:
 383         return "https://bugzilla.mozilla.org/show_bug.cgi?id=%s" % m.group(1)
 384
 385
 386 class WptMetaCollection(object):
 387     def __init__(self, root):
 388         self.root = root
 389         self.loaded = {}
 390
 391     def __enter__(self):
 392         return self
 393
 394     def __exit__(self, *args, **kwargs):
 395         for item in self.loaded.itervalues():
 396             item.write(self.root)
 397         self.loaded = {}
 398
 399     def get(self, dir_path):
 400         if dir_path not in self.loaded:
 401             meta = WptMeta.get_or_create(self.root, dir_path)
 402             self.loaded[dir_path] = meta
 403         return self.loaded[dir_path]
 404
 405
 406 class WptMeta(object):
 407     def __init__(self, dir_path, data):
 408         assert "links" in data and isinstance(data["links"], list)
 409         self.dir_path = dir_path
 410         self.data = data
 411
 412     @staticmethod
 413     def meta_path(meta_root, dir_path):
 414         return os.path.join(meta_root, dir_path, "META.yml")
 415
 416     def path(self, meta_root):
 417         return self.meta_path(meta_root, self.dir_path)
 418
 419     @classmethod
 420     def get_or_create(cls, meta_root, dir_path):
 421         if os.path.exists(cls.meta_path(meta_root, dir_path)):
 422             return cls.load(meta_root, dir_path)
 423         return cls(dir_path, {"links": []})
 424
 425     @classmethod
 426     def load(cls, meta_root, dir_path):
 427         with open(cls.meta_path(meta_root, dir_path), "r") as f:
 428             data = yaml.safe_load(f)
 429         return cls(dir_path, data)
 430
 431     def set(self, test, subtest, product, bug_url):
 432         target_link = None
 433         for link in self.data["links"]:
 434             link_product = link.get("product")
 435             if link_product:
 436                 link_product = link_product.split("-", 1)[0]
 437             if link_product is None or link_product == product:
 438                 if link["url"] == bug_url:
 439                     target_link = link
 440                     break
 441
 442         if target_link is None:
 443             target_link = {
 444                 "product": product.encode("utf8"),
 445                 "url": bug_url.encode("utf8"),
 446                 "results": [],
 447             }
 448             self.data["links"].append(target_link)
 449
 450         if "results" not in target_link:
 451             target_link["results"] = []
 452
 453         has_result = any(
 454             (result["test"] == test and result.get("subtest") == subtest)
 455             for result in target_link["results"]
 456         )
 457         if not has_result:
 458             data = {"test": test.encode("utf8")}
 459             if subtest:
 460                 data["subtest"] = subtest.encode("utf8")
 461             target_link["results"].append(data)
 462
 463     def write(self, meta_root):
 464         path = self.path(meta_root)
 465         dirname = os.path.dirname(path)
 466         if not os.path.exists(dirname):
 467             os.makedirs(dirname)
 468         with open(path, "wb") as f:
 469             yaml.safe_dump(self.data, f, default_flow_style=False, allow_unicode=True)