testing/web-platform/metasummary.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import json
   7 import logging
   8 import os
   9 import re
  10 from collections import defaultdict
  11 from urllib import parse as urlparse
  12
  13 import manifestupdate
  14 from wptrunner import expected
  15 from wptrunner.wptmanifest.backends import base
  16 from wptrunner.wptmanifest.serializer import serialize
  17
  18 here = os.path.dirname(__file__)
  19 logger = logging.getLogger(__name__)
  20 yaml = None
  21
  22
  23 class Compiler(base.Compiler):
  24     def visit_KeyValueNode(self, node):
  25         key_name = node.data
  26         values = []
  27         for child in node.children:
  28             values.append(self.visit(child))
  29
  30         self.output_node.set(key_name, values)
  31
  32     def visit_ConditionalNode(self, node):
  33         assert len(node.children) == 2
  34         # For conditional nodes, just return the subtree
  35         return node.children[0], self.visit(node.children[1])
  36
  37     def visit_UnaryExpressionNode(self, node):
  38         raise NotImplementedError
  39
  40     def visit_BinaryExpressionNode(self, node):
  41         raise NotImplementedError
  42
  43     def visit_UnaryOperatorNode(self, node):
  44         raise NotImplementedError
  45
  46     def visit_BinaryOperatorNode(self, node):
  47         raise NotImplementedError
  48
  49
  50 class ExpectedManifest(base.ManifestItem):
  51     def __init__(self, node, test_path, url_base):
  52         """Object representing all the tests in a particular manifest
  53
  54         :param name: Name of the AST Node associated with this object.
  55                      Should always be None since this should always be associated with
  56                      the root node of the AST.
  57         :param test_path: Path of the test file associated with this manifest.
  58         :param url_base: Base url for serving the tests in this manifest
  59         """
  60         if test_path is None:
  61             raise ValueError("ExpectedManifest requires a test path")
  62         if url_base is None:
  63             raise ValueError("ExpectedManifest requires a base url")
  64         base.ManifestItem.__init__(self, node)
  65         self.child_map = {}
  66         self.test_path = test_path
  67         self.url_base = url_base
  68
  69     def append(self, child):
  70         """Add a test to the manifest"""
  71         base.ManifestItem.append(self, child)
  72         self.child_map[child.id] = child
  73
  74     @property
  75     def url(self):
  76         return urlparse.urljoin(
  77             self.url_base, "/".join(self.test_path.split(os.path.sep))
  78         )
  79
  80
  81 class DirectoryManifest(base.ManifestItem):
  82     pass
  83
  84
  85 class TestManifestItem(base.ManifestItem):
  86     def __init__(self, node, **kwargs):
  87         """Tree node associated with a particular test in a manifest
  88
  89         :param name: name of the test"""
  90         base.ManifestItem.__init__(self, node)
  91         self.subtests = {}
  92
  93     @property
  94     def id(self):
  95         return urlparse.urljoin(self.parent.url, self.name)
  96
  97     def append(self, node):
  98         """Add a subtest to the current test
  99
 100         :param node: AST Node associated with the subtest"""
 101         child = base.ManifestItem.append(self, node)
 102         self.subtests[child.name] = child
 103
 104     def get_subtest(self, name):
 105         """Get the SubtestNode corresponding to a particular subtest, by name
 106
 107         :param name: Name of the node to return"""
 108         if name in self.subtests:
 109             return self.subtests[name]
 110         return None
 111
 112
 113 class SubtestManifestItem(TestManifestItem):
 114     pass
 115
 116
 117 def data_cls_getter(output_node, visited_node):
 118     # visited_node is intentionally unused
 119     if output_node is None:
 120         return ExpectedManifest
 121     if isinstance(output_node, ExpectedManifest):
 122         return TestManifestItem
 123     if isinstance(output_node, TestManifestItem):
 124         return SubtestManifestItem
 125     raise ValueError
 126
 127
 128 def get_manifest(metadata_root, test_path, url_base):
 129     """Get the ExpectedManifest for a particular test path, or None if there is no
 130     metadata stored for that test path.
 131
 132     :param metadata_root: Absolute path to the root of the metadata directory
 133     :param test_path: Path to the test(s) relative to the test root
 134     :param url_base: Base url for serving the tests in this manifest
 135     :param run_info: Dictionary of properties of the test run for which the expectation
 136                      values should be computed.
 137     """
 138     manifest_path = expected.expected_path(metadata_root, test_path)
 139     try:
 140         with open(manifest_path, "rb") as f:
 141             return compile(
 142                 f,
 143                 data_cls_getter=data_cls_getter,
 144                 test_path=test_path,
 145                 url_base=url_base,
 146             )
 147     except IOError:
 148         return None
 149
 150
 151 def get_dir_manifest(path):
 152     """Get the ExpectedManifest for a particular test path, or None if there is no
 153     metadata stored for that test path.
 154
 155     :param path: Full path to the ini file
 156     :param run_info: Dictionary of properties of the test run for which the expectation
 157                      values should be computed.
 158     """
 159     try:
 160         with open(path, "rb") as f:
 161             return compile(f, data_cls_getter=lambda x, y: DirectoryManifest)
 162     except IOError:
 163         return None
 164
 165
 166 def compile(stream, data_cls_getter=None, **kwargs):
 167     return base.compile(Compiler, stream, data_cls_getter=data_cls_getter, **kwargs)
 168
 169
 170 def create_parser():
 171     parser = argparse.ArgumentParser()
 172     parser.add_argument("--out-dir", help="Directory to store output files")
 173     parser.add_argument(
 174         "--meta-dir", help="Directory containing wpt-metadata " "checkout to update."
 175     )
 176     return parser
 177
 178
 179 def run(src_root, obj_root, logger_=None, **kwargs):
 180     logger_obj = logger_ if logger_ is not None else logger
 181
 182     manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs)
 183
 184     rv = {}
 185     dirs_seen = set()
 186
 187     for meta_root, test_path, test_metadata in iter_tests(manifests):
 188         for dir_path in get_dir_paths(meta_root, test_path):
 189             if dir_path not in dirs_seen:
 190                 dirs_seen.add(dir_path)
 191                 dir_manifest = get_dir_manifest(dir_path)
 192                 rel_path = os.path.relpath(dir_path, meta_root)
 193                 if dir_manifest:
 194                     add_manifest(rv, rel_path, dir_manifest)
 195             else:
 196                 break
 197         add_manifest(rv, test_path, test_metadata)
 198
 199     if kwargs["out_dir"]:
 200         if not os.path.exists(kwargs["out_dir"]):
 201             os.makedirs(kwargs["out_dir"])
 202         out_path = os.path.join(kwargs["out_dir"], "summary.json")
 203         with open(out_path, "w") as f:
 204             json.dump(rv, f)
 205     else:
 206         print(json.dumps(rv, indent=2))
 207
 208     if kwargs["meta_dir"]:
 209         update_wpt_meta(logger_obj, kwargs["meta_dir"], rv)
 210
 211
 212 def get_dir_paths(test_root, test_path):
 213     if not os.path.isabs(test_path):
 214         test_path = os.path.join(test_root, test_path)
 215     dir_path = os.path.dirname(test_path)
 216     while dir_path != test_root:
 217         yield os.path.join(dir_path, "__dir__.ini")
 218         dir_path = os.path.dirname(dir_path)
 219         assert len(dir_path) >= len(test_root)
 220
 221
 222 def iter_tests(manifests):
 223     for manifest in manifests.keys():
 224         for test_type, test_path, tests in manifest:
 225             url_base = manifests[manifest]["url_base"]
 226             metadata_base = manifests[manifest]["metadata_path"]
 227             expected_manifest = get_manifest(metadata_base, test_path, url_base)
 228             if expected_manifest:
 229                 yield metadata_base, test_path, expected_manifest
 230
 231
 232 def add_manifest(target, path, metadata):
 233     dir_name, file_name = path.rsplit(os.sep, 1)
 234     key = [dir_name]
 235
 236     add_metadata(target, key, metadata)
 237
 238     key.append("_tests")
 239
 240     for test_metadata in metadata.children:
 241         key.append(test_metadata.name)
 242         add_metadata(target, key, test_metadata)
 243         add_filename(target, key, file_name)
 244         key.append("_subtests")
 245         for subtest_metadata in test_metadata.children:
 246             key.append(subtest_metadata.name)
 247             add_metadata(target, key, subtest_metadata)
 248             key.pop()
 249         key.pop()
 250         key.pop()
 251
 252
 253 simple_props = [
 254     "disabled",
 255     "min-asserts",
 256     "max-asserts",
 257     "lsan-allowed",
 258     "leak-allowed",
 259     "bug",
 260 ]
 261 statuses = set(["CRASH"])
 262
 263
 264 def add_filename(target, key, filename):
 265     for part in key:
 266         if part not in target:
 267             target[part] = {}
 268         target = target[part]
 269
 270     target["_filename"] = filename
 271
 272
 273 def add_metadata(target, key, metadata):
 274     if not is_interesting(metadata):
 275         return
 276
 277     for part in key:
 278         if part not in target:
 279             target[part] = {}
 280         target = target[part]
 281
 282     for prop in simple_props:
 283         if metadata.has_key(prop):  # noqa W601
 284             target[prop] = get_condition_value_list(metadata, prop)
 285
 286     if metadata.has_key("expected"):  # noqa W601
 287         intermittent = []
 288         values = metadata.get("expected")
 289         by_status = defaultdict(list)
 290         for item in values:
 291             if isinstance(item, tuple):
 292                 condition, status = item
 293             else:
 294                 condition = None
 295                 status = item
 296             if isinstance(status, list):
 297                 intermittent.append((condition, status))
 298                 expected_status = status[0]
 299             else:
 300                 expected_status = status
 301             by_status[expected_status].append(condition)
 302         for status in statuses:
 303             if status in by_status:
 304                 target["expected_%s" % status] = [
 305                     serialize(item) if item else None for item in by_status[status]
 306                 ]
 307         if intermittent:
 308             target["intermittent"] = [
 309                 [serialize(cond) if cond else None, intermittent_statuses]
 310                 for cond, intermittent_statuses in intermittent
 311             ]
 312
 313
 314 def get_condition_value_list(metadata, key):
 315     conditions = []
 316     for item in metadata.get(key):
 317         if isinstance(item, tuple):
 318             assert len(item) == 2
 319             conditions.append((serialize(item[0]), item[1]))
 320         else:
 321             conditions.append((None, item))
 322     return conditions
 323
 324
 325 def is_interesting(metadata):
 326     if any(metadata.has_key(prop) for prop in simple_props):  # noqa W601
 327         return True
 328
 329     if metadata.has_key("expected"):  # noqa W601
 330         for expected_value in metadata.get("expected"):
 331             # Include both expected and known intermittent values
 332             if isinstance(expected_value, tuple):
 333                 expected_value = expected_value[1]
 334             if isinstance(expected_value, list):
 335                 return True
 336             if expected_value in statuses:
 337                 return True
 338             return True
 339     return False
 340
 341
 342 def update_wpt_meta(logger, meta_root, data):
 343     global yaml
 344     import yaml
 345
 346     if not os.path.exists(meta_root) or not os.path.isdir(meta_root):
 347         raise ValueError("%s is not a directory" % (meta_root,))
 348
 349     with WptMetaCollection(meta_root) as wpt_meta:
 350         for dir_path, dir_data in sorted(data.items()):
 351             for test, test_data in dir_data.get("_tests", {}).items():
 352                 add_test_data(logger, wpt_meta, dir_path, test, None, test_data)
 353                 for subtest, subtest_data in test_data.get("_subtests", {}).items():
 354                     add_test_data(
 355                         logger, wpt_meta, dir_path, test, subtest, subtest_data
 356                     )
 357
 358
 359 def add_test_data(logger, wpt_meta, dir_path, test, subtest, test_data):
 360     triage_keys = ["bug"]
 361
 362     for key in triage_keys:
 363         if key in test_data:
 364             value = test_data[key]
 365             for cond_value in value:
 366                 if cond_value[0] is not None:
 367                     logger.info("Skipping conditional metadata")
 368                     continue
 369                 cond_value = cond_value[1]
 370                 if not isinstance(cond_value, list):
 371                     cond_value = [cond_value]
 372                 for bug_value in cond_value:
 373                     bug_link = get_bug_link(bug_value)
 374                     if bug_link is None:
 375                         logger.info("Could not extract bug: %s" % value)
 376                         continue
 377                     meta = wpt_meta.get(dir_path)
 378                     meta.set(test, subtest, product="firefox", bug_url=bug_link)
 379
 380
 381 bugzilla_re = re.compile(r"https://bugzilla\.mozilla\.org/show_bug\.cgi\?id=\d+")
 382 bug_re = re.compile(r"(?:[Bb][Uu][Gg])?\s*(\d+)")
 383
 384
 385 def get_bug_link(value):
 386     value = value.strip()
 387     m = bugzilla_re.match(value)
 388     if m:
 389         return m.group(0)
 390     m = bug_re.match(value)
 391     if m:
 392         return "https://bugzilla.mozilla.org/show_bug.cgi?id=%s" % m.group(1)
 393
 394
 395 class WptMetaCollection(object):
 396     def __init__(self, root):
 397         self.root = root
 398         self.loaded = {}
 399
 400     def __enter__(self):
 401         return self
 402
 403     def __exit__(self, *args, **kwargs):
 404         for item in self.loaded.itervalues():
 405             item.write(self.root)
 406         self.loaded = {}
 407
 408     def get(self, dir_path):
 409         if dir_path not in self.loaded:
 410             meta = WptMeta.get_or_create(self.root, dir_path)
 411             self.loaded[dir_path] = meta
 412         return self.loaded[dir_path]
 413
 414
 415 class WptMeta(object):
 416     def __init__(self, dir_path, data):
 417         assert "links" in data and isinstance(data["links"], list)
 418         self.dir_path = dir_path
 419         self.data = data
 420
 421     @staticmethod
 422     def meta_path(meta_root, dir_path):
 423         return os.path.join(meta_root, dir_path, "META.yml")
 424
 425     def path(self, meta_root):
 426         return self.meta_path(meta_root, self.dir_path)
 427
 428     @classmethod
 429     def get_or_create(cls, meta_root, dir_path):
 430         if os.path.exists(cls.meta_path(meta_root, dir_path)):
 431             return cls.load(meta_root, dir_path)
 432         return cls(dir_path, {"links": []})
 433
 434     @classmethod
 435     def load(cls, meta_root, dir_path):
 436         with open(cls.meta_path(meta_root, dir_path), "r") as f:
 437             data = yaml.safe_load(f)
 438         return cls(dir_path, data)
 439
 440     def set(self, test, subtest, product, bug_url):
 441         target_link = None
 442         for link in self.data["links"]:
 443             link_product = link.get("product")
 444             if link_product:
 445                 link_product = link_product.split("-", 1)[0]
 446             if link_product is None or link_product == product:
 447                 if link["url"] == bug_url:
 448                     target_link = link
 449                     break
 450
 451         if target_link is None:
 452             target_link = {
 453                 "product": product.encode("utf8"),
 454                 "url": bug_url.encode("utf8"),
 455                 "results": [],
 456             }
 457             self.data["links"].append(target_link)
 458
 459         if "results" not in target_link:
 460             target_link["results"] = []
 461
 462         has_result = any(
 463             (result["test"] == test and result.get("subtest") == subtest)
 464             for result in target_link["results"]
 465         )
 466         if not has_result:
 467             data = {"test": test.encode("utf8")}
 468             if subtest:
 469                 data["subtest"] = subtest.encode("utf8")
 470             target_link["results"].append(data)
 471
 472     def write(self, meta_root):
 473         path = self.path(meta_root)
 474         dirname = os.path.dirname(path)
 475         if not os.path.exists(dirname):
 476             os.makedirs(dirname)
 477         with open(path, "wb") as f:
 478             yaml.safe_dump(self.data, f, default_flow_style=False, allow_unicode=True)