testing/web-platform/metasummary.py

   1 import argparse
   2 import json
   3 import logging
   4 import os
   5 import re
   6 import urlparse
   7 from collections import defaultdict
   8
   9 import manifestupdate
  10
  11 from wptrunner import expected
  12 from wptrunner.wptmanifest.serializer import serialize
  13 from wptrunner.wptmanifest.backends import base
  14
  15 here = os.path.dirname(__file__)
  16 logger = logging.getLogger(__name__)
  17 yaml = None
  18
  19
  20 class Compiler(base.Compiler):
  21     def visit_KeyValueNode(self, node):
  22         key_name = node.data
  23         values = []
  24         for child in node.children:
  25             values.append(self.visit(child))
  26
  27         self.output_node.set(key_name, values)
  28
  29     def visit_ConditionalNode(self, node):
  30         assert len(node.children) == 2
  31         # For conditional nodes, just return the subtree
  32         return node.children[0], self.visit(node.children[1])
  33
  34     def visit_UnaryExpressionNode(self, node):
  35         raise NotImplementedError
  36
  37     def visit_BinaryExpressionNode(self, node):
  38         raise NotImplementedError
  39
  40     def visit_UnaryOperatorNode(self, node):
  41         raise NotImplementedError
  42
  43     def visit_BinaryOperatorNode(self, node):
  44         raise NotImplementedError
  45
  46
  47 class ExpectedManifest(base.ManifestItem):
  48     def __init__(self, node, test_path, url_base):
  49         """Object representing all the tests in a particular manifest
  50
  51         :param name: Name of the AST Node associated with this object.
  52                      Should always be None since this should always be associated with
  53                      the root node of the AST.
  54         :param test_path: Path of the test file associated with this manifest.
  55         :param url_base: Base url for serving the tests in this manifest
  56         """
  57         if test_path is None:
  58             raise ValueError("ExpectedManifest requires a test path")
  59         if url_base is None:
  60             raise ValueError("ExpectedManifest requires a base url")
  61         base.ManifestItem.__init__(self, node)
  62         self.child_map = {}
  63         self.test_path = test_path
  64         self.url_base = url_base
  65
  66     def append(self, child):
  67         """Add a test to the manifest"""
  68         base.ManifestItem.append(self, child)
  69         self.child_map[child.id] = child
  70
  71     @property
  72     def url(self):
  73         return urlparse.urljoin(self.url_base,
  74                                 "/".join(self.test_path.split(os.path.sep)))
  75
  76
  77 class DirectoryManifest(base.ManifestItem):
  78     pass
  79
  80
  81 class TestManifestItem(base.ManifestItem):
  82     def __init__(self, node, **kwargs):
  83         """Tree node associated with a particular test in a manifest
  84
  85         :param name: name of the test"""
  86         base.ManifestItem.__init__(self, node)
  87         self.subtests = {}
  88
  89     @property
  90     def id(self):
  91         return urlparse.urljoin(self.parent.url, self.name)
  92
  93     def append(self, node):
  94         """Add a subtest to the current test
  95
  96         :param node: AST Node associated with the subtest"""
  97         child = base.ManifestItem.append(self, node)
  98         self.subtests[child.name] = child
  99
 100     def get_subtest(self, name):
 101         """Get the SubtestNode corresponding to a particular subtest, by name
 102
 103         :param name: Name of the node to return"""
 104         if name in self.subtests:
 105             return self.subtests[name]
 106         return None
 107
 108
 109 class SubtestManifestItem(TestManifestItem):
 110     pass
 111
 112
 113 def data_cls_getter(output_node, visited_node):
 114     # visited_node is intentionally unused
 115     if output_node is None:
 116         return ExpectedManifest
 117     if isinstance(output_node, ExpectedManifest):
 118         return TestManifestItem
 119     if isinstance(output_node, TestManifestItem):
 120         return SubtestManifestItem
 121     raise ValueError
 122
 123
 124 def get_manifest(metadata_root, test_path, url_base):
 125     """Get the ExpectedManifest for a particular test path, or None if there is no
 126     metadata stored for that test path.
 127
 128     :param metadata_root: Absolute path to the root of the metadata directory
 129     :param test_path: Path to the test(s) relative to the test root
 130     :param url_base: Base url for serving the tests in this manifest
 131     :param run_info: Dictionary of properties of the test run for which the expectation
 132                      values should be computed.
 133     """
 134     manifest_path = expected.expected_path(metadata_root, test_path)
 135     try:
 136         with open(manifest_path) as f:
 137             return compile(f,
 138                            data_cls_getter=data_cls_getter,
 139                            test_path=test_path,
 140                            url_base=url_base)
 141     except IOError:
 142         return None
 143
 144
 145 def get_dir_manifest(path):
 146     """Get the ExpectedManifest for a particular test path, or None if there is no
 147     metadata stored for that test path.
 148
 149     :param path: Full path to the ini file
 150     :param run_info: Dictionary of properties of the test run for which the expectation
 151                      values should be computed.
 152     """
 153     try:
 154         with open(path) as f:
 155             return compile(f, data_cls_getter=lambda x,y: DirectoryManifest)
 156     except IOError:
 157         return None
 158
 159
 160 def compile(stream, data_cls_getter=None, **kwargs):
 161     return base.compile(Compiler,
 162                         stream,
 163                         data_cls_getter=data_cls_getter,
 164                         **kwargs)
 165
 166
 167 def create_parser():
 168     parser = argparse.ArgumentParser()
 169     parser.add_argument("--out-dir", help="Directory to store output files")
 170     parser.add_argument("--meta-dir", help="Directory containing wpt-metadata "
 171                         "checkout to update.")
 172     return parser
 173
 174
 175 def run(src_root, obj_root, logger_=None, **kwargs):
 176     logger_obj = logger_ if logger_ is not None else logger
 177
 178     manifests = manifestupdate.run(src_root, obj_root, logger_obj, **kwargs)
 179
 180     rv = {}
 181     dirs_seen = set()
 182
 183     for meta_root, test_path, test_metadata in iter_tests(manifests):
 184         for dir_path in get_dir_paths(meta_root, test_path):
 185             if dir_path not in dirs_seen:
 186                 dirs_seen.add(dir_path)
 187                 dir_manifest = get_dir_manifest(dir_path)
 188                 rel_path = os.path.relpath(dir_path, meta_root)
 189                 if dir_manifest:
 190                     add_manifest(rv, rel_path, dir_manifest)
 191             else:
 192                 break
 193         add_manifest(rv, test_path, test_metadata)
 194
 195     if kwargs["out_dir"]:
 196         if not os.path.exists(kwargs["out_dir"]):
 197             os.makedirs(kwargs["out_dir"])
 198         out_path = os.path.join(kwargs["out_dir"], "summary.json")
 199         with open(out_path, "w") as f:
 200             json.dump(rv, f)
 201     else:
 202         print(json.dumps(rv, indent=2))
 203
 204     if kwargs["meta_dir"]:
 205         update_wpt_meta(logger_obj, kwargs["meta_dir"], rv)
 206
 207
 208 def get_dir_paths(test_root, test_path):
 209     if not os.path.isabs(test_path):
 210         test_path = os.path.join(test_root, test_path)
 211     dir_path = os.path.dirname(test_path)
 212     while dir_path != test_root:
 213         yield os.path.join(dir_path, "__dir__.ini")
 214         dir_path = os.path.dirname(dir_path)
 215         assert len(dir_path) >= len(test_root)
 216
 217
 218 def iter_tests(manifests):
 219     for manifest in manifests.iterkeys():
 220         for test_type, test_path, tests in manifest:
 221             url_base = manifests[manifest]["url_base"]
 222             metadata_base = manifests[manifest]["metadata_path"]
 223             expected_manifest = get_manifest(metadata_base, test_path, url_base)
 224             if expected_manifest:
 225                 yield metadata_base, test_path, expected_manifest
 226
 227
 228 def add_manifest(target, path, metadata):
 229     dir_name = os.path.dirname(path)
 230     key = [dir_name]
 231
 232     add_metadata(target, key, metadata)
 233
 234     key.append("_tests")
 235
 236     for test_metadata in metadata.children:
 237         key.append(test_metadata.name)
 238         add_metadata(target, key, test_metadata)
 239         key.append("_subtests")
 240         for subtest_metadata in test_metadata.children:
 241             key.append(subtest_metadata.name)
 242             add_metadata(target,
 243                          key,
 244                          subtest_metadata)
 245             key.pop()
 246         key.pop()
 247         key.pop()
 248
 249
 250 simple_props = ["disabled", "min-asserts", "max-asserts", "lsan-allowed",
 251                 "leak-allowed", "bug"]
 252 statuses = set(["CRASH"])
 253
 254
 255 def add_metadata(target, key, metadata):
 256     if not is_interesting(metadata):
 257         return
 258
 259     for part in key:
 260         if part not in target:
 261             target[part] = {}
 262         target = target[part]
 263
 264     for prop in simple_props:
 265         if metadata.has_key(prop):
 266             target[prop] = get_condition_value_list(metadata, prop)
 267
 268     if metadata.has_key("expected"):
 269         intermittent = []
 270         values = metadata.get("expected")
 271         by_status = defaultdict(list)
 272         for item in values:
 273             if isinstance(item, tuple):
 274                 condition, status = item
 275             else:
 276                 condition = None
 277                 status = item
 278             if isinstance(status, list):
 279                 intermittent.append((condition, status))
 280                 expected_status = status[0]
 281             else:
 282                 expected_status = status
 283             by_status[expected_status].append(condition)
 284         for status in statuses:
 285             if status in by_status:
 286                 target["expected_%s" % status] = [serialize(item) if item else None
 287                                                   for item in by_status[status]]
 288         if intermittent:
 289             target["intermittent"] = [[serialize(cond) if cond else None, intermittent_statuses]
 290                                       for cond, intermittent_statuses in intermittent]
 291
 292
 293 def get_condition_value_list(metadata, key):
 294     conditions = []
 295     for item in metadata.get(key):
 296         if isinstance(item, tuple):
 297             assert len(item) == 2
 298             conditions.append((serialize(item[0]), item[1]))
 299         else:
 300             conditions.append((None, item))
 301     return conditions
 302
 303
 304 def is_interesting(metadata):
 305     if any(metadata.has_key(prop) for prop in simple_props):
 306         return True
 307
 308     if metadata.has_key("expected"):
 309         for expected_value in metadata.get("expected"):
 310             # Include both expected and known intermittent values
 311             if isinstance(expected_value, tuple):
 312                 expected_value = expected_value[1]
 313             if isinstance(expected_value, list):
 314                 return True
 315             if expected_value in statuses:
 316                 return True
 317             return True
 318     return False
 319
 320
 321 def update_wpt_meta(logger, meta_root, data):
 322     global yaml
 323     import yaml
 324
 325     if not os.path.exists(meta_root) or not os.path.isdir(meta_root):
 326         raise ValueError("%s is not a directory" % (meta_root,))
 327
 328     with WptMetaCollection(meta_root) as wpt_meta:
 329         for dir_path, dir_data in sorted(data.iteritems()):
 330             for test, test_data in dir_data.get("_tests", {}).iteritems():
 331                 add_test_data(logger, wpt_meta, dir_path, test, None, test_data)
 332                 for subtest, subtest_data in test_data.get("_subtests", {}).iteritems():
 333                     add_test_data(logger, wpt_meta, dir_path, test, subtest, subtest_data)
 334
 335 def add_test_data(logger, wpt_meta, dir_path, test, subtest, test_data):
 336     triage_keys = ["bug"]
 337
 338     for key in triage_keys:
 339         if key in test_data:
 340             value = test_data[key]
 341             for cond_value in value:
 342                 if cond_value[0] is not None:
 343                     logger.info("Skipping conditional metadata")
 344                     continue
 345                 cond_value = cond_value[1]
 346                 if not isinstance(cond_value, list):
 347                     cond_value = [cond_value]
 348                 for bug_value in cond_value:
 349                     bug_link = get_bug_link(bug_value)
 350                     if bug_link is None:
 351                         logger.info("Could not extract bug: %s" % value)
 352                         continue
 353                     meta = wpt_meta.get(dir_path)
 354                     meta.set(test,
 355                              subtest,
 356                              product="firefox",
 357                              bug_url=bug_link)
 358
 359
 360 bugzilla_re = re.compile("https://bugzilla\.mozilla\.org/show_bug\.cgi\?id=\d+")
 361 bug_re = re.compile("(?:[Bb][Uu][Gg])?\s*(\d+)")
 362
 363
 364 def get_bug_link(value):
 365     value = value.strip()
 366     m = bugzilla_re.match(value)
 367     if m:
 368         return m.group(0)
 369     m = bug_re.match(value)
 370     if m:
 371         return "https://bugzilla.mozilla.org/show_bug.cgi?id=%s" % m.group(1)
 372
 373
 374 class WptMetaCollection(object):
 375     def __init__(self, root):
 376         self.root = root
 377         self.loaded = {}
 378
 379     def __enter__(self):
 380         return self
 381
 382     def __exit__(self, *args, **kwargs):
 383         for item in self.loaded.itervalues():
 384             item.write(self.root)
 385         self.loaded = {}
 386
 387     def get(self, dir_path):
 388         if dir_path not in self.loaded:
 389             meta = WptMeta.get_or_create(self.root, dir_path)
 390             self.loaded[dir_path] = meta
 391         return self.loaded[dir_path]
 392
 393
 394 class WptMeta(object):
 395     def __init__(self, dir_path, data):
 396         assert "links" in data and isinstance(data["links"], list)
 397         self.dir_path = dir_path
 398         self.data = data
 399
 400     @staticmethod
 401     def meta_path(meta_root, dir_path):
 402         return os.path.join(meta_root, dir_path, "META.yml")
 403
 404     def path(self, meta_root):
 405         return self.meta_path(meta_root, self.dir_path)
 406
 407     @classmethod
 408     def get_or_create(cls, meta_root, dir_path):
 409         if os.path.exists(cls.meta_path(meta_root, dir_path)):
 410             return cls.load(meta_root, dir_path)
 411         return cls(dir_path, {"links": []})
 412
 413     @classmethod
 414     def load(cls, meta_root, dir_path):
 415         with open(cls.meta_path(meta_root, dir_path), "r") as f:
 416             data = yaml.safe_load(f)
 417         return cls(dir_path, data)
 418
 419     def set(self, test, subtest, product, bug_url):
 420         target_link = None
 421         for link in self.data["links"]:
 422             link_product = link.get("product")
 423             if link_product:
 424                 link_product = link_product.split("-", 1)[0]
 425             if link_product is None or link_product == product:
 426                 if link["url"] == bug_url:
 427                     target_link = link
 428                     break
 429
 430         if target_link is None:
 431             target_link = {"product": product.encode("utf8"),
 432                            "url": bug_url.encode("utf8"),
 433                            "results": []}
 434             self.data["links"].append(target_link)
 435
 436         if not "results" in target_link:
 437             target_link["results"] = []
 438
 439         has_result = any((result["test"] == test and result.get("subtest") == subtest)
 440                           for result in target_link["results"])
 441         if not has_result:
 442             data = {"test": test.encode("utf8")}
 443             if subtest:
 444                 data["subtest"] = subtest.encode("utf8")
 445             target_link["results"].append(data)
 446
 447     def write(self, meta_root):
 448         path = self.path(meta_root)
 449         dirname = os.path.dirname(path)
 450         if not os.path.exists(dirname):
 451             os.makedirs(dirname)
 452         with open(path, "wb") as f:
 453             yaml.safe_dump(self.data, f,
 454                            default_flow_style=False,
 455                            allow_unicode=True)