testing/web-platform/metamerge.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import logging
   7 import os
   8 import sys
   9 from collections import namedtuple
  10
  11 from wptrunner.wptmanifest.serializer import serialize
  12 from wptrunner.wptmanifest.backends import base
  13 from wptrunner.wptmanifest.node import KeyValueNode
  14
  15 here = os.path.dirname(__file__)
  16 logger = logging.getLogger(__name__)
  17
  18
  19 class Compiler(base.Compiler):
  20     def visit_KeyValueNode(self, node):
  21         key_name = node.data
  22         values = []
  23         for child in node.children:
  24             values.append(self.visit(child))
  25
  26         self.output_node.set(key_name, values)
  27
  28     def visit_ConditionalNode(self, node):
  29         assert len(node.children) == 2
  30         # For conditional nodes, just return the subtree
  31         return serialize(node.children[0]), self.visit(node.children[1])
  32
  33     def visit_UnaryExpressionNode(self, node):
  34         raise NotImplementedError
  35
  36     def visit_BinaryExpressionNode(self, node):
  37         raise NotImplementedError
  38
  39     def visit_UnaryOperatorNode(self, node):
  40         raise NotImplementedError
  41
  42     def visit_BinaryOperatorNode(self, node):
  43         raise NotImplementedError
  44
  45
  46 class ExpectedManifest(base.ManifestItem):
  47     def __init__(self, node):
  48         """Object representing all the tests in a particular manifest"""
  49         base.ManifestItem.__init__(self, node)
  50         self.child_map = {}
  51
  52     def append(self, child):
  53         """Add a test to the manifest"""
  54         base.ManifestItem.append(self, child)
  55         self.child_map[child.name] = child
  56
  57     def insert(self, child):
  58         self.append(child)
  59         self.node.append(child.node)
  60
  61     def delete(self, child):
  62         del self.child_map[child.name]
  63         child.node.remove()
  64         child.remove()
  65
  66
  67 class TestManifestItem(ExpectedManifest):
  68     def set_expected(self, other_manifest):
  69         for item in self.node.children:
  70             if isinstance(item, KeyValueNode) and item.data == "expected":
  71                 assert "expected" in self._data
  72                 item.remove()
  73                 del self._data["expected"]
  74                 break
  75
  76         for item in other_manifest.node.children:
  77             if isinstance(item, KeyValueNode) and item.data == "expected":
  78                 assert "expected" in other_manifest._data
  79                 item.remove()
  80                 self.node.children.insert(0, item)
  81                 self._data["expected"] = other_manifest._data.pop("expected")
  82                 break
  83
  84
  85 def data_cls_getter(output_node, visited_node):
  86     # visited_node is intentionally unused
  87     if output_node is None:
  88         return ExpectedManifest
  89     if isinstance(output_node, ExpectedManifest):
  90         return TestManifestItem
  91     raise ValueError
  92
  93
  94 def compile(stream, data_cls_getter=None, **kwargs):
  95     return base.compile(Compiler, stream, data_cls_getter=data_cls_getter, **kwargs)
  96
  97
  98 def get_manifest(manifest_path):
  99     """Get the ExpectedManifest for a particular manifest path"""
 100     try:
 101         with open(manifest_path, "rb") as f:
 102             try:
 103                 return compile(f, data_cls_getter=data_cls_getter)
 104             except Exception:
 105                 f.seek(0)
 106                 sys.stderr.write("Error parsing:\n%s" % f.read().decode("utf8"))
 107                 raise
 108     except IOError:
 109         return None
 110
 111
 112 def indent(str_data, indent=2):
 113     rv = []
 114     for line in str_data.splitlines():
 115         rv.append("%s%s" % (" " * indent, line))
 116     return "\n".join(rv)
 117
 118
 119 class Differences(object):
 120     def __init__(self):
 121         self.added = []
 122         self.deleted = []
 123         self.modified = []
 124
 125     def __nonzero__(self):
 126         return bool(self.added or self.deleted or self.modified)
 127
 128     def __str__(self):
 129         modified = []
 130         for item in self.modified:
 131             if isinstance(item, TestModified):
 132                 modified.append(
 133                     "  %s\n    %s\n%s" % (item[0], item[1], indent(str(item[2]), 4))
 134                 )
 135             else:
 136                 assert isinstance(item, ExpectedModified)
 137                 modified.append("  %s\n    %s %s" % item)
 138         return "Added:\n%s\nDeleted:\n%s\nModified:\n%s\n" % (
 139             "\n".join("  %s:\n    %s" % item for item in self.added),
 140             "\n".join("  %s" % item for item in self.deleted),
 141             "\n".join(modified),
 142         )
 143
 144
 145 TestModified = namedtuple("TestModified", ["test", "test_manifest", "differences"])
 146
 147
 148 ExpectedModified = namedtuple(
 149     "ExpectedModified", ["test", "ancestor_manifest", "new_manifest"]
 150 )
 151
 152
 153 def compare_test(test, ancestor_manifest, new_manifest):
 154     changes = Differences()
 155
 156     compare_expected(changes, None, ancestor_manifest, new_manifest)
 157
 158     for subtest, ancestor_subtest_manifest in ancestor_manifest.child_map.items():
 159         compare_expected(
 160             changes,
 161             subtest,
 162             ancestor_subtest_manifest,
 163             new_manifest.child_map.get(subtest),
 164         )
 165
 166     for subtest, subtest_manifest in new_manifest.child_map.items():
 167         if subtest not in ancestor_manifest.child_map:
 168             changes.added.append((subtest, subtest_manifest))
 169
 170     return changes
 171
 172
 173 def compare_expected(changes, subtest, ancestor_manifest, new_manifest):
 174     if not (
 175         ancestor_manifest and ancestor_manifest.has_key("expected")  # noqa W601
 176     ) and (
 177         new_manifest and new_manifest.has_key("expected")  # noqa W601
 178     ):
 179         changes.modified.append(
 180             ExpectedModified(subtest, ancestor_manifest, new_manifest)
 181         )
 182     elif (
 183         ancestor_manifest
 184         and ancestor_manifest.has_key("expected")  # noqa W601
 185         and not (new_manifest and new_manifest.has_key("expected"))  # noqa W601
 186     ):
 187         changes.deleted.append(subtest)
 188     elif (
 189         ancestor_manifest
 190         and ancestor_manifest.has_key("expected")  # noqa W601
 191         and new_manifest
 192         and new_manifest.has_key("expected")  # noqa W601
 193     ):
 194         old_expected = ancestor_manifest.get("expected")
 195         new_expected = new_manifest.get("expected")
 196         if expected_values_changed(old_expected, new_expected):
 197             changes.modified.append(
 198                 ExpectedModified(subtest, ancestor_manifest, new_manifest)
 199             )
 200
 201
 202 def expected_values_changed(old_expected, new_expected):
 203     if len(old_expected) != len(new_expected):
 204         return True
 205
 206     old_dict = {}
 207     new_dict = {}
 208     for dest, cond_lines in [(old_dict, old_expected), (new_dict, new_expected)]:
 209         for cond_line in cond_lines:
 210             if isinstance(cond_line, tuple):
 211                 condition, value = cond_line
 212             else:
 213                 condition = None
 214                 value = cond_line
 215             dest[condition] = value
 216
 217     return new_dict != old_dict
 218
 219
 220 def record_changes(ancestor_manifest, new_manifest):
 221     changes = Differences()
 222
 223     for test, test_manifest in new_manifest.child_map.items():
 224         if test not in ancestor_manifest.child_map:
 225             changes.added.append((test, test_manifest))
 226         else:
 227             ancestor_test_manifest = ancestor_manifest.child_map[test]
 228             test_differences = compare_test(test, ancestor_test_manifest, test_manifest)
 229             if test_differences:
 230                 changes.modified.append(
 231                     TestModified(test, test_manifest, test_differences)
 232                 )
 233
 234     for test, test_manifest in ancestor_manifest.child_map.items():
 235         if test not in new_manifest.child_map:
 236             changes.deleted.append(test)
 237
 238     return changes
 239
 240
 241 def apply_changes(current_manifest, changes):
 242     for test, test_manifest in changes.added:
 243         if test in current_manifest.child_map:
 244             current_manifest.delete(current_manifest.child_map[test])
 245         current_manifest.insert(test_manifest)
 246
 247     for test in changes.deleted:
 248         if test in current_manifest.child_map:
 249             current_manifest.delete(current_manifest.child_map[test])
 250
 251     for item in changes.modified:
 252         if isinstance(item, TestModified):
 253             test, new_manifest, test_changes = item
 254             if test in current_manifest.child_map:
 255                 apply_changes(current_manifest.child_map[test], test_changes)
 256             else:
 257                 current_manifest.insert(new_manifest)
 258         else:
 259             assert isinstance(item, ExpectedModified)
 260             subtest, ancestor_manifest, new_manifest = item
 261             if not subtest:
 262                 current_manifest.set_expected(new_manifest)
 263             elif subtest in current_manifest.child_map:
 264                 current_manifest.child_map[subtest].set_expected(new_manifest)
 265             else:
 266                 current_manifest.insert(new_manifest)
 267
 268
 269 def get_parser():
 270     parser = argparse.ArgumentParser()
 271     parser.add_argument("ancestor")
 272     parser.add_argument("current")
 273     parser.add_argument("new")
 274     parser.add_argument("dest", nargs="?")
 275     return parser
 276
 277
 278 def get_parser_mergetool():
 279     parser = argparse.ArgumentParser()
 280     parser.add_argument("--no-overwrite", dest="overwrite", action="store_false")
 281     return parser
 282
 283
 284 def make_changes(ancestor_manifest, current_manifest, new_manifest):
 285     changes = record_changes(ancestor_manifest, new_manifest)
 286     apply_changes(current_manifest, changes)
 287
 288     return serialize(current_manifest.node)
 289
 290
 291 def run(ancestor, current, new, dest):
 292     ancestor_manifest = get_manifest(ancestor)
 293     current_manifest = get_manifest(current)
 294     new_manifest = get_manifest(new)
 295
 296     updated_current_str = make_changes(
 297         ancestor_manifest, current_manifest, new_manifest
 298     )
 299
 300     if dest != "-":
 301         with open(dest, "wb") as f:
 302             f.write(updated_current_str.encode("utf8"))
 303     else:
 304         print(updated_current_str)