testing/web-platform/metamerge.py

   1 import argparse
   2 import logging
   3 import os
   4 from collections import defaultdict, namedtuple
   5
   6 from wptrunner.wptmanifest.serializer import serialize
   7 from wptrunner.wptmanifest.backends import base
   8 from wptrunner.wptmanifest.node import KeyValueNode
   9
  10 here = os.path.dirname(__file__)
  11 logger = logging.getLogger(__name__)
  12
  13
  14 class Compiler(base.Compiler):
  15     def visit_KeyValueNode(self, node):
  16         key_name = node.data
  17         values = []
  18         for child in node.children:
  19             values.append(self.visit(child))
  20
  21         self.output_node.set(key_name, values)
  22
  23     def visit_ConditionalNode(self, node):
  24         assert len(node.children) == 2
  25         # For conditional nodes, just return the subtree
  26         return serialize(node.children[0]), self.visit(node.children[1])
  27
  28     def visit_UnaryExpressionNode(self, node):
  29         raise NotImplementedError
  30
  31     def visit_BinaryExpressionNode(self, node):
  32         raise NotImplementedError
  33
  34     def visit_UnaryOperatorNode(self, node):
  35         raise NotImplementedError
  36
  37     def visit_BinaryOperatorNode(self, node):
  38         raise NotImplementedError
  39
  40
  41 class ExpectedManifest(base.ManifestItem):
  42     def __init__(self, node):
  43         """Object representing all the tests in a particular manifest"""
  44         base.ManifestItem.__init__(self, node)
  45         self.child_map = {}
  46
  47     def append(self, child):
  48         """Add a test to the manifest"""
  49         base.ManifestItem.append(self, child)
  50         self.child_map[child.name] = child
  51
  52     def insert(self, child):
  53         self.append(child)
  54         self.node.append(child.node)
  55
  56     def delete(self, child):
  57         del self.child_map[child.name]
  58         child.node.remove()
  59         child.remove()
  60
  61
  62 class TestManifestItem(ExpectedManifest):
  63     def set_expected(self, other_manifest):
  64         for item in self.node.children:
  65             if isinstance(item, KeyValueNode) and item.data == "expected":
  66                 assert "expected" in self._data
  67                 item.remove()
  68                 del self._data["expected"]
  69                 break
  70
  71         for item in other_manifest.node.children:
  72             if isinstance(item, KeyValueNode) and item.data == "expected":
  73                 assert "expected" in other_manifest._data
  74                 item.remove()
  75                 self.node.children.insert(0, item)
  76                 self._data["expected"] = other_manifest._data.pop("expected")
  77                 break
  78
  79
  80 def data_cls_getter(output_node, visited_node):
  81     # visited_node is intentionally unused
  82     if output_node is None:
  83         return ExpectedManifest
  84     if isinstance(output_node, ExpectedManifest):
  85         return TestManifestItem
  86     raise ValueError
  87
  88
  89 def compile(stream, data_cls_getter=None, **kwargs):
  90     return base.compile(Compiler,
  91                         stream,
  92                         data_cls_getter=data_cls_getter,
  93                         **kwargs)
  94
  95
  96 def get_manifest(manifest_path):
  97     """Get the ExpectedManifest for a particular manifest path"""
  98     try:
  99         with open(manifest_path) as f:
 100             return compile(f,
 101                            data_cls_getter=data_cls_getter)
 102     except IOError:
 103         return None
 104
 105
 106 def indent(str_data, indent=2):
 107     rv = []
 108     for line in str_data.splitlines():
 109         rv.append("%s%s" % (" " * indent, line))
 110     return "\n".join(rv)
 111
 112
 113 class Differences(object):
 114     def __init__(self):
 115         self.added = []
 116         self.deleted = []
 117         self.modified = []
 118
 119     def __nonzero__(self):
 120         return bool(self.added or self.deleted or self.modified)
 121
 122     def __str__(self):
 123         modified = []
 124         for item in self.modified:
 125             if isinstance(item, TestModified):
 126                 modified.append("  %s\n    %s\n%s" % (item[0], item[1], indent(str(item[2]), 4)))
 127             else:
 128                 assert isinstance(item, ExpectedModified)
 129                 modified.append("  %s\n    %s %s" % item)
 130         return "Added:\n%s\nDeleted:\n%s\nModified:\n%s\n" % (
 131             "\n".join("  %s:\n    %s" % item for item in self.added),
 132             "\n".join("  %s" % item for item in self.deleted),
 133             "\n".join(modified))
 134
 135
 136 TestModified = namedtuple("TestModified", ["test", "test_manifest", "differences"])
 137
 138
 139 ExpectedModified = namedtuple("ExpectedModified", ["test", "ancestor_manifest", "new_manifest"])
 140
 141
 142 def compare_test(test, ancestor_manifest, new_manifest):
 143     changes = Differences()
 144
 145     compare_expected(changes, None, ancestor_manifest, new_manifest)
 146
 147     for subtest, ancestor_subtest_manifest in ancestor_manifest.child_map.iteritems():
 148         compare_expected(changes, subtest, ancestor_subtest_manifest,
 149                          new_manifest.child_map.get(subtest))
 150
 151     for subtest, subtest_manifest in new_manifest.child_map.iteritems():
 152         if subtest not in ancestor_manifest.child_map:
 153             changes.added.append((subtest, subtest_manifest))
 154
 155     return changes
 156
 157
 158 def compare_expected(changes, subtest, ancestor_manifest, new_manifest):
 159     if (not (ancestor_manifest and ancestor_manifest.has_key("expected")) and
 160         (new_manifest and new_manifest.has_key("expected"))):
 161         changes.modified.append(ExpectedModified(subtest, ancestor_manifest, new_manifest))
 162     elif (ancestor_manifest and ancestor_manifest.has_key("expected") and
 163           not (new_manifest and new_manifest.has_key("expected"))):
 164         changes.deleted.append(subtest)
 165     elif (ancestor_manifest and ancestor_manifest.has_key("expected") and
 166           new_manifest and new_manifest.has_key("expected")):
 167         old_expected = ancestor_manifest.get("expected")
 168         new_expected = new_manifest.get("expected")
 169         if expected_values_changed(old_expected, new_expected):
 170             changes.modified.append(ExpectedModified(subtest, ancestor_manifest, new_manifest))
 171
 172
 173 def expected_values_changed(old_expected, new_expected):
 174     if len(old_expected) != len(new_expected):
 175         return True
 176
 177     old_dict = {}
 178     new_dict = {}
 179     for dest, cond_lines in [(old_dict, old_expected), (new_dict, new_expected)]:
 180         for cond_line in cond_lines:
 181             if isinstance(cond_line, tuple):
 182                 condition, value = cond_line
 183             else:
 184                 condition = None
 185                 value = cond_line
 186             dest[condition] = value
 187
 188     return new_dict != old_dict
 189
 190
 191 def record_changes(ancestor_manifest, new_manifest):
 192     changes = Differences()
 193
 194     for test, test_manifest in new_manifest.child_map.iteritems():
 195         if test not in ancestor_manifest.child_map:
 196             changes.added.append((test, test_manifest))
 197         else:
 198             ancestor_test_manifest = ancestor_manifest.child_map[test]
 199             test_differences = compare_test(test,
 200                                             ancestor_test_manifest,
 201                                             test_manifest)
 202             if test_differences:
 203                 changes.modified.append(TestModified(test, test_manifest, test_differences))
 204
 205     for test, test_manifest in ancestor_manifest.child_map.iteritems():
 206         if test not in new_manifest.child_map:
 207             changes.deleted.append(test)
 208
 209     return changes
 210
 211
 212 def apply_changes(current_manifest, changes):
 213     for test, test_manifest in changes.added:
 214         if test in current_manifest.child_map:
 215             current_manifest.delete(current_manifest.child_map[test])
 216         current_manifest.insert(test_manifest)
 217
 218     for test in changes.deleted:
 219         if test in current_manifest.child_map:
 220             current_manifest.delete(current_manifest.child_map[test])
 221
 222     for item in changes.modified:
 223         if isinstance(item, TestModified):
 224             test, new_manifest, test_changes = item
 225             if test in current_manifest.child_map:
 226                 apply_changes(current_manifest.child_map[test], test_changes)
 227             else:
 228                 current_manifest.insert(new_manifest)
 229         else:
 230             assert isinstance(item, ExpectedModified)
 231             subtest, ancestor_manifest, new_manifest = item
 232             if not subtest:
 233                 current_manifest.set_expected(new_manifest)
 234             elif subtest in current_manifest.child_map:
 235                 current_manifest.child_map[subtest].set_expected(new_manifest)
 236             else:
 237                 current_manifest.insert(new_manifest)
 238
 239
 240 def get_parser():
 241     parser = argparse.ArgumentParser()
 242     parser.add_argument("ancestor")
 243     parser.add_argument("current")
 244     parser.add_argument("new")
 245     parser.add_argument("dest", nargs="?")
 246     return parser
 247
 248
 249 def get_parser_mergetool():
 250     parser = argparse.ArgumentParser()
 251     parser.add_argument("--no-overwrite", dest="overwrite", action="store_false")
 252     return parser
 253
 254
 255
 256 def make_changes(ancestor_manifest, current_manifest, new_manifest):
 257     changes = record_changes(ancestor_manifest, new_manifest)
 258     apply_changes(current_manifest, changes)
 259
 260     return serialize(current_manifest.node)
 261
 262
 263 def run(ancestor, current, new, dest):
 264     ancestor_manifest = get_manifest(ancestor)
 265     current_manifest = get_manifest(current)
 266     new_manifest = get_manifest(new)
 267
 268
 269     updated_current_str = make_changes(ancestor_manifest, current_manifest, new_manifest)
 270
 271     if dest != "-":
 272         with open(dest, "wb") as f:
 273             f.write(updated_current_str)
 274     else:
 275         print(updated_current_str)