svntest/verify.py

   1 #
   2 #  verify.py:  routines that handle comparison and display of expected
   3 #              vs. actual output
   4 #
   5 #  Subversion is a tool for revision control.
   6 #  See http://subversion.tigris.org for more information.
   7 #
   8 # ====================================================================
   9 #    Licensed to the Apache Software Foundation (ASF) under one
  10 #    or more contributor license agreements.  See the NOTICE file
  11 #    distributed with this work for additional information
  12 #    regarding copyright ownership.  The ASF licenses this file
  13 #    to you under the Apache License, Version 2.0 (the
  14 #    "License"); you may not use this file except in compliance
  15 #    with the License.  You may obtain a copy of the License at
  16 #
  17 #      http://www.apache.org/licenses/LICENSE-2.0
  18 #
  19 #    Unless required by applicable law or agreed to in writing,
  20 #    software distributed under the License is distributed on an
  21 #    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  22 #    KIND, either express or implied.  See the License for the
  23 #    specific language governing permissions and limitations
  24 #    under the License.
  25 ######################################################################
  26
  27 import re, sys
  28 from difflib import unified_diff, ndiff
  29 import pprint
  30 import logging
  31
  32 import svntest
  33
  34 logger = logging.getLogger()
  35
  36
  37 ######################################################################
  38 # Exception types
  39
  40 class SVNUnexpectedOutput(svntest.Failure):
  41   """Exception raised if an invocation of svn results in unexpected
  42   output of any kind."""
  43   pass
  44
  45 class SVNUnexpectedStdout(SVNUnexpectedOutput):
  46   """Exception raised if an invocation of svn results in unexpected
  47   output on STDOUT."""
  48   pass
  49
  50 class SVNUnexpectedStderr(SVNUnexpectedOutput):
  51   """Exception raised if an invocation of svn results in unexpected
  52   output on STDERR."""
  53   pass
  54
  55 class SVNExpectedStdout(SVNUnexpectedOutput):
  56   """Exception raised if an invocation of svn results in no output on
  57   STDOUT when output was expected."""
  58   pass
  59
  60 class SVNExpectedStderr(SVNUnexpectedOutput):
  61   """Exception raised if an invocation of svn results in no output on
  62   STDERR when output was expected."""
  63   pass
  64
  65 class SVNUnexpectedExitCode(SVNUnexpectedOutput):
  66   """Exception raised if an invocation of svn exits with a value other
  67   than what was expected."""
  68   pass
  69
  70 class SVNIncorrectDatatype(SVNUnexpectedOutput):
  71   """Exception raised if invalid input is passed to the
  72   run_and_verify_* API"""
  73   pass
  74
  75 class SVNDumpParseError(svntest.Failure):
  76   """Exception raised if parsing a dump file fails"""
  77   pass
  78
  79
  80 ######################################################################
  81 # Comparison of expected vs. actual output
  82
  83 def createExpectedOutput(expected, output_type, match_all=True):
  84   """Return EXPECTED, promoted to an ExpectedOutput instance if not
  85   None.  Raise SVNIncorrectDatatype if the data type of EXPECTED is
  86   not handled."""
  87   if isinstance(expected, list):
  88     expected = ExpectedOutput(expected)
  89   elif isinstance(expected, str):
  90     expected = RegexOutput(expected, match_all)
  91   elif isinstance(expected, int):
  92     expected = RegexOutput(".*: E%d:.*" % expected, False)
  93   elif expected is AnyOutput:
  94     expected = AnyOutput()
  95   elif expected is not None and not isinstance(expected, ExpectedOutput):
  96     raise SVNIncorrectDatatype("Unexpected type for '%s' data" % output_type)
  97   return expected
  98
  99 class ExpectedOutput:
 100   """Contains expected output, and performs comparisons."""
 101
 102   is_regex = False
 103   is_unordered = False
 104
 105   def __init__(self, output, match_all=True):
 106     """Initialize the expected output to OUTPUT which is a string, or a list
 107     of strings, or None meaning an empty list. If MATCH_ALL is True, the
 108     expected strings will be matched with the actual strings, one-to-one, in
 109     the same order. If False, they will be matched with a subset of the
 110     actual strings, one-to-one, in the same order, ignoring any other actual
 111     strings among the matching ones."""
 112     self.output = output
 113     self.match_all = match_all
 114
 115   def __str__(self):
 116     return str(self.output)
 117
 118   def __cmp__(self, other):
 119     raise Exception('badness')
 120
 121   def matches(self, other, except_re=None):
 122     """Return whether SELF.output matches OTHER (which may be a list
 123     of newline-terminated lines, or a single string).  Either value
 124     may be None."""
 125     if self.output is None:
 126       expected = []
 127     else:
 128       expected = self.output
 129     if other is None:
 130       actual = []
 131     else:
 132       actual = other
 133
 134     if not isinstance(actual, list):
 135       actual = [actual]
 136     if not isinstance(expected, list):
 137       expected = [expected]
 138
 139     if except_re:
 140       return self.matches_except(expected, actual, except_re)
 141     else:
 142       return self.is_equivalent_list(expected, actual)
 143
 144   def matches_except(self, expected, actual, except_re):
 145     "Return whether EXPECTED and ACTUAL match except for except_re."
 146     if not self.is_regex:
 147       i_expected = 0
 148       i_actual = 0
 149       while i_expected < len(expected) and i_actual < len(actual):
 150         if re.match(except_re, actual[i_actual]):
 151           i_actual += 1
 152         elif re.match(except_re, expected[i_expected]):
 153           i_expected += 1
 154         elif expected[i_expected] == actual[i_actual]:
 155           i_expected += 1
 156           i_actual += 1
 157         else:
 158           return False
 159       if i_expected == len(expected) and i_actual == len(actual):
 160             return True
 161       return False
 162     else:
 163       raise Exception("is_regex and except_re are mutually exclusive")
 164
 165   def is_equivalent_list(self, expected, actual):
 166     "Return whether EXPECTED and ACTUAL are equivalent."
 167     if not self.is_regex:
 168       if self.match_all:
 169         # The EXPECTED lines must match the ACTUAL lines, one-to-one, in
 170         # the same order.
 171         return expected == actual
 172
 173       # The EXPECTED lines must match a subset of the ACTUAL lines,
 174       # one-to-one, in the same order, with zero or more other ACTUAL
 175       # lines interspersed among the matching ACTUAL lines.
 176       i_expected = 0
 177       for actual_line in actual:
 178         if expected[i_expected] == actual_line:
 179           i_expected += 1
 180           if i_expected == len(expected):
 181             return True
 182       return False
 183
 184     expected_re = expected[0]
 185     # If we want to check that every line matches the regexp
 186     # assume they all match and look for any that don't.  If
 187     # only one line matching the regexp is enough, assume none
 188     # match and look for even one that does.
 189     if self.match_all:
 190       all_lines_match_re = True
 191     else:
 192       all_lines_match_re = False
 193
 194     # If a regex was provided assume that we actually require
 195     # some output. Fail if we don't have any.
 196     if len(actual) == 0:
 197       return False
 198
 199     for actual_line in actual:
 200       if self.match_all:
 201         if not re.match(expected_re, actual_line):
 202           return False
 203       else:
 204         # As soon an actual_line matches something, then we're good.
 205         if re.match(expected_re, actual_line):
 206           return True
 207
 208     return all_lines_match_re
 209
 210   def display_differences(self, message, label, actual):
 211     """Delegate to the display_lines() routine with the appropriate
 212     args.  MESSAGE is ignored if None."""
 213     display_lines(message, label, self.output, actual,
 214                   self.is_regex, self.is_unordered)
 215
 216
 217 class AnyOutput(ExpectedOutput):
 218   def __init__(self):
 219     ExpectedOutput.__init__(self, None, False)
 220
 221   def is_equivalent_list(self, ignored, actual):
 222     if len(actual) == 0:
 223       # No actual output. No match.
 224       return False
 225
 226     for line in actual:
 227       # If any line has some text, then there is output, so we match.
 228       if line:
 229         return True
 230
 231     # We did not find a line with text. No match.
 232     return False
 233
 234   def display_differences(self, message, label, actual):
 235     if message:
 236       logger.warn(message)
 237
 238
 239 class RegexOutput(ExpectedOutput):
 240   is_regex = True
 241
 242
 243 class UnorderedOutput(ExpectedOutput):
 244   """Marks unordered output, and performs comparisons."""
 245
 246   is_unordered = True
 247
 248   def __cmp__(self, other):
 249     raise Exception('badness')
 250
 251   def matches_except(self, expected, actual, except_re):
 252     assert type(actual) == type([]) # ### if this trips: fix it!
 253     return self.is_equivalent_list([l for l in expected if not except_re.match(l)],
 254                                    [l for l in actual if not except_re.match(l)])
 255
 256   def is_equivalent_list(self, expected, actual):
 257     "Disregard the order of ACTUAL lines during comparison."
 258
 259     e_set = set(expected)
 260     a_set = set(actual)
 261
 262     if self.match_all:
 263       if len(e_set) != len(a_set):
 264         return False
 265       if self.is_regex:
 266         for expect_re in e_set:
 267           for actual_line in a_set:
 268             if re.match(expect_re, actual_line):
 269               a_set.remove(actual_line)
 270               break
 271           else:
 272             # One of the regexes was not found
 273             return False
 274         return True
 275
 276       # All expected lines must be in the output.
 277       return e_set == a_set
 278
 279     if self.is_regex:
 280       # If any of the expected regexes are in the output, then we match.
 281       for expect_re in e_set:
 282         for actual_line in a_set:
 283           if re.match(expect_re, actual_line):
 284             return True
 285       return False
 286
 287     # If any of the expected lines are in the output, then we match.
 288     return len(e_set.intersection(a_set)) > 0
 289
 290
 291 class UnorderedRegexOutput(UnorderedOutput, RegexOutput):
 292   is_regex = True
 293   is_unordered = True
 294
 295
 296 ######################################################################
 297 # Displaying expected and actual output
 298
 299 def display_trees(message, label, expected, actual):
 300   'Print two trees, expected and actual.'
 301   if message is not None:
 302     logger.warn(message)
 303   if expected is not None:
 304     logger.warn('EXPECTED %s:', label)
 305     svntest.tree.dump_tree(expected)
 306   if actual is not None:
 307     logger.warn('ACTUAL %s:', label)
 308     svntest.tree.dump_tree(actual)
 309
 310
 311 def display_lines(message, label, expected, actual, expected_is_regexp=None,
 312                   expected_is_unordered=None):
 313   """Print MESSAGE, unless it is None, then print EXPECTED (labeled
 314   with LABEL) followed by ACTUAL (also labeled with LABEL).
 315   Both EXPECTED and ACTUAL may be strings or lists of strings."""
 316   if message is not None:
 317     logger.warn(message)
 318   if expected is not None:
 319     output = 'EXPECTED %s' % label
 320     if expected_is_regexp:
 321       output += ' (regexp)'
 322       expected = [expected + '\n']
 323     if expected_is_unordered:
 324       output += ' (unordered)'
 325     output += ':'
 326     logger.warn(output)
 327     for x in expected:
 328       sys.stdout.write(x)
 329   if actual is not None:
 330     logger.warn('ACTUAL %s:', label)
 331     for x in actual:
 332       sys.stdout.write(x)
 333
 334   # Additionally print unified diff
 335   if not expected_is_regexp:
 336     logger.warn('DIFF ' + ' '.join(output.split(' ')[1:]))
 337
 338     if type(expected) is str:
 339       expected = [expected]
 340
 341     if type(actual) is str:
 342       actual = [actual]
 343
 344     for x in unified_diff(expected, actual,
 345                           fromfile="EXPECTED %s" % label,
 346                           tofile="ACTUAL %s" % label):
 347       sys.stdout.write(x)
 348
 349 def compare_and_display_lines(message, label, expected, actual,
 350                               raisable=None, except_re=None):
 351   """Compare two sets of output lines, and print them if they differ,
 352   preceded by MESSAGE iff not None.  EXPECTED may be an instance of
 353   ExpectedOutput (and if not, it is wrapped as such).  RAISABLE is an
 354   exception class, an instance of which is thrown if ACTUAL doesn't
 355   match EXPECTED."""
 356   if raisable is None:
 357     raisable = svntest.main.SVNLineUnequal
 358   ### It'd be nicer to use createExpectedOutput() here, but its
 359   ### semantics don't match all current consumers of this function.
 360   if not isinstance(expected, ExpectedOutput):
 361     expected = ExpectedOutput(expected)
 362
 363   if isinstance(actual, str):
 364     actual = [actual]
 365   actual = [line for line in actual if not line.startswith('DBG:')]
 366
 367   if not expected.matches(actual, except_re):
 368     expected.display_differences(message, label, actual)
 369     raise raisable
 370
 371 def verify_outputs(message, actual_stdout, actual_stderr,
 372                    expected_stdout, expected_stderr, all_stdout=True):
 373   """Compare and display expected vs. actual stderr and stdout lines:
 374   if they don't match, print the difference (preceded by MESSAGE iff
 375   not None) and raise an exception.
 376
 377   If EXPECTED_STDERR or EXPECTED_STDOUT is a string the string is
 378   interpreted as a regular expression.  For EXPECTED_STDOUT and
 379   ACTUAL_STDOUT to match, every line in ACTUAL_STDOUT must match the
 380   EXPECTED_STDOUT regex, unless ALL_STDOUT is false.  For
 381   EXPECTED_STDERR regexes only one line in ACTUAL_STDERR need match."""
 382   expected_stderr = createExpectedOutput(expected_stderr, 'stderr', False)
 383   expected_stdout = createExpectedOutput(expected_stdout, 'stdout', all_stdout)
 384
 385   for (actual, expected, label, raisable) in (
 386       (actual_stderr, expected_stderr, 'STDERR', SVNExpectedStderr),
 387       (actual_stdout, expected_stdout, 'STDOUT', SVNExpectedStdout)):
 388     if expected is None:
 389       continue
 390
 391     if isinstance(expected, RegexOutput):
 392       raisable = svntest.main.SVNUnmatchedError
 393     elif not isinstance(expected, AnyOutput):
 394       raisable = svntest.main.SVNLineUnequal
 395
 396     compare_and_display_lines(message, label, expected, actual, raisable)
 397
 398 def verify_exit_code(message, actual, expected,
 399                      raisable=SVNUnexpectedExitCode):
 400   """Compare and display expected vs. actual exit codes:
 401   if they don't match, print the difference (preceded by MESSAGE iff
 402   not None) and raise an exception."""
 403
 404   if expected != actual:
 405     display_lines(message, "Exit Code",
 406                   str(expected) + '\n', str(actual) + '\n')
 407     raise raisable
 408
 409 # A simple dump file parser.  While sufficient for the current
 410 # testsuite it doesn't cope with all valid dump files.
 411 class DumpParser:
 412   def __init__(self, lines):
 413     self.current = 0
 414     self.lines = lines
 415     self.parsed = {}
 416
 417   def parse_line(self, regex, required=True):
 418     m = re.match(regex, self.lines[self.current])
 419     if not m:
 420       if required:
 421         raise SVNDumpParseError("expected '%s' at line %d\n%s"
 422                                 % (regex, self.current,
 423                                    self.lines[self.current]))
 424       else:
 425         return None
 426     self.current += 1
 427     return m.group(1)
 428
 429   def parse_blank(self, required=True):
 430     if self.lines[self.current] != '\n':  # Works on Windows
 431       if required:
 432         raise SVNDumpParseError("expected blank at line %d\n%s"
 433                                 % (self.current, self.lines[self.current]))
 434       else:
 435         return False
 436     self.current += 1
 437     return True
 438
 439   def parse_format(self):
 440     return self.parse_line('SVN-fs-dump-format-version: ([0-9]+)$')
 441
 442   def parse_uuid(self):
 443     return self.parse_line('UUID: ([0-9a-z-]+)$')
 444
 445   def parse_revision(self):
 446     return self.parse_line('Revision-number: ([0-9]+)$')
 447
 448   def parse_prop_length(self, required=True):
 449     return self.parse_line('Prop-content-length: ([0-9]+)$', required)
 450
 451   def parse_content_length(self, required=True):
 452     return self.parse_line('Content-length: ([0-9]+)$', required)
 453
 454   def parse_path(self):
 455     path = self.parse_line('Node-path: (.+)$', required=False)
 456     if not path and self.lines[self.current] == 'Node-path: \n':
 457       self.current += 1
 458       path = ''
 459     return path
 460
 461   def parse_kind(self):
 462     return self.parse_line('Node-kind: (.+)$', required=False)
 463
 464   def parse_action(self):
 465     return self.parse_line('Node-action: ([0-9a-z-]+)$')
 466
 467   def parse_copyfrom_rev(self):
 468     return self.parse_line('Node-copyfrom-rev: ([0-9]+)$', required=False)
 469
 470   def parse_copyfrom_path(self):
 471     path = self.parse_line('Node-copyfrom-path: (.+)$', required=False)
 472     if not path and self.lines[self.current] == 'Node-copyfrom-path: \n':
 473       self.current += 1
 474       path = ''
 475     return path
 476
 477   def parse_copy_md5(self):
 478     return self.parse_line('Text-copy-source-md5: ([0-9a-z]+)$', required=False)
 479
 480   def parse_copy_sha1(self):
 481     return self.parse_line('Text-copy-source-sha1: ([0-9a-z]+)$', required=False)
 482
 483   def parse_text_md5(self):
 484     return self.parse_line('Text-content-md5: ([0-9a-z]+)$', required=False)
 485
 486   def parse_text_sha1(self):
 487     return self.parse_line('Text-content-sha1: ([0-9a-z]+)$', required=False)
 488
 489   def parse_text_length(self):
 490     return self.parse_line('Text-content-length: ([0-9]+)$', required=False)
 491
 492   # One day we may need to parse individual property name/values into a map
 493   def get_props(self):
 494     props = []
 495     while not re.match('PROPS-END$', self.lines[self.current]):
 496       props.append(self.lines[self.current])
 497       self.current += 1
 498     self.current += 1
 499     return props
 500
 501   def get_content(self, length):
 502     content = ''
 503     while len(content) < length:
 504       content += self.lines[self.current]
 505       self.current += 1
 506     if len(content) == length + 1:
 507       content = content[:-1]
 508     elif len(content) != length:
 509       raise SVNDumpParseError("content length expected %d actual %d at line %d"
 510                               % (length, len(content), self.current))
 511     return content
 512
 513   def parse_one_node(self):
 514     node = {}
 515     node['kind'] = self.parse_kind()
 516     action = self.parse_action()
 517     node['copyfrom_rev'] = self.parse_copyfrom_rev()
 518     node['copyfrom_path'] = self.parse_copyfrom_path()
 519     node['copy_md5'] = self.parse_copy_md5()
 520     node['copy_sha1'] = self.parse_copy_sha1()
 521     node['prop_length'] = self.parse_prop_length(required=False)
 522     node['text_length'] = self.parse_text_length()
 523     node['text_md5'] = self.parse_text_md5()
 524     node['text_sha1'] = self.parse_text_sha1()
 525     node['content_length'] = self.parse_content_length(required=False)
 526     self.parse_blank()
 527     if node['prop_length']:
 528       node['props'] = self.get_props()
 529     if node['text_length']:
 530       node['content'] = self.get_content(int(node['text_length']))
 531     # Hard to determine how may blanks is 'correct' (a delete that is
 532     # followed by an add that is a replace and a copy has one fewer
 533     # than expected but that can't be predicted until seeing the add)
 534     # so allow arbitrary number
 535     blanks = 0
 536     while self.current < len(self.lines) and self.parse_blank(required=False):
 537       blanks += 1
 538     node['blanks'] = blanks
 539     return action, node
 540
 541   def parse_all_nodes(self):
 542     nodes = {}
 543     while True:
 544       if self.current >= len(self.lines):
 545         break
 546       path = self.parse_path()
 547       if not path and not path is '':
 548         break
 549       if not nodes.get(path):
 550         nodes[path] = {}
 551       action, node = self.parse_one_node()
 552       if nodes[path].get(action):
 553         raise SVNDumpParseError("duplicate action '%s' for node '%s' at line %d"
 554                                 % (action, path, self.current))
 555       nodes[path][action] = node
 556     return nodes
 557
 558   def parse_one_revision(self):
 559     revision = {}
 560     number = self.parse_revision()
 561     revision['prop_length'] = self.parse_prop_length()
 562     revision['content_length'] = self.parse_content_length()
 563     self.parse_blank()
 564     revision['props'] = self.get_props()
 565     self.parse_blank()
 566     revision['nodes'] = self.parse_all_nodes()
 567     return number, revision
 568
 569   def parse_all_revisions(self):
 570     while self.current < len(self.lines):
 571       number, revision = self.parse_one_revision()
 572       if self.parsed.get(number):
 573         raise SVNDumpParseError("duplicate revision %d at line %d"
 574                                 % (number, self.current))
 575       self.parsed[number] = revision
 576
 577   def parse(self):
 578     self.parsed['format'] = self.parse_format()
 579     self.parse_blank()
 580     self.parsed['uuid'] = self.parse_uuid()
 581     self.parse_blank()
 582     self.parse_all_revisions()
 583     return self.parsed
 584
 585 def compare_dump_files(message, label, expected, actual):
 586   """Parse two dump files EXPECTED and ACTUAL, both of which are lists
 587   of lines as returned by run_and_verify_dump, and check that the same
 588   revisions, nodes, properties, etc. are present in both dumps.
 589   """
 590
 591   parsed_expected = DumpParser(expected).parse()
 592   parsed_actual = DumpParser(actual).parse()
 593
 594   if parsed_expected != parsed_actual:
 595     raise svntest.Failure('\n' + '\n'.join(ndiff(
 596           pprint.pformat(parsed_expected).splitlines(),
 597           pprint.pformat(parsed_actual).splitlines())))