contrib/verify-cvs2svn.py

   1 #!/usr/bin/env python
   2 # (Be in -*- python -*- mode.)
   3 #
   4 # ====================================================================
   5 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   6 #
   7 # This software is licensed as described in the file COPYING, which
   8 # you should have received as part of this distribution.  The terms
   9 # are also available at http://subversion.tigris.org/license-1.html.
  10 # If newer versions of this license are posted there, you may use a
  11 # newer version instead, at your option.
  12 #
  13 # This software consists of voluntary contributions made by many
  14 # individuals.  For exact contribution history, see the revision
  15 # history and logs, available at http://cvs2svn.tigris.org/.
  16 # ====================================================================
  17 #
  18 # The purpose of verify-cvs2svn is to verify the result of a cvs2svn
  19 # repository conversion.  The following tests are performed:
  20 #
  21 # 1. Content checking of the HEAD revision of trunk, all tags and all
  22 #    branches.  Only the tags and branches in the Subversion
  23 #    repository are checked, i.e. there are no checks to verify that
  24 #    all tags and branches in the CVS repository are present.
  25 #
  26 # This program only works if you converted a subdirectory of a CVS
  27 # repository, and not the whole repository.  If you really did convert
  28 # a whole repository and need to check it, you must create a CVSROOT
  29 # directory above the current root using cvs init.
  30 #
  31 # ====================================================================
  32
  33 import os
  34 import sys
  35 import optparse
  36 import subprocess
  37 import shutil
  38 import re
  39
  40
  41 # CVS and Subversion command line client commands
  42 CVS_CMD = 'cvs'
  43 SVN_CMD = 'svn'
  44 HG_CMD = 'hg'
  45
  46
  47 def pipe(cmd):
  48   """Run cmd as a pipe.  Return (output, status)."""
  49   child = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  50   output = child.stdout.read()
  51   status = child.wait()
  52   return (output, status)
  53
  54 def cmd_failed(cmd, output, status):
  55   print 'CMD FAILED:', ' '.join(cmd)
  56   print 'Output:'
  57   sys.stdout.write(output)
  58   raise RuntimeError('%s command failed!' % cmd[0])
  59
  60 class CvsRepos:
  61   def __init__(self, path):
  62     """Open the CVS repository at PATH."""
  63     path = os.path.abspath(path)
  64     if not os.path.isdir(path):
  65       raise RuntimeError('CVS path is not a directory')
  66
  67     if os.path.exists(os.path.join(path, 'CVSROOT')):
  68       # The whole repository
  69       self.module = "."
  70       self.cvsroot = path
  71     else:
  72       self.cvsroot = os.path.dirname(path)
  73       self.module = os.path.basename(path)
  74       while not os.path.exists(os.path.join(self.cvsroot, 'CVSROOT')):
  75         parent = os.path.dirname(self.cvsroot)
  76         if parent == self.cvsroot:
  77           raise RuntimeError('Cannot find the CVSROOT')
  78         self.module = os.path.join(os.path.basename(self.cvsroot), self.module)
  79         self.cvsroot = parent
  80
  81   def __str__(self):
  82     return os.path.basename(self.cvsroot)
  83
  84   def export(self, dest_path, rev=None, keyword_opt=None):
  85     """Export revision REV to DEST_PATH where REV can be None to export
  86     the HEAD revision, or any valid CVS revision string to export that
  87     revision."""
  88     os.mkdir(dest_path)
  89     cmd = [ CVS_CMD, '-Q', '-d', ':local:' + self.cvsroot, 'export' ]
  90     if rev:
  91       cmd.extend([ '-r', rev ])
  92     else:
  93       cmd.extend([ '-D', 'now' ])
  94     if keyword_opt:
  95       cmd.append(keyword_opt)
  96     cmd.extend([ '-d', dest_path, self.module ])
  97     (output, status) = pipe(cmd)
  98     if status or output:
  99       cmd_failed(cmd, output, status)
 100
 101
 102 class SvnRepos:
 103   name = 'svn'
 104
 105   def __init__(self, url):
 106     """Open the Subversion repository at URL."""
 107     # Check if the user supplied an URL or a path
 108     if url.find('://') == -1:
 109       abspath = os.path.abspath(url)
 110       url = 'file://' + (abspath[0] != '/' and '/' or '') + abspath
 111       if os.sep != '/':
 112         url = url.replace(os.sep, '/')
 113
 114     self.url = url
 115
 116     # Cache a list of all tags and branches
 117     list = self.list('')
 118     if 'tags' in list:
 119       self.tag_list = self.list('tags')
 120     else:
 121       self.tag_list = []
 122     if 'branches' in list:
 123       self.branch_list = self.list('branches')
 124     else:
 125       self.branch_list = []
 126
 127   def __str__(self):
 128     return self.url.split('/')[-1]
 129
 130   def export(self, path, dest_path):
 131     """Export PATH to DEST_PATH."""
 132     url = '/'.join([self.url, path])
 133     cmd = [ SVN_CMD, 'export', '-q', url, dest_path ]
 134     (output, status) = pipe(cmd)
 135     if status or output:
 136       cmd_failed(cmd, output, status)
 137
 138   def export_trunk(self, dest_path):
 139     """Export trunk to DEST_PATH."""
 140     self.export('trunk', dest_path)
 141
 142   def export_tag(self, dest_path, tag):
 143     """Export the tag TAG to DEST_PATH."""
 144     self.export('tags/' + tag, dest_path)
 145
 146   def export_branch(self, dest_path, branch):
 147     """Export the branch BRANCH to DEST_PATH."""
 148     self.export('branches/' + branch, dest_path)
 149
 150   def list(self, path):
 151     """Return a list of all files and directories in PATH."""
 152     cmd = [ SVN_CMD, 'ls', self.url + '/' + path ]
 153     (output, status) = pipe(cmd)
 154     if status:
 155       cmd_failed(cmd, output, status)
 156     entries = []
 157     for line in output.split("\n"):
 158       if line:
 159         entries.append(line[:-1])
 160     return entries
 161
 162   def tags(self):
 163     """Return a list of all tags in the repository."""
 164     return self.tag_list
 165
 166   def branches(self):
 167     """Return a list of all branches in the repository."""
 168     return self.branch_list
 169
 170 class HgRepos:
 171   name = 'hg'
 172
 173   def __init__(self, path):
 174     self.path = path
 175     self.base_cmd = [HG_CMD, '-R', self.path]
 176
 177     self._branches = None               # cache result of branches()
 178     self._have_default = None           # so export_trunk() doesn't blow up
 179
 180   def __str__(self):
 181     return os.path.basename(self.path)
 182
 183   def _export(self, dest_path, rev):
 184     cmd = self.base_cmd + ['archive',
 185                            '--type', 'files',
 186                            '--rev', rev,
 187                            '--exclude', 're:^\.hg',
 188                            dest_path]
 189     (output, status) = pipe(cmd)
 190     if status or output:
 191       cmd_failed(cmd, output, status)
 192
 193     # If Mercurial has nothing to export, then it doesn't create
 194     # dest_path.  This breaks tree_compare(), so just check that the
 195     # manifest for the chosen revision really is empty, and if so create
 196     # the empty dir.
 197     if not os.path.exists(dest_path):
 198       cmd = self.base_cmd + ['manifest', '--rev', rev]
 199
 200       (output, status) = pipe(cmd)
 201       if status:
 202         cmd_failed(cmd, output, status)
 203       manifest = [fn for fn in output.split("\n")[:-1]
 204                   if not fn.startswith('.hg')]
 205       if not manifest:
 206         os.mkdir(dest_path)
 207
 208   def export_trunk(self, dest_path):
 209     self.branches()                     # ensure _have_default is set
 210     if self._have_default:
 211       self._export(dest_path, 'default')
 212     else:
 213       # same as CVS does when exporting empty trunk
 214       os.mkdir(dest_path)
 215
 216   def export_tag(self, dest_path, tag):
 217     self._export(dest_path, tag)
 218
 219   def export_branch(self, dest_path, branch):
 220     self._export(dest_path, branch)
 221
 222   def tags(self):
 223     cmd = self.base_cmd + ['tags', '-q']
 224     tags = self._split_output(cmd)
 225     tags.remove('tip')
 226     return tags
 227
 228   def branches(self):
 229     if self._branches is None:
 230       cmd = self.base_cmd + ['branches', '-q']
 231       self._branches = branches = self._split_output(cmd)
 232       try:
 233         branches.remove('default')
 234         self._have_default = True
 235       except ValueError:
 236         self._have_default = False
 237
 238     return self._branches
 239
 240   def _split_output(self, cmd):
 241     (output, status) = pipe(cmd)
 242     if status:
 243       cmd_failed(cmd, output, status)
 244     return output.split("\n")[:-1]
 245
 246 class GitRepos:
 247   name = 'git'
 248
 249   def __init__(self, path):
 250     raise NotImplementedError()
 251
 252 def transform_symbol(ctx, name):
 253   """Transform the symbol NAME using the renaming rules specified
 254   with --symbol-transform.  Return the transformed symbol name."""
 255
 256   for (pattern, replacement) in ctx.symbol_transforms:
 257     newname = pattern.sub(replacement, name)
 258     if newname != name:
 259       print "   symbol '%s' transformed to '%s'" % (name, newname)
 260       name = newname
 261
 262   return name
 263
 264
 265 class Failures(object):
 266   def __init__(self):
 267     self.count = 0                      # number of failures seen
 268
 269   def __str__(self):
 270     return str(self.count)
 271
 272   def __repr__(self):
 273     return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self.count)
 274
 275   def report(self, summary, details=None):
 276     self.count += 1
 277     sys.stdout.write(' FAIL: %s\n' % summary)
 278     if details:
 279       for line in details:
 280         sys.stdout.write('  %s\n' % line)
 281
 282   def __nonzero__(self):
 283     return self.count > 0
 284
 285
 286 def file_compare(failures, base1, base2, run_diff, rel_path):
 287   """Compare the mode and contents of two files.
 288
 289   The paths are specified as two base paths BASE1 and BASE2, and a
 290   path REL_PATH that is relative to the two base paths.  Return True
 291   iff the file mode and contents are identical."""
 292
 293   ok = True
 294   path1 = os.path.join(base1, rel_path)
 295   path2 = os.path.join(base2, rel_path)
 296   mode1 = os.stat(path1).st_mode & 0700   # only look at owner bits
 297   mode2 = os.stat(path2).st_mode & 0700
 298   if mode1 != mode2:
 299     failures.report('File modes differ for %s' % rel_path,
 300                     details=['%s: %o' % (path1, mode1),
 301                              '%s: %o' % (path2, mode2)])
 302     ok = False
 303
 304   file1 = open(path1, 'rb')
 305   file2 = open(path2, 'rb')
 306   while True:
 307     data1 = file1.read(8192)
 308     data2 = file2.read(8192)
 309     if data1 != data2:
 310       if run_diff:
 311         cmd = ['diff', '-u', path1, path2]
 312         (output, status) = pipe(cmd)
 313         diff = output.split('\n')
 314       else:
 315         diff = None
 316       failures.report('File contents differ for %s' % rel_path,
 317                       details=diff)
 318       ok = False
 319     if len(data1) == 0:
 320       # eof
 321       break
 322
 323   return ok
 324
 325
 326 def tree_compare(failures, base1, base2, run_diff, rel_path=''):
 327   """Compare the contents of two directory trees, including file contents.
 328
 329   The paths are specified as two base paths BASE1 and BASE2, and a
 330   path REL_PATH that is relative to the two base paths.  Return True
 331   iff the trees are identical."""
 332
 333   if not rel_path:
 334     path1 = base1
 335     path2 = base2
 336   else:
 337     path1 = os.path.join(base1, rel_path)
 338     path2 = os.path.join(base2, rel_path)
 339   if not os.path.exists(path1):
 340     failures.report('%s does not exist' % path1)
 341     return False
 342   if not os.path.exists(path2):
 343     failures.report('%s does not exist' % path2)
 344     return False
 345   if os.path.isfile(path1) and os.path.isfile(path2):
 346     return file_compare(failures, base1, base2, run_diff, rel_path)
 347   if not (os.path.isdir(path1) and os.path.isdir(path2)):
 348     failures.report('Path types differ for %r' % rel_path)
 349     return False
 350   entries1 = os.listdir(path1)
 351   entries1.sort()
 352   entries2 = os.listdir(path2)
 353   entries2.sort()
 354
 355   ok = True
 356
 357   missing = filter(lambda x: x not in entries2, entries1)
 358   extra = filter(lambda x: x not in entries1, entries2)
 359   if missing:
 360     failures.report('Directory /%s is missing entries: %s' %
 361                     (rel_path, ', '.join(missing)))
 362     ok = False
 363   if extra:
 364     failures.report('Directory /%s has extra entries: %s' %
 365                     (rel_path, ', '.join(extra)))
 366     ok = False
 367
 368   for entry in entries1:
 369     new_rel_path = os.path.join(rel_path, entry)
 370     if not tree_compare(failures, base1, base2, run_diff, new_rel_path):
 371       ok = False
 372   return ok
 373
 374
 375 def verify_contents_single(failures, cvsrepos, verifyrepos, kind, label, ctx):
 376   """Verify the HEAD revision of a trunk, tag, or branch.
 377
 378   Verify that the contents of the HEAD revision of all directories and
 379   files in the conversion repository VERIFYREPOS match the ones in the
 380   CVS repository CVSREPOS.  KIND can be either 'trunk', 'tag' or
 381   'branch'.  If KIND is either 'tag' or 'branch', LABEL is used to
 382   specify the name of the tag or branch.  CTX has the attributes:
 383   CTX.tmpdir: specifying the directory for all temporary files.
 384   CTX.skip_cleanup: if true, the temporary files are not deleted.
 385   CTX.run_diff: if true, run diff on differing files."""
 386
 387   itemname = kind + (kind != 'trunk' and '-' + label or '')
 388   cvs_export_dir = os.path.join(
 389     ctx.tmpdir, 'cvs-export-%s' % itemname)
 390   vrf_export_dir = os.path.join(
 391     ctx.tmpdir, '%s-export-%s' % (verifyrepos.name, itemname))
 392
 393   if label:
 394     cvslabel = transform_symbol(ctx, label)
 395   else:
 396     cvslabel = None
 397
 398   try:
 399     cvsrepos.export(cvs_export_dir, cvslabel, ctx.keyword_opt)
 400     if kind == 'trunk':
 401       verifyrepos.export_trunk(vrf_export_dir)
 402     elif kind == 'tag':
 403       verifyrepos.export_tag(vrf_export_dir, label)
 404     else:
 405       verifyrepos.export_branch(vrf_export_dir, label)
 406
 407     if not tree_compare(
 408           failures, cvs_export_dir, vrf_export_dir, ctx.run_diff
 409           ):
 410       return False
 411   finally:
 412     if not ctx.skip_cleanup:
 413       if os.path.exists(cvs_export_dir):
 414         shutil.rmtree(cvs_export_dir)
 415       if os.path.exists(vrf_export_dir):
 416         shutil.rmtree(vrf_export_dir)
 417   return True
 418
 419
 420 def verify_contents(failures, cvsrepos, verifyrepos, ctx):
 421   """Verify that the contents of the HEAD revision of all directories
 422   and files in the trunk, all tags and all branches in the conversion
 423   repository VERIFYREPOS matches the ones in the CVS repository CVSREPOS.
 424   CTX is passed through to verify_contents_single()."""
 425
 426   # branches/tags that failed:
 427   locations = []
 428
 429   # Verify contents of trunk
 430   print 'Verifying trunk'
 431   if not verify_contents_single(
 432         failures, cvsrepos, verifyrepos, 'trunk', None, ctx
 433         ):
 434     locations.append('trunk')
 435
 436   # Verify contents of all tags
 437   for tag in verifyrepos.tags():
 438     print 'Verifying tag', tag
 439     if not verify_contents_single(
 440           failures, cvsrepos, verifyrepos, 'tag', tag, ctx
 441           ):
 442       locations.append('tag:' + tag)
 443
 444   # Verify contents of all branches
 445   for branch in verifyrepos.branches():
 446     if branch[:10] == 'unlabeled-':
 447       print 'Skipped branch', branch
 448     else:
 449       print 'Verifying branch', branch
 450       if not verify_contents_single(
 451             failures, cvsrepos, verifyrepos, 'branch', branch, ctx
 452             ):
 453         locations.append('branch:' + branch)
 454
 455   assert bool(failures) == bool(locations), \
 456          "failures = %r\nlocations = %r" % (failures, locations)
 457
 458   # Show the results
 459   if failures:
 460     sys.stdout.write('FAIL: %s != %s: %d failure(s) in:\n'
 461                      % (cvsrepos, verifyrepos, failures.count))
 462     for location in locations:
 463       sys.stdout.write('  %s\n' % location)
 464   else:
 465     sys.stdout.write('PASS: %s == %s\n' % (cvsrepos, verifyrepos))
 466
 467 class OptionContext:
 468   pass
 469
 470
 471 def main(argv):
 472   parser = optparse.OptionParser(
 473     usage='%prog [options] cvs-repos verify-repos')
 474   parser.add_option('--branch',
 475                     help='verify contents of the branch BRANCH only')
 476   parser.add_option('--diff', action='store_true', dest='run_diff',
 477                     help='run diff on differing files')
 478   parser.add_option('--tag',
 479                     help='verify contents of the tag TAG only')
 480   parser.add_option('--tmpdir',
 481                     metavar='PATH',
 482                     help='path to store temporary files')
 483   parser.add_option('--trunk', action='store_true',
 484                     help='verify contents of trunk only')
 485   parser.add_option('--symbol-transform', action='append',
 486                     metavar='P:S',
 487                     help='transform symbol names from P to S like cvs2svn, '
 488                          'except transforms SVN symbol to CVS symbol')
 489   parser.add_option('--svn',
 490                     action='store_const', dest='repos_type', const='svn',
 491                     help='assume verify-repos is svn [default]')
 492   parser.add_option('--hg',
 493                     action='store_const', dest='repos_type', const='hg',
 494                     help='assume verify-repos is hg')
 495   parser.add_option('--git',
 496                     action='store_const', dest='repos_type', const='git',
 497                     help='assume verify-repos is git (not implemented!)')
 498   parser.add_option('--suppress-keywords',
 499                     action='store_const', dest='keyword_opt', const='-kk',
 500                     help='suppress CVS keyword expansion '
 501                          '(equivalent to --keyword-opt=-kk)')
 502   parser.add_option('--keyword-opt',
 503                     metavar='OPT',
 504                     help='control CVS keyword expansion by adding OPT to '
 505                          'cvs export command line')
 506
 507   parser.set_defaults(run_diff=False,
 508                       tmpdir='',
 509                       skip_cleanup=False,
 510                       symbol_transforms=[],
 511                       repos_type='svn')
 512   (options, args) = parser.parse_args()
 513
 514   symbol_transforms = []
 515   for value in options.symbol_transforms:
 516     # This is broken!
 517     [pattern, replacement] = value.split(":")
 518     try:
 519       symbol_transforms.append(
 520           RegexpSymbolTransform(pattern, replacement))
 521     except re.error:
 522       parser.error("'%s' is not a valid regexp." % (pattern,))
 523
 524   def error(msg):
 525     """Print an error to sys.stderr."""
 526     sys.stderr.write('Error: ' + str(msg) + '\n')
 527
 528   verify_branch = options.branch
 529   verify_tag = options.tag
 530   verify_trunk = options.trunk
 531
 532   # Consistency check for options and arguments.
 533   if len(args) != 2:
 534     parser.error("wrong number of arguments")
 535
 536   cvs_path = args[0]
 537   verify_path = args[1]
 538   verify_klass = {'svn': SvnRepos,
 539                   'hg':  HgRepos,
 540                   'git': GitRepos}[options.repos_type]
 541
 542   failures = Failures()
 543   try:
 544     # Open the repositories
 545     cvsrepos = CvsRepos(cvs_path)
 546     verifyrepos = verify_klass(verify_path)
 547
 548     # Do our thing...
 549     if verify_branch:
 550       print 'Verifying branch', verify_branch
 551       verify_contents_single(
 552           failures, cvsrepos, verifyrepos, 'branch', verify_branch, options
 553           )
 554     elif verify_tag:
 555       print 'Verifying tag', verify_tag
 556       verify_contents_single(
 557           failures, cvsrepos, verifyrepos, 'tag', verify_tag, options
 558           )
 559     elif verify_trunk:
 560       print 'Verifying trunk'
 561       verify_contents_single(
 562           failures, cvsrepos, verifyrepos, 'trunk', None, options
 563           )
 564     else:
 565       # Verify trunk, tags and branches
 566       verify_contents(failures, cvsrepos, verifyrepos, options)
 567   except RuntimeError, e:
 568     error(str(e))
 569   except KeyboardInterrupt:
 570     pass
 571
 572   sys.exit(failures and 1 or 0)
 573
 574 if __name__ == '__main__':
 575   main(sys.argv)
 576