2 # (Be in -*- python -*- mode.)
4 # ====================================================================
5 # Copyright (c) 2000-2009 CollabNet. All rights reserved.
7 # This software is licensed as described in the file COPYING, which
8 # you should have received as part of this distribution. The terms
9 # are also available at http://subversion.tigris.org/license-1.html.
10 # If newer versions of this license are posted there, you may use a
11 # newer version instead, at your option.
13 # This software consists of voluntary contributions made by many
14 # individuals. For exact contribution history, see the revision
15 # history and logs, available at http://cvs2svn.tigris.org/.
16 # ====================================================================
18 # The purpose of verify-cvs2svn is to verify the result of a cvs2svn
19 # repository conversion. The following tests are performed:
21 # 1. Content checking of the HEAD revision of trunk, all tags and all
22 # branches. Only the tags and branches in the Subversion
23 # repository are checked, i.e. there are no checks to verify that
24 # all tags and branches in the CVS repository are present.
26 # This program only works if you converted a subdirectory of a CVS
27 # repository, and not the whole repository. If you really did convert
28 # a whole repository and need to check it, you must create a CVSROOT
29 # directory above the current root using cvs init.
31 # ====================================================================
41 # CVS and Subversion command line client commands
48 """Run cmd as a pipe. Return (output, status)."""
49 child
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
)
50 output
= child
.stdout
.read()
52 return (output
, status
)
54 def cmd_failed(cmd
, output
, status
):
55 print 'CMD FAILED:', ' '.join(cmd
)
57 sys
.stdout
.write(output
)
58 raise RuntimeError('%s command failed!' % cmd
[0])
61 def __init__(self
, path
):
62 """Open the CVS repository at PATH."""
63 path
= os
.path
.abspath(path
)
64 if not os
.path
.isdir(path
):
65 raise RuntimeError('CVS path is not a directory')
67 if os
.path
.exists(os
.path
.join(path
, 'CVSROOT')):
68 # The whole repository
72 self
.cvsroot
= os
.path
.dirname(path
)
73 self
.module
= os
.path
.basename(path
)
74 while not os
.path
.exists(os
.path
.join(self
.cvsroot
, 'CVSROOT')):
75 parent
= os
.path
.dirname(self
.cvsroot
)
76 if parent
== self
.cvsroot
:
77 raise RuntimeError('Cannot find the CVSROOT')
78 self
.module
= os
.path
.join(os
.path
.basename(self
.cvsroot
), self
.module
)
82 return os
.path
.basename(self
.cvsroot
)
84 def export(self
, dest_path
, rev
=None, keyword_opt
=None):
85 """Export revision REV to DEST_PATH where REV can be None to export
86 the HEAD revision, or any valid CVS revision string to export that
89 cmd
= [ CVS_CMD
, '-Q', '-d', ':local:' + self
.cvsroot
, 'export' ]
91 cmd
.extend([ '-r', rev
])
93 cmd
.extend([ '-D', 'now' ])
95 cmd
.append(keyword_opt
)
96 cmd
.extend([ '-d', dest_path
, self
.module
])
97 (output
, status
) = pipe(cmd
)
99 cmd_failed(cmd
, output
, status
)
105 def __init__(self
, url
):
106 """Open the Subversion repository at URL."""
107 # Check if the user supplied an URL or a path
108 if url
.find('://') == -1:
109 abspath
= os
.path
.abspath(url
)
110 url
= 'file://' + (abspath
[0] != '/' and '/' or '') + abspath
112 url
= url
.replace(os
.sep
, '/')
116 # Cache a list of all tags and branches
119 self
.tag_list
= self
.list('tags')
122 if 'branches' in list:
123 self
.branch_list
= self
.list('branches')
125 self
.branch_list
= []
128 return self
.url
.split('/')[-1]
130 def export(self
, path
, dest_path
):
131 """Export PATH to DEST_PATH."""
132 url
= '/'.join([self
.url
, path
])
133 cmd
= [ SVN_CMD
, 'export', '-q', url
, dest_path
]
134 (output
, status
) = pipe(cmd
)
136 cmd_failed(cmd
, output
, status
)
138 def export_trunk(self
, dest_path
):
139 """Export trunk to DEST_PATH."""
140 self
.export('trunk', dest_path
)
142 def export_tag(self
, dest_path
, tag
):
143 """Export the tag TAG to DEST_PATH."""
144 self
.export('tags/' + tag
, dest_path
)
146 def export_branch(self
, dest_path
, branch
):
147 """Export the branch BRANCH to DEST_PATH."""
148 self
.export('branches/' + branch
, dest_path
)
150 def list(self
, path
):
151 """Return a list of all files and directories in PATH."""
152 cmd
= [ SVN_CMD
, 'ls', self
.url
+ '/' + path
]
153 (output
, status
) = pipe(cmd
)
155 cmd_failed(cmd
, output
, status
)
157 for line
in output
.split("\n"):
159 entries
.append(line
[:-1])
163 """Return a list of all tags in the repository."""
167 """Return a list of all branches in the repository."""
168 return self
.branch_list
173 def __init__(self
, path
):
175 self
.base_cmd
= [HG_CMD
, '-R', self
.path
]
177 self
._branches
= None # cache result of branches()
178 self
._have
_default
= None # so export_trunk() doesn't blow up
181 return os
.path
.basename(self
.path
)
183 def _export(self
, dest_path
, rev
):
184 cmd
= self
.base_cmd
+ ['archive',
187 '--exclude', 're:^\.hg',
189 (output
, status
) = pipe(cmd
)
191 cmd_failed(cmd
, output
, status
)
193 # If Mercurial has nothing to export, then it doesn't create
194 # dest_path. This breaks tree_compare(), so just check that the
195 # manifest for the chosen revision really is empty, and if so create
197 if not os
.path
.exists(dest_path
):
198 cmd
= self
.base_cmd
+ ['manifest', '--rev', rev
]
200 (output
, status
) = pipe(cmd
)
202 cmd_failed(cmd
, output
, status
)
203 manifest
= [fn
for fn
in output
.split("\n")[:-1]
204 if not fn
.startswith('.hg')]
208 def export_trunk(self
, dest_path
):
209 self
.branches() # ensure _have_default is set
210 if self
._have
_default
:
211 self
._export
(dest_path
, 'default')
213 # same as CVS does when exporting empty trunk
216 def export_tag(self
, dest_path
, tag
):
217 self
._export
(dest_path
, tag
)
219 def export_branch(self
, dest_path
, branch
):
220 self
._export
(dest_path
, branch
)
223 cmd
= self
.base_cmd
+ ['tags', '-q']
224 tags
= self
._split
_output
(cmd
)
229 if self
._branches
is None:
230 cmd
= self
.base_cmd
+ ['branches', '-q']
231 self
._branches
= branches
= self
._split
_output
(cmd
)
233 branches
.remove('default')
234 self
._have
_default
= True
236 self
._have
_default
= False
238 return self
._branches
240 def _split_output(self
, cmd
):
241 (output
, status
) = pipe(cmd
)
243 cmd_failed(cmd
, output
, status
)
244 return output
.split("\n")[:-1]
249 def __init__(self
, path
):
250 raise NotImplementedError()
252 def transform_symbol(ctx
, name
):
253 """Transform the symbol NAME using the renaming rules specified
254 with --symbol-transform. Return the transformed symbol name."""
256 for (pattern
, replacement
) in ctx
.symbol_transforms
:
257 newname
= pattern
.sub(replacement
, name
)
259 print " symbol '%s' transformed to '%s'" % (name
, newname
)
265 class Failures(object):
267 self
.count
= 0 # number of failures seen
270 return str(self
.count
)
273 return "<%s at 0x%x: %s>" % (self
.__class
__.__name
__, id(self
), self
.count
)
275 def report(self
, summary
, details
=None):
277 sys
.stdout
.write(' FAIL: %s\n' % summary
)
280 sys
.stdout
.write(' %s\n' % line
)
282 def __nonzero__(self
):
283 return self
.count
> 0
286 def file_compare(failures
, base1
, base2
, run_diff
, rel_path
):
287 """Compare the mode and contents of two files.
289 The paths are specified as two base paths BASE1 and BASE2, and a
290 path REL_PATH that is relative to the two base paths. Return True
291 iff the file mode and contents are identical."""
294 path1
= os
.path
.join(base1
, rel_path
)
295 path2
= os
.path
.join(base2
, rel_path
)
296 mode1
= os
.stat(path1
).st_mode
& 0700 # only look at owner bits
297 mode2
= os
.stat(path2
).st_mode
& 0700
299 failures
.report('File modes differ for %s' % rel_path
,
300 details
=['%s: %o' % (path1
, mode1
),
301 '%s: %o' % (path2
, mode2
)])
304 file1
= open(path1
, 'rb')
305 file2
= open(path2
, 'rb')
307 data1
= file1
.read(8192)
308 data2
= file2
.read(8192)
311 cmd
= ['diff', '-u', path1
, path2
]
312 (output
, status
) = pipe(cmd
)
313 diff
= output
.split('\n')
316 failures
.report('File contents differ for %s' % rel_path
,
326 def tree_compare(failures
, base1
, base2
, run_diff
, rel_path
=''):
327 """Compare the contents of two directory trees, including file contents.
329 The paths are specified as two base paths BASE1 and BASE2, and a
330 path REL_PATH that is relative to the two base paths. Return True
331 iff the trees are identical."""
337 path1
= os
.path
.join(base1
, rel_path
)
338 path2
= os
.path
.join(base2
, rel_path
)
339 if not os
.path
.exists(path1
):
340 failures
.report('%s does not exist' % path1
)
342 if not os
.path
.exists(path2
):
343 failures
.report('%s does not exist' % path2
)
345 if os
.path
.isfile(path1
) and os
.path
.isfile(path2
):
346 return file_compare(failures
, base1
, base2
, run_diff
, rel_path
)
347 if not (os
.path
.isdir(path1
) and os
.path
.isdir(path2
)):
348 failures
.report('Path types differ for %r' % rel_path
)
350 entries1
= os
.listdir(path1
)
352 entries2
= os
.listdir(path2
)
357 missing
= filter(lambda x
: x
not in entries2
, entries1
)
358 extra
= filter(lambda x
: x
not in entries1
, entries2
)
360 failures
.report('Directory /%s is missing entries: %s' %
361 (rel_path
, ', '.join(missing
)))
364 failures
.report('Directory /%s has extra entries: %s' %
365 (rel_path
, ', '.join(extra
)))
368 for entry
in entries1
:
369 new_rel_path
= os
.path
.join(rel_path
, entry
)
370 if not tree_compare(failures
, base1
, base2
, run_diff
, new_rel_path
):
375 def verify_contents_single(failures
, cvsrepos
, verifyrepos
, kind
, label
, ctx
):
376 """Verify the HEAD revision of a trunk, tag, or branch.
378 Verify that the contents of the HEAD revision of all directories and
379 files in the conversion repository VERIFYREPOS match the ones in the
380 CVS repository CVSREPOS. KIND can be either 'trunk', 'tag' or
381 'branch'. If KIND is either 'tag' or 'branch', LABEL is used to
382 specify the name of the tag or branch. CTX has the attributes:
383 CTX.tmpdir: specifying the directory for all temporary files.
384 CTX.skip_cleanup: if true, the temporary files are not deleted.
385 CTX.run_diff: if true, run diff on differing files."""
387 itemname
= kind
+ (kind
!= 'trunk' and '-' + label
or '')
388 cvs_export_dir
= os
.path
.join(
389 ctx
.tmpdir
, 'cvs-export-%s' % itemname
)
390 vrf_export_dir
= os
.path
.join(
391 ctx
.tmpdir
, '%s-export-%s' % (verifyrepos
.name
, itemname
))
394 cvslabel
= transform_symbol(ctx
, label
)
399 cvsrepos
.export(cvs_export_dir
, cvslabel
, ctx
.keyword_opt
)
401 verifyrepos
.export_trunk(vrf_export_dir
)
403 verifyrepos
.export_tag(vrf_export_dir
, label
)
405 verifyrepos
.export_branch(vrf_export_dir
, label
)
408 failures
, cvs_export_dir
, vrf_export_dir
, ctx
.run_diff
412 if not ctx
.skip_cleanup
:
413 if os
.path
.exists(cvs_export_dir
):
414 shutil
.rmtree(cvs_export_dir
)
415 if os
.path
.exists(vrf_export_dir
):
416 shutil
.rmtree(vrf_export_dir
)
420 def verify_contents(failures
, cvsrepos
, verifyrepos
, ctx
):
421 """Verify that the contents of the HEAD revision of all directories
422 and files in the trunk, all tags and all branches in the conversion
423 repository VERIFYREPOS matches the ones in the CVS repository CVSREPOS.
424 CTX is passed through to verify_contents_single()."""
426 # branches/tags that failed:
429 # Verify contents of trunk
430 print 'Verifying trunk'
431 if not verify_contents_single(
432 failures
, cvsrepos
, verifyrepos
, 'trunk', None, ctx
434 locations
.append('trunk')
436 # Verify contents of all tags
437 for tag
in verifyrepos
.tags():
438 print 'Verifying tag', tag
439 if not verify_contents_single(
440 failures
, cvsrepos
, verifyrepos
, 'tag', tag
, ctx
442 locations
.append('tag:' + tag
)
444 # Verify contents of all branches
445 for branch
in verifyrepos
.branches():
446 if branch
[:10] == 'unlabeled-':
447 print 'Skipped branch', branch
449 print 'Verifying branch', branch
450 if not verify_contents_single(
451 failures
, cvsrepos
, verifyrepos
, 'branch', branch
, ctx
453 locations
.append('branch:' + branch
)
455 assert bool(failures
) == bool(locations
), \
456 "failures = %r\nlocations = %r" % (failures
, locations
)
460 sys
.stdout
.write('FAIL: %s != %s: %d failure(s) in:\n'
461 % (cvsrepos
, verifyrepos
, failures
.count
))
462 for location
in locations
:
463 sys
.stdout
.write(' %s\n' % location
)
465 sys
.stdout
.write('PASS: %s == %s\n' % (cvsrepos
, verifyrepos
))
472 parser
= optparse
.OptionParser(
473 usage
='%prog [options] cvs-repos verify-repos')
474 parser
.add_option('--branch',
475 help='verify contents of the branch BRANCH only')
476 parser
.add_option('--diff', action
='store_true', dest
='run_diff',
477 help='run diff on differing files')
478 parser
.add_option('--tag',
479 help='verify contents of the tag TAG only')
480 parser
.add_option('--tmpdir',
482 help='path to store temporary files')
483 parser
.add_option('--trunk', action
='store_true',
484 help='verify contents of trunk only')
485 parser
.add_option('--symbol-transform', action
='append',
487 help='transform symbol names from P to S like cvs2svn, '
488 'except transforms SVN symbol to CVS symbol')
489 parser
.add_option('--svn',
490 action
='store_const', dest
='repos_type', const
='svn',
491 help='assume verify-repos is svn [default]')
492 parser
.add_option('--hg',
493 action
='store_const', dest
='repos_type', const
='hg',
494 help='assume verify-repos is hg')
495 parser
.add_option('--git',
496 action
='store_const', dest
='repos_type', const
='git',
497 help='assume verify-repos is git (not implemented!)')
498 parser
.add_option('--suppress-keywords',
499 action
='store_const', dest
='keyword_opt', const
='-kk',
500 help='suppress CVS keyword expansion '
501 '(equivalent to --keyword-opt=-kk)')
502 parser
.add_option('--keyword-opt',
504 help='control CVS keyword expansion by adding OPT to '
505 'cvs export command line')
507 parser
.set_defaults(run_diff
=False,
510 symbol_transforms
=[],
512 (options
, args
) = parser
.parse_args()
514 symbol_transforms
= []
515 for value
in options
.symbol_transforms
:
517 [pattern
, replacement
] = value
.split(":")
519 symbol_transforms
.append(
520 RegexpSymbolTransform(pattern
, replacement
))
522 parser
.error("'%s' is not a valid regexp." % (pattern
,))
525 """Print an error to sys.stderr."""
526 sys
.stderr
.write('Error: ' + str(msg
) + '\n')
528 verify_branch
= options
.branch
529 verify_tag
= options
.tag
530 verify_trunk
= options
.trunk
532 # Consistency check for options and arguments.
534 parser
.error("wrong number of arguments")
537 verify_path
= args
[1]
538 verify_klass
= {'svn': SvnRepos
,
540 'git': GitRepos
}[options
.repos_type
]
542 failures
= Failures()
544 # Open the repositories
545 cvsrepos
= CvsRepos(cvs_path
)
546 verifyrepos
= verify_klass(verify_path
)
550 print 'Verifying branch', verify_branch
551 verify_contents_single(
552 failures
, cvsrepos
, verifyrepos
, 'branch', verify_branch
, options
555 print 'Verifying tag', verify_tag
556 verify_contents_single(
557 failures
, cvsrepos
, verifyrepos
, 'tag', verify_tag
, options
560 print 'Verifying trunk'
561 verify_contents_single(
562 failures
, cvsrepos
, verifyrepos
, 'trunk', None, options
565 # Verify trunk, tags and branches
566 verify_contents(failures
, cvsrepos
, verifyrepos
, options
)
567 except RuntimeError, e
:
569 except KeyboardInterrupt:
572 sys
.exit(failures
and 1 or 0)
574 if __name__
== '__main__':