contrib/destroy_repository.py

   1 #! /usr/bin/python
   2
   3 # (Be in -*- python -*- mode.)
   4 #
   5 # ====================================================================
   6 # Copyright (c) 2006-2008 CollabNet.  All rights reserved.
   7 #
   8 # This software is licensed as described in the file COPYING, which
   9 # you should have received as part of this distribution.  The terms
  10 # are also available at http://subversion.tigris.org/license-1.html.
  11 # If newer versions of this license are posted there, you may use a
  12 # newer version instead, at your option.
  13 #
  14 # This software consists of voluntary contributions made by many
  15 # individuals.  For exact contribution history, see the revision
  16 # history and logs, available at http://cvs2svn.tigris.org/.
  17 # ====================================================================
  18
  19 """Usage: destroy_repository.py OPTION... PATH...
  20
  21 Strip the text content out of RCS-format files.
  22
  23 *** This script irretrievably destroys any RCS files that it is applied to!
  24
  25 This script attempts to strip the file text, log messages, and author
  26 names out of RCS files, in addition to renaming RCS files and directories.
  27 (This is useful to make test cases smaller and to remove much of the
  28 proprietary information that is stored in a repository.)  Note that this
  29 script does NOT obliterate other information that might also be considered
  30 proprietary, such as 'CVSROOT' directories and their contents, commit dates,
  31 etc.  In fact, it's not guaranteed even to obliterate all of the file text,
  32 or to do anything else for that matter.
  33
  34 The following OPTIONs are recognized:
  35   --all       destroy all data (this is the default if no options are given)
  36   --data      destroy revision data (file contents) only
  37   --metadata  destroy revision metadata (author, log message, description) only
  38   --symbols   destroy symbol names (branch/tag names) only
  39   --filenames destroy the filenames of RCS files
  40   --basenames destroy basenames only (keep filename extensions, such as '.txt')
  41               (--filenames overrides --basenames)
  42   --dirnames  destroy directory names within given PATH. PATH itself (if a
  43               directory) is not destroyed.
  44   --cvsroot   delete files within 'CVSROOT' directories, instead of leaving
  45               them untouched. The 'CVSROOT' directory itself is preserved.
  46   --no-<X>    where <X> is one of the above options negates the meaning of that
  47               option.
  48
  49 Each PATH that is a *,v file will be stripped.
  50
  51 Each PATH that is a directory will be traversed and all of its *,v
  52 files stripped.
  53
  54 Other PATHs will be ignored.
  55
  56
  57 Examples of usage:
  58   destroy_repository.py PATH
  59         destroys all data in PATH
  60
  61   destroy_repository.py --all PATH
  62         same as above
  63
  64   destroy_repository.py --data PATH
  65         destroys only revision data
  66
  67   destroy_repository.py --no-data PATH
  68         destroys everything but revision data
  69
  70   destroy_repository.py --data --metadata PATH
  71         destroys revision data and metadata only
  72
  73 ---->8----
  74
  75 The *,v files must be writable by the user running the script.
  76 Typically CVS repositories are read-only, so you might have to run
  77 something like
  78
  79     $ chmod -R ug+w my/repo/path
  80
  81 before running this script.
  82
  83 Most cvs2svn behavior is completely independent of the text contained
  84 in an RCS file.  (The text is not even looked at until OutputPass.)
  85
  86 The idea is to use this script when preparing test cases for problems
  87 that you experience with cvs2svn.  Instead of sending us your whole
  88 CVS repository, you should:
  89
  90 1. Make a copy of the original repository
  91
  92 2. Run this script on the copy (NEVER ON THE ORIGINAL!!!)
  93
  94 3. Verify that the problem still exists when you use cvs2svn to
  95    convert the 'destroyed' copy
  96
  97 4. Send us the 'destroyed' copy along with the exact cvs2svn version
  98    that you used, the exact command line that you used to start the
  99    conversion, and the options file if you used one.
 100
 101 Please also consider using shrink_test_case.py to localize the problem
 102 even further.
 103
 104 """
 105
 106 import sys
 107 import os
 108 import shutil
 109 import re
 110
 111 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 112
 113 from cvs2svn_lib.key_generator import KeyGenerator
 114 import cvs2svn_rcsparse
 115 from rcs_file_filter import WriteRCSFileSink
 116 from rcs_file_filter import FilterSink
 117
 118
 119 # Which components to be destroyed. Default to all.
 120 destroy = {
 121     'data': True,
 122     'metadata': True,
 123     'symbols': True,
 124     'filenames': True,
 125     'basenames': True,
 126     'dirnames': True,
 127     'cvsroot': True,
 128     }
 129
 130 tmpdir = 'destroy_repository-tmp'
 131
 132 file_key_generator = KeyGenerator(1)
 133
 134 def get_tmp_filename():
 135     return os.path.join(tmpdir, 'f%07d.tmp' % file_key_generator.gen_id())
 136
 137 # Mapping from "real" symbol name to rewritten symbol name
 138 symbol_map = {}
 139
 140 def rewrite_symbol(name):
 141     if name not in symbol_map:
 142         symbol_map[name] = "symbol%05d" % (len(symbol_map))
 143     return symbol_map[name]
 144
 145 # Mapping from "real" filename to rewritten filename
 146 filename_map = {
 147     # Empty filename should always map to empty filename. This is useful when
 148     # preserving the last component of filenames with only one component.
 149     '': '',
 150     }
 151
 152 # Set the following to true if we should not destroy the last filename
 153 # component (aka. filename extension)
 154 keep_last_filename_component = False
 155
 156 def rewrite_filename(pathname):
 157     if not destroy['filenames']:
 158         return pathname
 159     (dirname, filename) = os.path.split(pathname)
 160     extra = ''
 161
 162     # Strip trailing ',v' now, and re-append it to the rewritten filename
 163     if filename.endswith(',v'):
 164         extra += ',v'
 165         filename = filename[:-2]
 166
 167     if keep_last_filename_component:
 168         (filename, extension) = os.path.splitext(filename)
 169         if not extension:
 170             # filename has no extension.  Do not rewrite this filename
 171             # at all.
 172             return pathname
 173         extra = extension + extra
 174
 175     # Rewrite filename
 176     try:
 177         return os.path.join(dirname, filename_map[filename] + extra)
 178     except KeyError:
 179         # filename_map[filename] does not exist. Generate automatically:
 180         num = len(filename_map)
 181         while True:
 182             filename_map[filename] = "file%03d" % (num)
 183             retval = os.path.join(dirname, filename_map[filename] + extra)
 184             if not os.path.exists(retval):
 185                 return retval
 186             num += 1
 187
 188 # List of directory names to be renamed. This list is filled while we walk
 189 # the directory structure, and then processed afterwards, in order to not
 190 # mess up the directory structure while it is being walked.
 191 rename_dir_list = []
 192
 193 def rename_dirs():
 194     """Rename all directories occuring in rename_dir_list"""
 195     # Make sure we rename subdirs _before_ renaming their parents
 196     rename_dir_list.reverse()
 197     rename_map = {}
 198     num = 0
 199     for d in rename_dir_list:
 200         (parent, name) = os.path.split(d)
 201         # Skip rewriting 'Attic' directories
 202         if name == "Attic":
 203             continue
 204         if name not in rename_map:
 205             while True:
 206                 num += 1
 207                 rename_map[name] = "dir%03d" % (num)
 208                 if not os.path.exists(os.path.join(parent, rename_map[name])):
 209                     break
 210         new_d = os.path.join(parent, rename_map[name])
 211         assert not os.path.exists(new_d)
 212         shutil.move(d, new_d)
 213
 214
 215 class Substituter:
 216     def __init__(self, template):
 217         self.template = template
 218         self.key_generator = KeyGenerator(1)
 219
 220         # A map from old values to new ones.
 221         self.substitutions = {}
 222
 223     def get_substitution(self, s):
 224         r = self.substitutions.get(s)
 225         if r == None:
 226             r = self.template % self.key_generator.gen_id()
 227             self.substitutions[s] = r
 228         return r
 229
 230
 231 class LogSubstituter(Substituter):
 232     # If a log messages matches any of these regular expressions, it
 233     # is passed through untouched.
 234     untouchable_log_res = [
 235         re.compile(r'^Initial revision\n$'),
 236         re.compile(r'^file (?P<filename>.+) was initially added'
 237                    r' on branch (?P<symbol>.+)\.\n$'),
 238         re.compile(r'^\*\*\* empty log message \*\*\*\n$'),
 239         re.compile(r'^initial checkin$'),
 240         ]
 241
 242     def __init__(self):
 243         Substituter.__init__(self, 'log %d')
 244
 245     def get_substitution(self, log):
 246         keep_log = ''
 247         for untouchable_log_re in self.untouchable_log_res:
 248             m = untouchable_log_re.search(log)
 249             if m:
 250                 # We have matched one of the above regexps
 251                 # Keep log message
 252                 keep_log = log
 253                 # Check if we matched a regexp with a named subgroup
 254                 groups = m.groupdict()
 255                 if 'symbol' in groups and destroy['symbols']:
 256                     # Need to rewrite symbol name
 257                     symbol = groups['symbol']
 258                     keep_log = keep_log.replace(symbol, rewrite_symbol(symbol))
 259                 if 'filename' in groups and destroy['filenames']:
 260                     # Need to rewrite filename
 261                     filename = groups['filename']
 262                     keep_log = keep_log.replace(
 263                         filename, rewrite_filename(filename)
 264                         )
 265         if keep_log:
 266             return keep_log
 267         if destroy['metadata']:
 268             return Substituter.get_substitution(self, log)
 269         return log
 270
 271
 272 class DestroyerFilterSink(FilterSink):
 273     def __init__(self, author_substituter, log_substituter, sink):
 274         FilterSink.__init__(self, sink)
 275
 276         self.author_substituter = author_substituter
 277         self.log_substituter = log_substituter
 278
 279     def set_head_revision(self, revision):
 280         self.head_revision = revision
 281         FilterSink.set_head_revision(self, revision)
 282
 283     def define_tag(self, name, revision):
 284         if destroy['symbols']:
 285             name = rewrite_symbol(name)
 286         FilterSink.define_tag(self, name, revision)
 287
 288     def define_revision(
 289         self, revision, timestamp, author, state, branches, next
 290         ):
 291         if destroy['metadata']:
 292             author = self.author_substituter.get_substitution(author)
 293         FilterSink.define_revision(
 294             self, revision, timestamp, author, state, branches, next
 295             )
 296
 297     def set_description(self, description):
 298         if destroy['metadata']:
 299             description = ''
 300         FilterSink.set_description(self, description)
 301
 302     def set_revision_info(self, revision, log, text):
 303         if destroy['data']:
 304             if revision == self.head_revision:
 305                 # Set the HEAD text unconditionally.  (It could be
 306                 # that revision HEAD-1 has an empty deltatext, in
 307                 # which case the HEAD text was actually committed in
 308                 # an earlier commit.)
 309                 text = (
 310                     'This text was last seen in HEAD (revision %s)\n'
 311                     ) % (revision,)
 312             elif text == '':
 313                 # This is a no-op revision; preserve that fact.  (It
 314                 # might be relied on by cvs2svn).
 315                 pass
 316             else:
 317                 # Otherwise, replace the data.
 318                 if revision.count('.') == 1:
 319                     # On trunk, it could be that revision N-1 has an
 320                     # empty deltatext, in which case text for revision
 321                     # N was actually committed in an earlier commit.
 322                     text = (
 323                         'd1 1\n'
 324                         'a1 1\n'
 325                         'This text was last seen in revision %s\n'
 326                         ) % (revision,)
 327                 else:
 328                     # On a branch, we know that the text was changed
 329                     # in revision N (even though the same text might
 330                     # also be kept across later revisions N+1 etc.)
 331                     text = (
 332                         'd1 1\n'
 333                         'a1 1\n'
 334                         'This text was committed in revision %s\n'
 335                         ) % (revision,)
 336         if destroy['metadata'] or destroy['symbols'] or destroy['filenames']:
 337             log = self.log_substituter.get_substitution(log)
 338         FilterSink.set_revision_info(self, revision, log, text)
 339
 340
 341 class FileDestroyer:
 342     def __init__(self):
 343         self.log_substituter = LogSubstituter()
 344         self.author_substituter = Substituter('author%d')
 345
 346     def destroy_file(self, filename):
 347         tmp_filename = get_tmp_filename()
 348         f = open(tmp_filename, 'wb')
 349         new_filename = rewrite_filename(filename)
 350         cvs2svn_rcsparse.parse(
 351             open(filename, 'rb'),
 352             DestroyerFilterSink(
 353                 self.author_substituter,
 354                 self.log_substituter,
 355                 WriteRCSFileSink(f),
 356                 )
 357             )
 358         f.close()
 359
 360         # Replace the original file with the new one:
 361         assert filename == new_filename or not os.path.exists(new_filename)
 362         os.remove(filename)
 363         shutil.move(tmp_filename, new_filename)
 364
 365     def visit(self, dirname, names):
 366         # Special handling of CVSROOT directories
 367         if "CVSROOT" in names:
 368             path = os.path.join(dirname, "CVSROOT")
 369             if destroy['cvsroot']:
 370                 # Remove all contents within CVSROOT
 371                 sys.stderr.write('Deleting %s contents...' % path)
 372                 shutil.rmtree(path)
 373                 os.mkdir(path)
 374             else:
 375                 # Leave CVSROOT alone
 376                 sys.stderr.write('Skipping %s...' % path)
 377             del names[names.index("CVSROOT")]
 378             sys.stderr.write('done.\n')
 379         for name in names:
 380             path = os.path.join(dirname, name)
 381             if os.path.isfile(path) and path.endswith(',v'):
 382                 sys.stderr.write('Destroying %s...' % path)
 383                 self.destroy_file(path)
 384                 sys.stderr.write('done.\n')
 385             elif os.path.isdir(path):
 386                 if destroy['dirnames']:
 387                     rename_dir_list.append(path)
 388                 # Subdirectories are traversed automatically
 389                 pass
 390             else:
 391                 sys.stderr.write('File %s is being ignored.\n' % path)
 392
 393     def destroy_dir(self, path):
 394         os.path.walk(path, FileDestroyer.visit, self)
 395
 396
 397 def usage_abort(msg):
 398     if msg:
 399         print >>sys.stderr, "ERROR:", msg
 400         print >>sys.stderr
 401     # Use this file's docstring as a usage string, but only the first part
 402     print __doc__.split('\n---->8----', 1)[0]
 403     sys.exit(1)
 404
 405 if __name__ == '__main__':
 406     if not os.path.isdir(tmpdir):
 407         os.makedirs(tmpdir)
 408
 409     # Paths to be destroyed
 410     paths = []
 411
 412     # Command-line argument processing
 413     first_option = True
 414     for arg in sys.argv[1:]:
 415         if arg.startswith("--"):
 416             # Option processing
 417             option = arg[2:].lower()
 418             value = True
 419             if option.startswith("no-"):
 420                 value = False
 421                 option = option[3:]
 422             if first_option:
 423                 # Use the first option on the command-line to determine the
 424                 # default actions. If the first option is negated (i.e. --no-X)
 425                 # the default action should be to destroy everything.
 426                 # Otherwise, the default action should be to destroy nothing.
 427                 # This makes both positive and negative options work
 428                 # intuitively (e.g. "--data" will destroy only data, while
 429                 # "--no-data" will destroy everything BUT data).
 430                 for d in destroy.keys():
 431                     destroy[d] = not value
 432                 first_option = False
 433             if option in destroy:
 434                 destroy[option] = value
 435             elif option == "all":
 436                 for d in destroy.keys():
 437                     destroy[d] = value
 438             else:
 439                 usage_abort("Unknown OPTION '%s'" % arg)
 440         else:
 441             # Path argument
 442             paths.append(arg)
 443
 444     # If --basenames if given (and not also --filenames), we shall destroy
 445     # filenames, up to, but not including the last component.
 446     if destroy['basenames'] and not destroy['filenames']:
 447         destroy['filenames'] = True
 448         keep_last_filename_component = True
 449
 450     if not paths:
 451         usage_abort("No PATH given")
 452
 453     # Destroy given PATHs
 454     file_destroyer = FileDestroyer()
 455     for path in paths:
 456         if os.path.isfile(path) and path.endswith(',v'):
 457             file_destroyer.destroy_file(path)
 458         elif os.path.isdir(path):
 459             file_destroyer.destroy_dir(path)
 460         else:
 461             sys.stderr.write('PATH %s is being ignored.\n' % path)
 462
 463     if destroy['dirnames']:
 464         rename_dirs()
 465