contrib/destroy_repository.py

   1 #! /usr/bin/python
   2
   3 # (Be in -*- python -*- mode.)
   4 #
   5 # ====================================================================
   6 # Copyright (c) 2006-2008 CollabNet.  All rights reserved.
   7 #
   8 # This software is licensed as described in the file COPYING, which
   9 # you should have received as part of this distribution.  The terms
  10 # are also available at http://subversion.tigris.org/license-1.html.
  11 # If newer versions of this license are posted there, you may use a
  12 # newer version instead, at your option.
  13 #
  14 # This software consists of voluntary contributions made by many
  15 # individuals.  For exact contribution history, see the revision
  16 # history and logs, available at http://cvs2svn.tigris.org/.
  17 # ====================================================================
  18
  19 """Usage: destroy_repository.py OPTION... PATH...
  20
  21 Strip the text content out of RCS-format files.
  22
  23 *** This script irretrievably destroys any RCS files that it is applied to!
  24
  25 This script attempts to strip the file text, log messages, and author
  26 names out of RCS files, in addition to renaming RCS files and directories.
  27 (This is useful to make test cases smaller and to remove much of the
  28 proprietary information that is stored in a repository.)  Note that this
  29 script does NOT obliterate other information that might also be considered
  30 proprietary, such as 'CVSROOT' directories and their contents, commit dates,
  31 etc.  In fact, it's not guaranteed even to obliterate all of the file text,
  32 or to do anything else for that matter.
  33
  34 The following OPTIONs are recognized:
  35   --all       destroy all data (this is the default if no options are given)
  36   --data      destroy revision data (file contents) only
  37   --metadata  destroy revision metadata (author, log message, description) only
  38   --symbols   destroy symbol names (branch/tag names) only
  39   --filenames destroy the filenames of RCS files
  40   --basenames destroy basenames only (keep filename extensions, such as '.txt')
  41               (--filenames overrides --basenames)
  42   --dirnames  destroy directory names within given PATH. PATH itself (if a
  43               directory) is not destroyed.
  44   --cvsroot   delete files within 'CVSROOT' directories, instead of leaving
  45               them untouched. The 'CVSROOT' directory itself is preserved.
  46   --no-<X>    where <X> is one of the above options negates the meaning of that
  47               option.
  48
  49 Each PATH that is a *,v file will be stripped.
  50
  51 Each PATH that is a directory will be traversed and all of its *,v
  52 files stripped.
  53
  54 Other PATHs will be ignored.
  55
  56
  57 Examples of usage:
  58   destroy_repository.py PATH
  59         destroys all data in PATH
  60
  61   destroy_repository.py --all PATH
  62         same as above
  63
  64   destroy_repository.py --data PATH
  65         destroys only revision data
  66
  67   destroy_repository.py --no-data PATH
  68         destroys everything but revision data
  69
  70   destroy_repository.py --data --metadata PATH
  71         destroys revision data and metadata only
  72
  73 ---->8----
  74
  75 The *,v files must be writable by the user running the script.
  76 Typically CVS repositories are read-only, so you might have to run
  77 something like
  78
  79     $ chmod -R ug+w my/repo/path
  80
  81 before running this script.
  82
  83 Most cvs2svn behavior is completely independent of the text contained
  84 in an RCS file.  (The text is not even looked at until OutputPass.)
  85
  86 The idea is to use this script when preparing test cases for problems
  87 that you experience with cvs2svn.  Instead of sending us your whole
  88 CVS repository, you should:
  89
  90 1. Make a copy of the original repository
  91
  92 2. Run this script on the copy (NEVER ON THE ORIGINAL!!!)
  93
  94 3. Verify that the problem still exists when you use cvs2svn to
  95    convert the 'destroyed' copy
  96
  97 4. Send us the 'destroyed' copy along with the exact cvs2svn version
  98    that you used, the exact command line that you used to start the
  99    conversion, and the options file if you used one.
 100
 101 Please also consider using shrink_test_case.py to localize the problem
 102 even further.
 103
 104 """
 105
 106 import sys
 107 import os
 108 import shutil
 109 import re
 110
 111 sys.path.insert(0, os.path.dirname(os.path.dirname(sys.argv[0])))
 112
 113 from cvs2svn_lib.key_generator import KeyGenerator
 114 import cvs2svn_rcsparse
 115 from rcs_file_filter import WriteRCSFileSink
 116 from rcs_file_filter import FilterSink
 117
 118
 119 # Which components to be destroyed. Default to all.
 120 destroy = {
 121     'data': True,
 122     'metadata': True,
 123     'symbols': True,
 124     'filenames': True,
 125     'basenames': True,
 126     'dirnames': True,
 127     'cvsroot': True,
 128     }
 129
 130 tmpdir = 'destroy_repository-tmp'
 131
 132 file_key_generator = KeyGenerator(1)
 133
 134 def get_tmp_filename():
 135     return os.path.join(tmpdir, 'f%07d.tmp' % file_key_generator.gen_id())
 136
 137 # Mapping from "real" symbol name to rewritten symbol name
 138 symbol_map = {}
 139
 140 def rewrite_symbol(name):
 141     if name not in symbol_map:
 142         symbol_map[name] = "symbol%05d" % (len(symbol_map))
 143     return symbol_map[name]
 144
 145 # Mapping from "real" filename to rewritten filename
 146 filename_map = {
 147     # Empty filename should always map to empty filename. This is useful when
 148     # preserving the last component of filenames with only one component.
 149     '': '',
 150     }
 151
 152 # Set the following to true if we should not destroy the last filename
 153 # component (aka. filename extension)
 154 keep_last_filename_component = False
 155
 156 def rewrite_filename(pathname):
 157     if not destroy['filenames']:
 158         return pathname
 159     (dirname, filename) = os.path.split(pathname)
 160     extra = ''
 161
 162     # Strip trailing ',v' now, and re-append it to the rewritten filename
 163     if filename.endswith(',v'):
 164         extra += ',v'
 165         filename = filename[:-2]
 166
 167     if keep_last_filename_component:
 168         (filename, extension) = os.path.splitext(filename)
 169         if not extension:
 170             # filename has no extension.  Do not rewrite this filename
 171             # at all.
 172             return pathname
 173         extra = extension + extra
 174
 175     # Rewrite filename
 176     try:
 177         return os.path.join(dirname, filename_map[filename] + extra)
 178     except KeyError:
 179         # filename_map[filename] does not exist. Generate automatically:
 180         num = len(filename_map)
 181         while True:
 182             filename_map[filename] = "file%03d" % (num)
 183             retval = os.path.join(dirname, filename_map[filename] + extra)
 184             if not os.path.exists(retval):
 185                 return retval
 186             num += 1
 187
 188 # List of directory names to be renamed. This list is filled while we walk
 189 # the directory structure, and then processed afterwards, in order to not
 190 # mess up the directory structure while it is being walked.
 191 rename_dir_list = []
 192
 193 def rename_dirs():
 194     """Rename all directories occuring in rename_dir_list"""
 195     # Make sure we rename subdirs _before_ renaming their parents
 196     rename_dir_list.reverse()
 197     rename_map = {}
 198     num = 0
 199     for d in rename_dir_list:
 200         (parent, name) = os.path.split(d)
 201         # Skip rewriting 'Attic' directories
 202         if name == "Attic":
 203             continue
 204         if name not in rename_map:
 205             while True:
 206                 num += 1
 207                 rename_map[name] = "dir%03d" % (num)
 208                 if not os.path.exists(os.path.join(parent, rename_map[name])):
 209                     break
 210         new_d = os.path.join(parent, rename_map[name])
 211         assert not os.path.exists(new_d)
 212         shutil.move(d, new_d)
 213
 214
 215 class Substituter:
 216     def __init__(self, template):
 217         self.template = template
 218         self.key_generator = KeyGenerator(1)
 219
 220         # A map from old values to new ones.
 221         self.substitutions = {}
 222
 223     def get_substitution(self, s):
 224         r = self.substitutions.get(s)
 225         if r == None:
 226             r = self.template % self.key_generator.gen_id()
 227             self.substitutions[s] = r
 228         return r
 229
 230
 231 class LogSubstituter(Substituter):
 232     # If a log messages matches any of these regular expressions, it
 233     # is passed through untouched.
 234     untouchable_log_res = [
 235         re.compile(r'^Initial revision\n$'),
 236         re.compile(r'^file (?P<filename>.+) was initially added'
 237                    r' on branch (?P<symbol>.+)\.\n$'),
 238         re.compile(r'^\*\*\* empty log message \*\*\*\n$'),
 239         re.compile(r'^initial checkin$'),
 240         ]
 241
 242     def __init__(self):
 243         Substituter.__init__(self, 'log %d')
 244
 245     def get_substitution(self, log):
 246         keep_log = ''
 247         for untouchable_log_re in self.untouchable_log_res:
 248             m = untouchable_log_re.search(log)
 249             if m:
 250                 # We have matched one of the above regexps
 251                 # Keep log message
 252                 keep_log = log
 253                 # Check if we matched a regexp with a named subgroup
 254                 groups = m.groupdict()
 255                 if 'symbol' in groups and destroy['symbols']:
 256                     # Need to rewrite symbol name
 257                     symbol = groups['symbol']
 258                     keep_log = keep_log.replace(symbol, rewrite_symbol(symbol))
 259                 if 'filename' in groups and destroy['filenames']:
 260                     # Need to rewrite filename
 261                     filename = groups['filename']
 262                     keep_log = keep_log.replace(
 263                         filename, rewrite_filename(filename)
 264                         )
 265         if keep_log:
 266             return keep_log
 267         if destroy['metadata']:
 268             return Substituter.get_substitution(self, log)
 269         return log
 270
 271
 272 class DestroyerFilterSink(FilterSink):
 273     def __init__(self, author_substituter, log_substituter, sink):
 274         FilterSink.__init__(self, sink)
 275
 276         self.author_substituter = author_substituter
 277         self.log_substituter = log_substituter
 278
 279     def define_tag(self, name, revision):
 280         if destroy['symbols']:
 281             name = rewrite_symbol(name)
 282         self.sink.define_tag(name, revision)
 283
 284     def define_revision(
 285         self, revision, timestamp, author, state, branches, next
 286         ):
 287         if destroy['metadata']:
 288             author = self.author_substituter.get_substitution(author)
 289         FilterSink.define_revision(
 290             self, revision, timestamp, author, state, branches, next
 291             )
 292
 293     def set_description(self, description):
 294         if destroy['metadata']:
 295             description = ''
 296         FilterSink.set_description(self, description)
 297
 298     def set_revision_info(self, revision, log, text):
 299         if destroy['data']:
 300             text = ''
 301         if destroy['metadata'] or destroy['symbols'] or destroy['filenames']:
 302             log = self.log_substituter.get_substitution(log)
 303         FilterSink.set_revision_info(self, revision, log, text)
 304
 305
 306 class FileDestroyer:
 307     def __init__(self):
 308         self.log_substituter = LogSubstituter()
 309         self.author_substituter = Substituter('author%d')
 310
 311     def destroy_file(self, filename):
 312         tmp_filename = get_tmp_filename()
 313         f = open(tmp_filename, 'wb')
 314         new_filename = rewrite_filename(filename)
 315         cvs2svn_rcsparse.parse(
 316             open(filename, 'rb'),
 317             DestroyerFilterSink(
 318                 self.author_substituter,
 319                 self.log_substituter,
 320                 WriteRCSFileSink(f),
 321                 )
 322             )
 323         f.close()
 324
 325         # Replace the original file with the new one:
 326         assert filename == new_filename or not os.path.exists(new_filename)
 327         os.remove(filename)
 328         shutil.move(tmp_filename, new_filename)
 329
 330     def visit(self, dirname, names):
 331         # Special handling of CVSROOT directories
 332         if "CVSROOT" in names:
 333             path = os.path.join(dirname, "CVSROOT")
 334             if destroy['cvsroot']:
 335                 # Remove all contents within CVSROOT
 336                 sys.stderr.write('Deleting %s contents...' % path)
 337                 shutil.rmtree(path)
 338                 os.mkdir(path)
 339             else:
 340                 # Leave CVSROOT alone
 341                 sys.stderr.write('Skipping %s...' % path)
 342             del names[names.index("CVSROOT")]
 343             sys.stderr.write('done.\n')
 344         for name in names:
 345             path = os.path.join(dirname, name)
 346             if os.path.isfile(path) and path.endswith(',v'):
 347                 sys.stderr.write('Destroying %s...' % path)
 348                 self.destroy_file(path)
 349                 sys.stderr.write('done.\n')
 350             elif os.path.isdir(path):
 351                 if destroy['dirnames']:
 352                     rename_dir_list.append(path)
 353                 # Subdirectories are traversed automatically
 354                 pass
 355             else:
 356                 sys.stderr.write('File %s is being ignored.\n' % path)
 357
 358     def destroy_dir(self, path):
 359         os.path.walk(path, FileDestroyer.visit, self)
 360
 361
 362 def usage_abort(msg):
 363     if msg:
 364         print >>sys.stderr, "ERROR:", msg
 365         print >>sys.stderr
 366     # Use this file's docstring as a usage string, but only the first part
 367     print __doc__.split('\n---->8----', 1)[0]
 368     sys.exit(1)
 369
 370 if __name__ == '__main__':
 371     if not os.path.isdir(tmpdir):
 372         os.makedirs(tmpdir)
 373
 374     # Paths to be destroyed
 375     paths = []
 376
 377     # Command-line argument processing
 378     first_option = True
 379     for arg in sys.argv[1:]:
 380         if arg.startswith("--"):
 381             # Option processing
 382             option = arg[2:].lower()
 383             value = True
 384             if option.startswith("no-"):
 385                 value = False
 386                 option = option[3:]
 387             if first_option:
 388                 # Use the first option on the command-line to determine the
 389                 # default actions. If the first option is negated (i.e. --no-X)
 390                 # the default action should be to destroy everything.
 391                 # Otherwise, the default action should be to destroy nothing.
 392                 # This makes both positive and negative options work
 393                 # intuitively (e.g. "--data" will destroy only data, while
 394                 # "--no-data" will destroy everything BUT data).
 395                 for d in destroy.keys():
 396                     destroy[d] = not value
 397                 first_option = False
 398             if option in destroy:
 399                 destroy[option] = value
 400             elif option == "all":
 401                 for d in destroy.keys():
 402                     destroy[d] = value
 403             else:
 404                 usage_abort("Unknown OPTION '%s'" % arg)
 405         else:
 406             # Path argument
 407             paths.append(arg)
 408
 409     # If --basenames if given (and not also --filenames), we shall destroy
 410     # filenames, up to, but not including the last component.
 411     if destroy['basenames'] and not destroy['filenames']:
 412         destroy['filenames'] = True
 413         keep_last_filename_component = True
 414
 415     if not paths:
 416         usage_abort("No PATH given")
 417
 418     # Destroy given PATHs
 419     file_destroyer = FileDestroyer()
 420     for path in paths:
 421         if os.path.isfile(path) and path.endswith(',v'):
 422             file_destroyer.destroy_file(path)
 423         elif os.path.isdir(path):
 424             file_destroyer.destroy_dir(path)
 425         else:
 426             sys.stderr.write('PATH %s is being ignored.\n' % path)
 427
 428     if destroy['dirnames']:
 429         rename_dirs()
 430