3 # (Be in -*- python -*- mode.)
5 # ====================================================================
6 # Copyright (c) 2006-2008 CollabNet. All rights reserved.
8 # This software is licensed as described in the file COPYING, which
9 # you should have received as part of this distribution. The terms
10 # are also available at http://subversion.tigris.org/license-1.html.
11 # If newer versions of this license are posted there, you may use a
12 # newer version instead, at your option.
14 # This software consists of voluntary contributions made by many
15 # individuals. For exact contribution history, see the revision
16 # history and logs, available at http://cvs2svn.tigris.org/.
17 # ====================================================================
19 """Usage: destroy_repository.py OPTION... PATH...
21 Strip the text content out of RCS-format files.
23 *** This script irretrievably destroys any RCS files that it is applied to!
25 This script attempts to strip the file text, log messages, and author
26 names out of RCS files, in addition to renaming RCS files and directories.
27 (This is useful to make test cases smaller and to remove much of the
28 proprietary information that is stored in a repository.) Note that this
29 script does NOT obliterate other information that might also be considered
30 proprietary, such as 'CVSROOT' directories and their contents, commit dates,
31 etc. In fact, it's not guaranteed even to obliterate all of the file text,
32 or to do anything else for that matter.
34 The following OPTIONs are recognized:
35 --all destroy all data (this is the default if no options are given)
36 --data destroy revision data (file contents) only
37 --metadata destroy revision metadata (author, log message, description) only
38 --symbols destroy symbol names (branch/tag names) only
39 --filenames destroy the filenames of RCS files
40 --basenames destroy basenames only (keep filename extensions, such as '.txt')
41 (--filenames overrides --basenames)
42 --dirnames destroy directory names within given PATH. PATH itself (if a
43 directory) is not destroyed.
44 --cvsroot delete files within 'CVSROOT' directories, instead of leaving
45 them untouched. The 'CVSROOT' directory itself is preserved.
46 --no-<X> where <X> is one of the above options negates the meaning of that
49 Each PATH that is a *,v file will be stripped.
51 Each PATH that is a directory will be traversed and all of its *,v
54 Other PATHs will be ignored.
58 destroy_repository.py PATH
59 destroys all data in PATH
61 destroy_repository.py --all PATH
64 destroy_repository.py --data PATH
65 destroys only revision data
67 destroy_repository.py --no-data PATH
68 destroys everything but revision data
70 destroy_repository.py --data --metadata PATH
71 destroys revision data and metadata only
75 The *,v files must be writable by the user running the script.
76 Typically CVS repositories are read-only, so you might have to run
79 $ chmod -R ug+w my/repo/path
81 before running this script.
83 Most cvs2svn behavior is completely independent of the text contained
84 in an RCS file. (The text is not even looked at until OutputPass.)
86 The idea is to use this script when preparing test cases for problems
87 that you experience with cvs2svn. Instead of sending us your whole
88 CVS repository, you should:
90 1. Make a copy of the original repository
92 2. Run this script on the copy (NEVER ON THE ORIGINAL!!!)
94 3. Verify that the problem still exists when you use cvs2svn to
95 convert the 'destroyed' copy
97 4. Send us the 'destroyed' copy along with the exact cvs2svn version
98 that you used, the exact command line that you used to start the
99 conversion, and the options file if you used one.
101 Please also consider using shrink_test_case.py to localize the problem
111 sys
.path
.insert(0, os
.path
.dirname(os
.path
.dirname(sys
.argv
[0])))
113 from cvs2svn_lib
.key_generator
import KeyGenerator
114 import cvs2svn_rcsparse
115 from rcs_file_filter
import WriteRCSFileSink
116 from rcs_file_filter
import FilterSink
119 # Which components to be destroyed. Default to all.
130 tmpdir
= 'destroy_repository-tmp'
132 file_key_generator
= KeyGenerator(1)
134 def get_tmp_filename():
135 return os
.path
.join(tmpdir
, 'f%07d.tmp' % file_key_generator
.gen_id())
137 # Mapping from "real" symbol name to rewritten symbol name
140 def rewrite_symbol(name
):
141 if name
not in symbol_map
:
142 symbol_map
[name
] = "symbol%05d" % (len(symbol_map
))
143 return symbol_map
[name
]
145 # Mapping from "real" filename to rewritten filename
147 # Empty filename should always map to empty filename. This is useful when
148 # preserving the last component of filenames with only one component.
152 # Set the following to true if we should not destroy the last filename
153 # component (aka. filename extension)
154 keep_last_filename_component
= False
156 def rewrite_filename(pathname
):
157 if not destroy
['filenames']:
159 (dirname
, filename
) = os
.path
.split(pathname
)
162 # Strip trailing ',v' now, and re-append it to the rewritten filename
163 if filename
.endswith(',v'):
165 filename
= filename
[:-2]
167 if keep_last_filename_component
:
168 (filename
, extension
) = os
.path
.splitext(filename
)
170 # filename has no extension. Do not rewrite this filename
173 extra
= extension
+ extra
177 return os
.path
.join(dirname
, filename_map
[filename
] + extra
)
179 # filename_map[filename] does not exist. Generate automatically:
180 num
= len(filename_map
)
182 filename_map
[filename
] = "file%03d" % (num
)
183 retval
= os
.path
.join(dirname
, filename_map
[filename
] + extra
)
184 if not os
.path
.exists(retval
):
188 # List of directory names to be renamed. This list is filled while we walk
189 # the directory structure, and then processed afterwards, in order to not
190 # mess up the directory structure while it is being walked.
194 """Rename all directories occuring in rename_dir_list"""
195 # Make sure we rename subdirs _before_ renaming their parents
196 rename_dir_list
.reverse()
199 for d
in rename_dir_list
:
200 (parent
, name
) = os
.path
.split(d
)
201 # Skip rewriting 'Attic' directories
204 if name
not in rename_map
:
207 rename_map
[name
] = "dir%03d" % (num
)
208 if not os
.path
.exists(os
.path
.join(parent
, rename_map
[name
])):
210 new_d
= os
.path
.join(parent
, rename_map
[name
])
211 assert not os
.path
.exists(new_d
)
212 shutil
.move(d
, new_d
)
216 def __init__(self
, template
):
217 self
.template
= template
218 self
.key_generator
= KeyGenerator(1)
220 # A map from old values to new ones.
221 self
.substitutions
= {}
223 def get_substitution(self
, s
):
224 r
= self
.substitutions
.get(s
)
226 r
= self
.template
% self
.key_generator
.gen_id()
227 self
.substitutions
[s
] = r
231 class LogSubstituter(Substituter
):
232 # If a log messages matches any of these regular expressions, it
233 # is passed through untouched.
234 untouchable_log_res
= [
235 re
.compile(r
'^Initial revision\n$'),
236 re
.compile(r
'^file (?P<filename>.+) was initially added'
237 r
' on branch (?P<symbol>.+)\.\n$'),
238 re
.compile(r
'^\*\*\* empty log message \*\*\*\n$'),
239 re
.compile(r
'^initial checkin$'),
243 Substituter
.__init
__(self
, 'log %d')
245 def get_substitution(self
, log
):
247 for untouchable_log_re
in self
.untouchable_log_res
:
248 m
= untouchable_log_re
.search(log
)
250 # We have matched one of the above regexps
253 # Check if we matched a regexp with a named subgroup
254 groups
= m
.groupdict()
255 if 'symbol' in groups
and destroy
['symbols']:
256 # Need to rewrite symbol name
257 symbol
= groups
['symbol']
258 keep_log
= keep_log
.replace(symbol
, rewrite_symbol(symbol
))
259 if 'filename' in groups
and destroy
['filenames']:
260 # Need to rewrite filename
261 filename
= groups
['filename']
262 keep_log
= keep_log
.replace(
263 filename
, rewrite_filename(filename
)
267 if destroy
['metadata']:
268 return Substituter
.get_substitution(self
, log
)
272 class DestroyerFilterSink(FilterSink
):
273 def __init__(self
, author_substituter
, log_substituter
, sink
):
274 FilterSink
.__init
__(self
, sink
)
276 self
.author_substituter
= author_substituter
277 self
.log_substituter
= log_substituter
279 def define_tag(self
, name
, revision
):
280 if destroy
['symbols']:
281 name
= rewrite_symbol(name
)
282 self
.sink
.define_tag(name
, revision
)
285 self
, revision
, timestamp
, author
, state
, branches
, next
287 if destroy
['metadata']:
288 author
= self
.author_substituter
.get_substitution(author
)
289 FilterSink
.define_revision(
290 self
, revision
, timestamp
, author
, state
, branches
, next
293 def set_description(self
, description
):
294 if destroy
['metadata']:
296 FilterSink
.set_description(self
, description
)
298 def set_revision_info(self
, revision
, log
, text
):
301 if destroy
['metadata'] or destroy
['symbols'] or destroy
['filenames']:
302 log
= self
.log_substituter
.get_substitution(log
)
303 FilterSink
.set_revision_info(self
, revision
, log
, text
)
308 self
.log_substituter
= LogSubstituter()
309 self
.author_substituter
= Substituter('author%d')
311 def destroy_file(self
, filename
):
312 tmp_filename
= get_tmp_filename()
313 f
= open(tmp_filename
, 'wb')
314 new_filename
= rewrite_filename(filename
)
315 cvs2svn_rcsparse
.parse(
316 open(filename
, 'rb'),
318 self
.author_substituter
,
319 self
.log_substituter
,
325 # Replace the original file with the new one:
326 assert filename
== new_filename
or not os
.path
.exists(new_filename
)
328 shutil
.move(tmp_filename
, new_filename
)
330 def visit(self
, dirname
, names
):
331 # Special handling of CVSROOT directories
332 if "CVSROOT" in names
:
333 path
= os
.path
.join(dirname
, "CVSROOT")
334 if destroy
['cvsroot']:
335 # Remove all contents within CVSROOT
336 sys
.stderr
.write('Deleting %s contents...' % path
)
340 # Leave CVSROOT alone
341 sys
.stderr
.write('Skipping %s...' % path
)
342 del names
[names
.index("CVSROOT")]
343 sys
.stderr
.write('done.\n')
345 path
= os
.path
.join(dirname
, name
)
346 if os
.path
.isfile(path
) and path
.endswith(',v'):
347 sys
.stderr
.write('Destroying %s...' % path
)
348 self
.destroy_file(path
)
349 sys
.stderr
.write('done.\n')
350 elif os
.path
.isdir(path
):
351 if destroy
['dirnames']:
352 rename_dir_list
.append(path
)
353 # Subdirectories are traversed automatically
356 sys
.stderr
.write('File %s is being ignored.\n' % path
)
358 def destroy_dir(self
, path
):
359 os
.path
.walk(path
, FileDestroyer
.visit
, self
)
362 def usage_abort(msg
):
364 print >>sys
.stderr
, "ERROR:", msg
366 # Use this file's docstring as a usage string, but only the first part
367 print __doc__
.split('\n---->8----', 1)[0]
370 if __name__
== '__main__':
371 if not os
.path
.isdir(tmpdir
):
374 # Paths to be destroyed
377 # Command-line argument processing
379 for arg
in sys
.argv
[1:]:
380 if arg
.startswith("--"):
382 option
= arg
[2:].lower()
384 if option
.startswith("no-"):
388 # Use the first option on the command-line to determine the
389 # default actions. If the first option is negated (i.e. --no-X)
390 # the default action should be to destroy everything.
391 # Otherwise, the default action should be to destroy nothing.
392 # This makes both positive and negative options work
393 # intuitively (e.g. "--data" will destroy only data, while
394 # "--no-data" will destroy everything BUT data).
395 for d
in destroy
.keys():
396 destroy
[d
] = not value
398 if option
in destroy
:
399 destroy
[option
] = value
400 elif option
== "all":
401 for d
in destroy
.keys():
404 usage_abort("Unknown OPTION '%s'" % arg
)
409 # If --basenames if given (and not also --filenames), we shall destroy
410 # filenames, up to, but not including the last component.
411 if destroy
['basenames'] and not destroy
['filenames']:
412 destroy
['filenames'] = True
413 keep_last_filename_component
= True
416 usage_abort("No PATH given")
418 # Destroy given PATHs
419 file_destroyer
= FileDestroyer()
421 if os
.path
.isfile(path
) and path
.endswith(',v'):
422 file_destroyer
.destroy_file(path
)
423 elif os
.path
.isdir(path
):
424 file_destroyer
.destroy_dir(path
)
426 sys
.stderr
.write('PATH %s is being ignored.\n' % path
)
428 if destroy
['dirnames']: