cvs2svn

   1 #!/usr/bin/env python
   2 # (Be in -*- python -*- mode.)
   3 #
   4 # ====================================================================
   5 # Copyright (c) 2000-2006 CollabNet.  All rights reserved.
   6 #
   7 # This software is licensed as described in the file COPYING, which
   8 # you should have received as part of this distribution.  The terms
   9 # are also available at http://subversion.tigris.org/license-1.html.
  10 # If newer versions of this license are posted there, you may use a
  11 # newer version instead, at your option.
  12 #
  13 # This software consists of voluntary contributions made by many
  14 # individuals.  For exact contribution history, see the revision
  15 # history and logs, available at http://cvs2svn.tigris.org/.
  16 # ====================================================================
  17
  18 VERSION = '1.4.0-dev'
  19
  20 import sys
  21
  22 # Make sure this Python is recent enough.  Do this as early as possible,
  23 # using only code compatible with Python 1.5.2 before the check.
  24 if sys.hexversion < 0x02020000:
  25   sys.stderr.write("ERROR: Python 2.2 or higher required.\n")
  26   sys.exit(1)
  27
  28 import os
  29 import re
  30 import getopt
  31 try:
  32   my_getopt = getopt.gnu_getopt
  33 except AttributeError:
  34   my_getopt = getopt.getopt
  35 import errno
  36
  37 try:
  38   # Try to get access to a bunch of encodings for use with --encoding.
  39   # See http://cjkpython.i18n.org/ for details.
  40   import iconv_codec
  41 except ImportError:
  42   pass
  43
  44 from cvs2svn_lib.boolean import *
  45 from cvs2svn_lib import config
  46 from cvs2svn_lib.common import warning_prefix
  47 from cvs2svn_lib.common import error_prefix
  48 from cvs2svn_lib.common import FatalException
  49 from cvs2svn_lib.common import FatalError
  50 from cvs2svn_lib.log import Log
  51 from cvs2svn_lib.process import CommandFailedException
  52 from cvs2svn_lib.process import check_command_runs
  53 from cvs2svn_lib.context import Ctx
  54 from cvs2svn_lib.project import Project
  55 from cvs2svn_lib.pass_manager import PassManager
  56 from cvs2svn_lib.pass_manager import InvalidPassError
  57 from cvs2svn_lib.symbol_strategy import RuleBasedSymbolStrategy
  58 from cvs2svn_lib.symbol_strategy import ForceBranchRegexpStrategyRule
  59 from cvs2svn_lib.symbol_strategy import ForceTagRegexpStrategyRule
  60 from cvs2svn_lib.symbol_strategy import ExcludeRegexpStrategyRule
  61 from cvs2svn_lib.symbol_strategy import UnambiguousUsageRule
  62 from cvs2svn_lib.symbol_strategy import BranchIfCommitsRule
  63 from cvs2svn_lib.symbol_strategy import HeuristicStrategyRule
  64 from cvs2svn_lib.symbol_strategy import AllBranchRule
  65 from cvs2svn_lib.symbol_strategy import AllTagRule
  66 from cvs2svn_lib import property_setters
  67 from cvs2svn_lib import passes
  68
  69
  70 pass_manager = PassManager([
  71   passes.CollectRevsPass(),
  72   passes.CollateSymbolsPass(),
  73   passes.ResyncRevsPass(),
  74   passes.SortRevsPass(),
  75   passes.CreateDatabasesPass(),
  76   passes.AggregateRevsPass(),
  77   passes.SortSymbolsPass(),
  78   passes.IndexSymbolsPass(),
  79   passes.OutputPass(),
  80   ])
  81
  82
  83 usage_message_template = """\
  84 USAGE: %(progname)s [-v] [-s svn-repos-path] [-p pass] cvs-repos-path
  85   --help, -h           print this usage message and exit with success
  86   --help-passes        list the available passes and their numbers
  87   --version            print the version number
  88   -q                   quiet
  89   -v                   verbose
  90   -s PATH              path for SVN repos
  91   -p PASS              execute only specified PASS
  92   -p [START]:[END]     execute passes START through END, inclusive
  93                        (PASS, START, and END can be pass names or numbers)
  94   --existing-svnrepos  load into existing SVN repository
  95   --dump-only          just produce a dumpfile; don't commit to a repos
  96   --dumpfile=PATH      name dumpfile to output
  97   --tmpdir=PATH        directory to use for tmp data (default to cwd)
  98   --dry-run            do not create a repository or a dumpfile;
  99                        just print what would happen.
 100   --profile            profile with 'hotshot' (into file cvs2svn.hotshot)
 101   --use-cvs            use CVS instead of RCS 'co' to extract data
 102                        (only use this if having problems with RCS)
 103   --svnadmin=PATH      path to the svnadmin program
 104   --trunk-only         convert only trunk commits, not tags nor branches
 105   --trunk=PATH         path for trunk (default: %(trunk_base)s)
 106   --branches=PATH      path for branches (default: %(branches_base)s)
 107   --tags=PATH          path for tags (default: %(tags_base)s)
 108   --no-prune           don't prune empty directories
 109   --encoding=ENC       encoding of paths and log messages in CVS repos
 110                        Multiple of these options may be passed, where they
 111                        will be treated as an ordered list of encodings to
 112                        attempt (with "ascii" as a hardcoded last resort)
 113   --force-branch=REGEXP force symbols matching REGEXP to be branches
 114   --force-tag=REGEXP   force symbols matching REGEXP to be tags
 115   --exclude=REGEXP     exclude branches and tags matching REGEXP
 116   --symbol-default=OPT choose how ambiguous symbols are converted.  OPT is
 117                        "branch", "tag", or "heuristic", or "strict" (default)
 118   --symbol-transform=P:S transform symbol names from P to S where P and S
 119                        use Python regexp and reference syntax respectively
 120   --username=NAME      username for cvs2svn-synthesized commits
 121   --skip-cleanup       prevent the deletion of intermediate files
 122   --bdb-txn-nosync     pass --bdb-txn-nosync to "svnadmin create"
 123   --fs-type=TYPE       pass --fs-type=TYPE to "svnadmin create"
 124   --cvs-revnums        record CVS revision numbers as file properties
 125   --mime-types=FILE    specify an apache-style mime.types file for
 126                        setting svn:mime-type
 127   --auto-props=FILE    set file properties from the auto-props section
 128                        of a file in svn config format
 129   --auto-props-ignore-case Ignore case when matching auto-props patterns
 130   --eol-from-mime-type set svn:eol-style from mime type if known
 131   --no-default-eol     don't set svn:eol-style to 'native' for
 132                        non-binary files with undetermined mime types
 133   --keywords-off       don't set svn:keywords on any files (by default,
 134                        cvs2svn sets svn:keywords on non-binary files to
 135                        "%(svn_keywords_value)s")
 136 """
 137
 138 def usage():
 139   sys.stdout.write(usage_message_template % {
 140       'progname' : os.path.basename(sys.argv[0]),
 141       'trunk_base' : Ctx().trunk_base,
 142       'branches_base' : Ctx().branches_base,
 143       'tags_base' : Ctx().tags_base,
 144       'svn_keywords_value' : config.SVN_KEYWORDS_VALUE,
 145       })
 146
 147
 148 def main():
 149   # Convenience var, so we don't have to keep instantiating this Borg.
 150   ctx = Ctx()
 151   ctx.symbol_strategy = RuleBasedSymbolStrategy()
 152
 153   profiling = False
 154   start_pass = 1
 155   end_pass = pass_manager.num_passes
 156
 157   try:
 158     opts, args = my_getopt(sys.argv[1:], 'p:s:qvh',
 159                                [ "help", "help-passes", "create", "trunk=",
 160                                  "username=", "existing-svnrepos",
 161                                  "branches=", "tags=", "encoding=",
 162                                  "force-branch=", "force-tag=", "exclude=",
 163                                  "symbol-default=",
 164                                  "use-cvs", "mime-types=",
 165                                  "auto-props=", "auto-props-ignore-case",
 166                                  "eol-from-mime-type", "no-default-eol",
 167                                  "trunk-only", "no-prune", "dry-run",
 168                                  "dump-only", "dumpfile=", "tmpdir=",
 169                                  "svnadmin=", "skip-cleanup", "cvs-revnums",
 170                                  "bdb-txn-nosync", "fs-type=",
 171                                  "version", "profile",
 172                                  "keywords-off", "symbol-transform="])
 173   except getopt.GetoptError, e:
 174     sys.stderr.write(error_prefix + ': ' + str(e) + '\n\n')
 175     usage()
 176     sys.exit(1)
 177
 178   for opt, value in opts:
 179     if opt == '--version':
 180         print '%s version %s' % (os.path.basename(sys.argv[0]), VERSION)
 181         sys.exit(0)
 182     elif opt == '-p':
 183       if value.find(':') >= 0:
 184         start_pass, end_pass = value.split(':')
 185         start_pass = pass_manager.get_pass_number(start_pass, 1)
 186         end_pass = pass_manager.get_pass_number(end_pass,
 187                                                 pass_manager.num_passes)
 188       else:
 189         end_pass = start_pass = pass_manager.get_pass_number(value)
 190
 191       if not start_pass <= end_pass:
 192         raise InvalidPassError(
 193             'Ending pass must not come before starting pass.')
 194     elif (opt == '--help') or (opt == '-h'):
 195       ctx.print_help = True
 196     elif opt == '--help-passes':
 197       pass_manager.help_passes()
 198       sys.exit(0)
 199     elif opt == '-v':
 200       Log().log_level = Log.VERBOSE
 201       ctx.verbose = True
 202     elif opt == '-q':
 203       Log().log_level = Log.QUIET
 204       ctx.quiet = True
 205     elif opt == '-s':
 206       ctx.target = value
 207     elif opt == '--existing-svnrepos':
 208       ctx.existing_svnrepos = True
 209     elif opt == '--dumpfile':
 210       ctx.dumpfile = value
 211     elif opt == '--tmpdir':
 212       ctx.tmpdir = value
 213     elif opt == '--use-cvs':
 214       ctx.use_cvs = True
 215     elif opt == '--svnadmin':
 216       ctx.svnadmin = value
 217     elif opt == '--trunk-only':
 218       ctx.trunk_only = True
 219     elif opt == '--trunk':
 220       ctx.trunk_base = value
 221     elif opt == '--branches':
 222       ctx.branches_base = value
 223     elif opt == '--tags':
 224       ctx.tags_base = value
 225     elif opt == '--no-prune':
 226       ctx.prune = False
 227     elif opt == '--dump-only':
 228       ctx.dump_only = True
 229     elif opt == '--dry-run':
 230       ctx.dry_run = True
 231     elif opt == '--encoding':
 232       ctx.encoding.insert(-1, value)
 233     elif opt == '--force-branch':
 234       ctx.symbol_strategy.add_rule(ForceBranchRegexpStrategyRule(value))
 235     elif opt == '--force-tag':
 236       ctx.symbol_strategy.add_rule(ForceTagRegexpStrategyRule(value))
 237     elif opt == '--exclude':
 238       ctx.symbol_strategy.add_rule(ExcludeRegexpStrategyRule(value))
 239     elif opt == '--symbol-default':
 240       if value not in ['branch', 'tag', 'heuristic', 'strict']:
 241         raise FatalError(
 242             '%r is not a valid option for --symbol_default.' % (value,))
 243       ctx.symbol_strategy_default = value
 244     elif opt == '--mime-types':
 245       ctx.mime_types_file = value
 246     elif opt == '--auto-props':
 247       ctx.auto_props_file = value
 248     elif opt == '--auto-props-ignore-case':
 249       ctx.auto_props_ignore_case = True
 250     elif opt == '--eol-from-mime-type':
 251       ctx.eol_from_mime_type = True
 252     elif opt == '--no-default-eol':
 253       ctx.no_default_eol = True
 254     elif opt == '--keywords-off':
 255       ctx.keywords_off = True
 256     elif opt == '--username':
 257       ctx.username = value
 258     elif opt == '--skip-cleanup':
 259       ctx.skip_cleanup = True
 260     elif opt == '--cvs-revnums':
 261       ctx.svn_property_setters.append(
 262           property_setters.CVSRevisionNumberSetter())
 263     elif opt == '--bdb-txn-nosync':
 264       ctx.bdb_txn_nosync = True
 265     elif opt == '--fs-type':
 266       ctx.fs_type = value
 267     elif opt == '--create':
 268       sys.stderr.write(warning_prefix +
 269           ': The behaviour produced by the --create option is now the '
 270           'default,\nand passing the option is deprecated.\n')
 271     elif opt == '--profile':
 272       profiling = True
 273     elif opt == '--symbol-transform':
 274       [pattern, replacement] = value.split(":")
 275       try:
 276         pattern = re.compile(pattern)
 277       except re.error, e:
 278         raise FatalError("'%s' is not a valid regexp." % (pattern,))
 279       ctx.symbol_transforms.append((pattern, replacement,))
 280
 281   if ctx.print_help:
 282     usage()
 283     sys.exit(0)
 284
 285   # Consistency check for options and arguments.
 286   if len(args) == 0:
 287     usage()
 288     sys.exit(1)
 289
 290   if len(args) > 1:
 291     sys.stderr.write(error_prefix +
 292                      ": must pass only one CVS repository.\n")
 293     usage()
 294     sys.exit(1)
 295
 296   cvsroot = args[0]
 297
 298   if (not ctx.target) and (not ctx.dump_only) and (not ctx.dry_run):
 299     raise FatalError("must pass one of '-s' or '--dump-only'.")
 300
 301   def not_both(opt1val, opt1name, opt2val, opt2name):
 302     if opt1val and opt2val:
 303       raise FatalError("cannot pass both '%s' and '%s'."
 304                        % (opt1name, opt2name,))
 305
 306   not_both(ctx.target, '-s',
 307            ctx.dump_only, '--dump-only')
 308
 309   not_both(ctx.dump_only, '--dump-only',
 310            ctx.existing_svnrepos, '--existing-svnrepos')
 311
 312   not_both(ctx.bdb_txn_nosync, '--bdb-txn-nosync',
 313            ctx.existing_svnrepos, '--existing-svnrepos')
 314
 315   not_both(ctx.dump_only, '--dump-only',
 316            ctx.bdb_txn_nosync, '--bdb-txn-nosync')
 317
 318   not_both(ctx.quiet, '-q',
 319            ctx.verbose, '-v')
 320
 321   not_both(ctx.fs_type, '--fs-type',
 322            ctx.existing_svnrepos, '--existing-svnrepos')
 323
 324   if ctx.fs_type and ctx.fs_type != 'bdb' and ctx.bdb_txn_nosync:
 325     raise FatalError("cannot pass --bdb-txn-nosync with --fs-type=%s."
 326                      % ctx.fs_type)
 327
 328   # Create the default project (using ctx.trunk, ctx.branches, and ctx.tags):
 329   ctx.project = Project(
 330       cvsroot, ctx.trunk_base, ctx.branches_base, ctx.tags_base)
 331
 332   if ctx.existing_svnrepos and not os.path.isdir(ctx.target):
 333     raise FatalError("the svn-repos-path '%s' is not an "
 334                      "existing directory." % ctx.target)
 335
 336   if not ctx.dump_only and not ctx.existing_svnrepos \
 337      and (not ctx.dry_run) and os.path.exists(ctx.target):
 338     raise FatalError("the svn-repos-path '%s' exists.\n"
 339                      "Remove it, or pass '--existing-svnrepos'."
 340                      % ctx.target)
 341
 342   if ctx.target and not ctx.dry_run:
 343     # Verify that svnadmin can be executed.  The 'help' subcommand
 344     # should be harmless.
 345     try:
 346       check_command_runs([ctx.svnadmin, 'help'], 'svnadmin')
 347     except CommandFailedException, e:
 348       raise FatalError(
 349           '%s\n'
 350           'svnadmin could not be executed.  Please ensure that it is\n'
 351           'installed and/or use the --svnadmin option.' % (e,))
 352
 353   ctx.symbol_strategy.add_rule(UnambiguousUsageRule())
 354   if ctx.symbol_strategy_default == 'strict':
 355     pass
 356   elif ctx.symbol_strategy_default == 'branch':
 357     ctx.symbol_strategy.add_rule(AllBranchRule())
 358   elif ctx.symbol_strategy_default == 'tag':
 359     ctx.symbol_strategy.add_rule(AllTagRule())
 360   elif ctx.symbol_strategy_default == 'heuristic':
 361     ctx.symbol_strategy.add_rule(BranchIfCommitsRule())
 362     ctx.symbol_strategy.add_rule(HeuristicStrategyRule())
 363   else:
 364     assert False
 365
 366   ctx.svn_property_setters.append(
 367       property_setters.ExecutablePropertySetter())
 368
 369   ctx.svn_property_setters.append(
 370       property_setters.BinaryFileEOLStyleSetter())
 371
 372   if ctx.mime_types_file:
 373     ctx.svn_property_setters.append(
 374         property_setters.MimeMapper(ctx.mime_types_file))
 375
 376   if ctx.auto_props_file:
 377     ctx.svn_property_setters.append(
 378         property_setters.AutoPropsPropertySetter(
 379             ctx.auto_props_file, ctx.auto_props_ignore_case))
 380
 381   ctx.svn_property_setters.append(
 382       property_setters.BinaryFileDefaultMimeTypeSetter())
 383
 384   if ctx.eol_from_mime_type:
 385     ctx.svn_property_setters.append(
 386         property_setters.EOLStyleFromMimeTypeSetter())
 387
 388   if ctx.no_default_eol:
 389     ctx.svn_property_setters.append(
 390         property_setters.DefaultEOLStyleSetter(None))
 391   else:
 392     ctx.svn_property_setters.append(
 393         property_setters.DefaultEOLStyleSetter('native'))
 394
 395   if not ctx.keywords_off:
 396     ctx.svn_property_setters.append(
 397         property_setters.KeywordsPropertySetter(config.SVN_KEYWORDS_VALUE))
 398
 399   # Make sure the tmp directory exists.  Note that we don't check if
 400   # it's empty -- we want to be able to use, for example, "." to hold
 401   # tempfiles.  But if we *did* want check if it were empty, we'd do
 402   # something like os.stat(ctx.tmpdir)[stat.ST_NLINK], of course :-).
 403   if not os.path.exists(ctx.tmpdir):
 404     os.mkdir(ctx.tmpdir)
 405   elif not os.path.isdir(ctx.tmpdir):
 406     raise FatalError(
 407         "cvs2svn tried to use '%s' for temporary files, but that path\n"
 408         "  exists and is not a directory.  Please make it be a directory,\n"
 409         "  or specify some other directory for temporary files."
 410         % (ctx.tmpdir,))
 411
 412   # But do lock the tmpdir, to avoid process clash.
 413   try:
 414     os.mkdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock'))
 415   except OSError, e:
 416     if e.errno == errno.EACCES:
 417       raise FatalError("Permission denied:"
 418                        + " No write access to directory '%s'." % ctx.tmpdir)
 419     if e.errno == errno.EEXIST:
 420       raise FatalError(
 421           "cvs2svn is using directory '%s' for temporary files, but\n"
 422           "  subdirectory '%s/cvs2svn.lock' exists, indicating that another\n"
 423           "  cvs2svn process is currently using '%s' as its temporary\n"
 424           "  workspace.  If you are certain that is not the case,\n"
 425           "  then remove the '%s/cvs2svn.lock' subdirectory."
 426           % (ctx.tmpdir, ctx.tmpdir, ctx.tmpdir, ctx.tmpdir,))
 427     raise
 428
 429   try:
 430     if profiling:
 431       import hotshot
 432       prof = hotshot.Profile('cvs2svn.hotshot')
 433       prof.runcall(pass_manager.run, start_pass, end_pass)
 434       prof.close()
 435     else:
 436       pass_manager.run(start_pass, end_pass)
 437   finally:
 438     try: os.rmdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock'))
 439     except: pass
 440
 441
 442 if __name__ == '__main__':
 443   try:
 444     main()
 445   except FatalException, e:
 446     sys.stderr.write(str(e))
 447     sys.exit(1)
 448
 449