cvs2svn_lib/dvcs_common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2007-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Miscellaneous utility code common to DVCS backends (like
  18 Git, Mercurial, or Bazaar).
  19 """
  20
  21 import sys
  22
  23 from cvs2svn_lib import config
  24 from cvs2svn_lib.common import FatalError
  25 from cvs2svn_lib.common import InternalError
  26 from cvs2svn_lib.run_options import RunOptions
  27 from cvs2svn_lib.log import Log
  28 from cvs2svn_lib.common import error_prefix
  29 from cvs2svn_lib.context import Ctx
  30 from cvs2svn_lib.artifact_manager import artifact_manager
  31 from cvs2svn_lib.project import Project
  32 from cvs2svn_lib.cvs_item import CVSRevisionAdd
  33 from cvs2svn_lib.cvs_item import CVSRevisionChange
  34 from cvs2svn_lib.cvs_item import CVSRevisionDelete
  35 from cvs2svn_lib.cvs_item import CVSRevisionNoop
  36 from cvs2svn_lib.svn_revision_range import RevisionScores
  37 from cvs2svn_lib.openings_closings import SymbolingsReader
  38 from cvs2svn_lib.repository_mirror import RepositoryMirror
  39 from cvs2svn_lib.output_option import OutputOption
  40
  41
  42 class DVCSRunOptions(RunOptions):
  43   """Dumping ground for whatever is common to GitRunOptions and
  44   HgRunOptions."""
  45   def __init__(self, progname, cmd_args, pass_manager):
  46     Ctx().cross_project_commits = False
  47     Ctx().cross_branch_commits = False
  48     RunOptions.__init__(self, progname, cmd_args, pass_manager)
  49
  50   def set_project(
  51         self,
  52         project_cvs_repos_path,
  53         symbol_transforms=None,
  54         symbol_strategy_rules=[],
  55         ):
  56     """Set the project to be converted.
  57
  58     If a project had already been set, overwrite it.
  59
  60     Most arguments are passed straight through to the Project
  61     constructor.  SYMBOL_STRATEGY_RULES is an iterable of
  62     SymbolStrategyRules that will be applied to symbols in this
  63     project."""
  64
  65     symbol_strategy_rules = list(symbol_strategy_rules)
  66
  67     project = Project(
  68         0,
  69         project_cvs_repos_path,
  70         symbol_transforms=symbol_transforms,
  71         )
  72
  73     self.projects = [project]
  74     self.project_symbol_strategy_rules = [symbol_strategy_rules]
  75
  76   def process_options(self):
  77     # Consistency check for options and arguments.
  78     if len(self.args) == 0:
  79       self.usage()
  80       sys.exit(1)
  81
  82     if len(self.args) > 1:
  83       Log().error(error_prefix + ": must pass only one CVS repository.\n")
  84       self.usage()
  85       sys.exit(1)
  86
  87     cvsroot = self.args[0]
  88
  89     self.process_extraction_options()
  90     self.process_output_options()
  91     self.process_symbol_strategy_options()
  92     self.process_property_setter_options()
  93
  94     # Create the project:
  95     self.set_project(
  96         cvsroot,
  97         symbol_transforms=self.options.symbol_transforms,
  98         symbol_strategy_rules=self.options.symbol_strategy_rules,
  99         )
 100
 101
 102 class DVCSOutputOption(OutputOption):
 103   # name of output format (for error messages); must be set by
 104   # subclasses
 105   name = None
 106
 107   def __init__(self):
 108     self._mirror = RepositoryMirror()
 109     self._symbolings_reader = None
 110
 111   def normalize_author_transforms(self, author_transforms):
 112     """Return a new dict with the same content as author_transforms, but with
 113     all strings encoded to UTF-8 and the (name, email) tuple turned into a
 114     string.  Also turns None into the empty dict."""
 115     result = {}
 116     if author_transforms is not None:
 117       for (cvsauthor, (name, email,)) in author_transforms.iteritems():
 118         cvsauthor = to_utf8(cvsauthor)
 119         name = to_utf8(name)
 120         email = to_utf8(email)
 121         result[cvsauthor] = "%s <%s>" % (name, email,)
 122     return result
 123
 124   def register_artifacts(self, which_pass):
 125     # These artifacts are needed for SymbolingsReader:
 126     artifact_manager.register_temp_file_needed(
 127         config.SYMBOL_OPENINGS_CLOSINGS_SORTED, which_pass
 128         )
 129     artifact_manager.register_temp_file_needed(
 130         config.SYMBOL_OFFSETS_DB, which_pass
 131         )
 132     self._mirror.register_artifacts(which_pass)
 133
 134   def check(self):
 135     if Ctx().cross_project_commits:
 136       raise FatalError(
 137           '%s output is not supported with cross-project commits' % self.name
 138           )
 139     if Ctx().cross_branch_commits:
 140       raise FatalError(
 141           '%s output is not supported with cross-branch commits' % self.name
 142           )
 143     if Ctx().username is None:
 144       raise FatalError(
 145           '%s output requires a default commit username' % self.name
 146           )
 147
 148   def setup(self, svn_rev_count):
 149     self._symbolings_reader = SymbolingsReader()
 150     self._mirror.open()
 151
 152   def cleanup(self):
 153     self._mirror.close()
 154     self._symbolings_reader.close()
 155     del self._symbolings_reader
 156
 157   def _get_source_groups(self, svn_commit):
 158     """Return groups of sources for SVN_COMMIT.
 159
 160     SVN_COMMIT is an instance of SVNSymbolCommit.  Yield tuples
 161     (source_lod, svn_revnum, cvs_symbols) where source_lod is the line
 162     of development and svn_revnum is the revision that should serve as
 163     a source, and cvs_symbols is a list of CVSSymbolItems that can be
 164     copied from that source.  The groups are returned in arbitrary
 165     order."""
 166
 167     # Get a map {CVSSymbol : SVNRevisionRange}:
 168     range_map = self._symbolings_reader.get_range_map(svn_commit)
 169
 170     # range_map, split up into one map per LOD; i.e., {LOD :
 171     # {CVSSymbol : SVNRevisionRange}}:
 172     lod_range_maps = {}
 173
 174     for (cvs_symbol, range) in range_map.iteritems():
 175       lod_range_map = lod_range_maps.get(range.source_lod)
 176       if lod_range_map is None:
 177         lod_range_map = {}
 178         lod_range_maps[range.source_lod] = lod_range_map
 179       lod_range_map[cvs_symbol] = range
 180
 181     # Sort the sources so that the branch that serves most often as
 182     # parent is processed first:
 183     lod_ranges = lod_range_maps.items()
 184     lod_ranges.sort(
 185         lambda (lod1,lod_range_map1),(lod2,lod_range_map2):
 186         -cmp(len(lod_range_map1), len(lod_range_map2)) or cmp(lod1, lod2)
 187         )
 188
 189     for (lod, lod_range_map) in lod_ranges:
 190       while lod_range_map:
 191         revision_scores = RevisionScores(lod_range_map.values())
 192         (source_lod, revnum, score) = revision_scores.get_best_revnum()
 193         assert source_lod == lod
 194         cvs_symbols = []
 195         for (cvs_symbol, range) in lod_range_map.items():
 196           if revnum in range:
 197             cvs_symbols.append(cvs_symbol)
 198             del lod_range_map[cvs_symbol]
 199         yield (lod, revnum, cvs_symbols)
 200
 201   def _is_simple_copy(self, svn_commit, source_groups):
 202     """Return True iff SVN_COMMIT can be created as a simple copy.
 203
 204     SVN_COMMIT is an SVNTagCommit.  Return True iff it can be created
 205     as a simple copy from an existing revision (i.e., if the fixup
 206     branch can be avoided for this tag creation)."""
 207
 208     # The first requirement is that there be exactly one source:
 209     if len(source_groups) != 1:
 210       return False
 211
 212     (source_lod, svn_revnum, cvs_symbols) = source_groups[0]
 213
 214     # The second requirement is that the destination LOD not already
 215     # exist:
 216     try:
 217       self._mirror.get_current_lod_directory(svn_commit.symbol)
 218     except KeyError:
 219       # The LOD doesn't already exist.  This is good.
 220       pass
 221     else:
 222       # The LOD already exists.  It cannot be created by a copy.
 223       return False
 224
 225     # The third requirement is that the source LOD contains exactly
 226     # the same files as we need to add to the symbol:
 227     try:
 228       source_node = self._mirror.get_old_lod_directory(source_lod, svn_revnum)
 229     except KeyError:
 230       raise InternalError('Source %r does not exist' % (source_lod,))
 231     return (
 232         set([cvs_symbol.cvs_file for cvs_symbol in cvs_symbols])
 233         == set(self._get_all_files(source_node))
 234         )
 235
 236   def _get_all_files(self, node):
 237     """Generate all of the CVSFiles under NODE."""
 238
 239     for cvs_path in node:
 240       subnode = node[cvs_path]
 241       if subnode is None:
 242         yield cvs_path
 243       else:
 244         for sub_cvs_path in self._get_all_files(subnode):
 245           yield sub_cvs_path
 246
 247
 248 class MirrorUpdater(object):
 249   def register_artifacts(self, which_pass):
 250     pass
 251
 252   def start(self, mirror):
 253     self._mirror = mirror
 254
 255   def _mkdir_p(self, cvs_directory, lod):
 256     """Make sure that CVS_DIRECTORY exists in LOD.
 257
 258     If not, create it.  Return the node for CVS_DIRECTORY."""
 259
 260     try:
 261       node = self._mirror.get_current_lod_directory(lod)
 262     except KeyError:
 263       node = self._mirror.add_lod(lod)
 264
 265     for sub_path in cvs_directory.get_ancestry()[1:]:
 266       try:
 267         node = node[sub_path]
 268       except KeyError:
 269         node = node.mkdir(sub_path)
 270       if node is None:
 271         raise ExpectedDirectoryError(
 272             'File found at \'%s\' where directory was expected.' % (sub_path,)
 273             )
 274
 275     return node
 276
 277   def add_file(self, cvs_rev, post_commit):
 278     cvs_file = cvs_rev.cvs_file
 279     if post_commit:
 280       lod = cvs_file.project.get_trunk()
 281     else:
 282       lod = cvs_rev.lod
 283     parent_node = self._mkdir_p(cvs_file.parent_directory, lod)
 284     parent_node.add_file(cvs_file)
 285
 286   def modify_file(self, cvs_rev, post_commit):
 287     cvs_file = cvs_rev.cvs_file
 288     if post_commit:
 289       lod = cvs_file.project.get_trunk()
 290     else:
 291       lod = cvs_rev.lod
 292     if self._mirror.get_current_path(cvs_file, lod) is not None:
 293       raise ExpectedFileError(
 294           'Directory found at \'%s\' where file was expected.' % (cvs_file,)
 295           )
 296
 297   def delete_file(self, cvs_rev, post_commit):
 298     cvs_file = cvs_rev.cvs_file
 299     if post_commit:
 300       lod = cvs_file.project.get_trunk()
 301     else:
 302       lod = cvs_rev.lod
 303     parent_node = self._mirror.get_current_path(
 304         cvs_file.parent_directory, lod
 305         )
 306     if parent_node[cvs_file] is not None:
 307       raise ExpectedFileError(
 308           'Directory found at \'%s\' where file was expected.' % (cvs_file,)
 309           )
 310     del parent_node[cvs_file]
 311
 312   def process_revision(self, cvs_rev, post_commit):
 313     if isinstance(cvs_rev, CVSRevisionAdd):
 314       self.add_file(cvs_rev, post_commit)
 315     elif isinstance(cvs_rev, CVSRevisionChange):
 316       self.modify_file(cvs_rev, post_commit)
 317     elif isinstance(cvs_rev, CVSRevisionDelete):
 318       self.delete_file(cvs_rev, post_commit)
 319     elif isinstance(cvs_rev, CVSRevisionNoop):
 320       pass
 321     else:
 322       raise InternalError('Unexpected CVSRevision type: %s' % (cvs_rev,))
 323
 324   def branch_file(self, cvs_symbol):
 325     cvs_file = cvs_symbol.cvs_file
 326     parent_node = self._mkdir_p(cvs_file.parent_directory, cvs_symbol.symbol)
 327     parent_node.add_file(cvs_file)
 328
 329   def finish(self):
 330     del self._mirror
 331
 332
 333 def to_utf8(s):
 334   if isinstance(s, unicode):
 335     return s.encode('utf8')
 336   else:
 337     return s
 338
 339