cvs2svn_lib/git_output_option.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2007-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Classes for outputting the converted repository to git.
  18
  19 For information about the format allowed by git-fast-import, see:
  20
  21     http://www.kernel.org/pub/software/scm/git/docs/git-fast-import.html
  22
  23 """
  24
  25 import bisect
  26 import time
  27
  28 from cvs2svn_lib.common import InternalError
  29 from cvs2svn_lib.common import FatalError
  30 from cvs2svn_lib.log import Log
  31 from cvs2svn_lib.context import Ctx
  32 from cvs2svn_lib.symbol import Trunk
  33 from cvs2svn_lib.symbol import Branch
  34 from cvs2svn_lib.symbol import Tag
  35 from cvs2svn_lib.cvs_item import CVSSymbol
  36 from cvs2svn_lib.dvcs_common import DVCSOutputOption
  37 from cvs2svn_lib.dvcs_common import MirrorUpdater
  38 from cvs2svn_lib.key_generator import KeyGenerator
  39
  40
  41 class ExpectedDirectoryError(Exception):
  42   """A file was found where a directory was expected."""
  43
  44   pass
  45
  46
  47 class ExpectedFileError(Exception):
  48   """A directory was found where a file was expected."""
  49
  50   pass
  51
  52
  53 class GitRevisionWriter(MirrorUpdater):
  54
  55   def start(self, mirror, f):
  56     super(GitRevisionWriter, self).start(mirror)
  57     self.f = f
  58
  59   def _modify_file(self, cvs_item, post_commit):
  60     raise NotImplementedError()
  61
  62   def add_file(self, cvs_rev, post_commit):
  63     super(GitRevisionWriter, self).add_file(cvs_rev, post_commit)
  64     self._modify_file(cvs_rev, post_commit)
  65
  66   def modify_file(self, cvs_rev, post_commit):
  67     super(GitRevisionWriter, self).modify_file(cvs_rev, post_commit)
  68     self._modify_file(cvs_rev, post_commit)
  69
  70   def delete_file(self, cvs_rev, post_commit):
  71     super(GitRevisionWriter, self).delete_file(cvs_rev, post_commit)
  72     self.f.write('D %s\n' % (cvs_rev.cvs_file.cvs_path,))
  73
  74   def branch_file(self, cvs_symbol):
  75     super(GitRevisionWriter, self).branch_file(cvs_symbol)
  76     self._modify_file(cvs_symbol, post_commit=False)
  77
  78   def finish(self):
  79     super(GitRevisionWriter, self).finish()
  80     del self.f
  81
  82
  83 class GitRevisionMarkWriter(GitRevisionWriter):
  84   def _modify_file(self, cvs_item, post_commit):
  85     if cvs_item.cvs_file.executable:
  86       mode = '100755'
  87     else:
  88       mode = '100644'
  89
  90     self.f.write(
  91         'M %s :%d %s\n'
  92         % (mode, cvs_item.revision_recorder_token,
  93            cvs_item.cvs_file.cvs_path,)
  94         )
  95
  96
  97 class GitRevisionInlineWriter(GitRevisionWriter):
  98   def __init__(self, revision_reader):
  99     self.revision_reader = revision_reader
 100
 101   def register_artifacts(self, which_pass):
 102     GitRevisionWriter.register_artifacts(self, which_pass)
 103     self.revision_reader.register_artifacts(which_pass)
 104
 105   def start(self, mirror, f):
 106     GitRevisionWriter.start(self, mirror, f)
 107     self.revision_reader.start()
 108
 109   def _modify_file(self, cvs_item, post_commit):
 110     if cvs_item.cvs_file.executable:
 111       mode = '100755'
 112     else:
 113       mode = '100644'
 114
 115     self.f.write(
 116         'M %s inline %s\n'
 117         % (mode, cvs_item.cvs_file.cvs_path,)
 118         )
 119
 120     if isinstance(cvs_item, CVSSymbol):
 121       cvs_rev = cvs_item.get_cvs_revision_source(Ctx()._cvs_items_db)
 122     else:
 123       cvs_rev = cvs_item
 124
 125     # FIXME: We have to decide what to do about keyword substitution
 126     # and eol_style here:
 127     stream = self.revision_reader.get_content_stream(
 128         cvs_rev, suppress_keyword_substitution=False
 129         )
 130     fulltext = stream.read()
 131     stream.close()
 132
 133     self.f.write('data %d\n' % (len(fulltext),))
 134     self.f.write(fulltext)
 135     self.f.write('\n')
 136
 137   def finish(self):
 138     GitRevisionWriter.finish(self)
 139     self.revision_reader.finish()
 140
 141
 142 class GitOutputOption(DVCSOutputOption):
 143   """An OutputOption that outputs to a git-fast-import formatted file.
 144
 145   Members:
 146
 147     dump_filename -- (string) the name of the file to which the
 148         git-fast-import commands for defining revisions will be
 149         written.
 150
 151     author_transforms -- a map {cvsauthor : (fullname, email)} from
 152         CVS author names to git full name and email address.  All of
 153         the contents are 8-bit strings encoded as UTF-8.
 154
 155   """
 156
 157   name = "Git"
 158
 159   # The first mark number used for git-fast-import commit marks.  This
 160   # value needs to be large to avoid conflicts with blob marks.
 161   _first_commit_mark = 1000000000
 162
 163   def __init__(
 164         self, dump_filename, revision_writer,
 165         author_transforms=None,
 166         tie_tag_fixup_branches=False,
 167         ):
 168     """Constructor.
 169
 170     DUMP_FILENAME is the name of the file to which the git-fast-import
 171     commands for defining revisions should be written.  (Please note
 172     that depending on the style of revision writer, the actual file
 173     contents might not be written to this file.)
 174
 175     REVISION_WRITER is a GitRevisionWriter that is used to output
 176     either the content of revisions or a mark that was previously used
 177     to label a blob.
 178
 179     AUTHOR_TRANSFORMS is a map {cvsauthor : (fullname, email)} from
 180     CVS author names to git full name and email address.  All of the
 181     contents should either be Unicode strings or 8-bit strings encoded
 182     as UTF-8.
 183
 184     TIE_TAG_FIXUP_BRANCHES means whether after finishing with a tag fixup
 185     branch, it should be psuedo-merged (ancestry linked but no content changes)
 186     back into its source branch, to dispose of the open head.
 187
 188     """
 189     DVCSOutputOption.__init__(self)
 190     self.dump_filename = dump_filename
 191     self.revision_writer = revision_writer
 192
 193     self.author_transforms = self.normalize_author_transforms(author_transforms)
 194
 195     self.tie_tag_fixup_branches = tie_tag_fixup_branches
 196
 197     self._mark_generator = KeyGenerator(GitOutputOption._first_commit_mark)
 198
 199   def register_artifacts(self, which_pass):
 200     DVCSOutputOption.register_artifacts(self, which_pass)
 201     self.revision_writer.register_artifacts(which_pass)
 202
 203   def check_symbols(self, symbol_map):
 204     # FIXME: What constraints does git impose on symbols?
 205     pass
 206
 207   def setup(self, svn_rev_count):
 208     DVCSOutputOption.setup(self, svn_rev_count)
 209     self.f = open(self.dump_filename, 'wb')
 210
 211     # The youngest revnum that has been committed so far:
 212     self._youngest = 0
 213
 214     # A map {lod : [(revnum, mark)]} giving each of the revision
 215     # numbers in which there was a commit to lod, and the mark active
 216     # at the end of the revnum.
 217     self._marks = {}
 218
 219     self.revision_writer.start(self._mirror, self.f)
 220
 221   def _create_commit_mark(self, lod, revnum):
 222     mark = self._mark_generator.gen_id()
 223     self._set_lod_mark(lod, revnum, mark)
 224     return mark
 225
 226   def _set_lod_mark(self, lod, revnum, mark):
 227     """Record MARK as the status of LOD for REVNUM.
 228
 229     If there is already an entry for REVNUM, overwrite it.  If not,
 230     append a new entry to the self._marks list for LOD."""
 231
 232     assert revnum >= self._youngest
 233     entry = (revnum, mark)
 234     try:
 235       modifications = self._marks[lod]
 236     except KeyError:
 237       # This LOD hasn't appeared before; create a new list and add the
 238       # entry:
 239       self._marks[lod] = [entry]
 240     else:
 241       # A record exists, so it necessarily has at least one element:
 242       if modifications[-1][0] == revnum:
 243         modifications[-1] = entry
 244       else:
 245         modifications.append(entry)
 246     self._youngest = revnum
 247
 248   def _get_author(self, svn_commit):
 249     """Return the author to be used for SVN_COMMIT.
 250
 251     Return the author as a UTF-8 string in the form needed by git fast-import;
 252     that is, 'name <email>'."""
 253
 254     cvs_author = svn_commit.get_author()
 255     return self._map_author(cvs_author)
 256
 257   def _map_author(self, cvs_author):
 258     return self.author_transforms.get(cvs_author, "%s <>" % (cvs_author,))
 259
 260   @staticmethod
 261   def _get_log_msg(svn_commit):
 262     return svn_commit.get_log_msg()
 263
 264   def process_initial_project_commit(self, svn_commit):
 265     self._mirror.start_commit(svn_commit.revnum)
 266     self._mirror.end_commit()
 267
 268   def process_primary_commit(self, svn_commit):
 269     author = self._get_author(svn_commit)
 270     log_msg = self._get_log_msg(svn_commit)
 271
 272     lods = set()
 273     for cvs_rev in svn_commit.get_cvs_items():
 274       lods.add(cvs_rev.lod)
 275     if len(lods) != 1:
 276       raise InternalError('Commit affects %d LODs' % (len(lods),))
 277     lod = lods.pop()
 278
 279     self._mirror.start_commit(svn_commit.revnum)
 280     if isinstance(lod, Trunk):
 281       # FIXME: is this correct?:
 282       self.f.write('commit refs/heads/master\n')
 283     else:
 284       self.f.write('commit refs/heads/%s\n' % (lod.name,))
 285     self.f.write(
 286         'mark :%d\n'
 287         % (self._create_commit_mark(lod, svn_commit.revnum),)
 288         )
 289     self.f.write(
 290         'committer %s %d +0000\n' % (author, svn_commit.date,)
 291         )
 292     self.f.write('data %d\n' % (len(log_msg),))
 293     self.f.write('%s\n' % (log_msg,))
 294     for cvs_rev in svn_commit.get_cvs_items():
 295       self.revision_writer.process_revision(cvs_rev, post_commit=False)
 296
 297     self.f.write('\n')
 298     self._mirror.end_commit()
 299
 300   def process_post_commit(self, svn_commit):
 301     author = self._get_author(svn_commit)
 302     log_msg = self._get_log_msg(svn_commit)
 303
 304     source_lods = set()
 305     for cvs_rev in svn_commit.cvs_revs:
 306       source_lods.add(cvs_rev.lod)
 307     if len(source_lods) != 1:
 308       raise InternalError('Commit is from %d LODs' % (len(source_lods),))
 309     source_lod = source_lods.pop()
 310
 311     self._mirror.start_commit(svn_commit.revnum)
 312     # FIXME: is this correct?:
 313     self.f.write('commit refs/heads/master\n')
 314     self.f.write(
 315         'mark :%d\n'
 316         % (self._create_commit_mark(None, svn_commit.revnum),)
 317         )
 318     self.f.write(
 319         'committer %s %d +0000\n' % (author, svn_commit.date,)
 320         )
 321     self.f.write('data %d\n' % (len(log_msg),))
 322     self.f.write('%s\n' % (log_msg,))
 323     self.f.write(
 324         'merge :%d\n'
 325         % (self._get_source_mark(source_lod, svn_commit.revnum),)
 326         )
 327     for cvs_rev in svn_commit.cvs_revs:
 328       self.revision_writer.process_revision(cvs_rev, post_commit=True)
 329
 330     self.f.write('\n')
 331     self._mirror.end_commit()
 332
 333   def _get_source_mark(self, source_lod, revnum):
 334     """Return the mark active on SOURCE_LOD at the end of REVNUM."""
 335
 336     modifications = self._marks[source_lod]
 337     i = bisect.bisect_left(modifications, (revnum + 1,)) - 1
 338     (revnum, mark) = modifications[i]
 339     return mark
 340
 341   def describe_lod_to_user(self, lod):
 342     """This needs to make sense to users of the fastimported result."""
 343     if isinstance(lod, Trunk):
 344       return 'master'
 345     else:
 346       return lod.name
 347
 348   def _describe_commit(self, svn_commit, lod):
 349       author = self._map_author(svn_commit.get_author())
 350       if author.endswith(" <>"):
 351         author = author[:-3]
 352       date = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime(svn_commit.date))
 353       log_msg = svn_commit.get_log_msg()
 354       if log_msg.find('\n') != -1:
 355         log_msg = log_msg[:log_msg.index('\n')]
 356       return "%s %s %s '%s'" % (
 357           self.describe_lod_to_user(lod), date, author, log_msg,)
 358
 359   def _process_symbol_commit(
 360         self, svn_commit, git_branch, source_groups, mark
 361         ):
 362     author = self._get_author(svn_commit)
 363     log_msg = self._get_log_msg(svn_commit)
 364
 365     # Get the primary parent
 366     p_source_lod, p_source_revnum, p_cvs_symbols = source_groups[0]
 367     try:
 368       p_source_node = self._mirror.get_old_lod_directory(p_source_lod, p_source_revnum)
 369     except KeyError:
 370       raise InternalError('Source %r does not exist' % (p_source_lod,))
 371     cvs_files_to_delete = set(self._get_all_files(p_source_node))
 372
 373     for (source_lod, source_revnum, cvs_symbols,) in source_groups:
 374       for cvs_symbol in cvs_symbols:
 375         cvs_files_to_delete.discard(cvs_symbol.cvs_file)
 376
 377     # Write a trailer to the log message which describes the cherrypicks that
 378     # make up this symbol creation.
 379     log_msg += "\n"
 380     log_msg += "\nSprout from %s" % (
 381         self._describe_commit(Ctx()._persistence_manager.get_svn_commit(p_source_revnum),
 382           p_source_lod),)
 383     for (source_lod, source_revnum, cvs_symbols,) in source_groups[1:]:
 384       log_msg += "\nCherrypick from %s:" % (
 385           self._describe_commit(Ctx()._persistence_manager.get_svn_commit(source_revnum),
 386             source_lod),)
 387       for cvs_symbol in cvs_symbols:
 388         log_msg += "\n    %s" % (cvs_symbol.cvs_file.cvs_path,)
 389     if len(cvs_files_to_delete):
 390       log_msg += "\nDelete:"
 391       for cvs_file in sorted(cvs_files_to_delete):
 392         log_msg += "\n    %s" % (cvs_file.cvs_path,)
 393
 394     self.f.write('commit %s\n' % (git_branch,))
 395     self.f.write('mark :%d\n' % (mark,))
 396     self.f.write('committer %s %d +0000\n' % (author, svn_commit.date,))
 397     self.f.write('data %d\n' % (len(log_msg),))
 398     self.f.write('%s\n' % (log_msg,))
 399
 400     # Only record actual DVCS ancestry for the primary sprout parent,
 401     # all the rest are effectively cherrypicks.
 402     self.f.write(
 403         'merge :%d\n'
 404         % (self._get_source_mark(p_source_lod, p_source_revnum),)
 405         )
 406
 407     for (source_lod, source_revnum, cvs_symbols,) in source_groups:
 408       for cvs_symbol in cvs_symbols:
 409         self.revision_writer.branch_file(cvs_symbol)
 410
 411     for cvs_file in cvs_files_to_delete:
 412       self.f.write('D %s\n' % (cvs_file.cvs_path,))
 413
 414     self.f.write('\n')
 415
 416   def process_branch_commit(self, svn_commit):
 417     self._mirror.start_commit(svn_commit.revnum)
 418
 419     source_groups = list(self._get_source_groups(svn_commit))
 420     if self._is_simple_copy(svn_commit, source_groups):
 421       (source_lod, source_revnum, cvs_symbols) = source_groups[0]
 422       Log().debug(
 423           '%s will be created via a simple copy from %s:r%d'
 424           % (svn_commit.symbol, source_lod, source_revnum,)
 425           )
 426       mark = self._get_source_mark(source_lod, source_revnum)
 427       self._set_symbol(svn_commit.symbol, mark)
 428       self._mirror.copy_lod(source_lod, svn_commit.symbol, source_revnum)
 429     else:
 430       Log().debug(
 431           '%s will be created via fixup commit(s)' % (svn_commit.symbol,)
 432           )
 433       self._process_symbol_commit(
 434           svn_commit, 'refs/heads/%s' % (svn_commit.symbol.name,),
 435           source_groups,
 436           self._create_commit_mark(svn_commit.symbol, svn_commit.revnum),
 437           )
 438
 439     self._mirror.end_commit()
 440
 441   def _set_symbol(self, symbol, mark):
 442     if isinstance(symbol, Branch):
 443       category = 'heads'
 444     elif isinstance(symbol, Tag):
 445       category = 'tags'
 446     else:
 447       raise InternalError()
 448     self.f.write('reset refs/%s/%s\n' % (category, symbol.name,))
 449     self.f.write('from :%d\n' % (mark,))
 450
 451   def get_tag_fixup_branch_name(self, svn_commit):
 452     # The branch name to use for the "tag fixup branches".  The git-fast-import
 453     # documentation suggests using 'TAG_FIXUP' (outside of the refs/heads
 454     # namespace), but this is currently broken.
 455     # Use a name containing '.', which is not allowed in CVS symbols, to avoid
 456     # conflicts (though of course a conflict could still result if the user
 457     # requests symbol transformations).
 458     return 'refs/heads/TAG.FIXUP'
 459
 460   def process_tag_commit(self, svn_commit):
 461     # FIXME: For now we create a fixup branch with the same name as
 462     # the tag, then the tag.  We never delete the fixup branch.
 463     self._mirror.start_commit(svn_commit.revnum)
 464
 465     source_groups = list(self._get_source_groups(svn_commit))
 466     if self._is_simple_copy(svn_commit, source_groups):
 467       (source_lod, source_revnum, cvs_symbols) = source_groups[0]
 468       Log().debug(
 469           '%s will be created via a simple copy from %s:r%d'
 470           % (svn_commit.symbol, source_lod, source_revnum,)
 471           )
 472       mark = self._get_source_mark(source_lod, source_revnum)
 473       self._set_symbol(svn_commit.symbol, mark)
 474     else:
 475       Log().debug(
 476           '%s will be created via a fixup branch' % (svn_commit.symbol,)
 477           )
 478
 479       fixup_branch_name = self.get_tag_fixup_branch_name(svn_commit)
 480
 481       # Create the fixup branch (which might involve making more than
 482       # one commit):
 483       mark = self._create_commit_mark(svn_commit.symbol, svn_commit.revnum)
 484       self._process_symbol_commit(
 485           svn_commit, fixup_branch_name, source_groups, mark
 486           )
 487
 488       # Store the mark of the last commit to the fixup branch as the
 489       # value of the tag:
 490       self._set_symbol(svn_commit.symbol, mark)
 491       self.f.write('reset %s\n' % (fixup_branch_name,))
 492       self.f.write('\n')
 493
 494       if self.tie_tag_fixup_branches:
 495         source_lod = source_groups[0][0]
 496         source_lod_git_branch = 'refs/heads/%s' % (getattr(source_lod, 'name', 'master'),)
 497
 498         mark2 = self._create_commit_mark(source_lod, svn_commit.revnum)
 499         author = self._map_author(Ctx().username)
 500         log_msg = self._get_log_msg_for_ancestry_tie(svn_commit)
 501
 502         self.f.write('commit %s\n' % (source_lod_git_branch,))
 503         self.f.write('mark :%d\n' % (mark2,))
 504         self.f.write('committer %s %d +0000\n' % (author, svn_commit.date,))
 505         self.f.write('data %d\n' % (len(log_msg),))
 506         self.f.write('%s\n' % (log_msg,))
 507
 508         self.f.write(
 509             'merge :%d\n'
 510             % (mark,)
 511             )
 512
 513         self.f.write('\n')
 514
 515     self._mirror.end_commit()
 516
 517   def _get_log_msg_for_ancestry_tie(self, svn_commit):
 518     return Ctx().text_wrapper.fill(
 519         Ctx().tie_tag_ancestry_message % {
 520             'symbol_name' : svn_commit.symbol.name,
 521             }
 522         )
 523
 524   def cleanup(self):
 525     DVCSOutputOption.cleanup(self)
 526     self.revision_writer.finish()
 527     self.f.close()
 528     del self.f
 529
 530