cvs2svn_lib/git_output_option.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2007-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Classes for outputting the converted repository to git.
  18
  19 For information about the format allowed by git-fast-import, see:
  20
  21     http://www.kernel.org/pub/software/scm/git/docs/git-fast-import.html
  22
  23 """
  24
  25 import bisect
  26 import time
  27
  28 from cvs2svn_lib.common import InternalError
  29 from cvs2svn_lib.log import logger
  30 from cvs2svn_lib.context import Ctx
  31 from cvs2svn_lib.symbol import Trunk
  32 from cvs2svn_lib.symbol import Branch
  33 from cvs2svn_lib.symbol import Tag
  34 from cvs2svn_lib.cvs_item import CVSSymbol
  35 from cvs2svn_lib.dvcs_common import DVCSOutputOption
  36 from cvs2svn_lib.dvcs_common import MirrorUpdater
  37 from cvs2svn_lib.key_generator import KeyGenerator
  38
  39
  40 class ExpectedDirectoryError(Exception):
  41   """A file was found where a directory was expected."""
  42
  43   pass
  44
  45
  46 class ExpectedFileError(Exception):
  47   """A directory was found where a file was expected."""
  48
  49   pass
  50
  51
  52 class GitRevisionWriter(MirrorUpdater):
  53
  54   def start(self, mirror, f):
  55     super(GitRevisionWriter, self).start(mirror)
  56     self.f = f
  57
  58   def _modify_file(self, cvs_item, post_commit):
  59     raise NotImplementedError()
  60
  61   def add_file(self, cvs_rev, post_commit):
  62     super(GitRevisionWriter, self).add_file(cvs_rev, post_commit)
  63     self._modify_file(cvs_rev, post_commit)
  64
  65   def modify_file(self, cvs_rev, post_commit):
  66     super(GitRevisionWriter, self).modify_file(cvs_rev, post_commit)
  67     self._modify_file(cvs_rev, post_commit)
  68
  69   def delete_file(self, cvs_rev, post_commit):
  70     super(GitRevisionWriter, self).delete_file(cvs_rev, post_commit)
  71     self.f.write('D %s\n' % (cvs_rev.cvs_file.cvs_path,))
  72
  73   def branch_file(self, cvs_symbol):
  74     super(GitRevisionWriter, self).branch_file(cvs_symbol)
  75     self._modify_file(cvs_symbol, post_commit=False)
  76
  77   def finish(self):
  78     super(GitRevisionWriter, self).finish()
  79     del self.f
  80
  81
  82 class GitRevisionMarkWriter(GitRevisionWriter):
  83   def _modify_file(self, cvs_item, post_commit):
  84     if cvs_item.cvs_file.executable:
  85       mode = '100755'
  86     else:
  87       mode = '100644'
  88
  89     self.f.write(
  90         'M %s :%d %s\n'
  91         % (mode, cvs_item.revision_reader_token,
  92            cvs_item.cvs_file.cvs_path,)
  93         )
  94
  95
  96 class GitRevisionInlineWriter(GitRevisionWriter):
  97   def __init__(self, revision_reader):
  98     self.revision_reader = revision_reader
  99
 100   def register_artifacts(self, which_pass):
 101     GitRevisionWriter.register_artifacts(self, which_pass)
 102     self.revision_reader.register_artifacts(which_pass)
 103
 104   def start(self, mirror, f):
 105     GitRevisionWriter.start(self, mirror, f)
 106     self.revision_reader.start()
 107
 108   def _modify_file(self, cvs_item, post_commit):
 109     if cvs_item.cvs_file.executable:
 110       mode = '100755'
 111     else:
 112       mode = '100644'
 113
 114     self.f.write(
 115         'M %s inline %s\n'
 116         % (mode, cvs_item.cvs_file.cvs_path,)
 117         )
 118
 119     if isinstance(cvs_item, CVSSymbol):
 120       cvs_rev = cvs_item.get_cvs_revision_source(Ctx()._cvs_items_db)
 121     else:
 122       cvs_rev = cvs_item
 123
 124     # FIXME: We have to decide what to do about keyword substitution
 125     # and eol_style here:
 126     fulltext = self.revision_reader.get_content(cvs_rev)
 127
 128     self.f.write('data %d\n' % (len(fulltext),))
 129     self.f.write(fulltext)
 130     self.f.write('\n')
 131
 132   def finish(self):
 133     GitRevisionWriter.finish(self)
 134     self.revision_reader.finish()
 135
 136
 137 class GitOutputOption(DVCSOutputOption):
 138   """An OutputOption that outputs to a git-fast-import formatted file.
 139
 140   Members:
 141
 142     dump_filename -- (string) the name of the file to which the
 143         git-fast-import commands for defining revisions will be
 144         written.
 145
 146     author_transforms -- a map from CVS author names to git full name
 147         and email address.  See
 148         DVCSOutputOption.normalize_author_transforms() for information
 149         about the form of this parameter.
 150
 151   """
 152
 153   name = "Git"
 154
 155   # The first mark number used for git-fast-import commit marks.  This
 156   # value needs to be large to avoid conflicts with blob marks.
 157   _first_commit_mark = 1000000000
 158
 159   def __init__(
 160         self, dump_filename, revision_writer,
 161         author_transforms=None,
 162         tie_tag_fixup_branches=False,
 163         ):
 164     """Constructor.
 165
 166     DUMP_FILENAME is the name of the file to which the git-fast-import
 167     commands for defining revisions should be written.  (Please note
 168     that depending on the style of revision writer, the actual file
 169     contents might not be written to this file.)
 170
 171     REVISION_WRITER is a GitRevisionWriter that is used to output
 172     either the content of revisions or a mark that was previously used
 173     to label a blob.
 174
 175     AUTHOR_TRANSFORMS is a map {cvsauthor : (fullname, email)} from
 176     CVS author names to git full name and email address.  All of the
 177     contents should either be Unicode strings or 8-bit strings encoded
 178     as UTF-8.
 179
 180     TIE_TAG_FIXUP_BRANCHES means whether after finishing with a tag
 181     fixup branch, it should be psuedo-merged (ancestry linked but no
 182     content changes) back into its source branch, to dispose of the
 183     open head.
 184
 185     """
 186     DVCSOutputOption.__init__(self)
 187     self.dump_filename = dump_filename
 188     self.revision_writer = revision_writer
 189
 190     self.author_transforms = self.normalize_author_transforms(
 191         author_transforms
 192         )
 193
 194     self.tie_tag_fixup_branches = tie_tag_fixup_branches
 195
 196     self._mark_generator = KeyGenerator(GitOutputOption._first_commit_mark)
 197
 198   def register_artifacts(self, which_pass):
 199     DVCSOutputOption.register_artifacts(self, which_pass)
 200     self.revision_writer.register_artifacts(which_pass)
 201
 202   def check_symbols(self, symbol_map):
 203     # FIXME: What constraints does git impose on symbols?
 204     pass
 205
 206   def setup(self, svn_rev_count):
 207     DVCSOutputOption.setup(self, svn_rev_count)
 208     self.f = open(self.dump_filename, 'wb')
 209
 210     # The youngest revnum that has been committed so far:
 211     self._youngest = 0
 212
 213     # A map {lod : [(revnum, mark)]} giving each of the revision
 214     # numbers in which there was a commit to lod, and the mark active
 215     # at the end of the revnum.
 216     self._marks = {}
 217
 218     self.revision_writer.start(self._mirror, self.f)
 219
 220   def _create_commit_mark(self, lod, revnum):
 221     mark = self._mark_generator.gen_id()
 222     self._set_lod_mark(lod, revnum, mark)
 223     return mark
 224
 225   def _set_lod_mark(self, lod, revnum, mark):
 226     """Record MARK as the status of LOD for REVNUM.
 227
 228     If there is already an entry for REVNUM, overwrite it.  If not,
 229     append a new entry to the self._marks list for LOD."""
 230
 231     assert revnum >= self._youngest
 232     entry = (revnum, mark)
 233     try:
 234       modifications = self._marks[lod]
 235     except KeyError:
 236       # This LOD hasn't appeared before; create a new list and add the
 237       # entry:
 238       self._marks[lod] = [entry]
 239     else:
 240       # A record exists, so it necessarily has at least one element:
 241       if modifications[-1][0] == revnum:
 242         modifications[-1] = entry
 243       else:
 244         modifications.append(entry)
 245     self._youngest = revnum
 246
 247   def _get_author(self, svn_commit):
 248     """Return the author to be used for SVN_COMMIT.
 249
 250     Return the author as a UTF-8 string in the form needed by git
 251     fast-import; that is, 'name <email>'."""
 252
 253     cvs_author = svn_commit.get_author()
 254     return self._map_author(cvs_author)
 255
 256   def _map_author(self, cvs_author):
 257     return self.author_transforms.get(cvs_author, "%s <>" % (cvs_author,))
 258
 259   @staticmethod
 260   def _get_log_msg(svn_commit):
 261     return svn_commit.get_log_msg()
 262
 263   def process_initial_project_commit(self, svn_commit):
 264     self._mirror.start_commit(svn_commit.revnum)
 265     self._mirror.end_commit()
 266
 267   def process_primary_commit(self, svn_commit):
 268     author = self._get_author(svn_commit)
 269     log_msg = self._get_log_msg(svn_commit)
 270
 271     lods = set()
 272     for cvs_rev in svn_commit.get_cvs_items():
 273       lods.add(cvs_rev.lod)
 274     if len(lods) != 1:
 275       raise InternalError('Commit affects %d LODs' % (len(lods),))
 276     lod = lods.pop()
 277
 278     self._mirror.start_commit(svn_commit.revnum)
 279     if isinstance(lod, Trunk):
 280       # FIXME: is this correct?:
 281       self.f.write('commit refs/heads/master\n')
 282     else:
 283       self.f.write('commit refs/heads/%s\n' % (lod.name,))
 284     self.f.write(
 285         'mark :%d\n'
 286         % (self._create_commit_mark(lod, svn_commit.revnum),)
 287         )
 288     self.f.write(
 289         'committer %s %d +0000\n' % (author, svn_commit.date,)
 290         )
 291     self.f.write('data %d\n' % (len(log_msg),))
 292     self.f.write('%s\n' % (log_msg,))
 293     for cvs_rev in svn_commit.get_cvs_items():
 294       self.revision_writer.process_revision(cvs_rev, post_commit=False)
 295
 296     self.f.write('\n')
 297     self._mirror.end_commit()
 298
 299   def process_post_commit(self, svn_commit):
 300     author = self._get_author(svn_commit)
 301     log_msg = self._get_log_msg(svn_commit)
 302
 303     source_lods = set()
 304     for cvs_rev in svn_commit.cvs_revs:
 305       source_lods.add(cvs_rev.lod)
 306     if len(source_lods) != 1:
 307       raise InternalError('Commit is from %d LODs' % (len(source_lods),))
 308     source_lod = source_lods.pop()
 309
 310     self._mirror.start_commit(svn_commit.revnum)
 311     # FIXME: is this correct?:
 312     self.f.write('commit refs/heads/master\n')
 313     self.f.write(
 314         'mark :%d\n'
 315         % (self._create_commit_mark(None, svn_commit.revnum),)
 316         )
 317     self.f.write(
 318         'committer %s %d +0000\n' % (author, svn_commit.date,)
 319         )
 320     self.f.write('data %d\n' % (len(log_msg),))
 321     self.f.write('%s\n' % (log_msg,))
 322     self.f.write(
 323         'merge :%d\n'
 324         % (self._get_source_mark(source_lod, svn_commit.revnum),)
 325         )
 326     for cvs_rev in svn_commit.cvs_revs:
 327       self.revision_writer.process_revision(cvs_rev, post_commit=True)
 328
 329     self.f.write('\n')
 330     self._mirror.end_commit()
 331
 332   def _get_source_mark(self, source_lod, revnum):
 333     """Return the mark active on SOURCE_LOD at the end of REVNUM."""
 334
 335     modifications = self._marks[source_lod]
 336     i = bisect.bisect_left(modifications, (revnum + 1,)) - 1
 337     (revnum, mark) = modifications[i]
 338     return mark
 339
 340   def describe_lod_to_user(self, lod):
 341     """This needs to make sense to users of the fastimported result."""
 342     if isinstance(lod, Trunk):
 343       return 'master'
 344     else:
 345       return lod.name
 346
 347   def _describe_commit(self, svn_commit, lod):
 348       author = self._map_author(svn_commit.get_author())
 349       if author.endswith(" <>"):
 350         author = author[:-3]
 351       date = time.strftime(
 352           "%Y-%m-%d %H:%M:%S UTC", time.gmtime(svn_commit.date)
 353           )
 354       log_msg = svn_commit.get_log_msg()
 355       if log_msg.find('\n') != -1:
 356         log_msg = log_msg[:log_msg.index('\n')]
 357       return "%s %s %s '%s'" % (
 358           self.describe_lod_to_user(lod), date, author, log_msg,)
 359
 360   def _process_symbol_commit(self, svn_commit, git_branch, source_groups):
 361     author = self._get_author(svn_commit)
 362     log_msg = self._get_log_msg(svn_commit)
 363
 364     # There are two distinct cases we need to care for here:
 365     #  1. initial creation of a LOD
 366     #  2. fixup of an existing LOD to include more files, because the LOD in
 367     #     CVS was created piecemeal over time, with intervening commits
 368
 369     # We look at _marks here, but self._mirror._get_lod_history(lod).exists()
 370     # might be technically more correct (though _get_lod_history is currently
 371     # underscore-private)
 372     is_initial_lod_creation = svn_commit.symbol not in self._marks
 373
 374     # Create the mark, only after the check above
 375     mark = self._create_commit_mark(svn_commit.symbol, svn_commit.revnum)
 376
 377     if is_initial_lod_creation:
 378       # Get the primary parent
 379       p_source_revnum, p_source_lod, p_cvs_symbols = source_groups[0]
 380       try:
 381         p_source_node = self._mirror.get_old_lod_directory(
 382             p_source_lod, p_source_revnum
 383             )
 384       except KeyError:
 385         raise InternalError('Source %r does not exist' % (p_source_lod,))
 386       cvs_files_to_delete = set(self._get_all_files(p_source_node))
 387
 388       for (source_revnum, source_lod, cvs_symbols,) in source_groups:
 389         for cvs_symbol in cvs_symbols:
 390           cvs_files_to_delete.discard(cvs_symbol.cvs_file)
 391
 392     # Write a trailer to the log message which describes the cherrypicks that
 393     # make up this symbol creation.
 394     log_msg += "\n"
 395     if is_initial_lod_creation:
 396       log_msg += "\nSprout from %s" % (
 397           self._describe_commit(
 398               Ctx()._persistence_manager.get_svn_commit(p_source_revnum),
 399               p_source_lod
 400               ),
 401           )
 402     for (source_revnum, source_lod, cvs_symbols,) \
 403             in source_groups[(is_initial_lod_creation and 1 or 0):]:
 404       log_msg += "\nCherrypick from %s:" % (
 405           self._describe_commit(
 406               Ctx()._persistence_manager.get_svn_commit(source_revnum),
 407               source_lod
 408               ),
 409           )
 410       for cvs_symbol in cvs_symbols:
 411         log_msg += "\n    %s" % (cvs_symbol.cvs_file.cvs_path,)
 412     if is_initial_lod_creation:
 413       if cvs_files_to_delete:
 414         log_msg += "\nDelete:"
 415         for cvs_file in sorted(cvs_files_to_delete):
 416           log_msg += "\n    %s" % (cvs_file.cvs_path,)
 417
 418     self.f.write('commit %s\n' % (git_branch,))
 419     self.f.write('mark :%d\n' % (mark,))
 420     self.f.write('committer %s %d +0000\n' % (author, svn_commit.date,))
 421     self.f.write('data %d\n' % (len(log_msg),))
 422     self.f.write('%s\n' % (log_msg,))
 423
 424     # Only record actual DVCS ancestry for the primary sprout parent,
 425     # all the rest are effectively cherrypicks.
 426     if is_initial_lod_creation:
 427       self.f.write(
 428           'from :%d\n'
 429           % (self._get_source_mark(p_source_lod, p_source_revnum),)
 430           )
 431
 432     for (source_revnum, source_lod, cvs_symbols,) in source_groups:
 433       for cvs_symbol in cvs_symbols:
 434         self.revision_writer.branch_file(cvs_symbol)
 435
 436     if is_initial_lod_creation:
 437       for cvs_file in cvs_files_to_delete:
 438         self.f.write('D %s\n' % (cvs_file.cvs_path,))
 439
 440     self.f.write('\n')
 441     return mark
 442
 443   def process_branch_commit(self, svn_commit):
 444     self._mirror.start_commit(svn_commit.revnum)
 445
 446     source_groups = self._get_source_groups(svn_commit)
 447     if self._is_simple_copy(svn_commit, source_groups):
 448       (source_revnum, source_lod, cvs_symbols) = source_groups[0]
 449       logger.debug(
 450           '%s will be created via a simple copy from %s:r%d'
 451           % (svn_commit.symbol, source_lod, source_revnum,)
 452           )
 453       mark = self._get_source_mark(source_lod, source_revnum)
 454       self._set_symbol(svn_commit.symbol, mark)
 455       self._mirror.copy_lod(source_lod, svn_commit.symbol, source_revnum)
 456       self._set_lod_mark(svn_commit.symbol, svn_commit.revnum, mark)
 457     else:
 458       logger.debug(
 459           '%s will be created via fixup commit(s)' % (svn_commit.symbol,)
 460           )
 461       self._process_symbol_commit(
 462           svn_commit, 'refs/heads/%s' % (svn_commit.symbol.name,),
 463           source_groups,
 464           )
 465
 466     self._mirror.end_commit()
 467
 468   def _set_symbol(self, symbol, mark):
 469     if isinstance(symbol, Branch):
 470       category = 'heads'
 471     elif isinstance(symbol, Tag):
 472       category = 'tags'
 473     else:
 474       raise InternalError()
 475     self.f.write('reset refs/%s/%s\n' % (category, symbol.name,))
 476     self.f.write('from :%d\n' % (mark,))
 477
 478   def get_tag_fixup_branch_name(self, svn_commit):
 479     # The branch name to use for the "tag fixup branches".  The
 480     # git-fast-import documentation suggests using 'TAG_FIXUP'
 481     # (outside of the refs/heads namespace), but this is currently
 482     # broken.  Use a name containing '.', which is not allowed in CVS
 483     # symbols, to avoid conflicts (though of course a conflict could
 484     # still result if the user requests symbol transformations).
 485     return 'refs/heads/TAG.FIXUP'
 486
 487   def process_tag_commit(self, svn_commit):
 488     # FIXME: For now we create a fixup branch with the same name as
 489     # the tag, then the tag.  We never delete the fixup branch.
 490     self._mirror.start_commit(svn_commit.revnum)
 491
 492     source_groups = self._get_source_groups(svn_commit)
 493     if self._is_simple_copy(svn_commit, source_groups):
 494       (source_revnum, source_lod, cvs_symbols) = source_groups[0]
 495       logger.debug(
 496           '%s will be created via a simple copy from %s:r%d'
 497           % (svn_commit.symbol, source_lod, source_revnum,)
 498           )
 499       mark = self._get_source_mark(source_lod, source_revnum)
 500       self._set_symbol(svn_commit.symbol, mark)
 501       self._mirror.copy_lod(source_lod, svn_commit.symbol, source_revnum)
 502       self._set_lod_mark(svn_commit.symbol, svn_commit.revnum, mark)
 503     else:
 504       logger.debug(
 505           '%s will be created via a fixup branch' % (svn_commit.symbol,)
 506           )
 507
 508       fixup_branch_name = self.get_tag_fixup_branch_name(svn_commit)
 509
 510       # Create the fixup branch (which might involve making more than
 511       # one commit):
 512       mark = self._process_symbol_commit(
 513           svn_commit, fixup_branch_name, source_groups
 514           )
 515
 516       # Store the mark of the last commit to the fixup branch as the
 517       # value of the tag:
 518       self._set_symbol(svn_commit.symbol, mark)
 519       self.f.write('reset %s\n' % (fixup_branch_name,))
 520       self.f.write('\n')
 521
 522       if self.tie_tag_fixup_branches:
 523         source_lod = source_groups[0][1]
 524         source_lod_git_branch = \
 525             'refs/heads/%s' % (getattr(source_lod, 'name', 'master'),)
 526
 527         mark2 = self._create_commit_mark(source_lod, svn_commit.revnum)
 528         author = self._map_author(Ctx().username)
 529         log_msg = self._get_log_msg_for_ancestry_tie(svn_commit)
 530
 531         self.f.write('commit %s\n' % (source_lod_git_branch,))
 532         self.f.write('mark :%d\n' % (mark2,))
 533         self.f.write('committer %s %d +0000\n' % (author, svn_commit.date,))
 534         self.f.write('data %d\n' % (len(log_msg),))
 535         self.f.write('%s\n' % (log_msg,))
 536
 537         self.f.write(
 538             'merge :%d\n'
 539             % (mark,)
 540             )
 541
 542         self.f.write('\n')
 543
 544     self._mirror.end_commit()
 545
 546   def _get_log_msg_for_ancestry_tie(self, svn_commit):
 547     return Ctx().text_wrapper.fill(
 548         Ctx().tie_tag_ancestry_message % {
 549             'symbol_name' : svn_commit.symbol.name,
 550             }
 551         )
 552
 553   def cleanup(self):
 554     DVCSOutputOption.cleanup(self)
 555     self.revision_writer.finish()
 556     self.f.close()
 557     del self.f
 558
 559