cvs2svn_lib/git_output_option.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2007-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Classes for outputting the converted repository to git.
  18
  19 For information about the format allowed by git-fast-import, see:
  20
  21     http://www.kernel.org/pub/software/scm/git/docs/git-fast-import.html
  22
  23 """
  24
  25 import sys
  26 import bisect
  27 import time
  28 import shutil
  29
  30 from cvs2svn_lib import config
  31 from cvs2svn_lib.common import InternalError
  32 from cvs2svn_lib.log import logger
  33 from cvs2svn_lib.context import Ctx
  34 from cvs2svn_lib.symbol import Trunk
  35 from cvs2svn_lib.symbol import Branch
  36 from cvs2svn_lib.symbol import Tag
  37 from cvs2svn_lib.cvs_item import CVSSymbol
  38 from cvs2svn_lib.dvcs_common import DVCSOutputOption
  39 from cvs2svn_lib.dvcs_common import MirrorUpdater
  40 from cvs2svn_lib.key_generator import KeyGenerator
  41 from cvs2svn_lib.artifact_manager import artifact_manager
  42
  43
  44 class GitRevisionWriter(MirrorUpdater):
  45
  46   def start(self, mirror, f):
  47     MirrorUpdater.start(self, mirror)
  48     self.f = f
  49
  50   def _modify_file(self, cvs_item, post_commit):
  51     raise NotImplementedError()
  52
  53   def add_file(self, cvs_rev, post_commit):
  54     MirrorUpdater.add_file(self, cvs_rev, post_commit)
  55     self._modify_file(cvs_rev, post_commit)
  56
  57   def modify_file(self, cvs_rev, post_commit):
  58     MirrorUpdater.modify_file(self, cvs_rev, post_commit)
  59     self._modify_file(cvs_rev, post_commit)
  60
  61   def delete_file(self, cvs_rev, post_commit):
  62     MirrorUpdater.delete_file(self, cvs_rev, post_commit)
  63     self.f.write('D %s\n' % (cvs_rev.cvs_file.cvs_path,))
  64
  65   def branch_file(self, cvs_symbol):
  66     MirrorUpdater.branch_file(self, cvs_symbol)
  67     self._modify_file(cvs_symbol, post_commit=False)
  68
  69   def finish(self):
  70     MirrorUpdater.finish(self)
  71     del self.f
  72
  73
  74 class GitRevisionMarkWriter(GitRevisionWriter):
  75   def register_artifacts(self, which_pass):
  76     GitRevisionWriter.register_artifacts(self, which_pass)
  77     if Ctx().revision_collector.blob_filename is None:
  78       artifact_manager.register_temp_file_needed(
  79         config.GIT_BLOB_DATAFILE, which_pass,
  80         )
  81
  82   def start(self, mirror, f):
  83     GitRevisionWriter.start(self, mirror, f)
  84     if Ctx().revision_collector.blob_filename is None:
  85       # The revision collector wrote the blobs to a temporary file;
  86       # copy them into f:
  87       logger.normal('Copying blob data to output')
  88       blobf = open(
  89           artifact_manager.get_temp_file(config.GIT_BLOB_DATAFILE), 'rb',
  90           )
  91       shutil.copyfileobj(blobf, f)
  92       blobf.close()
  93
  94   def _modify_file(self, cvs_item, post_commit):
  95     if cvs_item.cvs_file.executable:
  96       mode = '100755'
  97     else:
  98       mode = '100644'
  99
 100     self.f.write(
 101         'M %s :%d %s\n'
 102         % (mode, cvs_item.revision_reader_token,
 103            cvs_item.cvs_file.cvs_path,)
 104         )
 105
 106
 107 class GitRevisionInlineWriter(GitRevisionWriter):
 108   def __init__(self, revision_reader):
 109     self.revision_reader = revision_reader
 110
 111   def register_artifacts(self, which_pass):
 112     GitRevisionWriter.register_artifacts(self, which_pass)
 113     self.revision_reader.register_artifacts(which_pass)
 114
 115   def start(self, mirror, f):
 116     GitRevisionWriter.start(self, mirror, f)
 117     self.revision_reader.start()
 118
 119   def _modify_file(self, cvs_item, post_commit):
 120     if cvs_item.cvs_file.executable:
 121       mode = '100755'
 122     else:
 123       mode = '100644'
 124
 125     self.f.write(
 126         'M %s inline %s\n'
 127         % (mode, cvs_item.cvs_file.cvs_path,)
 128         )
 129
 130     if isinstance(cvs_item, CVSSymbol):
 131       cvs_rev = cvs_item.get_cvs_revision_source(Ctx()._cvs_items_db)
 132     else:
 133       cvs_rev = cvs_item
 134
 135     # FIXME: We have to decide what to do about keyword substitution
 136     # and eol_style here:
 137     fulltext = self.revision_reader.get_content(cvs_rev)
 138
 139     self.f.write('data %d\n' % (len(fulltext),))
 140     self.f.write(fulltext)
 141     self.f.write('\n')
 142
 143   def finish(self):
 144     GitRevisionWriter.finish(self)
 145     self.revision_reader.finish()
 146
 147
 148 class GitOutputOption(DVCSOutputOption):
 149   """An OutputOption that outputs to a git-fast-import formatted file.
 150
 151   Members:
 152
 153     dump_filename -- (string or None) the name of the file to which
 154         the git-fast-import commands for defining revisions will be
 155         written.  If None, the data will be written to stdout.
 156
 157     author_transforms -- a map from CVS author names to git full name
 158         and email address.  See
 159         DVCSOutputOption.normalize_author_transforms() for information
 160         about the form of this parameter.
 161
 162   """
 163
 164   name = "Git"
 165
 166   # The first mark number used for git-fast-import commit marks.  This
 167   # value needs to be large to avoid conflicts with blob marks.
 168   _first_commit_mark = 1000000000
 169
 170   def __init__(
 171         self, revision_writer,
 172         dump_filename=None,
 173         author_transforms=None,
 174         tie_tag_fixup_branches=False,
 175         ):
 176     """Constructor.
 177
 178     REVISION_WRITER is a GitRevisionWriter that is used to output
 179     either the content of revisions or a mark that was previously used
 180     to label a blob.
 181
 182     DUMP_FILENAME is the name of the file to which the git-fast-import
 183     commands for defining revisions should be written.  (Please note
 184     that depending on the style of revision writer, the actual file
 185     contents might not be written to this file.)  If it is None, then
 186     the output is written to stdout.
 187
 188     AUTHOR_TRANSFORMS is a map {cvsauthor : (fullname, email)} from
 189     CVS author names to git full name and email address.  All of the
 190     contents should either be Unicode strings or 8-bit strings encoded
 191     as UTF-8.
 192
 193     TIE_TAG_FIXUP_BRANCHES means whether after finishing with a tag
 194     fixup branch, it should be psuedo-merged (ancestry linked but no
 195     content changes) back into its source branch, to dispose of the
 196     open head.
 197
 198     """
 199     DVCSOutputOption.__init__(self)
 200     self.dump_filename = dump_filename
 201     self.revision_writer = revision_writer
 202
 203     self.author_transforms = self.normalize_author_transforms(
 204         author_transforms
 205         )
 206
 207     self.tie_tag_fixup_branches = tie_tag_fixup_branches
 208
 209     self._mark_generator = KeyGenerator(GitOutputOption._first_commit_mark)
 210
 211   def register_artifacts(self, which_pass):
 212     DVCSOutputOption.register_artifacts(self, which_pass)
 213     self.revision_writer.register_artifacts(which_pass)
 214
 215   def check_symbols(self, symbol_map):
 216     # FIXME: What constraints does git impose on symbols?
 217     pass
 218
 219   def setup(self, svn_rev_count):
 220     DVCSOutputOption.setup(self, svn_rev_count)
 221     if self.dump_filename is None:
 222       self.f = sys.stdout
 223     else:
 224       self.f = open(self.dump_filename, 'wb')
 225
 226     # The youngest revnum that has been committed so far:
 227     self._youngest = 0
 228
 229     # A map {lod : [(revnum, mark)]} giving each of the revision
 230     # numbers in which there was a commit to lod, and the mark active
 231     # at the end of the revnum.
 232     self._marks = {}
 233
 234     self.revision_writer.start(self._mirror, self.f)
 235
 236   def _create_commit_mark(self, lod, revnum):
 237     mark = self._mark_generator.gen_id()
 238     self._set_lod_mark(lod, revnum, mark)
 239     return mark
 240
 241   def _set_lod_mark(self, lod, revnum, mark):
 242     """Record MARK as the status of LOD for REVNUM.
 243
 244     If there is already an entry for REVNUM, overwrite it.  If not,
 245     append a new entry to the self._marks list for LOD."""
 246
 247     assert revnum >= self._youngest
 248     entry = (revnum, mark)
 249     try:
 250       modifications = self._marks[lod]
 251     except KeyError:
 252       # This LOD hasn't appeared before; create a new list and add the
 253       # entry:
 254       self._marks[lod] = [entry]
 255     else:
 256       # A record exists, so it necessarily has at least one element:
 257       if modifications[-1][0] == revnum:
 258         modifications[-1] = entry
 259       else:
 260         modifications.append(entry)
 261     self._youngest = revnum
 262
 263   def _get_author(self, svn_commit):
 264     """Return the author to be used for SVN_COMMIT.
 265
 266     Return the author as a UTF-8 string in the form needed by git
 267     fast-import; that is, 'name <email>'."""
 268
 269     cvs_author = svn_commit.get_author()
 270     return self._map_author(cvs_author)
 271
 272   def _map_author(self, cvs_author):
 273     return self.author_transforms.get(cvs_author, "%s <>" % (cvs_author,))
 274
 275   @staticmethod
 276   def _get_log_msg(svn_commit):
 277     return svn_commit.get_log_msg()
 278
 279   def process_initial_project_commit(self, svn_commit):
 280     self._mirror.start_commit(svn_commit.revnum)
 281     self._mirror.end_commit()
 282
 283   def process_primary_commit(self, svn_commit):
 284     author = self._get_author(svn_commit)
 285     log_msg = self._get_log_msg(svn_commit)
 286
 287     lods = set()
 288     for cvs_rev in svn_commit.get_cvs_items():
 289       lods.add(cvs_rev.lod)
 290     if len(lods) != 1:
 291       raise InternalError('Commit affects %d LODs' % (len(lods),))
 292     lod = lods.pop()
 293
 294     self._mirror.start_commit(svn_commit.revnum)
 295     if isinstance(lod, Trunk):
 296       # FIXME: is this correct?:
 297       self.f.write('commit refs/heads/master\n')
 298     else:
 299       self.f.write('commit refs/heads/%s\n' % (lod.name,))
 300     mark = self._create_commit_mark(lod, svn_commit.revnum)
 301     logger.normal(
 302         'Writing commit r%d on %s (mark :%d)'
 303         % (svn_commit.revnum, lod, mark,)
 304         )
 305     self.f.write('mark :%d\n' % (mark,))
 306     self.f.write(
 307         'committer %s %d +0000\n' % (author, svn_commit.date,)
 308         )
 309     self.f.write('data %d\n' % (len(log_msg),))
 310     self.f.write('%s\n' % (log_msg,))
 311     for cvs_rev in svn_commit.get_cvs_items():
 312       self.revision_writer.process_revision(cvs_rev, post_commit=False)
 313
 314     self.f.write('\n')
 315     self._mirror.end_commit()
 316
 317   def process_post_commit(self, svn_commit):
 318     author = self._get_author(svn_commit)
 319     log_msg = self._get_log_msg(svn_commit)
 320
 321     source_lods = set()
 322     for cvs_rev in svn_commit.cvs_revs:
 323       source_lods.add(cvs_rev.lod)
 324     if len(source_lods) != 1:
 325       raise InternalError('Commit is from %d LODs' % (len(source_lods),))
 326     source_lod = source_lods.pop()
 327
 328     self._mirror.start_commit(svn_commit.revnum)
 329     # FIXME: is this correct?:
 330     self.f.write('commit refs/heads/master\n')
 331     mark = self._create_commit_mark(None, svn_commit.revnum)
 332     logger.normal(
 333         'Writing post-commit r%d on Trunk (mark :%d)'
 334         % (svn_commit.revnum, mark,)
 335         )
 336     self.f.write('mark :%d\n' % (mark,))
 337     self.f.write(
 338         'committer %s %d +0000\n' % (author, svn_commit.date,)
 339         )
 340     self.f.write('data %d\n' % (len(log_msg),))
 341     self.f.write('%s\n' % (log_msg,))
 342     self.f.write(
 343         'merge :%d\n'
 344         % (self._get_source_mark(source_lod, svn_commit.revnum),)
 345         )
 346     for cvs_rev in svn_commit.cvs_revs:
 347       self.revision_writer.process_revision(cvs_rev, post_commit=True)
 348
 349     self.f.write('\n')
 350     self._mirror.end_commit()
 351
 352   def _get_source_mark(self, source_lod, revnum):
 353     """Return the mark active on SOURCE_LOD at the end of REVNUM."""
 354
 355     modifications = self._marks[source_lod]
 356     i = bisect.bisect_left(modifications, (revnum + 1,)) - 1
 357     (revnum, mark) = modifications[i]
 358     return mark
 359
 360   def describe_lod_to_user(self, lod):
 361     """This needs to make sense to users of the fastimported result."""
 362     if isinstance(lod, Trunk):
 363       return 'master'
 364     else:
 365       return lod.name
 366
 367   def _describe_commit(self, svn_commit, lod):
 368       author = self._map_author(svn_commit.get_author())
 369       if author.endswith(" <>"):
 370         author = author[:-3]
 371       date = time.strftime(
 372           "%Y-%m-%d %H:%M:%S UTC", time.gmtime(svn_commit.date)
 373           )
 374       log_msg = svn_commit.get_log_msg()
 375       if log_msg.find('\n') != -1:
 376         log_msg = log_msg[:log_msg.index('\n')]
 377       return "%s %s %s '%s'" % (
 378           self.describe_lod_to_user(lod), date, author, log_msg,)
 379
 380   def _process_symbol_commit(self, svn_commit, git_branch, source_groups):
 381     author = self._get_author(svn_commit)
 382     log_msg = self._get_log_msg(svn_commit)
 383
 384     # There are two distinct cases we need to care for here:
 385     #  1. initial creation of a LOD
 386     #  2. fixup of an existing LOD to include more files, because the LOD in
 387     #     CVS was created piecemeal over time, with intervening commits
 388
 389     # We look at _marks here, but self._mirror._get_lod_history(lod).exists()
 390     # might be technically more correct (though _get_lod_history is currently
 391     # underscore-private)
 392     is_initial_lod_creation = svn_commit.symbol not in self._marks
 393
 394     # Create the mark, only after the check above
 395     mark = self._create_commit_mark(svn_commit.symbol, svn_commit.revnum)
 396
 397     if is_initial_lod_creation:
 398       # Get the primary parent
 399       p_source_revnum, p_source_lod, p_cvs_symbols = source_groups[0]
 400       try:
 401         p_source_node = self._mirror.get_old_lod_directory(
 402             p_source_lod, p_source_revnum
 403             )
 404       except KeyError:
 405         raise InternalError('Source %r does not exist' % (p_source_lod,))
 406       cvs_files_to_delete = set(self._get_all_files(p_source_node))
 407
 408       for (source_revnum, source_lod, cvs_symbols,) in source_groups:
 409         for cvs_symbol in cvs_symbols:
 410           cvs_files_to_delete.discard(cvs_symbol.cvs_file)
 411
 412     self.f.write('commit %s\n' % (git_branch,))
 413     self.f.write('mark :%d\n' % (mark,))
 414     self.f.write('committer %s %d +0000\n' % (author, svn_commit.date,))
 415     self.f.write('data %d\n' % (len(log_msg),))
 416     self.f.write('%s\n' % (log_msg,))
 417
 418     # Only record actual DVCS ancestry for the primary sprout parent,
 419     # all the rest are effectively cherrypicks.
 420     if is_initial_lod_creation:
 421       self.f.write(
 422           'from :%d\n'
 423           % (self._get_source_mark(p_source_lod, p_source_revnum),)
 424           )
 425
 426     for (source_revnum, source_lod, cvs_symbols,) in source_groups:
 427       for cvs_symbol in cvs_symbols:
 428         self.revision_writer.branch_file(cvs_symbol)
 429
 430     if is_initial_lod_creation:
 431       for cvs_file in cvs_files_to_delete:
 432         self.f.write('D %s\n' % (cvs_file.cvs_path,))
 433
 434     self.f.write('\n')
 435     return mark
 436
 437   def process_branch_commit(self, svn_commit):
 438     self._mirror.start_commit(svn_commit.revnum)
 439
 440     source_groups = self._get_source_groups(svn_commit)
 441     if self._is_simple_copy(svn_commit, source_groups):
 442       (source_revnum, source_lod, cvs_symbols) = source_groups[0]
 443       logger.debug(
 444           '%s will be created via a simple copy from %s:r%d'
 445           % (svn_commit.symbol, source_lod, source_revnum,)
 446           )
 447       mark = self._get_source_mark(source_lod, source_revnum)
 448       self._set_symbol(svn_commit.symbol, mark)
 449       self._mirror.copy_lod(source_lod, svn_commit.symbol, source_revnum)
 450       self._set_lod_mark(svn_commit.symbol, svn_commit.revnum, mark)
 451     else:
 452       logger.debug(
 453           '%s will be created via fixup commit(s)' % (svn_commit.symbol,)
 454           )
 455       self._process_symbol_commit(
 456           svn_commit, 'refs/heads/%s' % (svn_commit.symbol.name,),
 457           source_groups,
 458           )
 459
 460     self._mirror.end_commit()
 461
 462   def _set_symbol(self, symbol, mark):
 463     if isinstance(symbol, Branch):
 464       category = 'heads'
 465     elif isinstance(symbol, Tag):
 466       category = 'tags'
 467     else:
 468       raise InternalError()
 469     self.f.write('reset refs/%s/%s\n' % (category, symbol.name,))
 470     self.f.write('from :%d\n' % (mark,))
 471
 472   def get_tag_fixup_branch_name(self, svn_commit):
 473     # The branch name to use for the "tag fixup branches".  The
 474     # git-fast-import documentation suggests using 'TAG_FIXUP'
 475     # (outside of the refs/heads namespace), but this is currently
 476     # broken.  Use a name containing '.', which is not allowed in CVS
 477     # symbols, to avoid conflicts (though of course a conflict could
 478     # still result if the user requests symbol transformations).
 479     return 'refs/heads/TAG.FIXUP'
 480
 481   def process_tag_commit(self, svn_commit):
 482     # FIXME: For now we create a fixup branch with the same name as
 483     # the tag, then the tag.  We never delete the fixup branch.
 484     self._mirror.start_commit(svn_commit.revnum)
 485
 486     source_groups = self._get_source_groups(svn_commit)
 487     if self._is_simple_copy(svn_commit, source_groups):
 488       (source_revnum, source_lod, cvs_symbols) = source_groups[0]
 489       logger.debug(
 490           '%s will be created via a simple copy from %s:r%d'
 491           % (svn_commit.symbol, source_lod, source_revnum,)
 492           )
 493       mark = self._get_source_mark(source_lod, source_revnum)
 494       self._set_symbol(svn_commit.symbol, mark)
 495       self._mirror.copy_lod(source_lod, svn_commit.symbol, source_revnum)
 496       self._set_lod_mark(svn_commit.symbol, svn_commit.revnum, mark)
 497     else:
 498       logger.debug(
 499           '%s will be created via a fixup branch' % (svn_commit.symbol,)
 500           )
 501
 502       fixup_branch_name = self.get_tag_fixup_branch_name(svn_commit)
 503
 504       # Create the fixup branch (which might involve making more than
 505       # one commit):
 506       mark = self._process_symbol_commit(
 507           svn_commit, fixup_branch_name, source_groups
 508           )
 509
 510       # Store the mark of the last commit to the fixup branch as the
 511       # value of the tag:
 512       self._set_symbol(svn_commit.symbol, mark)
 513       self.f.write('reset %s\n' % (fixup_branch_name,))
 514       self.f.write('\n')
 515
 516       if self.tie_tag_fixup_branches:
 517         source_lod = source_groups[0][1]
 518         source_lod_git_branch = \
 519             'refs/heads/%s' % (getattr(source_lod, 'name', 'master'),)
 520
 521         mark2 = self._create_commit_mark(source_lod, svn_commit.revnum)
 522         author = self._map_author(Ctx().username)
 523         log_msg = self._get_log_msg_for_ancestry_tie(svn_commit)
 524
 525         self.f.write('commit %s\n' % (source_lod_git_branch,))
 526         self.f.write('mark :%d\n' % (mark2,))
 527         self.f.write('committer %s %d +0000\n' % (author, svn_commit.date,))
 528         self.f.write('data %d\n' % (len(log_msg),))
 529         self.f.write('%s\n' % (log_msg,))
 530
 531         self.f.write(
 532             'merge :%d\n'
 533             % (mark,)
 534             )
 535
 536         self.f.write('\n')
 537
 538     self._mirror.end_commit()
 539
 540   def _get_log_msg_for_ancestry_tie(self, svn_commit):
 541     return Ctx().text_wrapper.fill(
 542         Ctx().tie_tag_ancestry_message % {
 543             'symbol_name' : svn_commit.symbol.name,
 544             }
 545         )
 546
 547   def cleanup(self):
 548     DVCSOutputOption.cleanup(self)
 549     self.revision_writer.finish()
 550     if self.dump_filename is not None:
 551       self.f.close()
 552     del self.f
 553
 554