cvs2svn_lib/external_blob_generator.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2009-2010 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Use the generate_blobs.py script to generate git blobs.
  18
  19 Use a separate program, generate_blobs.py, to generate a git blob file
  20 directly from the RCS files, setting blob marks that we choose.  This
  21 method is very much faster then generating the blobs from within the
  22 main program for several reasons:
  23
  24 * The revision fulltexts are generated using internal code (rather
  25   than spawning rcs or cvs once per revision).  This gain is analogous
  26   to the benefit of using --use-internal-co rather than --use-cvs or
  27   --use-rcs for cvs2svn.
  28
  29 * Intermediate revisions' fulltext can usually be held in RAM rather
  30   than being written to temporary storage, and output as they are
  31   generated (git-fast-import doesn't care about their order).
  32
  33 * The generate_blobs.py script runs in parallel to the main cvs2git
  34   script, allowing benefits to be had from multiple CPUs.
  35
  36 """
  37
  38 import sys
  39 import os
  40 import subprocess
  41 import cPickle as pickle
  42
  43 from cvs2svn_lib import config
  44 from cvs2svn_lib.common import FatalError
  45 from cvs2svn_lib.log import logger
  46 from cvs2svn_lib.cvs_item import CVSRevisionDelete
  47 from cvs2svn_lib.revision_manager import RevisionCollector
  48 from cvs2svn_lib.key_generator import KeyGenerator
  49 from cvs2svn_lib.artifact_manager import artifact_manager
  50
  51
  52 class ExternalBlobGenerator(RevisionCollector):
  53   """Have generate_blobs.py output file revisions to a blob file."""
  54
  55   def __init__(self, blob_filename=None):
  56     self.blob_filename = blob_filename
  57
  58   def register_artifacts(self, which_pass):
  59     RevisionCollector.register_artifacts(self, which_pass)
  60     if self.blob_filename is None:
  61       artifact_manager.register_temp_file(
  62         config.GIT_BLOB_DATAFILE, which_pass,
  63         )
  64
  65   def start(self):
  66     self._mark_generator = KeyGenerator()
  67     logger.normal('Starting generate_blobs.py...')
  68     if self.blob_filename is None:
  69       blob_filename = artifact_manager.get_temp_file(config.GIT_BLOB_DATAFILE)
  70     else:
  71       blob_filename = self.blob_filename
  72     self._pipe = subprocess.Popen(
  73         [
  74             sys.executable,
  75             os.path.join(os.path.dirname(__file__), 'generate_blobs.py'),
  76             blob_filename,
  77             ],
  78         stdin=subprocess.PIPE,
  79         )
  80
  81   def _process_symbol(self, cvs_symbol, cvs_file_items):
  82     """Record the original source of CVS_SYMBOL.
  83
  84     Determine the original revision source of CVS_SYMBOL, and store it
  85     as the symbol's revision_reader_token."""
  86
  87     cvs_source = cvs_symbol.get_cvs_revision_source(cvs_file_items)
  88     cvs_symbol.revision_reader_token = cvs_source.revision_reader_token
  89
  90   def process_file(self, cvs_file_items):
  91     marks = {}
  92     for lod_items in cvs_file_items.iter_lods():
  93       for cvs_rev in lod_items.cvs_revisions:
  94         if not isinstance(cvs_rev, CVSRevisionDelete):
  95           mark = self._mark_generator.gen_id()
  96           cvs_rev.revision_reader_token = mark
  97           marks[cvs_rev.rev] = mark
  98
  99     if marks:
 100       # A separate pickler is used for each dump(), so that its memo
 101       # doesn't grow very large.  The default ASCII protocol is used so
 102       # that this works without changes on systems that distinguish
 103       # between text and binary files.
 104       pickle.dump((cvs_file_items.cvs_file.rcs_path, marks), self._pipe.stdin)
 105       self._pipe.stdin.flush()
 106
 107     # Now that all CVSRevisions' revision_reader_tokens are set,
 108     # iterate through symbols and set their tokens to those of their
 109     # original source revisions:
 110     for lod_items in cvs_file_items.iter_lods():
 111       if lod_items.cvs_branch is not None:
 112         self._process_symbol(lod_items.cvs_branch, cvs_file_items)
 113       for cvs_tag in lod_items.cvs_tags:
 114         self._process_symbol(cvs_tag, cvs_file_items)
 115
 116   def finish(self):
 117     self._pipe.stdin.close()
 118     logger.normal('Waiting for generate_blobs.py to finish...')
 119     returncode = self._pipe.wait()
 120     if returncode:
 121       raise FatalError(
 122           'generate_blobs.py failed with return code %s.' % (returncode,)
 123           )
 124     else:
 125       logger.normal('generate_blobs.py is done.')
 126
 127