cvs2svn_lib/pass_manager.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains tools to manage the passes of a conversion."""
  18
  19
  20 import time
  21 import gc
  22
  23 from cvs2svn_lib import config
  24 from cvs2svn_lib.common import FatalError
  25 from cvs2svn_lib.context import Ctx
  26 from cvs2svn_lib.log import logger
  27 from cvs2svn_lib.stats_keeper import StatsKeeper
  28 from cvs2svn_lib.stats_keeper import read_stats_keeper
  29 from cvs2svn_lib.artifact_manager import artifact_manager
  30
  31
  32 class InvalidPassError(FatalError):
  33   def __init__(self, msg):
  34     FatalError.__init__(
  35         self, msg + '\nUse --help-passes for more information.')
  36
  37
  38 def check_for_garbage():
  39   # We've turned off the garbage collector because we shouldn't
  40   # need it (we don't create circular dependencies) and because it
  41   # is therefore a waste of time.  So here we check for any
  42   # unreachable objects and generate a debug-level warning if any
  43   # occur:
  44   try:
  45     gc.set_debug(gc.DEBUG_SAVEALL)
  46     gc_count = gc.collect()
  47     if gc_count:
  48       if logger.is_on(logger.DEBUG):
  49         logger.debug(
  50             'INTERNAL: %d unreachable object(s) were garbage collected:'
  51             % (gc_count,)
  52             )
  53         for g in gc.garbage:
  54           logger.debug('    %s' % (g,))
  55       del gc.garbage[:]
  56   except (AttributeError, NotImplementedError):
  57     # Other Python implementations implement garbage collection
  58     # differently, so if errors occur just ignore them.
  59     pass
  60
  61
  62 class Pass(object):
  63   """Base class for one step of the conversion."""
  64
  65   def __init__(self):
  66     # By default, use the pass object's class name as the pass name:
  67     self.name = self.__class__.__name__
  68
  69   def register_artifacts(self):
  70     """Register artifacts (created and needed) in artifact_manager."""
  71
  72     raise NotImplementedError()
  73
  74   def _register_temp_file(self, basename):
  75     """Helper method; for brevity only."""
  76
  77     artifact_manager.register_temp_file(basename, self)
  78
  79   def _register_temp_file_needed(self, basename):
  80     """Helper method; for brevity only."""
  81
  82     artifact_manager.register_temp_file_needed(basename, self)
  83
  84   def run(self, run_options, stats_keeper):
  85     """Carry out this step of the conversion.
  86
  87     RUN_OPTIONS is an instance of RunOptions.  STATS_KEEPER is an
  88     instance of StatsKeeper."""
  89
  90     raise NotImplementedError()
  91
  92
  93 class PassManager:
  94   """Manage a list of passes that can be executed separately or all at once.
  95
  96   Passes are numbered starting with 1."""
  97
  98   def __init__(self, passes):
  99     """Construct a PassManager with the specified PASSES.
 100
 101     Internally, passes are numbered starting with 1.  So PASSES[0] is
 102     considered to be pass number 1."""
 103
 104     self.passes = passes
 105     self.num_passes = len(self.passes)
 106
 107   def get_pass_number(self, pass_name, default=None):
 108     """Return the number of the pass indicated by PASS_NAME.
 109
 110     PASS_NAME should be a string containing the name or number of a
 111     pass.  If a number, it should be in the range 1 <= value <=
 112     self.num_passes.  Return an integer in the same range.  If
 113     PASS_NAME is the empty string and DEFAULT is specified, return
 114     DEFAULT.  Raise InvalidPassError if PASS_NAME cannot be converted
 115     into a valid pass number."""
 116
 117     if not pass_name and default is not None:
 118       assert 1 <= default <= self.num_passes
 119       return default
 120
 121     try:
 122       # Does pass_name look like an integer?
 123       pass_number = int(pass_name)
 124       if not 1 <= pass_number <= self.num_passes:
 125         raise InvalidPassError(
 126             'illegal value (%d) for pass number.  Must be 1 through %d or\n'
 127             'the name of a known pass.'
 128             % (pass_number,self.num_passes,))
 129       return pass_number
 130     except ValueError:
 131       # Is pass_name the name of one of the passes?
 132       for (i, the_pass) in enumerate(self.passes):
 133         if the_pass.name == pass_name:
 134           return i + 1
 135       raise InvalidPassError('Unknown pass name (%r).' % (pass_name,))
 136
 137   def run(self, run_options):
 138     """Run the specified passes, one after another.
 139
 140     RUN_OPTIONS will be passed to the Passes' run() methods.
 141     RUN_OPTIONS.start_pass is the number of the first pass that should
 142     be run.  RUN_OPTIONS.end_pass is the number of the last pass that
 143     should be run.  It must be that 1 <= RUN_OPTIONS.start_pass <=
 144     RUN_OPTIONS.end_pass <= self.num_passes."""
 145
 146     # Convert start_pass and end_pass into the indices of the passes
 147     # to execute, using the Python index range convention (i.e., first
 148     # pass executed and first pass *after* the ones that should be
 149     # executed).
 150     index_start = run_options.start_pass - 1
 151     index_end = run_options.end_pass
 152
 153     # Inform the artifact manager when artifacts are created and used:
 154     for (i, the_pass) in enumerate(self.passes):
 155       the_pass.register_artifacts()
 156       # Each pass creates a new version of the statistics file:
 157       artifact_manager.register_temp_file(
 158           config.STATISTICS_FILE % (i + 1,), the_pass
 159           )
 160       if i != 0:
 161         # Each pass subsequent to the first reads the statistics file
 162         # from the preceding pass:
 163         artifact_manager.register_temp_file_needed(
 164             config.STATISTICS_FILE % (i + 1 - 1,), the_pass
 165             )
 166
 167     # Tell the artifact manager about passes that are being skipped this run:
 168     for the_pass in self.passes[0:index_start]:
 169       artifact_manager.pass_skipped(the_pass)
 170
 171     start_time = time.time()
 172     for i in range(index_start, index_end):
 173       the_pass = self.passes[i]
 174       logger.quiet('----- pass %d (%s) -----' % (i + 1, the_pass.name,))
 175       artifact_manager.pass_started(the_pass)
 176
 177       if i == 0:
 178         stats_keeper = StatsKeeper()
 179       else:
 180         stats_keeper = read_stats_keeper(
 181             artifact_manager.get_temp_file(
 182                 config.STATISTICS_FILE % (i + 1 - 1,)
 183                 )
 184             )
 185
 186       the_pass.run(run_options, stats_keeper)
 187       end_time = time.time()
 188       stats_keeper.log_duration_for_pass(
 189           end_time - start_time, i + 1, the_pass.name
 190           )
 191       logger.normal(stats_keeper.single_pass_timing(i + 1))
 192       stats_keeper.archive(
 193           artifact_manager.get_temp_file(config.STATISTICS_FILE % (i + 1,))
 194           )
 195       start_time = end_time
 196       Ctx().clean()
 197       # Allow the artifact manager to clean up artifacts that are no
 198       # longer needed:
 199       artifact_manager.pass_done(the_pass, Ctx().skip_cleanup)
 200
 201       check_for_garbage()
 202
 203     # Tell the artifact manager about passes that are being deferred:
 204     for the_pass in self.passes[index_end:]:
 205       artifact_manager.pass_deferred(the_pass)
 206
 207     logger.quiet(stats_keeper)
 208     logger.normal(stats_keeper.timings())
 209
 210     # Consistency check:
 211     artifact_manager.check_clean()
 212
 213   def help_passes(self):
 214     """Output (to sys.stdout) the indices and names of available passes."""
 215
 216     print 'PASSES:'
 217     for (i, the_pass) in enumerate(self.passes):
 218       print '%5d : %s' % (i + 1, the_pass.name,)
 219
 220