cvs2svn_lib/pass_manager.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains tools to manage the passes of a conversion."""
  18
  19
  20 import time
  21 import gc
  22
  23 from cvs2svn_lib import config
  24 from cvs2svn_lib.common import FatalError
  25 from cvs2svn_lib.context import Ctx
  26 from cvs2svn_lib.log import logger
  27 from cvs2svn_lib.stats_keeper import StatsKeeper
  28 from cvs2svn_lib.stats_keeper import read_stats_keeper
  29 from cvs2svn_lib.artifact_manager import artifact_manager
  30
  31
  32 class InvalidPassError(FatalError):
  33   def __init__(self, msg):
  34     FatalError.__init__(
  35         self, msg + '\nUse --help-passes for more information.')
  36
  37
  38 def check_for_garbage():
  39   # We've turned off the garbage collector because we shouldn't
  40   # need it (we don't create circular dependencies) and because it
  41   # is therefore a waste of time.  So here we check for any
  42   # unreachable objects and generate a debug-level warning if any
  43   # occur:
  44   gc.set_debug(gc.DEBUG_SAVEALL)
  45   gc_count = gc.collect()
  46   if gc_count:
  47     if logger.is_on(logger.DEBUG):
  48       logger.debug(
  49           'INTERNAL: %d unreachable object(s) were garbage collected:'
  50           % (gc_count,)
  51           )
  52       for g in gc.garbage:
  53         logger.debug('    %s' % (g,))
  54     del gc.garbage[:]
  55
  56
  57 class Pass(object):
  58   """Base class for one step of the conversion."""
  59
  60   def __init__(self):
  61     # By default, use the pass object's class name as the pass name:
  62     self.name = self.__class__.__name__
  63
  64   def register_artifacts(self):
  65     """Register artifacts (created and needed) in artifact_manager."""
  66
  67     raise NotImplementedError()
  68
  69   def _register_temp_file(self, basename):
  70     """Helper method; for brevity only."""
  71
  72     artifact_manager.register_temp_file(basename, self)
  73
  74   def _register_temp_file_needed(self, basename):
  75     """Helper method; for brevity only."""
  76
  77     artifact_manager.register_temp_file_needed(basename, self)
  78
  79   def run(self, run_options, stats_keeper):
  80     """Carry out this step of the conversion.
  81
  82     RUN_OPTIONS is an instance of RunOptions.  STATS_KEEPER is an
  83     instance of StatsKeeper."""
  84
  85     raise NotImplementedError()
  86
  87
  88 class PassManager:
  89   """Manage a list of passes that can be executed separately or all at once.
  90
  91   Passes are numbered starting with 1."""
  92
  93   def __init__(self, passes):
  94     """Construct a PassManager with the specified PASSES.
  95
  96     Internally, passes are numbered starting with 1.  So PASSES[0] is
  97     considered to be pass number 1."""
  98
  99     self.passes = passes
 100     self.num_passes = len(self.passes)
 101
 102   def get_pass_number(self, pass_name, default=None):
 103     """Return the number of the pass indicated by PASS_NAME.
 104
 105     PASS_NAME should be a string containing the name or number of a
 106     pass.  If a number, it should be in the range 1 <= value <=
 107     self.num_passes.  Return an integer in the same range.  If
 108     PASS_NAME is the empty string and DEFAULT is specified, return
 109     DEFAULT.  Raise InvalidPassError if PASS_NAME cannot be converted
 110     into a valid pass number."""
 111
 112     if not pass_name and default is not None:
 113       assert 1 <= default <= self.num_passes
 114       return default
 115
 116     try:
 117       # Does pass_name look like an integer?
 118       pass_number = int(pass_name)
 119       if not 1 <= pass_number <= self.num_passes:
 120         raise InvalidPassError(
 121             'illegal value (%d) for pass number.  Must be 1 through %d or\n'
 122             'the name of a known pass.'
 123             % (pass_number,self.num_passes,))
 124       return pass_number
 125     except ValueError:
 126       # Is pass_name the name of one of the passes?
 127       for (i, the_pass) in enumerate(self.passes):
 128         if the_pass.name == pass_name:
 129           return i + 1
 130       raise InvalidPassError('Unknown pass name (%r).' % (pass_name,))
 131
 132   def run(self, run_options):
 133     """Run the specified passes, one after another.
 134
 135     RUN_OPTIONS will be passed to the Passes' run() methods.
 136     RUN_OPTIONS.start_pass is the number of the first pass that should
 137     be run.  RUN_OPTIONS.end_pass is the number of the last pass that
 138     should be run.  It must be that 1 <= RUN_OPTIONS.start_pass <=
 139     RUN_OPTIONS.end_pass <= self.num_passes."""
 140
 141     # Convert start_pass and end_pass into the indices of the passes
 142     # to execute, using the Python index range convention (i.e., first
 143     # pass executed and first pass *after* the ones that should be
 144     # executed).
 145     index_start = run_options.start_pass - 1
 146     index_end = run_options.end_pass
 147
 148     # Inform the artifact manager when artifacts are created and used:
 149     for (i, the_pass) in enumerate(self.passes):
 150       the_pass.register_artifacts()
 151       # Each pass creates a new version of the statistics file:
 152       artifact_manager.register_temp_file(
 153           config.STATISTICS_FILE % (i + 1,), the_pass
 154           )
 155       if i != 0:
 156         # Each pass subsequent to the first reads the statistics file
 157         # from the preceding pass:
 158         artifact_manager.register_temp_file_needed(
 159             config.STATISTICS_FILE % (i + 1 - 1,), the_pass
 160             )
 161
 162     # Tell the artifact manager about passes that are being skipped this run:
 163     for the_pass in self.passes[0:index_start]:
 164       artifact_manager.pass_skipped(the_pass)
 165
 166     start_time = time.time()
 167     for i in range(index_start, index_end):
 168       the_pass = self.passes[i]
 169       logger.quiet('----- pass %d (%s) -----' % (i + 1, the_pass.name,))
 170       artifact_manager.pass_started(the_pass)
 171
 172       if i == 0:
 173         stats_keeper = StatsKeeper()
 174       else:
 175         stats_keeper = read_stats_keeper(
 176             artifact_manager.get_temp_file(
 177                 config.STATISTICS_FILE % (i + 1 - 1,)
 178                 )
 179             )
 180
 181       the_pass.run(run_options, stats_keeper)
 182       end_time = time.time()
 183       stats_keeper.log_duration_for_pass(
 184           end_time - start_time, i + 1, the_pass.name
 185           )
 186       logger.normal(stats_keeper.single_pass_timing(i + 1))
 187       stats_keeper.archive(
 188           artifact_manager.get_temp_file(config.STATISTICS_FILE % (i + 1,))
 189           )
 190       start_time = end_time
 191       Ctx().clean()
 192       # Allow the artifact manager to clean up artifacts that are no
 193       # longer needed:
 194       artifact_manager.pass_done(the_pass, Ctx().skip_cleanup)
 195
 196       check_for_garbage()
 197
 198     # Tell the artifact manager about passes that are being deferred:
 199     for the_pass in self.passes[index_end:]:
 200       artifact_manager.pass_deferred(the_pass)
 201
 202     logger.quiet(stats_keeper)
 203     logger.normal(stats_keeper.timings())
 204
 205     # Consistency check:
 206     artifact_manager.check_clean()
 207
 208   def help_passes(self):
 209     """Output (to sys.stdout) the indices and names of available passes."""
 210
 211     print 'PASSES:'
 212     for (i, the_pass) in enumerate(self.passes):
 213       print '%5d : %s' % (i + 1, the_pass.name,)
 214
 215