Encapsulate pass_manager.convert() in new PassManager class.

[cvs2svn.git] / cvs2svn

#!/usr/bin/env python
# (Be in -*- python -*- mode.)
#
# cvs2svn: ...
#
# ====================================================================
# Copyright (c) 2000-2004 CollabNet.  All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.  The terms
# are also available at http://subversion.tigris.org/license-1.html.
# If newer versions of this license are posted there, you may use a
# newer version instead, at your option.
#
# This software consists of voluntary contributions made by many
# individuals.  For exact contribution history, see the revision
# history and logs, available at http://cvs2svn.tigris.org/.
# ====================================================================

VERSION = 'r' + "$LastChangedRevision$"[22:-2]

from __future__ import generators

import os
import sys
import sha
import re
import time
import fileinput
import getopt
import marshal
import errno
import types

try:
  # Try to get access to a bunch of encodings for use with --encoding.
  # See http://cjkpython.i18n.org/ for details.
  import iconv_codec
except ImportError:
  pass

import cvs2svn_rcsparse

from cvs2svn_lib.boolean import *

from cvs2svn_lib import config

from cvs2svn_lib import common
from cvs2svn_lib.common import \
        warning_prefix, \
        error_prefix, \
        FatalException, \
        FatalError

from cvs2svn_lib.log import Log

from cvs2svn_lib.process import \
        run_command, \
        CommandFailedException, \
        check_command_runs

from cvs2svn_lib import database
from cvs2svn_lib.context import Ctx
from cvs2svn_lib.artifact_manager import artifact_manager
from cvs2svn_lib.stats_keeper import StatsKeeper
from cvs2svn_lib import key_generator
from cvs2svn_lib import cvs_revision
from cvs2svn_lib import cvs_repository
from cvs2svn_lib import property_setters
from cvs2svn_lib.svn_revision_range import SVNRevisionRange
from cvs2svn_lib.tags_database import TagsDatabase
from cvs2svn_lib.cvs_revision_database import CVSRevisionDatabase
from cvs2svn_lib.openings_closings import \
        OpeningsClosingsMap, \
        SymbolingsLogger
from cvs2svn_lib.fill_source import FillSource
from cvs2svn_lib.last_symbolic_name_database import LastSymbolicNameDatabase
from cvs2svn_lib.symbol_database import SymbolDatabase
from cvs2svn_lib.project import Project
from cvs2svn_lib import collect_data
from cvs2svn_lib.symbolings_reader import SymbolingsReader
from cvs2svn_lib.svn_commit_item import SVNCommitItem
from cvs2svn_lib.svn_commit import SVNCommit
from cvs2svn_lib.svn_repository_mirror import \
        SVNRepositoryMirror, \
        SVNRepositoryMirrorDelegate
from cvs2svn_lib.dumpfile_delegate import DumpfileDelegate
from cvs2svn_lib.repository_delegate import RepositoryDelegate
from cvs2svn_lib.stdout_delegate import StdoutDelegate
from cvs2svn_lib.persistence_manager import PersistenceManager
from cvs2svn_lib.pass_manager import PassManager


# Make sure this Python is recent enough.
if sys.hexversion < 0x02020000:
  sys.stderr.write("'%s: Python 2.2 or higher required, "
                   "see www.python.org.\n" % error_prefix)
  sys.exit(1)


DIGEST_END_IDX = 9 + (sha.digestsize * 2)


ctrl_characters_regexp = re.compile('[\\\x00-\\\x1f\\\x7f]')

def verify_filename_legal(filename):
  """Verify that FILENAME does not include any control characters.  If
  it does, raise a FatalError."""

  m = ctrl_characters_regexp.search(filename)
  if m:
    raise FatalError(
        "Character %r in filename %r is not supported by subversion."
        % (m.group(), filename,))


def sort_file(infilename, outfilename):
  """Sort file INFILENAME, storing the results to OUTFILENAME."""

  # GNU sort will sort our dates differently (incorrectly!) if our
  # LC_ALL is anything but 'C', so if LC_ALL is set, temporarily set
  # it to 'C'
  lc_all_tmp = os.environ.get('LC_ALL', None)
  os.environ['LC_ALL'] = 'C'
  try:
    # The -T option to sort has a nice side effect.  The Win32 sort is
    # case insensitive and cannot be used, and since it does not
    # understand the -T option and dies if we try to use it, there is
    # no risk that we use that sort by accident.
    run_command('sort -T %s %s > %s'
                % (Ctx().tmpdir, infilename, outfilename))
  finally:
    if lc_all_tmp is None:
      del os.environ['LC_ALL']
    else:
      os.environ['LC_ALL'] = lc_all_tmp


class CVSCommit:
  """Each instance of this class contains a number of CVS Revisions
  that correspond to one or more Subversion Commits.  After all CVS
  Revisions are added to the grouping, calling process_revisions will
  generate a Subversion Commit (or Commits) for the set of CVS
  Revisions in the grouping."""

  def __init__(self, digest, author, log):
    self.digest = digest
    self.author = author
    self.log = log

    # Symbolic names for which the last source revision has already
    # been seen and for which the CVSRevisionAggregator has already
    # generated a fill SVNCommit.  See self.process_revisions().
    self.done_symbols = [ ]

    self.files = { }
    # Lists of CVSRevisions
    self.changes = [ ]
    self.deletes = [ ]

    # Start out with a t_min higher than any incoming time T, and a
    # t_max lower than any incoming T.  This way the first T will
    # push t_min down to T, and t_max up to T, naturally (without any
    # special-casing), and successive times will then ratchet them
    # outward as appropriate.
    self.t_min = 1L<<32
    self.t_max = 0

    # This will be set to the SVNCommit that occurs in self._commit.
    self.motivating_commit = None

    # This is a list of all non-primary commits motivated by the main
    # commit.  We gather these so that we can set their dates to the
    # same date as the primary commit.
    self.secondary_commits = [ ]

    # State for handling default branches.
    #
    # Here is a tempting, but ultimately nugatory, bit of logic, which
    # I share with you so you may appreciate the less attractive, but
    # refreshingly non-nugatory, logic which follows it:
    #
    # If some of the commits in this txn happened on a non-trunk
    # default branch, then those files will have to be copied into
    # trunk manually after being changed on the branch (because the
    # RCS "default branch" appears as head, i.e., trunk, in practice).
    # As long as those copies don't overwrite any trunk paths that
    # were also changed in this commit, then we can do the copies in
    # the same revision, because they won't cover changes that don't
    # appear anywhere/anywhen else.  However, if some of the trunk dst
    # paths *did* change in this commit, then immediately copying the
    # branch changes would lose those trunk mods forever.  So in this
    # case, we need to do at least that copy in its own revision.  And
    # for simplicity's sake, if we're creating the new revision for
    # even one file, then we just do all such copies together in the
    # new revision.
    #
    # Doesn't that sound nice?
    #
    # Unfortunately, Subversion doesn't support copies with sources
    # in the current txn.  All copies must be based in committed
    # revisions.  Therefore, we generate the above-described new
    # revision unconditionally.
    #
    # This is a list of c_revs, and a c_rev is appended for each
    # default branch commit that will need to be copied to trunk (or
    # deleted from trunk) in some generated revision following the
    # "regular" revision.
    self.default_branch_cvs_revisions = [ ]

  def __cmp__(self, other):
    # Commits should be sorted by t_max.  If both self and other have
    # the same t_max, break the tie using t_min, and lastly, digest.
    # If all those are equal, then compare based on ids, to ensure
    # that no two instances compare equal.
    return (cmp(self.t_max, other.t_max) or cmp(self.t_min, other.t_min)
            or cmp(self.digest, other.digest) or cmp(id(self), id(other)))

  def __hash__(self):
    return id(self)

  def has_file(self, fname):
    return self.files.has_key(fname)

  def revisions(self):
    return self.changes + self.deletes

  def opens_symbolic_name(self, name):
    """Return True if any CVSRevision in this commit is on a tag or a
    branch or is the origin of a tag or branch."""

    for c_rev in self.revisions():
      if c_rev.opens_symbolic_name(name):
        return True
    return False

  def add_revision(self, c_rev):
    # Record the time range of this commit.
    #
    # ### ISSUE: It's possible, though unlikely, that the time range
    # of a commit could get gradually expanded to be arbitrarily
    # longer than COMMIT_THRESHOLD.  I'm not sure this is a huge
    # problem, and anyway deciding where to break it up would be a
    # judgement call.  For now, we just print a warning in commit() if
    # this happens.
    if c_rev.timestamp < self.t_min:
      self.t_min = c_rev.timestamp
    if c_rev.timestamp > self.t_max:
      self.t_max = c_rev.timestamp

    if c_rev.op == common.OP_DELETE:
      self.deletes.append(c_rev)
    else:
      # OP_CHANGE or OP_ADD
      self.changes.append(c_rev)

    self.files[c_rev.fname] = 1

  def _pre_commit(self):
    """Generates any SVNCommits that must exist before the main commit."""

    # There may be multiple c_revs in this commit that would cause
    # branch B to be filled, but we only want to fill B once.  On the
    # other hand, there might be multiple branches committed on in
    # this commit.  Whatever the case, we should count exactly one
    # commit per branch, because we only fill a branch once per
    # CVSCommit.  This list tracks which branches we've already
    # counted.
    accounted_for_sym_names = [ ]

    def fill_needed(c_rev, pm):
      """Return 1 if this is the first commit on a new branch (for
      this file) and we need to fill the branch; else return 0
      (meaning that some other file's first commit on the branch has
      already done the fill for us).

      If C_REV.op is OP_ADD, only return 1 if the branch that this
      commit is on has no last filled revision.

      PM is a PersistenceManager to query."""

      # Different '.' counts indicate that c_rev is now on a different
      # line of development (and may need a fill)
      if c_rev.rev.count('.') != c_rev.prev_rev.rev.count('.'):
        svn_revnum = pm.get_svn_revnum(c_rev.prev_rev.unique_key())
        # It should be the case that when we have a file F that
        # is added on branch B (thus, F on trunk is in state
        # 'dead'), we generate an SVNCommit to fill B iff the branch
        # has never been filled before.
        #
        # If this c_rev.op == OP_ADD, *and* the branch has never
        # been filled before, then fill it now.  Otherwise, no need to
        # fill it.
        if c_rev.op == common.OP_ADD:
          if pm.last_filled.get(c_rev.branch_name, None) is None:
            return 1
        elif c_rev.op == common.OP_CHANGE:
          if svn_revnum > pm.last_filled.get(c_rev.branch_name, 0):
            return 1
        elif c_rev.op == common.OP_DELETE:
          if pm.last_filled.get(c_rev.branch_name, None) is None:
            return 1
      return 0

    for c_rev in self.changes + self.deletes:
      # If a commit is on a branch, we must ensure that the branch
      # path being committed exists (in HEAD of the Subversion
      # repository).  If it doesn't exist, we will need to fill the
      # branch.  After the fill, the path on which we're committing
      # will exist.
      if c_rev.branch_name \
          and c_rev.branch_name not in accounted_for_sym_names \
          and c_rev.branch_name not in self.done_symbols \
          and fill_needed(c_rev, Ctx()._persistence_manager):
        svn_commit = SVNCommit("pre-commit symbolic name '%s'"
                               % c_rev.branch_name)
        svn_commit.set_symbolic_name(c_rev.branch_name)
        self.secondary_commits.append(svn_commit)
        accounted_for_sym_names.append(c_rev.branch_name)

  def _commit(self):
    """Generates the primary SVNCommit that corresponds to this
    CVSCommit."""

    # Generate an SVNCommit unconditionally.  Even if the only change
    # in this CVSCommit is a deletion of an already-deleted file (that
    # is, a CVS revision in state 'dead' whose predecessor was also in
    # state 'dead'), the conversion will still generate a Subversion
    # revision containing the log message for the second dead
    # revision, because we don't want to lose that information.
    svn_commit = SVNCommit("commit")
    self.motivating_commit = svn_commit

    for c_rev in self.changes:
      svn_commit.add_revision(c_rev)
      # Only make a change if we need to:
      if c_rev.rev == "1.1.1.1" and not c_rev.deltatext_exists:
        # When 1.1.1.1 has an empty deltatext, the explanation is
        # almost always that we're looking at an imported file whose
        # 1.1 and 1.1.1.1 are identical.  On such imports, CVS creates
        # an RCS file where 1.1 has the content, and 1.1.1.1 has an
        # empty deltatext, i.e, the same content as 1.1.  There's no
        # reason to reflect this non-change in the repository, so we
        # want to do nothing in this case.  (If we were really
        # paranoid, we could make sure 1.1's log message is the
        # CVS-generated "Initial revision\n", but I think the
        # conditions above are strict enough.)
        pass
      else:
        if c_rev.is_default_branch_revision():
          self.default_branch_cvs_revisions.append(c_rev)

    for c_rev in self.deletes:
      # When a file is added on a branch, CVS not only adds the file
      # on the branch, but generates a trunk revision (typically
      # 1.1) for that file in state 'dead'.  We only want to add
      # this revision if the log message is not the standard cvs
      # fabricated log message.
      if c_rev.prev_rev is None:
        # c_rev.branches may be empty if the originating branch
        # has been excluded.
        if not c_rev.branches:
          continue
        cvs_generated_msg = ('file %s was initially added on branch %s.\n'
                             % (c_rev.filename(),
                                c_rev.branches[0]))
        author, log_msg = \
            Ctx()._persistence_manager.svn_commit_metadata[c_rev.digest]
        if log_msg == cvs_generated_msg:
          continue

      svn_commit.add_revision(c_rev)
      if c_rev.is_default_branch_revision():
        self.default_branch_cvs_revisions.append(c_rev)

    # There is a slight chance that we didn't actually register any
    # CVSRevisions with our SVNCommit (see loop over self.deletes
    # above), so if we have no CVSRevisions, we don't flush the
    # svn_commit to disk and roll back our revnum.
    if len(svn_commit.cvs_revs) > 0:
      svn_commit.set_date(self.t_max)
      svn_commit.flush()
    else:
      # We will not be flushing this SVNCommit, so rollback the
      # SVNCommit revision counter.
      SVNCommit.revnum -= 1

    if not Ctx().trunk_only:
      for c_rev in self.revisions():
        Ctx()._symbolings_logger.log_revision(c_rev, svn_commit.revnum)

  def _post_commit(self):
    """Generates any SVNCommits that we can perform now that _commit
    has happened.  That is, handle non-trunk default branches.
    Sometimes an RCS file has a non-trunk default branch, so a commit
    on that default branch would be visible in a default CVS checkout
    of HEAD.  If we don't copy that commit over to Subversion's trunk,
    then there will be no Subversion tree which corresponds to that
    CVS checkout.  Of course, in order to copy the path over, we may
    first need to delete the existing trunk there."""

    # Only generate a commit if we have default branch revs
    if len(self.default_branch_cvs_revisions):
      # Generate an SVNCommit for all of our default branch c_revs.
      svn_commit = SVNCommit("post-commit default branch(es)")
      svn_commit.set_motivating_revnum(self.motivating_commit.revnum)
      for c_rev in self.default_branch_cvs_revisions:
        svn_commit.add_revision(c_rev)
        Ctx()._symbolings_logger.log_default_branch_closing(
            c_rev, svn_commit.revnum)
      self.secondary_commits.append(svn_commit)

  def process_revisions(self, done_symbols):
    """Process all the CVSRevisions that this instance has, creating
    one or more SVNCommits in the process.  Generate fill SVNCommits
    only for symbols not in DONE_SYMBOLS (avoids unnecessary
    fills).

    Return the primary SVNCommit that corresponds to this CVSCommit.
    The returned SVNCommit is the commit that motivated any other
    SVNCommits generated in this CVSCommit."""

    self.done_symbols = done_symbols
    seconds = self.t_max - self.t_min + 1

    Log().write(Log.VERBOSE, '-' * 60)
    Log().write(Log.VERBOSE, 'CVS Revision grouping:')
    if seconds == 1:
      Log().write(Log.VERBOSE, '  Start time: %s (duration: 1 second)'
                  % time.ctime(self.t_max))
    else:
      Log().write(Log.VERBOSE, '  Start time: %s' % time.ctime(self.t_min))
      Log().write(Log.VERBOSE, '  End time:   %s (duration: %d seconds)'
                  % (time.ctime(self.t_max), seconds))

    if seconds > config.COMMIT_THRESHOLD + 1:
      Log().write(Log.WARN, '%s: grouping spans more than %d seconds'
                  % (warning_prefix, config.COMMIT_THRESHOLD))

    if Ctx().trunk_only: # Only do the primary commit if we're trunk-only
      self._commit()
      return self.motivating_commit

    self._pre_commit()
    self._commit()
    self._post_commit()

    for svn_commit in self.secondary_commits:
      svn_commit.set_date(self.motivating_commit.get_date())
      svn_commit.flush()

    return self.motivating_commit


class CVSRevisionAggregator:
  """This class groups CVSRevisions into CVSCommits that represent
  at least one SVNCommit."""

  def __init__(self):
    self.metadata_db = database.Database(
        artifact_manager.get_temp_file(config.METADATA_DB),
        database.DB_OPEN_READ)
    if not Ctx().trunk_only:
      self.last_revs_db = database.Database(
          artifact_manager.get_temp_file(config.SYMBOL_LAST_CVS_REVS_DB),
          database.DB_OPEN_READ)

    # A map { key : CVSCommit } of CVS commits currently being
    # accumulated.  If the CVSCommit is still open to further
    # CVSRevisions, then key is CVSRevision.digest.  If not (because
    # an inbound commit wanted to affect a file that was already
    # within the CVSCommit), then key is CVSRevision.digest plus some
    # number of appended '-'.
    self.cvs_commits = {}

    # List of ready commits.
    self.ready_queue = [ ]

    # A map { symbol : None } of symbolic names for which the last
    # source CVSRevision has already been processed but which haven't
    # been closed yet.
    self.pending_symbols = {}

    # A list of closed symbols.  That is, we've already encountered
    # the last CVSRevision that is a source for that symbol, the final
    # fill for this symbol has been done, and we never need to fill it
    # again.
    self.done_symbols = [ ]

    # This variable holds the most recently created primary svn_commit
    # object.  CVSRevisionAggregator maintains this variable merely
    # for its date, so that it can set dates for the SVNCommits
    # created in self._attempt_to_commit_symbols().
    self.latest_primary_svn_commit = None

    Ctx()._symbolings_logger = SymbolingsLogger()
    Ctx()._persistence_manager = PersistenceManager(database.DB_OPEN_NEW)
    Ctx()._default_branches_db = database.SDatabase(
        artifact_manager.get_temp_file(config.DEFAULT_BRANCHES_DB),
        database.DB_OPEN_READ)

  def _extract_ready_commits(self, timestamp=None):
    """Extract any active commits that expire by TIMESTAMP from
    self.cvs_commits and append them to self.ready_queue.  If
    TIMESTAMP is not specified, then extract all commits."""

    for digest_key, cvs_commit in self.cvs_commits.items():
      if timestamp is None \
             or cvs_commit.t_max + config.COMMIT_THRESHOLD < timestamp:
        self.ready_queue.append(cvs_commit)
        del self.cvs_commits[digest_key]

  def _commit_ready_commits(self, timestamp=None):
    """Sort the commits from self.ready_queue by time, then process
    them in order.  If TIMESTAMP is specified, only process commits
    that have timestamp previous to TIMESTAMP."""

    self.ready_queue.sort()
    while self.ready_queue and \
              (timestamp is None or self.ready_queue[0].t_max < timestamp):
      cvs_commit = self.ready_queue.pop(0)
      self.latest_primary_svn_commit = \
          cvs_commit.process_revisions(self.done_symbols)
      self._attempt_to_commit_symbols()

  def process_revision(self, c_rev):
    # Each time we read a new line, scan the accumulating commits to
    # see if any are ready for processing.
    self._extract_ready_commits(c_rev.timestamp)

    for digest_key, cvs_commit in self.cvs_commits.items():
      # If the inbound commit is on the same file as a pending commit,
      # close the pending commit to further changes.  Don't flush it though,
      # as there may be other pending commits dated before this one.
      # ### ISSUE: the has_file() check below is not optimal.
      # It does fix the dataloss bug where revisions would get lost
      # if checked in too quickly, but it can also break apart the
      # commits.  The correct fix would require tracking the dependencies
      # between change sets and committing them in proper order.
      if cvs_commit.has_file(c_rev.fname):
        unused_id = digest_key + '-'
        # Find a string that does is not already a key in
        # the self.cvs_commits dict
        while self.cvs_commits.has_key(unused_id):
          unused_id += '-'
        self.cvs_commits[unused_id] = cvs_commit
        del self.cvs_commits[digest_key]

    # Add this item into the set of still-available commits.
    if self.cvs_commits.has_key(c_rev.digest):
      cvs_commit = self.cvs_commits[c_rev.digest]
    else:
      author, log = self.metadata_db[c_rev.digest]
      cvs_commit = CVSCommit(c_rev.digest, author, log)
      self.cvs_commits[c_rev.digest] = cvs_commit
    cvs_commit.add_revision(c_rev)

    # Any elements in self.ready_queue at this point need to be
    # processed, because this latest rev couldn't possibly be part of
    # any of them.
    self._commit_ready_commits()

    self._add_pending_symbols(c_rev)

  def flush(self):
    """Commit anything left in self.cvs_commits.  Then inform the
    SymbolingsLogger that all commits are done."""

    self._extract_ready_commits()
    self._commit_ready_commits()

    if not Ctx().trunk_only:
      Ctx()._symbolings_logger.close()

  def _add_pending_symbols(self, c_rev):
    """Add to self.pending_symbols any symbols from C_REV for which
    C_REV is the last CVSRevision.

    If we're not doing a trunk-only conversion, get the symbolic names
    that this c_rev is the last *source* CVSRevision for and add them
    to those left over from previous passes through the aggregator."""

    if not Ctx().trunk_only:
      for sym in self.last_revs_db.get(c_rev.unique_key(), []):
        self.pending_symbols[sym] = None

  def _attempt_to_commit_symbols(self):
    """Generate one SVNCommit for each symbol in self.pending_symbols
    that doesn't have an opening CVSRevision in either self.ready_queue
    or self.cvs_commits.values()."""

    # Make a list of all symbols from self.pending_symbols that do not
    # have *source* CVSRevisions in the pending commit queues
    # (self.cvs_commits or self.ready_queue):
    closeable_symbols = []
    pending_commits = self.cvs_commits.values() + self.ready_queue
    for sym in self.pending_symbols:
      for cvs_commit in pending_commits:
        if cvs_commit.opens_symbolic_name(sym):
          break
      else:
        closeable_symbols.append(sym)

    # Sort the closeable symbols so that we will always process the
    # symbols in the same order, regardless of the order in which the
    # dict hashing algorithm hands them back to us.  We do this so
    # that our tests will get the same results on all platforms.
    closeable_symbols.sort()
    for sym in closeable_symbols:
      svn_commit = SVNCommit("closing tag/branch '%s'" % sym)
      svn_commit.set_symbolic_name(sym)
      svn_commit.set_date(self.latest_primary_svn_commit.get_date())
      svn_commit.flush()
      self.done_symbols.append(sym)
      del self.pending_symbols[sym]


class Pass:
  """Base class for one step of the conversion."""

  def __init__(self):
    # By default, use the pass object's class name as the pass name:
    self.name = self.__class__.__name__

  def register_artifacts(self):
    """Register artifacts (created and needed) in artifact_manager."""

    raise NotImplementedError

  def _register_temp_file(self, basename):
    """Helper method; for brevity only."""

    artifact_manager.register_temp_file(basename, self)

  def _register_temp_file_needed(self, basename):
    """Helper method; for brevity only."""

    artifact_manager.register_temp_file_needed(basename, self)

  def run(self):
    """Carry out this step of the conversion."""

    raise NotImplementedError


class CollectRevsPass(Pass):
  """This pass was formerly known as pass1."""

  def register_artifacts(self):
    self._register_temp_file(config.TAGS_LIST)
    self._register_temp_file(config.BRANCHES_LIST)
    self._register_temp_file(config.REVS_DATAFILE)
    self._register_temp_file(config.RESYNC_DATAFILE)
    self._register_temp_file(config.DEFAULT_BRANCHES_DB)
    self._register_temp_file(config.METADATA_DB)

  def run(self):
    OS_SEP_PLUS_ATTIC = os.sep + 'Attic'
    Log().write(Log.QUIET, "Examining all CVS ',v' files...")
    cd = collect_data.CollectData()

    def visit_file(baton, dirname, files):
      cd = baton
      for fname in files:
        verify_filename_legal(fname)
        if not fname.endswith(',v'):
          continue
        cd.found_valid_file = 1
        pathname = os.path.join(dirname, fname)
        if dirname.endswith(OS_SEP_PLUS_ATTIC):
          # drop the 'Attic' portion from the pathname for the canonical name.
          fdc = collect_data.FileDataCollector(
              cd, os.path.join(dirname[:-6], fname), pathname)
        else:
          # If this file also exists in the attic, it's a fatal error
          attic_path = os.path.join(dirname, 'Attic', fname)
          if os.path.exists(attic_path):
            err = "%s: A CVS repository cannot contain both %s and %s" \
                  % (error_prefix, pathname, attic_path)
            sys.stderr.write(err + '\n')
            cd.fatal_errors.append(err)
          fdc = collect_data.FileDataCollector(cd, pathname, pathname)
        Log().write(Log.NORMAL, pathname)
        try:
          cvs2svn_rcsparse.parse(open(pathname, 'rb'), fdc)
        except (cvs2svn_rcsparse.common.RCSParseError, ValueError,
                RuntimeError):
          err = "%s: '%s' is not a valid ,v file" \
                % (error_prefix, pathname)
          sys.stderr.write(err + '\n')
          cd.fatal_errors.append(err)
        except:
          Log().write(Log.WARN,
                      "Exception occurred while parsing %s" % pathname)
          raise

    os.path.walk(Ctx().project.project_cvs_repos_path, visit_file, cd)
    Log().write(Log.VERBOSE, 'Processed', cd.num_files, 'files')

    cd.write_symbol_db()

    if len(cd.fatal_errors) > 0:
      raise FatalException("Pass 1 complete.\n"
                           + "=" * 75 + "\n"
                           + "Error summary:\n"
                           + "\n".join(cd.fatal_errors) + "\n"
                           + "Exited due to fatal error(s).\n")

    if cd.found_valid_file is None:
      raise FatalException(
          "\n"
          "No RCS files found in your CVS Repository!\n"
          "Are you absolutely certain you are pointing cvs2svn\n"
          "at a CVS repository?\n"
          "\n"
          "Exited due to fatal error(s).\n")

    StatsKeeper().reset_c_rev_info()
    StatsKeeper().archive()
    Log().write(Log.QUIET, "Done")


class ResyncRevsPass(Pass):
  """Clean up the revision information.

  This pass was formerly known as pass2."""

  def register_artifacts(self):
    self._register_temp_file(config.TAGS_DB)
    self._register_temp_file(config.CLEAN_REVS_DATAFILE)
    self._register_temp_file(config.TWEAKED_TIMESTAMPS_DB)
    self._register_temp_file_needed(config.TAGS_LIST)
    self._register_temp_file_needed(config.BRANCHES_LIST)
    self._register_temp_file_needed(config.REVS_DATAFILE)
    self._register_temp_file_needed(config.RESYNC_DATAFILE)

  def run(self):
    symbol_db = SymbolDatabase()
    symbol_db.read()

    # Convert the list of regexps to a list of strings
    excludes = symbol_db.find_excluded_symbols(Ctx().excludes)

    error_detected = 0

    Log().write(Log.QUIET, "Checking for blocked exclusions...")
    blocked_excludes = symbol_db.find_blocked_excludes(excludes)
    if blocked_excludes:
      for branch, blockers in blocked_excludes.items():
        sys.stderr.write(error_prefix + ": The branch '%s' cannot be "
                         "excluded because the following symbols depend "
                         "on it:\n" % (branch))
        for blocker in blockers:
          sys.stderr.write("    '%s'\n" % (blocker))
      sys.stderr.write("\n")
      error_detected = 1

    Log().write(Log.QUIET, "Checking for forced tags with commits...")
    invalid_forced_tags = [ ]
    for forced_tag in Ctx().forced_tags:
      if excludes.has_key(forced_tag):
        continue
      if symbol_db.branch_has_commit(forced_tag):
        invalid_forced_tags.append(forced_tag)
    if invalid_forced_tags:
      sys.stderr.write(error_prefix + ": The following branches cannot be "
                       "forced to be tags because they have commits:\n")
      for tag in invalid_forced_tags:
        sys.stderr.write("    '%s'\n" % (tag))
      sys.stderr.write("\n")
      error_detected = 1

    Log().write(Log.QUIET, "Checking for tag/branch mismatches...")
    mismatches = symbol_db.find_mismatches(excludes)
    def is_not_forced(mismatch):
      name = mismatch[0]
      return not (name in Ctx().forced_tags or name in Ctx().forced_branches)
    mismatches = filter(is_not_forced, mismatches)
    if mismatches:
      sys.stderr.write(error_prefix + ": The following symbols are tags "
                       "in some files and branches in others.\nUse "
                       "--force-tag, --force-branch and/or --exclude to "
                       "resolve the symbols.\n")
      for name, tag_count, branch_count, commit_count in mismatches:
        sys.stderr.write("    '%s' is a tag in %d files, a branch in "
                         "%d files and has commits in %d files.\n"
                         % (name, tag_count, branch_count, commit_count))
      error_detected = 1

    # Bail out now if we found errors
    if error_detected:
      sys.exit(1)

    # Create the tags database
    tags_db = TagsDatabase(database.DB_OPEN_NEW)
    for tag in symbol_db.tags:
      if tag not in Ctx().forced_branches:
        tags_db.add(tag)
    for tag in Ctx().forced_tags:
      tags_db.add(tag)

    Log().write(Log.QUIET, "Re-synchronizing CVS revision timestamps...")

    # We may have recorded some changes in revisions' timestamp.  We need to
    # scan for any other files which may have had the same log message and
    # occurred at "the same time" and change their timestamps, too.

    # read the resync data file
    def read_resync(fname):
      """Read the .resync file into memory."""

      ### note that we assume that we can hold the entire resync file in
      ### memory. really large repositories with whacky timestamps could
      ### bust this assumption. should that ever happen, then it is possible
      ### to split the resync file into pieces and make multiple passes,
      ### using each piece.

      #
      # A digest maps to a sequence of lists which specify a lower and upper
      # time bound for matching up the commit.  We keep a sequence of these
      # because a number of checkins with the same log message (e.g. an empty
      # log message) could need to be remapped.  We also make them a list
      # because we will dynamically expand the lower/upper bound as we find
      # commits that fall into a particular msg and time range.
      #
      # resync == digest -> [[old_time_lower, old_time_upper, new_time], ...]
      #
      resync = { }

      for line in fileinput.FileInput(fname):
        t1 = int(line[:8], 16)
        digest = line[9:DIGEST_END_IDX]
        t2 = int(line[DIGEST_END_IDX+1:], 16)
        t1_l = t1 - config.COMMIT_THRESHOLD/2
        t1_u = t1 + config.COMMIT_THRESHOLD/2
        resync.setdefault(digest, []).append([t1_l, t1_u, t2])

      # For each digest, sort the resync items in it in increasing order,
      # based on the lower time bound.
      for val in resync.values():
        val.sort()

      return resync

    resync = read_resync(
        artifact_manager.get_temp_file(config.RESYNC_DATAFILE))

    output = open(artifact_manager.get_temp_file(config.CLEAN_REVS_DATAFILE),
                  'w')

    tweaked_timestamps_db = database.Database(
        artifact_manager.get_temp_file(config.TWEAKED_TIMESTAMPS_DB),
        database.DB_OPEN_NEW)

    # process the revisions file, looking for items to clean up
    for line in fileinput.FileInput(
            artifact_manager.get_temp_file(config.REVS_DATAFILE)):
      c_rev = cvs_revision.parse_cvs_revision(Ctx(), line[:-1])

      # Skip this entire revision if it's on an excluded branch
      if excludes.has_key(c_rev.branch_name):
        continue

      new_prev_ts = None
      if c_rev.prev_rev is not None:
        new_prev_ts = tweaked_timestamps_db.get(
            c_rev.prev_rev.unique_key(), None)
      if new_prev_ts:
        c_rev.prev_timestamp = new_prev_ts

      new_next_ts = None
      if c_rev.next_rev is not None:
        new_next_ts = tweaked_timestamps_db.get(
          c_rev.next_rev.unique_key(), None)
      if new_next_ts:
        c_rev.next_timestamp = new_next_ts

      # Remove all references to excluded tags and branches
      def not_excluded(symbol, excludes=excludes):
        return not excludes.has_key(symbol)
      c_rev.branches = filter(not_excluded, c_rev.branches)
      c_rev.tags = filter(not_excluded, c_rev.tags)

      # Convert all branches that are forced to be tags
      for forced_tag in Ctx().forced_tags:
        if forced_tag in c_rev.branches:
          c_rev.branches.remove(forced_tag)
          c_rev.tags.append(forced_tag)

      # Convert all tags that are forced to be branches
      for forced_branch in Ctx().forced_branches:
        if forced_branch in c_rev.tags:
          c_rev.tags.remove(forced_branch)
          c_rev.branches.append(forced_branch)

      # see if this is "near" any of the resync records we
      # have recorded for this digest [of the log message].
      for record in resync.get(c_rev.digest, []):
        if record[2] == c_rev.timestamp:
          # This means that either c_rev is the same revision that
          # caused the resync record to exist, or c_rev is a different
          # CVS revision that happens to have the same timestamp.  In
          # either case, we don't have to do anything, so we...
          continue

        if record[0] <= c_rev.timestamp <= record[1]:
          # bingo!  We probably want to remap the time on this c_rev,
          # unless the remapping would be useless because the new time
          # would fall outside the COMMIT_THRESHOLD window for this
          # commit group.
          new_timestamp = record[2]
          # If the new timestamp is earlier than that of our previous revision
          if new_timestamp < c_rev.prev_timestamp:
            desc = ("%s: Attempt to set timestamp of revision %s on file %s"
                    + " to time %s, which is before previous the time of"
                    + " revision %s (%s):")
            Log().write(Log.WARN, desc % (warning_prefix, c_rev.rev,
                                          c_rev.cvs_path, new_timestamp,
                                          c_rev.prev_rev.rev,
                                          c_rev.prev_timestamp))
            # If resyncing our rev to c_rev.prev_timestamp + 1 will place
            # the timestamp of c_rev within COMMIT_THRESHOLD of the
            # attempted resync time, then sync back to c_rev.prev_timestamp
            # + 1...
            if ((c_rev.prev_timestamp + 1) - new_timestamp) \
                   < config.COMMIT_THRESHOLD:
              new_timestamp = c_rev.prev_timestamp + 1
              Log().write(Log.WARN, "%s: Time set to %s" % (warning_prefix,
                                                            new_timestamp))
            else:
              Log().write(Log.WARN, "%s: Timestamp left untouched" %
                          warning_prefix)
              continue

          # If the new timestamp is later than that of our next revision
          elif c_rev.next_timestamp and new_timestamp > c_rev.next_timestamp:
            desc = ("%s: Attempt to set timestamp of revision %s on file %s"
                    + " to time %s, which is after time of next"
                    + " revision %s (%s):")
            Log().write(Log.WARN, desc % (warning_prefix, c_rev.rev,
                                          c_rev.cvs_path, new_timestamp,
                                          c_rev.next_rev.rev,
                                          c_rev.next_timestamp))
            # If resyncing our rev to c_rev.next_timestamp - 1 will place
            # the timestamp of c_rev within COMMIT_THRESHOLD of the
            # attempted resync time, then sync forward to c_rev.next_timestamp
            # - 1...
            if (new_timestamp - (c_rev.next_timestamp - 1)) \
                   < config.COMMIT_THRESHOLD:
              new_timestamp = c_rev.next_timestamp - 1
              Log().write(Log.WARN, "%s: Time set to %s" % (warning_prefix,
                                                            new_timestamp))
            else:
              Log().write(Log.WARN, "%s: Timestamp left untouched" %
                          warning_prefix)
              continue

          # Fix for Issue #71: Avoid resyncing two consecutive revisions
          # to the same timestamp.
          elif (new_timestamp == c_rev.prev_timestamp
                or new_timestamp == c_rev.next_timestamp):
            continue

          # adjust the time range. we want the COMMIT_THRESHOLD from the
          # bounds of the earlier/latest commit in this group.
          record[0] = min(record[0],
                          c_rev.timestamp - config.COMMIT_THRESHOLD/2)
          record[1] = max(record[1],
                          c_rev.timestamp + config.COMMIT_THRESHOLD/2)

          msg = "PASS2 RESYNC: '%s' (%s): old time='%s' delta=%ds" \
                % (c_rev.cvs_path, c_rev.rev, time.ctime(c_rev.timestamp),
                   new_timestamp - c_rev.timestamp)
          Log().write(Log.VERBOSE, msg)

          c_rev.timestamp = new_timestamp
          tweaked_timestamps_db[c_rev.unique_key()] = new_timestamp

          # stop looking for hits
          break

      output.write(str(c_rev) + "\n")
    Log().write(Log.QUIET, "Done")


class SortRevsPass(Pass):
  """This pass was formerly known as pass3."""

  def register_artifacts(self):
    self._register_temp_file(config.SORTED_REVS_DATAFILE)
    self._register_temp_file_needed(config.CLEAN_REVS_DATAFILE)

  def run(self):
    Log().write(Log.QUIET, "Sorting CVS revisions...")
    sort_file(artifact_manager.get_temp_file(config.CLEAN_REVS_DATAFILE),
              artifact_manager.get_temp_file(config.SORTED_REVS_DATAFILE))
    Log().write(Log.QUIET, "Done")


class CreateDatabasesPass(Pass):
  """This pass was formerly known as pass4."""

  def register_artifacts(self):
    if not Ctx().trunk_only:
      self._register_temp_file(config.SYMBOL_LAST_CVS_REVS_DB)
    self._register_temp_file(config.CVS_REVS_DB)
    self._register_temp_file_needed(config.SORTED_REVS_DATAFILE)

  def run(self):
    """Iterate through sorted revs, storing them in a database.
    If we're not doing a trunk-only conversion, generate the
    LastSymbolicNameDatabase, which contains the last CVSRevision
    that is a source for each tag or branch."""

    Log().write(Log.QUIET,
        "Copying CVS revision data from flat file to database...")
    cvs_revs_db = CVSRevisionDatabase(database.DB_OPEN_NEW)
    if not Ctx().trunk_only:
      Log().write(Log.QUIET,
          "Finding last CVS revisions for all symbolic names...")
      last_sym_name_db = LastSymbolicNameDatabase()
    else:
      # This is to avoid testing Ctx().trunk_only every time around the loop
      class DummyLSNDB:
        def noop(*args): pass
        log_revision = noop
        create_database = noop
      last_sym_name_db = DummyLSNDB()

    for line in fileinput.FileInput(
            artifact_manager.get_temp_file(config.SORTED_REVS_DATAFILE)):
      c_rev = cvs_revision.parse_cvs_revision(Ctx(), line[:-1])
      cvs_revs_db.log_revision(c_rev)
      last_sym_name_db.log_revision(c_rev)
      StatsKeeper().record_c_rev(c_rev)

    StatsKeeper().set_stats_reflect_exclude(True)

    last_sym_name_db.create_database()
    StatsKeeper().archive()
    Log().write(Log.QUIET, "Done")


class AggregateRevsPass(Pass):
  """Generate the SVNCommit <-> CVSRevision mapping databases.
  CVSCommit._commit also calls SymbolingsLogger to register
  CVSRevisions that represent an opening or closing for a path on a
  branch or tag.  See SymbolingsLogger for more details.

  This pass was formerly known as pass5."""

  def register_artifacts(self):
    self._register_temp_file(config.SYMBOL_OPENINGS_CLOSINGS)
    self._register_temp_file(config.SYMBOL_CLOSINGS_TMP)
    self._register_temp_file(config.SVN_REVNUMS_TO_CVS_REVS)
    self._register_temp_file(config.CVS_REVS_TO_SVN_REVNUMS)
    if not Ctx().trunk_only:
      self._register_temp_file_needed(config.SYMBOL_LAST_CVS_REVS_DB)
    self._register_temp_file_needed(config.CVS_REVS_DB)
    self._register_temp_file_needed(config.TAGS_DB)
    self._register_temp_file_needed(config.DEFAULT_BRANCHES_DB)
    self._register_temp_file_needed(config.METADATA_DB)
    self._register_temp_file_needed(config.SORTED_REVS_DATAFILE)

  def run(self):
    Log().write(Log.QUIET, "Mapping CVS revisions to Subversion commits...")

    aggregator = CVSRevisionAggregator()
    for line in fileinput.FileInput(
            artifact_manager.get_temp_file(config.SORTED_REVS_DATAFILE)):
      c_rev = cvs_revision.parse_cvs_revision(Ctx(), line[:-1])
      if not (Ctx().trunk_only and c_rev.branch_name is not None):
        aggregator.process_revision(c_rev)
    aggregator.flush()

    StatsKeeper().set_svn_rev_count(SVNCommit.revnum - 1)
    StatsKeeper().archive()
    Log().write(Log.QUIET, "Done")


class SortSymbolsPass(Pass):
  """This pass was formerly known as pass6."""

  def register_artifacts(self):
    if not Ctx().trunk_only:
      self._register_temp_file(config.SYMBOL_OPENINGS_CLOSINGS_SORTED)
    self._register_temp_file_needed(config.SYMBOL_OPENINGS_CLOSINGS)

  def run(self):
    Log().write(Log.QUIET, "Sorting symbolic name source revisions...")

    if not Ctx().trunk_only:
      sort_file(
          artifact_manager.get_temp_file(config.SYMBOL_OPENINGS_CLOSINGS),
          artifact_manager.get_temp_file(
              config.SYMBOL_OPENINGS_CLOSINGS_SORTED))
    Log().write(Log.QUIET, "Done")


class IndexSymbolsPass(Pass):
  """This pass was formerly known as pass7."""

  def register_artifacts(self):
    if not Ctx().trunk_only:
      self._register_temp_file(config.SYMBOL_OFFSETS_DB)
      self._register_temp_file_needed(config.SYMBOL_OPENINGS_CLOSINGS_SORTED)

  def run(self):
    Log().write(Log.QUIET, "Determining offsets for all symbolic names...")

    def generate_offsets_for_symbolings():
      """This function iterates through all the lines in
      SYMBOL_OPENINGS_CLOSINGS_SORTED, writing out a file mapping
      SYMBOLIC_NAME to the file offset in SYMBOL_OPENINGS_CLOSINGS_SORTED
      where SYMBOLIC_NAME is first encountered.  This will allow us to
      seek to the various offsets in the file and sequentially read only
      the openings and closings that we need."""

      ###PERF This is a fine example of a db that can be in-memory and
      #just flushed to disk when we're done.  Later, it can just be sucked
      #back into memory.
      offsets_db = database.Database(
          artifact_manager.get_temp_file(config.SYMBOL_OFFSETS_DB),
          database.DB_OPEN_NEW)

      file = open(
          artifact_manager.get_temp_file(
              config.SYMBOL_OPENINGS_CLOSINGS_SORTED),
          'r')
      old_sym = ""
      while 1:
        fpos = file.tell()
        line = file.readline()
        if not line:
          break
        sym, svn_revnum, cvs_rev_key = line.split(" ", 2)
        if sym != old_sym:
          Log().write(Log.VERBOSE, " ", sym)
          old_sym = sym
          offsets_db[sym] = fpos

    if not Ctx().trunk_only:
      generate_offsets_for_symbolings()
    Log().write(Log.QUIET, "Done.")


class OutputPass(Pass):
  """This pass was formerly known as pass8."""

  def register_artifacts(self):
    self._register_temp_file(config.SVN_MIRROR_REVISIONS_DB)
    self._register_temp_file(config.SVN_MIRROR_NODES_DB)
    self._register_temp_file_needed(config.CVS_REVS_DB)
    self._register_temp_file_needed(config.TAGS_DB)
    self._register_temp_file_needed(config.METADATA_DB)
    self._register_temp_file_needed(config.SVN_REVNUMS_TO_CVS_REVS)
    self._register_temp_file_needed(config.CVS_REVS_TO_SVN_REVNUMS)
    if not Ctx().trunk_only:
      self._register_temp_file_needed(config.SYMBOL_OPENINGS_CLOSINGS_SORTED)
      self._register_temp_file_needed(config.SYMBOL_OFFSETS_DB)

  def run(self):
    svncounter = 2 # Repository initialization is 1.
    repos = SVNRepositoryMirror()
    persistence_manager = PersistenceManager(database.DB_OPEN_READ)

    if Ctx().target:
      if not Ctx().dry_run:
        repos.add_delegate(RepositoryDelegate())
      Log().write(Log.QUIET, "Starting Subversion Repository.")
    else:
      if not Ctx().dry_run:
        repos.add_delegate(DumpfileDelegate())
      Log().write(Log.QUIET, "Starting Subversion Dumpfile.")

    repos.add_delegate(StdoutDelegate(StatsKeeper().svn_rev_count()))

    while 1:
      svn_commit = persistence_manager.get_svn_commit(svncounter)
      if not svn_commit:
        break
      repos.commit(svn_commit)
      svncounter += 1

    repos.finish()


pass_manager = PassManager([
  CollectRevsPass(),
  ResyncRevsPass(),
  SortRevsPass(),
  CreateDatabasesPass(),
  AggregateRevsPass(),
  SortSymbolsPass(),
  IndexSymbolsPass(),
  OutputPass(),
  ])


def help_passes():
  print 'PASSES:'
  for i in range(len(pass_manager.passes)):
    print '%5d : %s' % (i + 1, pass_manager.passes[i].name,)


def normalize_ttb_path(opt, path):
  """Normalize a path to be used for --trunk, --tags, or --branches.

  1. Strip leading, trailing, and duplicated '/'.
  2. Verify that the path is not empty.

  Return the normalized path.

  If the path is invalid, write an error message and exit."""

  norm_path = common.path_join(*path.split('/'))
  if not norm_path:
    raise FatalError("cannot pass an empty path to %s." % (opt,))
  return norm_path


def usage():
  print 'USAGE: %s [-v] [-s svn-repos-path] [-p pass] cvs-repos-path' \
        % os.path.basename(sys.argv[0])
  print '  --help, -h           print this usage message and exit with success'
  print '  --help-passes        list the available passes and their numbers'
  print '  --version            print the version number'
  print '  -q                   quiet'
  print '  -v                   verbose'
  print '  -s PATH              path for SVN repos'
  print '  -p START[:END]       start at pass START, end at pass END of %d' \
        % len(pass_manager.passes)
  print '                       If only START is given, run only pass START'
  print '                       (implicitly enables --skip-cleanup)'
  print '  --existing-svnrepos  load into existing SVN repository'
  print '  --dumpfile=PATH      name of intermediate svn dumpfile'
  print '  --tmpdir=PATH        directory to use for tmp data (default to cwd)'
  print '  --profile            profile with \'hotshot\' (into file cvs2svn.hotshot)'
  print '  --dry-run            do not create a repository or a dumpfile;'
  print '                       just print what would happen.'
  print '  --use-cvs            use CVS instead of RCS \'co\' to extract data'
  print '                       (only use this if having problems with RCS)'
  print '  --svnadmin=PATH      path to the svnadmin program'
  print '  --trunk-only         convert only trunk commits, not tags nor branches'
  print '  --trunk=PATH         path for trunk (default: %s)'    \
        % Ctx().trunk_base
  print '  --branches=PATH      path for branches (default: %s)' \
        % Ctx().branches_base
  print '  --tags=PATH          path for tags (default: %s)'     \
        % Ctx().tags_base
  print '  --no-prune           don\'t prune empty directories'
  print '  --dump-only          just produce a dumpfile, don\'t commit to a repos'
  print '  --encoding=ENC       encoding of paths and log messages in CVS repos'
  print '                       Multiple of these options may be passed, where they'
  print '                       will be treated as an ordered list of encodings to'
  print '                       attempt (with "ascii" as a hardcoded last resort)'
  print '  --force-branch=NAME  force NAME to be a branch'
  print '  --force-tag=NAME     force NAME to be a tag'
  print '  --exclude=REGEXP     exclude branches and tags matching REGEXP'
  print '  --symbol-transform=P:S transform symbol names from P to S where P and S'
  print '                       use Python regexp and reference syntax respectively'
  print '  --username=NAME      username for cvs2svn-synthesized commits'
  print '  --skip-cleanup       prevent the deletion of intermediate files'
  print '  --bdb-txn-nosync     pass --bdb-txn-nosync to "svnadmin create"'
  print '  --fs-type=TYPE       pass --fs-type=TYPE to "svnadmin create"'
  print '  --cvs-revnums        record CVS revision numbers as file properties'
  print '  --auto-props=FILE    set file properties from the auto-props section'
  print '                       of a file in svn config format'
  print '  --auto-props-ignore-case Ignore case when matching auto-props patterns'
  print '  --mime-types=FILE    specify an apache-style mime.types file for'
  print '                       setting svn:mime-type'
  print '  --eol-from-mime-type set svn:eol-style from mime type if known'
  print '  --no-default-eol     don\'t set svn:eol-style to \'native\' for'
  print '                       non-binary files with undetermined mime types'
  print '  --keywords-off       don\'t set svn:keywords on any files (by default,'
  print '                       cvs2svn sets svn:keywords on non-binary files to'
  print '                       "%s")' % config.SVN_KEYWORDS_VALUE


def main():
  # Convenience var, so we don't have to keep instantiating this Borg.
  ctx = Ctx()

  profiling = None
  start_pass = 1
  end_pass = len(pass_manager.passes)

  try:
    opts, args = getopt.getopt(sys.argv[1:], 'p:s:qvh',
                               [ "help", "help-passes", "create", "trunk=",
                                 "username=", "existing-svnrepos",
                                 "branches=", "tags=", "encoding=",
                                 "force-branch=", "force-tag=", "exclude=",
                                 "use-cvs", "mime-types=",
                                 "auto-props=", "auto-props-ignore-case",
                                 "eol-from-mime-type", "no-default-eol",
                                 "trunk-only", "no-prune", "dry-run",
                                 "dump-only", "dumpfile=", "tmpdir=",
                                 "svnadmin=", "skip-cleanup", "cvs-revnums",
                                 "bdb-txn-nosync", "fs-type=",
                                 "version", "profile",
                                 "keywords-off", "symbol-transform="])
  except getopt.GetoptError, e:
    sys.stderr.write(error_prefix + ': ' + str(e) + '\n\n')
    usage()
    sys.exit(1)

  for opt, value in opts:
    if opt == '--version':
        print '%s version %s' % (os.path.basename(sys.argv[0]), VERSION)
        sys.exit(0)
    elif opt == '-p':
      # Don't cleanup if we're doing incrementals.
      ctx.skip_cleanup = 1
      if value.find(':') > 0:
        start_pass, end_pass = map(int, value.split(':'))
      else:
        end_pass = start_pass = int(value)
      if start_pass > len(pass_manager.passes) or start_pass < 1:
        raise FatalError(
            'illegal value (%d) for starting pass.  Must be 1 through %d.'
            % (int(start_pass), len(pass_manager.passes),))
      if end_pass < start_pass or end_pass > len(pass_manager.passes):
        raise FatalError(
            'illegal value (%d) for ending pass.  Must be %d through %d.'
            % (int(end_pass), int(start_pass), len(pass_manager.passes),))
    elif (opt == '--help') or (opt == '-h'):
      ctx.print_help = 1
    elif opt == '--help-passes':
      help_passes()
      sys.exit(0)
    elif opt == '-v':
      Log().log_level = Log.VERBOSE
      ctx.verbose = 1
    elif opt == '-q':
      Log().log_level = Log.QUIET
      ctx.quiet = 1
    elif opt == '-s':
      ctx.target = value
    elif opt == '--existing-svnrepos':
      ctx.existing_svnrepos = 1
    elif opt == '--dumpfile':
      ctx.dumpfile = value
    elif opt == '--tmpdir':
      ctx.tmpdir = value
    elif opt == '--use-cvs':
      ctx.use_cvs = 1
    elif opt == '--svnadmin':
      ctx.svnadmin = value
    elif opt == '--trunk-only':
      ctx.trunk_only = 1
    elif opt == '--trunk':
      ctx.trunk_base = normalize_ttb_path(opt, value)
    elif opt == '--branches':
      ctx.branches_base = normalize_ttb_path(opt, value)
    elif opt == '--tags':
      ctx.tags_base = normalize_ttb_path(opt, value)
    elif opt == '--no-prune':
      ctx.prune = None
    elif opt == '--dump-only':
      ctx.dump_only = 1
    elif opt == '--dry-run':
      ctx.dry_run = 1
    elif opt == '--encoding':
      ctx.encoding.insert(-1, value)
    elif opt == '--force-branch':
      ctx.forced_branches.append(value)
    elif opt == '--force-tag':
      ctx.forced_tags.append(value)
    elif opt == '--exclude':
      try:
        ctx.excludes.append(re.compile('^' + value + '$'))
      except re.error, e:
        raise FatalError("'%s' is not a valid regexp." % (value,))
    elif opt == '--mime-types':
      ctx.mime_types_file = value
    elif opt == '--auto-props':
      ctx.auto_props_file = value
    elif opt == '--auto-props-ignore-case':
      ctx.auto_props_ignore_case = True
    elif opt == '--eol-from-mime-type':
      ctx.eol_from_mime_type = 1
    elif opt == '--no-default-eol':
      ctx.no_default_eol = 1
    elif opt == '--keywords-off':
      ctx.keywords_off = 1
    elif opt == '--username':
      ctx.username = value
    elif opt == '--skip-cleanup':
      ctx.skip_cleanup = 1
    elif opt == '--cvs-revnums':
      ctx.svn_property_setters.append(
          property_setters.CVSRevisionNumberSetter())
    elif opt == '--bdb-txn-nosync':
      ctx.bdb_txn_nosync = 1
    elif opt == '--fs-type':
      ctx.fs_type = value
    elif opt == '--create':
      sys.stderr.write(warning_prefix +
          ': The behaviour produced by the --create option is now the '
          'default,\nand passing the option is deprecated.\n')
    elif opt == '--profile':
      profiling = 1
    elif opt == '--symbol-transform':
      [pattern, replacement] = value.split(":")
      try:
        pattern = re.compile(pattern)
      except re.error, e:
        raise FatalError("'%s' is not a valid regexp." % (pattern,))
      ctx.symbol_transforms.append((pattern, replacement,))

  if ctx.print_help:
    usage()
    sys.exit(0)

  # Consistency check for options and arguments.
  if len(args) == 0:
    usage()
    sys.exit(1)

  if len(args) > 1:
    sys.stderr.write(error_prefix +
                     ": must pass only one CVS repository.\n")
    usage()
    sys.exit(1)

  cvsroot = args[0]

  if ctx.use_cvs:
    ctx.cvs_repository = cvs_repository.CVSRepositoryViaCVS(cvsroot)
  else:
    ctx.cvs_repository = cvs_repository.CVSRepositoryViaRCS(cvsroot)

  if (not ctx.target) and (not ctx.dump_only) and (not ctx.dry_run):
    raise FatalError("must pass one of '-s' or '--dump-only'.")

  def not_both(opt1val, opt1name, opt2val, opt2name):
    if opt1val and opt2val:
      raise FatalError("cannot pass both '%s' and '%s'."
                       % (opt1name, opt2name,))

  not_both(ctx.target, '-s',
           ctx.dump_only, '--dump-only')

  not_both(ctx.dump_only, '--dump-only',
           ctx.existing_svnrepos, '--existing-svnrepos')

  not_both(ctx.bdb_txn_nosync, '--bdb-txn-nosync',
           ctx.existing_svnrepos, '--existing-svnrepos')

  not_both(ctx.dump_only, '--dump-only',
           ctx.bdb_txn_nosync, '--bdb-txn-nosync')

  not_both(ctx.quiet, '-q',
           ctx.verbose, '-v')

  not_both(ctx.fs_type, '--fs-type',
           ctx.existing_svnrepos, '--existing-svnrepos')

  if ctx.fs_type and ctx.fs_type != 'bdb' and ctx.bdb_txn_nosync:
    raise FatalError("cannot pass --bdb-txn-nosync with --fs-type=%s."
                     % ctx.fs_type)

  # Create the default project (using ctx.trunk, ctx.branches, and ctx.tags):
  ctx.project = Project(ctx.cvs_repository.cvs_repos_path,
                        ctx.trunk_base, ctx.branches_base, ctx.tags_base)

  if ctx.existing_svnrepos and not os.path.isdir(ctx.target):
    raise FatalError("the svn-repos-path '%s' is not an "
                     "existing directory." % ctx.target)

  if not ctx.dump_only and not ctx.existing_svnrepos \
     and (not ctx.dry_run) and os.path.exists(ctx.target):
    raise FatalError("the svn-repos-path '%s' exists.\n"
                     "Remove it, or pass '--existing-svnrepos'."
                     % ctx.target)

  if ctx.target and not ctx.dry_run:
    # Verify that svnadmin can be executed.  The 'help' subcommand
    # should be harmless.
    try:
      check_command_runs([ctx.svnadmin, 'help'], 'svnadmin')
    except CommandFailedException, e:
      raise FatalError(
          '%s\n'
          'svnadmin could not be executed.  Please ensure that it is\n'
          'installed and/or use the --svnadmin option.' % (e,))

  ctx.svn_property_setters.append(
      property_setters.ExecutablePropertySetter())

  ctx.svn_property_setters.append(
      property_setters.BinaryFileEOLStyleSetter())

  if ctx.mime_types_file:
    ctx.svn_property_setters.append(
        property_setters.MimeMapper(ctx.mime_types_file))

  if ctx.auto_props_file:
    ctx.svn_property_setters.append(
        property_setters.AutoPropsPropertySetter(
            ctx.auto_props_file, ctx.auto_props_ignore_case))

  ctx.svn_property_setters.append(
      property_setters.BinaryFileDefaultMimeTypeSetter())

  if ctx.eol_from_mime_type:
    ctx.svn_property_setters.append(
        property_setters.EOLStyleFromMimeTypeSetter())

  if ctx.no_default_eol:
    ctx.svn_property_setters.append(
        property_setters.DefaultEOLStyleSetter(None))
  else:
    ctx.svn_property_setters.append(
        property_setters.DefaultEOLStyleSetter('native'))

  if not ctx.keywords_off:
    ctx.svn_property_setters.append(
        property_setters.KeywordsPropertySetter(config.SVN_KEYWORDS_VALUE))

  # Make sure the tmp directory exists.  Note that we don't check if
  # it's empty -- we want to be able to use, for example, "." to hold
  # tempfiles.  But if we *did* want check if it were empty, we'd do
  # something like os.stat(ctx.tmpdir)[stat.ST_NLINK], of course :-).
  if not os.path.exists(ctx.tmpdir):
    os.mkdir(ctx.tmpdir)
  elif not os.path.isdir(ctx.tmpdir):
    raise FatalError(
        "cvs2svn tried to use '%s' for temporary files, but that path\n"
        "  exists and is not a directory.  Please make it be a directory,\n"
        "  or specify some other directory for temporary files."
        % (ctx.tmpdir,))

  # But do lock the tmpdir, to avoid process clash.
  try:
    os.mkdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock'))
  except OSError, e:
    if e.errno == errno.EACCES:
      raise FatalError("Permission denied:"
                       + " No write access to directory '%s'." % ctx.tmpdir)
    if e.errno == errno.EEXIST:
      raise FatalError(
          "cvs2svn is using directory '%s' for temporary files, but\n"
          "  subdirectory '%s/cvs2svn.lock' exists, indicating that another\n"
          "  cvs2svn process is currently using '%s' as its temporary\n"
          "  workspace.  If you are certain that is not the case,\n"
          "  then remove the '%s/cvs2svn.lock' subdirectory."
          % (ctx.tmpdir, ctx.tmpdir, ctx.tmpdir, ctx.tmpdir,))
    raise
  try:
    if profiling:
      import hotshot
      prof = hotshot.Profile('cvs2svn.hotshot')
      prof.runcall(pass_manager.run, start_pass, end_pass)
      prof.close()
    else:
      pass_manager.run(start_pass, end_pass)
  finally:
    try: os.rmdir(os.path.join(ctx.tmpdir, 'cvs2svn.lock'))
    except: pass


if __name__ == '__main__':
  try:
    main()
  except FatalException, e:
    sys.stderr.write(str(e))
    sys.exit(1)