cvs2svn_lib/repository_walker.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Walk through a CVS project, generating CVSPaths."""
  18
  19
  20 import os
  21 import stat
  22
  23 from cvs2svn_lib.common import path_join
  24 from cvs2svn_lib.common import FatalError
  25 from cvs2svn_lib.common import warning_prefix
  26 from cvs2svn_lib.common import IllegalSVNPathError
  27 from cvs2svn_lib.log import logger
  28 from cvs2svn_lib.context import Ctx
  29 from cvs2svn_lib.project import FileInAndOutOfAtticException
  30 from cvs2svn_lib.cvs_path import CVSDirectory
  31 from cvs2svn_lib.cvs_path import CVSFile
  32
  33
  34 class _RepositoryWalker(object):
  35   def __init__(self, file_key_generator, error_handler):
  36     self.file_key_generator = file_key_generator
  37     self.error_handler = error_handler
  38
  39   def _get_cvs_file(
  40         self, parent_directory, basename,
  41         file_in_attic=False, leave_in_attic=False,
  42         ):
  43     """Return a CVSFile describing the file with name BASENAME.
  44
  45     PARENT_DIRECTORY is the CVSDirectory instance describing the
  46     directory that physically holds this file in the filesystem.
  47     BASENAME must be the base name of a *,v file within
  48     PARENT_DIRECTORY.
  49
  50     FILE_IN_ATTIC is a boolean telling whether the specified file is
  51     in an Attic subdirectory.  If FILE_IN_ATTIC is True, then:
  52
  53     - If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in
  54       the filename.
  55
  56     - Otherwise, raise FileInAndOutOfAtticException if a file with the
  57       same filename appears outside of Attic.
  58
  59     The CVSFile is assigned a new unique id.  All of the CVSFile
  60     information is filled in except mode (which can only be determined
  61     by parsing the file).
  62
  63     Raise FatalError if the resulting filename would not be legal in
  64     SVN."""
  65
  66     filename = os.path.join(parent_directory.rcs_path, basename)
  67     try:
  68       Ctx().output_option.verify_filename_legal(basename[:-2])
  69     except IllegalSVNPathError, e:
  70       raise FatalError(
  71           'File %r would result in an illegal SVN filename: %s'
  72           % (filename, e,)
  73           )
  74
  75     if file_in_attic and not leave_in_attic:
  76       in_attic = True
  77       logical_parent_directory = parent_directory.parent_directory
  78
  79       # If this file also exists outside of the attic, it's a fatal
  80       # error:
  81       non_attic_filename = os.path.join(
  82           logical_parent_directory.rcs_path, basename,
  83           )
  84       if os.path.exists(non_attic_filename):
  85         raise FileInAndOutOfAtticException(non_attic_filename, filename)
  86     else:
  87       in_attic = False
  88       logical_parent_directory = parent_directory
  89
  90     file_stat = os.stat(filename)
  91
  92     # The size of the file in bytes:
  93     file_size = file_stat.st_size
  94
  95     # Whether or not the executable bit is set:
  96     file_executable = bool(file_stat.st_mode & stat.S_IXUSR)
  97
  98     # mode is not known, so we temporarily set it to None.
  99     return CVSFile(
 100         self.file_key_generator.gen_id(),
 101         parent_directory.project, logical_parent_directory, basename[:-2],
 102         in_attic, file_executable, file_size, None, None
 103         )
 104
 105   def _get_attic_file(self, parent_directory, basename):
 106     """Return a CVSFile object for the Attic file at BASENAME.
 107
 108     PARENT_DIRECTORY is the CVSDirectory that physically contains the
 109     file on the filesystem (i.e., the Attic directory).  It is not
 110     necessarily the parent_directory of the CVSFile that will be
 111     returned.
 112
 113     Return CVSFile, whose parent directory is usually
 114     PARENT_DIRECTORY.parent_directory, but might be PARENT_DIRECTORY
 115     iff CVSFile will remain in the Attic directory."""
 116
 117     try:
 118       return self._get_cvs_file(
 119           parent_directory, basename, file_in_attic=True,
 120           )
 121     except FileInAndOutOfAtticException, e:
 122       if Ctx().retain_conflicting_attic_files:
 123         logger.warn(
 124             "%s: %s;\n"
 125             "   storing the latter into 'Attic' subdirectory.\n"
 126             % (warning_prefix, e)
 127             )
 128       else:
 129         self.error_handler(str(e))
 130
 131       # Either way, return a CVSFile object so that the rest of the
 132       # file processing can proceed:
 133       return self._get_cvs_file(
 134           parent_directory, basename, file_in_attic=True, leave_in_attic=True,
 135           )
 136
 137   def _generate_attic_cvs_files(self, cvs_directory):
 138     """Generate CVSFiles for the files in Attic directory CVS_DIRECTORY.
 139
 140     Also yield CVS_DIRECTORY if any files are being retained in the
 141     Attic.
 142
 143     Silently ignore subdirectories named '.svn', but emit a warning if
 144     any other directories are found within the Attic directory."""
 145
 146     retained_attic_files = []
 147
 148     fnames = os.listdir(cvs_directory.rcs_path)
 149     fnames.sort()
 150     for fname in fnames:
 151       pathname = os.path.join(cvs_directory.rcs_path, fname)
 152       if os.path.isdir(pathname):
 153         if fname == '.svn':
 154           logger.debug(
 155               "Directory %s found within Attic; ignoring" % (pathname,)
 156               )
 157         else:
 158           logger.warn(
 159               "Directory %s found within Attic; ignoring" % (pathname,)
 160               )
 161       elif fname.endswith(',v'):
 162         cvs_file = self._get_attic_file(cvs_directory, fname)
 163         if cvs_file.parent_directory == cvs_directory:
 164           # This file will be retained in the Attic directory.
 165           retained_attic_files.append(cvs_file)
 166         else:
 167           # This is a normal Attic file, which is treated as if it
 168           # were located one directory up:
 169           yield cvs_file
 170
 171     if retained_attic_files:
 172       # There was at least one file in the attic that will be retained
 173       # in the attic.  First include the Attic directory itself in the
 174       # output, then the retained attic files:
 175       yield cvs_directory
 176       for cvs_file in retained_attic_files:
 177         yield cvs_file
 178
 179   def generate_cvs_paths(self, cvs_directory, exclude_paths):
 180     """Generate the CVSPaths under non-Attic directory CVS_DIRECTORY.
 181
 182     Yield CVSDirectory and CVSFile instances as they are found.
 183     Process directories recursively, including Attic directories.
 184     Also look for conflicts between the filenames that will result
 185     from files, attic files, and subdirectories.
 186
 187     Silently ignore subdirectories named '.svn', as these don't make
 188     much sense in a real conversion, but they are present in our test
 189     suite."""
 190
 191     yield cvs_directory
 192
 193     # Map {cvs_file.rcs_basename : cvs_file.rcs_path} for files
 194     # directly in cvs_directory:
 195     rcsfiles = {}
 196
 197     attic_dir = None
 198
 199     # Non-Attic subdirectories of cvs_directory (to be recursed into):
 200     dirs = []
 201
 202     fnames = os.listdir(cvs_directory.rcs_path)
 203     fnames.sort()
 204     for fname in fnames:
 205       pathname = os.path.join(cvs_directory.rcs_path, fname)
 206       path_in_repository = path_join(cvs_directory.get_cvs_path(), fname)
 207       if path_in_repository in exclude_paths:
 208         logger.normal(
 209             "Excluding file from conversion: %s" % (path_in_repository,)
 210             )
 211         pass
 212       elif os.path.isdir(pathname):
 213         if fname == 'Attic':
 214           attic_dir = fname
 215         elif fname == '.svn':
 216           logger.debug("Directory %s ignored" % (pathname,))
 217         else:
 218           dirs.append(fname)
 219       elif fname.endswith(',v'):
 220         cvs_file = self._get_cvs_file(cvs_directory, fname)
 221         rcsfiles[cvs_file.rcs_basename] = cvs_file.rcs_path
 222         yield cvs_file
 223       else:
 224         # Silently ignore other files:
 225         pass
 226
 227     # Map {cvs_file.rcs_basename : cvs_file.rcs_path} for files in an
 228     # Attic directory within cvs_directory:
 229     attic_rcsfiles = {}
 230
 231     if attic_dir is not None:
 232       attic_directory = CVSDirectory(
 233           self.file_key_generator.gen_id(),
 234           cvs_directory.project, cvs_directory, 'Attic',
 235           )
 236
 237       for cvs_path in self._generate_attic_cvs_files(attic_directory):
 238         if isinstance(cvs_path, CVSFile) \
 239                and cvs_path.parent_directory == cvs_directory:
 240           attic_rcsfiles[cvs_path.rcs_basename] = cvs_path.rcs_path
 241
 242         yield cvs_path
 243
 244       alldirs = dirs + [attic_dir]
 245     else:
 246       alldirs = dirs
 247
 248     # Check for conflicts between directory names and the filenames
 249     # that will result from the rcs files (both in this directory and
 250     # in attic).  (We recurse into the subdirectories nevertheless, to
 251     # try to detect more problems.)
 252     for fname in alldirs:
 253       for rcsfile_list in [rcsfiles, attic_rcsfiles]:
 254         if fname in rcsfile_list:
 255           self.error_handler(
 256               'Directory name conflicts with filename.  Please remove or '
 257               'rename one\n'
 258               'of the following:\n'
 259               '    "%s"\n'
 260               '    "%s"' % (
 261                   os.path.join(cvs_directory.rcs_path, fname),
 262                   rcsfile_list[fname],
 263                   )
 264               )
 265
 266     # Now recurse into the other subdirectories:
 267     for fname in dirs:
 268       dirname = os.path.join(cvs_directory.rcs_path, fname)
 269
 270       # Verify that the directory name does not contain any illegal
 271       # characters:
 272       try:
 273         Ctx().output_option.verify_filename_legal(fname)
 274       except IllegalSVNPathError, e:
 275         raise FatalError(
 276             'Directory %r would result in an illegal SVN path name: %s'
 277             % (dirname, e,)
 278             )
 279
 280       sub_directory = CVSDirectory(
 281           self.file_key_generator.gen_id(),
 282           cvs_directory.project, cvs_directory, fname,
 283           )
 284
 285       for cvs_path in self.generate_cvs_paths(sub_directory, exclude_paths):
 286         yield cvs_path
 287
 288
 289 def walk_repository(project, file_key_generator, error_handler):
 290   """Generate CVSDirectories and CVSFiles within PROJECT.
 291
 292   Use FILE_KEY_GENERATOR to generate the IDs used for files.  If there
 293   is a fatal error, register it by calling ERROR_HANDLER with a string
 294   argument describing the problem.  (The error will be logged but
 295   processing will continue through the end of the pass.)  Also:
 296
 297   * Set PROJECT.root_cvs_directory_id.
 298
 299   * Handle files in the Attic by generating CVSFile instances with the
 300     _in_attic member set.
 301
 302   * Check for naming conflicts that will result from files in and out
 303     of the Attic.  If Ctx().retain_conflicting_attic_files is set, fix
 304     the conflicts by leaving the Attic file in the attic.  Otherwise,
 305     register a fatal error.
 306
 307   * Check for naming conflicts between files (in or out of the Attic)
 308     and directories.
 309
 310   * Check for filenames that contain characters not allowed by
 311     Subversion.
 312
 313   """
 314
 315   root_cvs_directory = CVSDirectory(
 316       file_key_generator.gen_id(), project, None, ''
 317       )
 318   project.root_cvs_directory_id = root_cvs_directory.id
 319   repository_walker = _RepositoryWalker(file_key_generator, error_handler)
 320   for cvs_path in repository_walker.generate_cvs_paths(
 321         root_cvs_directory, project.exclude_paths
 322         ):
 323     yield cvs_path
 324
 325