cvs2svn_lib/repository_walker.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Walk through a CVS project, generating CVSPaths."""
  18
  19
  20 import os
  21 import stat
  22
  23 from cvs2svn_lib.common import FatalError
  24 from cvs2svn_lib.common import warning_prefix
  25 from cvs2svn_lib.common import IllegalSVNPathError
  26 from cvs2svn_lib.common import verify_svn_filename_legal
  27 from cvs2svn_lib.log import logger
  28 from cvs2svn_lib.context import Ctx
  29 from cvs2svn_lib.project import FileInAndOutOfAtticException
  30 from cvs2svn_lib.cvs_path import CVSDirectory
  31 from cvs2svn_lib.cvs_path import CVSFile
  32
  33
  34 class _RepositoryWalker(object):
  35   def __init__(self, file_key_generator, error_handler):
  36     self.file_key_generator = file_key_generator
  37     self.error_handler = error_handler
  38
  39   def _get_cvs_file(
  40         self, parent_directory, basename,
  41         file_in_attic=False, leave_in_attic=False,
  42         ):
  43     """Return a CVSFile describing the file with name BASENAME.
  44
  45     PARENT_DIRECTORY is the CVSDirectory instance describing the
  46     directory that physically holds this file in the filesystem.
  47     BASENAME must be the base name of a *,v file within
  48     PARENT_DIRECTORY.
  49
  50     FILE_IN_ATTIC is a boolean telling whether the specified file is
  51     in an Attic subdirectory.  If FILE_IN_ATTIC is True, then:
  52
  53     - If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in
  54       the filename.
  55
  56     - Otherwise, raise FileInAndOutOfAtticException if a file with the
  57       same filename appears outside of Attic.
  58
  59     The CVSFile is assigned a new unique id.  All of the CVSFile
  60     information is filled in except mode (which can only be determined
  61     by parsing the file).
  62
  63     Raise FatalError if the resulting filename would not be legal in
  64     SVN."""
  65
  66     filename = os.path.join(parent_directory.filename, basename)
  67     try:
  68       verify_svn_filename_legal(basename[:-2])
  69     except IllegalSVNPathError, e:
  70       raise FatalError(
  71           'File %r would result in an illegal SVN filename: %s'
  72           % (filename, e,)
  73           )
  74
  75     if file_in_attic and not leave_in_attic:
  76       in_attic = True
  77       logical_parent_directory = parent_directory.parent_directory
  78
  79       # If this file also exists outside of the attic, it's a fatal
  80       # error:
  81       non_attic_filename = os.path.join(
  82           logical_parent_directory.filename, basename,
  83           )
  84       if os.path.exists(non_attic_filename):
  85         raise FileInAndOutOfAtticException(non_attic_filename, filename)
  86     else:
  87       in_attic = False
  88       logical_parent_directory = parent_directory
  89
  90     file_stat = os.stat(filename)
  91
  92     # The size of the file in bytes:
  93     file_size = file_stat.st_size
  94
  95     # Whether or not the executable bit is set:
  96     file_executable = bool(file_stat.st_mode & stat.S_IXUSR)
  97
  98     # mode is not known, so we temporarily set it to None.
  99     return CVSFile(
 100         self.file_key_generator.gen_id(),
 101         parent_directory.project, logical_parent_directory, basename[:-2],
 102         in_attic, file_executable, file_size, None, None
 103         )
 104
 105   def _get_attic_file(self, parent_directory, basename):
 106     """Return a CVSFile object for the Attic file at BASENAME.
 107
 108     PARENT_DIRECTORY is the CVSDirectory that physically contains the
 109     file on the filesystem (i.e., the Attic directory).  It is not
 110     necessarily the parent_directory of the CVSFile that will be
 111     returned.
 112
 113     Return CVSFile, whose parent directory is usually
 114     PARENT_DIRECTORY.parent_directory, but might be PARENT_DIRECTORY
 115     iff CVSFile will remain in the Attic directory."""
 116
 117     try:
 118       return self._get_cvs_file(
 119           parent_directory, basename, file_in_attic=True,
 120           )
 121     except FileInAndOutOfAtticException, e:
 122       if Ctx().retain_conflicting_attic_files:
 123         logger.warn(
 124             "%s: %s;\n"
 125             "   storing the latter into 'Attic' subdirectory.\n"
 126             % (warning_prefix, e)
 127             )
 128       else:
 129         self.error_handler(str(e))
 130
 131       # Either way, return a CVSFile object so that the rest of the
 132       # file processing can proceed:
 133       return self._get_cvs_file(
 134           parent_directory, basename, file_in_attic=True, leave_in_attic=True,
 135           )
 136
 137   def _generate_attic_cvs_files(self, cvs_directory):
 138     """Generate CVSFiles for the files in Attic directory CVS_DIRECTORY.
 139
 140     Also yield CVS_DIRECTORY if any files are being retained in the
 141     Attic.
 142
 143     Silently ignore subdirectories named '.svn', but emit a warning if
 144     any other directories are found within the Attic directory."""
 145
 146     retained_attic_files = []
 147
 148     fnames = os.listdir(cvs_directory.filename)
 149     fnames.sort()
 150     for fname in fnames:
 151       pathname = os.path.join(cvs_directory.filename, fname)
 152       if os.path.isdir(pathname):
 153         if fname == '.svn':
 154           logger.debug(
 155               "Directory %s found within Attic; ignoring" % (pathname,)
 156               )
 157         else:
 158           logger.warn(
 159               "Directory %s found within Attic; ignoring" % (pathname,)
 160               )
 161       elif fname.endswith(',v'):
 162         cvs_file = self._get_attic_file(cvs_directory, fname)
 163         if cvs_file.parent_directory == cvs_directory:
 164           # This file will be retained in the Attic directory.
 165           retained_attic_files.append(cvs_file)
 166         else:
 167           # This is a normal Attic file, which is treated as if it
 168           # were located one directory up:
 169           yield cvs_file
 170
 171     if retained_attic_files:
 172       # There was at least one file in the attic that will be retained
 173       # in the attic.  First include the Attic directory itself in the
 174       # output, then the retained attic files:
 175       yield cvs_directory
 176       for cvs_file in retained_attic_files:
 177         yield cvs_file
 178
 179   def generate_cvs_paths(self, cvs_directory):
 180     """Generate the CVSPaths under non-Attic directory CVS_DIRECTORY.
 181
 182     Yield CVSDirectory and CVSFile instances as they are found.
 183     Process directories recursively, including Attic directories.
 184     Also look for conflicts between the filenames that will result
 185     from files, attic files, and subdirectories.
 186
 187     Silently ignore subdirectories named '.svn', as these don't make
 188     much sense in a real conversion, but they are present in our test
 189     suite."""
 190
 191     yield cvs_directory
 192
 193     # Map {cvs_file.basename : cvs_file.filename} for files directly
 194     # in cvs_directory:
 195     rcsfiles = {}
 196
 197     attic_dir = None
 198
 199     # Non-Attic subdirectories of cvs_directory (to be recursed into):
 200     dirs = []
 201
 202     fnames = os.listdir(cvs_directory.filename)
 203     fnames.sort()
 204     for fname in fnames:
 205       pathname = os.path.join(cvs_directory.filename, fname)
 206       if os.path.isdir(pathname):
 207         if fname == 'Attic':
 208           attic_dir = fname
 209         elif fname == '.svn':
 210           logger.debug("Directory %s ignored" % (pathname,))
 211         else:
 212           dirs.append(fname)
 213       elif fname.endswith(',v'):
 214         cvs_file = self._get_cvs_file(cvs_directory, fname)
 215         rcsfiles[cvs_file.basename] = cvs_file.filename
 216         yield cvs_file
 217       else:
 218         # Silently ignore other files:
 219         pass
 220
 221     # Map {cvs_file.basename : cvs_file.filename} for files in an
 222     # Attic directory within cvs_directory:
 223     attic_rcsfiles = {}
 224
 225     if attic_dir is not None:
 226       attic_directory = CVSDirectory(
 227           self.file_key_generator.gen_id(),
 228           cvs_directory.project, cvs_directory, 'Attic',
 229           )
 230
 231       for cvs_path in self._generate_attic_cvs_files(attic_directory):
 232         if isinstance(cvs_path, CVSFile) \
 233                and cvs_path.parent_directory == cvs_directory:
 234           attic_rcsfiles[cvs_path.basename] = cvs_path.filename
 235
 236         yield cvs_path
 237
 238       alldirs = dirs + [attic_dir]
 239     else:
 240       alldirs = dirs
 241
 242     # Check for conflicts between directory names and the filenames
 243     # that will result from the rcs files (both in this directory and
 244     # in attic).  (We recurse into the subdirectories nevertheless, to
 245     # try to detect more problems.)
 246     for fname in alldirs:
 247       for rcsfile_list in [rcsfiles, attic_rcsfiles]:
 248         if fname in rcsfile_list:
 249           self.error_handler(
 250               'Directory name conflicts with filename.  Please remove or '
 251               'rename one\n'
 252               'of the following:\n'
 253               '    "%s"\n'
 254               '    "%s"' % (
 255                   os.path.join(cvs_directory.filename, fname),
 256                   rcsfile_list[fname],
 257                   )
 258               )
 259
 260     # Now recurse into the other subdirectories:
 261     for fname in dirs:
 262       dirname = os.path.join(cvs_directory.filename, fname)
 263
 264       # Verify that the directory name does not contain any illegal
 265       # characters:
 266       try:
 267         verify_svn_filename_legal(fname)
 268       except IllegalSVNPathError, e:
 269         raise FatalError(
 270             'Directory %r would result in an illegal SVN path name: %s'
 271             % (dirname, e,)
 272             )
 273
 274       sub_directory = CVSDirectory(
 275           self.file_key_generator.gen_id(),
 276           cvs_directory.project, cvs_directory, fname,
 277           )
 278
 279       for cvs_path in self.generate_cvs_paths(sub_directory):
 280         yield cvs_path
 281
 282
 283 def walk_repository(project, file_key_generator, error_handler):
 284   """Generate CVSDirectories and CVSFiles within PROJECT.
 285
 286   Use FILE_KEY_GENERATOR to generate the IDs used for files.  If there
 287   is a fatal error, register it by calling ERROR_HANDLER with a string
 288   argument describing the problem.  (The error will be logged but
 289   processing will continue through the end of the pass.)  Also:
 290
 291   * Set PROJECT.root_cvs_directory_id.
 292
 293   * Handle files in the Attic by generating CVSFile instances with the
 294     _in_attic member set.
 295
 296   * Check for naming conflicts that will result from files in and out
 297     of the Attic.  If Ctx().retain_conflicting_attic_files is set, fix
 298     the conflicts by leaving the Attic file in the attic.  Otherwise,
 299     register a fatal error.
 300
 301   * Check for naming conflicts between files (in or out of the Attic)
 302     and directories.
 303
 304   * Check for filenames that contain characters not allowed by
 305     Subversion.
 306
 307   """
 308
 309   root_cvs_directory = CVSDirectory(
 310       file_key_generator.gen_id(), project, None, ''
 311       )
 312   project.root_cvs_directory_id = root_cvs_directory.id
 313   repository_walker = _RepositoryWalker(file_key_generator, error_handler)
 314   for cvs_path in repository_walker.generate_cvs_paths(root_cvs_directory):
 315     yield cvs_path
 316
 317