cvs2svn_lib/repository_walker.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Walk through a CVS project, generating CVSPaths."""
  18
  19
  20 import os
  21 import stat
  22
  23 from cvs2svn_lib.common import FatalError
  24 from cvs2svn_lib.common import warning_prefix
  25 from cvs2svn_lib.common import IllegalSVNPathError
  26 from cvs2svn_lib.common import verify_svn_filename_legal
  27 from cvs2svn_lib.log import Log
  28 from cvs2svn_lib.context import Ctx
  29 from cvs2svn_lib.project import FileInAndOutOfAtticException
  30 from cvs2svn_lib.cvs_path import CVSDirectory
  31 from cvs2svn_lib.cvs_path import CVSFile
  32
  33
  34 class _RepositoryWalker(object):
  35   def __init__(self, file_key_generator, error_handler):
  36     self.file_key_generator = file_key_generator
  37     self.error_handler = error_handler
  38
  39   def _get_cvs_file(
  40         self, parent_directory, basename,
  41         file_in_attic=False, leave_in_attic=False,
  42         ):
  43     """Return a CVSFile describing the file with name BASENAME.
  44
  45     PARENT_DIRECTORY is the CVSDirectory instance describing the
  46     directory that physically holds this file in the filesystem.
  47     BASENAME must be the base name of a *,v file within
  48     PARENT_DIRECTORY.
  49
  50     FILE_IN_ATTIC is a boolean telling whether the specified file is
  51     in an Attic subdirectory.  If FILE_IN_ATTIC is True, then:
  52
  53     - If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in
  54       the filename.
  55
  56     - Otherwise, raise FileInAndOutOfAtticException if a file with the
  57       same filename appears outside of Attic.
  58
  59     The CVSFile is assigned a new unique id.  All of the CVSFile
  60     information is filled in except mode (which can only be determined
  61     by parsing the file).
  62
  63     Raise FatalError if the resulting filename would not be legal in
  64     SVN."""
  65
  66     filename = os.path.join(parent_directory.filename, basename)
  67     try:
  68       verify_svn_filename_legal(basename[:-2])
  69     except IllegalSVNPathError, e:
  70       raise FatalError(
  71           'File %r would result in an illegal SVN filename: %s'
  72           % (filename, e,)
  73           )
  74
  75     if file_in_attic and not leave_in_attic:
  76       in_attic = True
  77       logical_parent_directory = parent_directory.parent_directory
  78
  79       # If this file also exists outside of the attic, it's a fatal
  80       # error:
  81       non_attic_filename = os.path.join(
  82           logical_parent_directory.filename, basename,
  83           )
  84       if os.path.exists(non_attic_filename):
  85         raise FileInAndOutOfAtticException(non_attic_filename, filename)
  86     else:
  87       in_attic = False
  88       logical_parent_directory = parent_directory
  89
  90     file_stat = os.stat(filename)
  91
  92     # The size of the file in bytes:
  93     file_size = file_stat[stat.ST_SIZE]
  94
  95     # Whether or not the executable bit is set:
  96     file_executable = bool(file_stat[0] & stat.S_IXUSR)
  97
  98     # mode is not known, so we temporarily set it to None.
  99     return CVSFile(
 100         self.file_key_generator.gen_id(),
 101         parent_directory.project, logical_parent_directory, basename[:-2],
 102         in_attic, file_executable, file_size, None, None
 103         )
 104
 105   def _get_attic_file(self, parent_directory, basename):
 106     """Return a CVSFile object for the Attic file at BASENAME.
 107
 108     PARENT_DIRECTORY is the CVSDirectory that physically contains the
 109     file on the filesystem (i.e., the Attic directory).  It is not
 110     necessarily the parent_directory of the CVSFile that will be
 111     returned.
 112
 113     Return CVSFile, whose parent directory is usually
 114     PARENT_DIRECTORY.parent_directory, but might be PARENT_DIRECTORY
 115     iff CVSFile will remain in the Attic directory."""
 116
 117     try:
 118       return self._get_cvs_file(
 119           parent_directory, basename, file_in_attic=True,
 120           )
 121     except FileInAndOutOfAtticException, e:
 122       if Ctx().retain_conflicting_attic_files:
 123         Log().warn(
 124             "%s: %s;\n"
 125             "   storing the latter into 'Attic' subdirectory.\n"
 126             % (warning_prefix, e)
 127             )
 128       else:
 129         self.error_handler(str(e))
 130
 131       # Either way, return a CVSFile object so that the rest of the
 132       # file processing can proceed:
 133       return self._get_cvs_file(
 134           parent_directory, basename, file_in_attic=True, leave_in_attic=True,
 135           )
 136
 137   def _generate_attic_cvs_files(self, cvs_directory):
 138     """Generate CVSFiles for the files in Attic directory CVS_DIRECTORY.
 139
 140     Also yield CVS_DIRECTORY if any files are being retained in the
 141     Attic."""
 142
 143     retained_attic_files = []
 144
 145     fnames = os.listdir(cvs_directory.filename)
 146     fnames.sort()
 147     for fname in fnames:
 148       pathname = os.path.join(cvs_directory.filename, fname)
 149       if os.path.isdir(pathname):
 150         Log().warn("Directory %s found within Attic; ignoring" % (pathname,))
 151       elif fname.endswith(',v'):
 152         cvs_file = self._get_attic_file(cvs_directory, fname)
 153         if cvs_file.parent_directory == cvs_directory:
 154           # This file will be retained in the Attic directory.
 155           retained_attic_files.append(cvs_file)
 156         else:
 157           # This is a normal Attic file, which is treated as if it
 158           # were located one directory up:
 159           yield cvs_file
 160
 161     if retained_attic_files:
 162       # There was at least one file in the attic that will be retained
 163       # in the attic.  First include the Attic directory itself in the
 164       # output, then the retained attic files:
 165       yield cvs_directory
 166       for cvs_file in retained_attic_files:
 167         yield cvs_file
 168
 169   def _get_non_attic_file(self, parent_directory, basename):
 170     """Return a CVSFile object for the non-Attic file at BASENAME."""
 171
 172     return self._get_cvs_file(parent_directory, basename)
 173
 174   def generate_cvs_paths(self, cvs_directory):
 175     """Generate the CVSPaths under non-Attic directory CVS_DIRECTORY.
 176
 177     Yield CVSDirectory and CVSFile instances as they are found.
 178     Process directories recursively, including Attic directories.
 179     Also look for conflicts between the filenames that will result
 180     from files, attic files, and subdirectories."""
 181
 182     yield cvs_directory
 183
 184     # Map {cvs_file.basename : cvs_file.filename} for files directly
 185     # in cvs_directory:
 186     rcsfiles = {}
 187
 188     attic_dir = None
 189
 190     # Non-Attic subdirectories of cvs_directory (to be recursed into):
 191     dirs = []
 192
 193     fnames = os.listdir(cvs_directory.filename)
 194     fnames.sort()
 195     for fname in fnames:
 196       pathname = os.path.join(cvs_directory.filename, fname)
 197       if os.path.isdir(pathname):
 198         if fname == 'Attic':
 199           attic_dir = fname
 200         else:
 201           dirs.append(fname)
 202       elif fname.endswith(',v'):
 203         cvs_file = self._get_non_attic_file(cvs_directory, fname)
 204         rcsfiles[cvs_file.basename] = cvs_file.filename
 205         yield cvs_file
 206       else:
 207         # Silently ignore other files:
 208         pass
 209
 210     # Map {cvs_file.basename : cvs_file.filename} for files in an
 211     # Attic directory within cvs_directory:
 212     attic_rcsfiles = {}
 213
 214     if attic_dir is not None:
 215       attic_directory = CVSDirectory(
 216           self.file_key_generator.gen_id(),
 217           cvs_directory.project, cvs_directory, 'Attic',
 218           )
 219
 220       for cvs_path in self._generate_attic_cvs_files(attic_directory):
 221         if isinstance(cvs_path, CVSFile) \
 222                and cvs_path.parent_directory == cvs_directory:
 223           attic_rcsfiles[cvs_path.basename] = cvs_path.filename
 224
 225         yield cvs_path
 226
 227       alldirs = dirs + [attic_dir]
 228     else:
 229       alldirs = dirs
 230
 231     # Check for conflicts between directory names and the filenames
 232     # that will result from the rcs files (both in this directory and
 233     # in attic).  (We recurse into the subdirectories nevertheless, to
 234     # try to detect more problems.)
 235     for fname in alldirs:
 236       pathname = os.path.join(cvs_directory.filename, fname)
 237       for rcsfile_list in [rcsfiles, attic_rcsfiles]:
 238         if fname in rcsfile_list:
 239           self.error_handler(
 240               'Directory name conflicts with filename.  Please remove or '
 241               'rename one\n'
 242               'of the following:\n'
 243               '    "%s"\n'
 244               '    "%s"'
 245               % (pathname, rcsfile_list[fname],)
 246               )
 247
 248     # Now recurse into the other subdirectories:
 249     for fname in dirs:
 250       dirname = os.path.join(cvs_directory.filename, fname)
 251
 252       # Verify that the directory name does not contain any illegal
 253       # characters:
 254       try:
 255         verify_svn_filename_legal(fname)
 256       except IllegalSVNPathError, e:
 257         raise FatalError(
 258             'Directory %r would result in an illegal SVN path name: %s'
 259             % (dirname, e,)
 260             )
 261
 262       sub_directory = CVSDirectory(
 263           self.file_key_generator.gen_id(),
 264           cvs_directory.project, cvs_directory, fname,
 265           )
 266
 267       for cvs_path in self.generate_cvs_paths(sub_directory):
 268         yield cvs_path
 269
 270
 271 def walk_repository(project, file_key_generator, error_handler):
 272   """Generate CVSDirectories and CVSFiles within PROJECT.
 273
 274   Use FILE_KEY_GENERATOR to generate the IDs used for files.  If there
 275   is a fatal error, register it by calling ERROR_HANDLER with a string
 276   argument describing the problem.  (The error will be logged but
 277   processing will continue through the end of the pass.)  Also:
 278
 279   * Set PROJECT.root_cvs_directory_id.
 280
 281   * Handle files in the Attic by generating CVSFile instances with the
 282     _in_attic member set.
 283
 284   * Check for naming conflicts that will result from files in and out
 285     of the Attic.  If Ctx().retain_conflicting_attic_files is set, fix
 286     the conflicts by leaving the Attic file in the attic.  Otherwise,
 287     register a fatal error.
 288
 289   * Check for naming conflicts between files (in or out of the Attic)
 290     and directories.
 291
 292   * Check for filenames that contain characters not allowed by
 293     Subversion.
 294
 295   """
 296
 297   root_cvs_directory = CVSDirectory(
 298       file_key_generator.gen_id(), project, None, ''
 299       )
 300   project.root_cvs_directory_id = root_cvs_directory.id
 301   repository_walker = _RepositoryWalker(file_key_generator, error_handler)
 302   for cvs_path in repository_walker.generate_cvs_paths(root_cvs_directory):
 303     yield cvs_path
 304
 305