cvs2svn_lib/repository_walker.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Walk through a CVS project, generating CVSPaths."""
  18
  19
  20 import os
  21 import stat
  22
  23 from cvs2svn_lib.common import FatalError
  24 from cvs2svn_lib.common import warning_prefix
  25 from cvs2svn_lib.common import IllegalSVNPathError
  26 from cvs2svn_lib.common import verify_svn_filename_legal
  27 from cvs2svn_lib.log import Log
  28 from cvs2svn_lib.context import Ctx
  29 from cvs2svn_lib.project import FileInAndOutOfAtticException
  30 from cvs2svn_lib.cvs_path import CVSDirectory
  31 from cvs2svn_lib.cvs_path import CVSFile
  32
  33
  34 class _RepositoryWalker(object):
  35   def __init__(self, file_key_generator, error_handler):
  36     self.file_key_generator = file_key_generator
  37     self.error_handler = error_handler
  38
  39   def _get_cvs_file(
  40         self, parent_directory, basename,
  41         file_in_attic=False, leave_in_attic=False,
  42         ):
  43     """Return a CVSFile describing the file with name BASENAME.
  44
  45     PARENT_DIRECTORY is the CVSDirectory instance describing the
  46     directory that physically holds this file in the filesystem.
  47     BASENAME must be the base name of a *,v file within
  48     PARENT_DIRECTORY.
  49
  50     FILE_IN_ATTIC is a boolean telling whether the specified file is
  51     in an Attic subdirectory.  If FILE_IN_ATTIC is True, then:
  52
  53     - If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in
  54       the filename.
  55
  56     - Otherwise, raise FileInAndOutOfAtticException if a file with the
  57       same filename appears outside of Attic.
  58
  59     The CVSFile is assigned a new unique id.  All of the CVSFile
  60     information is filled in except mode (which can only be determined
  61     by parsing the file).
  62
  63     Raise FatalError if the resulting filename would not be legal in
  64     SVN."""
  65
  66     filename = os.path.join(parent_directory.filename, basename)
  67     try:
  68       verify_svn_filename_legal(basename[:-2])
  69     except IllegalSVNPathError, e:
  70       raise FatalError(
  71           'File %r would result in an illegal SVN filename: %s'
  72           % (filename, e,)
  73           )
  74
  75     if file_in_attic and not leave_in_attic:
  76       in_attic = True
  77       logical_parent_directory = parent_directory.parent_directory
  78
  79       # If this file also exists outside of the attic, it's a fatal
  80       # error:
  81       non_attic_filename = os.path.join(
  82           logical_parent_directory.filename, basename,
  83           )
  84       if os.path.exists(non_attic_filename):
  85         raise FileInAndOutOfAtticException(non_attic_filename, filename)
  86     else:
  87       in_attic = False
  88       logical_parent_directory = parent_directory
  89
  90     file_stat = os.stat(filename)
  91
  92     # The size of the file in bytes:
  93     file_size = file_stat[stat.ST_SIZE]
  94
  95     # Whether or not the executable bit is set:
  96     file_executable = bool(file_stat[0] & stat.S_IXUSR)
  97
  98     # mode is not known, so we temporarily set it to None.
  99     return CVSFile(
 100         self.file_key_generator.gen_id(),
 101         parent_directory.project, logical_parent_directory, basename[:-2],
 102         in_attic, file_executable, file_size, None, None
 103         )
 104
 105   def _get_attic_file(self, parent_directory, basename):
 106     """Return a CVSFile object for the Attic file at BASENAME.
 107
 108     PARENT_DIRECTORY is the CVSDirectory that physically contains the
 109     file on the filesystem (i.e., the Attic directory).  It is not
 110     necessarily the parent_directory of the CVSFile that will be
 111     returned.
 112
 113     Return CVSFile, whose parent directory is usually
 114     PARENT_DIRECTORY.parent_directory, but might be PARENT_DIRECTORY
 115     iff CVSFile will remain in the Attic directory."""
 116
 117     try:
 118       return self._get_cvs_file(
 119           parent_directory, basename, file_in_attic=True,
 120           )
 121     except FileInAndOutOfAtticException, e:
 122       if Ctx().retain_conflicting_attic_files:
 123         Log().warn(
 124             "%s: %s;\n"
 125             "   storing the latter into 'Attic' subdirectory.\n"
 126             % (warning_prefix, e)
 127             )
 128       else:
 129         self.error_handler(str(e))
 130
 131       # Either way, return a CVSFile object so that the rest of the
 132       # file processing can proceed:
 133       return self._get_cvs_file(
 134           parent_directory, basename, file_in_attic=True, leave_in_attic=True,
 135           )
 136
 137   def _generate_attic_cvs_files(self, cvs_directory):
 138     """Generate CVSFiles for the files in Attic directory CVS_DIRECTORY.
 139
 140     Also yield CVS_DIRECTORY if any files are being retained in the
 141     Attic."""
 142
 143     retained_attic_files = []
 144
 145     fnames = os.listdir(cvs_directory.filename)
 146     fnames.sort()
 147     for fname in fnames:
 148       pathname = os.path.join(cvs_directory.filename, fname)
 149       if os.path.isdir(pathname):
 150         Log().warn("Directory %s found within Attic; ignoring" % (pathname,))
 151       elif fname.endswith(',v'):
 152         cvs_file = self._get_attic_file(cvs_directory, fname)
 153         if cvs_file.parent_directory == cvs_directory:
 154           # This file will be retained in the Attic directory.
 155           retained_attic_files.append(cvs_file)
 156         else:
 157           # This is a normal Attic file, which is treated as if it
 158           # were located one directory up:
 159           yield cvs_file
 160
 161     if retained_attic_files:
 162       # There was at least one file in the attic that will be retained
 163       # in the attic.  First include the Attic directory itself in the
 164       # output, then the retained attic files:
 165       yield cvs_directory
 166       for cvs_file in retained_attic_files:
 167         yield cvs_file
 168
 169   def generate_cvs_paths(self, cvs_directory):
 170     """Generate the CVSPaths under non-Attic directory CVS_DIRECTORY.
 171
 172     Yield CVSDirectory and CVSFile instances as they are found.
 173     Process directories recursively, including Attic directories.
 174     Also look for conflicts between the filenames that will result
 175     from files, attic files, and subdirectories."""
 176
 177     yield cvs_directory
 178
 179     # Map {cvs_file.basename : cvs_file.filename} for files directly
 180     # in cvs_directory:
 181     rcsfiles = {}
 182
 183     attic_dir = None
 184
 185     # Non-Attic subdirectories of cvs_directory (to be recursed into):
 186     dirs = []
 187
 188     fnames = os.listdir(cvs_directory.filename)
 189     fnames.sort()
 190     for fname in fnames:
 191       pathname = os.path.join(cvs_directory.filename, fname)
 192       if os.path.isdir(pathname):
 193         if fname == 'Attic':
 194           attic_dir = fname
 195         else:
 196           dirs.append(fname)
 197       elif fname.endswith(',v'):
 198         cvs_file = self._get_cvs_file(cvs_directory, fname)
 199         rcsfiles[cvs_file.basename] = cvs_file.filename
 200         yield cvs_file
 201       else:
 202         # Silently ignore other files:
 203         pass
 204
 205     # Map {cvs_file.basename : cvs_file.filename} for files in an
 206     # Attic directory within cvs_directory:
 207     attic_rcsfiles = {}
 208
 209     if attic_dir is not None:
 210       attic_directory = CVSDirectory(
 211           self.file_key_generator.gen_id(),
 212           cvs_directory.project, cvs_directory, 'Attic',
 213           )
 214
 215       for cvs_path in self._generate_attic_cvs_files(attic_directory):
 216         if isinstance(cvs_path, CVSFile) \
 217                and cvs_path.parent_directory == cvs_directory:
 218           attic_rcsfiles[cvs_path.basename] = cvs_path.filename
 219
 220         yield cvs_path
 221
 222       alldirs = dirs + [attic_dir]
 223     else:
 224       alldirs = dirs
 225
 226     # Check for conflicts between directory names and the filenames
 227     # that will result from the rcs files (both in this directory and
 228     # in attic).  (We recurse into the subdirectories nevertheless, to
 229     # try to detect more problems.)
 230     for fname in alldirs:
 231       pathname = os.path.join(cvs_directory.filename, fname)
 232       for rcsfile_list in [rcsfiles, attic_rcsfiles]:
 233         if fname in rcsfile_list:
 234           self.error_handler(
 235               'Directory name conflicts with filename.  Please remove or '
 236               'rename one\n'
 237               'of the following:\n'
 238               '    "%s"\n'
 239               '    "%s"'
 240               % (pathname, rcsfile_list[fname],)
 241               )
 242
 243     # Now recurse into the other subdirectories:
 244     for fname in dirs:
 245       dirname = os.path.join(cvs_directory.filename, fname)
 246
 247       # Verify that the directory name does not contain any illegal
 248       # characters:
 249       try:
 250         verify_svn_filename_legal(fname)
 251       except IllegalSVNPathError, e:
 252         raise FatalError(
 253             'Directory %r would result in an illegal SVN path name: %s'
 254             % (dirname, e,)
 255             )
 256
 257       sub_directory = CVSDirectory(
 258           self.file_key_generator.gen_id(),
 259           cvs_directory.project, cvs_directory, fname,
 260           )
 261
 262       for cvs_path in self.generate_cvs_paths(sub_directory):
 263         yield cvs_path
 264
 265
 266 def walk_repository(project, file_key_generator, error_handler):
 267   """Generate CVSDirectories and CVSFiles within PROJECT.
 268
 269   Use FILE_KEY_GENERATOR to generate the IDs used for files.  If there
 270   is a fatal error, register it by calling ERROR_HANDLER with a string
 271   argument describing the problem.  (The error will be logged but
 272   processing will continue through the end of the pass.)  Also:
 273
 274   * Set PROJECT.root_cvs_directory_id.
 275
 276   * Handle files in the Attic by generating CVSFile instances with the
 277     _in_attic member set.
 278
 279   * Check for naming conflicts that will result from files in and out
 280     of the Attic.  If Ctx().retain_conflicting_attic_files is set, fix
 281     the conflicts by leaving the Attic file in the attic.  Otherwise,
 282     register a fatal error.
 283
 284   * Check for naming conflicts between files (in or out of the Attic)
 285     and directories.
 286
 287   * Check for filenames that contain characters not allowed by
 288     Subversion.
 289
 290   """
 291
 292   root_cvs_directory = CVSDirectory(
 293       file_key_generator.gen_id(), project, None, ''
 294       )
 295   project.root_cvs_directory_id = root_cvs_directory.id
 296   repository_walker = _RepositoryWalker(file_key_generator, error_handler)
 297   for cvs_path in repository_walker.generate_cvs_paths(root_cvs_directory):
 298     yield cvs_path
 299
 300