cvs2svn_lib/repository_walker.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """Walk through a CVS project, generating CVSPaths."""
  18
  19
  20 import os
  21 import stat
  22
  23 from cvs2svn_lib.common import FatalError
  24 from cvs2svn_lib.common import warning_prefix
  25 from cvs2svn_lib.common import IllegalSVNPathError
  26 from cvs2svn_lib.common import verify_svn_filename_legal
  27 from cvs2svn_lib.log import Log
  28 from cvs2svn_lib.context import Ctx
  29 from cvs2svn_lib.project import FileInAndOutOfAtticException
  30 from cvs2svn_lib.cvs_path import CVSDirectory
  31 from cvs2svn_lib.cvs_path import CVSFile
  32
  33
  34 class _RepositoryWalker(object):
  35   def __init__(self, file_key_generator, error_handler):
  36     self.file_key_generator = file_key_generator
  37     self.error_handler = error_handler
  38
  39   def _get_cvs_file(
  40         self, parent_directory, basename, file_in_attic, leave_in_attic=False
  41         ):
  42     """Return a CVSFile describing the file with name BASENAME.
  43
  44     PARENT_DIRECTORY is the CVSDirectory instance describing the
  45     directory that physically holds this file in the filesystem.
  46     BASENAME must be the base name of a *,v file within
  47     PARENT_DIRECTORY.
  48
  49     FILE_IN_ATTIC is a boolean telling whether the specified file is
  50     in an Attic subdirectory.  If FILE_IN_ATTIC is True, then:
  51
  52     - If LEAVE_IN_ATTIC is True, then leave the 'Attic' component in
  53       the filename.
  54
  55     - Otherwise, raise FileInAndOutOfAtticException if a file with the
  56       same filename appears outside of Attic.
  57
  58     The CVSFile is assigned a new unique id.  All of the CVSFile
  59     information is filled in except mode (which can only be determined
  60     by parsing the file).
  61
  62     Raise FatalError if the resulting filename would not be legal in
  63     SVN."""
  64
  65     filename = os.path.join(parent_directory.filename, basename)
  66     try:
  67       verify_svn_filename_legal(basename[:-2])
  68     except IllegalSVNPathError, e:
  69       raise FatalError(
  70           'File %r would result in an illegal SVN filename: %s'
  71           % (filename, e,)
  72           )
  73
  74     if file_in_attic and not leave_in_attic:
  75       in_attic = True
  76       logical_parent_directory = parent_directory.parent_directory
  77
  78       # If this file also exists outside of the attic, it's a fatal
  79       # error:
  80       non_attic_filename = os.path.join(
  81           logical_parent_directory.filename, basename,
  82           )
  83       if os.path.exists(non_attic_filename):
  84         raise FileInAndOutOfAtticException(non_attic_filename, filename)
  85     else:
  86       in_attic = False
  87       logical_parent_directory = parent_directory
  88
  89     file_stat = os.stat(filename)
  90
  91     # The size of the file in bytes:
  92     file_size = file_stat[stat.ST_SIZE]
  93
  94     # Whether or not the executable bit is set:
  95     file_executable = bool(file_stat[0] & stat.S_IXUSR)
  96
  97     # mode is not known, so we temporarily set it to None.
  98     return CVSFile(
  99         self.file_key_generator.gen_id(),
 100         parent_directory.project, logical_parent_directory, basename[:-2],
 101         in_attic, file_executable, file_size, None, None
 102         )
 103
 104   def _get_attic_file(self, parent_directory, basename):
 105     """Return a CVSFile object for the Attic file at BASENAME.
 106
 107     PARENT_DIRECTORY is the CVSDirectory that physically contains the
 108     file on the filesystem (i.e., the Attic directory).  It is not
 109     necessarily the parent_directory of the CVSFile that will be
 110     returned.
 111
 112     Return CVSFile, whose parent directory is usually
 113     PARENT_DIRECTORY.parent_directory, but might be PARENT_DIRECTORY
 114     iff CVSFile will remain in the Attic directory."""
 115
 116     try:
 117       return self._get_cvs_file(parent_directory, basename, True)
 118     except FileInAndOutOfAtticException, e:
 119       if Ctx().retain_conflicting_attic_files:
 120         Log().warn(
 121             "%s: %s;\n"
 122             "   storing the latter into 'Attic' subdirectory.\n"
 123             % (warning_prefix, e)
 124             )
 125       else:
 126         self.error_handler(str(e))
 127
 128       # Either way, return a CVSFile object so that the rest of the
 129       # file processing can proceed:
 130       return self._get_cvs_file(
 131           parent_directory, basename, True, leave_in_attic=True
 132           )
 133
 134   def _generate_attic_cvs_files(self, cvs_directory):
 135     """Generate CVSFiles for the files in Attic directory CVS_DIRECTORY.
 136
 137     Also yield CVS_DIRECTORY if any files are being retained in the
 138     Attic."""
 139
 140     retained_attic_files = []
 141
 142     fnames = os.listdir(cvs_directory.filename)
 143     fnames.sort()
 144     for fname in fnames:
 145       pathname = os.path.join(cvs_directory.filename, fname)
 146       if os.path.isdir(pathname):
 147         Log().warn("Directory %s found within Attic; ignoring" % (pathname,))
 148       elif fname.endswith(',v'):
 149         cvs_file = self._get_attic_file(cvs_directory, fname)
 150         if cvs_file.parent_directory == cvs_directory:
 151           # This file will be retained in the Attic directory.
 152           retained_attic_files.append(cvs_file)
 153         else:
 154           # This is a normal Attic file, which is treated as if it
 155           # were located one directory up:
 156           yield cvs_file
 157
 158     if retained_attic_files:
 159       # There was at least one file in the attic that will be retained
 160       # in the attic.  First include the Attic directory itself in the
 161       # output, then the retained attic files:
 162       yield cvs_directory
 163       for cvs_file in retained_attic_files:
 164         yield cvs_file
 165
 166   def _get_non_attic_file(self, parent_directory, basename):
 167     """Return a CVSFile object for the non-Attic file at BASENAME."""
 168
 169     return self._get_cvs_file(parent_directory, basename, False)
 170
 171   def generate_cvs_paths(self, cvs_directory):
 172     """Generate the CVSPaths under non-Attic directory CVS_DIRECTORY.
 173
 174     Yield CVSDirectory and CVSFile instances as they are found.
 175     Process directories recursively, including Attic directories.
 176     Also look for conflicts between the filenames that will result
 177     from files, attic files, and subdirectories."""
 178
 179     yield cvs_directory
 180
 181     # Map {cvs_file.basename : cvs_file.filename} for files directly
 182     # in cvs_directory:
 183     rcsfiles = {}
 184
 185     attic_dir = None
 186
 187     # Non-Attic subdirectories of cvs_directory (to be recursed into):
 188     dirs = []
 189
 190     fnames = os.listdir(cvs_directory.filename)
 191     fnames.sort()
 192     for fname in fnames:
 193       pathname = os.path.join(cvs_directory.filename, fname)
 194       if os.path.isdir(pathname):
 195         if fname == 'Attic':
 196           attic_dir = fname
 197         else:
 198           dirs.append(fname)
 199       elif fname.endswith(',v'):
 200         cvs_file = self._get_non_attic_file(cvs_directory, fname)
 201         rcsfiles[cvs_file.basename] = cvs_file.filename
 202         yield cvs_file
 203       else:
 204         # Silently ignore other files:
 205         pass
 206
 207     # Map {cvs_file.basename : cvs_file.filename} for files in an
 208     # Attic directory within cvs_directory:
 209     attic_rcsfiles = {}
 210
 211     if attic_dir is not None:
 212       attic_directory = CVSDirectory(
 213           self.file_key_generator.gen_id(),
 214           cvs_directory.project, cvs_directory, 'Attic',
 215           )
 216
 217       for cvs_path in self._generate_attic_cvs_files(attic_directory):
 218         if isinstance(cvs_path, CVSFile) \
 219                and cvs_path.parent_directory == cvs_directory:
 220           attic_rcsfiles[cvs_path.basename] = cvs_path.filename
 221
 222         yield cvs_path
 223
 224       alldirs = dirs + [attic_dir]
 225     else:
 226       alldirs = dirs
 227
 228     # Check for conflicts between directory names and the filenames
 229     # that will result from the rcs files (both in this directory and
 230     # in attic).  (We recurse into the subdirectories nevertheless, to
 231     # try to detect more problems.)
 232     for fname in alldirs:
 233       pathname = os.path.join(cvs_directory.filename, fname)
 234       for rcsfile_list in [rcsfiles, attic_rcsfiles]:
 235         if fname in rcsfile_list:
 236           self.error_handler(
 237               'Directory name conflicts with filename.  Please remove or '
 238               'rename one\n'
 239               'of the following:\n'
 240               '    "%s"\n'
 241               '    "%s"'
 242               % (pathname, rcsfile_list[fname],)
 243               )
 244
 245     # Now recurse into the other subdirectories:
 246     for fname in dirs:
 247       dirname = os.path.join(cvs_directory.filename, fname)
 248
 249       # Verify that the directory name does not contain any illegal
 250       # characters:
 251       try:
 252         verify_svn_filename_legal(fname)
 253       except IllegalSVNPathError, e:
 254         raise FatalError(
 255             'Directory %r would result in an illegal SVN path name: %s'
 256             % (dirname, e,)
 257             )
 258
 259       sub_directory = CVSDirectory(
 260           self.file_key_generator.gen_id(),
 261           cvs_directory.project, cvs_directory, fname,
 262           )
 263
 264       for cvs_path in self.generate_cvs_paths(sub_directory):
 265         yield cvs_path
 266
 267
 268 def walk_repository(project, file_key_generator, error_handler):
 269   """Generate CVSDirectories and CVSFiles within PROJECT.
 270
 271   Use FILE_KEY_GENERATOR to generate the IDs used for files.  If there
 272   is a fatal error, register it by calling ERROR_HANDLER with a string
 273   argument describing the problem.  (The error will be logged but
 274   processing will continue through the end of the pass.)  Also:
 275
 276   * Set PROJECT.root_cvs_directory_id.
 277
 278   * Handle files in the Attic by generating CVSFile instances with the
 279     _in_attic member set.
 280
 281   * Check for naming conflicts that will result from files in and out
 282     of the Attic.  If Ctx().retain_conflicting_attic_files is set, fix
 283     the conflicts by leaving the Attic file in the attic.  Otherwise,
 284     register a fatal error.
 285
 286   * Check for naming conflicts between files (in or out of the Attic)
 287     and directories.
 288
 289   * Check for filenames that contain characters not allowed by
 290     Subversion.
 291
 292   """
 293
 294   root_cvs_directory = CVSDirectory(
 295       file_key_generator.gen_id(), project, None, ''
 296       )
 297   project.root_cvs_directory_id = root_cvs_directory.id
 298   repository_walker = _RepositoryWalker(file_key_generator, error_handler)
 299   for cvs_path in repository_walker.generate_cvs_paths(root_cvs_directory):
 300     yield cvs_path
 301
 302