cvs2svn_lib/common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains common facilities used by cvs2svn."""
  18
  19
  20 import re
  21 import time
  22 import codecs
  23
  24 from cvs2svn_lib.log import Log
  25
  26
  27 # Always use these constants for opening databases.
  28 DB_OPEN_READ = 'r'
  29 DB_OPEN_WRITE = 'w'
  30 DB_OPEN_NEW = 'n'
  31
  32
  33 SVN_INVALID_REVNUM = -1
  34
  35
  36 # Warnings and errors start with these strings.  They are typically
  37 # followed by a colon and a space, as in "%s: " ==> "WARNING: ".
  38 warning_prefix = "WARNING"
  39 error_prefix = "ERROR"
  40
  41
  42 class FatalException(Exception):
  43   """Exception thrown on a non-recoverable error.
  44
  45   If this exception is thrown by main(), it is caught by the global
  46   layer of the program, its string representation is printed (followed
  47   by a newline), and the program is ended with an exit code of 1."""
  48
  49   pass
  50
  51
  52 class InternalError(Exception):
  53   """Exception thrown in the case of a cvs2svn internal error (aka, bug)."""
  54
  55   pass
  56
  57
  58 class FatalError(FatalException):
  59   """A FatalException that prepends error_prefix to the message."""
  60
  61   def __init__(self, msg):
  62     """Use (error_prefix + ': ' + MSG) as the error message."""
  63
  64     FatalException.__init__(self, '%s: %s' % (error_prefix, msg,))
  65
  66
  67 class CommandError(FatalError):
  68   """A FatalError caused by a failed command invocation.
  69
  70   The error message includes the command name, exit code, and output."""
  71
  72   def __init__(self, command, exit_status, error_output=''):
  73     self.command = command
  74     self.exit_status = exit_status
  75     self.error_output = error_output
  76     if error_output.rstrip():
  77       FatalError.__init__(
  78           self,
  79           'The command %r failed with exit status=%s\n'
  80           'and the following output:\n'
  81           '%s'
  82           % (self.command, self.exit_status, self.error_output.rstrip()))
  83     else:
  84       FatalError.__init__(
  85           self,
  86           'The command %r failed with exit status=%s and no output'
  87           % (self.command, self.exit_status))
  88
  89
  90 def path_join(*components):
  91   """Join two or more pathname COMPONENTS, inserting '/' as needed.
  92   Empty component are skipped."""
  93
  94   return '/'.join(filter(None, components))
  95
  96
  97 def path_split(path):
  98   """Split the svn pathname PATH into a pair, (HEAD, TAIL).
  99
 100   This is similar to os.path.split(), but always uses '/' as path
 101   separator.  PATH is an svn path, which should not start with a '/'.
 102   HEAD is everything before the last slash, and TAIL is everything
 103   after.  If PATH ends in a slash, TAIL will be empty.  If there is no
 104   slash in PATH, HEAD will be empty.  If PATH is empty, both HEAD and
 105   TAIL are empty."""
 106
 107   pos = path.rfind('/')
 108   if pos == -1:
 109     return ('', path,)
 110   else:
 111     return (path[:pos], path[pos+1:],)
 112
 113
 114 class IllegalSVNPathError(FatalException):
 115   pass
 116
 117
 118 # Control characters (characters not allowed in Subversion filenames):
 119 ctrl_characters_regexp = re.compile('[\\\x00-\\\x1f\\\x7f]')
 120
 121
 122 def verify_svn_filename_legal(filename):
 123   """Verify that FILENAME is a legal filename.
 124
 125   FILENAME is a path component of a CVS path.  Check that it won't
 126   choke SVN:
 127
 128   - Check that it is not empty.
 129
 130   - Check that it is not equal to '.' or '..'.
 131
 132   - Check that the filename does not include any control characters.
 133
 134   If any of these tests fail, raise an IllegalSVNPathError."""
 135
 136   if filename == '':
 137     raise IllegalSVNPathError("Empty filename component.")
 138
 139   if filename in ['.', '..']:
 140     raise IllegalSVNPathError("Illegal filename component %r." % (filename,))
 141
 142   m = ctrl_characters_regexp.search(filename)
 143   if m:
 144     raise IllegalSVNPathError(
 145         "Character %r in filename %r is not supported by Subversion."
 146         % (m.group(), filename,)
 147         )
 148
 149
 150 def verify_svn_path_legal(path):
 151   """Verify that PATH is a legitimate SVN path.
 152
 153   If not, raise an IllegalSVNPathError."""
 154
 155   if path.startswith('/'):
 156     raise IllegalSVNPathError("Path %r must not start with '/'." % (path,))
 157   head = path
 158   while head != '':
 159     (head,tail) = path_split(head)
 160     try:
 161         verify_svn_filename_legal(tail)
 162     except IllegalSVNPathError, e:
 163         raise IllegalSVNPathError('Problem with path %r: %s' % (path, e,))
 164
 165
 166 def normalize_svn_path(path, allow_empty=False):
 167   """Normalize an SVN path (e.g., one supplied by a user).
 168
 169   1. Strip leading, trailing, and duplicated '/'.
 170   2. If ALLOW_EMPTY is not set, verify that PATH is not empty.
 171
 172   Return the normalized path.
 173
 174   If the path is invalid, raise an IllegalSVNPathError."""
 175
 176   norm_path = path_join(*path.split('/'))
 177   if not allow_empty and not norm_path:
 178     raise IllegalSVNPathError("Path is empty")
 179   return norm_path
 180
 181
 182 class PathRepeatedException(Exception):
 183   def __init__(self, path, count):
 184     self.path = path
 185     self.count = count
 186     Exception.__init__(
 187         self, 'Path %s is repeated %d times' % (self.path, self.count,)
 188         )
 189
 190
 191 class PathsNestedException(Exception):
 192   def __init__(self, nest, nestlings):
 193     self.nest = nest
 194     self.nestlings = nestlings
 195     Exception.__init__(
 196         self,
 197         'Path %s contains the following other paths: %s'
 198         % (self.nest, ', '.join(self.nestlings),)
 199         )
 200
 201
 202 class PathsNotDisjointException(FatalException):
 203   """An exception that collects multiple other disjointness exceptions."""
 204
 205   def __init__(self, problems):
 206     self.problems = problems
 207     Exception.__init__(
 208         self,
 209         'The following paths are not disjoint:\n'
 210         '    %s\n'
 211         % ('\n    '.join([str(problem) for problem in self.problems]),)
 212         )
 213
 214
 215 def verify_paths_disjoint(*paths):
 216   """Verify that all of the paths in the argument list are disjoint.
 217
 218   If any of the paths is nested in another one (i.e., in the sense
 219   that 'a/b/c/d' is nested in 'a/b'), or any two paths are identical,
 220   raise a PathsNotDisjointException containing exceptions detailing
 221   the individual problems."""
 222
 223   def split(path):
 224     if not path:
 225       return []
 226     else:
 227       return path.split('/')
 228
 229   def contains(split_path1, split_path2):
 230     """Return True iff SPLIT_PATH1 contains SPLIT_PATH2."""
 231
 232     return (
 233         len(split_path1) < len(split_path2)
 234         and split_path2[:len(split_path1)] == split_path1
 235         )
 236
 237   paths = [(split(path), path) for path in paths]
 238   # If all overlapping elements are equal, a shorter list is
 239   # considered "less than" a longer one.  Therefore if any paths are
 240   # nested, this sort will leave at least one such pair adjacent, in
 241   # the order [nest,nestling].
 242   paths.sort()
 243
 244   problems = []
 245
 246   # Create exceptions for any repeated paths, and delete the repeats
 247   # from the paths array:
 248   i = 0
 249   while i < len(paths):
 250     split_path, path = paths[i]
 251     j = i + 1
 252     while j < len(paths) and split_path == paths[j][0]:
 253       j += 1
 254     if j - i > 1:
 255       problems.append(PathRepeatedException(path, j - i))
 256       # Delete all but the first copy:
 257       del paths[i + 1:j]
 258     i += 1
 259
 260   # Create exceptions for paths nested in each other:
 261   i = 0
 262   while i < len(paths):
 263     split_path, path = paths[i]
 264     j = i + 1
 265     while j < len(paths) and contains(split_path, paths[j][0]):
 266       j += 1
 267     if j - i > 1:
 268       problems.append(PathsNestedException(
 269           path, [path2 for (split_path2, path2) in paths[i + 1:j]]
 270           ))
 271     i += 1
 272
 273   if problems:
 274     raise PathsNotDisjointException(problems)
 275
 276
 277 def format_date(date):
 278   """Return an svn-compatible date string for DATE (seconds since epoch).
 279
 280   A Subversion date looks like '2002-09-29T14:44:59.000000Z'."""
 281
 282   return time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", time.gmtime(date))
 283
 284
 285 class CVSTextDecoder:
 286   """Callable that decodes CVS strings into Unicode."""
 287
 288   def __init__(self, encodings, fallback_encoding=None):
 289     """Create a CVSTextDecoder instance.
 290
 291     ENCODINGS is a list containing the names of encodings that are
 292     attempted to be used as source encodings in 'strict' mode.
 293
 294     FALLBACK_ENCODING, if specified, is the name of an encoding that
 295     should be used as a source encoding in lossy 'replace' mode if all
 296     of ENCODINGS failed.
 297
 298     Raise LookupError if any of the specified encodings is unknown."""
 299
 300     self.decoders = [
 301         (encoding, codecs.lookup(encoding)[1])
 302         for encoding in encodings]
 303
 304     if fallback_encoding is None:
 305       self.fallback_decoder = None
 306     else:
 307       self.fallback_decoder = (
 308           fallback_encoding, codecs.lookup(fallback_encoding)[1]
 309           )
 310
 311   def add_encoding(self, encoding):
 312     """Add an encoding to be tried in 'strict' mode.
 313
 314     ENCODING is the name of an encoding.  If it is unknown, raise a
 315     LookupError."""
 316
 317     for (name, decoder) in self.decoders:
 318       if name == encoding:
 319         return
 320     else:
 321       self.decoders.append( (encoding, codecs.lookup(encoding)[1]) )
 322
 323   def set_fallback_encoding(self, encoding):
 324     """Set the fallback encoding, to be tried in 'replace' mode.
 325
 326     ENCODING is the name of an encoding.  If it is unknown, raise a
 327     LookupError."""
 328
 329     if encoding is None:
 330       self.fallback_decoder = None
 331     else:
 332       self.fallback_decoder = (encoding, codecs.lookup(encoding)[1])
 333
 334   def __call__(self, s):
 335     """Try to decode string S using our configured source encodings.
 336
 337     Return the string as a Unicode string.  If S is already a unicode
 338     string, do nothing.
 339
 340     Raise UnicodeError if the string cannot be decoded using any of
 341     the source encodings and no fallback encoding was specified."""
 342
 343     if isinstance(s, unicode):
 344       return s
 345     for (name, decoder) in self.decoders:
 346       try:
 347         return decoder(s)[0]
 348       except ValueError:
 349         Log().verbose("Encoding '%s' failed for string %r" % (name, s))
 350
 351     if self.fallback_decoder is not None:
 352       (name, decoder) = self.fallback_decoder
 353       return decoder(s, 'replace')[0]
 354     else:
 355       raise UnicodeError
 356
 357
 358 class Timestamper:
 359   """Return monotonic timestamps derived from changeset timestamps."""
 360
 361   def __init__(self):
 362     # The last timestamp that has been returned:
 363     self.timestamp = 0.0
 364
 365     # The maximum timestamp that is considered reasonable:
 366     self.max_timestamp = time.time() + 24.0 * 60.0 * 60.0
 367
 368   def get(self, timestamp, change_expected):
 369     """Return a reasonable timestamp derived from TIMESTAMP.
 370
 371     Push TIMESTAMP into the future if necessary to ensure that it is
 372     at least one second later than every other timestamp that has been
 373     returned by previous calls to this method.
 374
 375     If CHANGE_EXPECTED is not True, then log a message if the
 376     timestamp has to be changed."""
 377
 378     if timestamp > self.max_timestamp:
 379       # If a timestamp is in the future, it is assumed that it is
 380       # bogus.  Shift it backwards in time to prevent it forcing other
 381       # timestamps to be pushed even further in the future.
 382
 383       # Note that this is not nearly a complete solution to the bogus
 384       # timestamp problem.  A timestamp in the future still affects
 385       # the ordering of changesets, and a changeset having such a
 386       # timestamp will not be committed until all changesets with
 387       # earlier timestamps have been committed, even if other
 388       # changesets with even earlier timestamps depend on this one.
 389       self.timestamp = self.timestamp + 1.0
 390       if not change_expected:
 391         Log().warn(
 392             'Timestamp "%s" is in the future; changed to "%s".'
 393             % (time.asctime(time.gmtime(timestamp)),
 394                time.asctime(time.gmtime(self.timestamp)),)
 395             )
 396     elif timestamp < self.timestamp + 1.0:
 397       self.timestamp = self.timestamp + 1.0
 398       if not change_expected and Log().is_on(Log.VERBOSE):
 399         Log().verbose(
 400             'Timestamp "%s" adjusted to "%s" to ensure monotonicity.'
 401             % (time.asctime(time.gmtime(timestamp)),
 402                time.asctime(time.gmtime(self.timestamp)),)
 403             )
 404     else:
 405       self.timestamp = timestamp
 406
 407     return self.timestamp
 408
 409