cvs2svn_lib/common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains common facilities used by cvs2svn."""
  18
  19
  20 import re
  21 import time
  22 import codecs
  23
  24 from cvs2svn_lib.log import Log
  25
  26
  27 # Always use these constants for opening databases.
  28 DB_OPEN_READ = 'r'
  29 DB_OPEN_WRITE = 'w'
  30 DB_OPEN_NEW = 'n'
  31
  32
  33 SVN_INVALID_REVNUM = -1
  34
  35
  36 # Warnings and errors start with these strings.  They are typically
  37 # followed by a colon and a space, as in "%s: " ==> "WARNING: ".
  38 warning_prefix = "WARNING"
  39 error_prefix = "ERROR"
  40
  41
  42 class FatalException(Exception):
  43   """Exception thrown on a non-recoverable error.
  44
  45   If this exception is thrown by main(), it is caught by the global
  46   layer of the program, its string representation is printed (followed
  47   by a newline), and the program is ended with an exit code of 1."""
  48
  49   pass
  50
  51
  52 class InternalError(Exception):
  53   """Exception thrown in the case of a cvs2svn internal error (aka, bug)."""
  54
  55   pass
  56
  57
  58 class FatalError(FatalException):
  59   """A FatalException that prepends error_prefix to the message."""
  60
  61   def __init__(self, msg):
  62     """Use (error_prefix + ': ' + MSG) as the error message."""
  63
  64     FatalException.__init__(self, '%s: %s' % (error_prefix, msg,))
  65
  66
  67 class CommandError(FatalError):
  68   """A FatalError caused by a failed command invocation.
  69
  70   The error message includes the command name, exit code, and output."""
  71
  72   def __init__(self, command, exit_status, error_output=''):
  73     self.command = command
  74     self.exit_status = exit_status
  75     self.error_output = error_output
  76     if error_output.rstrip():
  77       FatalError.__init__(
  78           self,
  79           'The command %r failed with exit status=%s\n'
  80           'and the following output:\n'
  81           '%s'
  82           % (self.command, self.exit_status, self.error_output.rstrip()))
  83     else:
  84       FatalError.__init__(
  85           self,
  86           'The command %r failed with exit status=%s and no output'
  87           % (self.command, self.exit_status))
  88
  89
  90 def path_join(*components):
  91   """Join two or more pathname COMPONENTS, inserting '/' as needed.
  92   Empty component are skipped."""
  93
  94   return '/'.join(filter(None, components))
  95
  96
  97 def path_split(path):
  98   """Split the svn pathname PATH into a pair, (HEAD, TAIL).
  99
 100   This is similar to os.path.split(), but always uses '/' as path
 101   separator.  PATH is an svn path, which should not start with a '/'.
 102   HEAD is everything before the last slash, and TAIL is everything
 103   after.  If PATH ends in a slash, TAIL will be empty.  If there is no
 104   slash in PATH, HEAD will be empty.  If PATH is empty, both HEAD and
 105   TAIL are empty."""
 106
 107   pos = path.rfind('/')
 108   if pos == -1:
 109     return ('', path,)
 110   else:
 111     return (path[:pos], path[pos+1:],)
 112
 113
 114 class IllegalSVNPathError(FatalException):
 115   pass
 116
 117
 118 # Control characters (characters not allowed in Subversion filenames):
 119 ctrl_characters_regexp = re.compile('[\\\x00-\\\x1f\\\x7f]')
 120
 121
 122 def verify_svn_filename_legal(filename):
 123   """Verify that FILENAME is a legal filename.
 124
 125   FILENAME is a path component of a CVS path.  Check that it won't
 126   choke SVN:
 127
 128   - Check that it is not empty.
 129
 130   - Check that it is not equal to '.' or '..'.
 131
 132   - Check that the filename does not include any control characters.
 133
 134   If any of these tests fail, raise an IllegalSVNPathError."""
 135
 136   if filename == '':
 137     raise IllegalSVNPathError("Empty filename component.")
 138
 139   if filename in ['.', '..']:
 140     raise IllegalSVNPathError("Illegal filename component %r." % (filename,))
 141
 142   m = ctrl_characters_regexp.search(filename)
 143   if m:
 144     raise IllegalSVNPathError(
 145         "Character %r in filename %r is not supported by Subversion."
 146         % (m.group(), filename,)
 147         )
 148
 149
 150 def verify_svn_path_legal(path):
 151   """Verify that PATH is a legitimate SVN path.
 152
 153   If not, raise an IllegalSVNPathError."""
 154
 155   if path.startswith('/'):
 156     raise IllegalSVNPathError("Path %r must not start with '/'." % (path,))
 157   head = path
 158   while head != '':
 159     (head,tail) = path_split(head)
 160     try:
 161         verify_svn_filename_legal(tail)
 162     except IllegalSVNPathError, e:
 163         raise IllegalSVNPathError('Problem with path %r: %s' % (path, e,))
 164
 165
 166 def normalize_svn_path(path, allow_empty=False):
 167   """Normalize an SVN path (e.g., one supplied by a user).
 168
 169   1. Strip leading, trailing, and duplicated '/'.
 170   2. If ALLOW_EMPTY is not set, verify that PATH is not empty.
 171
 172   Return the normalized path.
 173
 174   If the path is invalid, raise an IllegalSVNPathError."""
 175
 176   norm_path = path_join(*path.split('/'))
 177   if not allow_empty and not norm_path:
 178     raise IllegalSVNPathError("Path is empty")
 179   return norm_path
 180
 181
 182 class PathRepeatedException(Exception):
 183   def __init__(self, path, count):
 184     self.path = path
 185     self.count = count
 186     Exception.__init__(
 187         self, 'Path %s is repeated %d times' % (self.path, self.count,)
 188         )
 189
 190
 191 class PathsNestedException(Exception):
 192   def __init__(self, nest, nestlings):
 193     self.nest = nest
 194     self.nestlings = nestlings
 195     Exception.__init__(
 196         self,
 197         'Path %s contains the following other paths: %s'
 198         % (self.nest, ', '.join(self.nestlings),)
 199         )
 200
 201
 202 class PathsNotDisjointException(FatalException):
 203   """An exception that collects multiple other disjointness exceptions."""
 204
 205   def __init__(self, problems):
 206     self.problems = problems
 207     Exception.__init__(
 208         self,
 209         'The following paths are not disjoint:\n'
 210         '    %s\n'
 211         % ('\n    '.join([str(problem) for problem in self.problems]),)
 212         )
 213
 214
 215 def verify_paths_disjoint(*paths):
 216   """Verify that all of the paths in the argument list are disjoint.
 217
 218   If any of the paths is nested in another one (i.e., in the sense
 219   that 'a/b/c/d' is nested in 'a/b'), or any two paths are identical,
 220   raise a PathsNotDisjointException containing exceptions detailing
 221   the individual problems."""
 222
 223   def split(path):
 224     if not path:
 225       return []
 226     else:
 227       return path.split('/')
 228
 229   def contains(split_path1, split_path2):
 230     """Return True iff SPLIT_PATH1 contains SPLIT_PATH2."""
 231
 232     return (
 233         len(split_path1) < len(split_path2)
 234         and split_path2[:len(split_path1)] == split_path1
 235         )
 236
 237   paths = [(split(path), path) for path in paths]
 238   # If all overlapping elements are equal, a shorter list is
 239   # considered "less than" a longer one.  Therefore if any paths are
 240   # nested, this sort will leave at least one such pair adjacent, in
 241   # the order [nest,nestling].
 242   paths.sort()
 243
 244   problems = []
 245
 246   # Create exceptions for any repeated paths, and delete the repeats
 247   # from the paths array:
 248   i = 0
 249   while i < len(paths):
 250     split_path, path = paths[i]
 251     j = i + 1
 252     while j < len(paths) and split_path == paths[j][0]:
 253       j += 1
 254     if j - i > 1:
 255       problems.append(PathRepeatedException(path, j - i))
 256       # Delete all but the first copy:
 257       del paths[i + 1:j]
 258     i += 1
 259
 260   # Create exceptions for paths nested in each other:
 261   i = 0
 262   while i < len(paths):
 263     split_path, path = paths[i]
 264     j = i + 1
 265     while j < len(paths) and contains(split_path, paths[j][0]):
 266       j += 1
 267     if j - i > 1:
 268       problems.append(PathsNestedException(
 269           path, [path2 for (split_path2, path2) in paths[i + 1:j]]
 270           ))
 271     i += 1
 272
 273   if problems:
 274     raise PathsNotDisjointException(problems)
 275
 276
 277 def is_trunk_revision(rev):
 278   """Return True iff REV is a trunk revision.
 279
 280   REV is a CVS revision number (e.g., '1.6' or '1.6.4.5').  Return
 281   True iff the revision is on trunk."""
 282
 283   return rev.count('.') == 1
 284
 285
 286 def format_date(date):
 287   """Return an svn-compatible date string for DATE (seconds since epoch).
 288
 289   A Subversion date looks like '2002-09-29T14:44:59.000000Z'."""
 290
 291   return time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", time.gmtime(date))
 292
 293
 294 class CVSTextDecoder:
 295   """Callable that decodes CVS strings into Unicode."""
 296
 297   def __init__(self, encodings, fallback_encoding=None):
 298     """Create a CVSTextDecoder instance.
 299
 300     ENCODINGS is a list containing the names of encodings that are
 301     attempted to be used as source encodings in 'strict' mode.
 302
 303     FALLBACK_ENCODING, if specified, is the name of an encoding that
 304     should be used as a source encoding in lossy 'replace' mode if all
 305     of ENCODINGS failed.
 306
 307     Raise LookupError if any of the specified encodings is unknown."""
 308
 309     self.decoders = [
 310         (encoding, codecs.lookup(encoding)[1])
 311         for encoding in encodings]
 312
 313     if fallback_encoding is None:
 314       self.fallback_decoder = None
 315     else:
 316       self.fallback_decoder = (
 317           fallback_encoding, codecs.lookup(fallback_encoding)[1]
 318           )
 319
 320   def add_encoding(self, encoding):
 321     """Add an encoding to be tried in 'strict' mode.
 322
 323     ENCODING is the name of an encoding.  If it is unknown, raise a
 324     LookupError."""
 325
 326     for (name, decoder) in self.decoders:
 327       if name == encoding:
 328         return
 329     else:
 330       self.decoders.append( (encoding, codecs.lookup(encoding)[1]) )
 331
 332   def set_fallback_encoding(self, encoding):
 333     """Set the fallback encoding, to be tried in 'replace' mode.
 334
 335     ENCODING is the name of an encoding.  If it is unknown, raise a
 336     LookupError."""
 337
 338     if encoding is None:
 339       self.fallback_decoder = None
 340     else:
 341       self.fallback_decoder = (encoding, codecs.lookup(encoding)[1])
 342
 343   def __call__(self, s):
 344     """Try to decode string S using our configured source encodings.
 345
 346     Return the string as a Unicode string.  If S is already a unicode
 347     string, do nothing.
 348
 349     Raise UnicodeError if the string cannot be decoded using any of
 350     the source encodings and no fallback encoding was specified."""
 351
 352     if isinstance(s, unicode):
 353       return s
 354     for (name, decoder) in self.decoders:
 355       try:
 356         return decoder(s)[0]
 357       except ValueError:
 358         Log().verbose("Encoding '%s' failed for string %r" % (name, s))
 359
 360     if self.fallback_decoder is not None:
 361       (name, decoder) = self.fallback_decoder
 362       return decoder(s, 'replace')[0]
 363     else:
 364       raise UnicodeError
 365
 366
 367 class Timestamper:
 368   """Return monotonic timestamps derived from changeset timestamps."""
 369
 370   def __init__(self):
 371     # The last timestamp that has been returned:
 372     self.timestamp = 0.0
 373
 374     # The maximum timestamp that is considered reasonable:
 375     self.max_timestamp = time.time() + 24.0 * 60.0 * 60.0
 376
 377   def get(self, timestamp, change_expected):
 378     """Return a reasonable timestamp derived from TIMESTAMP.
 379
 380     Push TIMESTAMP into the future if necessary to ensure that it is
 381     at least one second later than every other timestamp that has been
 382     returned by previous calls to this method.
 383
 384     If CHANGE_EXPECTED is not True, then log a message if the
 385     timestamp has to be changed."""
 386
 387     if timestamp > self.max_timestamp:
 388       # If a timestamp is in the future, it is assumed that it is
 389       # bogus.  Shift it backwards in time to prevent it forcing other
 390       # timestamps to be pushed even further in the future.
 391
 392       # Note that this is not nearly a complete solution to the bogus
 393       # timestamp problem.  A timestamp in the future still affects
 394       # the ordering of changesets, and a changeset having such a
 395       # timestamp will not be committed until all changesets with
 396       # earlier timestamps have been committed, even if other
 397       # changesets with even earlier timestamps depend on this one.
 398       self.timestamp = self.timestamp + 1.0
 399       if not change_expected:
 400         Log().warn(
 401             'Timestamp "%s" is in the future; changed to "%s".'
 402             % (time.asctime(time.gmtime(timestamp)),
 403                time.asctime(time.gmtime(self.timestamp)),)
 404             )
 405     elif timestamp < self.timestamp + 1.0:
 406       self.timestamp = self.timestamp + 1.0
 407       if not change_expected and Log().is_on(Log.VERBOSE):
 408         Log().verbose(
 409             'Timestamp "%s" adjusted to "%s" to ensure monotonicity.'
 410             % (time.asctime(time.gmtime(timestamp)),
 411                time.asctime(time.gmtime(self.timestamp)),)
 412             )
 413     else:
 414       self.timestamp = timestamp
 415
 416     return self.timestamp
 417
 418