cvs2svn_lib/common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains common facilities used by cvs2svn."""
  18
  19
  20 import re
  21 import time
  22 import codecs
  23
  24 from cvs2svn_lib.log import Log
  25
  26
  27 # Always use these constants for opening databases.
  28 DB_OPEN_READ = 'r'
  29 DB_OPEN_WRITE = 'w'
  30 DB_OPEN_NEW = 'n'
  31
  32
  33 SVN_INVALID_REVNUM = -1
  34
  35
  36 # Warnings and errors start with these strings.  They are typically
  37 # followed by a colon and a space, as in "%s: " ==> "WARNING: ".
  38 warning_prefix = "WARNING"
  39 error_prefix = "ERROR"
  40
  41
  42 class FatalException(Exception):
  43   """Exception thrown on a non-recoverable error.
  44
  45   If this exception is thrown by main(), it is caught by the global
  46   layer of the program, its string representation is printed (followed
  47   by a newline), and the program is ended with an exit code of 1."""
  48
  49   pass
  50
  51
  52 class InternalError(Exception):
  53   """Exception thrown in the case of a cvs2svn internal error (aka, bug)."""
  54
  55   pass
  56
  57
  58 class FatalError(FatalException):
  59   """A FatalException that prepends error_prefix to the message."""
  60
  61   def __init__(self, msg):
  62     """Use (error_prefix + ': ' + MSG) as the error message."""
  63
  64     FatalException.__init__(self, '%s: %s' % (error_prefix, msg,))
  65
  66
  67 class CommandError(FatalError):
  68   """A FatalError caused by a failed command invocation.
  69
  70   The error message includes the command name, exit code, and output."""
  71
  72   def __init__(self, command, exit_status, error_output=''):
  73     self.command = command
  74     self.exit_status = exit_status
  75     self.error_output = error_output
  76     if error_output.rstrip():
  77       FatalError.__init__(
  78           self,
  79           'The command %r failed with exit status=%s\n'
  80           'and the following output:\n'
  81           '%s'
  82           % (self.command, self.exit_status, self.error_output.rstrip()))
  83     else:
  84       FatalError.__init__(
  85           self,
  86           'The command %r failed with exit status=%s and no output'
  87           % (self.command, self.exit_status))
  88
  89
  90 def path_join(*components):
  91   """Join two or more pathname COMPONENTS, inserting '/' as needed.
  92   Empty component are skipped."""
  93
  94   return '/'.join(filter(None, components))
  95
  96
  97 def path_split(path):
  98   """Split the svn pathname PATH into a pair, (HEAD, TAIL).
  99
 100   This is similar to os.path.split(), but always uses '/' as path
 101   separator.  PATH is an svn path, which should not start with a '/'.
 102   HEAD is everything before the last slash, and TAIL is everything
 103   after.  If PATH ends in a slash, TAIL will be empty.  If there is no
 104   slash in PATH, HEAD will be empty.  If PATH is empty, both HEAD and
 105   TAIL are empty."""
 106
 107   pos = path.rfind('/')
 108   if pos == -1:
 109     return ('', path,)
 110   else:
 111     return (path[:pos], path[pos+1:],)
 112
 113
 114 class IllegalSVNPathError(FatalException):
 115   pass
 116
 117
 118 # Control characters (characters not allowed in Subversion filenames):
 119 ctrl_characters_regexp = re.compile('[\\\x00-\\\x1f\\\x7f]')
 120
 121
 122 def verify_svn_filename_legal(filename):
 123   """Verify that FILENAME is a legal filename.
 124
 125   FILENAME is a path component of a CVS path.  Check that it won't
 126   choke SVN:
 127
 128   - Check that it is not empty.
 129
 130   - Check that it is not equal to '.' or '..'.
 131
 132   - Check that the filename does not include any control characters.
 133
 134   If any of these tests fail, raise an IllegalSVNPathError."""
 135
 136   if filename == '':
 137     raise IllegalSVNPathError("Empty filename component.")
 138
 139   if filename in ['.', '..']:
 140     raise IllegalSVNPathError("Illegal filename component %r." % (filename,))
 141
 142   m = ctrl_characters_regexp.search(filename)
 143   if m:
 144     raise IllegalSVNPathError(
 145         "Character %r in filename %r is not supported by Subversion."
 146         % (m.group(), filename,)
 147         )
 148
 149
 150 def verify_svn_path_legal(path):
 151   """Verify that PATH is a legitimate SVN path.
 152
 153   If not, raise an IllegalSVNPathError."""
 154
 155   if path.startswith('/'):
 156     raise IllegalSVNPathError("Path %r must not start with '/'." % (path,))
 157   head = path
 158   while head != '':
 159     (head,tail) = path_split(head)
 160     try:
 161         verify_svn_filename_legal(tail)
 162     except IllegalSVNPathError, e:
 163         raise IllegalSVNPathError('Problem with path %r: %s' % (path, e,))
 164
 165
 166 def normalize_svn_path(path, allow_empty=False):
 167   """Normalize an SVN path (e.g., one supplied by a user).
 168
 169   1. Strip leading, trailing, and duplicated '/'.
 170   2. If ALLOW_EMPTY is not set, verify that PATH is not empty.
 171
 172   Return the normalized path.
 173
 174   If the path is invalid, raise an IllegalSVNPathError."""
 175
 176   norm_path = path_join(*path.split('/'))
 177   if not allow_empty and not norm_path:
 178     raise IllegalSVNPathError("Path is empty")
 179   return norm_path
 180
 181
 182 class PathRepeatedException(Exception):
 183   def __init__(self, path, count):
 184     self.path = path
 185     self.count = count
 186     Exception.__init__(
 187         self, 'Path %s is repeated %d times' % (self.path, self.count,)
 188         )
 189
 190
 191 class PathsNestedException(Exception):
 192   def __init__(self, nest, nestlings):
 193     self.nest = nest
 194     self.nestlings = nestlings
 195     Exception.__init__(
 196         self,
 197         'Path %s contains the following other paths: %s'
 198         % (self.nest, ', '.join(self.nestlings),)
 199         )
 200
 201
 202 class PathsNotDisjointException(FatalException):
 203   """An exception that collects multiple other disjointness exceptions."""
 204
 205   def __init__(self, problems):
 206     self.problems = problems
 207     Exception.__init__(
 208         self,
 209         'The following paths are not disjoint:\n'
 210         '    %s\n'
 211         % ('\n    '.join([str(problem) for problem in self.problems]),)
 212         )
 213
 214
 215 def verify_paths_disjoint(*paths):
 216   """Verify that all of the paths in the argument list are disjoint.
 217
 218   If any of the paths is nested in another one (i.e., in the sense
 219   that 'a/b/c/d' is nested in 'a/b'), or any two paths are identical,
 220   raise a PathsNotDisjointException containing exceptions detailing
 221   the individual problems."""
 222
 223   def split(path):
 224     if not path:
 225       return []
 226     else:
 227       return path.split('/')
 228
 229   def contains(split_path1, split_path2):
 230     """Return True iff SPLIT_PATH1 contains SPLIT_PATH2."""
 231
 232     return (
 233         len(split_path1) < len(split_path2)
 234         and split_path2[:len(split_path1)] == split_path1
 235         )
 236
 237   paths = [(split(path), path) for path in paths]
 238   # If all overlapping elements are equal, a shorter list is
 239   # considered "less than" a longer one.  Therefore if any paths are
 240   # nested, this sort will leave at least one such pair adjacent, in
 241   # the order [nest,nestling].
 242   paths.sort()
 243
 244   problems = []
 245
 246   # Create exceptions for any repeated paths, and delete the repeats
 247   # from the paths array:
 248   i = 0
 249   while i < len(paths):
 250     split_path, path = paths[i]
 251     j = i + 1
 252     while j < len(paths) and split_path == paths[j][0]:
 253       j += 1
 254     if j - i > 1:
 255       problems.append(PathRepeatedException(path, j - i))
 256       # Delete all but the first copy:
 257       del paths[i + 1:j]
 258     i += 1
 259
 260   # Create exceptions for paths nested in each other:
 261   i = 0
 262   while i < len(paths):
 263     split_path, path = paths[i]
 264     j = i + 1
 265     while j < len(paths) and contains(split_path, paths[j][0]):
 266       j += 1
 267     if j - i > 1:
 268       problems.append(PathsNestedException(
 269           path, [path2 for (split_path2, path2) in paths[i + 1:j]]
 270           ))
 271     i += 1
 272
 273   if problems:
 274     raise PathsNotDisjointException(problems)
 275
 276
 277 def is_trunk_revision(rev):
 278   """Return True iff REV is a trunk revision.
 279
 280   REV is a CVS revision number (e.g., '1.6' or '1.6.4.5').  Return
 281   True iff the revision is on trunk."""
 282
 283   return rev.count('.') == 1
 284
 285
 286 def is_branch_revision_number(rev):
 287   """Return True iff REV is a branch revision number.
 288
 289   REV is a CVS revision number in canonical form (i.e., with zeros
 290   removed).  Return True iff it refers to a whole branch, as opposed
 291   to a single revision."""
 292
 293   return rev.count('.') % 2 == 0
 294
 295
 296 def format_date(date):
 297   """Return an svn-compatible date string for DATE (seconds since epoch).
 298
 299   A Subversion date looks like '2002-09-29T14:44:59.000000Z'."""
 300
 301   return time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", time.gmtime(date))
 302
 303
 304 class CVSTextDecoder:
 305   """Callable that decodes CVS strings into Unicode."""
 306
 307   def __init__(self, encodings, fallback_encoding=None):
 308     """Create a CVSTextDecoder instance.
 309
 310     ENCODINGS is a list containing the names of encodings that are
 311     attempted to be used as source encodings in 'strict' mode.
 312
 313     FALLBACK_ENCODING, if specified, is the name of an encoding that
 314     should be used as a source encoding in lossy 'replace' mode if all
 315     of ENCODINGS failed.
 316
 317     Raise LookupError if any of the specified encodings is unknown."""
 318
 319     self.decoders = [
 320         (encoding, codecs.lookup(encoding)[1])
 321         for encoding in encodings]
 322
 323     if fallback_encoding is None:
 324       self.fallback_decoder = None
 325     else:
 326       self.fallback_decoder = (
 327           fallback_encoding, codecs.lookup(fallback_encoding)[1]
 328           )
 329
 330   def add_encoding(self, encoding):
 331     """Add an encoding to be tried in 'strict' mode.
 332
 333     ENCODING is the name of an encoding.  If it is unknown, raise a
 334     LookupError."""
 335
 336     for (name, decoder) in self.decoders:
 337       if name == encoding:
 338         return
 339     else:
 340       self.decoders.append( (encoding, codecs.lookup(encoding)[1]) )
 341
 342   def set_fallback_encoding(self, encoding):
 343     """Set the fallback encoding, to be tried in 'replace' mode.
 344
 345     ENCODING is the name of an encoding.  If it is unknown, raise a
 346     LookupError."""
 347
 348     if encoding is None:
 349       self.fallback_decoder = None
 350     else:
 351       self.fallback_decoder = (encoding, codecs.lookup(encoding)[1])
 352
 353   def __call__(self, s):
 354     """Try to decode string S using our configured source encodings.
 355
 356     Return the string as a Unicode string.  If S is already a unicode
 357     string, do nothing.
 358
 359     Raise UnicodeError if the string cannot be decoded using any of
 360     the source encodings and no fallback encoding was specified."""
 361
 362     if isinstance(s, unicode):
 363       return s
 364     for (name, decoder) in self.decoders:
 365       try:
 366         return decoder(s)[0]
 367       except ValueError:
 368         Log().verbose("Encoding '%s' failed for string %r" % (name, s))
 369
 370     if self.fallback_decoder is not None:
 371       (name, decoder) = self.fallback_decoder
 372       return decoder(s, 'replace')[0]
 373     else:
 374       raise UnicodeError
 375
 376
 377 class Timestamper:
 378   """Return monotonic timestamps derived from changeset timestamps."""
 379
 380   def __init__(self):
 381     # The last timestamp that has been returned:
 382     self.timestamp = 0.0
 383
 384     # The maximum timestamp that is considered reasonable:
 385     self.max_timestamp = time.time() + 24.0 * 60.0 * 60.0
 386
 387   def get(self, timestamp, change_expected):
 388     """Return a reasonable timestamp derived from TIMESTAMP.
 389
 390     Push TIMESTAMP into the future if necessary to ensure that it is
 391     at least one second later than every other timestamp that has been
 392     returned by previous calls to this method.
 393
 394     If CHANGE_EXPECTED is not True, then log a message if the
 395     timestamp has to be changed."""
 396
 397     if timestamp > self.max_timestamp:
 398       # If a timestamp is in the future, it is assumed that it is
 399       # bogus.  Shift it backwards in time to prevent it forcing other
 400       # timestamps to be pushed even further in the future.
 401
 402       # Note that this is not nearly a complete solution to the bogus
 403       # timestamp problem.  A timestamp in the future still affects
 404       # the ordering of changesets, and a changeset having such a
 405       # timestamp will not be committed until all changesets with
 406       # earlier timestamps have been committed, even if other
 407       # changesets with even earlier timestamps depend on this one.
 408       self.timestamp = self.timestamp + 1.0
 409       if not change_expected:
 410         Log().warn(
 411             'Timestamp "%s" is in the future; changed to "%s".'
 412             % (time.asctime(time.gmtime(timestamp)),
 413                time.asctime(time.gmtime(self.timestamp)),)
 414             )
 415     elif timestamp < self.timestamp + 1.0:
 416       self.timestamp = self.timestamp + 1.0
 417       if not change_expected and Log().is_on(Log.VERBOSE):
 418         Log().verbose(
 419             'Timestamp "%s" adjusted to "%s" to ensure monotonicity.'
 420             % (time.asctime(time.gmtime(timestamp)),
 421                time.asctime(time.gmtime(self.timestamp)),)
 422             )
 423     else:
 424       self.timestamp = timestamp
 425
 426     return self.timestamp
 427
 428