cvs2svn_lib/common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains common facilities used by cvs2svn."""
  18
  19
  20 import time
  21 import codecs
  22
  23 from cvs2svn_lib.log import logger
  24
  25
  26 # Always use these constants for opening databases.
  27 DB_OPEN_READ = 'r'
  28 DB_OPEN_WRITE = 'w'
  29 DB_OPEN_NEW = 'n'
  30
  31
  32 SVN_INVALID_REVNUM = -1
  33
  34
  35 # Warnings and errors start with these strings.  They are typically
  36 # followed by a colon and a space, as in "%s: " ==> "WARNING: ".
  37 warning_prefix = "WARNING"
  38 error_prefix = "ERROR"
  39
  40
  41 class FatalException(Exception):
  42   """Exception thrown on a non-recoverable error.
  43
  44   If this exception is thrown by main(), it is caught by the global
  45   layer of the program, its string representation is printed (followed
  46   by a newline), and the program is ended with an exit code of 1."""
  47
  48   pass
  49
  50
  51 class InternalError(Exception):
  52   """Exception thrown in the case of a cvs2svn internal error (aka, bug)."""
  53
  54   pass
  55
  56
  57 class FatalError(FatalException):
  58   """A FatalException that prepends error_prefix to the message."""
  59
  60   def __init__(self, msg):
  61     """Use (error_prefix + ': ' + MSG) as the error message."""
  62
  63     FatalException.__init__(self, '%s: %s' % (error_prefix, msg,))
  64
  65
  66 class CommandError(FatalError):
  67   """A FatalError caused by a failed command invocation.
  68
  69   The error message includes the command name, exit code, and output."""
  70
  71   def __init__(self, command, exit_status, error_output=''):
  72     self.command = command
  73     self.exit_status = exit_status
  74     self.error_output = error_output
  75     if error_output.rstrip():
  76       FatalError.__init__(
  77           self,
  78           'The command %r failed with exit status=%s\n'
  79           'and the following output:\n'
  80           '%s'
  81           % (self.command, self.exit_status, self.error_output.rstrip()))
  82     else:
  83       FatalError.__init__(
  84           self,
  85           'The command %r failed with exit status=%s and no output'
  86           % (self.command, self.exit_status))
  87
  88
  89 def canonicalize_eol(text, eol):
  90   """Replace any end-of-line sequences in TEXT with the string EOL."""
  91
  92   text = text.replace('\r\n', '\n')
  93   text = text.replace('\r', '\n')
  94   if eol != '\n':
  95     text = text.replace('\n', eol)
  96   return text
  97
  98
  99 def path_join(*components):
 100   """Join two or more pathname COMPONENTS, inserting '/' as needed.
 101   Empty component are skipped."""
 102
 103   return '/'.join(filter(None, components))
 104
 105
 106 def path_split(path):
 107   """Split the svn pathname PATH into a pair, (HEAD, TAIL).
 108
 109   This is similar to os.path.split(), but always uses '/' as path
 110   separator.  PATH is an svn path, which should not start with a '/'.
 111   HEAD is everything before the last slash, and TAIL is everything
 112   after.  If PATH ends in a slash, TAIL will be empty.  If there is no
 113   slash in PATH, HEAD will be empty.  If PATH is empty, both HEAD and
 114   TAIL are empty."""
 115
 116   pos = path.rfind('/')
 117   if pos == -1:
 118     return ('', path,)
 119   else:
 120     return (path[:pos], path[pos+1:],)
 121
 122
 123 class IllegalSVNPathError(FatalException):
 124   pass
 125
 126
 127 def normalize_svn_path(path, allow_empty=False):
 128   """Normalize an SVN path (e.g., one supplied by a user).
 129
 130   1. Strip leading, trailing, and duplicated '/'.
 131   2. If ALLOW_EMPTY is not set, verify that PATH is not empty.
 132
 133   Return the normalized path.
 134
 135   If the path is invalid, raise an IllegalSVNPathError."""
 136
 137   norm_path = path_join(*path.split('/'))
 138   if not allow_empty and not norm_path:
 139     raise IllegalSVNPathError("Path is empty")
 140   return norm_path
 141
 142
 143 class PathRepeatedException(Exception):
 144   def __init__(self, path, count):
 145     self.path = path
 146     self.count = count
 147     Exception.__init__(
 148         self, 'Path %s is repeated %d times' % (self.path, self.count,)
 149         )
 150
 151
 152 class PathsNestedException(Exception):
 153   def __init__(self, nest, nestlings):
 154     self.nest = nest
 155     self.nestlings = nestlings
 156     Exception.__init__(
 157         self,
 158         'Path %s contains the following other paths: %s'
 159         % (self.nest, ', '.join(self.nestlings),)
 160         )
 161
 162
 163 class PathsNotDisjointException(FatalException):
 164   """An exception that collects multiple other disjointness exceptions."""
 165
 166   def __init__(self, problems):
 167     self.problems = problems
 168     Exception.__init__(
 169         self,
 170         'The following paths are not disjoint:\n'
 171         '    %s\n'
 172         % ('\n    '.join([str(problem) for problem in self.problems]),)
 173         )
 174
 175
 176 def verify_paths_disjoint(*paths):
 177   """Verify that all of the paths in the argument list are disjoint.
 178
 179   If any of the paths is nested in another one (i.e., in the sense
 180   that 'a/b/c/d' is nested in 'a/b'), or any two paths are identical,
 181   raise a PathsNotDisjointException containing exceptions detailing
 182   the individual problems."""
 183
 184   def split(path):
 185     if not path:
 186       return []
 187     else:
 188       return path.split('/')
 189
 190   def contains(split_path1, split_path2):
 191     """Return True iff SPLIT_PATH1 contains SPLIT_PATH2."""
 192
 193     return (
 194         len(split_path1) < len(split_path2)
 195         and split_path2[:len(split_path1)] == split_path1
 196         )
 197
 198   paths = [(split(path), path) for path in paths]
 199   # If all overlapping elements are equal, a shorter list is
 200   # considered "less than" a longer one.  Therefore if any paths are
 201   # nested, this sort will leave at least one such pair adjacent, in
 202   # the order [nest,nestling].
 203   paths.sort()
 204
 205   problems = []
 206
 207   # Create exceptions for any repeated paths, and delete the repeats
 208   # from the paths array:
 209   i = 0
 210   while i < len(paths):
 211     split_path, path = paths[i]
 212     j = i + 1
 213     while j < len(paths) and split_path == paths[j][0]:
 214       j += 1
 215     if j - i > 1:
 216       problems.append(PathRepeatedException(path, j - i))
 217       # Delete all but the first copy:
 218       del paths[i + 1:j]
 219     i += 1
 220
 221   # Create exceptions for paths nested in each other:
 222   i = 0
 223   while i < len(paths):
 224     split_path, path = paths[i]
 225     j = i + 1
 226     while j < len(paths) and contains(split_path, paths[j][0]):
 227       j += 1
 228     if j - i > 1:
 229       problems.append(PathsNestedException(
 230           path, [path2 for (split_path2, path2) in paths[i + 1:j]]
 231           ))
 232     i += 1
 233
 234   if problems:
 235     raise PathsNotDisjointException(problems)
 236
 237
 238 def is_trunk_revision(rev):
 239   """Return True iff REV is a trunk revision.
 240
 241   REV is a CVS revision number (e.g., '1.6' or '1.6.4.5').  Return
 242   True iff the revision is on trunk."""
 243
 244   return rev.count('.') == 1
 245
 246
 247 def is_branch_revision_number(rev):
 248   """Return True iff REV is a branch revision number.
 249
 250   REV is a CVS revision number in canonical form (i.e., with zeros
 251   removed).  Return True iff it refers to a whole branch, as opposed
 252   to a single revision."""
 253
 254   return rev.count('.') % 2 == 0
 255
 256
 257 def format_date(date):
 258   """Return an svn-compatible date string for DATE (seconds since epoch).
 259
 260   A Subversion date looks like '2002-09-29T14:44:59.000000Z'."""
 261
 262   return time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", time.gmtime(date))
 263
 264
 265 class CVSTextDecoder:
 266   """Callable that decodes CVS strings into Unicode."""
 267
 268   def __init__(self, encodings, fallback_encoding=None, eol_fix=None):
 269     """Create a CVSTextDecoder instance.
 270
 271     ENCODINGS is a list containing the names of encodings that are
 272     attempted to be used as source encodings in 'strict' mode.
 273
 274     FALLBACK_ENCODING, if specified, is the name of an encoding that
 275     should be used as a source encoding in lossy 'replace' mode if all
 276     of ENCODINGS failed.
 277
 278     EOL_FIX is the string to which all EOL sequences should be
 279     converted.  If it is set to None, then EOL sequences are left
 280     unchanged.
 281
 282     Raise LookupError if any of the specified encodings is unknown."""
 283
 284     self.decoders = [
 285         (encoding, codecs.lookup(encoding)[1])
 286         for encoding in encodings]
 287
 288     if fallback_encoding is None:
 289       self.fallback_decoder = None
 290     else:
 291       self.fallback_decoder = (
 292           fallback_encoding, codecs.lookup(fallback_encoding)[1]
 293           )
 294     self.eol_fix = eol_fix
 295
 296   def add_encoding(self, encoding):
 297     """Add an encoding to be tried in 'strict' mode.
 298
 299     ENCODING is the name of an encoding.  If it is unknown, raise a
 300     LookupError."""
 301
 302     for (name, decoder) in self.decoders:
 303       if name == encoding:
 304         return
 305     else:
 306       self.decoders.append( (encoding, codecs.lookup(encoding)[1]) )
 307
 308   def set_fallback_encoding(self, encoding):
 309     """Set the fallback encoding, to be tried in 'replace' mode.
 310
 311     ENCODING is the name of an encoding.  If it is unknown, raise a
 312     LookupError."""
 313
 314     if encoding is None:
 315       self.fallback_decoder = None
 316     else:
 317       self.fallback_decoder = (encoding, codecs.lookup(encoding)[1])
 318
 319   def decode(self, s):
 320     """Try to decode string S using our configured source encodings.
 321
 322     Return the string as a Unicode string.  If S is already a unicode
 323     string, do nothing.
 324
 325     Raise UnicodeError if the string cannot be decoded using any of
 326     the source encodings and no fallback encoding was specified."""
 327
 328     if isinstance(s, unicode):
 329       return s
 330     for (name, decoder) in self.decoders:
 331       try:
 332         return decoder(s)[0]
 333       except ValueError:
 334         logger.verbose("Encoding '%s' failed for string %r" % (name, s))
 335
 336     if self.fallback_decoder is not None:
 337       (name, decoder) = self.fallback_decoder
 338       return decoder(s, 'replace')[0]
 339     else:
 340       raise UnicodeError()
 341
 342   def __call__(self, s):
 343     s = self.decode(s)
 344     if self.eol_fix is not None:
 345       s = canonicalize_eol(s, self.eol_fix)
 346     return s
 347
 348
 349 class Timestamper:
 350   """Return monotonic timestamps derived from changeset timestamps."""
 351
 352   def __init__(self):
 353     # The last timestamp that has been returned:
 354     self.timestamp = 0.0
 355
 356     # The maximum timestamp that is considered reasonable:
 357     self.max_timestamp = time.time() + 24.0 * 60.0 * 60.0
 358
 359   def get(self, timestamp, change_expected):
 360     """Return a reasonable timestamp derived from TIMESTAMP.
 361
 362     Push TIMESTAMP into the future if necessary to ensure that it is
 363     at least one second later than every other timestamp that has been
 364     returned by previous calls to this method.
 365
 366     If CHANGE_EXPECTED is not True, then log a message if the
 367     timestamp has to be changed."""
 368
 369     if timestamp > self.max_timestamp:
 370       # If a timestamp is in the future, it is assumed that it is
 371       # bogus.  Shift it backwards in time to prevent it forcing other
 372       # timestamps to be pushed even further in the future.
 373
 374       # Note that this is not nearly a complete solution to the bogus
 375       # timestamp problem.  A timestamp in the future still affects
 376       # the ordering of changesets, and a changeset having such a
 377       # timestamp will not be committed until all changesets with
 378       # earlier timestamps have been committed, even if other
 379       # changesets with even earlier timestamps depend on this one.
 380       self.timestamp = self.timestamp + 1.0
 381       if not change_expected:
 382         logger.warn(
 383             'Timestamp "%s" is in the future; changed to "%s".'
 384             % (time.asctime(time.gmtime(timestamp)),
 385                time.asctime(time.gmtime(self.timestamp)),)
 386             )
 387     elif timestamp < self.timestamp + 1.0:
 388       self.timestamp = self.timestamp + 1.0
 389       if not change_expected and logger.is_on(logger.VERBOSE):
 390         logger.verbose(
 391             'Timestamp "%s" adjusted to "%s" to ensure monotonicity.'
 392             % (time.asctime(time.gmtime(timestamp)),
 393                time.asctime(time.gmtime(self.timestamp)),)
 394             )
 395     else:
 396       self.timestamp = timestamp
 397
 398     return self.timestamp
 399
 400