cvs2svn_lib/common.py

   1 # (Be in -*- python -*- mode.)
   2 #
   3 # ====================================================================
   4 # Copyright (c) 2000-2009 CollabNet.  All rights reserved.
   5 #
   6 # This software is licensed as described in the file COPYING, which
   7 # you should have received as part of this distribution.  The terms
   8 # are also available at http://subversion.tigris.org/license-1.html.
   9 # If newer versions of this license are posted there, you may use a
  10 # newer version instead, at your option.
  11 #
  12 # This software consists of voluntary contributions made by many
  13 # individuals.  For exact contribution history, see the revision
  14 # history and logs, available at http://cvs2svn.tigris.org/.
  15 # ====================================================================
  16
  17 """This module contains common facilities used by cvs2svn."""
  18
  19
  20 import time
  21 import codecs
  22
  23 from cvs2svn_lib.log import logger
  24
  25
  26 # Always use these constants for opening databases.
  27 DB_OPEN_READ = 'r'
  28 DB_OPEN_WRITE = 'w'
  29 DB_OPEN_NEW = 'n'
  30
  31
  32 SVN_INVALID_REVNUM = -1
  33
  34
  35 # Warnings and errors start with these strings.  They are typically
  36 # followed by a colon and a space, as in "%s: " ==> "WARNING: ".
  37 warning_prefix = "WARNING"
  38 error_prefix = "ERROR"
  39
  40
  41 class FatalException(Exception):
  42   """Exception thrown on a non-recoverable error.
  43
  44   If this exception is thrown by main(), it is caught by the global
  45   layer of the program, its string representation is printed (followed
  46   by a newline), and the program is ended with an exit code of 1."""
  47
  48   pass
  49
  50
  51 class InternalError(Exception):
  52   """Exception thrown in the case of a cvs2svn internal error (aka, bug)."""
  53
  54   pass
  55
  56
  57 class FatalError(FatalException):
  58   """A FatalException that prepends error_prefix to the message."""
  59
  60   def __init__(self, msg):
  61     """Use (error_prefix + ': ' + MSG) as the error message."""
  62
  63     FatalException.__init__(self, '%s: %s' % (error_prefix, msg,))
  64
  65
  66 class CommandError(FatalError):
  67   """A FatalError caused by a failed command invocation.
  68
  69   The error message includes the command name, exit code, and output."""
  70
  71   def __init__(self, command, exit_status, error_output=''):
  72     self.command = command
  73     self.exit_status = exit_status
  74     self.error_output = error_output
  75     if error_output.rstrip():
  76       FatalError.__init__(
  77           self,
  78           'The command %r failed with exit status=%s\n'
  79           'and the following output:\n'
  80           '%s'
  81           % (self.command, self.exit_status, self.error_output.rstrip()))
  82     else:
  83       FatalError.__init__(
  84           self,
  85           'The command %r failed with exit status=%s and no output'
  86           % (self.command, self.exit_status))
  87
  88
  89 def canonicalize_eol(text, eol):
  90   """Replace any end-of-line sequences in TEXT with the string EOL."""
  91
  92   text = text.replace('\r\n', '\n')
  93   text = text.replace('\r', '\n')
  94   if eol != '\n':
  95     text = text.replace('\n', eol)
  96   return text
  97
  98
  99 def path_join(*components):
 100   """Join two or more pathname COMPONENTS, inserting '/' as needed.
 101   Empty component are skipped."""
 102
 103   return '/'.join(filter(None, components))
 104
 105
 106 def path_split(path):
 107   """Split the svn pathname PATH into a pair, (HEAD, TAIL).
 108
 109   This is similar to os.path.split(), but always uses '/' as path
 110   separator.  PATH is an svn path, which should not start with a '/'.
 111   HEAD is everything before the last slash, and TAIL is everything
 112   after.  If PATH ends in a slash, TAIL will be empty.  If there is no
 113   slash in PATH, HEAD will be empty.  If PATH is empty, both HEAD and
 114   TAIL are empty."""
 115
 116   pos = path.rfind('/')
 117   if pos == -1:
 118     return ('', path,)
 119   else:
 120     return (path[:pos], path[pos+1:],)
 121
 122
 123 class IllegalSVNPathError(FatalException):
 124   pass
 125
 126
 127 def normalize_svn_path(path, allow_empty=False):
 128   """Normalize an SVN path (e.g., one supplied by a user).
 129
 130   1. Strip leading, trailing, and duplicated '/'.
 131   2. If ALLOW_EMPTY is not set, verify that PATH is not empty.
 132
 133   Return the normalized path.
 134
 135   If the path is invalid, raise an IllegalSVNPathError."""
 136
 137   norm_path = path_join(*path.split('/'))
 138   if not allow_empty and not norm_path:
 139     raise IllegalSVNPathError("Path is empty")
 140   return norm_path
 141
 142
 143 class PathRepeatedException(Exception):
 144   def __init__(self, path, count):
 145     self.path = path
 146     self.count = count
 147     Exception.__init__(
 148         self, 'Path %s is repeated %d times' % (self.path, self.count,)
 149         )
 150
 151
 152 class PathsNestedException(Exception):
 153   def __init__(self, nest, nestlings):
 154     self.nest = nest
 155     self.nestlings = nestlings
 156     Exception.__init__(
 157         self,
 158         'Path %s contains the following other paths: %s'
 159         % (self.nest, ', '.join(self.nestlings),)
 160         )
 161
 162
 163 class PathsNotDisjointException(FatalException):
 164   """An exception that collects multiple other disjointness exceptions."""
 165
 166   def __init__(self, problems):
 167     self.problems = problems
 168     Exception.__init__(
 169         self,
 170         'The following paths are not disjoint:\n'
 171         '    %s\n'
 172         % ('\n    '.join([str(problem) for problem in self.problems]),)
 173         )
 174
 175
 176 def verify_paths_disjoint(*paths):
 177   """Verify that all of the paths in the argument list are disjoint.
 178
 179   If any of the paths is nested in another one (i.e., in the sense
 180   that 'a/b/c/d' is nested in 'a/b'), or any two paths are identical,
 181   raise a PathsNotDisjointException containing exceptions detailing
 182   the individual problems."""
 183
 184   def split(path):
 185     if not path:
 186       return []
 187     else:
 188       return path.split('/')
 189
 190   def contains(split_path1, split_path2):
 191     """Return True iff SPLIT_PATH1 contains SPLIT_PATH2."""
 192
 193     return (
 194         len(split_path1) < len(split_path2)
 195         and split_path2[:len(split_path1)] == split_path1
 196         )
 197
 198   paths = [(split(path), path) for path in paths]
 199   # If all overlapping elements are equal, a shorter list is
 200   # considered "less than" a longer one.  Therefore if any paths are
 201   # nested, this sort will leave at least one such pair adjacent, in
 202   # the order [nest,nestling].
 203   paths.sort()
 204
 205   problems = []
 206
 207   # Create exceptions for any repeated paths, and delete the repeats
 208   # from the paths array:
 209   i = 0
 210   while i < len(paths):
 211     split_path, path = paths[i]
 212     j = i + 1
 213     while j < len(paths) and split_path == paths[j][0]:
 214       j += 1
 215     if j - i > 1:
 216       problems.append(PathRepeatedException(path, j - i))
 217       # Delete all but the first copy:
 218       del paths[i + 1:j]
 219     i += 1
 220
 221   # Create exceptions for paths nested in each other:
 222   i = 0
 223   while i < len(paths):
 224     split_path, path = paths[i]
 225     j = i + 1
 226     while j < len(paths) and contains(split_path, paths[j][0]):
 227       j += 1
 228     if j - i > 1:
 229       problems.append(PathsNestedException(
 230           path, [path2 for (split_path2, path2) in paths[i + 1:j]]
 231           ))
 232     i += 1
 233
 234   if problems:
 235     raise PathsNotDisjointException(problems)
 236
 237
 238 def is_trunk_revision(rev):
 239   """Return True iff REV is a trunk revision.
 240
 241   REV is a CVS revision number (e.g., '1.6' or '1.6.4.5').  Return
 242   True iff the revision is on trunk."""
 243
 244   return rev.count('.') == 1
 245
 246
 247 def is_branch_revision_number(rev):
 248   """Return True iff REV is a branch revision number.
 249
 250   REV is a CVS revision number in canonical form (i.e., with zeros
 251   removed).  Return True iff it refers to a whole branch, as opposed
 252   to a single revision."""
 253
 254   return rev.count('.') % 2 == 0
 255
 256
 257 def format_date(date):
 258   """Return an svn-compatible date string for DATE (seconds since epoch).
 259
 260   A Subversion date looks like '2002-09-29T14:44:59.000000Z'."""
 261
 262   return time.strftime("%Y-%m-%dT%H:%M:%S.000000Z", time.gmtime(date))
 263
 264
 265 class CVSTextDecoder:
 266   """Callable that decodes CVS strings into Unicode.
 267
 268   Members:
 269
 270     decoders -- a list [(name, CodecInfo.decode), ...] containing the
 271         names and decoders that will be used in 'strict' mode to
 272         attempt to decode inputs.
 273
 274     fallback_decoder -- a tuple (name, CodecInfo.decode) containing
 275         the name and decoder that will be used in 'replace' mode if
 276         none of the decoders in DECODERS succeeds.  If no
 277         fallback_decoder has been specified, this member contains
 278         None.
 279
 280     eol_fix -- a string to which all EOL sequences will be converted,
 281         or None if they should be left unchanged.
 282
 283   """
 284
 285   def __init__(self, encodings, fallback_encoding=None, eol_fix=None):
 286     """Create a CVSTextDecoder instance.
 287
 288     ENCODINGS is a list containing the names of encodings that are
 289     attempted to be used as source encodings in 'strict' mode.
 290
 291     FALLBACK_ENCODING, if specified, is the name of an encoding that
 292     should be used as a source encoding in lossy 'replace' mode if all
 293     of ENCODINGS failed.
 294
 295     EOL_FIX is the string to which all EOL sequences should be
 296     converted.  If it is set to None, then EOL sequences are left
 297     unchanged.
 298
 299     Raise LookupError if any of the specified encodings is unknown."""
 300
 301     self.decoders = []
 302
 303     for encoding in encodings:
 304       self.add_encoding(encoding)
 305
 306     self.set_fallback_encoding(fallback_encoding)
 307     self.eol_fix = eol_fix
 308
 309   def add_encoding(self, encoding):
 310     """Add an encoding to be tried in 'strict' mode.
 311
 312     ENCODING is the name of an encoding.  If it is unknown, raise a
 313     LookupError."""
 314
 315     for (name, decoder) in self.decoders:
 316       if name == encoding:
 317         return
 318     else:
 319       self.decoders.append( (encoding, codecs.lookup(encoding)[1]) )
 320
 321   def set_fallback_encoding(self, encoding):
 322     """Set the fallback encoding, to be tried in 'replace' mode.
 323
 324     ENCODING is the name of an encoding.  If it is unknown, raise a
 325     LookupError."""
 326
 327     if encoding is None:
 328       self.fallback_decoder = None
 329     else:
 330       self.fallback_decoder = (encoding, codecs.lookup(encoding)[1])
 331
 332   def decode(self, s):
 333     """Try to decode string S using our configured source encodings.
 334
 335     Return the string as a Unicode string.  If S is already a unicode
 336     string, do nothing.
 337
 338     Raise UnicodeError if the string cannot be decoded using any of
 339     the source encodings and no fallback encoding was specified."""
 340
 341     if isinstance(s, unicode):
 342       return s
 343     for (name, decoder) in self.decoders:
 344       try:
 345         return decoder(s)[0]
 346       except ValueError:
 347         logger.verbose("Encoding '%s' failed for string %r" % (name, s))
 348
 349     if self.fallback_decoder is not None:
 350       (name, decoder) = self.fallback_decoder
 351       return decoder(s, 'replace')[0]
 352     else:
 353       raise UnicodeError()
 354
 355   def __call__(self, s):
 356     s = self.decode(s)
 357     if self.eol_fix is not None:
 358       s = canonicalize_eol(s, self.eol_fix)
 359     return s
 360
 361   def decode_path(self, path):
 362     """Try to decode PATH using our configured source encodings.
 363
 364     Decode each path component separately (as they may each use
 365     different encodings)."""
 366
 367     return u'/'.join([
 368         self.decode(piece)
 369         for piece in path.split('/')
 370         ])
 371
 372
 373 class Timestamper:
 374   """Return monotonic timestamps derived from changeset timestamps."""
 375
 376   def __init__(self):
 377     # The last timestamp that has been returned:
 378     self.timestamp = 0.0
 379
 380     # The maximum timestamp that is considered reasonable:
 381     self.max_timestamp = time.time() + 24.0 * 60.0 * 60.0
 382
 383   def get(self, timestamp, change_expected):
 384     """Return a reasonable timestamp derived from TIMESTAMP.
 385
 386     Push TIMESTAMP into the future if necessary to ensure that it is
 387     at least one second later than every other timestamp that has been
 388     returned by previous calls to this method.
 389
 390     If CHANGE_EXPECTED is not True, then log a message if the
 391     timestamp has to be changed."""
 392
 393     if timestamp > self.max_timestamp:
 394       # If a timestamp is in the future, it is assumed that it is
 395       # bogus.  Shift it backwards in time to prevent it forcing other
 396       # timestamps to be pushed even further in the future.
 397
 398       # Note that this is not nearly a complete solution to the bogus
 399       # timestamp problem.  A timestamp in the future still affects
 400       # the ordering of changesets, and a changeset having such a
 401       # timestamp will not be committed until all changesets with
 402       # earlier timestamps have been committed, even if other
 403       # changesets with even earlier timestamps depend on this one.
 404       self.timestamp = self.timestamp + 1.0
 405       if not change_expected:
 406         logger.warn(
 407             'Timestamp "%s" is in the future; changed to "%s".'
 408             % (time.asctime(time.gmtime(timestamp)),
 409                time.asctime(time.gmtime(self.timestamp)),)
 410             )
 411     elif timestamp < self.timestamp + 1.0:
 412       self.timestamp = self.timestamp + 1.0
 413       if not change_expected and logger.is_on(logger.VERBOSE):
 414         logger.verbose(
 415             'Timestamp "%s" adjusted to "%s" to ensure monotonicity.'
 416             % (time.asctime(time.gmtime(timestamp)),
 417                time.asctime(time.gmtime(self.timestamp)),)
 418             )
 419     else:
 420       self.timestamp = timestamp
 421
 422     return self.timestamp
 423
 424