Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 # Notes for authors of new mailbox subclasses:
   6 #
   7 # Remember to fsync() changes to disk before closing a modified file
   8 # or returning from a flush() method.  See functions _sync_flush() and
   9 # _sync_close().
  10
  11 import sys
  12 import os
  13 import time
  14 import calendar
  15 import socket
  16 import errno
  17 import copy
  18 import email
  19 import email.message
  20 import email.generator
  21 import rfc822
  22 import StringIO
  23 try:
  24     if sys.platform == 'os2emx':
  25         # OS/2 EMX fcntl() not adequate
  26         raise ImportError
  27     import fcntl
  28 except ImportError:
  29     fcntl = None
  30
  31 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  32             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  33             'BabylMessage', 'MMDFMessage', 'UnixMailbox',
  34             'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
  35
  36 class Mailbox:
  37     """A group of messages in a particular place."""
  38
  39     def __init__(self, path, factory=None, create=True):
  40         """Initialize a Mailbox instance."""
  41         self._path = os.path.abspath(os.path.expanduser(path))
  42         self._factory = factory
  43
  44     def add(self, message):
  45         """Add message and return assigned key."""
  46         raise NotImplementedError('Method must be implemented by subclass')
  47
  48     def remove(self, key):
  49         """Remove the keyed message; raise KeyError if it doesn't exist."""
  50         raise NotImplementedError('Method must be implemented by subclass')
  51
  52     def __delitem__(self, key):
  53         self.remove(key)
  54
  55     def discard(self, key):
  56         """If the keyed message exists, remove it."""
  57         try:
  58             self.remove(key)
  59         except KeyError:
  60             pass
  61
  62     def __setitem__(self, key, message):
  63         """Replace the keyed message; raise KeyError if it doesn't exist."""
  64         raise NotImplementedError('Method must be implemented by subclass')
  65
  66     def get(self, key, default=None):
  67         """Return the keyed message, or default if it doesn't exist."""
  68         try:
  69             return self.__getitem__(key)
  70         except KeyError:
  71             return default
  72
  73     def __getitem__(self, key):
  74         """Return the keyed message; raise KeyError if it doesn't exist."""
  75         if not self._factory:
  76             return self.get_message(key)
  77         else:
  78             return self._factory(self.get_file(key))
  79
  80     def get_message(self, key):
  81         """Return a Message representation or raise a KeyError."""
  82         raise NotImplementedError('Method must be implemented by subclass')
  83
  84     def get_string(self, key):
  85         """Return a string representation or raise a KeyError."""
  86         raise NotImplementedError('Method must be implemented by subclass')
  87
  88     def get_file(self, key):
  89         """Return a file-like representation or raise a KeyError."""
  90         raise NotImplementedError('Method must be implemented by subclass')
  91
  92     def iterkeys(self):
  93         """Return an iterator over keys."""
  94         raise NotImplementedError('Method must be implemented by subclass')
  95
  96     def keys(self):
  97         """Return a list of keys."""
  98         return list(self.iterkeys())
  99
 100     def itervalues(self):
 101         """Return an iterator over all messages."""
 102         for key in self.iterkeys():
 103             try:
 104                 value = self[key]
 105             except KeyError:
 106                 continue
 107             yield value
 108
 109     def __iter__(self):
 110         return self.itervalues()
 111
 112     def values(self):
 113         """Return a list of messages. Memory intensive."""
 114         return list(self.itervalues())
 115
 116     def iteritems(self):
 117         """Return an iterator over (key, message) tuples."""
 118         for key in self.iterkeys():
 119             try:
 120                 value = self[key]
 121             except KeyError:
 122                 continue
 123             yield (key, value)
 124
 125     def items(self):
 126         """Return a list of (key, message) tuples. Memory intensive."""
 127         return list(self.iteritems())
 128
 129     def has_key(self, key):
 130         """Return True if the keyed message exists, False otherwise."""
 131         raise NotImplementedError('Method must be implemented by subclass')
 132
 133     def __contains__(self, key):
 134         return self.has_key(key)
 135
 136     def __len__(self):
 137         """Return a count of messages in the mailbox."""
 138         raise NotImplementedError('Method must be implemented by subclass')
 139
 140     def clear(self):
 141         """Delete all messages."""
 142         for key in self.iterkeys():
 143             self.discard(key)
 144
 145     def pop(self, key, default=None):
 146         """Delete the keyed message and return it, or default."""
 147         try:
 148             result = self[key]
 149         except KeyError:
 150             return default
 151         self.discard(key)
 152         return result
 153
 154     def popitem(self):
 155         """Delete an arbitrary (key, message) pair and return it."""
 156         for key in self.iterkeys():
 157             return (key, self.pop(key))     # This is only run once.
 158         else:
 159             raise KeyError('No messages in mailbox')
 160
 161     def update(self, arg=None):
 162         """Change the messages that correspond to certain keys."""
 163         if hasattr(arg, 'iteritems'):
 164             source = arg.iteritems()
 165         elif hasattr(arg, 'items'):
 166             source = arg.items()
 167         else:
 168             source = arg
 169         bad_key = False
 170         for key, message in source:
 171             try:
 172                 self[key] = message
 173             except KeyError:
 174                 bad_key = True
 175         if bad_key:
 176             raise KeyError('No message with key(s)')
 177
 178     def flush(self):
 179         """Write any pending changes to the disk."""
 180         raise NotImplementedError('Method must be implemented by subclass')
 181
 182     def lock(self):
 183         """Lock the mailbox."""
 184         raise NotImplementedError('Method must be implemented by subclass')
 185
 186     def unlock(self):
 187         """Unlock the mailbox if it is locked."""
 188         raise NotImplementedError('Method must be implemented by subclass')
 189
 190     def close(self):
 191         """Flush and close the mailbox."""
 192         raise NotImplementedError('Method must be implemented by subclass')
 193
 194     def _dump_message(self, message, target, mangle_from_=False):
 195         # Most files are opened in binary mode to allow predictable seeking.
 196         # To get native line endings on disk, the user-friendly \n line endings
 197         # used in strings and by email.Message are translated here.
 198         """Dump message contents to target file."""
 199         if isinstance(message, email.message.Message):
 200             buffer = StringIO.StringIO()
 201             gen = email.generator.Generator(buffer, mangle_from_, 0)
 202             gen.flatten(message)
 203             buffer.seek(0)
 204             target.write(buffer.read().replace('\n', os.linesep))
 205         elif isinstance(message, str):
 206             if mangle_from_:
 207                 message = message.replace('\nFrom ', '\n>From ')
 208             message = message.replace('\n', os.linesep)
 209             target.write(message)
 210         elif hasattr(message, 'read'):
 211             while True:
 212                 line = message.readline()
 213                 if line == '':
 214                     break
 215                 if mangle_from_ and line.startswith('From '):
 216                     line = '>From ' + line[5:]
 217                 line = line.replace('\n', os.linesep)
 218                 target.write(line)
 219         else:
 220             raise TypeError('Invalid message type: %s' % type(message))
 221
 222
 223 class Maildir(Mailbox):
 224     """A qmail-style Maildir mailbox."""
 225
 226     colon = ':'
 227
 228     def __init__(self, dirname, factory=rfc822.Message, create=True):
 229         """Initialize a Maildir instance."""
 230         Mailbox.__init__(self, dirname, factory, create)
 231         if not os.path.exists(self._path):
 232             if create:
 233                 os.mkdir(self._path, 0700)
 234                 os.mkdir(os.path.join(self._path, 'tmp'), 0700)
 235                 os.mkdir(os.path.join(self._path, 'new'), 0700)
 236                 os.mkdir(os.path.join(self._path, 'cur'), 0700)
 237             else:
 238                 raise NoSuchMailboxError(self._path)
 239         self._toc = {}
 240         self._last_read = None          # Records last time we read cur/new
 241
 242     def add(self, message):
 243         """Add message and return assigned key."""
 244         tmp_file = self._create_tmp()
 245         try:
 246             self._dump_message(message, tmp_file)
 247         finally:
 248             _sync_close(tmp_file)
 249         if isinstance(message, MaildirMessage):
 250             subdir = message.get_subdir()
 251             suffix = self.colon + message.get_info()
 252             if suffix == self.colon:
 253                 suffix = ''
 254         else:
 255             subdir = 'new'
 256             suffix = ''
 257         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 258         dest = os.path.join(self._path, subdir, uniq + suffix)
 259         try:
 260             if hasattr(os, 'link'):
 261                 os.link(tmp_file.name, dest)
 262                 os.remove(tmp_file.name)
 263             else:
 264                 os.rename(tmp_file.name, dest)
 265         except OSError, e:
 266             os.remove(tmp_file.name)
 267             if e.errno == errno.EEXIST:
 268                 raise ExternalClashError('Name clash with existing message: %s'
 269                                          % dest)
 270             else:
 271                 raise
 272         if isinstance(message, MaildirMessage):
 273             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 274         return uniq
 275
 276     def remove(self, key):
 277         """Remove the keyed message; raise KeyError if it doesn't exist."""
 278         os.remove(os.path.join(self._path, self._lookup(key)))
 279
 280     def discard(self, key):
 281         """If the keyed message exists, remove it."""
 282         # This overrides an inapplicable implementation in the superclass.
 283         try:
 284             self.remove(key)
 285         except KeyError:
 286             pass
 287         except OSError, e:
 288             if e.errno != errno.ENOENT:
 289                 raise
 290
 291     def __setitem__(self, key, message):
 292         """Replace the keyed message; raise KeyError if it doesn't exist."""
 293         old_subpath = self._lookup(key)
 294         temp_key = self.add(message)
 295         temp_subpath = self._lookup(temp_key)
 296         if isinstance(message, MaildirMessage):
 297             # temp's subdir and suffix were specified by message.
 298             dominant_subpath = temp_subpath
 299         else:
 300             # temp's subdir and suffix were defaults from add().
 301             dominant_subpath = old_subpath
 302         subdir = os.path.dirname(dominant_subpath)
 303         if self.colon in dominant_subpath:
 304             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 305         else:
 306             suffix = ''
 307         self.discard(key)
 308         new_path = os.path.join(self._path, subdir, key + suffix)
 309         os.rename(os.path.join(self._path, temp_subpath), new_path)
 310         if isinstance(message, MaildirMessage):
 311             os.utime(new_path, (os.path.getatime(new_path),
 312                                 message.get_date()))
 313
 314     def get_message(self, key):
 315         """Return a Message representation or raise a KeyError."""
 316         subpath = self._lookup(key)
 317         f = open(os.path.join(self._path, subpath), 'r')
 318         try:
 319             if self._factory:
 320                 msg = self._factory(f)
 321             else:
 322                 msg = MaildirMessage(f)
 323         finally:
 324             f.close()
 325         subdir, name = os.path.split(subpath)
 326         msg.set_subdir(subdir)
 327         if self.colon in name:
 328             msg.set_info(name.split(self.colon)[-1])
 329         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 330         return msg
 331
 332     def get_string(self, key):
 333         """Return a string representation or raise a KeyError."""
 334         f = open(os.path.join(self._path, self._lookup(key)), 'r')
 335         try:
 336             return f.read()
 337         finally:
 338             f.close()
 339
 340     def get_file(self, key):
 341         """Return a file-like representation or raise a KeyError."""
 342         f = open(os.path.join(self._path, self._lookup(key)), 'rb')
 343         return _ProxyFile(f)
 344
 345     def iterkeys(self):
 346         """Return an iterator over keys."""
 347         self._refresh()
 348         for key in self._toc:
 349             try:
 350                 self._lookup(key)
 351             except KeyError:
 352                 continue
 353             yield key
 354
 355     def has_key(self, key):
 356         """Return True if the keyed message exists, False otherwise."""
 357         self._refresh()
 358         return key in self._toc
 359
 360     def __len__(self):
 361         """Return a count of messages in the mailbox."""
 362         self._refresh()
 363         return len(self._toc)
 364
 365     def flush(self):
 366         """Write any pending changes to disk."""
 367         return  # Maildir changes are always written immediately.
 368
 369     def lock(self):
 370         """Lock the mailbox."""
 371         return
 372
 373     def unlock(self):
 374         """Unlock the mailbox if it is locked."""
 375         return
 376
 377     def close(self):
 378         """Flush and close the mailbox."""
 379         return
 380
 381     def list_folders(self):
 382         """Return a list of folder names."""
 383         result = []
 384         for entry in os.listdir(self._path):
 385             if len(entry) > 1 and entry[0] == '.' and \
 386                os.path.isdir(os.path.join(self._path, entry)):
 387                 result.append(entry[1:])
 388         return result
 389
 390     def get_folder(self, folder):
 391         """Return a Maildir instance for the named folder."""
 392         return Maildir(os.path.join(self._path, '.' + folder),
 393                        factory=self._factory,
 394                        create=False)
 395
 396     def add_folder(self, folder):
 397         """Create a folder and return a Maildir instance representing it."""
 398         path = os.path.join(self._path, '.' + folder)
 399         result = Maildir(path, factory=self._factory)
 400         maildirfolder_path = os.path.join(path, 'maildirfolder')
 401         if not os.path.exists(maildirfolder_path):
 402             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
 403                 0666))
 404         return result
 405
 406     def remove_folder(self, folder):
 407         """Delete the named folder, which must be empty."""
 408         path = os.path.join(self._path, '.' + folder)
 409         for entry in os.listdir(os.path.join(path, 'new')) + \
 410                      os.listdir(os.path.join(path, 'cur')):
 411             if len(entry) < 1 or entry[0] != '.':
 412                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 413         for entry in os.listdir(path):
 414             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 415                os.path.isdir(os.path.join(path, entry)):
 416                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 417                                     (folder, entry))
 418         for root, dirs, files in os.walk(path, topdown=False):
 419             for entry in files:
 420                 os.remove(os.path.join(root, entry))
 421             for entry in dirs:
 422                 os.rmdir(os.path.join(root, entry))
 423         os.rmdir(path)
 424
 425     def clean(self):
 426         """Delete old files in "tmp"."""
 427         now = time.time()
 428         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 429             path = os.path.join(self._path, 'tmp', entry)
 430             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 431                 os.remove(path)
 432
 433     _count = 1  # This is used to generate unique file names.
 434
 435     def _create_tmp(self):
 436         """Create a file in the tmp subdirectory and open and return it."""
 437         now = time.time()
 438         hostname = socket.gethostname()
 439         if '/' in hostname:
 440             hostname = hostname.replace('/', r'\057')
 441         if ':' in hostname:
 442             hostname = hostname.replace(':', r'\072')
 443         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 444                                     Maildir._count, hostname)
 445         path = os.path.join(self._path, 'tmp', uniq)
 446         try:
 447             os.stat(path)
 448         except OSError, e:
 449             if e.errno == errno.ENOENT:
 450                 Maildir._count += 1
 451                 try:
 452                     return _create_carefully(path)
 453                 except OSError, e:
 454                     if e.errno != errno.EEXIST:
 455                         raise
 456             else:
 457                 raise
 458
 459         # Fall through to here if stat succeeded or open raised EEXIST.
 460         raise ExternalClashError('Name clash prevented file creation: %s' %
 461                                  path)
 462
 463     def _refresh(self):
 464         """Update table of contents mapping."""
 465         new_mtime = os.path.getmtime(os.path.join(self._path, 'new'))
 466         cur_mtime = os.path.getmtime(os.path.join(self._path, 'cur'))
 467
 468         if (self._last_read is not None and
 469             new_mtime <= self._last_read and cur_mtime <= self._last_read):
 470             return
 471
 472         self._toc = {}
 473         def update_dir (subdir):
 474             path = os.path.join(self._path, subdir)
 475             for entry in os.listdir(path):
 476                 p = os.path.join(path, entry)
 477                 if os.path.isdir(p):
 478                     continue
 479                 uniq = entry.split(self.colon)[0]
 480                 self._toc[uniq] = os.path.join(subdir, entry)
 481
 482         update_dir('new')
 483         update_dir('cur')
 484
 485         # We record the current time - 1sec so that, if _refresh() is called
 486         # again in the same second, we will always re-read the mailbox
 487         # just in case it's been modified.  (os.path.mtime() only has
 488         # 1sec resolution.)  This results in a few unnecessary re-reads
 489         # when _refresh() is called multiple times in the same second,
 490         # but once the clock ticks over, we will only re-read as needed.
 491         now = int(time.time() - 1)
 492         self._last_read = time.time() - 1
 493
 494     def _lookup(self, key):
 495         """Use TOC to return subpath for given key, or raise a KeyError."""
 496         try:
 497             if os.path.exists(os.path.join(self._path, self._toc[key])):
 498                 return self._toc[key]
 499         except KeyError:
 500             pass
 501         self._refresh()
 502         try:
 503             return self._toc[key]
 504         except KeyError:
 505             raise KeyError('No message with key: %s' % key)
 506
 507     # This method is for backward compatibility only.
 508     def next(self):
 509         """Return the next message in a one-time iteration."""
 510         if not hasattr(self, '_onetime_keys'):
 511             self._onetime_keys = self.iterkeys()
 512         while True:
 513             try:
 514                 return self[self._onetime_keys.next()]
 515             except StopIteration:
 516                 return None
 517             except KeyError:
 518                 continue
 519
 520
 521 class _singlefileMailbox(Mailbox):
 522     """A single-file mailbox."""
 523
 524     def __init__(self, path, factory=None, create=True):
 525         """Initialize a single-file mailbox."""
 526         Mailbox.__init__(self, path, factory, create)
 527         try:
 528             f = open(self._path, 'rb+')
 529         except IOError, e:
 530             if e.errno == errno.ENOENT:
 531                 if create:
 532                     f = open(self._path, 'wb+')
 533                 else:
 534                     raise NoSuchMailboxError(self._path)
 535             elif e.errno == errno.EACCES:
 536                 f = open(self._path, 'rb')
 537             else:
 538                 raise
 539         self._file = f
 540         self._toc = None
 541         self._next_key = 0
 542         self._pending = False   # No changes require rewriting the file.
 543         self._locked = False
 544         self._file_length = None        # Used to record mailbox size
 545
 546     def add(self, message):
 547         """Add message and return assigned key."""
 548         self._lookup()
 549         self._toc[self._next_key] = self._append_message(message)
 550         self._next_key += 1
 551         self._pending = True
 552         return self._next_key - 1
 553
 554     def remove(self, key):
 555         """Remove the keyed message; raise KeyError if it doesn't exist."""
 556         self._lookup(key)
 557         del self._toc[key]
 558         self._pending = True
 559
 560     def __setitem__(self, key, message):
 561         """Replace the keyed message; raise KeyError if it doesn't exist."""
 562         self._lookup(key)
 563         self._toc[key] = self._append_message(message)
 564         self._pending = True
 565
 566     def iterkeys(self):
 567         """Return an iterator over keys."""
 568         self._lookup()
 569         for key in self._toc.keys():
 570             yield key
 571
 572     def has_key(self, key):
 573         """Return True if the keyed message exists, False otherwise."""
 574         self._lookup()
 575         return key in self._toc
 576
 577     def __len__(self):
 578         """Return a count of messages in the mailbox."""
 579         self._lookup()
 580         return len(self._toc)
 581
 582     def lock(self):
 583         """Lock the mailbox."""
 584         if not self._locked:
 585             _lock_file(self._file)
 586             self._locked = True
 587
 588     def unlock(self):
 589         """Unlock the mailbox if it is locked."""
 590         if self._locked:
 591             _unlock_file(self._file)
 592             self._locked = False
 593
 594     def flush(self):
 595         """Write any pending changes to disk."""
 596         if not self._pending:
 597             return
 598
 599         # In order to be writing anything out at all, self._toc must
 600         # already have been generated (and presumably has been modified
 601         # by adding or deleting an item).
 602         assert self._toc is not None
 603
 604         # Check length of self._file; if it's changed, some other process
 605         # has modified the mailbox since we scanned it.
 606         self._file.seek(0, 2)
 607         cur_len = self._file.tell()
 608         if cur_len != self._file_length:
 609             raise ExternalClashError('Size of mailbox file changed '
 610                                      '(expected %i, found %i)' %
 611                                      (self._file_length, cur_len))
 612
 613         new_file = _create_temporary(self._path)
 614         try:
 615             new_toc = {}
 616             self._pre_mailbox_hook(new_file)
 617             for key in sorted(self._toc.keys()):
 618                 start, stop = self._toc[key]
 619                 self._file.seek(start)
 620                 self._pre_message_hook(new_file)
 621                 new_start = new_file.tell()
 622                 while True:
 623                     buffer = self._file.read(min(4096,
 624                                                  stop - self._file.tell()))
 625                     if buffer == '':
 626                         break
 627                     new_file.write(buffer)
 628                 new_toc[key] = (new_start, new_file.tell())
 629                 self._post_message_hook(new_file)
 630         except:
 631             new_file.close()
 632             os.remove(new_file.name)
 633             raise
 634         _sync_close(new_file)
 635         # self._file is about to get replaced, so no need to sync.
 636         self._file.close()
 637         try:
 638             os.rename(new_file.name, self._path)
 639         except OSError, e:
 640             if e.errno == errno.EEXIST or \
 641               (os.name == 'os2' and e.errno == errno.EACCES):
 642                 os.remove(self._path)
 643                 os.rename(new_file.name, self._path)
 644             else:
 645                 raise
 646         self._file = open(self._path, 'rb+')
 647         self._toc = new_toc
 648         self._pending = False
 649         if self._locked:
 650             _lock_file(self._file, dotlock=False)
 651
 652     def _pre_mailbox_hook(self, f):
 653         """Called before writing the mailbox to file f."""
 654         return
 655
 656     def _pre_message_hook(self, f):
 657         """Called before writing each message to file f."""
 658         return
 659
 660     def _post_message_hook(self, f):
 661         """Called after writing each message to file f."""
 662         return
 663
 664     def close(self):
 665         """Flush and close the mailbox."""
 666         self.flush()
 667         if self._locked:
 668             self.unlock()
 669         self._file.close()  # Sync has been done by self.flush() above.
 670
 671     def _lookup(self, key=None):
 672         """Return (start, stop) or raise KeyError."""
 673         if self._toc is None:
 674             self._generate_toc()
 675         if key is not None:
 676             try:
 677                 return self._toc[key]
 678             except KeyError:
 679                 raise KeyError('No message with key: %s' % key)
 680
 681     def _append_message(self, message):
 682         """Append message to mailbox and return (start, stop) offsets."""
 683         self._file.seek(0, 2)
 684         self._pre_message_hook(self._file)
 685         offsets = self._install_message(message)
 686         self._post_message_hook(self._file)
 687         self._file.flush()
 688         self._file_length = self._file.tell()  # Record current length of mailbox
 689         return offsets
 690
 691
 692
 693 class _mboxMMDF(_singlefileMailbox):
 694     """An mbox or MMDF mailbox."""
 695
 696     _mangle_from_ = True
 697
 698     def get_message(self, key):
 699         """Return a Message representation or raise a KeyError."""
 700         start, stop = self._lookup(key)
 701         self._file.seek(start)
 702         from_line = self._file.readline().replace(os.linesep, '')
 703         string = self._file.read(stop - self._file.tell())
 704         msg = self._message_factory(string.replace(os.linesep, '\n'))
 705         msg.set_from(from_line[5:])
 706         return msg
 707
 708     def get_string(self, key, from_=False):
 709         """Return a string representation or raise a KeyError."""
 710         start, stop = self._lookup(key)
 711         self._file.seek(start)
 712         if not from_:
 713             self._file.readline()
 714         string = self._file.read(stop - self._file.tell())
 715         return string.replace(os.linesep, '\n')
 716
 717     def get_file(self, key, from_=False):
 718         """Return a file-like representation or raise a KeyError."""
 719         start, stop = self._lookup(key)
 720         self._file.seek(start)
 721         if not from_:
 722             self._file.readline()
 723         return _PartialFile(self._file, self._file.tell(), stop)
 724
 725     def _install_message(self, message):
 726         """Format a message and blindly write to self._file."""
 727         from_line = None
 728         if isinstance(message, str) and message.startswith('From '):
 729             newline = message.find('\n')
 730             if newline != -1:
 731                 from_line = message[:newline]
 732                 message = message[newline + 1:]
 733             else:
 734                 from_line = message
 735                 message = ''
 736         elif isinstance(message, _mboxMMDFMessage):
 737             from_line = 'From ' + message.get_from()
 738         elif isinstance(message, email.message.Message):
 739             from_line = message.get_unixfrom()  # May be None.
 740         if from_line is None:
 741             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 742         start = self._file.tell()
 743         self._file.write(from_line + os.linesep)
 744         self._dump_message(message, self._file, self._mangle_from_)
 745         stop = self._file.tell()
 746         return (start, stop)
 747
 748
 749 class mbox(_mboxMMDF):
 750     """A classic mbox mailbox."""
 751
 752     _mangle_from_ = True
 753
 754     def __init__(self, path, factory=None, create=True):
 755         """Initialize an mbox mailbox."""
 756         self._message_factory = mboxMessage
 757         _mboxMMDF.__init__(self, path, factory, create)
 758
 759     def _pre_message_hook(self, f):
 760         """Called before writing each message to file f."""
 761         if f.tell() != 0:
 762             f.write(os.linesep)
 763
 764     def _generate_toc(self):
 765         """Generate key-to-(start, stop) table of contents."""
 766         starts, stops = [], []
 767         self._file.seek(0)
 768         while True:
 769             line_pos = self._file.tell()
 770             line = self._file.readline()
 771             if line.startswith('From '):
 772                 if len(stops) < len(starts):
 773                     stops.append(line_pos - len(os.linesep))
 774                 starts.append(line_pos)
 775             elif line == '':
 776                 stops.append(line_pos)
 777                 break
 778         self._toc = dict(enumerate(zip(starts, stops)))
 779         self._next_key = len(self._toc)
 780         self._file_length = self._file.tell()
 781
 782
 783 class MMDF(_mboxMMDF):
 784     """An MMDF mailbox."""
 785
 786     def __init__(self, path, factory=None, create=True):
 787         """Initialize an MMDF mailbox."""
 788         self._message_factory = MMDFMessage
 789         _mboxMMDF.__init__(self, path, factory, create)
 790
 791     def _pre_message_hook(self, f):
 792         """Called before writing each message to file f."""
 793         f.write('\001\001\001\001' + os.linesep)
 794
 795     def _post_message_hook(self, f):
 796         """Called after writing each message to file f."""
 797         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 798
 799     def _generate_toc(self):
 800         """Generate key-to-(start, stop) table of contents."""
 801         starts, stops = [], []
 802         self._file.seek(0)
 803         next_pos = 0
 804         while True:
 805             line_pos = next_pos
 806             line = self._file.readline()
 807             next_pos = self._file.tell()
 808             if line.startswith('\001\001\001\001' + os.linesep):
 809                 starts.append(next_pos)
 810                 while True:
 811                     line_pos = next_pos
 812                     line = self._file.readline()
 813                     next_pos = self._file.tell()
 814                     if line == '\001\001\001\001' + os.linesep:
 815                         stops.append(line_pos - len(os.linesep))
 816                         break
 817                     elif line == '':
 818                         stops.append(line_pos)
 819                         break
 820             elif line == '':
 821                 break
 822         self._toc = dict(enumerate(zip(starts, stops)))
 823         self._next_key = len(self._toc)
 824         self._file.seek(0, 2)
 825         self._file_length = self._file.tell()
 826
 827
 828 class MH(Mailbox):
 829     """An MH mailbox."""
 830
 831     def __init__(self, path, factory=None, create=True):
 832         """Initialize an MH instance."""
 833         Mailbox.__init__(self, path, factory, create)
 834         if not os.path.exists(self._path):
 835             if create:
 836                 os.mkdir(self._path, 0700)
 837                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 838                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
 839             else:
 840                 raise NoSuchMailboxError(self._path)
 841         self._locked = False
 842
 843     def add(self, message):
 844         """Add message and return assigned key."""
 845         keys = self.keys()
 846         if len(keys) == 0:
 847             new_key = 1
 848         else:
 849             new_key = max(keys) + 1
 850         new_path = os.path.join(self._path, str(new_key))
 851         f = _create_carefully(new_path)
 852         try:
 853             if self._locked:
 854                 _lock_file(f)
 855             try:
 856                 self._dump_message(message, f)
 857                 if isinstance(message, MHMessage):
 858                     self._dump_sequences(message, new_key)
 859             finally:
 860                 if self._locked:
 861                     _unlock_file(f)
 862         finally:
 863             _sync_close(f)
 864         return new_key
 865
 866     def remove(self, key):
 867         """Remove the keyed message; raise KeyError if it doesn't exist."""
 868         path = os.path.join(self._path, str(key))
 869         try:
 870             f = open(path, 'rb+')
 871         except IOError, e:
 872             if e.errno == errno.ENOENT:
 873                 raise KeyError('No message with key: %s' % key)
 874             else:
 875                 raise
 876         try:
 877             if self._locked:
 878                 _lock_file(f)
 879             try:
 880                 f.close()
 881                 os.remove(os.path.join(self._path, str(key)))
 882             finally:
 883                 if self._locked:
 884                     _unlock_file(f)
 885         finally:
 886             f.close()
 887
 888     def __setitem__(self, key, message):
 889         """Replace the keyed message; raise KeyError if it doesn't exist."""
 890         path = os.path.join(self._path, str(key))
 891         try:
 892             f = open(path, 'rb+')
 893         except IOError, e:
 894             if e.errno == errno.ENOENT:
 895                 raise KeyError('No message with key: %s' % key)
 896             else:
 897                 raise
 898         try:
 899             if self._locked:
 900                 _lock_file(f)
 901             try:
 902                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 903                 self._dump_message(message, f)
 904                 if isinstance(message, MHMessage):
 905                     self._dump_sequences(message, key)
 906             finally:
 907                 if self._locked:
 908                     _unlock_file(f)
 909         finally:
 910             _sync_close(f)
 911
 912     def get_message(self, key):
 913         """Return a Message representation or raise a KeyError."""
 914         try:
 915             if self._locked:
 916                 f = open(os.path.join(self._path, str(key)), 'r+')
 917             else:
 918                 f = open(os.path.join(self._path, str(key)), 'r')
 919         except IOError, e:
 920             if e.errno == errno.ENOENT:
 921                 raise KeyError('No message with key: %s' % key)
 922             else:
 923                 raise
 924         try:
 925             if self._locked:
 926                 _lock_file(f)
 927             try:
 928                 msg = MHMessage(f)
 929             finally:
 930                 if self._locked:
 931                     _unlock_file(f)
 932         finally:
 933             f.close()
 934         for name, key_list in self.get_sequences().iteritems():
 935             if key in key_list:
 936                 msg.add_sequence(name)
 937         return msg
 938
 939     def get_string(self, key):
 940         """Return a string representation or raise a KeyError."""
 941         try:
 942             if self._locked:
 943                 f = open(os.path.join(self._path, str(key)), 'r+')
 944             else:
 945                 f = open(os.path.join(self._path, str(key)), 'r')
 946         except IOError, e:
 947             if e.errno == errno.ENOENT:
 948                 raise KeyError('No message with key: %s' % key)
 949             else:
 950                 raise
 951         try:
 952             if self._locked:
 953                 _lock_file(f)
 954             try:
 955                 return f.read()
 956             finally:
 957                 if self._locked:
 958                     _unlock_file(f)
 959         finally:
 960             f.close()
 961
 962     def get_file(self, key):
 963         """Return a file-like representation or raise a KeyError."""
 964         try:
 965             f = open(os.path.join(self._path, str(key)), 'rb')
 966         except IOError, e:
 967             if e.errno == errno.ENOENT:
 968                 raise KeyError('No message with key: %s' % key)
 969             else:
 970                 raise
 971         return _ProxyFile(f)
 972
 973     def iterkeys(self):
 974         """Return an iterator over keys."""
 975         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 976                                       if entry.isdigit()))
 977
 978     def has_key(self, key):
 979         """Return True if the keyed message exists, False otherwise."""
 980         return os.path.exists(os.path.join(self._path, str(key)))
 981
 982     def __len__(self):
 983         """Return a count of messages in the mailbox."""
 984         return len(list(self.iterkeys()))
 985
 986     def lock(self):
 987         """Lock the mailbox."""
 988         if not self._locked:
 989             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
 990             _lock_file(self._file)
 991             self._locked = True
 992
 993     def unlock(self):
 994         """Unlock the mailbox if it is locked."""
 995         if self._locked:
 996             _unlock_file(self._file)
 997             _sync_close(self._file)
 998             del self._file
 999             self._locked = False
1000
1001     def flush(self):
1002         """Write any pending changes to the disk."""
1003         return
1004
1005     def close(self):
1006         """Flush and close the mailbox."""
1007         if self._locked:
1008             self.unlock()
1009
1010     def list_folders(self):
1011         """Return a list of folder names."""
1012         result = []
1013         for entry in os.listdir(self._path):
1014             if os.path.isdir(os.path.join(self._path, entry)):
1015                 result.append(entry)
1016         return result
1017
1018     def get_folder(self, folder):
1019         """Return an MH instance for the named folder."""
1020         return MH(os.path.join(self._path, folder),
1021                   factory=self._factory, create=False)
1022
1023     def add_folder(self, folder):
1024         """Create a folder and return an MH instance representing it."""
1025         return MH(os.path.join(self._path, folder),
1026                   factory=self._factory)
1027
1028     def remove_folder(self, folder):
1029         """Delete the named folder, which must be empty."""
1030         path = os.path.join(self._path, folder)
1031         entries = os.listdir(path)
1032         if entries == ['.mh_sequences']:
1033             os.remove(os.path.join(path, '.mh_sequences'))
1034         elif entries == []:
1035             pass
1036         else:
1037             raise NotEmptyError('Folder not empty: %s' % self._path)
1038         os.rmdir(path)
1039
1040     def get_sequences(self):
1041         """Return a name-to-key-list dictionary to define each sequence."""
1042         results = {}
1043         f = open(os.path.join(self._path, '.mh_sequences'), 'r')
1044         try:
1045             all_keys = set(self.keys())
1046             for line in f:
1047                 try:
1048                     name, contents = line.split(':')
1049                     keys = set()
1050                     for spec in contents.split():
1051                         if spec.isdigit():
1052                             keys.add(int(spec))
1053                         else:
1054                             start, stop = (int(x) for x in spec.split('-'))
1055                             keys.update(range(start, stop + 1))
1056                     results[name] = [key for key in sorted(keys) \
1057                                          if key in all_keys]
1058                     if len(results[name]) == 0:
1059                         del results[name]
1060                 except ValueError:
1061                     raise FormatError('Invalid sequence specification: %s' %
1062                                       line.rstrip())
1063         finally:
1064             f.close()
1065         return results
1066
1067     def set_sequences(self, sequences):
1068         """Set sequences using the given name-to-key-list dictionary."""
1069         f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
1070         try:
1071             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
1072             for name, keys in sequences.iteritems():
1073                 if len(keys) == 0:
1074                     continue
1075                 f.write('%s:' % name)
1076                 prev = None
1077                 completing = False
1078                 for key in sorted(set(keys)):
1079                     if key - 1 == prev:
1080                         if not completing:
1081                             completing = True
1082                             f.write('-')
1083                     elif completing:
1084                         completing = False
1085                         f.write('%s %s' % (prev, key))
1086                     else:
1087                         f.write(' %s' % key)
1088                     prev = key
1089                 if completing:
1090                     f.write(str(prev) + '\n')
1091                 else:
1092                     f.write('\n')
1093         finally:
1094             _sync_close(f)
1095
1096     def pack(self):
1097         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1098         sequences = self.get_sequences()
1099         prev = 0
1100         changes = []
1101         for key in self.iterkeys():
1102             if key - 1 != prev:
1103                 changes.append((key, prev + 1))
1104                 if hasattr(os, 'link'):
1105                     os.link(os.path.join(self._path, str(key)),
1106                             os.path.join(self._path, str(prev + 1)))
1107                     os.unlink(os.path.join(self._path, str(key)))
1108                 else:
1109                     os.rename(os.path.join(self._path, str(key)),
1110                               os.path.join(self._path, str(prev + 1)))
1111             prev += 1
1112         self._next_key = prev + 1
1113         if len(changes) == 0:
1114             return
1115         for name, key_list in sequences.items():
1116             for old, new in changes:
1117                 if old in key_list:
1118                     key_list[key_list.index(old)] = new
1119         self.set_sequences(sequences)
1120
1121     def _dump_sequences(self, message, key):
1122         """Inspect a new MHMessage and update sequences appropriately."""
1123         pending_sequences = message.get_sequences()
1124         all_sequences = self.get_sequences()
1125         for name, key_list in all_sequences.iteritems():
1126             if name in pending_sequences:
1127                 key_list.append(key)
1128             elif key in key_list:
1129                 del key_list[key_list.index(key)]
1130         for sequence in pending_sequences:
1131             if sequence not in all_sequences:
1132                 all_sequences[sequence] = [key]
1133         self.set_sequences(all_sequences)
1134
1135
1136 class Babyl(_singlefileMailbox):
1137     """An Rmail-style Babyl mailbox."""
1138
1139     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1140                                  'forwarded', 'edited', 'resent'))
1141
1142     def __init__(self, path, factory=None, create=True):
1143         """Initialize a Babyl mailbox."""
1144         _singlefileMailbox.__init__(self, path, factory, create)
1145         self._labels = {}
1146
1147     def add(self, message):
1148         """Add message and return assigned key."""
1149         key = _singlefileMailbox.add(self, message)
1150         if isinstance(message, BabylMessage):
1151             self._labels[key] = message.get_labels()
1152         return key
1153
1154     def remove(self, key):
1155         """Remove the keyed message; raise KeyError if it doesn't exist."""
1156         _singlefileMailbox.remove(self, key)
1157         if key in self._labels:
1158             del self._labels[key]
1159
1160     def __setitem__(self, key, message):
1161         """Replace the keyed message; raise KeyError if it doesn't exist."""
1162         _singlefileMailbox.__setitem__(self, key, message)
1163         if isinstance(message, BabylMessage):
1164             self._labels[key] = message.get_labels()
1165
1166     def get_message(self, key):
1167         """Return a Message representation or raise a KeyError."""
1168         start, stop = self._lookup(key)
1169         self._file.seek(start)
1170         self._file.readline()   # Skip '1,' line specifying labels.
1171         original_headers = StringIO.StringIO()
1172         while True:
1173             line = self._file.readline()
1174             if line == '*** EOOH ***' + os.linesep or line == '':
1175                 break
1176             original_headers.write(line.replace(os.linesep, '\n'))
1177         visible_headers = StringIO.StringIO()
1178         while True:
1179             line = self._file.readline()
1180             if line == os.linesep or line == '':
1181                 break
1182             visible_headers.write(line.replace(os.linesep, '\n'))
1183         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1184                                                                  '\n')
1185         msg = BabylMessage(original_headers.getvalue() + body)
1186         msg.set_visible(visible_headers.getvalue())
1187         if key in self._labels:
1188             msg.set_labels(self._labels[key])
1189         return msg
1190
1191     def get_string(self, key):
1192         """Return a string representation or raise a KeyError."""
1193         start, stop = self._lookup(key)
1194         self._file.seek(start)
1195         self._file.readline()   # Skip '1,' line specifying labels.
1196         original_headers = StringIO.StringIO()
1197         while True:
1198             line = self._file.readline()
1199             if line == '*** EOOH ***' + os.linesep or line == '':
1200                 break
1201             original_headers.write(line.replace(os.linesep, '\n'))
1202         while True:
1203             line = self._file.readline()
1204             if line == os.linesep or line == '':
1205                 break
1206         return original_headers.getvalue() + \
1207                self._file.read(stop - self._file.tell()).replace(os.linesep,
1208                                                                  '\n')
1209
1210     def get_file(self, key):
1211         """Return a file-like representation or raise a KeyError."""
1212         return StringIO.StringIO(self.get_string(key).replace('\n',
1213                                                               os.linesep))
1214
1215     def get_labels(self):
1216         """Return a list of user-defined labels in the mailbox."""
1217         self._lookup()
1218         labels = set()
1219         for label_list in self._labels.values():
1220             labels.update(label_list)
1221         labels.difference_update(self._special_labels)
1222         return list(labels)
1223
1224     def _generate_toc(self):
1225         """Generate key-to-(start, stop) table of contents."""
1226         starts, stops = [], []
1227         self._file.seek(0)
1228         next_pos = 0
1229         label_lists = []
1230         while True:
1231             line_pos = next_pos
1232             line = self._file.readline()
1233             next_pos = self._file.tell()
1234             if line == '\037\014' + os.linesep:
1235                 if len(stops) < len(starts):
1236                     stops.append(line_pos - len(os.linesep))
1237                 starts.append(next_pos)
1238                 labels = [label.strip() for label
1239                                         in self._file.readline()[1:].split(',')
1240                                         if label.strip() != '']
1241                 label_lists.append(labels)
1242             elif line == '\037' or line == '\037' + os.linesep:
1243                 if len(stops) < len(starts):
1244                     stops.append(line_pos - len(os.linesep))
1245             elif line == '':
1246                 stops.append(line_pos - len(os.linesep))
1247                 break
1248         self._toc = dict(enumerate(zip(starts, stops)))
1249         self._labels = dict(enumerate(label_lists))
1250         self._next_key = len(self._toc)
1251         self._file.seek(0, 2)
1252         self._file_length = self._file.tell()
1253
1254     def _pre_mailbox_hook(self, f):
1255         """Called before writing the mailbox to file f."""
1256         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1257                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1258                  os.linesep))
1259
1260     def _pre_message_hook(self, f):
1261         """Called before writing each message to file f."""
1262         f.write('\014' + os.linesep)
1263
1264     def _post_message_hook(self, f):
1265         """Called after writing each message to file f."""
1266         f.write(os.linesep + '\037')
1267
1268     def _install_message(self, message):
1269         """Write message contents and return (start, stop)."""
1270         start = self._file.tell()
1271         if isinstance(message, BabylMessage):
1272             special_labels = []
1273             labels = []
1274             for label in message.get_labels():
1275                 if label in self._special_labels:
1276                     special_labels.append(label)
1277                 else:
1278                     labels.append(label)
1279             self._file.write('1')
1280             for label in special_labels:
1281                 self._file.write(', ' + label)
1282             self._file.write(',,')
1283             for label in labels:
1284                 self._file.write(' ' + label + ',')
1285             self._file.write(os.linesep)
1286         else:
1287             self._file.write('1,,' + os.linesep)
1288         if isinstance(message, email.message.Message):
1289             orig_buffer = StringIO.StringIO()
1290             orig_generator = email.generator.Generator(orig_buffer, False, 0)
1291             orig_generator.flatten(message)
1292             orig_buffer.seek(0)
1293             while True:
1294                 line = orig_buffer.readline()
1295                 self._file.write(line.replace('\n', os.linesep))
1296                 if line == '\n' or line == '':
1297                     break
1298             self._file.write('*** EOOH ***' + os.linesep)
1299             if isinstance(message, BabylMessage):
1300                 vis_buffer = StringIO.StringIO()
1301                 vis_generator = email.generator.Generator(vis_buffer, False, 0)
1302                 vis_generator.flatten(message.get_visible())
1303                 while True:
1304                     line = vis_buffer.readline()
1305                     self._file.write(line.replace('\n', os.linesep))
1306                     if line == '\n' or line == '':
1307                         break
1308             else:
1309                 orig_buffer.seek(0)
1310                 while True:
1311                     line = orig_buffer.readline()
1312                     self._file.write(line.replace('\n', os.linesep))
1313                     if line == '\n' or line == '':
1314                         break
1315             while True:
1316                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1317                 if buffer == '':
1318                     break
1319                 self._file.write(buffer.replace('\n', os.linesep))
1320         elif isinstance(message, str):
1321             body_start = message.find('\n\n') + 2
1322             if body_start - 2 != -1:
1323                 self._file.write(message[:body_start].replace('\n',
1324                                                               os.linesep))
1325                 self._file.write('*** EOOH ***' + os.linesep)
1326                 self._file.write(message[:body_start].replace('\n',
1327                                                               os.linesep))
1328                 self._file.write(message[body_start:].replace('\n',
1329                                                               os.linesep))
1330             else:
1331                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1332                 self._file.write(message.replace('\n', os.linesep))
1333         elif hasattr(message, 'readline'):
1334             original_pos = message.tell()
1335             first_pass = True
1336             while True:
1337                 line = message.readline()
1338                 self._file.write(line.replace('\n', os.linesep))
1339                 if line == '\n' or line == '':
1340                     self._file.write('*** EOOH ***' + os.linesep)
1341                     if first_pass:
1342                         first_pass = False
1343                         message.seek(original_pos)
1344                     else:
1345                         break
1346             while True:
1347                 buffer = message.read(4096)     # Buffer size is arbitrary.
1348                 if buffer == '':
1349                     break
1350                 self._file.write(buffer.replace('\n', os.linesep))
1351         else:
1352             raise TypeError('Invalid message type: %s' % type(message))
1353         stop = self._file.tell()
1354         return (start, stop)
1355
1356
1357 class Message(email.message.Message):
1358     """Message with mailbox-format-specific properties."""
1359
1360     def __init__(self, message=None):
1361         """Initialize a Message instance."""
1362         if isinstance(message, email.message.Message):
1363             self._become_message(copy.deepcopy(message))
1364             if isinstance(message, Message):
1365                 message._explain_to(self)
1366         elif isinstance(message, str):
1367             self._become_message(email.message_from_string(message))
1368         elif hasattr(message, "read"):
1369             self._become_message(email.message_from_file(message))
1370         elif message is None:
1371             email.message.Message.__init__(self)
1372         else:
1373             raise TypeError('Invalid message type: %s' % type(message))
1374
1375     def _become_message(self, message):
1376         """Assume the non-format-specific state of message."""
1377         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1378                      'preamble', 'epilogue', 'defects', '_default_type'):
1379             self.__dict__[name] = message.__dict__[name]
1380
1381     def _explain_to(self, message):
1382         """Copy format-specific state to message insofar as possible."""
1383         if isinstance(message, Message):
1384             return  # There's nothing format-specific to explain.
1385         else:
1386             raise TypeError('Cannot convert to specified type')
1387
1388
1389 class MaildirMessage(Message):
1390     """Message with Maildir-specific properties."""
1391
1392     def __init__(self, message=None):
1393         """Initialize a MaildirMessage instance."""
1394         self._subdir = 'new'
1395         self._info = ''
1396         self._date = time.time()
1397         Message.__init__(self, message)
1398
1399     def get_subdir(self):
1400         """Return 'new' or 'cur'."""
1401         return self._subdir
1402
1403     def set_subdir(self, subdir):
1404         """Set subdir to 'new' or 'cur'."""
1405         if subdir == 'new' or subdir == 'cur':
1406             self._subdir = subdir
1407         else:
1408             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1409
1410     def get_flags(self):
1411         """Return as a string the flags that are set."""
1412         if self._info.startswith('2,'):
1413             return self._info[2:]
1414         else:
1415             return ''
1416
1417     def set_flags(self, flags):
1418         """Set the given flags and unset all others."""
1419         self._info = '2,' + ''.join(sorted(flags))
1420
1421     def add_flag(self, flag):
1422         """Set the given flag(s) without changing others."""
1423         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1424
1425     def remove_flag(self, flag):
1426         """Unset the given string flag(s) without changing others."""
1427         if self.get_flags() != '':
1428             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1429
1430     def get_date(self):
1431         """Return delivery date of message, in seconds since the epoch."""
1432         return self._date
1433
1434     def set_date(self, date):
1435         """Set delivery date of message, in seconds since the epoch."""
1436         try:
1437             self._date = float(date)
1438         except ValueError:
1439             raise TypeError("can't convert to float: %s" % date)
1440
1441     def get_info(self):
1442         """Get the message's "info" as a string."""
1443         return self._info
1444
1445     def set_info(self, info):
1446         """Set the message's "info" string."""
1447         if isinstance(info, str):
1448             self._info = info
1449         else:
1450             raise TypeError('info must be a string: %s' % type(info))
1451
1452     def _explain_to(self, message):
1453         """Copy Maildir-specific state to message insofar as possible."""
1454         if isinstance(message, MaildirMessage):
1455             message.set_flags(self.get_flags())
1456             message.set_subdir(self.get_subdir())
1457             message.set_date(self.get_date())
1458         elif isinstance(message, _mboxMMDFMessage):
1459             flags = set(self.get_flags())
1460             if 'S' in flags:
1461                 message.add_flag('R')
1462             if self.get_subdir() == 'cur':
1463                 message.add_flag('O')
1464             if 'T' in flags:
1465                 message.add_flag('D')
1466             if 'F' in flags:
1467                 message.add_flag('F')
1468             if 'R' in flags:
1469                 message.add_flag('A')
1470             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1471         elif isinstance(message, MHMessage):
1472             flags = set(self.get_flags())
1473             if 'S' not in flags:
1474                 message.add_sequence('unseen')
1475             if 'R' in flags:
1476                 message.add_sequence('replied')
1477             if 'F' in flags:
1478                 message.add_sequence('flagged')
1479         elif isinstance(message, BabylMessage):
1480             flags = set(self.get_flags())
1481             if 'S' not in flags:
1482                 message.add_label('unseen')
1483             if 'T' in flags:
1484                 message.add_label('deleted')
1485             if 'R' in flags:
1486                 message.add_label('answered')
1487             if 'P' in flags:
1488                 message.add_label('forwarded')
1489         elif isinstance(message, Message):
1490             pass
1491         else:
1492             raise TypeError('Cannot convert to specified type: %s' %
1493                             type(message))
1494
1495
1496 class _mboxMMDFMessage(Message):
1497     """Message with mbox- or MMDF-specific properties."""
1498
1499     def __init__(self, message=None):
1500         """Initialize an mboxMMDFMessage instance."""
1501         self.set_from('MAILER-DAEMON', True)
1502         if isinstance(message, email.message.Message):
1503             unixfrom = message.get_unixfrom()
1504             if unixfrom is not None and unixfrom.startswith('From '):
1505                 self.set_from(unixfrom[5:])
1506         Message.__init__(self, message)
1507
1508     def get_from(self):
1509         """Return contents of "From " line."""
1510         return self._from
1511
1512     def set_from(self, from_, time_=None):
1513         """Set "From " line, formatting and appending time_ if specified."""
1514         if time_ is not None:
1515             if time_ is True:
1516                 time_ = time.gmtime()
1517             from_ += ' ' + time.asctime(time_)
1518         self._from = from_
1519
1520     def get_flags(self):
1521         """Return as a string the flags that are set."""
1522         return self.get('Status', '') + self.get('X-Status', '')
1523
1524     def set_flags(self, flags):
1525         """Set the given flags and unset all others."""
1526         flags = set(flags)
1527         status_flags, xstatus_flags = '', ''
1528         for flag in ('R', 'O'):
1529             if flag in flags:
1530                 status_flags += flag
1531                 flags.remove(flag)
1532         for flag in ('D', 'F', 'A'):
1533             if flag in flags:
1534                 xstatus_flags += flag
1535                 flags.remove(flag)
1536         xstatus_flags += ''.join(sorted(flags))
1537         try:
1538             self.replace_header('Status', status_flags)
1539         except KeyError:
1540             self.add_header('Status', status_flags)
1541         try:
1542             self.replace_header('X-Status', xstatus_flags)
1543         except KeyError:
1544             self.add_header('X-Status', xstatus_flags)
1545
1546     def add_flag(self, flag):
1547         """Set the given flag(s) without changing others."""
1548         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1549
1550     def remove_flag(self, flag):
1551         """Unset the given string flag(s) without changing others."""
1552         if 'Status' in self or 'X-Status' in self:
1553             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1554
1555     def _explain_to(self, message):
1556         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1557         if isinstance(message, MaildirMessage):
1558             flags = set(self.get_flags())
1559             if 'O' in flags:
1560                 message.set_subdir('cur')
1561             if 'F' in flags:
1562                 message.add_flag('F')
1563             if 'A' in flags:
1564                 message.add_flag('R')
1565             if 'R' in flags:
1566                 message.add_flag('S')
1567             if 'D' in flags:
1568                 message.add_flag('T')
1569             del message['status']
1570             del message['x-status']
1571             maybe_date = ' '.join(self.get_from().split()[-5:])
1572             try:
1573                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1574                                                       '%a %b %d %H:%M:%S %Y')))
1575             except (ValueError, OverflowError):
1576                 pass
1577         elif isinstance(message, _mboxMMDFMessage):
1578             message.set_flags(self.get_flags())
1579             message.set_from(self.get_from())
1580         elif isinstance(message, MHMessage):
1581             flags = set(self.get_flags())
1582             if 'R' not in flags:
1583                 message.add_sequence('unseen')
1584             if 'A' in flags:
1585                 message.add_sequence('replied')
1586             if 'F' in flags:
1587                 message.add_sequence('flagged')
1588             del message['status']
1589             del message['x-status']
1590         elif isinstance(message, BabylMessage):
1591             flags = set(self.get_flags())
1592             if 'R' not in flags:
1593                 message.add_label('unseen')
1594             if 'D' in flags:
1595                 message.add_label('deleted')
1596             if 'A' in flags:
1597                 message.add_label('answered')
1598             del message['status']
1599             del message['x-status']
1600         elif isinstance(message, Message):
1601             pass
1602         else:
1603             raise TypeError('Cannot convert to specified type: %s' %
1604                             type(message))
1605
1606
1607 class mboxMessage(_mboxMMDFMessage):
1608     """Message with mbox-specific properties."""
1609
1610
1611 class MHMessage(Message):
1612     """Message with MH-specific properties."""
1613
1614     def __init__(self, message=None):
1615         """Initialize an MHMessage instance."""
1616         self._sequences = []
1617         Message.__init__(self, message)
1618
1619     def get_sequences(self):
1620         """Return a list of sequences that include the message."""
1621         return self._sequences[:]
1622
1623     def set_sequences(self, sequences):
1624         """Set the list of sequences that include the message."""
1625         self._sequences = list(sequences)
1626
1627     def add_sequence(self, sequence):
1628         """Add sequence to list of sequences including the message."""
1629         if isinstance(sequence, str):
1630             if not sequence in self._sequences:
1631                 self._sequences.append(sequence)
1632         else:
1633             raise TypeError('sequence must be a string: %s' % type(sequence))
1634
1635     def remove_sequence(self, sequence):
1636         """Remove sequence from the list of sequences including the message."""
1637         try:
1638             self._sequences.remove(sequence)
1639         except ValueError:
1640             pass
1641
1642     def _explain_to(self, message):
1643         """Copy MH-specific state to message insofar as possible."""
1644         if isinstance(message, MaildirMessage):
1645             sequences = set(self.get_sequences())
1646             if 'unseen' in sequences:
1647                 message.set_subdir('cur')
1648             else:
1649                 message.set_subdir('cur')
1650                 message.add_flag('S')
1651             if 'flagged' in sequences:
1652                 message.add_flag('F')
1653             if 'replied' in sequences:
1654                 message.add_flag('R')
1655         elif isinstance(message, _mboxMMDFMessage):
1656             sequences = set(self.get_sequences())
1657             if 'unseen' not in sequences:
1658                 message.add_flag('RO')
1659             else:
1660                 message.add_flag('O')
1661             if 'flagged' in sequences:
1662                 message.add_flag('F')
1663             if 'replied' in sequences:
1664                 message.add_flag('A')
1665         elif isinstance(message, MHMessage):
1666             for sequence in self.get_sequences():
1667                 message.add_sequence(sequence)
1668         elif isinstance(message, BabylMessage):
1669             sequences = set(self.get_sequences())
1670             if 'unseen' in sequences:
1671                 message.add_label('unseen')
1672             if 'replied' in sequences:
1673                 message.add_label('answered')
1674         elif isinstance(message, Message):
1675             pass
1676         else:
1677             raise TypeError('Cannot convert to specified type: %s' %
1678                             type(message))
1679
1680
1681 class BabylMessage(Message):
1682     """Message with Babyl-specific properties."""
1683
1684     def __init__(self, message=None):
1685         """Initialize an BabylMessage instance."""
1686         self._labels = []
1687         self._visible = Message()
1688         Message.__init__(self, message)
1689
1690     def get_labels(self):
1691         """Return a list of labels on the message."""
1692         return self._labels[:]
1693
1694     def set_labels(self, labels):
1695         """Set the list of labels on the message."""
1696         self._labels = list(labels)
1697
1698     def add_label(self, label):
1699         """Add label to list of labels on the message."""
1700         if isinstance(label, str):
1701             if label not in self._labels:
1702                 self._labels.append(label)
1703         else:
1704             raise TypeError('label must be a string: %s' % type(label))
1705
1706     def remove_label(self, label):
1707         """Remove label from the list of labels on the message."""
1708         try:
1709             self._labels.remove(label)
1710         except ValueError:
1711             pass
1712
1713     def get_visible(self):
1714         """Return a Message representation of visible headers."""
1715         return Message(self._visible)
1716
1717     def set_visible(self, visible):
1718         """Set the Message representation of visible headers."""
1719         self._visible = Message(visible)
1720
1721     def update_visible(self):
1722         """Update and/or sensibly generate a set of visible headers."""
1723         for header in self._visible.keys():
1724             if header in self:
1725                 self._visible.replace_header(header, self[header])
1726             else:
1727                 del self._visible[header]
1728         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1729             if header in self and header not in self._visible:
1730                 self._visible[header] = self[header]
1731
1732     def _explain_to(self, message):
1733         """Copy Babyl-specific state to message insofar as possible."""
1734         if isinstance(message, MaildirMessage):
1735             labels = set(self.get_labels())
1736             if 'unseen' in labels:
1737                 message.set_subdir('cur')
1738             else:
1739                 message.set_subdir('cur')
1740                 message.add_flag('S')
1741             if 'forwarded' in labels or 'resent' in labels:
1742                 message.add_flag('P')
1743             if 'answered' in labels:
1744                 message.add_flag('R')
1745             if 'deleted' in labels:
1746                 message.add_flag('T')
1747         elif isinstance(message, _mboxMMDFMessage):
1748             labels = set(self.get_labels())
1749             if 'unseen' not in labels:
1750                 message.add_flag('RO')
1751             else:
1752                 message.add_flag('O')
1753             if 'deleted' in labels:
1754                 message.add_flag('D')
1755             if 'answered' in labels:
1756                 message.add_flag('A')
1757         elif isinstance(message, MHMessage):
1758             labels = set(self.get_labels())
1759             if 'unseen' in labels:
1760                 message.add_sequence('unseen')
1761             if 'answered' in labels:
1762                 message.add_sequence('replied')
1763         elif isinstance(message, BabylMessage):
1764             message.set_visible(self.get_visible())
1765             for label in self.get_labels():
1766                 message.add_label(label)
1767         elif isinstance(message, Message):
1768             pass
1769         else:
1770             raise TypeError('Cannot convert to specified type: %s' %
1771                             type(message))
1772
1773
1774 class MMDFMessage(_mboxMMDFMessage):
1775     """Message with MMDF-specific properties."""
1776
1777
1778 class _ProxyFile:
1779     """A read-only wrapper of a file."""
1780
1781     def __init__(self, f, pos=None):
1782         """Initialize a _ProxyFile."""
1783         self._file = f
1784         if pos is None:
1785             self._pos = f.tell()
1786         else:
1787             self._pos = pos
1788
1789     def read(self, size=None):
1790         """Read bytes."""
1791         return self._read(size, self._file.read)
1792
1793     def readline(self, size=None):
1794         """Read a line."""
1795         return self._read(size, self._file.readline)
1796
1797     def readlines(self, sizehint=None):
1798         """Read multiple lines."""
1799         result = []
1800         for line in self:
1801             result.append(line)
1802             if sizehint is not None:
1803                 sizehint -= len(line)
1804                 if sizehint <= 0:
1805                     break
1806         return result
1807
1808     def __iter__(self):
1809         """Iterate over lines."""
1810         return iter(self.readline, "")
1811
1812     def tell(self):
1813         """Return the position."""
1814         return self._pos
1815
1816     def seek(self, offset, whence=0):
1817         """Change position."""
1818         if whence == 1:
1819             self._file.seek(self._pos)
1820         self._file.seek(offset, whence)
1821         self._pos = self._file.tell()
1822
1823     def close(self):
1824         """Close the file."""
1825         del self._file
1826
1827     def _read(self, size, read_method):
1828         """Read size bytes using read_method."""
1829         if size is None:
1830             size = -1
1831         self._file.seek(self._pos)
1832         result = read_method(size)
1833         self._pos = self._file.tell()
1834         return result
1835
1836
1837 class _PartialFile(_ProxyFile):
1838     """A read-only wrapper of part of a file."""
1839
1840     def __init__(self, f, start=None, stop=None):
1841         """Initialize a _PartialFile."""
1842         _ProxyFile.__init__(self, f, start)
1843         self._start = start
1844         self._stop = stop
1845
1846     def tell(self):
1847         """Return the position with respect to start."""
1848         return _ProxyFile.tell(self) - self._start
1849
1850     def seek(self, offset, whence=0):
1851         """Change position, possibly with respect to start or stop."""
1852         if whence == 0:
1853             self._pos = self._start
1854             whence = 1
1855         elif whence == 2:
1856             self._pos = self._stop
1857             whence = 1
1858         _ProxyFile.seek(self, offset, whence)
1859
1860     def _read(self, size, read_method):
1861         """Read size bytes using read_method, honoring start and stop."""
1862         remaining = self._stop - self._pos
1863         if remaining <= 0:
1864             return ''
1865         if size is None or size < 0 or size > remaining:
1866             size = remaining
1867         return _ProxyFile._read(self, size, read_method)
1868
1869
1870 def _lock_file(f, dotlock=True):
1871     """Lock file f using lockf and dot locking."""
1872     dotlock_done = False
1873     try:
1874         if fcntl:
1875             try:
1876                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1877             except IOError, e:
1878                 if e.errno in (errno.EAGAIN, errno.EACCES):
1879                     raise ExternalClashError('lockf: lock unavailable: %s' %
1880                                              f.name)
1881                 else:
1882                     raise
1883         if dotlock:
1884             try:
1885                 pre_lock = _create_temporary(f.name + '.lock')
1886                 pre_lock.close()
1887             except IOError, e:
1888                 if e.errno == errno.EACCES:
1889                     return  # Without write access, just skip dotlocking.
1890                 else:
1891                     raise
1892             try:
1893                 if hasattr(os, 'link'):
1894                     os.link(pre_lock.name, f.name + '.lock')
1895                     dotlock_done = True
1896                     os.unlink(pre_lock.name)
1897                 else:
1898                     os.rename(pre_lock.name, f.name + '.lock')
1899                     dotlock_done = True
1900             except OSError, e:
1901                 if e.errno == errno.EEXIST or \
1902                   (os.name == 'os2' and e.errno == errno.EACCES):
1903                     os.remove(pre_lock.name)
1904                     raise ExternalClashError('dot lock unavailable: %s' %
1905                                              f.name)
1906                 else:
1907                     raise
1908     except:
1909         if fcntl:
1910             fcntl.lockf(f, fcntl.LOCK_UN)
1911         if dotlock_done:
1912             os.remove(f.name + '.lock')
1913         raise
1914
1915 def _unlock_file(f):
1916     """Unlock file f using lockf and dot locking."""
1917     if fcntl:
1918         fcntl.lockf(f, fcntl.LOCK_UN)
1919     if os.path.exists(f.name + '.lock'):
1920         os.remove(f.name + '.lock')
1921
1922 def _create_carefully(path):
1923     """Create a file if it doesn't exist and open for reading and writing."""
1924     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
1925     try:
1926         return open(path, 'rb+')
1927     finally:
1928         os.close(fd)
1929
1930 def _create_temporary(path):
1931     """Create a temp file based on path and open for reading and writing."""
1932     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1933                                               socket.gethostname(),
1934                                               os.getpid()))
1935
1936 def _sync_flush(f):
1937     """Ensure changes to file f are physically on disk."""
1938     f.flush()
1939     if hasattr(os, 'fsync'):
1940         os.fsync(f.fileno())
1941
1942 def _sync_close(f):
1943     """Close file f, ensuring all changes are physically on disk."""
1944     _sync_flush(f)
1945     f.close()
1946
1947 ## Start: classes from the original module (for backward compatibility).
1948
1949 # Note that the Maildir class, whose name is unchanged, itself offers a next()
1950 # method for backward compatibility.
1951
1952 class _Mailbox:
1953
1954     def __init__(self, fp, factory=rfc822.Message):
1955         self.fp = fp
1956         self.seekp = 0
1957         self.factory = factory
1958
1959     def __iter__(self):
1960         return iter(self.next, None)
1961
1962     def next(self):
1963         while 1:
1964             self.fp.seek(self.seekp)
1965             try:
1966                 self._search_start()
1967             except EOFError:
1968                 self.seekp = self.fp.tell()
1969                 return None
1970             start = self.fp.tell()
1971             self._search_end()
1972             self.seekp = stop = self.fp.tell()
1973             if start != stop:
1974                 break
1975         return self.factory(_PartialFile(self.fp, start, stop))
1976
1977 # Recommended to use PortableUnixMailbox instead!
1978 class UnixMailbox(_Mailbox):
1979
1980     def _search_start(self):
1981         while 1:
1982             pos = self.fp.tell()
1983             line = self.fp.readline()
1984             if not line:
1985                 raise EOFError
1986             if line[:5] == 'From ' and self._isrealfromline(line):
1987                 self.fp.seek(pos)
1988                 return
1989
1990     def _search_end(self):
1991         self.fp.readline()      # Throw away header line
1992         while 1:
1993             pos = self.fp.tell()
1994             line = self.fp.readline()
1995             if not line:
1996                 return
1997             if line[:5] == 'From ' and self._isrealfromline(line):
1998                 self.fp.seek(pos)
1999                 return
2000
2001     # An overridable mechanism to test for From-line-ness.  You can either
2002     # specify a different regular expression or define a whole new
2003     # _isrealfromline() method.  Note that this only gets called for lines
2004     # starting with the 5 characters "From ".
2005     #
2006     # BAW: According to
2007     #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
2008     # the only portable, reliable way to find message delimiters in a BSD (i.e
2009     # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
2010     # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
2011     # like a good idea, in practice, there are too many variations for more
2012     # strict parsing of the line to be completely accurate.
2013     #
2014     # _strict_isrealfromline() is the old version which tries to do stricter
2015     # parsing of the From_ line.  _portable_isrealfromline() simply returns
2016     # true, since it's never called if the line doesn't already start with
2017     # "From ".
2018     #
2019     # This algorithm, and the way it interacts with _search_start() and
2020     # _search_end() may not be completely correct, because it doesn't check
2021     # that the two characters preceding "From " are \n\n or the beginning of
2022     # the file.  Fixing this would require a more extensive rewrite than is
2023     # necessary.  For convenience, we've added a PortableUnixMailbox class
2024     # which does no checking of the format of the 'From' line.
2025
2026     _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
2027                         r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
2028                         r"[^\s]*\s*"
2029                         "$")
2030     _regexp = None
2031
2032     def _strict_isrealfromline(self, line):
2033         if not self._regexp:
2034             import re
2035             self._regexp = re.compile(self._fromlinepattern)
2036         return self._regexp.match(line)
2037
2038     def _portable_isrealfromline(self, line):
2039         return True
2040
2041     _isrealfromline = _strict_isrealfromline
2042
2043
2044 class PortableUnixMailbox(UnixMailbox):
2045     _isrealfromline = UnixMailbox._portable_isrealfromline
2046
2047
2048 class MmdfMailbox(_Mailbox):
2049
2050     def _search_start(self):
2051         while 1:
2052             line = self.fp.readline()
2053             if not line:
2054                 raise EOFError
2055             if line[:5] == '\001\001\001\001\n':
2056                 return
2057
2058     def _search_end(self):
2059         while 1:
2060             pos = self.fp.tell()
2061             line = self.fp.readline()
2062             if not line:
2063                 return
2064             if line == '\001\001\001\001\n':
2065                 self.fp.seek(pos)
2066                 return
2067
2068
2069 class MHMailbox:
2070
2071     def __init__(self, dirname, factory=rfc822.Message):
2072         import re
2073         pat = re.compile('^[1-9][0-9]*$')
2074         self.dirname = dirname
2075         # the three following lines could be combined into:
2076         # list = map(long, filter(pat.match, os.listdir(self.dirname)))
2077         list = os.listdir(self.dirname)
2078         list = filter(pat.match, list)
2079         list = map(long, list)
2080         list.sort()
2081         # This only works in Python 1.6 or later;
2082         # before that str() added 'L':
2083         self.boxes = map(str, list)
2084         self.boxes.reverse()
2085         self.factory = factory
2086
2087     def __iter__(self):
2088         return iter(self.next, None)
2089
2090     def next(self):
2091         if not self.boxes:
2092             return None
2093         fn = self.boxes.pop()
2094         fp = open(os.path.join(self.dirname, fn))
2095         msg = self.factory(fp)
2096         try:
2097             msg._mh_msgno = fn
2098         except (AttributeError, TypeError):
2099             pass
2100         return msg
2101
2102
2103 class BabylMailbox(_Mailbox):
2104
2105     def _search_start(self):
2106         while 1:
2107             line = self.fp.readline()
2108             if not line:
2109                 raise EOFError
2110             if line == '*** EOOH ***\n':
2111                 return
2112
2113     def _search_end(self):
2114         while 1:
2115             pos = self.fp.tell()
2116             line = self.fp.readline()
2117             if not line:
2118                 return
2119             if line == '\037\014\n' or line == '\037':
2120                 self.fp.seek(pos)
2121                 return
2122
2123 ## End: classes from the original module (for backward compatibility).
2124
2125
2126 class Error(Exception):
2127     """Raised for module-specific errors."""
2128
2129 class NoSuchMailboxError(Error):
2130     """The specified mailbox does not exist and won't be created."""
2131
2132 class NotEmptyError(Error):
2133     """The specified mailbox is not empty and deletion was requested."""
2134
2135 class ExternalClashError(Error):
2136     """Another process caused an action to fail."""
2137
2138 class FormatError(Error):
2139     """A file appears to have an invalid format."""